pipeline: print a console message before the shader cache warm-up

launcher: close the terminal after a clean exit and run several games at once

Co-authored-by: nmzik <Nmzik@mail.ru>

CMakeLists.txt

@@ -337,7 +337,7 @@ if (KYTY_CLANG_CL)
 	set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED")
 endif()
 
-set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus OpenXR::openxr_loader)
+set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus OpenXR::openxr_loader libzstd_static)
 
 # Linux system libraries.
 if(LINUX)
@@ -451,6 +451,11 @@ add_executable(lru_cache_tests EXCLUDE_FROM_ALL
 )
 target_include_directories(lru_cache_tests PRIVATE ${inc_headers})
 
+add_executable(slot_vector_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/SlotVectorTests.cpp"
+)
+target_include_directories(slot_vector_tests PRIVATE ${inc_headers})
+
 add_executable(ime_dialog_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ImeDialogTests.cpp"
 	"${KYTY_SOURCE_DIR}/libs/imeCommon.cpp"
@@ -541,6 +546,19 @@ add_executable(png_enc_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/PngEncTests.cpp
 target_link_libraries(png_enc_tests common fmt::fmt)
 target_include_directories(png_enc_tests PRIVATE ${inc_headers})
 
+add_executable(jpeg_dec_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/JpegDecTests.cpp")
+target_link_libraries(jpeg_dec_tests common fmt::fmt)
+target_include_directories(jpeg_dec_tests PRIVATE ${inc_headers})
+
+add_executable(jpeg_enc_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/JpegEncTests.cpp")
+target_link_libraries(jpeg_enc_tests common fmt::fmt)
+target_include_directories(jpeg_enc_tests PRIVATE ${inc_headers})
+
+add_executable(occlusion_queries_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/OcclusionQueriesTests.cpp")
+target_link_libraries(occlusion_queries_tests common fmt::fmt Vulkan::Headers)
+target_include_directories(occlusion_queries_tests PRIVATE ${inc_headers})
+
 add_executable(ces_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/CesTests.cpp")
 target_link_libraries(ces_tests common fmt::fmt SDL3::SDL3-static)
 target_include_directories(ces_tests PRIVATE ${inc_headers})
@@ -652,6 +670,9 @@ if(BUILD_TESTING)
 	add_test(NAME host_input COMMAND $<TARGET_FILE:host_input_tests>)
 	add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
 	add_test(NAME png_enc COMMAND $<TARGET_FILE:png_enc_tests>)
+	add_test(NAME jpeg_dec COMMAND $<TARGET_FILE:jpeg_dec_tests>)
+	add_test(NAME jpeg_enc COMMAND $<TARGET_FILE:jpeg_enc_tests>)
+	add_test(NAME occlusion_queries COMMAND $<TARGET_FILE:occlusion_queries_tests>)
 	add_test(NAME ces COMMAND $<TARGET_FILE:ces_tests>)
 	add_test(NAME save_data_dialog COMMAND $<TARGET_FILE:save_data_dialog_tests>)
 	add_test(NAME http_uri_parse COMMAND $<TARGET_FILE:http_uri_parse_tests>)
@@ -678,6 +699,8 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
 	add_test(NAME wait_until_safe_for_rendering
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --wait-until-safe-for-rendering-only)
+	add_test(NAME pm4_flip
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --flip-only)
 	add_test(NAME depth_bits
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-bits-only)
 	add_test(NAME msaa_storage
@@ -742,6 +765,9 @@ if(BUILD_TESTING)
 		host_input_tests
 		save_data_memory_tests
 		png_enc_tests
+		jpeg_dec_tests
+		jpeg_enc_tests
+		occlusion_queries_tests
 		ces_tests
 		save_data_dialog_tests
 		http_uri_parse_tests

src/common/additionalContent.cpp

@@ -109,4 +109,23 @@ std::vector<AddContentEntry> FindAddContent(const std::filesystem::path& dir,
 	return entries;
 }
 
+std::optional<AddContentEntry> ParseAddContent(std::string_view value) {
+	static const std::regex label_pattern("^[A-Z0-9]{16}$");
+	const auto              eq    = value.find('=');
+	const auto              label = value.substr(0, eq);
+	if (!std::regex_match(label.begin(), label.end(), label_pattern)) {
+		return std::nullopt;
+	}
+
+	AddContentEntry entry {{}, std::string(label), {}, {}};
+	if (eq != std::string_view::npos) {
+		entry.mount_dir = PathFromUtf8(FixFilenameSlash(value.substr(eq + 1)));
+		std::error_code ec;
+		if (!std::filesystem::is_directory(entry.mount_dir, ec)) {
+			return std::nullopt;
+		}
+	}
+	return entry;
+}
+
 } // namespace Common

src/common/additionalContent.h

@@ -2,7 +2,9 @@
 #define KYTY_COMMON_ADDITIONAL_CONTENT_H_
 
 #include <filesystem>
+#include <optional>
 #include <string>
+#include <string_view>
 #include <vector>
 
 namespace Common {
@@ -10,7 +12,7 @@ namespace Common {
 struct AddContentEntry {
 	std::string           content_id;
 	std::string           entitlement_label;
-	std::filesystem::path mount_dir;
+	std::filesystem::path mount_dir; // Empty for an add-on without extra data.
 	std::string           title;
 };
 
@@ -18,6 +20,9 @@ struct AddContentEntry {
 std::vector<AddContentEntry> FindAddContent(const std::filesystem::path& dir,
                                             const std::string&           title_id);
 
+// Parses a user add-on: "LABEL" without extra data, or "LABEL=DIR" with its data in DIR.
+std::optional<AddContentEntry> ParseAddContent(std::string_view value);
+
 } // namespace Common
 
 #endif /* KYTY_COMMON_ADDITIONAL_CONTENT_H_ */

src/common/emulatorConfig.cpp

@@ -133,6 +133,10 @@ bool ShaderValidationEnabled() {
 	return g_config->shader_validation_enabled;
 }
 
+bool ShaderCacheEnabled() {
+	return g_config->shader_cache_enabled;
+}
+
 ShaderOptimizationType GetShaderOptimizationType() {
 	return g_config->shader_optimization_type;
 }
@@ -197,6 +201,10 @@ bool TessellationEnabled() {
 	return g_config->tessellation_enabled;
 }
 
+bool OcclusionQueriesEnabled() {
+	return g_config->occlusion_queries_enabled;
+}
+
 bool TrophyEnabled() {
 	return g_config->trophy_enabled;
 }

src/common/emulatorConfig.h

@@ -72,6 +72,7 @@ struct ConfigOptions {
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
+	bool                   shader_cache_enabled        = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
 	LogDirection           shader_log_direction        = LogDirection::Silent;
 	std::filesystem::path  shader_log_folder           = "_Shaders";
@@ -88,6 +89,7 @@ struct ConfigOptions {
 	bool                   readback_tiled_images       = false;
 	bool                   sync_raw_image_buffers      = false;
 	bool                   tessellation_enabled        = false;
+	bool                   occlusion_queries_enabled   = false;
 	bool                   trophy_enabled              = true;
 	uint32_t               trophy_duration_seconds     = 6;
 	bool                   perf_overlay_enabled            = false;
@@ -131,6 +133,7 @@ uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
+bool                   ShaderCacheEnabled();
 ShaderOptimizationType GetShaderOptimizationType();
 LogDirection           GetShaderLogDirection();
 std::filesystem::path  GetShaderLogFolder();
@@ -154,6 +157,7 @@ bool ReadbackLinearImagesEnabled();
 bool ReadbackTiledImagesEnabled();
 bool SyncRawImageBuffersEnabled();
 bool TessellationEnabled();
+bool OcclusionQueriesEnabled();
 bool TrophyEnabled();
 uint32_t GetTrophyDurationSeconds();
 bool PlayGoHackEnabled();

src/common/slotVector.h

@@ -1,15 +1,15 @@
 #ifndef EMULATOR_SRC_COMMON_SLOTVECTOR_H_
 #define EMULATOR_SRC_COMMON_SLOTVECTOR_H_
 
-#include "common/assert.h"
-#include "common/abi.h"
+#include "common/common.h"
 
 #include <compare>
 #include <cstdint>
-#include <deque>
 #include <functional>
 #include <limits>
-#include <optional>
+#include <memory>
+#include <new>
+#include <numeric>
 #include <utility>
 #include <vector>
 
@@ -19,105 +19,105 @@ struct SlotId {
 	static constexpr uint32_t INVALID_INDEX = std::numeric_limits<uint32_t>::max();
 
 	constexpr SlotId() noexcept = default;
-	constexpr SlotId(uint32_t value) noexcept: index(value), generation(1) {}
-	constexpr SlotId(uint32_t value, uint32_t slot_generation) noexcept
-	    : index(value), generation(slot_generation) {}
+	constexpr SlotId(uint32_t index) noexcept: index(index) {}
 
-	[[nodiscard]] constexpr explicit operator bool() const noexcept { return index != INVALID_INDEX; }
+	[[nodiscard]] constexpr explicit operator bool() const noexcept {
+		return index != INVALID_INDEX;
+	}
 	constexpr auto operator<=>(const SlotId&) const noexcept = default;
 
 	uint32_t index = INVALID_INDEX;
-	uint32_t generation = 0;
 };
 
-// Stable-address slot storage for cache resources that are intentionally non-movable.
+// Fixed-size chunks keep the stored, non-movable resources in place. Occupancy is a byte per
+// slot, not a bitset: a deferred erase runs while fault threads read other slots.
 template <typename T>
 class SlotVector {
+	static constexpr uint32_t ChunkSize = 256;
+
 public:
 	SlotVector() = default;
 	KYTY_CLASS_NO_COPY(SlotVector);
 
+	~SlotVector() noexcept {
+		ForEach([](SlotId, T& value) { value.~T(); });
+	}
+
 	[[nodiscard]] T& operator[](SlotId id) noexcept {
-		EXIT_IF(!is_allocated(id));
-		return *m_values[id.index].value;
+		return m_chunks[id.index / ChunkSize][id.index % ChunkSize].object;
 	}
 
 	[[nodiscard]] const T& operator[](SlotId id) const noexcept {
-		EXIT_IF(!is_allocated(id));
-		return *m_values[id.index].value;
+		return m_chunks[id.index / ChunkSize][id.index % ChunkSize].object;
 	}
 
 	[[nodiscard]] T* try_get(SlotId id) noexcept {
-		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+		return is_allocated(id) ? &(*this)[id] : nullptr;
 	}
 
 	[[nodiscard]] const T* try_get(SlotId id) const noexcept {
-		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+		return is_allocated(id) ? &(*this)[id] : nullptr;
 	}
 
 	[[nodiscard]] bool is_allocated(SlotId id) const noexcept {
-		return id && id.index < m_values.size() &&
-		       m_values[id.index].generation == id.generation && m_values[id.index].value.has_value();
+		return id.index < m_stored.size() && m_stored[id.index] != 0;
 	}
 
 	template <typename... Args>
 	[[nodiscard]] SlotId insert(Args&&... args) {
-		uint32_t index = 0;
 		if (m_free_list.empty()) {
-			index = static_cast<uint32_t>(m_values.size());
-			m_values.emplace_back();
-			m_values.back().value.emplace(std::forward<Args>(args)...);
-		} else {
-			index = m_free_list.back();
-			m_free_list.pop_back();
-			EXIT_IF(m_values[index].value.has_value());
-			m_values[index].value.emplace(std::forward<Args>(args)...);
+			Reserve();
 		}
-		++m_size;
-		return SlotId {index, m_values[index].generation};
+		const uint32_t index = m_free_list.back();
+		m_free_list.pop_back();
+		new (&(*this)[index]) T(std::forward<Args>(args)...);
+		m_stored[index] = 1;
+		return SlotId {index};
 	}
 
 	void erase(SlotId id) noexcept {
-		EXIT_IF(!is_allocated(id));
-		auto& slot = m_values[id.index];
-		slot.value.reset();
-		if (++slot.generation == 0) {
-			slot.generation = 1;
-		}
+		(*this)[id].~T();
+		m_stored[id.index] = 0;
 		m_free_list.push_back(id.index);
-		--m_size;
 	}
 
-	[[nodiscard]] size_t size() const noexcept { return m_size; }
-	[[nodiscard]] size_t capacity() const noexcept { return m_values.size(); }
-
 	template <typename F>
 	void ForEach(F&& fn) {
-		for (uint32_t index = 0; index < m_values.size(); ++index) {
-			if (m_values[index].value) {
-				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+		for (uint32_t index = 0; index < m_stored.size(); ++index) {
+			if (m_stored[index] != 0) {
+				fn(SlotId {index}, (*this)[index]);
 			}
 		}
 	}
 
 	template <typename F>
 	void ForEach(F&& fn) const {
-		for (uint32_t index = 0; index < m_values.size(); ++index) {
-			if (m_values[index].value) {
-				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+		for (uint32_t index = 0; index < m_stored.size(); ++index) {
+			if (m_stored[index] != 0) {
+				fn(SlotId {index}, (*this)[index]);
 			}
 		}
 	}
 
 private:
-	struct Slot {
-		std::optional<T> value;
-		uint32_t         generation = 1;
+	union Slot {
+		Slot() noexcept {}
+		~Slot() noexcept {}
+
+		T object;
 	};
 
-	std::deque<Slot>             m_values;
-	std::vector<uint32_t>        m_free_list;
-	size_t                       m_size = 0;
+	void Reserve() {
+		const auto first = static_cast<uint32_t>(m_stored.size());
+		m_chunks.push_back(std::make_unique<Slot[]>(ChunkSize));
+		m_stored.resize(first + ChunkSize);
+		m_free_list.resize(ChunkSize);
+		std::iota(m_free_list.rbegin(), m_free_list.rend(), first);
+	}
+
+	std::vector<std::unique_ptr<Slot[]>> m_chunks;
+	std::vector<uint8_t>                 m_stored;
+	std::vector<uint32_t>                m_free_list;
 };
 
 } // namespace Common
@@ -125,7 +125,7 @@ private:
 template <>
 struct std::hash<Common::SlotId> {
 	[[nodiscard]] size_t operator()(Common::SlotId id) const noexcept {
-		return std::hash<uint64_t> {}((static_cast<uint64_t>(id.generation) << 32u) | id.index);
+		return std::hash<uint32_t> {}(id.index);
 	}
 };
 

src/emulator.cpp

@@ -188,7 +188,7 @@ void Run(const RunOptions& options) {
 
 	MountSandboxDirs();
 
-	Loader::AddContentLoad(options.addcont_dir, title_id);
+	Loader::AddContentLoad(options.addcont_dir, options.addcont, title_id);
 	auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();
 	Libs::InitAll(rt->Symbols());
 

src/emulator.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_EMULATOR_H_
 #define EMULATOR_INCLUDE_EMULATOR_EMULATOR_H_
 
+#include "common/additionalContent.h"
 #include "common/emulatorConfig.h"
 #include "common/stringUtils.h"
 
@@ -9,11 +10,12 @@
 namespace Emulator {
 
 struct RunOptions {
-	Config::ConfigOptions config;
-	std::filesystem::path app0_dir;
-	std::filesystem::path elf;
-	std::filesystem::path game_patch;
-	std::filesystem::path addcont_dir;
+	Config::ConfigOptions                config;
+	std::filesystem::path                app0_dir;
+	std::filesystem::path                elf;
+	std::filesystem::path                game_patch;
+	std::filesystem::path                addcont_dir;
+	std::vector<Common::AddContentEntry> addcont;
 };
 
 void Run(const RunOptions& options);

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -50,13 +50,6 @@ private:
 
 class CommandProcessor {
 public:
-	struct FlipInfo {
-		int     handle    = 0;
-		int     index     = 0;
-		int     flip_mode = 0;
-		int64_t flip_arg  = 0;
-	};
-
 	CommandProcessor(RenderContext& renderer, int interrupt_event_id)
 	    : m_renderer(renderer), m_interrupt_event_id(interrupt_event_id) {}
 	~CommandProcessor() = default;
@@ -90,40 +83,17 @@ public:
 	void DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
 	                       const volatile uint32_t* count_addr, uint32_t stride_in_bytes,
 	                       IndirectDrawRegisters registers, uint32_t draw_initiator, bool indexed);
-	void WriteAtEndOfPipe32(uint32_t cache_policy, uint32_t event_write_dest,
-	                        uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index,
-	                        uint32_t event_write_source, void* dst_gpu_addr, uint32_t value,
-	                        uint32_t interrupt_selector, uint32_t interrupt_context_id = 0);
-	void WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_write_dest,
-	                        uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index,
-	                        uint32_t event_write_source, void* dst_gpu_addr, uint64_t value,
-	                        uint32_t interrupt_selector, uint32_t interrupt_context_id = 0);
-	void Flip();
-	void Flip(void* dst_gpu_addr, uint32_t value);
-	void FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action, void* dst_gpu_addr,
-	                       uint32_t value);
+	void ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector, void* address, uint64_t data,
+	                uint32_t interrupt_context_id);
+	void Flip(int handle, int index, int flip_mode, int64_t flip_arg);
 	void PrepareCpuFlip(uint64_t request_id);
-	void SynchronizeGpu();
-	void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id);
 	void DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z,
 	                    uint32_t mode);
 	void DispatchIndirect(uint64_t args_addr, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
 	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);
 
-	void SetUserDataMarker(HW::UserSgprType type) { m_user_data_marker = type; }
-	[[nodiscard]] HW::UserSgprType GetUserDataMarker() const { return m_user_data_marker; }
-
-	void ResetDeCe();
-	void SetCeComplete(bool complete) { m_ce_complete = complete; }
-	void WaitCe();
-	void WaitDeDiff(uint32_t diff);
 	void WaitForRewind(bool valid);
-	void IncrementDe();
-	void IncrementCe();
-
-	void WriteConstRam(uint32_t offset, const uint32_t* src, uint32_t dw_num);
-	void DumpConstRam(uint32_t* dst, uint32_t offset, uint32_t dw_num);
 
 	template <typename T>
 	void WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, uint32_t wait_op);
@@ -140,18 +110,11 @@ public:
 	Pm4ProcessResult Process(Pm4Execution& execution, std::span<const uint32_t> commands);
 	void             ProcessIndirectBuffer(std::span<const uint32_t> commands, bool chain);
 
-	void SetFlip(const FlipInfo& flip) { m_flip = flip; }
-
 	[[nodiscard]] uint64_t GetSubmitId() const { return m_submit_id; }
 	void                   SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; }
 	[[nodiscard]] bool     IsAsyncComputeQueue() const { return m_interrupt_event_id >= 0x20; }
 
 private:
-	template <typename T>
-	void WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_write_dest, uint32_t eop_event_type,
-	                      uint32_t cache_action, uint32_t event_index, uint32_t event_write_source,
-	                      void* dst_gpu_addr, T value, uint32_t interrupt_selector,
-	                      uint32_t interrupt_context_id);
 	void ProcessPm4(Pm4Execution& execution);
 	void SuspendPm4();
 	uint32_t            NumInstances();
@@ -159,30 +122,22 @@ private:
 	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
 
-	RenderContext&   m_renderer;
-	HW::Context      m_ctx;
-	HW::Context      m_saved_ctx;
-	bool             m_context_state_pushed = false;
-	HW::UserConfig   m_ucfg;
-	HW::Shader       m_sh_ctx;
-	HW::UserSgprType m_user_data_marker                 = HW::UserSgprType::Unknown;
-	uint32_t         m_index_type_and_size              = 0;
-	uint32_t         m_index_buffer_size                = 0;
-	uint64_t         m_index_base_addr                  = 0;
-	uint64_t         m_draw_indirect_args_base_addr     = 0;
-	uint64_t         m_dispatch_indirect_args_base_addr = 0;
+	RenderContext& m_renderer;
+	HW::Context    m_ctx;
+	HW::Context    m_saved_ctx;
+	bool           m_context_state_pushed = false;
+	HW::UserConfig m_ucfg;
+	HW::Shader     m_sh_ctx;
+	uint32_t       m_index_type_and_size              = 0;
+	uint32_t       m_index_buffer_size                = 0;
+	uint64_t       m_index_base_addr                  = 0;
+	uint64_t       m_draw_indirect_args_base_addr     = 0;
+	uint64_t       m_dispatch_indirect_args_base_addr = 0;
 	// Persistent draw state. Indirect draws leave the address of their last instance count,
 	// read only when a later draw inherits it.
 	uint32_t m_num_instances      = 1;
 	uint64_t m_num_instances_addr = 0;
 
-	uint32_t m_de_count    = 0;
-	uint32_t m_ce_count    = 0;
-	bool     m_ce_complete = false;
-
-	uint32_t m_const_ram[0x3000] = {0};
-
-	FlipInfo  m_flip;
 	const int m_interrupt_event_id;
 	uint64_t  m_submit_id                   = 0;
 	uint64_t  m_synthetic_occlusion_counter = 0;

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -199,10 +199,9 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_INDIRECT_BUFFER]           = CpOpIndirectBuffer;
 	g_cp_op_func[Pm4::IT_COPY_DATA]                 = CpOpCopyData;
 	g_cp_op_func[Pm4::IT_EVENT_WRITE]               = CpOpEventWrite;
-	g_cp_op_func[Pm4::IT_EVENT_WRITE_EOP]           = CpOpEventWriteEop;
-	g_cp_op_func[Pm4::IT_EVENT_WRITE_EOS]           = CpOpEventWriteEos;
 	g_cp_op_func[Pm4::IT_DMA_DATA]                  = CpOpDmaData;
 	g_cp_op_func[Pm4::IT_ACQUIRE_MEM]               = CpOpAcquireMem;
+	g_cp_op_func[Pm4::IT_RELEASE_MEM]               = CpOpReleaseMem;
 	g_cp_op_func[Pm4::IT_REWIND]                    = CpOpRewind;
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG]           = CpOpSetContextReg;
 	g_cp_op_func[Pm4::IT_SET_SH_REG]                = CpOpSetShaderReg;
@@ -211,12 +210,6 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG_INDIRECT]  = CpOpIndirectCxRegs;
 	g_cp_op_func[Pm4::IT_SET_SH_REG_INDIRECT]       = CpOpIndirectShRegs;
 	g_cp_op_func[Pm4::IT_SET_UCONFIG_REG_INDIRECT]  = CpOpIndirectUcRegs;
-	g_cp_op_func[Pm4::IT_WRITE_CONST_RAM]           = CpOpWriteConstRam;
-	g_cp_op_func[Pm4::IT_DUMP_CONST_RAM]            = CpOpDumpConstRam;
-	g_cp_op_func[Pm4::IT_INCREMENT_CE_COUNTER]      = CpOpIncrementCeCounter;
-	g_cp_op_func[Pm4::IT_INCREMENT_DE_COUNTER]      = CpOpIncrementDeCounter;
-	g_cp_op_func[Pm4::IT_WAIT_ON_CE_COUNTER]        = CpOpWaitOnCeCounter;
-	g_cp_op_func[Pm4::IT_WAIT_ON_DE_COUNTER_DIFF]   = CpOpWaitOnDeCounterDiff;
 	g_cp_op_func[Pm4::IT_GET_LOD_STATS]             = CpOpGetLodStats;
 	g_cp_op_func[Pm4::IT_WAIT_REG_MEM_64]           = CpOpWaitRegMem64;
 
@@ -231,7 +224,6 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	g_cp_op_custom_func[Pm4::R_PUSH_MARKER]    = CpOpPushMarker;
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
-	g_cp_op_custom_func[Pm4::R_RELEASE_MEM]    = CpOpReleaseMem;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;
 
 	return g_cp_op_custom_func;

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -133,22 +133,15 @@ uint32_t CpOpWriteData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, u
 uint32_t CpOpIndirectBuffer(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpCopyData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpEventWrite(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpEventWriteEop(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpEventWriteEos(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDmaData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpAcquireMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpReleaseMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetContextReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetShaderReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetUconfigReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpIndirectCxRegs(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpIndirectShRegs(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpIndirectUcRegs(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpWriteConstRam(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpDumpConstRam(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpIncrementCeCounter(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpIncrementDeCounter(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpWaitOnCeCounter(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpWaitOnDeCounterDiff(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpGetLodStats(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDispatchReset(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpWaitRegMem32(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
@@ -157,7 +150,6 @@ uint32_t CpOpPushMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t,
 uint32_t CpOpPopMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpWaitRegMem64(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpFlip(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpReleaseMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 
 } // namespace Libs::Graphics
 

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -54,34 +54,12 @@ namespace Libs::Graphics {
 
 namespace {
 
-constexpr uint32_t GcrGl2MetadataInvalidate = 1u << 1u;
-constexpr uint32_t GcrGl0VectorInvalidate   = 1u << 2u;
-constexpr uint32_t GcrGl1Invalidate         = 1u << 3u;
-constexpr uint32_t GcrGl2Unshared           = 1u << 4u;
-constexpr uint32_t GcrGl2Invalidate         = 1u << 8u;
-constexpr uint32_t GcrGl2Writeback          = 1u << 9u;
-constexpr uint32_t GcrOrder012              = 1u << 10u;
-constexpr uint32_t GcrOrder210              = 2u << 10u;
-constexpr uint32_t GcrKnownMask             = GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate |
-                                              GcrGl1Invalidate | GcrGl2Unshared | GcrGl2Invalidate |
-                                              GcrGl2Writeback | GcrOrder012 | GcrOrder210;
-constexpr uint32_t RegisterSelectorMask     = 0x70000000u;
+constexpr uint32_t RegisterSelectorMask = 0x70000000u;
 
 constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
 	return raw_offset & ~RegisterSelectorMask;
 }
 
-uint32_t ReleaseMemCacheActionFromGcr(uint32_t gcr_cntl) {
-	return ((gcr_cntl & GcrGl2Writeback) != 0 ? 0x38u : 0x00u);
-}
-
-void LogUnknownReleaseMemGcr(uint32_t gcr_cntl) {
-	const uint32_t unknown = (gcr_cntl & ~GcrKnownMask);
-	if (unknown != 0) {
-		LOGF("\t warning: release_mem uses unknown GCR bits: 0x%04" PRIx32 "\n", unknown);
-	}
-}
-
 } // namespace
 
 static HW::BlendControl DecodeBlendControl(uint32_t value) {
@@ -1068,9 +1046,8 @@ KYTY_HW_SH_PARSER(HwShSetCsUserSgpr) {
 	auto write_num = UserSgprWriteNum(slot, reg_num, 16u, "cs");
 
 	for (uint32_t i = 0; i < write_num; i++) {
-		cp.GetShCtx().SetCsUserSgpr(slot + i, buffer[i], cp.GetUserDataMarker());
+		cp.GetShCtx().SetCsUserSgpr(slot + i, buffer[i]);
 	}
-	cp.SetUserDataMarker(HW::UserSgprType::Unknown);
 
 	return reg_num;
 }
@@ -1085,9 +1062,8 @@ KYTY_HW_SH_PARSER(HwShSetPsUserSgpr) {
 	auto write_num = UserSgprWriteNum(slot, reg_num, 32u, "ps");
 
 	for (uint32_t i = 0; i < write_num; i++) {
-		cp.GetShCtx().SetPsUserSgpr(slot + i, buffer[i], cp.GetUserDataMarker());
+		cp.GetShCtx().SetPsUserSgpr(slot + i, buffer[i]);
 	}
-	cp.SetUserDataMarker(HW::UserSgprType::Unknown);
 
 	return reg_num;
 }
@@ -1102,9 +1078,8 @@ KYTY_HW_SH_PARSER(HwShSetGsUserSgpr) {
 	auto write_num = UserSgprWriteNum(slot, reg_num, 32u, "gs");
 
 	for (uint32_t i = 0; i < write_num; i++) {
-		cp.GetShCtx().SetGsUserSgpr(slot + i, buffer[i], cp.GetUserDataMarker());
+		cp.GetShCtx().SetGsUserSgpr(slot + i, buffer[i]);
 	}
-	cp.SetUserDataMarker(HW::UserSgprType::Unknown);
 
 	return reg_num;
 }
@@ -1119,9 +1094,8 @@ KYTY_HW_SH_PARSER(HwShSetHsUserSgpr) {
 	auto write_num = UserSgprWriteNum(slot, reg_num, 32u, "hs");
 
 	for (uint32_t i = 0; i < write_num; i++) {
-		cp.GetShCtx().SetHsUserSgpr(slot + i, buffer[i], cp.GetUserDataMarker());
+		cp.GetShCtx().SetHsUserSgpr(slot + i, buffer[i]);
 	}
-	cp.SetUserDataMarker(HW::UserSgprType::Unknown);
 
 	return reg_num;
 }
@@ -1700,24 +1674,6 @@ KYTY_CP_OP_PARSER(CpOpContextState) {
 	return packet_size_dw - 1u;
 }
 
-KYTY_CP_OP_PARSER(CpOpDumpConstRam) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0038300);
-
-	auto  offset = buffer[0];
-	auto  dw_num = buffer[1];
-	auto* dst = reinterpret_cast<uint32_t*>(buffer[2] | (static_cast<uint64_t>(buffer[3]) << 32u));
-
-	EXIT_NOT_IMPLEMENTED(dw_num >= 0x3000);
-	EXIT_NOT_IMPLEMENTED(offset > 0xbffc);
-	EXIT_NOT_IMPLEMENTED((offset & 0x3u) != 0);
-
-	cp.DumpConstRam(dst, offset, dw_num);
-
-	return 4;
-}
-
 KYTY_CP_OP_PARSER(CpOpEventWrite) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -1738,90 +1694,16 @@ KYTY_CP_OP_PARSER(CpOpEventWrite) {
 	return packet_size_dw - 1u;
 }
 
-KYTY_CP_OP_PARSER(CpOpEventWriteEop) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0044700);
-
-	uint32_t cache_policy       = (buffer[0] >> 25u) & 0x3u;
-	uint32_t event_write_dest   = ((buffer[0] >> 23u) & 0x10u) | ((buffer[2] >> 16u) & 0x01u);
-	uint32_t eop_event_type     = (buffer[0]) & 0x3fu;
-	uint32_t cache_action       = (buffer[0] >> 12u) & 0x3fu;
-	uint32_t event_index        = (buffer[0] >> 8u) & 0x7u;
-	uint32_t event_write_source = ((buffer[2] >> 29u) & 0x7u);
-	uint32_t interrupt_selector = (buffer[2] >> 24u) & 0x7u;
-	auto*    dst_gpu_addr =
-	    reinterpret_cast<void*>(buffer[1] | (static_cast<uint64_t>(buffer[2] & 0xffffu) << 32u));
-	uint64_t value = (buffer[3] | (static_cast<uint64_t>(buffer[4]) << 32u));
-
-	cp.WriteAtEndOfPipe64(cache_policy, event_write_dest, eop_event_type, cache_action, event_index,
-	                      event_write_source, dst_gpu_addr, value, interrupt_selector);
-
-	return 5;
-}
-
-KYTY_CP_OP_PARSER(CpOpEventWriteEos) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0034802);
-
-	uint32_t cache_policy       = (buffer[0] >> 25u) & 0x3u;
-	uint32_t event_write_dest   = 0;
-	uint32_t eop_event_type     = (buffer[0]) & 0x3fu;
-	uint32_t cache_action       = (buffer[0] >> 12u) & 0x3fu;
-	uint32_t event_index        = (buffer[0] >> 8u) & 0x7u;
-	uint32_t event_write_source = ((buffer[2] >> 29u) & 0x7u);
-	uint32_t interrupt_selector = (buffer[2] >> 24u) & 0x7u;
-
-	auto* dst_gpu_addr =
-	    reinterpret_cast<void*>(buffer[1] | (static_cast<uint64_t>(buffer[2] & 0xffffu) << 32u));
-	uint32_t value = buffer[3];
-
-	cp.WriteAtEndOfPipe32(cache_policy, event_write_dest, eop_event_type, cache_action, event_index,
-	                      event_write_source, dst_gpu_addr, value, interrupt_selector);
-
-	return 4;
-}
-
 KYTY_CP_OP_PARSER(CpOpFlip) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc004105c);
 
-	CommandProcessor::FlipInfo f;
-
-	f.handle    = static_cast<int>(buffer[0]);
-	f.index     = static_cast<int>(buffer[1]);
-	f.flip_mode = static_cast<int>(buffer[2]);
-	f.flip_arg  = static_cast<int64_t>(buffer[3] | (static_cast<uint64_t>(buffer[4]) << 32u));
-
-	cp.SetFlip(f);
-	cp.Flip();
+	cp.Flip(static_cast<int>(buffer[0]), static_cast<int>(buffer[1]), static_cast<int>(buffer[2]),
+	        static_cast<int64_t>(buffer[3] | (static_cast<uint64_t>(buffer[4]) << 32u)));
 	return 5;
 }
 
-KYTY_CP_OP_PARSER(CpOpIncrementCeCounter) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008400);
-	EXIT_NOT_IMPLEMENTED(buffer[0] != 1);
-
-	cp.IncrementCe();
-
-	return 1;
-}
-
-KYTY_CP_OP_PARSER(CpOpIncrementDeCounter) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008500);
-	EXIT_NOT_IMPLEMENTED(buffer[0] != 0);
-
-	cp.IncrementDe();
-
-	return 1;
-}
-
 KYTY_CP_OP_PARSER(CpOpIndexType) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -2103,50 +1985,12 @@ KYTY_CP_OP_PARSER(CpOpIndirectUcRegs) {
 	return KYTY_PM4_LEN(cmd_id) - 1u;
 }
 
-KYTY_CP_OP_PARSER(CpOpMarker) {
-	KYTY_PROFILER_FUNCTION();
-
-	uint32_t len_dw = ((cmd_id >> 16u) & 0x3fffu);
-
-	switch (buffer[0]) {
-		case 0x68750004: cp.SetUserDataMarker(HW::UserSgprType::Vsharp); break;
-		case 0x6875000d: cp.SetUserDataMarker(HW::UserSgprType::Region); break;
-		case 0x68750777: {
-			cp.Flip();
-			break;
-		}
-		case 0x68750778: {
-			auto* addr =
-			    reinterpret_cast<void*>(buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u));
-			uint32_t value = buffer[3];
-			cp.Flip(addr, value);
-			break;
-		}
-		case 0x68750781: {
-			auto* addr =
-			    reinterpret_cast<void*>(buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u));
-			uint32_t value          = buffer[3];
-			uint32_t eop_event_type = buffer[4];
-			uint32_t cache_action   = buffer[5];
-			cp.FlipWithInterrupt(eop_event_type, cache_action, addr, value);
-			break;
-		}
-		// NOP payloads are caller-owned data; only complete known marker tags execute.
-		default: break;
-	}
-
-	return len_dw + 1;
-}
-
 KYTY_CP_OP_PARSER(CpOpNop) {
 	KYTY_PROFILER_FUNCTION();
 
 	auto r = KYTY_PM4_R(cmd_id);
 
 	if (r == Pm4::R_ZERO) {
-		if ((buffer[0] & 0xffff0000u) == 0x68750000u) {
-			return CpOpMarker(cp, cmd_id, buffer, dw, num_dw);
-		}
 		return KYTY_PM4_LEN(cmd_id) - 1;
 	}
 
@@ -2202,108 +2046,14 @@ KYTY_CP_OP_PARSER(CpOpPushMarker) {
 KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 	KYTY_PROFILER_FUNCTION();
 
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0061060);
+	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0064900);
 
-	uint32_t cache_policy       = (buffer[0] >> 25u) & 0x3u;
-	uint32_t eop_event_type     = buffer[0] & 0x3fu;
-	uint32_t event_index        = (buffer[0] >> 8u) & 0x7u;
-	uint32_t gcr_cntl           = (buffer[0] >> 12u) & 0xfffu;
-	uint32_t release_dst        = (buffer[1] >> 16u) & 0x3u;
-	uint32_t event_write_dest   = 0;
-	uint32_t data_sel           = (buffer[1] >> 29u) & 0x7u;
-	uint32_t interrupt_selector = (buffer[1] >> 24u) & 0x7u;
-	auto*    dst_gpu_addr =
-	    reinterpret_cast<void*>(buffer[2] | (static_cast<uint64_t>(buffer[3]) << 32u));
-	uint64_t value                = buffer[4] | (static_cast<uint64_t>(buffer[5]) << 32u);
-	uint32_t interrupt_context_id = buffer[6] & 0x07ffffffu;
-
-	constexpr uint32_t ReleaseMemDstMemory = 0;
-	constexpr uint32_t ReleaseMemDstTcL2   = 1;
-	EXIT_NOT_IMPLEMENTED(release_dst != ReleaseMemDstMemory && release_dst != ReleaseMemDstTcL2);
-	EXIT_NOT_IMPLEMENTED(data_sel != 0 && data_sel != 1 && data_sel != 2 && data_sel != 3 &&
-	                     data_sel != 5);
-
-	LogUnknownReleaseMemGcr(gcr_cntl);
-
-	const bool gl2_writeback = ((gcr_cntl & GcrGl2Writeback) != 0);
-
-	auto trigger_interrupt = [&]() {
-		switch (interrupt_selector) {
-			case 0x00:
-			case 0x03: break;
-			case 0x01:
-			case 0x02:
-			case 0x04:
-				cp.TriggerEopEventAtEndOfPipe(interrupt_context_id);
-				cp.BufferFlush();
-				break;
-			default: EXIT("unknown release_mem interrupt selector\n");
-		}
-	};
-
-	// Cache actions only order GPU work, which the renderer's resource barriers already order.
-	if (data_sel == 0 || interrupt_selector == 4 ||
-	    (release_dst == ReleaseMemDstMemory && dst_gpu_addr == nullptr)) {
-		trigger_interrupt();
-
-		return 7;
-	}
-
-	auto cache_action = ReleaseMemCacheActionFromGcr(gcr_cntl);
-	cache_policy      = 0;
-
-	if (data_sel == 1) {
-		eop_event_type    = 0x2f;
-		event_index       = 6;
-		auto event_source = 2u;
-
-		cp.WriteAtEndOfPipe32(cache_policy, event_write_dest, eop_event_type, cache_action,
-		                      event_index, event_source, dst_gpu_addr, static_cast<uint32_t>(value),
-		                      interrupt_selector, interrupt_context_id);
-		if (interrupt_selector == 0x01 || interrupt_selector == 0x02) {
-			cp.BufferFlush();
-		}
-
-		return 7;
-	}
-
-	if (data_sel == 5) {
-		if (gl2_writeback) {
-			cache_action = 0;
-		}
-
-		eop_event_type    = 0x2f;
-		event_index       = 6;
-		auto event_source = 1u;
-
-		cp.WriteAtEndOfPipe32(cache_policy, event_write_dest, eop_event_type, cache_action,
-		                      event_index, event_source, dst_gpu_addr, static_cast<uint32_t>(value),
-		                      interrupt_selector, interrupt_context_id);
-		if (interrupt_selector == 0x01) {
-			cp.BufferFlush();
-		}
-
-		return 7;
-	}
-
-	const bool keep_native_event_index =
-	    (eop_event_type == 0x04 && cache_action == 0x00) ||
-	    (eop_event_type == 0x2f && event_index == 0x06 && cache_action == 0x38);
-
-	if (data_sel == 2) {
-		if (!keep_native_event_index) {
-			event_index = 0;
-		}
-	} else {
-		EXIT_IF(data_sel != 3);
-		if (!keep_native_event_index) {
-			event_index = 0;
-		}
-		data_sel = 4;
-	}
-
-	cp.WriteAtEndOfPipe64(cache_policy, event_write_dest, eop_event_type, cache_action, event_index,
-	                      data_sel, dst_gpu_addr, value, interrupt_selector, interrupt_context_id);
+	// The event type, cache actions and destination cache only order GPU work, which the renderer's
+	// resource barriers already order.
+	cp.ReleaseMem(
+	    (buffer[1] >> 29u) & 0x7u, (buffer[1] >> 24u) & 0x7u,
+	    reinterpret_cast<void*>((buffer[2] & ~0x3u) | (static_cast<uint64_t>(buffer[3]) << 32u)),
+	    buffer[4] | (static_cast<uint64_t>(buffer[5]) << 32u), buffer[6] & 0x07ffffffu);
 
 	return 7;
 }
@@ -2524,42 +2274,6 @@ KYTY_CP_OP_PARSER(CpOpWaitRegMem64) {
 	return CpOpWaitRegMemSized<uint64_t>(cp, cmd_id, buffer);
 }
 
-KYTY_CP_OP_PARSER(CpOpWaitOnCeCounter) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008600);
-	EXIT_NOT_IMPLEMENTED(buffer[0] != 1);
-
-	cp.WaitCe();
-
-	return 1;
-}
-
-KYTY_CP_OP_PARSER(CpOpWaitOnDeCounterDiff) {
-	KYTY_PROFILER_FUNCTION();
-
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0008800);
-
-	cp.WaitDeDiff(buffer[0]);
-
-	return 1;
-}
-
-KYTY_CP_OP_PARSER(CpOpWriteConstRam) {
-	KYTY_PROFILER_FUNCTION();
-
-	auto dw_num = (cmd_id >> 16u) & 0x3fffu;
-	auto offset = buffer[0];
-
-	EXIT_NOT_IMPLEMENTED(dw_num >= 0x3000);
-	EXIT_NOT_IMPLEMENTED(offset > 0xbffc);
-	EXIT_NOT_IMPLEMENTED((offset & 0x3u) != 0);
-
-	cp.WriteConstRam(offset, buffer + 1, dw_num);
-
-	return 1 + dw_num;
-}
-
 KYTY_CP_OP_PARSER(CpOpWriteData) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -3519,9 +3233,7 @@ void GraphicsInitJmpTablesShIndirect() {
 
 	for (uint32_t slot = 0; slot < 16; slot++) {
 		g_hw_sh_indirect_func[Pm4::COMPUTE_USER_DATA_0 + slot] = [](KYTY_HW_SH_INDIRECT_ARGS) {
-			auto sgpr = cmd_offset - Pm4::COMPUTE_USER_DATA_0;
-			cp.GetShCtx().SetCsUserSgpr(sgpr, value, cp.GetUserDataMarker());
-			cp.SetUserDataMarker(HW::UserSgprType::Unknown);
+			cp.GetShCtx().SetCsUserSgpr(cmd_offset - Pm4::COMPUTE_USER_DATA_0, value);
 		};
 	}
 	for (uint32_t slot = 0; slot < 4; slot++) {
@@ -3532,21 +3244,15 @@ void GraphicsInitJmpTablesShIndirect() {
 	for (uint32_t slot = 0; slot < 32; slot++) {
 		g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_PS_0 + slot] =
 		    [](KYTY_HW_SH_INDIRECT_ARGS) {
-			    auto sgpr = cmd_offset - Pm4::SPI_SHADER_USER_DATA_PS_0;
-			    cp.GetShCtx().SetPsUserSgpr(sgpr, value, cp.GetUserDataMarker());
-			    cp.SetUserDataMarker(HW::UserSgprType::Unknown);
+			    cp.GetShCtx().SetPsUserSgpr(cmd_offset - Pm4::SPI_SHADER_USER_DATA_PS_0, value);
 		    };
 		g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_GS_0 + slot] =
 		    [](KYTY_HW_SH_INDIRECT_ARGS) {
-			    auto sgpr = cmd_offset - Pm4::SPI_SHADER_USER_DATA_GS_0;
-			    cp.GetShCtx().SetGsUserSgpr(sgpr, value, cp.GetUserDataMarker());
-			    cp.SetUserDataMarker(HW::UserSgprType::Unknown);
+			    cp.GetShCtx().SetGsUserSgpr(cmd_offset - Pm4::SPI_SHADER_USER_DATA_GS_0, value);
 		    };
 		g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_HS_0 + slot] =
 		    [](KYTY_HW_SH_INDIRECT_ARGS) {
-			    auto sgpr = cmd_offset - Pm4::SPI_SHADER_USER_DATA_HS_0;
-			    cp.GetShCtx().SetHsUserSgpr(sgpr, value, cp.GetUserDataMarker());
-			    cp.SetUserDataMarker(HW::UserSgprType::Unknown);
+			    cp.GetShCtx().SetHsUserSgpr(cmd_offset - Pm4::SPI_SHADER_USER_DATA_HS_0, value);
 		    };
 	}
 	for (uint32_t slot = 0; slot < 4; slot++) {

src/graphics/guest_gpu/graphicsRun.cpp

@@ -127,16 +127,14 @@ void GuestGpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	done.acquire();
 }
 
-void GuestGpu::Submit(std::span<const uint32_t> draw_commands,
-                      std::span<const uint32_t> constant_commands) {
-	if (draw_commands.empty()) {
+void GuestGpu::Submit(std::span<const uint32_t> commands) {
+	if (commands.empty()) {
 		return;
 	}
 	Submission submission;
-	submission.type              = SubmissionType::Graphics;
-	submission.queue_id          = 0;
-	submission.commands          = draw_commands;
-	submission.constant_commands = constant_commands;
+	submission.type     = SubmissionType::Graphics;
+	submission.queue_id = 0;
+	submission.commands = commands;
 	Enqueue(std::move(submission));
 }
 
@@ -200,11 +198,8 @@ void CommandProcessor::Reset() {
 	m_num_instances                    = 1;
 	m_num_instances_addr               = 0;
 	m_predicate_skip                   = false;
-	m_user_data_marker                 = HW::UserSgprType::Unknown;
 	m_draw_indirect_args_base_addr     = 0;
 	m_dispatch_indirect_args_base_addr = 0;
-
-	std::memset(m_const_ram, 0, sizeof(m_const_ram));
 }
 
 void CommandProcessor::ApplyContextStateOperation(ContextStateOperation operation) {
@@ -237,6 +232,8 @@ void CommandProcessor::BufferInit() {
 
 void CommandProcessor::BufferFlush() {
 	GetScheduler().Flush();
+	// The occlusion counters reach a guest that polls them without waiting for a later fence.
+	m_renderer.GetOcclusionQueries().Publish();
 }
 
 void CommandProcessor::BufferWait() {
@@ -244,47 +241,12 @@ void CommandProcessor::BufferWait() {
 	GetScheduler().Finish();
 }
 
-void CommandProcessor::ResetDeCe() {
-	m_de_count    = 0;
-	m_ce_count    = 0;
-	m_ce_complete = false;
-}
-
-void CommandProcessor::WaitCe() {
-	if (m_ce_count <= m_de_count && !m_ce_complete) {
-		SuspendPm4();
-	}
-}
-
-void CommandProcessor::WaitDeDiff(uint32_t diff) {
-	EXIT_IF(m_de_count > m_ce_count);
-	if (m_ce_count - m_de_count >= diff) {
-		SuspendPm4();
-	}
-}
-
 void CommandProcessor::WaitForRewind(bool valid) {
 	if (!valid) {
 		SuspendPm4();
 	}
 }
 
-void CommandProcessor::IncrementDe() {
-	m_de_count++;
-}
-
-void CommandProcessor::IncrementCe() {
-	m_ce_count++;
-}
-
-void CommandProcessor::WriteConstRam(uint32_t offset, const uint32_t* src, uint32_t dw_num) {
-	memcpy(m_const_ram + offset / 4, src, static_cast<size_t>(dw_num) * 4);
-}
-
-void CommandProcessor::DumpConstRam(uint32_t* dst, uint32_t offset, uint32_t dw_num) {
-	memcpy(dst, m_const_ram + offset / 4, static_cast<size_t>(dw_num) * 4);
-}
-
 bool TestWaitRegMemValue(uint64_t value, uint64_t ref, uint64_t mask, uint32_t func) {
 	switch (func) {
 		case 0: return true;
@@ -539,64 +501,34 @@ bool GuestGpu::Process(Submission& submission) {
 	if (first_slice) {
 		submission.started = true;
 		cp.SetSubmitId(++m_submit_id);
-		cp.ResetDeCe();
-		cp.SetFlip({});
 	}
 
 	cp.BufferInit();
 	bool complete = true;
 
 	switch (submission.type) {
-		case SubmissionType::Graphics: {
-			bool progressed = false;
-			submission.constant_complete |= submission.constant_commands.empty();
-			for (;;) {
-				bool round_progress = false;
-				if (!submission.constant_complete) {
-					submission.constant_complete =
-					    cp.Process(submission.constant_execution, submission.constant_commands) ==
-					    Pm4ProcessResult::Complete;
-					round_progress |= submission.constant_execution.MadeProgress();
-				}
-				cp.SetCeComplete(submission.constant_complete);
-				if (!submission.command_complete) {
-					submission.command_complete =
-					    cp.Process(submission.command_execution, submission.commands) ==
-					    Pm4ProcessResult::Complete;
-					round_progress |= submission.command_execution.MadeProgress();
-				}
-				progressed |= round_progress;
-				complete = submission.command_complete && submission.constant_complete;
-				if (complete || !round_progress) {
-					break;
-				}
-			}
-			if (progressed) {
-				cp.BufferFlush();
-			}
-			break;
-		}
-		case SubmissionType::Compute: {
-			const auto      num_dw = static_cast<uint32_t>(submission.commands.size());
-			const auto*     buffer = submission.commands.data();
-			static uint32_t compute_batch_log_count = 0;
-			if (first_slice && num_dw <= 128 && compute_batch_log_count++ < 32) {
-				LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n",
-				     reinterpret_cast<uint64_t>(buffer), num_dw);
-				for (uint32_t i = 0; i < std::min<uint32_t>(num_dw, 16); i++) {
-					LOGF("\t compute[%02" PRIu32 "] = 0x%08" PRIx32 "\n", i, buffer[i]);
-				}
-			}
+		case SubmissionType::Compute:
 			if (first_slice) {
+				const auto      num_dw = static_cast<uint32_t>(submission.commands.size());
+				const auto*     buffer = submission.commands.data();
+				static uint32_t compute_batch_log_count = 0;
+				if (num_dw <= 128 && compute_batch_log_count++ < 32) {
+					LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n",
+					     reinterpret_cast<uint64_t>(buffer), num_dw);
+					for (uint32_t i = 0; i < std::min<uint32_t>(num_dw, 16); i++) {
+						LOGF("\t compute[%02" PRIu32 "] = 0x%08" PRIx32 "\n", i, buffer[i]);
+					}
+				}
 				GraphicsDbgDumpDcb("cc", num_dw, buffer);
 			}
-			complete = cp.Process(submission.command_execution, submission.commands) ==
-			           Pm4ProcessResult::Complete;
-			if (submission.command_execution.MadeProgress()) {
+			[[fallthrough]];
+		case SubmissionType::Graphics:
+			complete =
+			    cp.Process(submission.execution, submission.commands) == Pm4ProcessResult::Complete;
+			if (submission.execution.MadeProgress()) {
 				cp.BufferFlush();
 			}
 			break;
-		}
 		case SubmissionType::FlipPreparation:
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
@@ -699,18 +631,6 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution) {
 				     opcode, KYTY_PM4_R(packet_header), packet_dw,
 				     reinterpret_cast<uint64_t>(packet), packet_header);
 			}
-			if (opcode == Pm4::IT_NOP && KYTY_PM4_R(packet_header) == Pm4::R_RELEASE_MEM &&
-			    packet_dw >= 7) {
-				static std::atomic<uint32_t> log_count {0};
-				if (log_count.fetch_add(1) < 128) {
-					const auto dst = packet[3] | (static_cast<uint64_t>(packet[4]) << 32u);
-					const auto val = packet[5] | (static_cast<uint64_t>(packet[6]) << 32u);
-					LOGF("\t predicated skip: R_RELEASE_MEM dst=0x%016" PRIx64
-					     ", value=0x%016" PRIx64 ", action=0x%08" PRIx32
-					     ", gcr/data/int=0x%08" PRIx32 "\n",
-					     dst, val, packet[1], packet[2]);
-				}
-			}
 			cursor.offset_dw += packet_dw;
 			execution.m_made_progress = true;
 			continue;
@@ -803,6 +723,7 @@ void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t
 			return;
 		case 0x01: {
 			EXIT_NOT_IMPLEMENTED(address == nullptr);
+			m_renderer.GetOcclusionQueries().Publish();
 			// One begin/end pair per DB; bit 63 marks each counter ready.
 			constexpr uint64_t ready_bit = 1ull << 63u;
 			const auto* results = reinterpret_cast<const volatile uint64_t*>(address);
@@ -1015,209 +936,61 @@ void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_
 	}
 }
 
-template <typename T>
-void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_write_dest,
-                                        uint32_t eop_event_type, uint32_t cache_action,
-                                        uint32_t event_index, uint32_t event_write_source,
-                                        void* dst_gpu_addr, T value, uint32_t interrupt_selector,
-                                        uint32_t interrupt_context_id) {
-	static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
-
-	auto& command = CurrentBuffer();
-
+void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector, void* address,
+                                  uint64_t data, uint32_t interrupt_context_id) {
 	if (GraphicsRunDebugDumpEnabled()) {
-		const auto bits      = static_cast<unsigned>(sizeof(T) * 8u);
-		const auto log_width = static_cast<int>(sizeof(T) * 2u);
-
-		LOGF("CommandProcessor::WriteAtEndOfPipe%u()\n"
-		     "\t cache_policy        = 0x%08" PRIx32 "\n"
-		     "\t event_write_dest    = 0x%08" PRIx32 "\n"
-		     "\t eop_event_type      = 0x%08" PRIx32 "\n"
-		     "\t cache_action        = 0x%08" PRIx32 "\n"
-		     "\t event_index         = 0x%08" PRIx32 "\n"
-		     "\t event_write_source  = 0x%08" PRIx32 "\n"
-		     "\t interrupt_selector  = 0x%08" PRIx32 "\n"
-		     "\t interrupt_context   = 0x%08" PRIx32 "\n"
-		     "\t dst_gpu_addr        = 0x%016" PRIx64 "\n"
-		     "\t value               = 0x%0*" PRIx64 "\n",
-		     bits, cache_policy, event_write_dest, eop_event_type, cache_action, event_index,
-		     event_write_source, interrupt_selector, interrupt_context_id,
-		     reinterpret_cast<uint64_t>(dst_gpu_addr), log_width, static_cast<uint64_t>(value));
+		LOGF("CommandProcessor::ReleaseMem()\n"
+		     "\t data_sel           = %" PRIu32 "\n"
+		     "\t interrupt_selector = %" PRIu32 "\n"
+		     "\t address            = 0x%016" PRIx64 "\n"
+		     "\t data               = 0x%016" PRIx64 "\n"
+		     "\t interrupt_context  = 0x%08" PRIx32 "\n",
+		     data_sel, interrupt_selector, reinterpret_cast<uint64_t>(address), data,
+		     interrupt_context_id);
 	}
 
-	EXIT_NOT_IMPLEMENTED(cache_policy != 0x00000000);
-	EXIT_NOT_IMPLEMENTED(event_write_dest != 0x00000000);
+	// QueuedInterrupt: kNone, kOnly, kAfterWriteConfirm, kDataAfterWriteConfirm, kSendIntCtx.
+	EXIT_NOT_IMPLEMENTED(interrupt_selector > 4);
+	const bool interrupt = (interrupt_selector != 0 && interrupt_selector != 3);
+	// kSendIntCtx has no label, and kOnly writes none on graphics queues.
+	if (address == nullptr || interrupt_selector == 4 ||
+	    (interrupt_selector == 1 && !IsAsyncComputeQueue())) {
+		data_sel = 0;
+	}
+	if (data_sel == 0 && !interrupt) {
+		return;
+	}
 
-	bool with_interrupt = false;
-	switch (interrupt_selector) {
-		case 0x00:
-		case 0x03: with_interrupt = false; break;
-		case 0x01:
-			if (!IsAsyncComputeQueue()) {
-				TriggerEopEventAtEndOfPipe(interrupt_context_id);
-				return;
-			}
-			with_interrupt = true;
+	// Dumped occlusion counters are written before any end-of-pipe signal reaches the guest.
+	m_renderer.GetOcclusionQueries().Publish();
+
+	// EndOfPipeData: kNone, kImmediate32b, kImmediate64b, kRefClk; 5 reads GDS.
+	switch (data_sel) {
+		case 0: break;
+		case 1: {
+			const auto value = static_cast<uint32_t>(data);
+			std::memcpy(address, &value, sizeof(value));
 			break;
-		case 0x02: with_interrupt = true; break;
-		default: EXIT("unknown interrupt_selector\n");
-	}
-
-	auto write32 = [&] {
-		auto* dst  = static_cast<uint32_t*>(dst_gpu_addr);
-		auto  data = static_cast<uint32_t>(value);
-		EXIT_IF(dst == nullptr);
-		(void)command.Handle();
-		std::memcpy(dst, &data, sizeof(data));
-
-		if (with_interrupt) {
-			TriggerEopEventAtEndOfPipe(interrupt_context_id);
 		}
-	};
-
-	switch (event_write_source) {
-		case 0x01:
-			if constexpr (sizeof(T) == sizeof(uint32_t)) {
-				if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
-					auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
-					SynchronizeGpu();
-					Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
-					              value >> 16u);
-					if (with_interrupt) {
-						m_renderer.TriggerInterrupt(m_interrupt_event_id, interrupt_context_id);
-					}
-					return;
-				}
-			} else if (eop_event_type == 0x04 && cache_action == 0x00 && event_index == 0x05) {
-				write32();
-				return;
-			}
+		case 3: data = Sync::ReadReferenceClock(); [[fallthrough]];
+		case 2: std::memcpy(address, &data, sizeof(data)); break;
+		case 5: {
+			GetScheduler().Finish();
+			const auto gds = static_cast<uint32_t>(data);
+			Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(),
+			              static_cast<uint32_t*>(address), gds & 0xffffu, gds >> 16u);
 			break;
-		case 0x02:
-		case 0x04:
-			if constexpr (sizeof(T) == sizeof(uint32_t)) {
-				if (event_write_source == 0x02 && eop_event_type == 0x2f && event_index == 0x06) {
-					switch (cache_action) {
-						case 0x00:
-						case 0x38: write32(); return;
-						default: break;
-					}
-				}
-			} else {
-				if (event_write_source == 0x04) {
-					value = Sync::ReadReferenceClock();
-				}
-				auto write64 = [&] {
-					auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
-					EXIT_IF(dst == nullptr);
-					(void)command.Handle();
-					std::memcpy(dst, &value, sizeof(value));
-
-					if (with_interrupt) {
-						TriggerEopEventAtEndOfPipe(interrupt_context_id);
-					}
-				};
-
-				switch (cache_action) {
-					case 0x00:
-						switch (eop_event_type) {
-							case 0x04:
-								if (event_index == 0x05) {
-									write64();
-									return;
-								}
-								break;
-							case 0x14:
-							case 0x28:
-							case 0x2f:
-								if (event_index == 0x00) {
-									write64();
-									return;
-								}
-								break;
-							case 0x2b:
-							case 0x2d:
-							case 0x30:
-								if (event_index == 0x00 && !with_interrupt) {
-									write64();
-									return;
-								}
-								break;
-							default: break;
-						}
-						break;
-					case 0x38:
-						switch (eop_event_type) {
-							case 0x04:
-							case 0x14:
-							case 0x28:
-								if (((eop_event_type == 0x04 || eop_event_type == 0x28) &&
-								     event_index == 0x05) ||
-								    (event_index == 0x00)) {
-									write64();
-									return;
-								}
-								break;
-							case 0x2b:
-							case 0x2d:
-								if (event_index == 0x00 && !with_interrupt) {
-									write64();
-									return;
-								}
-								break;
-							case 0x2f:
-								if (event_index == 0x06 && !with_interrupt) {
-									write64();
-									return;
-								}
-								break;
-							default: break;
-						}
-						break;
-					case 0x3b:
-						if (eop_event_type == 0x04 && event_index == 0x05 && with_interrupt) {
-							write64();
-							return;
-						}
-						break;
-					default: break;
-				}
-			}
-			break;
-		default: break;
+		}
+		default: EXIT("unknown release_mem data selector: %" PRIu32 "\n", data_sel);
 	}
 
-	EXIT("unknown event type\n");
-}
-
-void CommandProcessor::WriteAtEndOfPipe32(uint32_t cache_policy, uint32_t event_write_dest,
-                                          uint32_t eop_event_type, uint32_t cache_action,
-                                          uint32_t event_index, uint32_t event_write_source,
-                                          void* dst_gpu_addr, uint32_t value,
-                                          uint32_t interrupt_selector,
-                                          uint32_t interrupt_context_id) {
-	WriteAtEndOfPipe(cache_policy, event_write_dest, eop_event_type, cache_action, event_index,
-	                 event_write_source, dst_gpu_addr, value, interrupt_selector,
-	                 interrupt_context_id);
-}
-
-void CommandProcessor::WriteAtEndOfPipe64(uint32_t cache_policy, uint32_t event_write_dest,
-                                          uint32_t eop_event_type, uint32_t cache_action,
-                                          uint32_t event_index, uint32_t event_write_source,
-                                          void* dst_gpu_addr, uint64_t value,
-                                          uint32_t interrupt_selector,
-                                          uint32_t interrupt_context_id) {
-	WriteAtEndOfPipe(cache_policy, event_write_dest, eop_event_type, cache_action, event_index,
-	                 event_write_source, dst_gpu_addr, value, interrupt_selector,
-	                 interrupt_context_id);
-}
-
-void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) {
-	(void)CurrentBuffer().Handle();
-	GetScheduler().DeferPriorityOperation(
-	    [&renderer = m_renderer, event_id = m_interrupt_event_id, interrupt_context_id] {
-		    renderer.TriggerInterrupt(event_id, interrupt_context_id);
-	    });
+	// GuestGpu::Process's end-of-slice flush submits the work this interrupt waits on.
+	if (interrupt) {
+		GetScheduler().DeferPriorityOperation(
+		    [&renderer = m_renderer, event_id = m_interrupt_event_id, interrupt_context_id] {
+			    renderer.TriggerInterrupt(event_id, interrupt_context_id);
+		    });
+	}
 }
 
 void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
@@ -1270,15 +1043,19 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
 				     "\n",
 				     event_index, event_address);
 			}
+			if (auto& queries = m_renderer.GetOcclusionQueries(); queries.Enabled()) {
+				queries.Dump(event_address);
+				break;
+			}
 			static std::once_flag warning_once;
 			std::call_once(warning_once, [] {
-				std::printf("Warning: game uses occlusion queries, which are currently treated as "
-				            "always visible; GPU usage may be higher and FPS may be lower.\n");
+				std::printf("Warning: game uses occlusion queries, which are treated as always "
+				            "visible without --occlusion-queries; GPU usage may be higher and FPS "
+				            "may be lower.\n");
 			});
 
-			// Until host occlusion queries are implemented, publish an always-visible result. The
-			// PS5 layout contains one interleaved begin/end pair per DB, and bit 63 marks a result
-			// ready.
+			// Without host occlusion queries, publish an always-visible result. The PS5 layout
+			// contains one interleaved begin/end pair per DB, and bit 63 marks a result ready.
 			constexpr uint64_t ready_bit    = 1ull << 63u;
 			constexpr uint64_t counter_mask = ready_bit - 1u;
 			auto*              results      = reinterpret_cast<volatile uint64_t*>(event_address);
@@ -1294,69 +1071,20 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
 	}
 }
 
-void CommandProcessor::Flip() {
+void CommandProcessor::Flip(int handle, int index, int flip_mode, int64_t flip_arg) {
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::Flip()\n");
 	}
+	m_renderer.GetOcclusionQueries().Publish();
 
 	auto& command = CurrentBuffer();
-	auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode,
-	                                         m_flip.flip_arg);
+	auto  request = Sync::PrepareVideoOutFlip(command, handle, index, flip_mode, flip_arg);
 	(void)command.Handle();
 	GetScheduler().DeferPriorityOperation(
 	    [&renderer = m_renderer, request] { renderer.GetVideoOut().CompleteFlip(request); });
 	GetScheduler().Flush();
 }
 
-void CommandProcessor::Flip(void* dst_gpu_addr, uint32_t value) {
-	auto& command = CurrentBuffer();
-
-	if (GraphicsRunDebugDumpEnabled()) {
-		LOGF("CommandProcessor::Flip()\n"
-		     "\t dst_gpu_addr = 0x%016" PRIx64 "\n"
-		     "\t value        = 0x%08" PRIx32 "\n",
-		     reinterpret_cast<uint64_t>(dst_gpu_addr), value);
-	}
-
-	EXIT_IF(dst_gpu_addr == nullptr);
-	std::memcpy(dst_gpu_addr, &value, sizeof(value));
-	auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode,
-	                                         m_flip.flip_arg);
-	(void)command.Handle();
-	GetScheduler().DeferPriorityOperation(
-	    [&renderer = m_renderer, request] { renderer.GetVideoOut().CompleteFlip(request); });
-	GetScheduler().Flush();
-}
-
-void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache_action,
-                                         void* dst_gpu_addr, uint32_t value) {
-	auto& command = CurrentBuffer();
-
-	if (GraphicsRunDebugDumpEnabled()) {
-		LOGF("CommandProcessor::FlipWithInterrupt()\n"
-		     "\t eop_event_type      = 0x%08" PRIx32 "\n"
-		     "\t cache_action        = 0x%08" PRIx32 "\n"
-		     "\t dst_gpu_addr        = 0x%016" PRIx64 "\n"
-		     "\t value               = 0x%08" PRIx32 "\n",
-		     eop_event_type, cache_action, reinterpret_cast<uint64_t>(dst_gpu_addr), value);
-	}
-
-	if (eop_event_type != 0x00000004 || cache_action != 0x00000038) {
-		EXIT("unknown event type\n");
-	}
-	EXIT_IF(dst_gpu_addr == nullptr);
-	std::memcpy(dst_gpu_addr, &value, sizeof(value));
-	auto request = Sync::PrepareVideoOutFlip(command, m_flip.handle, m_flip.index, m_flip.flip_mode,
-	                                         m_flip.flip_arg);
-	(void)command.Handle();
-	GetScheduler().DeferPriorityOperation(
-	    [&renderer = m_renderer, event_id = m_interrupt_event_id, request] {
-		    renderer.GetVideoOut().CompleteFlip(request);
-		    renderer.TriggerInterrupt(event_id, 0);
-	    });
-	GetScheduler().Flush();
-}
-
 void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 	auto& command = CurrentBuffer();
 	if (g_current_processor != nullptr) {
@@ -1374,10 +1102,6 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 	GetScheduler().Flush();
 }
 
-void CommandProcessor::SynchronizeGpu() {
-	GetScheduler().Finish();
-}
-
 bool GuestGpu::IsGpuThread() noexcept {
 	return g_gpu_thread;
 }

src/graphics/guest_gpu/graphicsRun.h

@@ -35,8 +35,7 @@ public:
 	void               Wake();
 
 	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
-	void Submit(std::span<const uint32_t> draw_commands,
-	            std::span<const uint32_t> constant_commands);
+	void Submit(std::span<const uint32_t> commands);
 	void SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
 	void SubmitFlipPreparation(uint64_t request_id);
 	// Insert an ordered graphics drain; only a previous suspend point can block the caller.
@@ -60,14 +59,10 @@ private:
 		SubmissionType            type     = SubmissionType::Graphics;
 		uint32_t                  queue_id = 0;
 		std::span<const uint32_t> commands;
-		std::span<const uint32_t> constant_commands;
-		Pm4Execution              command_execution;
-		Pm4Execution              constant_execution;
-		bool                      started           = false;
-		bool                      command_complete  = false;
-		bool                      constant_complete = false;
-		bool                      blocked           = false;
-		uint64_t                  flip_request_id   = 0;
+		Pm4Execution              execution;
+		bool                      started         = false;
+		bool                      blocked         = false;
+		uint64_t                  flip_request_id = 0;
 	};
 
 	void              Enqueue(Submission submission);

src/graphics/guest_gpu/hardwareContext.h

@@ -597,14 +597,10 @@ struct ShaderRegisters {
 	}
 };
 
-enum class UserSgprType { Unknown, Region, Vsharp };
-
 struct UserSgprInfo {
 	static constexpr int SGPRS_MAX = 32;
 
-	uint32_t     value[SGPRS_MAX] = {0};
-	UserSgprType type[SGPRS_MAX]  = {};
-	uint32_t     count            = 0;
+	uint32_t value[SGPRS_MAX] = {0};
 };
 
 struct VertexShaderInfo {
@@ -1056,30 +1052,10 @@ public:
 	void SetCsShader(const CsStageRegisters& cs_regs) { m_cs.cs_regs = cs_regs; }
 	void SetCsWaveSize(uint8_t wave_size) { m_cs.cs_regs.wave_size = wave_size; }
 
-	void SetPsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
-		m_ps.ps_user_sgpr.value[id] = value;
-		m_ps.ps_user_sgpr.type[id]  = type;
-		m_ps.ps_user_sgpr.count =
-		    ((id + 1) > m_ps.ps_user_sgpr.count ? (id + 1) : m_ps.ps_user_sgpr.count);
-	}
-	void SetCsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
-		m_cs.cs_user_sgpr.value[id] = value;
-		m_cs.cs_user_sgpr.type[id]  = type;
-		m_cs.cs_user_sgpr.count =
-		    ((id + 1) > m_cs.cs_user_sgpr.count ? (id + 1) : m_cs.cs_user_sgpr.count);
-	}
-	void SetHsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
-		m_vs.hs_user_sgpr.value[id] = value;
-		m_vs.hs_user_sgpr.type[id]  = type;
-		m_vs.hs_user_sgpr.count =
-		    ((id + 1) > m_vs.hs_user_sgpr.count ? (id + 1) : m_vs.hs_user_sgpr.count);
-	}
-	void SetGsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
-		m_vs.gs_user_sgpr.value[id] = value;
-		m_vs.gs_user_sgpr.type[id]  = type;
-		m_vs.gs_user_sgpr.count =
-		    ((id + 1) > m_vs.gs_user_sgpr.count ? (id + 1) : m_vs.gs_user_sgpr.count);
-	}
+	void SetPsUserSgpr(uint32_t id, uint32_t value) { m_ps.ps_user_sgpr.value[id] = value; }
+	void SetCsUserSgpr(uint32_t id, uint32_t value) { m_cs.cs_user_sgpr.value[id] = value; }
+	void SetHsUserSgpr(uint32_t id, uint32_t value) { m_vs.hs_user_sgpr.value[id] = value; }
+	void SetGsUserSgpr(uint32_t id, uint32_t value) { m_vs.gs_user_sgpr.value[id] = value; }
 
 	[[nodiscard]] const PixelShaderInfo&   GetPs() const { return m_ps; }
 	[[nodiscard]] const VertexShaderInfo&  GetVs() const { return m_vs; }

src/graphics/guest_gpu/pm4.cpp

@@ -25,7 +25,6 @@ constexpr auto MakeRegisterNames() {
 	names[R_POP_MARKER]     = "R_POP_MARKER";
 	names[R_WRITE_DATA]     = "R_WRITE_DATA";
 	names[R_FLIP]           = "R_FLIP";
-	names[R_RELEASE_MEM]    = "R_RELEASE_MEM";
 	names[R_DMA_DATA]       = "R_DMA_DATA";
 	names[R_CONTEXT_STATE]  = "R_CONTEXT_STATE";
 	return names;
@@ -54,7 +53,6 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_DRAW_INDIRECT_MULTI]       = "IT_DRAW_INDIRECT_MULTI";
 	names[IT_DRAW_INDEX_AUTO]           = "IT_DRAW_INDEX_AUTO";
 	names[IT_NUM_INSTANCES]             = "IT_NUM_INSTANCES";
-	names[IT_INDIRECT_BUFFER_CNST]      = "IT_INDIRECT_BUFFER_CNST";
 	names[IT_DRAW_INDEX_OFFSET_2]       = "IT_DRAW_INDEX_OFFSET_2";
 	names[IT_WRITE_DATA]                = "IT_WRITE_DATA";
 	names[IT_MEM_SEMAPHORE]             = "IT_MEM_SEMAPHORE";
@@ -66,8 +64,6 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_PFP_SYNC_ME]               = "IT_PFP_SYNC_ME";
 	names[IT_SURFACE_SYNC]              = "IT_SURFACE_SYNC";
 	names[IT_EVENT_WRITE]               = "IT_EVENT_WRITE";
-	names[IT_EVENT_WRITE_EOP]           = "IT_EVENT_WRITE_EOP";
-	names[IT_EVENT_WRITE_EOS]           = "IT_EVENT_WRITE_EOS";
 	names[IT_RELEASE_MEM]               = "IT_RELEASE_MEM";
 	names[IT_DMA_DATA]                  = "IT_DMA_DATA";
 	names[IT_ACQUIRE_MEM]               = "IT_ACQUIRE_MEM";
@@ -80,12 +76,6 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_SET_QUEUE_REG]             = "IT_SET_QUEUE_REG";
 	names[IT_SET_UCONFIG_REG]           = "IT_SET_UCONFIG_REG";
 	names[IT_SET_UCONFIG_REG_INDEX]     = "IT_SET_UCONFIG_REG_INDEX";
-	names[IT_WRITE_CONST_RAM]           = "IT_WRITE_CONST_RAM";
-	names[IT_DUMP_CONST_RAM]            = "IT_DUMP_CONST_RAM";
-	names[IT_INCREMENT_CE_COUNTER]      = "IT_INCREMENT_CE_COUNTER";
-	names[IT_INCREMENT_DE_COUNTER]      = "IT_INCREMENT_DE_COUNTER";
-	names[IT_WAIT_ON_CE_COUNTER]        = "IT_WAIT_ON_CE_COUNTER";
-	names[IT_WAIT_ON_DE_COUNTER_DIFF]   = "IT_WAIT_ON_DE_COUNTER_DIFF";
 	names[IT_DISPATCH_DRAW_PREAMBLE]    = "IT_DISPATCH_DRAW_PREAMBLE";
 	names[IT_DISPATCH_DRAW]             = "IT_DISPATCH_DRAW";
 	names[IT_GET_LOD_STATS]             = "IT_GET_LOD_STATS";

src/graphics/guest_gpu/pm4.h

@@ -40,7 +40,6 @@ constexpr uint32_t IT_INDEX_TYPE                = 0x2A;
 constexpr uint32_t IT_DRAW_INDIRECT_MULTI       = 0x2C;
 constexpr uint32_t IT_DRAW_INDEX_AUTO           = 0x2D;
 constexpr uint32_t IT_NUM_INSTANCES             = 0x2F;
-constexpr uint32_t IT_INDIRECT_BUFFER_CNST      = 0x33;
 constexpr uint32_t IT_DRAW_INDEX_OFFSET_2       = 0x35;
 constexpr uint32_t IT_WRITE_DATA                = 0x37;
 constexpr uint32_t IT_MEM_SEMAPHORE             = 0x39;
@@ -53,8 +52,6 @@ constexpr uint32_t IT_CP_DMA                    = 0x41;
 constexpr uint32_t IT_PFP_SYNC_ME               = 0x42;
 constexpr uint32_t IT_SURFACE_SYNC              = 0x43;
 constexpr uint32_t IT_EVENT_WRITE               = 0x46;
-constexpr uint32_t IT_EVENT_WRITE_EOP           = 0x47;
-constexpr uint32_t IT_EVENT_WRITE_EOS           = 0x48;
 constexpr uint32_t IT_RELEASE_MEM               = 0x49;
 constexpr uint32_t IT_DMA_DATA                  = 0x50;
 constexpr uint32_t IT_ACQUIRE_MEM               = 0x58;
@@ -67,12 +64,6 @@ constexpr uint32_t IT_SET_SH_REG                = 0x76;
 constexpr uint32_t IT_SET_QUEUE_REG             = 0x78;
 constexpr uint32_t IT_SET_UCONFIG_REG           = 0x79;
 constexpr uint32_t IT_SET_UCONFIG_REG_INDEX     = 0x7A;
-constexpr uint32_t IT_WRITE_CONST_RAM           = 0x81;
-constexpr uint32_t IT_DUMP_CONST_RAM            = 0x83;
-constexpr uint32_t IT_INCREMENT_CE_COUNTER      = 0x84;
-constexpr uint32_t IT_INCREMENT_DE_COUNTER      = 0x85;
-constexpr uint32_t IT_WAIT_ON_CE_COUNTER        = 0x86;
-constexpr uint32_t IT_WAIT_ON_DE_COUNTER_DIFF   = 0x88;
 constexpr uint32_t IT_DISPATCH_DRAW             = 0x8D;
 constexpr uint32_t IT_GET_LOD_STATS             = 0x8E;
 constexpr uint32_t IT_WAIT_REG_MEM_64           = 0x93;
@@ -88,7 +79,6 @@ constexpr uint32_t R_PUSH_MARKER    = 0x0B;
 constexpr uint32_t R_POP_MARKER     = 0x0C;
 constexpr uint32_t R_WRITE_DATA     = 0x15;
 constexpr uint32_t R_FLIP           = 0x17;
-constexpr uint32_t R_RELEASE_MEM    = 0x18;
 constexpr uint32_t R_DMA_DATA       = 0x19;
 constexpr uint32_t R_CONTEXT_STATE  = 0x1A;
 

src/graphics/host_gpu/graphicContext.h

@@ -36,6 +36,8 @@ struct GraphicContext {
 	bool                               provoking_vertex_last_enabled         = false;
 	bool                               maintenance8_enabled                  = false;
 	bool                               conditional_rendering_enabled         = false;
+	bool                               occlusion_queries_enabled             = false;
+	bool                               occlusion_query_precise_enabled       = false;
 	bool                               supports_block_texel_view              = false;
 	bool                                      mesh_shader_enabled                   = false;
 	vk::PhysicalDeviceMeshShaderPropertiesEXT mesh_shader_properties                = {};
@@ -43,6 +45,7 @@ struct GraphicContext {
 	uint32_t                           min_subgroup_size                     = 0;
 	uint32_t                           max_subgroup_size                     = 0;
 	uint32_t                           max_push_descriptors                  = 0;
+	uint32_t                           device_clock_period_ps                = 0;
 	vk::DescriptorSetLayout            image_heap_layout                     = nullptr;
 	vk::ShaderStageFlags               required_subgroup_size_stages         = {};
 	Common::Mutex                      queue_mutex;

src/graphics/host_gpu/memoryTracker.cpp

@@ -66,18 +66,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 	return ptr;
 }
 
-void MemoryTracker::QueueBdaSync(uint64_t vaddr, uint64_t size) {
-	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
-	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
-	std::lock_guard lock(m_bda_sync_mutex);
-	m_bda_sync.Add(begin, end - begin);
-}
-
-RangeSet MemoryTracker::TakeBdaSync() {
-	std::lock_guard lock(m_bda_sync_mutex);
-	return std::exchange(m_bda_sync, RangeSet {});
-}
-
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -100,7 +88,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 		std::scoped_lock lock(manager->lock);
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
-	QueueBdaSync(vaddr, size);
 }
 
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
@@ -141,8 +128,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
 	});
-	locks.clear();
-	QueueBdaSync(vaddr, size);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/memoryTracker.h

@@ -29,10 +29,6 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
-	// Queue after changing ownership; a concurrent drain must not consume an event
-	// before the dirty pages it describes become visible.
-	void                   QueueBdaSync(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] RangeSet TakeBdaSync();
 	// Removes protection from a range and flushes GPU-owned data when required.
 	template <typename Flush>
 	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
@@ -55,7 +51,6 @@ public:
 				on_flush();
 			}
 		});
-		QueueBdaSync(vaddr, size);
 	}
 #if KYTY_BUILD == KYTY_BUILD_DEBUG
 	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
@@ -154,8 +149,6 @@ private:
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
 	std::vector<std::unique_ptr<RegionManager>>    m_region_storage;
 	std::mutex                                     m_region_mutex;
-	std::mutex                                     m_bda_sync_mutex;
-	RangeSet                                       m_bda_sync;
 	PageManager&                                   m_page_manager;
 };
 

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -127,7 +127,7 @@ void BufferCache::ChangeRegister(BufferId id) {
 		}
 		WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
 		                addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
-		m_memory_tracker.QueueBdaSync(buffer.CpuAddress(), buffer.Size());
+		QueueBdaSync(buffer.CpuAddress(), buffer.Size());
 	} else {
 		const auto found = m_buffers.find(buffer.CpuAddress());
 		EXIT_IF(found == m_buffers.end() || found->second != id);
@@ -299,6 +299,7 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 	}
 	m_memory_tracker.InvalidateRegion(vaddr, size,
 	                                  [this, vaddr, size] { ReadMemory(vaddr, size, true); });
+	QueueBdaSync(vaddr, size);
 }
 
 void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
@@ -318,6 +319,7 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		}
 		if (is_write) {
 			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
+			QueueBdaSync(vaddr, size);
 		}
 	});
 }
@@ -719,14 +721,16 @@ void BufferCache::ProcessFaultBuffer() {
 	m_fault_manager.ProcessFaultBuffer();
 }
 
-void BufferCache::SynchronizeBda(const RangeSet& mapped) {
-	// Take before walking: writes queued during synchronization remain pending.
-	const auto pending = m_memory_tracker.TakeBdaSync();
-	pending.ForEach([&](uint64_t begin, uint64_t end) {
-		mapped.ForEachInRange(begin, end - begin, [this](uint64_t start, uint64_t finish) {
-			SynchronizeBuffersInRange(start, finish - start);
-		});
-	});
+void BufferCache::QueueBdaSync(uint64_t vaddr, uint64_t size) {
+	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
+	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
+	std::lock_guard lock(m_bda_sync_mutex);
+	m_bda_sync.Add(begin, end - begin);
+}
+
+RangeSet BufferCache::TakeBdaSync() {
+	std::lock_guard lock(m_bda_sync_mutex);
+	return std::exchange(m_bda_sync, {});
 }
 
 void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -13,6 +13,7 @@
 
 #include <array>
 #include <map>
+#include <mutex>
 #include <span>
 #include <utility>
 #include <vector>
@@ -85,19 +86,16 @@ public:
 	void               MarkColorMetadataChecked(uint64_t vaddr, uint64_t size, uint32_t keys);
 	[[nodiscard]] bool IsColorMetadataChecked(uint64_t vaddr, uint64_t size, uint32_t keys) const;
 	void               ProcessFaultBuffer();
-	void               SynchronizeBda(const RangeSet& mapped);
-	void               QueueBdaSync(uint64_t vaddr, uint64_t size) {
-		m_memory_tracker.QueueBdaSync(vaddr, size);
-	}
+	void               SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
+	// Queue after the range's state change becomes visible, or a drain could consume it first.
+	void                   QueueBdaSync(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] RangeSet TakeBdaSync();
 	void               RunGarbageCollector();
 
 private:
 	friend struct BufferCacheTestAccess;
 
-	bool IsBufferInvalid(BufferId id) const {
-		const auto* buffer = m_slot_buffers.try_get(id);
-		return buffer == nullptr || buffer->is_deleted;
-	}
+	bool IsBufferInvalid(BufferId id) const { return !id || m_slot_buffers[id].is_deleted; }
 
 	using BufferMap = std::map<uint64_t, BufferId>;
 	struct OverlapResult {
@@ -122,7 +120,6 @@ private:
 	void DeleteBuffer(BufferId id);
 	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
 	                                     bool is_written, bool is_texel_buffer);
-	void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
@@ -144,6 +141,8 @@ private:
 	// One per TileManager::ColorClearSlots key slot.
 	std::array<RangeSet, 8>                           m_checked_color_metadata;
 	MemoryTracker                                     m_memory_tracker;
+	std::mutex                                        m_bda_sync_mutex;
+	RangeSet                                          m_bda_sync;
 	StreamBuffer                                      m_staging_buffer;
 	StreamBuffer                                      m_stream_buffer;
 	StreamBuffer                                      m_download_buffer;

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -251,24 +251,24 @@ void TextureCache::UnregisterImage(ImageId id) {
 }
 
 void TextureCache::DeleteImage(ImageId id) {
-	auto* image = m_slot_images.try_get(id);
-	if (image == nullptr || !image->registered) {
+	auto& image = m_slot_images[id];
+	if (!image.registered) {
 		return;
 	}
-	if (image->depth_id) {
-		std::erase(m_slot_images[image->depth_id].stencil_ids, id);
+	if (image.depth_id) {
+		std::erase(m_slot_images[image.depth_id].stencil_ids, id);
 	} else {
 		// Detach first: deleting each association also unlinks it from its depth owner.
-		for (const auto association: std::exchange(image->stencil_ids, {})) {
+		for (const auto association: std::exchange(image.stencil_ids, {})) {
 			FreeImage(association);
 		}
 	}
-	if (image->IsGpuModified()) {
+	if (image.IsGpuModified()) {
 		EXIT("TextureCache: deleting a GPU-modified image without resolving its contents\n");
 	}
 	m_download_images.erase(id);
-	if (image->info.metadata.kind == ImageMetadataKind::Htile) {
-		m_surface_metas.erase(image->info.metadata.range.address);
+	if (image.info.metadata.kind == ImageMetadataKind::Htile) {
+		m_surface_metas.erase(image.info.metadata.range.address);
 	}
 	UnregisterImage(id);
 	if (m_scheduler.Active()) {
@@ -429,12 +429,6 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 		return {};
 	}
 
-	uint32_t query_epoch = ++m_image_query_epoch;
-	if (query_epoch == 0) {
-		m_slot_images.ForEach([](ImageId, const Image& image) { image.query_epoch = 0; });
-		query_epoch = ++m_image_query_epoch;
-	}
-
 	ImageIds result;
 	ForEachPage(address, size, [&](uint64_t page) {
 		const auto* owners = m_image_page_table.Find(page);
@@ -442,19 +436,17 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 			return;
 		}
 		owners->ForEach([&](ImageId id) {
-			auto* image = m_slot_images.try_get(id);
-			if (image == nullptr) {
+			const auto& image = m_slot_images[id];
+			if (image.picked || !image.Overlaps(address, size, page_overlap)) {
 				return;
 			}
-			if (image->query_epoch == query_epoch) {
-				return;
-			}
-			image->query_epoch = query_epoch;
-			if (image->Overlaps(address, size, page_overlap)) {
-				result.push_back(id);
-			}
+			image.picked = true;
+			result.push_back(id);
 		});
 	});
+	for (const auto id: result) {
+		m_slot_images[id].picked = false;
+	}
 	return result;
 }
 
@@ -787,15 +779,14 @@ void Resolve(TextureCache& cache, ImageId cached_id, ImageId merged_id) {
 TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& requested,
                                                          BindingType binding, ImageId cached_id,
                                                          ImageId merged_id) {
-	auto owner = m_slot_images.try_get(cached_id);
-	if (owner == nullptr || !owner->registered) {
+	auto& cached = m_slot_images[cached_id];
+	if (!cached.registered) {
 		return {merged_id};
 	}
-	if (BcAlias::Matches(requested, owner->info) && !owner->IsCpuDirty()) {
+	if (BcAlias::Matches(requested, cached.info) && !cached.IsCpuDirty()) {
 		BcAlias::Resolve(*this, cached_id, merged_id);
 		return {merged_id};
 	}
-	auto&      cached       = *owner;
 	const auto current_tick = m_scheduler.CurrentTick();
 	const bool safe_to_delete =
 	    current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval &&
@@ -869,9 +860,8 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		if (requested.pixel_format != cached.info.pixel_format ||
 		    requested.data.size <= cached.info.data.size) {
 			const auto result_id = merged_id ? merged_id : cached_id;
-			const auto result    = m_slot_images.try_get(result_id);
-			return {result != nullptr && ImageViewOps::FormatsCompatible(result->info.pixel_format,
-			                                                             requested.pixel_format)
+			return {ImageViewOps::FormatsCompatible(m_slot_images[result_id].info.pixel_format,
+			                                        requested.pixel_format)
 			            ? result_id
 			            : ImageId {}};
 		}
@@ -1218,9 +1208,8 @@ ImageId TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
 
 	ImageId association {};
 	for (const auto id: FindImagesInRegion(stencil.address, stencil.size, false)) {
-		const auto owner = m_slot_images.try_get(id);
-		if (owner != nullptr && owner->info.data == stencil &&
-		    owner->info.extent == depth.info.extent) {
+		const auto& owner = m_slot_images[id];
+		if (owner.info.data == stencil && owner.info.extent == depth.info.extent) {
 			association = id;
 		}
 	}
@@ -1315,10 +1304,9 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 			}
 		} else if (block_alias) {
 			for (const auto candidate: candidates) {
-				const auto* owner = m_slot_images.try_get(candidate);
-				if (candidate != result && owner != nullptr && owner->registered &&
-				    !owner->IsCpuDirty() &&
-				    BcAlias::Matches(m_slot_images[result].info, owner->info)) {
+				const auto& owner = m_slot_images[candidate];
+				if (candidate != result && owner.registered && !owner.IsCpuDirty() &&
+				    BcAlias::Matches(m_slot_images[result].info, owner.info)) {
 					BcAlias::Resolve(*this, candidate, result);
 				}
 			}
@@ -1381,14 +1369,14 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e
 	std::scoped_lock lock {m_lock};
 	ImageIds         matches;
 	for (const auto id: FindImagesInRegion(address, size, false)) {
-		auto owner = m_slot_images.try_get(id);
-		if (owner == nullptr || owner->info.data.address != address) {
+		const auto* owner = &m_slot_images[id];
+		if (owner->info.data.address != address) {
 			continue;
 		}
 		if (ensure_valid && owner->depth_id) {
-			owner = m_slot_images.try_get(owner->depth_id);
+			owner = &m_slot_images[owner->depth_id];
 		}
-		if (owner == nullptr || (ensure_valid && !owner->SafeToDownload())) {
+		if (ensure_valid && !owner->SafeToDownload()) {
 			continue;
 		}
 		matches.push_back(id);
@@ -1405,11 +1393,8 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e
 			}
 		}
 	}
-	if (selected && ensure_valid) {
-		const auto owner = m_slot_images.try_get(selected);
-		if (owner != nullptr && owner->depth_id) {
-			selected = owner->depth_id;
-		}
+	if (selected && ensure_valid && m_slot_images[selected].depth_id) {
+		selected = m_slot_images[selected].depth_id;
 	}
 	return selected;
 }
@@ -1430,9 +1415,8 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
 	    desc.info.data.End() <= image.info.stencil.End()) {
 		for (const auto stencil_id:
 		     FindImagesInRegion(image.info.stencil.address, image.info.stencil.size, false)) {
-			const auto* stencil = m_slot_images.try_get(stencil_id);
-			if (stencil != nullptr && stencil->depth_id == id &&
-			    stencil->info.data == image.info.stencil) {
+			const auto& stencil = m_slot_images[stencil_id];
+			if (stencil.depth_id == id && stencil.info.data == image.info.stencil) {
 				RefreshImage(stencil_id);
 				break;
 			}
@@ -1543,27 +1527,24 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	vk::ImageAspectFlags  aspect {};
 	ImageSubresourceRange subresources;
 	for (const auto id: FindImagesInRegion(address, size, false)) {
-		auto owner = m_slot_images.try_get(id);
-		if (owner == nullptr) {
-			continue;
-		}
+		const auto&          owner = m_slot_images[id];
 		vk::ImageAspectFlags candidate {};
 		ImageId              candidate_id = id;
 		ImageSubresourceRange candidate_subresources {
-		    0, owner->info.resources.levels, 0, owner->info.TransferLayers()};
-		if (owner->depth_id && owner->info.data.address == address &&
-		    owner->info.data.size == size) {
+		    0, owner.info.resources.levels, 0, owner.info.TransferLayers()};
+		if (owner.depth_id && owner.info.data.address == address &&
+		    owner.info.data.size == size) {
 			candidate              = vk::ImageAspectFlagBits::eStencil;
-			candidate_id           = owner->depth_id;
-			candidate_subresources = owner->stencil_subresources;
-			owner        = m_slot_images.try_get(candidate_id);
-			if (owner == nullptr || owner->backing.image == nullptr || !owner->info.HasStencil()) {
+			candidate_id           = owner.depth_id;
+			candidate_subresources = owner.stencil_subresources;
+			const auto& depth      = m_slot_images[candidate_id];
+			if (depth.backing.image == nullptr || !depth.info.HasStencil()) {
 				continue;
 			}
-		} else if (!owner->depth_id && owner->info.data.address == address &&
-		           owner->info.data.size == size) {
-			candidate = owner->info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
-			                                  : vk::ImageAspectFlagBits::eColor;
+		} else if (!owner.depth_id && owner.info.data.address == address &&
+		           owner.info.data.size == size) {
+			candidate = owner.info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
+			                                 : vk::ImageAspectFlagBits::eColor;
 		}
 		if (!candidate) {
 			continue;
@@ -1904,23 +1885,20 @@ void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
 	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
-		auto owner = m_slot_images.try_get(id);
-		if (owner == nullptr) {
-			continue;
-		}
-		if (owner->Overlaps(address, size)) {
-			owner->MarkCpuDirty();
+		auto& owner = m_slot_images[id];
+		if (owner.Overlaps(address, size)) {
+			owner.MarkCpuDirty();
 			UntrackImage(id);
 			continue;
 		}
-		const auto image_begin = owner->info.data.address;
-		const auto image_end   = owner->info.data.End();
+		const auto image_begin = owner.info.data.address;
+		const auto image_end   = owner.info.data.End();
 		if (page_end < image_end) {
 			UntrackImageHead(id);
 		} else if (image_begin < page_begin) {
 			UntrackImageTail(id);
 		} else {
-			MarkAsMaybeDirty(id, *owner);
+			MarkAsMaybeDirty(id, owner);
 		}
 	}
 }
@@ -2060,8 +2038,7 @@ void TextureCache::ProcessDownloadImages() {
 	}
 	std::scoped_lock lock {m_lock};
 	for (const auto id: m_download_images) {
-		const auto owner = m_slot_images.try_get(id);
-		if (owner != nullptr && owner->registered && owner->IsGpuModified()) {
+		if (m_slot_images[id].IsGpuModified()) {
 			(void)DownloadImageMemory(id);
 		}
 	}

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -184,7 +184,6 @@ private:
 	uint64_t                                          m_pressure_gc_memory = 1536ull * 1024 * 1024;
 	uint64_t         m_critical_gc_memory     = 3ull * 1024 * 1024 * 1024;
 	uint64_t         m_gc_tick                = 0;
-	mutable uint32_t m_image_query_epoch      = 0;
 	bool             m_readback_linear_images = false;
 	bool             m_readback_tiled_images  = false;
 

src/graphics/host_gpu/renderer/debug.cpp

@@ -595,13 +595,6 @@ void LogDrawPhase(const char* draw_name, const char* phase) {
 	}
 }
 
-void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id) {
-	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: %s VS=%" PRIu64 " PS=%" PRIu64 "\n", phase, vertex_program_id,
-		     pixel_program_id);
-	}
-}
-
 static void VpPrint(const char* func, const HW::ScreenViewport& vp,
                     const HW::ScanModeControl& smc) {
 	LOGF("%s\n", func);

src/graphics/host_gpu/renderer/image/image.h

@@ -139,14 +139,17 @@ public:
 	}
 
 	static inline std::atomic<uint64_t> heap_epoch {0};
+	static inline std::atomic<uint64_t> serials {0};
 
+	// Unique per image, unlike ImageId: freed slots are reused.
+	const uint64_t   serial = serials.fetch_add(1u, std::memory_order_relaxed) + 1u;
 	ImageInfo        info;
 	VulkanImage      backing;
 	std::vector<CachedImageView> views;
 	ImageUsage       usage;
 	ImageBinding     binding;
 	bool             registered     = false;
-	mutable uint32_t query_epoch    = 0;
+	mutable bool     picked         = false; // Set only while FindImagesInRegion collects it.
 	uint64_t         track_addr     = 0;
 	uint64_t         track_addr_end = 0;
 	ImageId          depth_id {};

src/graphics/host_gpu/renderer/occlusionQueries.cpp

@@ -0,0 +1,82 @@
+#include "graphics/host_gpu/renderer/occlusionQueries.h"
+
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics {
+
+// Queries no dump has read yet; past them a draw counts one sample.
+constexpr uint32_t QUERY_COUNT = 16384;
+
+OcclusionQueries::OcclusionQueries(GraphicContext& graphics, CommandScheduler& scheduler)
+    : m_scheduler(scheduler) {
+	if (!graphics.occlusion_queries_enabled) {
+		return;
+	}
+	m_device = graphics.device;
+	const vk::QueryPoolCreateInfo info {.queryType  = vk::QueryType::eOcclusion,
+	                                    .queryCount = QUERY_COUNT};
+	RequireVulkanSuccess(m_device.createQueryPool(&info, nullptr, &m_pool), "vkCreateQueryPool");
+	m_device.resetQueryPool(m_pool, 0, QUERY_COUNT);
+	m_results.resize(QUERY_COUNT);
+	if (graphics.occlusion_query_precise_enabled) {
+		m_control = vk::QueryControlFlagBits::ePrecise;
+	}
+}
+
+OcclusionQueries::~OcclusionQueries() {
+	if (m_pool) {
+		m_device.destroyQueryPool(m_pool, nullptr);
+	}
+}
+
+int32_t OcclusionQueries::BeginQuery(vk::CommandBuffer command) {
+	if (m_begun - m_read == QUERY_COUNT) {
+		m_uncounted++;
+		return -1;
+	}
+	const auto query = static_cast<uint32_t>(m_begun++ % QUERY_COUNT);
+	command.beginQuery(m_pool, query, m_control);
+	return static_cast<int32_t>(query);
+}
+
+void OcclusionQueries::Dump(uint64_t address) {
+	// Begins dump to the results, ends 8 bytes into them.
+	if ((address & 8u) == 0) {
+		m_open++;
+	} else if (m_open != 0) {
+		m_open--;
+	}
+	m_dumps.push_back({address, m_scheduler.CurrentTick(), m_begun, m_uncounted});
+}
+
+void OcclusionQueries::PublishDumps() {
+	if (const auto last = m_dumps.back().queries; last != m_read) {
+		m_scheduler.Wait(m_dumps.back().tick);
+		// The unread queries wrap around the pool at most once.
+		for (uint64_t read = m_read; read < last;) {
+			const auto first = static_cast<uint32_t>(read % QUERY_COUNT);
+			const auto count =
+			    static_cast<uint32_t>(std::min<uint64_t>(last - read, QUERY_COUNT - first));
+			RequireVulkanSuccess(
+			    m_device.getQueryPoolResults(m_pool, first, count, count * sizeof(uint64_t),
+				                             m_results.data() + first, sizeof(uint64_t),
+				                             vk::QueryResultFlagBits::e64),
+			    "vkGetQueryPoolResults");
+			m_device.resetQueryPool(m_pool, first, count);
+			read += count;
+		}
+	}
+	for (const auto& dump: m_dumps) {
+		for (; m_read < dump.queries; m_read++) {
+			m_samples += m_results[m_read % QUERY_COUNT];
+		}
+		WriteZPassCounters(reinterpret_cast<volatile uint64_t*>(dump.address),
+		                   m_samples + dump.uncounted, (dump.address & 8u) == 0);
+	}
+	m_dumps.clear();
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/occlusionQueries.h

@@ -0,0 +1,85 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_OCCLUSIONQUERIES_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_OCCLUSIONQUERIES_H_
+
+#include "common/common.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+
+#include <cstdint>
+#include <vector>
+
+namespace Libs::Graphics {
+
+class CommandScheduler;
+struct GraphicContext;
+
+// The guest's query results (16-byte aligned): a begin/end pair of running z-pass counters per DB,
+// 16 bytes apart; bit 63 marks a written counter. A query's samples are the sum of end - begin.
+constexpr uint32_t OCCLUSION_QUERY_DBS   = 16;
+constexpr uint64_t OCCLUSION_QUERY_READY = 1ull << 63u;
+
+// A dump of the running count, all of it in DB 0. A begin also clears its end counters.
+inline void WriteZPassCounters(volatile uint64_t* results, uint64_t samples, bool begin) {
+	for (uint32_t db = 0; db < OCCLUSION_QUERY_DBS; db++) {
+		results[db * 2] = OCCLUSION_QUERY_READY | (db == 0 ? samples : 0);
+		if (begin) {
+			results[db * 2 + 1] = 0;
+		}
+	}
+}
+
+// The guest's z-pass counters from host occlusion queries: a draw counts its samples while a guest
+// query is open, and the dumps' counters are written before the guest can see the GPU ran them.
+class OcclusionQueries {
+public:
+	OcclusionQueries(GraphicContext& graphics, CommandScheduler& scheduler);
+	~OcclusionQueries();
+	KYTY_CLASS_NO_COPY(OcclusionQueries);
+
+	[[nodiscard]] bool Enabled() const { return m_pool != nullptr; }
+
+	// In the draw's rendering: the query that counts its samples, or -1.
+	[[nodiscard]] int32_t Begin(vk::CommandBuffer command) {
+		return m_open == 0 ? -1 : BeginQuery(command);
+	}
+	void End(vk::CommandBuffer command, int32_t query) const {
+		if (query >= 0) {
+			command.endQuery(m_pool, static_cast<uint32_t>(query));
+		}
+	}
+
+	// A PIXEL_PIPE_STAT_DUMP to address, after the draws recorded so far.
+	void Dump(uint64_t address);
+	// Waits for the GPU to run the dumps and writes their counters.
+	void Publish() {
+		if (!m_dumps.empty()) {
+			PublishDumps();
+		}
+	}
+
+private:
+	struct PendingDump {
+		uint64_t address;
+		uint64_t tick;
+		uint64_t queries;
+		uint64_t uncounted;
+	};
+
+	int32_t BeginQuery(vk::CommandBuffer command);
+	void    PublishDumps();
+
+	CommandScheduler&        m_scheduler;
+	vk::Device               m_device;
+	vk::QueryPool            m_pool;
+	vk::QueryControlFlags    m_control;
+	std::vector<PendingDump> m_dumps;
+	std::vector<uint64_t>    m_results;
+	uint32_t                 m_open      = 0;
+	uint64_t                 m_begun     = 0;
+	uint64_t                 m_read      = 0;
+	uint64_t                 m_samples   = 0;
+	uint64_t                 m_uncounted = 0;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_OCCLUSIONQUERIES_H_

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -672,9 +672,7 @@ void RenderExecutor::BindRenderTarget(ImageId id) {
 
 void RenderExecutor::ResetBindings() {
 	for (const auto id: m_bound_images) {
-		if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(id); image != nullptr) {
-			image->binding = {};
-		}
+		m_context.GetTextureCache().m_slot_images[id].binding = {};
 	}
 	m_bound_images.clear();
 }
@@ -683,7 +681,7 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
                                      PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(!runtime);
-	const auto& program  = *runtime.program;
+	const auto& program  = runtime.module->info;
 	const auto& snapshot = *runtime.resources;
 	prepared.runtime = &runtime;
 	prepared.gds = {nullptr, 0, VK_WHOLE_SIZE};
@@ -725,7 +723,7 @@ void RenderExecutor::FindBuffers(std::span<PreparedBindings* const> stages) {
 	for (auto* stage: stages) {
 		auto& prepared = *stage;
 		EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
-		const auto& program  = *prepared.runtime->program;
+		const auto& program  = prepared.runtime->module->info;
 		const auto& snapshot = *prepared.runtime->resources;
 		const auto& layout = program.bindings;
 		prepared.buffer_sources.resize(layout.descriptor_counts[static_cast<size_t>(BindingKind::Buffers)]);
@@ -782,7 +780,7 @@ void RenderExecutor::FindBuffers(std::span<PreparedBindings* const> stages) {
 void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
-	const auto& program   = *prepared.runtime->program;
+	const auto& program   = prepared.runtime->module->info;
 	const auto& snapshot  = *prepared.runtime->resources;
 	const auto& layout    = program.bindings;
 	EXIT_IF(prepared.buffer_sources.size() != layout.descriptor_counts[static_cast<size_t>(BindingKind::Buffers)]);
@@ -815,7 +813,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
-	const auto& program  = *prepared.runtime->program;
+	const auto& program  = prepared.runtime->module->info;
 	const auto& snapshot = *prepared.runtime->resources;
 	auto&       images   = prepared.images;
 	EXIT_IF(images.size() != program.info.images.size());
@@ -823,12 +821,9 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	// Acquire each view before a later overlapping descriptor can replace its image.
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		if (!images[i].image_id) continue;
-		const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id);
-		if (old_image == nullptr || !old_image->registered ||
-		    old_image->binding.needs_rebind) {
-			if (old_image != nullptr) {
-				old_image->binding = {};
-			}
+		auto& old_image = texture_cache.m_slot_images[images[i].image_id];
+		if (!old_image.registered || old_image.binding.needs_rebind) {
+			old_image.binding = {};
 			images[i] = ResolveTexture(program.info.images[i], snapshot.images[i]);
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
@@ -864,8 +859,8 @@ void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const>
 	bool uses_image_heap = false;
 	FindBuffers(stages);
 	for (auto* stage: stages) {
-		uses_dma |= stage->runtime->program->info.uses_dma;
-		uses_image_heap |= stage->runtime->program->info.uses_image_heap;
+		uses_dma |= stage->runtime->module->info.info.uses_dma;
+		uses_image_heap |= stage->runtime->module->info.info.uses_image_heap;
 	}
 	if (uses_dma) {
 		m_context.PrepareBda();
@@ -879,12 +874,9 @@ void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const>
 	auto& cache = m_context.GetTextureCache();
 	for (auto& target: colors) {
 		EXIT_IF(!target.image_id);
-		const auto old_image = cache.m_slot_images.try_get(target.image_id);
-		if (old_image == nullptr || !old_image->registered ||
-		    old_image->binding.needs_rebind) {
-			if (old_image != nullptr) {
-				old_image->binding = {};
-			}
+		auto& old_image = cache.m_slot_images[target.image_id];
+		if (!old_image.registered || old_image.binding.needs_rebind) {
+			old_image.binding = {};
 			target.desc.view_info.base_level = target.guest_mip_level;
 			target.desc.view_info.base_layer = target.guest_array_layer;
 			target.image_id = cache.FindImage(target.desc);
@@ -917,7 +909,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 	                                       : vk::ShaderStageFlags {};
 	for (const auto* prepared: prepared_bindings) {
 		EXIT_IF(prepared == nullptr || prepared->runtime == nullptr || !*prepared->runtime);
-		const auto& program = *prepared->runtime->program;
+		const auto& program = prepared->runtime->module->info;
 		write_count += std::popcount(program.bindings.descriptor_mask);
 		for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
 			const auto index = std::countr_zero(mask);
@@ -938,17 +930,16 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 		const auto& reads = reader->runtime->resources->specialization_reads;
 		if (reads.empty()) continue;
 		for (const auto* writer: prepared_bindings) {
-			if (writer->runtime->program->has_address_writes) {
+			if (writer->runtime->module->info.has_address_writes) {
 				EXIT("scalar resource reads cannot be proven disjoint from shader address writes\n");
 			}
 		}
 		for (const auto [address, size]: reads) {
 			for (const auto id: m_bound_images) {
-				const auto* image = m_context.GetTextureCache().m_slot_images.try_get(id);
-				if (image == nullptr ||
-				    (!image->binding.shader_write && !image->binding.is_target)) continue;
-				for (const auto written: {image->info.data, image->info.stencil,
-				                          image->info.metadata.range}) {
+				const auto& image = m_context.GetTextureCache().m_slot_images[id];
+				if (!image.binding.shader_write && !image.binding.is_target) continue;
+				for (const auto written: {image.info.data, image.info.stencil,
+				                          image.info.metadata.range}) {
 					if (written.size != 0 && ImageRangeOverlaps(address, size,
 					                                          written.address, written.size)) {
 						EXIT("scalar resource reads overlap an image or attachment write\n");
@@ -956,7 +947,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 				}
 			}
 			for (const auto* writer: prepared_bindings) {
-				const auto& program = *writer->runtime->program;
+				const auto& program = writer->runtime->module->info;
 				for (const auto& resource: program.info.buffers) {
 					if (!resource.written || resource.descriptor_index == UINT32_MAX) continue;
 					const auto& written = writer->buffer_sources[resource.descriptor_index];
@@ -982,7 +973,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 	std::array<vk::DescriptorBufferInfo, 2> address_buffers;
 
 	for (auto* prepared: prepared_bindings) {
-		const auto& program       = *prepared->runtime->program;
+		const auto& program       = prepared->runtime->module->info;
 		auto&       descriptors   = *prepared;
 		const auto  shader_stage  = NativeShaderStage(program.stage);
 		const auto  shader_stages = ShaderPipelineStages(shader_stage);

src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp

@@ -350,12 +350,17 @@ void ImageHeap::Release(View& view, uint32_t entry) {
 }
 
 void ImageHeap::Unreference(Element& element) {
-	if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(element.binding.image_id)) {
+	if (auto* image = ImageOf(element)) {
 		image->heap_refs.fetch_sub(1u, std::memory_order_relaxed);
 		image->tick_accessed_last = m_context.GetCommandScheduler().CurrentTick();
 	}
 }
 
+Image* ImageHeap::ImageOf(const Element& element) const {
+	auto* image = m_context.GetTextureCache().m_slot_images.try_get(element.binding.image_id);
+	return image != nullptr && image->serial == element.serial ? image : nullptr;
+}
+
 uint32_t ImageHeap::Allocate() {
 	auto& scheduler = m_context.GetCommandScheduler();
 	while (!m_retired.empty() && scheduler.IsFree(m_retired.front().first)) {
@@ -393,7 +398,7 @@ void ImageHeap::Rebind(RenderExecutor& executor) {
 			for (const auto& [value, index]: view->elements) {
 				const auto& binding = m_elements[index].binding;
 				if (!binding.image_id) continue;
-				const auto* image = textures.m_slot_images.try_get(binding.image_id);
+				const auto* image = ImageOf(m_elements[index]);
 				if (image == nullptr || !image->registered || image->depth_id ||
 				    image->binding.needs_rebind) {
 					mark(index);
@@ -434,6 +439,7 @@ void ImageHeap::Rebind(RenderExecutor& executor) {
 			if (binding.image_id) {
 				auto& image = textures.GetImage(binding.image_id);
 				image.heap_refs.fetch_add(1u, std::memory_order_relaxed);
+				m_elements[index].serial = image.serial;
 				binding.layout     = image.ReadLayout();
 				binding.image_view = textures.FindTexture(binding.image_id, binding.desc);
 				if (binding.desc.info.metadata.kind == ImageMetadataKind::Dcc) {
@@ -471,8 +477,7 @@ void ImageHeap::WriteDescriptors() {
 void ImageHeap::Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point,
                        vk::PipelineLayout layout) {
 	KYTY_PROFILER_FUNCTION();
-	auto&      textures = m_context.GetTextureCache();
-	const auto epoch    = Image::heap_epoch.load(std::memory_order_relaxed);
+	const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
 	for (auto* view: m_active) {
 		if (view->dirty_begin != view->dirty_end) {
 			const auto slots = std::span(view->slots).subspan(view->dirty_begin,
@@ -493,7 +498,7 @@ void ImageHeap::Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point,
 		// this draw's discovery replaced lives until the tick ends, and its slots still select it.
 		std::erase_if(view->away, [&](uint32_t index) {
 			const auto& element = m_elements[index];
-			auto*       image   = textures.m_slot_images.try_get(element.binding.image_id);
+			auto*       image   = ImageOf(element);
 			if (element.refs == 0u || image == nullptr) return true;
 			if (image->binding.is_target || image->binding.shader_write ||
 			    image->binding.force_general) {

src/graphics/host_gpu/renderer/pipeline/imageHeap.h

@@ -57,9 +57,10 @@ private:
 	struct Element {
 		DescriptorValue value;
 		TextureBinding  binding;
-		uint32_t        kind  = 0;
-		uint32_t        refs  = 0; // Slots of the owning view that select it.
-		bool            stale = false;
+		uint32_t        kind   = 0;
+		uint32_t        refs   = 0; // Slots of the owning view that select it.
+		bool            stale  = false;
+		uint64_t        serial = 0; // Image::serial of binding.image_id.
 	};
 
 	struct Memory;
@@ -109,6 +110,7 @@ private:
 	uint32_t Intern(View& view, uint32_t kind, const DescriptorValue& value);
 	void     Release(View& view, uint32_t entry);
 	void     Unreference(Element& element);
+	Image*   ImageOf(const Element& element) const;
 	uint32_t Allocate();
 	void     Retire(View& view);
 	void     WriteDescriptors();

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -4,42 +4,57 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
+#include "common/file.h"
+#include "common/uniqueFunction.h"
+#include "graphics/host_gpu/renderer/colorRenderTarget.h"
+#include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
 
 #include <chrono>
+#include <condition_variable>
 #include <cstddef>
 #include <filesystem>
 #include <memory>
+#include <mutex>
+#include <queue>
 #include <span>
+#include <thread>
 #include <type_traits>
 #include <unordered_map>
+#include <vector>
 
 namespace Libs::Graphics {
 
 struct GraphicContext;
-struct RenderColorInfo;
-struct RenderDepthInfo;
+struct ShaderParams;
 class CommandBuffer;
 
-namespace HW {
-class Context;
-class Shader;
-class UserConfig;
-struct ComputeShaderInfo;
-} // namespace HW
+namespace ShaderRecompiler {
+struct CompileOptions;
+struct ShaderSource;
+struct TranslateResult;
+} // namespace ShaderRecompiler
 
 namespace ShaderRecompiler::IR {
 class ImageHeapSource;
 } // namespace ShaderRecompiler::IR
 
+namespace ShaderRecompiler::Spirv {
+struct Profile;
+} // namespace ShaderRecompiler::Spirv
+
 #pragma pack(push, 1)
 
 struct PipelineStaticParameters {
 	bool                       negative_one_to_one      = false;
 	bool                       depth_clip_enable        = true;
 	vk::PrimitiveTopology      topology                 = vk::PrimitiveTopology::ePointList;
+	// 0 without tessellation, 3 for a rectangle list, else the input control points.
+	uint8_t                    patch_control_points     = 0;
 	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
@@ -58,57 +73,108 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
-
-	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
 
-#pragma pack(pop)
-
-static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
-static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
-static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 117);
-
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};
 	vk::Format                                           depth_format   = vk::Format::eUndefined;
 	vk::Format                                           stencil_format = vk::Format::eUndefined;
 	uint32_t                                             color_count    = 0;
-
-	bool operator==(const PipelineRenderingState&) const = default;
 };
 
 struct PipelineVertexInputState {
 	struct Binding {
-		uint32_t stride                           = 0;
-		bool     instance                         = false;
-		bool     operator==(const Binding&) const = default;
+		uint32_t stride   = 0;
+		uint8_t  instance = 0;
 	};
 	struct Attribute {
-		uint32_t offset                             = 0;
-		uint8_t  binding                            = 0;
-		bool     operator==(const Attribute&) const = default;
+		uint32_t offset  = 0;
+		uint8_t  binding = 0;
 	};
 
 	std::array<Binding, ShaderVertexInputInfo::RES_MAX>   bindings {};
 	std::array<Attribute, ShaderVertexInputInfo::RES_MAX> attributes {};
 	uint8_t                                               binding_count   = 0;
 	uint8_t                                               attribute_count = 0;
-
-	bool operator==(const PipelineVertexInputState&) const = default;
 };
 
-struct ShaderProgram {
-	uint64_t         id     = 0;
-	vk::ShaderModule module = nullptr;
+// Everything a graphics pipeline is made of besides the stage modules its ids name. Aligned so
+// that rendering.color_formats can be passed to Vulkan.
+struct alignas(4) GraphicsPipelineKey {
+	// VS, LS or mesh; HS; TES; PS. 0 marks an absent stage.
+	std::array<uint64_t, 4>  stage_ids {};
+	PipelineRenderingState   rendering;
+	PipelineVertexInputState vertex_input;
+	PipelineStaticParameters static_params;
 
-	explicit operator bool() const { return id != 0 && module != nullptr; }
+	bool operator==(const GraphicsPipelineKey& other) const;
+};
+
+#pragma pack(pop)
+
+// Keys are compared and hashed as bytes.
+static_assert(std::has_unique_object_representations_v<GraphicsPipelineKey>);
+
+// A compiled stage with what pipeline creation needs from it besides the key.
+struct ShaderModule {
+	uint64_t                                     id     = 0;
+	vk::ShaderModule                             module = nullptr;
+	ShaderRecompiler::IR::CompiledShaderInfo     info;
+	ShaderRecompiler::IR::ResourceSpecialization specialization;
+	std::vector<uint32_t>                        function_code;
+	// VS or LS: the host format of each vertex input attribute.
+	std::vector<vk::Format>                      vertex_formats;
+	// Rectangle-list PS: the parameters it reads.
+	std::vector<RectListParameter>               rect_parameters;
+};
+
+// A draw's topology, shader stages (from PipelineCache::GetGraphicsPrograms) and targets.
+struct DrawRenderState {
+	RenderDepthInfo                      depth_info;
+	RenderColorInfo                      color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
+	uint32_t                             color_count                              = 0;
+	// Color targets the pixel program writes.
+	uint32_t                             mrt_mask                                 = 0;
+	vk::PrimitiveTopology                topology = vk::PrimitiveTopology::ePointList;
+	bool                                 primitive_restart_enable                 = false;
+	bool                                 ps_active                                = false;
+	std::array<ShaderVertexInputInfo, 3> vertex_info;
+	ShaderPixelInputInfo                 ps_input_info;
+
+	[[nodiscard]] std::span<const ShaderVertexInputInfo> VertexStages() const {
+		return {vertex_info.data(), vertex_info[1].stage ? 3u : 1u};
+	}
+};
+
+// Writes the shader store on its own thread, running jobs in the order they were posted.
+class ShaderStoreWriter {
+public:
+	// Keeps the first `kept` bytes of the store at `path`, or starts it over with `header`. The
+	// thread runs only if that succeeded.
+	void Open(const std::filesystem::path& path, std::span<const uint8_t> header, uint64_t kept);
+	explicit operator bool() const { return m_thread.joinable(); }
+	// Compresses `record` and appends it on the store thread, in call order.
+	void Append(std::vector<uint8_t>&& record);
+
+private:
+	friend struct PipelineCacheTestAccess;
+
+	void Post(Common::UniqueFunction<void>&& job);
+
+	Common::File                             m_file;
+	std::mutex                               m_mutex;
+	std::condition_variable_any              m_ready;
+	std::queue<Common::UniqueFunction<void>> m_jobs;
+	// Declared last, so it drains the queue and joins before the rest is destroyed.
+	std::jthread m_thread;
 };
 
 // The owning renderer serializes access, including saves while the GPU is running.
 class PipelineCache {
 public:
-	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap);
+	// Without a shader store path nothing is kept across runs.
+	PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap,
+	              const std::filesystem::path& shader_store = {});
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 	void Save();
@@ -121,87 +187,106 @@ public:
 		bool                    uses_image_heap       = false;
 	};
 
-	struct GraphicsPrograms {
-		std::array<ShaderProgram, 3> vertex;
-		ShaderProgram pixel;
+	// Reads the shaders and registers bound in `command` into the state's shader stages.
+	void GetGraphicsPrograms(const CommandBuffer& command, DrawRenderState& state);
+	// Null skips the dispatch.
+	const ShaderModule* GetComputeProgram(const HW::ComputeShaderInfo& regs,
+	                                      ShaderComputeInputInfo&      input_info);
 
-		[[nodiscard]] uint32_t VertexStageCount() const { return vertex[1] ? 3u : 1u; }
-	};
-
-	GraphicsPrograms
-	GetGraphicsPrograms(const HW::VertexShaderInfo& vertex_regs,
-	                    const HW::PixelShaderInfo& pixel_regs, const HW::ShaderRegisters& sh,
-	                    const HW::Context& context, const HW::UserConfig& user_config,
-	                    std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
-	                    bool pixel_active, std::array<ShaderVertexInputInfo, 3>& vertex_info,
-	                    ShaderPixelInputInfo& pixel_info);
-	ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs,
-	                                const HW::ShaderRegisters&   sh,
-	                                ShaderComputeInputInfo&      input_info);
-
-	Pipeline& GetGraphicsPipeline(std::span<const RenderColorInfo>       colors,
-	                              const RenderDepthInfo&                 depth,
-	                              std::span<const ShaderVertexInputInfo> vertex_info,
-	                              CommandBuffer& command, const ShaderPixelInputInfo* ps_input_info,
-	                              vk::PrimitiveTopology topology, bool primitive_restart_enable,
-	                              const GraphicsPrograms& programs);
-	Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info,
-	                             const ShaderProgram&          compute_program);
+	Pipeline& GetGraphicsPipeline(const CommandBuffer& command, const DrawRenderState& state);
+	Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info);
 
 private:
-	struct ProgramCache;
+	friend struct PipelineCacheTestAccess;
 
-	struct GraphicsPipelineKey {
-		PipelineRenderingState   rendering;
-		std::array<uint64_t, 3>  vertex_shader_ids {};
-		uint64_t                 ps_shader_id = 0;
-		PipelineVertexInputState vertex_input;
-		PipelineStaticParameters static_params;
+	struct Program {
+		std::unique_ptr<ShaderRecompiler::ShaderSource> call_source;
+		ShaderRecompiler::IR::ResourcePlan           resource_plan;
+		ShaderRecompiler::IR::ResourceSnapshot       resources;
+		ShaderRecompiler::IR::ResourceSpecialization specialization;
+		std::vector<ShaderModule>                    modules;
+		bool                                         skip_dispatch = false;
 
-		bool operator==(const GraphicsPipelineKey& other) const {
-			return rendering == other.rendering && vertex_shader_ids == other.vertex_shader_ids &&
-			       ps_shader_id == other.ps_shader_id && vertex_input == other.vertex_input &&
-			       static_params == other.static_params;
-		}
+		[[nodiscard]] std::span<const uint32_t> FunctionCode() const;
+		bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime);
+		template <bool HasCalls>
+		ShaderModule* FindModule(const ShaderRecompiler::IR::SrtRuntime& runtime,
+		                         uint32_t                                push_data_cursor);
 	};
 
-	struct PipelineKeyHash {
-		static void Mix(std::size_t& hash, std::size_t value) {
-			hash ^= value + static_cast<std::size_t>(0x9e3779b97f4a7c15ull) + (hash << 6u) +
-			        (hash >> 2u);
-		}
+	struct ProgramKey {
+		ShaderType            stage           = ShaderType::Unknown;
+		uint64_t              hash            = 0;
+		uint32_t              user_data_count = 0;
+		std::vector<uint32_t> static_state;
+
+		bool operator==(const ProgramKey&) const = default;
+	};
+
+	// Variants of one source share a bucket; ProgramKey equality compares the state.
+	struct ProgramKeyHash {
+		std::size_t operator()(const ProgramKey& key) const { return key.hash; }
 	};
 
 	struct GraphicsPipelineKeyHash {
 		std::size_t operator()(const GraphicsPipelineKey& key) const;
 	};
 
-	GraphicContext&               m_graphics;
-	std::unique_ptr<ProgramCache> m_program_cache;
+	using ModuleMap = std::unordered_map<uint64_t, const ShaderModule*>;
+
+	GraphicContext&                        m_graphics;
+	ShaderRecompiler::IR::ImageHeapSource* m_image_heap;
+	std::unordered_map<ProgramKey, Program, ProgramKeyHash> m_program_cache;
+	// Modules compiled per stage.
+	std::array<uint32_t, static_cast<size_t>(ShaderType::TessellationEvaluation) + 1> m_compiled {};
+	ProgramKey                             m_lookup_key;
 	vk::PipelineCache             m_driver_cache = nullptr;
 	std::filesystem::path         m_driver_cache_path;
 	std::chrono::steady_clock::time_point m_next_driver_cache_save;
 	bool                          m_driver_cache_dirty = false;
-	std::unordered_map<GraphicsPipelineKey, std::unique_ptr<Pipeline>, GraphicsPipelineKeyHash>
-	                                                        m_graphics_pipelines;
-	std::unordered_map<uint64_t, std::unique_ptr<Pipeline>> m_compute_pipelines;
+	std::unordered_map<GraphicsPipelineKey, Pipeline, GraphicsPipelineKeyHash> m_graphics_pipelines;
+	std::unordered_map<uint64_t, Pipeline>                                     m_compute_pipelines;
+	std::unordered_map<uint64_t, ShaderTessellationInputInfo>                  m_tess_layouts;
+	// Records everything first created; closed without a shader store.
+	ShaderStoreWriter m_store;
 
+	template <typename InputInfo>
+	const ShaderModule* GetProgram(const ShaderParams& params, InputInfo& input_info,
+	                               uint32_t& push_data_cursor);
+	ShaderModule& CompileModule(Program& program, const ProgramKey& key,
+	                            ShaderRecompiler::CompileOptions  options,
+	                            ShaderRecompiler::TranslateResult translated,
+	                            uint32_t                          push_data_cursor);
 	void InitializeDriverCache();
 	void CheckpointDriverCache();
+
+	// pipelineSerialization.cpp
+	void WarmUp(const std::filesystem::path& path);
+	bool LoadRecord(std::span<const uint8_t>& records, std::vector<GraphicsPipelineKey>& pipelines);
+	void StoreModule(const ProgramKey& key, const Program& program, const ShaderModule& module,
+	                 const std::vector<uint32_t>& spirv);
+	void StorePipeline(const GraphicsPipelineKey& key);
+	void StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout);
+	// Every module by id, with id 0 naming an absent stage.
+	[[nodiscard]] ModuleMap MapModules() const;
+	// False when a stage id of the key is not in `modules`.
+	bool CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key,
+	                            const ModuleMap& modules);
 };
 
-void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id);
+// The shader store of the running title; empty unless --shader-cache is on in a Release build
+// without debug dumps, which change the SPIR-V.
+std::filesystem::path ShaderStorePath();
+// Host capabilities that change the emitted SPIR-V.
+ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics);
+
+// Stages are indexed like GraphicsPipelineKey::stage_ids.
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-                            const PipelineRenderingState&          rendering,
-                            const PipelineVertexInputState&        vertex_input,
-                            std::span<const ShaderVertexInputInfo> vertex_info,
-                            const ShaderPixelInputInfo*            ps_input_info,
-                            const PipelineCache::GraphicsPrograms& programs,
-                            const PipelineStaticParameters&        static_params,
-                            vk::PipelineCache                      driver_cache);
+                            const GraphicsPipelineKey&                 key,
+                            const std::array<const ShaderModule*, 4>& stages,
+                            vk::PipelineCache                          driver_cache);
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-                            const ShaderComputeInputInfo& input_info,
-                            vk::ShaderModule compute_module, vk::PipelineCache driver_cache);
+                            const ShaderModule& module, vk::PipelineCache driver_cache);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp

@@ -0,0 +1,329 @@
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+
+#include "common/logging/log.h"
+#include "common/stringUtils.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/ir/Serialize.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+#include <fmt/format.h>
+#include <ranges>
+#include <xxhash.h>
+#include <zstd.h>
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#elif defined(__APPLE__)
+#include <mach-o/dyld.h>
+#endif
+
+// The shader store: a header, then zstd-compressed records of everything a run created first.
+// Records are only appended, so a store cut short by a crash keeps every complete record.
+namespace Libs::Graphics {
+
+namespace {
+
+using ShaderRecompiler::IR::IoAll;
+using ShaderRecompiler::IR::Reader;
+using ShaderRecompiler::IR::Writer;
+
+enum class Record : uint8_t { Module, GraphicsPipeline, TessellationLayout };
+
+// Precedes each record's zstd frame.
+struct Frame {
+	uint32_t compressed_size = 0;
+	uint32_t raw_size        = 0;
+	uint64_t checksum        = 0; // XXH3 of the compressed bytes.
+};
+
+// Changes with every build, so records never meet code that encodes them differently. Zero when
+// the executable cannot be found.
+uint64_t BuildStamp() {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	std::array<wchar_t, 32768> name {};
+	GetModuleFileNameW(nullptr, name.data(), static_cast<DWORD>(name.size()));
+	const std::filesystem::path path = name.data();
+#elif defined(__APPLE__)
+	std::array<char, 4096> name {};
+	auto size = static_cast<uint32_t>(name.size());
+	_NSGetExecutablePath(name.data(), &size);
+	const std::filesystem::path path = name.data();
+#else
+	const std::filesystem::path path = "/proc/self/exe";
+#endif
+	std::error_code error;
+	const auto      time = std::filesystem::last_write_time(path, error).time_since_epoch().count();
+	return error ? 0 : static_cast<uint64_t>(time);
+}
+
+// What every record depends on besides its own fields; empty when the build is unknown.
+std::vector<uint8_t> StoreHeader(const GraphicContext& graphics) {
+	const auto stamp = BuildStamp();
+	if (stamp == 0) {
+		return {};
+	}
+	Writer header;
+	IoAll(header, stamp, SpirvProfile(graphics),
+	      graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize);
+	return std::move(header.bytes);
+}
+
+// Record fields, shared by writing and reading.
+template <class S, class Key> void IoKey(S& io, Key& key) {
+	auto& [stage, hash, user_data_count, static_state] = key;
+	IoAll(io, stage, hash, user_data_count, static_state);
+}
+
+template <class S, class Module> void IoModule(S& io, Module& stage) {
+	// The handle comes from the SPIR-V; stored programs have no function code.
+	auto& [id, handle, info, specialization, function_code, vertex_formats, rect_parameters] = stage;
+	IoAll(io, id, info, specialization, vertex_formats, rect_parameters);
+}
+
+} // namespace
+
+void ShaderStoreWriter::Open(const std::filesystem::path& path, std::span<const uint8_t> header,
+                             uint64_t kept) {
+	Common::File::CreateDirectories(path.parent_path());
+	if (kept == 0) {
+		if (m_file.Create(path)) {
+			m_file.Write(header.data(), static_cast<uint32_t>(header.size()));
+		}
+	} else if (m_file.Open(path, Common::File::Mode::ReadWrite)) {
+		m_file.Truncate(kept);
+		m_file.Seek(kept);
+	}
+	if (m_file.IsInvalid()) {
+		return;
+	}
+	m_thread = std::jthread([this](std::stop_token stop) {
+		std::unique_lock lock(m_mutex);
+		// Once stop is requested the wait returns false only when the queue is empty.
+		while (m_ready.wait(lock, stop, [this] { return !m_jobs.empty(); })) {
+			const auto job = std::move(m_jobs.front());
+			m_jobs.pop();
+			lock.unlock();
+			job();
+			lock.lock();
+		}
+	});
+}
+
+void ShaderStoreWriter::Append(std::vector<uint8_t>&& record) {
+	Post([this, record = std::move(record)] {
+		std::vector<uint8_t> bytes(sizeof(Frame) + ZSTD_compressBound(record.size()));
+		const auto size = ZSTD_compress(bytes.data() + sizeof(Frame), bytes.size() - sizeof(Frame),
+		                                record.data(), record.size(), 3);
+		EXIT_IF(ZSTD_isError(size));
+		const Frame frame {.compressed_size = static_cast<uint32_t>(size),
+		                   .raw_size        = static_cast<uint32_t>(record.size()),
+		                   .checksum        = XXH3_64bits(bytes.data() + sizeof(Frame), size)};
+		std::memcpy(bytes.data(), &frame, sizeof(frame));
+		// A failed write leaves a partial record; the next run drops it and everything after it.
+		m_file.Write(bytes.data(), static_cast<uint32_t>(sizeof(Frame) + size));
+	});
+}
+
+void ShaderStoreWriter::Post(Common::UniqueFunction<void>&& job) {
+	{
+		std::scoped_lock lock(m_mutex);
+		m_jobs.push(std::move(job));
+	}
+	m_ready.notify_one();
+}
+
+void PipelineCache::WarmUp(const std::filesystem::path& path) {
+	if (path.empty()) {
+		return;
+	}
+	const auto header = StoreHeader(m_graphics);
+	if (header.empty()) {
+		return;
+	}
+	const auto           start = std::chrono::steady_clock::now();
+	std::vector<uint8_t> bytes;
+	if (Common::File file;
+	    Common::File::IsFileExisting(path) && file.Open(path, Common::File::Mode::Read)) {
+		bytes.resize(file.Size());
+		uint32_t read = 0;
+		file.Read(bytes.data(), static_cast<uint32_t>(bytes.size()), &read);
+		bytes.resize(read);
+	}
+	Log::WriteToConsoleAndLog(
+	    fmt::format("Shader cache: compiling shaders and pipelines from {} ({:.1f} MB)\n",
+	                Common::PathToString(path), static_cast<double>(bytes.size()) / 1'000'000.0));
+	std::vector<GraphicsPipelineKey> pipelines;
+	auto                             records = std::span<const uint8_t>(bytes);
+	const bool reusable =
+	    records.size() >= header.size() && std::ranges::equal(records.first(header.size()), header);
+	if (reusable) {
+		records = records.subspan(header.size());
+		while (LoadRecord(records, pipelines)) {
+		}
+	}
+	// A damaged record ends the store; the records before it stay.
+	m_store.Open(path, header, reusable ? bytes.size() - records.size() : 0);
+
+	const auto modules = MapModules();
+	for (const auto& key: pipelines) {
+		Pipeline pipeline;
+		if (CreateGraphicsPipeline(pipeline, key, modules)) {
+			m_graphics_pipelines.emplace(key, pipeline);
+		}
+	}
+	// A compute pipeline depends only on its module.
+	for (const auto* module: modules | std::views::values) {
+		if (module != nullptr && module->info.stage == ShaderType::Compute) {
+			CreatePipelineInternal(m_graphics, m_compute_pipelines[module->id], *module,
+			                       m_driver_cache);
+		}
+	}
+	if (!m_graphics_pipelines.empty() || !m_compute_pipelines.empty()) {
+		CheckpointDriverCache();
+	}
+	Log::WriteToConsoleAndLog(fmt::format(
+	    "Shader cache: loaded {} programs, {} modules, {} pipelines in {} ms\n",
+	    m_program_cache.size(), modules.size() - 1,
+	    m_graphics_pipelines.size() + m_compute_pipelines.size(),
+	    std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() -
+	                                                          start)
+	        .count()));
+}
+
+bool PipelineCache::LoadRecord(std::span<const uint8_t>&        records,
+                               std::vector<GraphicsPipelineKey>& pipelines) {
+	Frame frame;
+	if (records.size() < sizeof(frame)) {
+		return false;
+	}
+	std::memcpy(&frame, records.data(), sizeof(frame));
+	if (records.size() - sizeof(frame) < frame.compressed_size) {
+		return false;
+	}
+	const auto compressed = records.subspan(sizeof(frame), frame.compressed_size);
+	if (XXH3_64bits(compressed.data(), compressed.size()) != frame.checksum ||
+	    ZSTD_getFrameContentSize(compressed.data(), compressed.size()) != frame.raw_size) {
+		return false;
+	}
+	std::vector<uint8_t> raw(frame.raw_size);
+	if (ZSTD_decompress(raw.data(), raw.size(), compressed.data(), compressed.size()) != raw.size()) {
+		return false;
+	}
+	Reader     reader {raw};
+	auto       type     = Record::Module;
+	const auto complete = [&] { return reader.ok && reader.bytes.empty(); };
+	IoAll(reader, type);
+	switch (type) {
+		case Record::Module: {
+			ProgramKey            key;
+			std::vector<uint8_t>  plan;
+			ShaderModule          module;
+			std::vector<uint32_t> spirv;
+			IoKey(reader, key);
+			IoAll(reader, plan);
+			IoModule(reader, module);
+			IoAll(reader, spirv);
+			if (!complete()) {
+				return false;
+			}
+			const auto [program, inserted] = m_program_cache.try_emplace(std::move(key));
+			if (inserted &&
+			    !ShaderRecompiler::IR::LoadResourcePlan(plan, program->second.resource_plan)) {
+				m_program_cache.erase(program);
+				return false;
+			}
+			module.module = CompileSPV(spirv, m_graphics.device);
+			if (module.module == nullptr) {
+				return false;
+			}
+			program->second.modules.push_back(std::move(module));
+			break;
+		}
+		case Record::GraphicsPipeline: {
+			GraphicsPipelineKey key;
+			reader.Raw(&key, sizeof(key));
+			if (!complete()) {
+				return false;
+			}
+			pipelines.push_back(key);
+			break;
+		}
+		case Record::TessellationLayout: {
+			uint64_t                    key = 0;
+			ShaderTessellationInputInfo layout;
+			IoAll(reader, key, layout);
+			if (!complete()) {
+				return false;
+			}
+			m_tess_layouts.try_emplace(key, layout);
+			break;
+		}
+		default: return false;
+	}
+	records = records.subspan(sizeof(frame) + compressed.size());
+	return true;
+}
+
+// Records are encoded here, since programs and modules change later.
+void PipelineCache::StoreModule(const ProgramKey& key, const Program& program,
+                                const ShaderModule& module, const std::vector<uint32_t>& spirv) {
+	if (!m_store) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::Module);
+	IoKey(record, key);
+	IoAll(record, ShaderRecompiler::IR::SaveResourcePlan(program.resource_plan));
+	IoModule(record, module);
+	IoAll(record, spirv);
+	m_store.Append(std::move(record.bytes));
+}
+
+void PipelineCache::StorePipeline(const GraphicsPipelineKey& key) {
+	if (!m_store) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::GraphicsPipeline);
+	record.Raw(&key, sizeof(key));
+	m_store.Append(std::move(record.bytes));
+}
+
+void PipelineCache::StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout) {
+	if (!m_store) {
+		return;
+	}
+	Writer record;
+	IoAll(record, Record::TessellationLayout, key, layout);
+	m_store.Append(std::move(record.bytes));
+}
+
+PipelineCache::ModuleMap PipelineCache::MapModules() const {
+	ModuleMap modules {{0, nullptr}};
+	for (const auto& program: m_program_cache | std::views::values) {
+		for (const auto& module: program.modules) {
+			modules.emplace(module.id, &module);
+		}
+	}
+	return modules;
+}
+
+bool PipelineCache::CreateGraphicsPipeline(Pipeline& pipeline, const GraphicsPipelineKey& key,
+                                           const ModuleMap& modules) {
+	std::array<const ShaderModule*, 4> stages {};
+	for (size_t i = 0; i < stages.size(); i++) {
+		const auto found = modules.find(key.stage_ids[i]);
+		if (found == modules.end()) {
+			return false;
+		}
+		stages[i] = found->second;
+	}
+	CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache);
+	return true;
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 
 #include "common/assert.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/shader/shader.h"
 #include "graphics/shader/shaderBindings.h"
 
@@ -90,7 +91,7 @@ vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer) {
 
 bool HasShaderBufferWrites(const ShaderStageRuntime& runtime) {
 	EXIT_IF(!runtime);
-	const auto& program   = *runtime.program;
+	const auto& program   = runtime.module->info;
 	const auto& resources = *runtime.resources;
 	EXIT_IF(resources.buffers.size() != program.info.buffers.size());
 	// Address writes reach memory outside the bound buffer descriptors.

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -13,7 +13,6 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
@@ -21,124 +20,11 @@
 #include <algorithm>
 #include <array>
 #include <bit>
-#include <limits>
 #include <span>
 #include <vector>
 
 namespace Libs::Graphics {
 
-// IDK: maybe we can remove it?
-constexpr uint8_t kTemporaryVertexAttribFormat113 =
-    static_cast<uint8_t>(Prospero::VertexAttribFormat::k16_16SInt);
-constexpr uint32_t kTemporaryPs5BufferFormat121 = 121u;
-
-static bool NarrowInputFormat(vk::Format& format, uint32_t& size, uint32_t used_components) {
-	if (used_components == 0 || used_components >= size) {
-		return false;
-	}
-
-	switch (format) {
-		case vk::Format::eR32G32B32A32Sfloat:
-			switch (used_components) {
-				case 1: format = vk::Format::eR32Sfloat; break;
-				case 2: format = vk::Format::eR32G32Sfloat; break;
-				case 3: format = vk::Format::eR32G32B32Sfloat; break;
-				default: return false;
-			}
-			size = used_components;
-			return true;
-		case vk::Format::eR32G32B32Sfloat:
-			switch (used_components) {
-				case 1: format = vk::Format::eR32Sfloat; break;
-				case 2: format = vk::Format::eR32G32Sfloat; break;
-				default: return false;
-			}
-			size = used_components;
-			return true;
-		case vk::Format::eR16G16B16A16Sfloat:
-			switch (used_components) {
-				case 1: format = vk::Format::eR16Sfloat; break;
-				case 2: format = vk::Format::eR16G16Sfloat; break;
-				default: return false;
-			}
-			size = used_components;
-			return true;
-		case vk::Format::eR8G8B8A8Unorm:
-			switch (used_components) {
-				case 1: format = vk::Format::eR8Unorm; break;
-				case 2: format = vk::Format::eR8G8Unorm; break;
-				default: return false;
-			}
-			size = used_components;
-			return true;
-		case vk::Format::eR8G8B8A8Snorm:
-			if (used_components != 2) {
-				return false;
-			}
-			format = vk::Format::eR8G8Snorm;
-			size   = 2;
-			return true;
-		case vk::Format::eR8G8B8A8Uint:
-			switch (used_components) {
-				case 1: format = vk::Format::eR8Uint; break;
-				case 2: format = vk::Format::eR8G8Uint; break;
-				default: return false;
-			}
-			size = used_components;
-			return true;
-		default: break;
-	}
-
-	return false;
-}
-
-static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format, uint32_t& size,
-                           uint32_t used_components) {
-	const auto fmt        = res.Format();
-	const auto raw_format = res.RawFormat();
-	if (raw_format == kTemporaryVertexAttribFormat113) {
-		static bool logged_113 = false;
-		if (!logged_113) {
-			LOGF("InputFormat: temporary: accepting invalid PS5 buffer format 113 as "
-			     "vk::Format::eR32G32B32A32Sfloat\n");
-			logged_113 = true;
-		}
-		format = vk::Format::eR32G32B32A32Sfloat;
-		size   = 4;
-		if (NarrowInputFormat(format, size, used_components)) {
-			LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", raw_format,
-			     vk::to_string(format).c_str(), used_components);
-		}
-		return;
-	}
-	if (raw_format == kTemporaryPs5BufferFormat121) {
-		static bool logged_121 = false;
-		if (!logged_121) {
-			LOGF("InputFormat: accepting PS5 buffer format 121 as vk::Format::eR16G16Sfloat\n");
-			logged_121 = true;
-		}
-		format = vk::Format::eR16G16Sfloat;
-		size   = 2;
-		return;
-	}
-
-	format = VulkanFormat(fmt);
-	size   = ShaderRecompiler::Format::GetFormatInfo(fmt).component_count;
-	if (format == vk::Format::eUndefined || size == 0) {
-		EXIT("unknown vertex format: fmt = %u\n", raw_format);
-	}
-
-	if (NarrowInputFormat(format, size, used_components)) {
-		static std::atomic<uint64_t> log_count = 0;
-		auto                         log_id    = log_count.fetch_add(1);
-		if (log_id < 32) {
-			LOGF("VertexInput: narrowed vertex format to %" PRIu32
-			     " component(s) for shader fetch\n",
-			     used_components);
-		}
-	}
-}
-
 static vk::BlendFactor GetBlendFactor(uint32_t factor) {
 	switch (static_cast<Prospero::BlendFactor>(factor)) {
 		case Prospero::BlendFactor::kZero: return vk::BlendFactor::eZero;
@@ -242,33 +128,27 @@ static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipe
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-                            const PipelineRenderingState&          rendering,
-                            const PipelineVertexInputState&        vertex_input,
-                            std::span<const ShaderVertexInputInfo> vertex_info,
-                            const ShaderPixelInputInfo*            ps_input_info,
-                            const PipelineCache::GraphicsPrograms& programs,
-                            const PipelineStaticParameters&        static_params,
-                            vk::PipelineCache                      driver_cache) {
-	const auto& vs_input_info  = vertex_info.front();
-	const auto& vertex_program = programs.vertex[0];
-	const auto& pixel_program  = programs.pixel;
-	const bool  tessellation   = vertex_info.size() == 3;
-	const bool ps_active = ps_input_info != nullptr;
-	EXIT_IF(!vertex_program || (ps_active && !pixel_program));
-	const bool with_depth = rendering.depth_format != vk::Format::eUndefined ||
-	                        rendering.stencil_format != vk::Format::eUndefined;
-	EXIT_IF(!vs_input_info.stage);
-	const bool mesh = vs_input_info.stage.program->stage == ShaderType::Mesh;
+                            const GraphicsPipelineKey&                 key,
+                            const std::array<const ShaderModule*, 4>& stages,
+                            vk::PipelineCache                          driver_cache) {
+	const auto& rendering     = key.rendering;
+	const auto& vertex_input  = key.vertex_input;
+	const auto& static_params = key.static_params;
+	const auto& vertex        = *stages[0];
+	const auto* pixel         = stages[3];
+	const bool  with_depth    = rendering.depth_format != vk::Format::eUndefined ||
+	                            rendering.stencil_format != vk::Format::eUndefined;
+	const bool  mesh          = vertex.info.stage == ShaderType::Mesh;
 	EXIT_NOT_IMPLEMENTED(mesh && !graphics.mesh_shader_enabled);
-	const bool rect_list =
-	    !mesh && !tessellation && static_params.topology == vk::PrimitiveTopology::ePatchList;
+	const bool rect_list = static_params.patch_control_points != 0 && stages[1] == nullptr;
 
 	vk::ShaderModule tess_control_shader_module = nullptr;
 	vk::ShaderModule tess_eval_shader_module    = nullptr;
 
 	if (rect_list) {
-		const auto shaders =
-		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
+		std::span<const RectListParameter> parameters;
+		if (pixel != nullptr) parameters = pixel->rect_parameters;
+		const auto shaders = BuildRectListShaders(parameters, vertex.info.param_export_mask);
 		tess_control_shader_module = CompileSPV(shaders.control, graphics.device);
 		if (graphics_debug_dump_enabled()) {
 			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done module=%p\n",
@@ -287,11 +167,12 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 
 	vk::PipelineShaderStageCreateInfo shader_stages[4] {};
 	uint32_t                          shader_stage_count = 0;
-	for (uint32_t i = 0; i < vertex_info.size(); i++) {
-		shader_stages[shader_stage_count++] = {.stage =
-		                                           NativeShaderStage(vertex_info[i].logical_stage),
-		                                       .module = programs.vertex[i].module,
-		                                       .pName  = "main"};
+	for (const auto* stage: stages) {
+		if (stage != nullptr) {
+			shader_stages[shader_stage_count++] = {.stage  = NativeShaderStage(stage->info.stage),
+			                                       .module = stage->module,
+			                                       .pName  = "main"};
+		}
 	}
 	if (rect_list) {
 		shader_stages[shader_stage_count++] = {.stage =
@@ -303,11 +184,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		                                       .module = tess_eval_shader_module,
 		                                       .pName  = "main"};
 	}
-	if (ps_active) {
-		shader_stages[shader_stage_count++] = {.stage  = vk::ShaderStageFlagBits::eFragment,
-		                                       .module = pixel_program.module,
-		                                       .pName  = "main"};
-	}
 
 	vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX] {};
 	vk::VertexInputBindingDescription   input_desc[ShaderVertexInputInfo::RES_MAX] {};
@@ -323,46 +199,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		input_attr[index].binding  = vertex_input.attributes[index].binding;
 		input_attr[index].location = index;
 		input_attr[index].offset   = vertex_input.attributes[index].offset;
-
-		uint32_t   attr_size     = 4;
-		const auto registers_num = vs_input_info.resources_dst[index].registers_num;
-		const auto compiled_components =
-		    vs_input_info.stage.program->info.vertex_fetch_components[index];
-		const auto used_components =
-		    compiled_components > 0 ? static_cast<int>(compiled_components) : registers_num;
-		GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size,
-		               static_cast<uint32_t>(used_components));
-
-		if (graphics_debug_dump_enabled()) {
-			static std::atomic_uint log_count = 0;
-			const auto              log_id    = log_count.fetch_add(1, std::memory_order_relaxed);
-			if (log_id < 128) {
-				LOGF("VertexInputState[%u]: attr=%u binding=%u offset=%u stride=%u fmt=%d "
-				     "src_fmt=%u dst=v%u regs=%u"
-				     " fetched_components=%u attr_size=%u swizzle=%u,%u,%u,%u\n",
-				     log_id, index, input_attr[index].binding, input_attr[index].offset,
-				     input_desc[input_attr[index].binding].stride,
-				     static_cast<int>(input_attr[index].format),
-				     static_cast<uint32_t>(vs_input_info.resources[index].Format()),
-				     static_cast<uint32_t>(vs_input_info.resources_dst[index].register_start),
-				     static_cast<uint32_t>(registers_num), static_cast<uint32_t>(used_components),
-				     attr_size, static_cast<uint32_t>(vs_input_info.resources[index].DstSelX()),
-				     static_cast<uint32_t>(vs_input_info.resources[index].DstSelY()),
-				     static_cast<uint32_t>(vs_input_info.resources[index].DstSelZ()),
-				     static_cast<uint32_t>(vs_input_info.resources[index].DstSelW()));
-			}
-		}
-
-		if (vs_input_info.resources[index].OutOfBounds() != 0) {
-			static bool logged = false;
-			if (!logged) {
-				LOGF("VertexInput: temporary: accepting PS5 out-of-bounds behavior %" PRIu8 "\n",
-				     vs_input_info.resources[index].OutOfBounds());
-				logged = true;
-			}
-		}
-
-		EXIT_IF(registers_num < 1 || registers_num > 4);
+		input_attr[index].format   = vertex.vertex_formats[index];
 	}
 
 	vk::PipelineVertexInputStateCreateInfo vertex_input_info {};
@@ -470,17 +307,13 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 
 	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
 	vk::ShaderStageFlags graphics_stages = vk::ShaderStageFlagBits::eFragment;
-	for (const auto& stage: vertex_info) {
-		const auto native_stage = NativeShaderStage(stage.logical_stage);
-		AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage);
-		pipeline.uses_image_heap |= stage.stage.program->info.uses_image_heap;
-		graphics_stages |= native_stage;
-	}
-	if (ps_active) {
-		EXIT_IF(!ps_input_info->stage);
-		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
-		                  vk::ShaderStageFlagBits::eFragment);
-		pipeline.uses_image_heap |= ps_input_info->stage.program->info.uses_image_heap;
+	for (const auto* stage: stages) {
+		if (stage != nullptr) {
+			const auto native_stage = NativeShaderStage(stage->info.stage);
+			AddLayoutBindings(descriptor_bindings, stage->info, native_stage);
+			pipeline.uses_image_heap |= stage->info.info.uses_image_heap;
+			graphics_stages |= native_stage;
+		}
 	}
 	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
 	const vk::PushConstantRange push_constants {graphics_stages, 0,
@@ -498,7 +331,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	if (graphics_debug_dump_enabled()) {
 		LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=%" PRIu64 " PS=%" PRIu64
 		     " set_layouts=1 push_constants=%" PRIu32 "\n",
-		     vertex_program.id, ps_active ? pixel_program.id : 0, 1u);
+		     key.stage_ids[0], key.stage_ids[3], 1u);
 	}
 	auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
 	                                                   &pipeline.pipeline_layout);
@@ -508,8 +341,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
-	EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr);
-
 	vk::PipelineDepthStencilStateCreateInfo depth_stencil_info {};
 
 	std::vector<vk::DynamicState> dynamic_states {
@@ -555,9 +386,9 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	pipeline_info.pVertexInputState        = mesh ? nullptr : &vertex_input_info;
 	pipeline_info.pInputAssemblyState      = mesh ? nullptr : &input_assembly;
 	vk::PipelineTessellationStateCreateInfo tessellation_state {};
-	tessellation_state.patchControlPoints =
-	    tessellation ? vs_input_info.tess.input_control_points : 3u;
-	pipeline_info.pTessellationState = (rect_list || tessellation) ? &tessellation_state : nullptr;
+	tessellation_state.patchControlPoints = static_params.patch_control_points;
+	pipeline_info.pTessellationState =
+	    static_params.patch_control_points != 0 ? &tessellation_state : nullptr;
 	pipeline_info.pViewportState          = &viewport_state;
 	pipeline_info.pRasterizationState     = &rasterizer;
 	pipeline_info.pMultisampleState       = &multisampling;
@@ -573,7 +404,7 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=%" PRIu64 " PS=%" PRIu64
 		     " topology=%" PRIu32 " color_mask=0x%08" PRIx32
 		     " depth=%s blend=%s dyn_states=%" PRIu32 "\n",
-		     vertex_program.id, ps_active ? pixel_program.id : 0,
+		     key.stage_ids[0], key.stage_ids[3],
 		     static_cast<uint32_t>(static_params.topology), static_params.color_mask[0],
 		     (with_depth ? "true" : "false"), (static_params.blend_enable[0] ? "true" : "false"),
 		     dynamic_state.dynamicStateCount);
@@ -586,29 +417,18 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
-	EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr);
-
-	if (tess_control_shader_module != nullptr) {
-		graphics.device.destroyShaderModule(tess_control_shader_module, nullptr);
-	}
-	if (tess_eval_shader_module != nullptr) {
-		graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr);
-	}
+	graphics.device.destroyShaderModule(tess_control_shader_module, nullptr);
+	graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr);
 }
 
-// NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-                            const ShaderComputeInputInfo& input_info,
-                            vk::ShaderModule compute_module, vk::PipelineCache driver_cache) {
-	EXIT_IF(compute_module == nullptr);
-
+                            const ShaderModule& module, vk::PipelineCache driver_cache) {
 	vk::PipelineShaderStageCreateInfo                     comp_shader_stage_info {};
 	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo comp_subgroup_size {};
 	comp_shader_stage_info.stage  = vk::ShaderStageFlagBits::eCompute;
-	comp_shader_stage_info.module = compute_module;
+	comp_shader_stage_info.module = module.module;
 	comp_shader_stage_info.pName  = "main";
-	EXIT_IF(!input_info.stage);
-	const auto wave_size = input_info.stage.program->wave_size;
+	const auto wave_size = module.info.wave_size;
 	if (graphics.compute_subgroup_size_control_enabled &&
 	    wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) {
 		comp_subgroup_size.requiredSubgroupSize = wave_size;
@@ -616,9 +436,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	}
 
 	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
-	AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
-	                  vk::ShaderStageFlagBits::eCompute);
-	pipeline.uses_image_heap = input_info.stage.program->info.uses_image_heap;
+	AddLayoutBindings(descriptor_bindings, module.info, vk::ShaderStageFlagBits::eCompute);
+	pipeline.uses_image_heap = module.info.info.uses_image_heap;
 	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
 	const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
 	                                            ShaderRecompiler::IR::NativePushConstantSize};
@@ -632,18 +451,12 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
-	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n",
-	     1u);
 	auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
 	                                                  &pipeline.pipeline_layout);
-	LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n",
-	     vk::to_string(result).c_str(), static_cast<void*>(pipeline.pipeline_layout));
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
-	EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr);
-
 	vk::ComputePipelineCreateInfo info {};
-	if (input_info.stage.program->info.uses_gds) {
+	if (module.info.info.uses_gds) {
 		info.flags = vk::PipelineCreateFlagBits::eDispatchBase;
 	}
 	info.stage             = comp_shader_stage_info;
@@ -652,15 +465,9 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 
 	EXIT_IF(pipeline.pipeline != nullptr);
 
-	LOGF("PipelineTrace: vkCreateComputePipelines begin layout=%p\n",
-	     static_cast<void*>(pipeline.pipeline_layout));
 	result = graphics.device.createComputePipelines(driver_cache, 1, &info, nullptr,
 	                                                &pipeline.pipeline);
-	LOGF("PipelineTrace: vkCreateComputePipelines done result=%s pipeline=%p\n",
-	     vk::to_string(result).c_str(), static_cast<void*>(pipeline.pipeline));
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/render.h

@@ -23,14 +23,9 @@ class Shader;
 } // namespace HW
 
 struct GraphicContext;
-struct ShaderBufferResource;
-struct ShaderComputeInputInfo;
-struct RenderDepthInfo;
-struct RenderColorInfo;
 struct DrawCallInfo;
 struct DrawEmitInfo;
 struct DrawIndexBufferSource;
-struct DrawRenderState;
 class RenderContext;
 class CommandScheduler;
 struct RenderExecutorTestAccess;
@@ -198,9 +193,8 @@ private:
 	                                          uint32_t            render_target_slice_offset,
 	                                          DrawRenderState& state);
 	void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer, const DrawCallInfo& draw,
-	                         DrawRenderState& state, vk::PrimitiveTopology topology,
-	                         const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
-	                         bool primitive_restart_enable);
+	                         DrawRenderState& state, const DrawEmitInfo& emit,
+	                         const DrawIndexBufferSource& index_source);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth,
 	                                               vk::ImageAspectFlags& feedback_aspects,

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -53,7 +53,7 @@ static bool FillSourcesDisjoint(std::span<const ShaderRecompiler::IR::Descriptor
 
 bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
                                                 const CommandBuffer&          buffer) {
-	const auto& program   = *input.stage.program;
+	const auto& program   = input.stage.module->info;
 	const auto& resources = *input.stage.resources;
 	if (resources.buffers.size() != program.info.buffers.size()) {
 		EXIT("compute runtime buffer count does not match shader metadata\n");
@@ -135,7 +135,7 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 bool RenderExecutor::TryConsumeComputeFill(const ShaderComputeInputInfo& input,
                                           CommandBuffer& command, uint32_t group_x,
                                           uint32_t group_y, uint32_t group_z, uint32_t mode) {
-	const auto& program   = *input.stage.program;
+	const auto& program   = input.stage.module->info;
 	const auto& resources = *input.stage.resources;
 	const auto& fill      = resources.uniform_fill;
 	auto&       cache     = command.GetContext().GetTextureCache();
@@ -208,7 +208,7 @@ bool RenderExecutor::TryConsumeComputeFill(const ShaderComputeInputInfo& input,
 	if (logged_clears.fetch_add(1, std::memory_order_relaxed) < 32) {
 		LOGF("GraphicsRenderDispatchDirect: compute %s shader=0x%016" PRIx64
 		     " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n",
-		     image_clear ? "image clear" : "buffer fill", input.stage.program->shader_hash,
+		     image_clear ? "image clear" : "buffer fill", input.stage.module->info.shader_hash,
 		     descriptor.Base48(), size, packed_clear);
 	}
 	return true;
@@ -216,7 +216,7 @@ bool RenderExecutor::TryConsumeComputeFill(const ShaderComputeInputInfo& input,
 
 static void BindSharedMemory(RenderContext& context, ShaderComputeInputInfo& input,
                              PreparedBindings& bindings, uint64_t indirect_args = 0) {
-	if (input.stage.program->bindings.descriptor_counts[
+	if (input.stage.module->info.bindings.descriptor_counts[
 	        static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::SharedMemory)] == 0) {
 		return;
 	}
@@ -248,7 +248,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
                                     uint32_t thread_group_z, uint32_t mode, bool async_compute) {
 	EXIT_IF(buffer.IsInvalid());
 	m_context.GetCommandScheduler().PopPendingOperations();
-	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
 
 	buffer.SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::DispatchDirect), submit_id,
@@ -293,7 +292,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	}
 
 	const auto& cs_regs = sh_ctx.GetCs();
-	const auto& sh_regs = ctx.GetShaderRegisters();
 
 	ShaderComputeInputInfo input_info {};
 	input_info.async_compute = async_compute;
@@ -312,16 +310,14 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		}
 	}
 	m_context.GetImageHeap().BeginDraw();
-	const auto compute_program =
-	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
-	if (!compute_program) return;
+	if (!m_context.GetPipelineCache().GetComputeProgram(cs_regs, input_info)) return;
 	if (use_thread_dimensions) {
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
 		input_info.dispatch_threads_num[2]    = thread_group_z;
 	}
 
-	const auto& program   = *input_info.stage.program;
+	const auto& program   = input_info.stage.module->info;
 	const auto& resources = *input_info.stage.resources;
 	if (resources.specialization_reads.empty() &&
 	    (TryConsumeComputeMetaClear(input_info, buffer) ||
@@ -351,8 +347,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	}
 
 	buffer.EndRendering();
-	auto& pipeline =
-	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
+	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info);
 	auto& bindings = m_compute_bindings;
 	PrepareBindings(input_info.stage, bindings);
 	if (program.bindings.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
@@ -502,16 +497,14 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	input_info.async_compute = async_compute;
 	input_info.indirect_args_address = args_addr;
 	m_context.GetImageHeap().BeginDraw();
-	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
-	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
-	if (!compute_program) return;
+	if (!m_context.GetPipelineCache().GetComputeProgram(cs_regs, input_info)) return;
 	buffer.EndRendering();
-	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
+	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info);
 	auto& bindings = m_compute_bindings;
 	PrepareBindings(input_info.stage, bindings);
 	PreparedBindings* descriptor_stage = &bindings;
 	FindBuffers(std::span {&descriptor_stage, 1u});
-	const auto& program = *input_info.stage.program;
+	const auto& program = input_info.stage.module->info;
 	if (program.info.uses_dma) {
 		m_context.PrepareBda();
 	}

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -10,13 +10,13 @@
 
 namespace Libs::Graphics {
 
-RenderContext::RenderContext(GraphicContext& graphics)
+RenderContext::RenderContext(GraphicContext& graphics, const std::filesystem::path& shader_store)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
-      m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics),
+      m_pipeline_cache(graphics, &m_image_heap, shader_store), m_sampler_cache(graphics),
       m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
       m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache),
-      m_image_heap(*this, m_page_manager) {
+      m_image_heap(*this, m_page_manager), m_occlusion_queries(graphics, m_command_scheduler) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -134,7 +134,11 @@ void RenderContext::PrepareBda() {
 		m_bda_logged = true;
 	}
 	std::shared_lock lock(m_mapped_ranges_mutex);
-	m_buffer_cache.SynchronizeBda(m_mapped_ranges);
+	m_buffer_cache.TakeBdaSync().ForEach([this](uint64_t begin, uint64_t end) {
+		m_mapped_ranges.ForEachInRange(begin, end - begin, [this](uint64_t start, uint64_t finish) {
+			m_buffer_cache.SynchronizeBuffersInRange(start, finish - start);
+		});
+	});
 	m_fault_process_pending = true;
 }
 

src/graphics/host_gpu/renderer/renderContext.h

@@ -11,6 +11,7 @@
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
+#include "graphics/host_gpu/renderer/occlusionQueries.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
 #include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
@@ -30,7 +31,8 @@ class GuestGpu;
 
 class RenderContext {
 public:
-	explicit RenderContext(GraphicContext& graphics);
+	explicit RenderContext(GraphicContext&              graphics,
+	                       const std::filesystem::path& shader_store = ShaderStorePath());
 	~RenderContext();
 	KYTY_CLASS_NO_COPY(RenderContext);
 
@@ -49,6 +51,7 @@ public:
 	BufferCache&        GetBufferCache() { return m_buffer_cache; }
 	TextureCache&       GetTextureCache() { return m_texture_cache; }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
+	OcclusionQueries&   GetOcclusionQueries() { return m_occlusion_queries; }
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
@@ -79,6 +82,7 @@ private:
 	BufferCache               m_buffer_cache;
 	TextureCache              m_texture_cache;
 	ImageHeap                 m_image_heap;
+	OcclusionQueries          m_occlusion_queries;
 	mutable std::shared_mutex m_mapped_ranges_mutex;
 	RangeSet                  m_mapped_ranges;
 	std::unique_ptr<GuestGpu> m_gpu;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -17,7 +17,6 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
@@ -58,7 +57,7 @@ std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
 	}
 
 	EXIT_IF(!vs_input_info.stage);
-	const auto& program   = *vs_input_info.stage.program;
+	const auto& program   = vs_input_info.stage.module->info;
 	const auto& resources = *vs_input_info.stage.resources;
 	if (index_offset == 0 &&
 	    program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
@@ -198,7 +197,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
 	const auto& dc             = ctx.GetDepthControl();
 	const auto& vp             = ctx.GetScreenViewport();
 	const auto& vp0            = vp.viewports[0];
-	const auto& ps_resources   = ps_input_info.stage.program->info;
+	const auto& ps_resources   = ps_input_info.stage.module->info.info;
 	const auto  sampled_images = std::count_if(
 	    ps_resources.images.begin(), ps_resources.images.end(), [](const auto& image) {
 		    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
@@ -321,7 +320,7 @@ static void UpdateGraphicsDynamicState(const CommandBuffer& buffer, DynamicState
 	const auto& ctx = buffer.GetRegisters();
 	const auto&        vp  = ctx.GetScreenViewport();
 	const vk::Extent2D framebuffer_extent {rendering.width, rendering.height};
-	const auto& outputs = vs_input_info.stage.program->info.outputs;
+	const auto& outputs = vs_input_info.stage.module->info.info.outputs;
 	const bool  indexed_viewports =
 	    std::any_of(outputs.begin(), outputs.end(), [](const auto& output) {
 		    return output.kind == ShaderRecompiler::IR::StageOutputKind::ViewportIndex;
@@ -433,22 +432,6 @@ static bool DrawHasValidVertexShader(const HW::Shader& sh_ctx) {
 	return vs.es_regs.data_addr != 0;
 }
 
-static bool PixelShaderHasDepthOrCoverageSideEffects(const HW::ShaderRegisters& sh_regs) {
-	const auto& db = sh_regs.db_shader_control;
-	return db.shader_kill_enable || db.shader_z_export_enable || db.shader_mask_export_enable ||
-	       db.shader_dual_export_enable || db.shader_execute_on_noop || sh_regs.AlphaToMask();
-}
-
-struct DrawRenderState {
-	RenderDepthInfo       depth_info;
-	RenderColorInfo       color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
-	uint32_t              color_count                              = 0;
-	bool                  ps_active                                = true;
-	std::array<ShaderVertexInputInfo, 3> vertex_info;
-	ShaderPixelInputInfo  ps_input_info;
-	PipelineCache::GraphicsPrograms programs;
-};
-
 struct DrawCallInfo {
 	CommandBufferDebugOp debug_op       = CommandBufferDebugOp::DrawIndex;
 	uint32_t             index_count    = 0;
@@ -474,9 +457,8 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
-		const auto owner = cache.m_slot_images.try_get(target.image_id);
-		if (owner == nullptr || (!owner->registered && !owner->info.data.Empty()) ||
-		    owner->binding.needs_rebind) {
+		const auto& owner = cache.m_slot_images[target.image_id];
+		if ((!owner.registered && !owner.info.data.Empty()) || owner.binding.needs_rebind) {
 			EXIT("color target changed after render-state discovery\n");
 		}
 		const auto image_view = cache.FindRenderTarget(target.image_id, target.desc);
@@ -502,8 +484,8 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.image_layout = layout;
 	}
 	if (depth.image_id) {
-		const auto owner = cache.m_slot_images.try_get(depth.image_id);
-		if (owner == nullptr || !owner->registered || owner->binding.needs_rebind) {
+		const auto& owner = cache.m_slot_images[depth.image_id];
+		if (!owner.registered || owner.binding.needs_rebind) {
 			EXIT("depth target changed after render-state discovery\n");
 		}
 		const auto  image_view = cache.FindDepthTarget(depth.image_id, depth.desc);
@@ -589,23 +571,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	return state;
 }
 
-static uint32_t DrawColorOutputMask(const HW::Context& ctx) {
-	const auto& sh_regs     = ctx.GetShaderRegisters();
-	const auto  write_mask  = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask;
-	uint32_t    output_mask = ctx.HasObjectPrimitiveIdOutput() ? 1u << 7u : 0;
-	for (uint32_t index = 0; index < RENDER_COLOR_ATTACHMENTS_MAX; index++) {
-		const auto slot = ShaderPixelExportTarget(sh_regs.m_cbShaderMask, index);
-		if (slot >= RENDER_COLOR_ATTACHMENTS_MAX) {
-			break;
-		}
-		if (sh_regs.target_output_mode[index] != 0 &&
-		    render_target_mask_slot(write_mask, slot) != 0) {
-			output_mask |= 1u << slot;
-		}
-	}
-	return output_mask;
-}
-
 enum class CbColorMode : uint8_t {
 	Disable            = 0,
 	Normal             = 1,
@@ -870,62 +835,19 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
 	return false;
 }
 
-static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
-                           uint32_t color_output_mask, DrawRenderState& state) {
-	auto& ctx    = buffer.GetRegisters();
-	auto& sh_ctx = buffer.GetShaders();
-
-	const auto& vertex_shader_info = sh_ctx.GetVs();
-	const auto& pixel_shader_info  = sh_ctx.GetPs();
-	const auto& shader_regs        = ctx.GetShaderRegisters();
-
-	state.programs      = {};
-	state.ps_input_info = {};
-	std::array<Prospero::ColorComponentMapping, RENDER_COLOR_ATTACHMENTS_MAX>
-	    target_export_mapping {};
-	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
-		const auto& rt = ctx.GetRenderTarget(slot);
-		if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) {
-			target_export_mapping[slot] =
-			    TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type,
-			                                 rt.info.channel_order)
-			        .export_mapping;
-		}
-	}
-	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
-	buffer.GetContext().GetImageHeap().BeginDraw();
-	if (draw.IsIndexed()) {
-		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
-	}
-	state.programs = pipeline_cache.GetGraphicsPrograms(
-	    vertex_shader_info, pixel_shader_info, shader_regs, ctx, buffer.GetUserConfig(),
-	    target_export_mapping, state.ps_active, state.vertex_info, state.ps_input_info);
-}
-
 bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
 	                                        DrawRenderState& state) {
-	const auto& shader_regs       = buffer.GetRegisters().GetShaderRegisters();
-	const auto  color_output_mask = DrawColorOutputMask(buffer.GetRegisters());
-	state.ps_active = buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
-	                  (color_output_mask != 0 ||
-	                   PixelShaderHasDepthOrCoverageSideEffects(shader_regs));
-	RefreshShaders(buffer, draw, color_output_mask, state);
-	uint32_t mrt_mask = 0;
-	if (state.ps_active) {
-		for (const auto& output: state.ps_input_info.stage.program->info.outputs) {
-			if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt ||
-			    output.kind == ShaderRecompiler::IR::StageOutputKind::ObjectPrimitiveId) {
-				mrt_mask |= 1u << output.location;
-			}
-		}
+	m_context.GetImageHeap().BeginDraw();
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
 	}
-	mrt_mask &= color_output_mask;
+	m_context.GetPipelineCache().GetGraphicsPrograms(buffer, state);
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}
 	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
-		if ((mrt_mask & (1u << slot)) != 0) {
+		if ((state.mrt_mask & (1u << slot)) != 0) {
 			ResolveRenderColorTarget(buffer, state.color_info[state.color_count],
 			                         render_target_slice_offset, slot);
 			if (state.color_info[state.color_count].image_id) {
@@ -1124,16 +1046,15 @@ bool RenderExecutor::CopyDepthToColor(CommandBuffer& buffer, const DrawCallInfo&
 
 void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer,
                                          const DrawCallInfo& draw, DrawRenderState& state,
-                                         vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
-                                         const DrawIndexBufferSource& index_source,
-	                                     bool primitive_restart_enable) {
-	auto& ucfg = buffer.GetUserConfig();
+                                         const DrawEmitInfo&          emit,
+                                         const DrawIndexBufferSource& index_source) {
+	auto&      ucfg                     = buffer.GetUserConfig();
+	const bool primitive_restart_enable = state.primitive_restart_enable;
 	if (CopyDepthToColor(buffer, draw, emit, state)) {
 		return;
 	}
-	const auto vertex_stages =
-	    std::span {state.vertex_info.data(), state.programs.VertexStageCount()};
-	const bool mesh_active = state.vertex_info[0].stage.program->stage == ShaderType::Mesh;
+	const auto vertex_stages = state.VertexStages();
+	const bool mesh_active = state.vertex_info[0].stage.module->info.stage == ShaderType::Mesh;
 	uint32_t   mesh_groups = 0;
 	if (mesh_active) {
 		const auto& mesh = state.vertex_info[0].mesh;
@@ -1203,10 +1124,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "CreatePipeline");
 	}
-	auto& pipeline = m_context.GetPipelineCache().GetGraphicsPipeline(
-	    std::span {state.color_info, state.color_count}, state.depth_info, vertex_stages, buffer,
-	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
-	    state.programs);
+	auto& pipeline = m_context.GetPipelineCache().GetGraphicsPipeline(buffer, state);
 	vk::ImageAspectFlags feedback_aspects;
 	const auto rendering =
 	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
@@ -1284,6 +1202,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
 	}
+	auto&         occlusion_queries = m_context.GetOcclusionQueries();
+	const int32_t occlusion_query   = occlusion_queries.Begin(vk_buffer);
 	if (mesh_active) {
 		vk_buffer.drawMeshTasksEXT(mesh_groups, draw.instance_count, 1);
 	} else if (args_buffer.first == nullptr) {
@@ -1295,6 +1215,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		vk_buffer.drawIndirect(args_buffer.first->Handle(), args_buffer.second,
 		                       emit.indirect.count, emit.indirect.stride);
 	}
+	occlusion_queries.End(vk_buffer, occlusion_query);
 	Common::DebugCounters::Add(Common::DebugCounters::Counter::Draws);
 	Common::DebugCounters::Add(Common::DebugCounters::Counter::DrawInstances, draw.instance_count);
 
@@ -1320,7 +1241,7 @@ static std::optional<DrawEmitInfo> ResolveDrawEmit(const HW::UserConfig&
                                                    uint32_t                     first_instance,
                                                    const DrawIndirectArgs&      indirect) {
 	if (indirect.address != 0) {
-		if (vs.stage.program->stage == ShaderType::Mesh ||
+		if (vs.stage.module->info.stage == ShaderType::Mesh ||
 		    ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy) {
 			return std::nullopt;
 		}
@@ -1385,8 +1306,8 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 
 	hw_check(buffer);
 
-	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
-	if (!GetDrawTopology(ucfg, topology)) {
+	DrawRenderState state {};
+	if (!GetDrawTopology(ucfg, state.topology)) {
 		return true;
 	}
 
@@ -1407,7 +1328,7 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		default: EXIT("unknown index_type_and_size: %u\n", args.index_type_and_size);
 	}
 	index_source.size = static_cast<uint64_t>(args.index_count) * index_source.guest_element_size;
-	const bool primitive_restart = ResolvePrimitiveRestart(buffer, index_source);
+	state.primitive_restart_enable = ResolvePrimitiveRestart(buffer, index_source);
 
 	std::vector<uint16_t> expanded_indices;
 	if (index_source.guest_element_size == 1) {
@@ -1415,7 +1336,8 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		const auto* src = static_cast<const uint8_t*>(args.index_addr);
 		expanded_indices.resize(args.index_count);
 		for (uint32_t i = 0; i < args.index_count; i++) {
-			expanded_indices[i] = primitive_restart && src[i] == 0xffu ? 0xffffu : src[i];
+			expanded_indices[i] =
+			    state.primitive_restart_enable && src[i] == 0xffu ? 0xffffu : src[i];
 		}
 		index_source.host_data = expanded_indices.data();
 		index_source.size      = expanded_indices.size() * sizeof(uint16_t);
@@ -1423,7 +1345,6 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 
 	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndex, args.index_count,
 	                        args.instance_count, args.first_instance};
-	DrawRenderState state {};
 	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return true;
@@ -1434,8 +1355,7 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	const auto emit = ResolveDrawEmit(ucfg, state.vertex_info[0], args.offset_source,
 	                                  args.base_vertex, args.first_instance, args.indirect);
 	if (emit) {
-		ExecutePreparedDraw(submit_id, buffer, draw, state, topology, *emit, index_source,
-		                    primitive_restart);
+		ExecutePreparedDraw(submit_id, buffer, draw, state, *emit, index_source);
 	}
 	ResetBindings();
 	return emit.has_value();
@@ -1491,12 +1411,11 @@ bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndexAuto,
 	                         args.vertex_count, args.instance_count, args.first_instance};
 
-	vk::PrimitiveTopology topology = vk::PrimitiveTopology::ePointList;
-	if (!GetDrawTopology(ucfg, topology)) {
+	DrawRenderState state {};
+	if (!GetDrawTopology(ucfg, state.topology)) {
 		ResetBindings();
 		return true;
 	}
-	DrawRenderState state {};
 	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return true;
@@ -1504,7 +1423,7 @@ bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 
 	const bool rect_list = Prospero::IsRectList(ucfg.GetPrimType());
 	if (rect_list && state.vertex_info[0].buffers_num == 0 &&
-	    state.vertex_info[0].stage.program->param_export_mask == 0 &&
+	    state.vertex_info[0].stage.module->info.param_export_mask == 0 &&
 	    state.ps_input_info.input_num != 0) {
 		if (graphics_debug_dump_enabled()) {
 			LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: "
@@ -1521,7 +1440,7 @@ bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	                                  static_cast<int32_t>(args.first_vertex), args.first_instance,
 	                                  args.indirect);
 	if (emit) {
-		ExecutePreparedDraw(submit_id, buffer, draw, state, topology, *emit, {}, false);
+		ExecutePreparedDraw(submit_id, buffer, draw, state, *emit, {});
 	}
 	ResetBindings();
 	return emit.has_value();

src/graphics/presentation/window/vulkanWindow.cpp

@@ -24,6 +24,7 @@
 #include <SDL3/SDL.h>
 #include <SDL3/SDL_vulkan.h>
 #include <algorithm>
+#include <cmath>
 #include <cstdio>
 #include <cstring>
 #include <fmt/format.h>
@@ -506,6 +507,13 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		maintenance8.pNext = supported_features2.pNext;
 		supported_features2.pNext = &maintenance8;
 	}
+	const bool shader_clock_extension =
+	    HasExtension(device_extensions, VK_KHR_SHADER_CLOCK_EXTENSION_NAME);
+	vk::PhysicalDeviceShaderClockFeaturesKHR shader_clock {};
+	if (shader_clock_extension) {
+		shader_clock.pNext = supported_features2.pNext;
+		supported_features2.pNext = &shader_clock;
+	}
 	const bool image_atomic_int64_extension =
 	    HasExtension(device_extensions, VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME);
 	vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic_int64 {};
@@ -593,6 +601,11 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	    feedback_extensions && feedback_layout.attachmentFeedbackLoopLayout &&
 	    feedback_dynamic.attachmentFeedbackLoopDynamicState;
 	graphics.maintenance8_enabled = maintenance8_extension && maintenance8.maintenance8;
+	// The device clock ticks at timestampPeriod nanoseconds.
+	if (shader_clock_extension && shader_clock.shaderDeviceClock) {
+		graphics.device_clock_period_ps = static_cast<uint32_t>(
+		    std::lround(graphics.physical_device_properties.limits.timestampPeriod * 1000.0f));
+	}
 	graphics.conditional_rendering_enabled =
 	    conditional_rendering_extension && conditional_rendering.conditionalRendering;
 	LOGF("Vulkan maintenance8 support: %s\n",
@@ -642,6 +655,16 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		    vk::SampleCountFlagBits::e1;
 	}
 	graphics.sample_rate_shading_enabled                 = true;
+	// Occlusion queries reset their queries from the host and count samples exactly where they can.
+	graphics.occlusion_queries_enabled =
+	    Config::OcclusionQueriesEnabled() && supported_features12.hostQueryReset;
+	graphics.occlusion_query_precise_enabled =
+	    graphics.occlusion_queries_enabled && supported_features2.features.occlusionQueryPrecise;
+	features12.hostQueryReset             = graphics.occlusion_queries_enabled;
+	device_features.occlusionQueryPrecise = graphics.occlusion_query_precise_enabled;
+	LOGF("Vulkan occlusion queries: %s, precise: %s\n",
+	     graphics.occlusion_queries_enabled ? "true" : "false",
+	     graphics.occlusion_query_precise_enabled ? "true" : "false");
 	device_features.shaderInt64 = VK_TRUE;
 	device_features.shaderFloat64 =
 	    supported_features2.features.shaderFloat64 &&
@@ -702,6 +725,10 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		maintenance8.pNext = const_cast<void*>(create_info.pNext);
 		create_info.pNext = &maintenance8;
 	}
+	if (graphics.device_clock_period_ps != 0u) {
+		shader_clock.pNext = const_cast<void*>(create_info.pNext);
+		create_info.pNext = &shader_clock;
+	}
 	if (graphics.conditional_rendering_enabled) {
 		conditional_rendering.pNext = const_cast<void*>(create_info.pNext);
 		create_info.pNext = &conditional_rendering;
@@ -1077,6 +1104,7 @@ void WindowContext::CreateVulkan() {
 			graphic_ctx.memory_budget_ext_enabled = true;
 		}
 		for (const auto* extension: {VK_KHR_MAINTENANCE_8_EXTENSION_NAME,
+		                             VK_KHR_SHADER_CLOCK_EXTENSION_NAME,
 		                             VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME,
 		                             VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME,
 		                             VK_EXT_MESH_SHADER_EXTENSION_NAME,

src/graphics/presentation/window/window.cpp

@@ -32,8 +32,6 @@
 #include <vector>
 #include <vulkan/vk_platform.h>
 
-#define STB_IMAGE_IMPLEMENTATION
-#define STBI_NO_SIMD
 #include "stb_image.h"
 
 // IWYU pragma: no_include <intrin.h>

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -30,10 +30,6 @@ namespace Libs::Graphics::ShaderRecompiler {
 
 namespace {
 
-const char* GetDumpLabel(const CompileOptions& options) {
-	return options.dump_label != nullptr ? options.dump_label : "ShaderRecompiler";
-}
-
 std::string MakeIrDump(std::string_view cfg, const IR::Program& ir) {
 	std::string dump = "CFG:\n";
 	dump += cfg;
@@ -44,30 +40,17 @@ std::string MakeIrDump(std::string_view cfg, const IR::Program& ir) {
 	return dump;
 }
 
-const char* StageName(ShaderType stage) {
-	switch (stage) {
-		case ShaderType::Compute: return "CS";
-		case ShaderType::Vertex: return "VS";
-		case ShaderType::Local: return "LS";
-		case ShaderType::TessellationControl: return "HS";
-		case ShaderType::TessellationEvaluation: return "TES";
-		case ShaderType::Mesh: return "MS";
-		case ShaderType::Pixel: return "PS";
-		default: return "unknown";
-	}
-}
-
 void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase) {
 	const auto* block        = cfg.FindBlock(cfg.failure_block);
 	const auto  start        = block != nullptr ? block->start_pc : UINT32_MAX;
 	const auto  end          = block != nullptr ? block->end_pc : UINT32_MAX;
 	const auto  predecessors = block != nullptr ? block->predecessors.size() : 0u;
 	const auto  successors   = block != nullptr ? block->successors.size() : 0u;
-	LOGF("%s CFG dispatcher fallback: stage=%s hash=0x%016" PRIx64
+	LOGF("ShaderRecompiler %s CFG dispatcher fallback: hash=0x%016" PRIx64
 	     " phase=%s failure=%s block=%" PRIu32 " pc=0x%08" PRIx32 "..0x%08" PRIx32 " preds=%" PRIu64
 	     " succs=%" PRIu64 " blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64
 	     " reason=%s\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash, phase,
+	     StageName(options.stage), options.shader_hash, phase,
 	     CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
 	     static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
@@ -584,14 +567,30 @@ void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
 	for (auto& inst: block) {
 		inst.SetParent(nullptr);
 		if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 || inst.GetOpcode() == ValueOpcode::ReadConstBuffer)
-			inst.SetFlags(SrtReadFlags {.index = inst.Flags<MemoryFlags>().index});
+			inst.SetFlags(MemoryFlags {.index = inst.Flags<MemoryFlags>().index});
 	}
 	query.value_storage.splice(query.value_storage.end(), block.Instructions());
 	source.call_targets = std::move(static_cast<ResourcePlan&>(query));
+	std::vector<Value> targets;
+	for (const auto& site: source.calls) targets.insert(targets.end(), site.target.begin(), site.target.end());
+	CompileSrtWalker(source.call_targets, targets);
 }
 
 } // namespace
 
+const char* StageName(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Compute: return "CS";
+		case ShaderType::Vertex: return "VS";
+		case ShaderType::Local: return "LS";
+		case ShaderType::TessellationControl: return "HS";
+		case ShaderType::TessellationEvaluation: return "TES";
+		case ShaderType::Mesh: return "MS";
+		case ShaderType::Pixel: return "PS";
+		default: return "unknown";
+	}
+}
+
 ShaderSource PrepareShaderSource(std::span<const uint32_t> code, const CompileOptions& options) {
 	EXIT_IF(code.empty());
 	ShaderSource source;
@@ -617,12 +616,18 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
 	EXIT_IF(source.unsupported_call_pc != UINT32_MAX);
 	auto& reads = source.reads;
 	reads.clear();
-	IR::SrtReadCapture capture(runtime, reads);
-	const auto clean = IR::CleanRuntime(capture.ObservedRuntime());
-	IR::SrtWalker walker(source.call_targets, clean);
+	std::vector<uint32_t> flat;
+	const bool walked = IR::WalkSrt(source.call_targets, runtime, flat, &reads);
+	const auto read_code = [&](uint64_t address, std::span<uint32_t> words) {
+		if (runtime.read_specialization_memory == nullptr ||
+		    !runtime.read_specialization_memory(runtime.userdata, address, words)) return false;
+		reads.emplace_back(address, words.size_bytes());
+		return true;
+	};
 	for (auto& site: source.calls) {
 		uint32_t low = 0, high = 0;
-		if (!walker.Evaluate(site.target[0], low) || !walker.Evaluate(site.target[1], high))
+		if (!walked || !IR::WalkedValue(source.call_targets, site.target[0], low) ||
+		    !IR::WalkedValue(source.call_targets, site.target[1], high))
 			EXIT("shader call at pc 0x%08x has an unavailable scalar target",
 			     source.decoded.instructions[site.instruction].pc);
 		// SWAPPC reads the old aliased pair and ignores the target's low two bits.
@@ -646,7 +651,7 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
 				break;
 			}
 			linked.observed_function.resize(function.size);
-			if (!clean.read_memory(clean.userdata, source.calls[i].address, linked.observed_function) ||
+			if (!read_code(source.calls[i].address, linked.observed_function) ||
 			    !std::ranges::equal(expected, linked.observed_function)) {
 				unchanged = false;
 				break;
@@ -676,8 +681,7 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
 				uint32_t count = 1;
 				Decoder::Instruction inst;
 				for (;;) {
-					if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
-					                       std::span(words).first(count)))
+					if (!read_code(address + uint64_t {index} * sizeof(uint32_t), std::span(words).first(count)))
 						EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
 					inst = {};
 					Decoder::DecodeInstruction(words, 0u, inst);
@@ -756,6 +760,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 		     static_cast<unsigned>(options.stage));
 	}
 
+	const auto* stage        = StageName(options.stage);
 	const auto compile_begin = std::chrono::steady_clock::now();
 	const auto phase_ms = [&compile_begin]() {
 		return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -765,27 +770,25 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 	std::string decoded_dump;
 	if (options.dump_ir) {
 		decoded_dump = Decoder::ProgramToString(decoded);
-		if (options.early_dump) {
-			LOGF("%s decoded RDNA2 (early):\n%s", GetDumpLabel(options), decoded_dump.c_str());
-		}
+		LOGF("ShaderRecompiler %s decoded RDNA2:\n%s", stage, decoded_dump.c_str());
 	}
 
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
-	     StageName(options.stage), options.shader_hash);
+	LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " CFG BuildGraph\n", stage,
+	     options.shader_hash);
 	auto native_cfg = CFG::BuildGraph(decoded);
 	CFG::Graph structured_cfg;
 	auto* selected_cfg = &native_cfg;
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
+	LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
 	     " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     stage, options.shader_hash,
 	     static_cast<uint64_t>(native_cfg.blocks.size()),
 	     static_cast<uint64_t>(native_cfg.natural_loops.size()),
 	     static_cast<uint64_t>(native_cfg.back_edges.size()), phase_ms());
 	if (native_cfg.irreducible) {
 		LogDispatcherFallback(options, native_cfg, "build");
 	} else {
-		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
-		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+		LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " CFG Structurize\n", stage,
+		     options.shader_hash);
 		structured_cfg = CFG::Structurize(native_cfg);
 		if (structured_cfg.unsupported) {
 			native_cfg.unsupported = true;
@@ -795,12 +798,12 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 			LogDispatcherFallback(options, native_cfg, "structurize");
 		} else {
 			selected_cfg = &structured_cfg;
-			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
+			LOGF("ShaderRecompiler %s structured CFG success: blocks=%" PRIu64 "\n", stage,
 			     static_cast<uint64_t>(selected_cfg->blocks.size()));
 		}
-		LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
+		LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
 		     " loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
-		     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+		     stage, options.shader_hash,
 		     static_cast<uint64_t>(selected_cfg->blocks.size()),
 		     static_cast<uint64_t>(selected_cfg->natural_loops.size()), phase_ms());
 	}
@@ -813,7 +816,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 		    decoded, *options.input_info.vertex, options.user_data_base,
 		    static_cast<uint32_t>(options.user_data.size()), options.wave_size);
 		if (!embedded_fetch.loads.empty()) {
-			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
+			LOGF("ShaderRecompiler %s embedded vertex fetch plan: detected=%" PRIu64 "\n", stage,
 			     static_cast<uint64_t>(embedded_fetch.loads.size()));
 		}
 	}
@@ -826,12 +829,12 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 	    .input_info       = options.input_info,
 	    .embedded_fetch   = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
 	};
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " IR TranslateProgram\n", stage,
+	     options.shader_hash);
 	auto ir = Frontend::TranslateProgram(decoded, cfg, translate_options);
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
+	LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     stage, options.shader_hash,
 	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
 	IR::RewriteToSsa(ir.blocks);
 	const auto simplify = [&ir] {
@@ -842,8 +845,9 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 	simplify();
 	const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size);
 	if (read_lane_stats.rewritten_reads != 0 || read_lane_stats.rewritten_reductions != 0) {
-		LOGF("%s read-lane elimination: reads=%" PRIu32 " reductions=%" PRIu32 "\n",
-		     GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.rewritten_reductions);
+		LOGF("ShaderRecompiler %s read-lane elimination: reads=%" PRIu32
+		     " reductions=%" PRIu32 "\n",
+		     stage, read_lane_stats.rewritten_reads, read_lane_stats.rewritten_reductions);
 	}
 	// Cleanup also exposes profitable folds by removing dead predicate consumers.
 	simplify();
@@ -857,10 +861,8 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 	std::string cfg_dump;
 	if (options.dump_ir) {
 		cfg_dump = CFG::GraphToString(cfg);
-		if (options.early_dump) {
-			LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options),
-			     MakeIrDump(cfg_dump, ir).c_str());
-		}
+		LOGF("ShaderRecompiler %s native IR before resource tracking:\n%s", stage,
+		     MakeIrDump(cfg_dump, ir).c_str());
 	}
 	IR::TrackResources(ir, decoded, native_cfg, options.input_info);
 	TranslateResult result;
@@ -877,6 +879,7 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
                              uint32_t push_data_start_dword) {
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
+	const auto* stage = StageName(ir.stage);
 	IR::ApplyResourceSpecialization(ir, specialization);
 	// The resource plan owns host descriptor evaluation now. Keep only dependencies consumed
 	// by GPU memory operations; bound descriptor dwords must not retain shader instructions.
@@ -923,19 +926,16 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(translated.cfg_dump, ir);
-		if (options.early_dump) {
-			LOGF("%s native IR (early):\n%s", GetDumpLabel(options), ir_dump.c_str());
-		}
+		LOGF("ShaderRecompiler %s native IR:\n%s", stage, ir_dump.c_str());
 	}
 
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
-	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
+	LOGF("ShaderRecompiler %s phase begin: hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", stage,
+	     ir.shader_hash);
 	auto spirv = Spirv::EmitProgram(ir, options.input_info, push_data_start_dword, *options.profile,
 	                                options.emit_debug_info);
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
-	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,
-	     static_cast<uint64_t>(spirv.size()),
+	LOGF("ShaderRecompiler %s phase end: hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
+	     " wave_size=%u elapsed_ms=%" PRIu64 "\n",
+	     stage, ir.shader_hash, static_cast<uint64_t>(spirv.size()), ir.wave_size,
 	     static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
 	                               std::chrono::steady_clock::now() - emit_begin)
 	                               .count()));

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -22,10 +22,8 @@ struct CompileOptions {
 	uint32_t                    user_data_base  = 0;
 	uint64_t                    shader_hash     = 0;
 	bool                        dump_ir                    = true;
-	bool                        early_dump                 = false;
 	bool                        emit_debug_info            = false;
 	const Spirv::Profile*       profile                    = &Spirv::DefaultProfile;
-	const char*                 dump_label                 = nullptr;
 	std::span<const uint32_t>   user_data;
 	std::span<const uint32_t>   back_code;
 	ShaderStageInputInfo        input_info;
@@ -69,6 +67,7 @@ struct ShaderSource {
 	std::vector<std::pair<uint64_t, uint64_t>> reads;
 };
 
+[[nodiscard]] const char* StageName(ShaderType stage);
 [[nodiscard]] ShaderSource PrepareShaderSource(std::span<const uint32_t> code,
 	                                           const CompileOptions& options);
 const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime);

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -13,6 +13,8 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv {
 struct Profile {
 	bool maintenance8         = false; // Dynamic texel offsets on image samples.
 	bool native_float_min_max = false; // NMin/NMax give the guest's NaN and signed-zero results.
+	// OpReadClockKHR Device tick in picoseconds; 0 without shaderDeviceClock.
+	uint32_t device_clock_period_ps = 0;
 };
 
 inline constexpr Profile DefaultProfile {};

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -54,9 +54,30 @@ uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
 	return result;
 }
 
-uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t value) {
+uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
+	// Clamping first keeps the truncating conversion defined; NaN and values past the largest
+	// float below the integer limit are selected afterwards.
+	const auto clamped =
+	    EmitExt(state, TypeF32(state), GLSLstd450NClamp,
+	            {src, ConstantF32(state, signed_value ? 0xcf000000u : 0u),
+	             ConstantF32(state, signed_value ? 0x4effffffu : 0x4f7fffffu)});
+	const auto converted = signed_value
+	                           ? Unary(state, spv::OpBitcast, TypeU32(state),
+	                                   Unary(state, spv::OpConvertFToS, TypeI32(state), clamped))
+	                           : Unary(state, spv::OpConvertFToU, TypeU32(state), clamped);
+	const auto above     = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
+	                              ConstantF32(state, signed_value ? 0x4f000000u : 0x4f800000u));
+	const auto high =
+	    Select(state, TypeU32(state), above,
+	           ConstantU32(state, signed_value ? 0x7fffffffu : 0xffffffffu), converted);
+	return Select(state, TypeU32(state), Unary(state, spv::OpIsNan, TypeBool(state), src),
+	              ConstantU32(state, 0), high);
+}
+
+} // namespace
+
+uint32_t EmitF32BitsToF16RtzBits(EmitterState& state, uint32_t bits) {
 	// Native half conversion causes regressions; use compact integer RTZ conversion.
-	const auto bits = EmitBitcastF32ToU32(state, value);
 	const auto abs  = EmitAndConstant(state, bits, 0x7fffffffu);
 	const auto sign = EmitAndConstant(
 	    state, Binary(state, spv::OpShiftRightLogical, TypeU32(state), bits, ConstantU32(state, 16)),
@@ -92,28 +113,6 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t value) {
 	return EmitOrU32(state, sign, magnitude);
 }
 
-uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
-	// Clamping first keeps the truncating conversion defined; NaN and values past the largest
-	// float below the integer limit are selected afterwards.
-	const auto clamped =
-	    EmitExt(state, TypeF32(state), GLSLstd450NClamp,
-	            {src, ConstantF32(state, signed_value ? 0xcf000000u : 0u),
-	             ConstantF32(state, signed_value ? 0x4effffffu : 0x4f7fffffu)});
-	const auto converted = signed_value
-	                           ? Unary(state, spv::OpBitcast, TypeU32(state),
-	                                   Unary(state, spv::OpConvertFToS, TypeI32(state), clamped))
-	                           : Unary(state, spv::OpConvertFToU, TypeU32(state), clamped);
-	const auto above     = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
-	                              ConstantF32(state, signed_value ? 0x4f000000u : 0x4f800000u));
-	const auto high =
-	    Select(state, TypeU32(state), above,
-	           ConstantU32(state, signed_value ? 0x7fffffffu : 0xffffffffu), converted);
-	return Select(state, TypeU32(state), Unary(state, spv::OpIsNan, TypeBool(state), src),
-	              ConstantU32(state, 0), high);
-}
-
-} // namespace
-
 uint32_t EmitFPFma32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
 	const auto result = EmitGlsl<GLSLstd450Fma, IR::Type::F32>(state, a, b, c);
 	state.builder.AddAnnotation(spv::OpDecorate, result, spv::DecorationNoContraction);
@@ -199,7 +198,8 @@ uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value
 
 uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return EmitNative<spv::OpBitFieldInsert, IR::Type::U32>(
-	    state, EmitF32ToF16RtzBits(state, arg0), EmitF32ToF16RtzBits(state, arg1),
+	    state, EmitF32BitsToF16RtzBits(state, EmitBitcastF32ToU32(state, arg0)),
+	    EmitF32BitsToF16RtzBits(state, EmitBitcastF32ToU32(state, arg1)),
 	    ConstantU32(state, 16), ConstantU32(state, 16));
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -4,6 +4,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <bit>
 #include <optional>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -113,6 +114,18 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
 	return EmitInputComponentU32(state, kind, component);
 }
 
+// Whether EXEC enables a source lane within the caller's 32-lane half.
+uint32_t EmitSourceLaneActive(ValueEmitContext& ctx, uint32_t exec, uint32_t lane) {
+	auto&      state  = ctx.state;
+	const auto scope  = ConstantU32(state, spv::ScopeSubgroup);
+	const auto ballot = EmitNative<spv::OpGroupNonUniformBallot, IR::Type::U32x4>(state, scope, exec);
+	if (ctx.half != 0) {
+		lane = EmitAndConstant(state, lane, 31);
+	}
+	return EmitNative<spv::OpGroupNonUniformBallotBitExtract, IR::Type::U1>(state, scope, ballot,
+	                                                                        lane);
+}
+
 uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
                                uint32_t exec) {
 	auto&    state    = ctx.state;
@@ -134,9 +147,24 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	// Only row shifts (0x101-0x10f, 0x111-0x11f) can vacate a lane; other controls always read one.
 	const bool row_shift = flags.control > 0x100u && flags.control < 0x120u &&
 	                       flags.control != 0x110u;
-	if (!flags.bound_control && row_shift) {
+	if (flags.bound_control || (flags.fetch_inactive && !row_shift)) {
+		return writable;
+	}
+	// Without BOUND_CTRL, a vacated source or, without FI, an EXEC-disabled one disables the write.
+	const auto target = EmitDppTargetLane(state, flags);
+	if (row_shift) {
+		writable = Binary(state, spv::OpLogicalAnd, TypeBool(state), writable, target.valid);
+	}
+	if (!flags.fetch_inactive) {
+		// Vacated lanes target outside their row; wrapping keeps the ballot index in range.
+		auto source = target.lane;
+		if (row_shift) {
+			lane   = lane != 0 ? lane : EmitSubgroupLocalInvocationId(state);
+			source = EmitNative<spv::OpBitFieldInsert, IR::Type::U32>(
+			    state, lane, source, ConstantU32(state, 0), ConstantU32(state, 4));
+		}
 		writable = Binary(state, spv::OpLogicalAnd, TypeBool(state), writable,
-		                  EmitDppTargetLane(state, flags).valid);
+		                  EmitSourceLaneActive(ctx, exec, source));
 	}
 	return writable;
 }
@@ -815,27 +843,23 @@ uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
 }
 
 uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
-	auto&      state     = ctx.state;
-	const auto flags     = inst.Flags<IR::PermlaneFlags>();
-	const auto subid     = EmitSubgroupLocalInvocationId(state);
-	const auto row       = state.builder.AllocateId();
-	const auto row_value = state.builder.AllocateId();
-	const auto lane      = state.builder.AllocateId();
-	const auto lane8     = state.builder.AllocateId();
-	const auto shift     = state.builder.AllocateId();
-	const auto upper     = state.builder.AllocateId();
-	const auto selected  = state.builder.AllocateId();
-	const auto shifted   = state.builder.AllocateId();
-	const auto index     = state.builder.AllocateId();
-	const auto target    = state.builder.AllocateId();
+	auto&      state    = ctx.state;
+	const auto flags    = inst.Flags<IR::PermlaneFlags>();
+	const auto subid    = EmitSubgroupLocalInvocationId(state);
+	const auto row      = state.builder.AllocateId();
+	const auto lane     = state.builder.AllocateId();
+	const auto lane8    = state.builder.AllocateId();
+	const auto shift    = state.builder.AllocateId();
+	const auto upper    = state.builder.AllocateId();
+	const auto selected = state.builder.AllocateId();
+	const auto shifted  = state.builder.AllocateId();
+	const auto index    = state.builder.AllocateId();
+	const auto target   = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
 	                          ConstantU32(state, 0xfffffff0u));
-	if (flags.x16) {
-		state.builder.AddFunction(spv::OpBitwiseXor, TypeU32(state), row_value, row,
-		                          ConstantU32(state, 16));
-	} else {
-		state.builder.AddFunction(spv::OpCopyObject, TypeU32(state), row_value, row);
-	}
+	const auto row_value =
+	    flags.x16 ? Binary(state, spv::OpBitwiseXor, TypeU32(state), row, ConstantU32(state, 16))
+		          : row;
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
 	                          ConstantU32(state, 15));
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane8, lane,
@@ -851,14 +875,11 @@ uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	                          ConstantU32(state, 15));
 	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row_value, index);
 	const auto shuffled = ctx.Shuffle(inst, 0, target);
-	uint32_t   result   = shuffled;
-	if (!flags.fetch_inactive) {
-		const auto source_exec = ctx.Shuffle(inst, 3, target);
-		result                 = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpSelect, TypeU32(state), result, source_exec, shuffled,
-		                          ConstantU32(state, 0));
+	if (flags.fetch_inactive) {
+		return shuffled;
 	}
-	return result;
+	return Select(state, TypeU32(state), EmitSourceLaneActive(ctx, ctx.Arg(inst, 3), target),
+	              shuffled, ctx.Arg(inst, 4));
 }
 
 uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -881,6 +902,22 @@ uint32_t EmitGetShaderBase(ValueEmitContext& ctx) {
 	return ctx.Def(IR::Value(uint64_t {0}));
 }
 
+uint32_t EmitReadRealtimeClock(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state  = ctx.state;
+	const auto period = state.profile.device_clock_period_ps;
+	if (period == 0u) {
+		ctx.Fail(inst, "requires shaderDeviceClock");
+	}
+	state.builder.RequireExtension("SPV_KHR_shader_clock");
+	state.builder.RequireCapability(spv::CapabilityShaderClockKHR);
+	const auto clock =
+	    Unary(state, spv::OpReadClockKHR, TypeU64(state), ConstantU32(state, spv::ScopeDevice));
+	// S_MEMREALTIME counts at 100 MHz. Shift the host clock by the most bits that keep its tick at
+	// most 15000 ps (2/3 of 100 MHz): 1 GHz shifts by 3 to 125 MHz; 100 MHz and slower stay.
+	const uint32_t shift = std::bit_width(7500u / period);
+	return EmitShiftRightLogical64(ctx, clock, IR::Value(shift));
+}
+
 void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) {
 	ctx.Fail(inst, "must be lowered before SPIR-V emission");
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -56,6 +56,7 @@ EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint3
 EMIT_NATIVE(SelectU1, OpSelect, U1, uint32_t, uint32_t, uint32_t)
 EMIT_NATIVE(SelectU32, OpSelect, U32, uint32_t, uint32_t, uint32_t)
 EMIT_NATIVE(SelectF32, OpSelect, F32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectF64, OpSelect, F64, uint32_t, uint32_t, uint32_t)
 EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
 EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
@@ -240,6 +241,7 @@ uint32_t              EmitGetAttributeF16(ValueEmitContext& ctx, const IR::Inst&
 uint32_t              EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetShaderBase(ValueEmitContext& ctx);
+uint32_t              EmitReadRealtimeClock(ValueEmitContext& ctx, const IR::Inst& inst);
 inline constexpr auto EmitGetSrtResource     = EmitVoid;
 inline constexpr auto EmitGetBufferResource  = EmitGetSrtResource;
 uint32_t              EmitGetBufferResourceIndex(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -531,6 +531,7 @@ inline constexpr auto EmitFNegateValue = EmitNative<spv::OpFNegate, IR::Type::F3
 inline constexpr auto EmitFAbsValue = EmitGlsl<GLSLstd450FAbs, IR::Type::F32, uint32_t>;
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
+uint32_t EmitF32BitsToF16RtzBits(EmitterState& state, uint32_t bits);
 
 void EmitProgram(EmitterState& state);
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -472,6 +472,17 @@ uint32_t EncodeFormattedStoreComponent(EmitterState& state, const Format::Buffer
                                        UnormRounding rounding) {
 	EXIT_IF(rounding != UnormRounding::RoundEven && info.type != Format::ComponentType::Unorm);
 	const auto bits = info.component_bits[component];
+	if (info.type == Format::ComponentType::Float && bits < 16u) {
+		// Truncate to f16, whose bias the unsigned 11/10-bit floats share, and drop mantissa
+		// bits; negatives and NaN, whose f16 bits lie above +inf, store zero.
+		const auto half = data_bits == 16u ? EmitAndConstant(state, data, 0xffffu)
+		                                   : EmitF32BitsToF16RtzBits(state, data);
+		return Select(state, TypeU32(state),
+		              EmitCompareU32Constant(state, spv::OpUGreaterThan, half, 0x7c00u),
+		              ConstantU32(state, 0),
+		              EmitBinaryU32(state, spv::OpShiftRightLogical, half,
+		                            ConstantU32(state, 15u - bits)));
+	}
 	if (data_bits == 16u) {
 		if (info.type == Format::ComponentType::Uint) {
 			data = EmitBitFieldUExtract(state, data, ConstantU32(state, 0), ConstantU32(state, 16));

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -187,7 +187,7 @@ void EmitMeshEntryPoint(EmitterState& state) {
 				vertex[component] = state.builder.AllocateId();
 				state.builder.AddFunction(
 				    spv::OpBitFieldUExtract, TypeU32(state), vertex[component], packed,
-				    ConstantU32(state, component * 10u), ConstantU32(state, 10));
+				    ConstantU32(state, component * 10u), ConstantU32(state, 9));
 			}
 			const auto triangle = state.builder.AllocateId();
 			state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 3), triangle,

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -235,9 +235,9 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto op = inst.GetOpcode();
 	if (ctx.half != 0 &&
 	    (op == Op::Ballot || op == Op::ReadFirstLane || op == Op::SubgroupUMinU32 ||
-	     op == Op::ReadConstBuffer ||
+	     op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock ||
 	     (op == Op::LoadAddressU32 && ctx.Memory(inst).kind == IR::ResourceKind::ScalarAddress))) {
-		// Whole-wave results and SMEM loads (SGPR operands, no EXEC) are identical in both halves.
+		// Whole-wave results, clocks and SMEM loads (SGPR operands, no EXEC) match in both halves.
 		ctx.Define(inst, ctx.other_half->Result(inst));
 		return;
 	}

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -57,6 +57,8 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x14u, Opcode::S_FF1_I32_B64},
     {0x15u, Opcode::S_FLBIT_I32_B32},
     {0x16u, Opcode::S_FLBIT_I32_B64},
+    {0x17u, Opcode::S_FLBIT_I32},
+    {0x18u, Opcode::S_FLBIT_I32_I64},
     {0x19u, Opcode::S_SEXT_I32_I8},
     {0x1au, Opcode::S_SEXT_I32_I16},
     {0x1bu, Opcode::S_BITSET0_B32},

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -60,6 +60,8 @@ enum class Opcode {
 	S_FF1_I32_B64,
 	S_FLBIT_I32_B32,
 	S_FLBIT_I32_B64,
+	S_FLBIT_I32,
+	S_FLBIT_I32_I64,
 	S_BITREPLICATE_B64_B32,
 	S_QUADMASK_B64,
 	S_GETPC_B64,

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -465,9 +465,13 @@ void Translator::V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16) {
 	    .fetch_inactive = inst.dst.op_sel,
 	    .bound_control  = inst.dst.op_sel_hi,
 	};
-	const auto result =
-	    ir.Emit(IR::ValueOpcode::Permlane16U32,
-	            {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec()}, flags);
+	// An EXEC-disabled source lane without FI disables the write unless BOUND_CTRL reads zero.
+	const auto inactive = flags.fetch_inactive || flags.bound_control
+	                          ? IR::U32(IR::Value(0u))
+	                          : ir.GetVectorReg(static_cast<IR::VectorReg>(inst.dst.reg));
+	const auto result   = ir.Emit(
+	    IR::ValueOpcode::Permlane16U32,
+	    {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec(), inactive}, flags);
 	auto dst      = DestinationOperand(inst);
 	dst.op_sel    = false;
 	dst.op_sel_hi = false;

src/graphics/shader/recompiler/frontend/translate/Convert.cpp

@@ -49,18 +49,11 @@ void Translator::V_CVT_F16_16(const Decoder::Instruction& inst, bool signed_valu
 }
 
 void Translator::V_CVT_16_F16(const Decoder::Instruction& inst, bool signed_value) {
-	const auto value = ReadF16AsF32(inst.src0);
 	// The 32-bit conversion already saturates and maps NaN to zero.
-	if (signed_value) {
-		const auto converted = ir.Emit(IR::ValueOpcode::ConvertS32F32, {value});
-		const auto low = ir.Emit(IR::ValueOpcode::SMax32, {converted, IR::Value(0xffff8000u)});
-		Write16Bits(DestinationOperand(inst),
-		            IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {low, IR::Value(0x7fffu)})));
-		return;
-	}
-	const auto converted = ir.Emit(IR::ValueOpcode::ConvertU32F32, {value});
-	Write16Bits(DestinationOperand(inst),
-	            IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {converted, IR::Value(0xffffu)})));
+	const auto converted = IR::U32(
+	    ir.Emit(signed_value ? IR::ValueOpcode::ConvertS32F32 : IR::ValueOpcode::ConvertU32F32,
+	            {ReadF16AsF32(inst.src0)}));
+	Write16Bits(DestinationOperand(inst), Saturate16(converted, signed_value));
 }
 
 void Translator::V_CVT_RPI_I32_F32(const Decoder::Instruction& inst) {
@@ -130,6 +123,12 @@ void Translator::V_CVT_PK_U8_F32(const Decoder::Instruction& inst) {
 	                     {ReadU32(SourceAt(inst, 2)), byte, shift, IR::Value(8u)}));
 }
 
+void Translator::V_CVT_PK_16_32(const Decoder::Instruction& inst, bool signed_value) {
+	WriteOperand(DestinationOperand(inst),
+	             PackU16Lanes(Saturate16(ReadU32(inst.src0), signed_value),
+	                          Saturate16(ReadU32(inst.src1), signed_value)));
+}
+
 void Translator::V_PACK_B32_F16(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst),
 	             PackU16Lanes(Read16LaneBits(inst.src0, false), Read16LaneBits(inst.src1, false)));
@@ -140,4 +139,12 @@ IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
 	    ir.Emit(IR::ValueOpcode::BitFieldInsert, {low, high, IR::Value(16u), IR::Value(16u)}));
 }
 
+IR::U32 Translator::Saturate16(IR::U32 value, bool signed_value) {
+	if (!signed_value) {
+		return IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {value, IR::Value(0xffffu)}));
+	}
+	const auto low = ir.Emit(IR::ValueOpcode::SMax32, {value, IR::Value(0xffff8000u)});
+	return IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {low, IR::Value(0x7fffu)}));
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -1,6 +1,7 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
 #include <array>
+#include <bit>
 #include <utility>
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
@@ -118,6 +119,33 @@ void Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode op
 	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {ReadOperand(inst.src0, type)}));
 }
 
+void Translator::V_FRACT(const Decoder::Instruction& inst, IR::Type type) {
+	// x - floor(x) rounds to 1.0 for a tiny negative x; v_fract returns the largest value below
+	// one instead. The ordered compare lets NaN through.
+	const auto destination = DestinationOperand(inst);
+	if (type == IR::Type::F64) {
+		const auto result    = ir.Emit(IR::ValueOpcode::FPFract64, {ReadOperand(inst.src0, type)});
+		const auto below_one = ir.Emit(IR::ValueOpcode::BitCastF64U64,
+		                               {IR::Value(uint64_t {0x3fefffffffffffff})});
+		const auto above     = ir.Emit(IR::ValueOpcode::FPOrdGreaterThan64, {result, below_one});
+		WriteOperand(destination, ir.Emit(IR::ValueOpcode::SelectF64, {above, below_one, result}));
+		return;
+	}
+	const bool half   = type == IR::Type::F16;
+	const auto source = half ? ReadF16AsF32(inst.src0) : IR::F32(ReadOperand(inst.src0, type));
+	const auto result = IR::F32(ir.Emit(IR::ValueOpcode::FPFract32, {source}));
+	// Clamp in the destination precision: 0x3f7fe000 is the half 0x3bff.
+	const auto below_one =
+	    IR::F32(IR::Value::F32(std::bit_cast<float>(half ? 0x3f7fe000u : 0x3f7fffffu)));
+	const auto above   = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThan32, {result, below_one}));
+	const auto clamped = SelectF32(above, below_one, result);
+	if (half) {
+		WriteF16(destination, clamped);
+	} else {
+		WriteOperand(destination, clamped);
+	}
+}
+
 void Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                              bool reverse) {
 	std::array<IR::Value, 2> args;

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -25,8 +25,7 @@ void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
 	const auto rhs    = ReadU16AsU32(inst.src1, true);
 	auto       result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, true));
 	if (inst.dst.clamp) {
-		result = IR::U32(ir.Emit(IR::ValueOpcode::SMax32, {result, IR::Value(0xffff8000u)}));
-		result = IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {result, IR::Value(0x7fffu)}));
+		result = Saturate16(result, true);
 	}
 	Write16Bits(DestinationOperand(inst), result);
 }
@@ -221,8 +220,17 @@ void Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), result);
 }
 
-void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb) {
+void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb,
+                              bool sign) {
 	auto source = ReadU32Pair(inst.src0);
+	if (sign) {
+		// A signed source sign-extends a literal; then bits equal to the sign become zero.
+		if (inst.src0.kind == Decoder::OperandKind::LiteralConstant) {
+			source[1] = ir.ShiftRightArithmetic(source[0], IR::U32(IR::Value(31u)));
+		}
+		const auto mask = ir.ShiftRightArithmetic(source[1], IR::U32(IR::Value(31u)));
+		source          = {ir.BitwiseXor(source[0], mask), ir.BitwiseXor(source[1], mask)};
+	}
 	for (auto& word: source) {
 		word = find_zero ? ir.BitwiseNot(word) : word;
 	}
@@ -240,12 +248,19 @@ void Translator::S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero,
 
 void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
 	const auto source = ReadU32(inst.src0);
-	const auto value  = sign ? ir.BitwiseXor(
-	                              source, ir.ShiftRightArithmetic(source, IR::U32(IR::Value(31u))))
-	                         : source;
-	const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {value}));
+	if (sign) {
+		// Bits equal to the sign become zero; FindILsb of the reversed bits is ~0 when none.
+		const auto bits =
+		    ir.BitwiseXor(source, ir.ShiftRightArithmetic(source, IR::U32(IR::Value(31u))));
+		const auto reversed = ir.Emit(IR::ValueOpcode::BitReverse32, {bits});
+		WriteOperand(DestinationOperand(inst),
+		             IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {reversed})));
+		return;
+	}
+	// ResourceTracking matches this 31 - FindUMsb32 form.
+	const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {source}));
 	const auto position = ir.ISub(IR::U32(IR::Value(31u)), msb);
-	const auto result   = ir.Select(ir.INotEqual(value, IR::U32(IR::Value(0u))), position,
+	const auto result   = ir.Select(ir.INotEqual(source, IR::U32(IR::Value(0u))), position,
 	                                IR::U32(IR::Value(0xffffffffu)));
 	WriteOperand(DestinationOperand(inst), result);
 }

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -1,11 +1,9 @@
 #include "common/assert.h"
-#include "common/logging/log.h"
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
 
 #include <algorithm>
 #include <array>
-#include <atomic>
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
@@ -969,17 +967,9 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4:
 		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8:
 		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return S_LOAD(inst, false);
-		case Decoder::Opcode::S_MEMREALTIME: {
-			static std::atomic_flag warned = ATOMIC_FLAG_INIT;
-			if (!warned.test_and_set(std::memory_order_relaxed)) {
-				Log::WriteToConsoleAndLog(
-				    "Warning: S_MEMREALTIME uses placeholder UINT64_MAX; real-time clock not implemented.\n");
-			}
-			for (uint32_t component = 0; component < 2; component++) {
-				WriteOperand(ScalarDestinationOperand(inst.dst, component), IR::Value(UINT32_MAX));
-			}
-			return;
-		}
+		case Decoder::Opcode::S_MEMREALTIME:
+			return WriteU32Pair(inst.dst,
+			                    ExtractU64(IR::U64(ir.Emit(IR::ValueOpcode::ReadRealtimeClock))));
 
 		case Decoder::Opcode::BUFFER_LOAD_UBYTE:
 		case Decoder::Opcode::BUFFER_LOAD_UBYTE_D16:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -238,10 +238,12 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, false, true, true);
 		case O::S_XNOR_B32:
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, true);
-		case O::S_FF0_I32_B64: return S_FF_I32_B64(inst, true, false);
-		case O::S_FF1_I32_B64: return S_FF_I32_B64(inst, false, false);
+		case O::S_FF0_I32_B64: return S_FF_I32_B64(inst, true, false, false);
+		case O::S_FF1_I32_B64: return S_FF_I32_B64(inst, false, false, false);
 		case O::S_FLBIT_I32_B32: return V_FFBH_32(inst, false);
-		case O::S_FLBIT_I32_B64: return S_FF_I32_B64(inst, false, true);
+		case O::S_FLBIT_I32_B64: return S_FF_I32_B64(inst, false, true, false);
+		case O::S_FLBIT_I32: return V_FFBH_32(inst, true);
+		case O::S_FLBIT_I32_I64: return S_FF_I32_B64(inst, false, true, true);
 
 		case O::S_BITSET0_B32: return S_BITSET_B32(inst, false);
 		case O::S_BITSET1_B32: return S_BITSET_B32(inst, true);

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -114,6 +114,7 @@ private:
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
+	IR::U32 Saturate16(IR::U32 value, bool signed_value);
 	void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, bool cmpx);
 	void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, bool cmpx);
 	void EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
@@ -141,6 +142,7 @@ private:
 	void V_CVT_PKRTZ_F16_F32(const Decoder::Instruction& inst);
 	void V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void V_CVT_PK_U8_F32(const Decoder::Instruction& inst);
+	void V_CVT_PK_16_32(const Decoder::Instruction& inst, bool signed_value);
 	void V_PACK_B32_F16(const Decoder::Instruction& inst);
 	void PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
 	                   bool quiet_snan);
@@ -151,6 +153,7 @@ private:
 	void Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
 	                    bool mix);
 	void FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void V_FRACT(const Decoder::Instruction& inst, IR::Type type);
 	void FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
 	void FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
 	                  bool mix);
@@ -182,7 +185,7 @@ private:
 	void V_AND_OR_B32(const Decoder::Instruction& inst);
 	void V_OR3_B32(const Decoder::Instruction& inst);
 	void V_XOR3_B32(const Decoder::Instruction& inst);
-	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb);
+	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb, bool sign);
 	void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
 	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
 	void V_MAD_U64_U32(const Decoder::Instruction& inst);

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -495,8 +495,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 			return;
 		case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return;
 		case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return;
-		case O::V_CVT_PK_U16_U32:
-		case O::V_CVT_PK_I16_I32: return PackB16(inst, false, false);
+		case O::V_CVT_PK_U16_U32: return V_CVT_PK_16_32(inst, false);
+		case O::V_CVT_PK_I16_I32: return V_CVT_PK_16_32(inst, true);
 
 		case O::V_LSHLREV_B16:
 			return Integer16Shift(inst, IR::ValueOpcode::ShiftLeftLogical32, false);
@@ -563,7 +563,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CEIL_F16: return Float16Unary(inst, IR::ValueOpcode::FPCeil32, false);
 		case O::V_TRUNC_F16: return Float16Unary(inst, IR::ValueOpcode::FPTrunc32, false);
 		case O::V_RNDNE_F16: return Float16Unary(inst, IR::ValueOpcode::FPRoundEven32, false);
-		case O::V_FRACT_F16: return Float16Unary(inst, IR::ValueOpcode::FPFract32, false);
+		case O::V_FRACT_F16: return V_FRACT(inst, IR::Type::F16);
 		case O::V_SIN_F16: return Float16Trig(inst, IR::ValueOpcode::FPSin);
 		case O::V_COS_F16: return Float16Trig(inst, IR::ValueOpcode::FPCos);
 		case O::V_MIN3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMinTri32, false, false);
@@ -574,8 +574,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_RCP_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecip32);
 		case O::V_RCP_F64: return FloatUnary(inst, IR::ValueOpcode::FPRecip64);
 		case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32);
-		case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32);
-		case O::V_FRACT_F64: return FloatUnary(inst, IR::ValueOpcode::FPFract64);
+		case O::V_FRACT_F32: return V_FRACT(inst, IR::Type::F32);
+		case O::V_FRACT_F64: return V_FRACT(inst, IR::Type::F64);
 		case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32);
 		case O::V_TRUNC_F64: return FloatUnary(inst, IR::ValueOpcode::FPTrunc64);
 		case O::V_CEIL_F32: return FloatUnary(inst, IR::ValueOpcode::FPCeil32);

src/graphics/shader/recompiler/ir/Serialize.h

@@ -0,0 +1,181 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SERIALIZE_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SERIALIZE_H_
+
+#include "common/assert.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/rectListShader.h"
+
+#include <algorithm>
+#include <bit>
+#include <optional>
+#include <span>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+#include <vector>
+
+// Shader cache records. Io(io, value) appends value to a Writer or reads it back from a Reader
+// field by field, so padding never reaches the bytes; instructions travel as plan-local indices.
+namespace Libs::Graphics::ShaderRecompiler::IR {
+
+struct Writer {
+	std::vector<uint8_t>                      bytes;
+	std::unordered_map<const Inst*, uint32_t> index; // Instructions that values may refer to.
+
+	void Raw(const void* data, size_t size) {
+		const auto* begin = static_cast<const uint8_t*>(data);
+		bytes.insert(bytes.end(), begin, begin + size);
+	}
+};
+
+struct Reader {
+	std::span<const uint8_t> bytes;
+	std::vector<Inst*>       insts;     // Instructions by index.
+	bool                     ok = true; // False once a read ran past the end or was invalid.
+
+	void Raw(void* data, size_t size) {
+		ok = ok && size <= bytes.size();
+		if (!ok) return;
+		std::copy_n(bytes.data(), size, static_cast<uint8_t*>(data));
+		bytes = bytes.subspan(size);
+	}
+	// Every element takes at least one byte, which bounds counts read from damaged records.
+	uint32_t Count(uint32_t count) {
+		ok = ok && count <= bytes.size();
+		return ok ? count : 0u;
+	}
+};
+
+template <class S> constexpr bool Reading = std::is_same_v<S, Reader>;
+template <class T, template <class...> class Of> constexpr bool IsA = false;
+template <template <class...> class Of, class... A> constexpr bool IsA<Of<A...>, Of> = true;
+template <class T, class U> concept Is = std::is_same_v<std::remove_const_t<T>, U>;
+
+template <class S, class T> void Io(S& io, T& value);
+
+template <class S, class... T> void IoAll(S& io, T&&... values) {
+	(Io(io, values), ...);
+}
+
+// The value's Type, Opaque for an instruction, then the instruction index or the zero-extended
+// immediate.
+inline void IoValue(Writer& writer, const Value& value) {
+	auto     type = Type::Opaque;
+	uint64_t bits = 0;
+	if (const auto* inst = value.TryInstruction()) {
+		bits = writer.index.at(inst);
+	} else {
+		type = value.GetType();
+		switch (type) {
+			case Type::Void: break;
+			case Type::ScalarReg: bits = RegIndex(value.ScalarRegister()); break;
+			case Type::VectorReg: bits = RegIndex(value.VectorRegister()); break;
+			case Type::U1: bits = value.U1(); break;
+			case Type::U8: bits = value.U8(); break;
+			case Type::U16: bits = value.U16(); break;
+			case Type::U32: bits = value.U32(); break;
+			case Type::U64: bits = value.U64(); break;
+			case Type::F16: bits = value.F16Bits(); break;
+			case Type::F32: bits = std::bit_cast<uint32_t>(value.F32Value()); break;
+			default: EXIT("value type %u has no cache encoding\n", static_cast<uint32_t>(type));
+		}
+	}
+	IoAll(writer, type, bits);
+}
+
+inline void IoValue(Reader& reader, Value& value) {
+	auto     type = Type::Void;
+	uint64_t bits = 0;
+	IoAll(reader, type, bits);
+	switch (type) {
+		case Type::Void: value = {}; break;
+		case Type::Opaque:
+			reader.ok = reader.ok && bits < reader.insts.size();
+			if (reader.ok) value = Value(reader.insts[bits]);
+			break;
+		case Type::ScalarReg: value = Value(static_cast<ScalarReg>(bits)); break;
+		case Type::VectorReg: value = Value(static_cast<VectorReg>(bits)); break;
+		case Type::U1: value = Value(bits != 0u); break;
+		case Type::U8: value = Value(static_cast<uint8_t>(bits)); break;
+		case Type::U16: value = Value(static_cast<uint16_t>(bits)); break;
+		case Type::U32: value = Value(static_cast<uint32_t>(bits)); break;
+		case Type::U64: value = Value(bits); break;
+		case Type::F16: value = Value::F16(static_cast<uint16_t>(bits)); break;
+		case Type::F32: value = Value::F32(std::bit_cast<float>(static_cast<uint32_t>(bits))); break;
+		default: reader.ok = false; break;
+	}
+}
+
+// Scalars and standard containers; structs have their own overloads below.
+template <class S, class T> void Io(S& io, T& value) {
+	using U = std::remove_const_t<T>;
+	static_assert(!std::is_pointer_v<U>, "pointers do not persist");
+	if constexpr (std::is_arithmetic_v<U> || std::is_enum_v<U>) {
+		io.Raw(&value, sizeof(U));
+	} else if constexpr (std::is_same_v<U, Value>) {
+		IoValue(io, value);
+	} else if constexpr (IsA<U, std::vector> || IsA<U, std::basic_string>) {
+		auto count = static_cast<uint32_t>(value.size());
+		Io(io, count);
+		if constexpr (Reading<S>) value.resize(io.Count(count));
+		using E = typename U::value_type;
+		if constexpr (std::is_arithmetic_v<E> || std::is_enum_v<E>) {
+			io.Raw(value.data(), value.size() * sizeof(E));
+		} else {
+			for (auto& element: value) Io(io, element);
+		}
+	} else if constexpr (IsA<U, std::optional>) {
+		bool has = value.has_value();
+		Io(io, has);
+		if constexpr (Reading<S>) {
+			if (has) value = typename U::value_type {}; // Not emplace(): clang rejects nested aggregates.
+			else value.reset();
+		}
+		if (has) Io(io, *value);
+	} else {
+		static_assert(requires { std::tuple_size<U>::value; }, "struct without an Io overload");
+		std::apply([&](auto&... elements) { IoAll(io, elements...); }, value); // std::array, std::pair
+	}
+}
+
+// Binds every field, so that a field added later fails to compile until it is listed.
+#define KYTY_IO_FIELDS(Struct, ...)                                                                \
+	template <class S, Is<Struct> T> void Io(S& io, T& object) {                                   \
+		auto& [__VA_ARGS__] = object;                                                              \
+		IoAll(io, __VA_ARGS__);                                                                    \
+	}
+KYTY_IO_FIELDS(MemoryInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y)
+KYTY_IO_FIELDS(BufferResource, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s)
+KYTY_IO_FIELDS(ImageResource, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w)
+KYTY_IO_FIELDS(SamplerResource, a, b, c, d, e, f, g, h, i, j, k)
+KYTY_IO_FIELDS(SampledResourcePair, a, b, c)
+KYTY_IO_FIELDS(StageInput, a, b, c, d, e)
+KYTY_IO_FIELDS(StageOutput, a, b, c, d)
+KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
+               A, B, C, D, E, F, G, H, I, J)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::PointerTable, a, b, c, d)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)
+KYTY_IO_FIELDS(DescriptorSource, a, b, c)
+KYTY_IO_FIELDS(SrtRead, a, b)
+KYTY_IO_FIELDS(ResourceBlock, a, b, c, d)
+KYTY_IO_FIELDS(UniformFill, a, b, c, d, e, f)
+KYTY_IO_FIELDS(UniformFillPlan, a, b, c, d)
+KYTY_IO_FIELDS(ResourceSpecialization::Buffer, a, b, c, d, e, f, g, h)
+KYTY_IO_FIELDS(ResourceSpecialization::Image, a, b, c, d, e, f, g, h, i, j, k)
+KYTY_IO_FIELDS(ResourceSpecialization::Sampler, a, b, c)
+KYTY_IO_FIELDS(ResourceSpecialization, a, b, c)
+KYTY_IO_FIELDS(Bindings, a, b, c, d, e)
+KYTY_IO_FIELDS(CompiledShaderInfo, a, b, c, d, e, f, g, h, i, j)
+KYTY_IO_FIELDS(Spirv::Profile, a, b, c)
+KYTY_IO_FIELDS(ShaderTessellationInputInfo, a, b, c, d, e, f, g, h, i)
+KYTY_IO_FIELDS(RectListParameter, a, b, c)
+#undef KYTY_IO_FIELDS
+
+} // namespace Libs::Graphics::ShaderRecompiler::IR
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SERIALIZE_H_ */

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -14,7 +14,6 @@
 
 #include <array>
 #include <bit>
-#include <deque>
 #include <list>
 #include <memory>
 #include <optional>
@@ -28,8 +27,6 @@ class CodeGenerator;
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-class SrtWalker;
-
 // Finite descriptor domains also bound the arithmetic used to prove their indices.
 constexpr uint32_t MaxIndirectDescriptorProbes = 65536u;
 
@@ -658,24 +655,10 @@ struct UniformFillPlan {
 // condition and fill values without translated blocks, plus reusable evaluation scratch.
 struct ResourcePlan {
 	struct SelectorKeys {
-		uint32_t              source = UINT32_MAX;
 		std::vector<uint32_t> values;
 		bool                  resolved = false;
 	};
 
-	struct EvaluationContext {
-		struct Entry {
-			uint64_t value      = 0;
-			uint64_t generation = 0;
-		};
-
-		std::vector<Entry> values;
-		uint64_t           generation = 0;
-		const SrtWalker*    session = nullptr;
-		Value              active_mask;
-		uint8_t            read_policy = 0;
-	};
-
 	ResourcePlan();
 	~ResourcePlan();
 
@@ -695,21 +678,18 @@ struct ResourcePlan {
 	std::vector<ResourceBlock>          control_flow;
 	std::unordered_map<const Inst*, std::vector<uint32_t>> control_phi_predecessors;
 	std::vector<SrtRead>                srt_reads;
-	bool                                requires_specialization_memory = false;
 	std::vector<std::pair<uint64_t, uint64_t>> source_reads;
 	bool                                capture_specialization_reads = false;
 	bool                                srt_plan_complete          = false;
 	bool                                resource_tracking_complete = false;
 	ShaderInfo                          info;
 	UniformFillPlan                     uniform_fill;
+	// x86 walker over the retained SRT, descriptor and condition values.
 	std::unique_ptr<Xbyak::CodeGenerator> srt_code;
-	// GPU-thread scratch for nested clean/EXEC memos, activity and material keys.
-	mutable std::deque<EvaluationContext> evaluation_contexts;
-	mutable uint32_t                       evaluation_value_count = 0;
-	mutable std::vector<uint8_t>            active_sources;
-	mutable std::vector<uint8_t>            visited_blocks;
-	mutable std::vector<uint8_t>            reachable_blocks;
-	mutable std::vector<uint32_t>           pending_blocks;
+	uint32_t                              srt_user_data = 0; // User-data DWORDs the walker reads.
+	uint32_t                              srt_captures  = 0; // Scratch offset of loaded addresses.
+	// GPU-thread scratch for walker results and material keys.
+	mutable std::vector<uint64_t>           srt_scratch;
 	mutable std::vector<uint32_t>           material_keys;
 	mutable std::vector<SelectorKeys>       selector_keys;
 };

src/graphics/shader/recompiler/ir/Value.h

@@ -123,13 +123,9 @@ public:
 	[[nodiscard]] Block*                  PhiBlock(size_t index) const;
 	[[nodiscard]] Block*                  Parent() const;
 	[[nodiscard]] const std::vector<Use>& Uses() const;
-	// Runtime indices belong to the resource plan that owns this instruction.
-	[[nodiscard]] uint32_t EvaluationIndex(uint32_t& count) const {
-		if (evaluation_index == UINT32_MAX) {
-			evaluation_index = count++;
-		}
-		return evaluation_index;
-	}
+	// SRT walker output index, assigned by the resource plan that owns this instruction.
+	[[nodiscard]] uint32_t WalkOutput() const { return walk_output; }
+	void SetWalkOutput(uint32_t index) const { walk_output = index; }
 	// Backend definitions belong to one emission. Wave64 emulation needs one per half.
 	[[nodiscard]] uint32_t Definition(uint32_t half) const { return definitions[half]; }
 	void SetDefinition(uint32_t half, uint32_t definition) const {
@@ -171,7 +167,7 @@ private:
 	ValueOpcode         opcode;
 	uint8_t             num_args;
 	bool                live = false;
-	mutable uint32_t    evaluation_index = UINT32_MAX;
+	mutable uint32_t    walk_output = UINT32_MAX;
 	mutable std::array<uint32_t, 2> definitions {};
 	uint64_t            flags;
 	Block*              parent = nullptr;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -10,6 +10,7 @@ VALUE_OPCODE(ConditionRef, U1, U1)
 VALUE_OPCODE(GetUserData, U32, ScalarReg)
 VALUE_OPCODE(GetDispatchThreadExtent, U32, U32)
 VALUE_OPCODE(GetShaderBase, U64)
+VALUE_OPCODE(ReadRealtimeClock, U64)
 VALUE_OPCODE(MeshDrawParameter, U32, U32)
 VALUE_OPCODE(MeshAllocate, Void, U32)
 VALUE_OPCODE(TessellationBase, U32, U32)
@@ -95,6 +96,7 @@ VALUE_OPCODE(WqmU64, U64, U64)
 // Core value operations used to expand architectural side effects.
 VALUE_OPCODE(SelectU1, U1, U1, U1, U1)
 VALUE_OPCODE(SelectF32, F32, U1, F32, F32)
+VALUE_OPCODE(SelectF64, F64, U1, F64, F64)
 VALUE_OPCODE(IAdd32, U32, U32, U32)
 VALUE_OPCODE(IAdd64, U64, U64, U64)
 VALUE_OPCODE(ISub32, U32, U32, U32)
@@ -221,7 +223,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
 VALUE_OPCODE(ReadLane, U32, U32, U32)
 VALUE_OPCODE(SubgroupUMinU32, U32, U32)
 VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
-VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1)
+VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1, U32)
 VALUE_OPCODE(PermuteU32, U32, U32, U32, U1)
 VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1)
 

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -3,6 +3,7 @@
 #include "common/assert.h"
 #include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
+#include "graphics/shader/recompiler/ir/Serialize.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shaderBindings.h"
 
@@ -186,35 +187,6 @@ const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
 	return &program.descriptor_sources[source];
 }
 
-void MarkCleanReads(const std::unordered_set<const Inst*>& planned_reads,
-                    const DescriptorSource* source, Value extra = {}) {
-	if (source == nullptr && extra.IsEmpty()) {
-		return;
-	}
-	std::vector<Value> pending;
-	if (source != nullptr) {
-		pending.assign(source->dwords.begin(), source->dwords.begin() + source->dword_count);
-	}
-	if (!extra.IsEmpty()) pending.push_back(extra);
-	std::vector<const Inst*> visited;
-	while (!pending.empty()) {
-		auto* inst = pending.back().Resolve().TryInstruction();
-		pending.pop_back();
-		if (inst == nullptr || std::ranges::find(visited, inst) != visited.end()) {
-			continue;
-		}
-		visited.push_back(inst);
-		if (planned_reads.contains(inst)) {
-			auto flags = inst->Flags<SrtReadFlags>();
-			flags.clean = 1u;
-			inst->SetFlags(flags);
-		}
-		for (size_t arg = 0; arg < inst->NumArgs(); arg++) {
-			pending.push_back(inst->Arg(arg));
-		}
-	}
-}
-
 bool ReadTable(uint64_t base, uint64_t size, uint64_t offset,
                const SrtRuntime& runtime, std::span<uint32_t> words) {
 	const auto count = std::min<uint64_t>(words.size(), offset < size ? (size - offset) / 4u : 0u);
@@ -309,22 +281,22 @@ bool CanReadIndexedSelector(const ShaderBufferResource& material, uint32_t offse
 	           uint64_t {UINT32_MAX} + 1u;
 }
 
-bool EvaluateIndirectCount(const DescriptorSource::IndirectDescriptor& indirect,
-                           const SrtRuntime& runtime, SrtWalker& clean, uint32_t& count) {
+bool EvaluateIndirectCount(const ResourcePlan& program, const DescriptorSource::IndirectDescriptor& indirect,
+                           const SrtRuntime& runtime, uint32_t& count) {
 	if (indirect.workgroup_axis != UINT32_MAX) {
 		if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false;
 		count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end()
 		    ? 0u : runtime.workgroup_counts[indirect.workgroup_axis];
 		return true;
 	}
-	if (!clean.Evaluate(indirect.key_count, count)) return false;
+	if (!WalkedValue(program, indirect.key_count, count)) return false;
 	if (indirect.signed_count && std::bit_cast<int32_t>(count) < 0) count = 0;
 	return true;
 }
 
-bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
+bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::IndirectDescriptor& indirect,
                          const DescriptorValue& material_value, const SrtRuntime& runtime,
-                         SrtWalker& clean, std::vector<uint32_t>& keys) {
+                         std::vector<uint32_t>& keys) {
 	const auto* selector = &*indirect.selector;
 	keys.clear();
 	const auto read_selector = [&](uint64_t base, uint64_t size, uint64_t offset,
@@ -363,7 +335,7 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
 		const auto& rows = *selector->rows;
 		DescriptorValue count_value;
 		ShaderBufferResource counts, material;
-		if (!clean.EvaluateDescriptor(rows.source, count_value) ||
+		if (!WalkedDescriptor(program, rows.source, count_value) ||
 		    !DecodeBufferDescriptor(count_value, counts) ||
 		    !DecodeBufferDescriptor(material_value, material) || selector->stride != 4u ||
 		    selector->offset != 0u || rows.stride == 0u || (rows.stride & 3u) != 0u ||
@@ -392,7 +364,7 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
 				if (candidate_end - first > MaxIndirectDescriptorProbes) break;
 				end = candidate_end;
 			}
-			// Coalesce small gaps, but interpret only selected prefixes. The strict
+			// Coalesce small gaps, but interpret only selected prefixes. The specialization
 			// reader captures the entire fetched span for writable-alias rejection.
 			batch.resize(end - first);
 			if (!ReadScalarTable(material.Base48(), material.GetSize(), first * 4u, runtime, batch))
@@ -408,8 +380,8 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
 		std::array<uint32_t, 4> masks {};
 		uint32_t first = 0, count = 0;
 		if (!DecodeBufferDescriptor(material_value, material) ||
-		    !clean.Evaluate(indirect.selector_first, first) ||
-		    !EvaluateIndirectCount(indirect, runtime, clean, count) || count != masks.size() * 32u ||
+		    !WalkedValue(program, indirect.selector_first, first) ||
+		    !EvaluateIndirectCount(program, indirect, runtime, count) || count != masks.size() * 32u ||
 		    selector->stride != sizeof(masks) || selector->offset != 0u ||
 		    material.Type() != 0u || material.Stride() != sizeof(masks) ||
 		    material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
@@ -462,8 +434,8 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
 			base = material.Base48();
 			size = material.GetSize();
 		}
-		if (!clean.Evaluate(indirect.selector_first, first) ||
-		    !EvaluateIndirectCount(indirect, runtime, clean, count) || count > MaxIndirectDescriptorProbes)
+		if (!WalkedValue(program, indirect.selector_first, first) ||
+		    !EvaluateIndirectCount(program, indirect, runtime, count) || count > MaxIndirectDescriptorProbes)
 			return false;
 		// SMEM uses its instruction's byte stride; V# stride only supplies its bounds.
 		const auto step = selector->stride != 0u ? selector->stride : material.Stride();
@@ -483,8 +455,8 @@ bool ResolveSelectorKeys(const DescriptorSource::IndirectDescriptor& indirect,
 		uint32_t mask = 0;
 		uint32_t count = 0;
 		if (material_value.dword_count != 2u ||
-		    !clean.Evaluate(indirect.selector_mask, mask) ||
-		    !EvaluateIndirectCount(indirect, runtime, clean, count) || count == 0u || count > 32u) {
+		    !WalkedValue(program, indirect.selector_mask, mask) ||
+		    !EvaluateIndirectCount(program, indirect, runtime, count) || count == 0u || count > 32u) {
 			return false;
 		}
 		if (count < 32u) mask &= (1u << count) - 1u;
@@ -541,8 +513,7 @@ template <typename Specialization, typename Normalize>
 bool MaterializeIndirectDescriptor(const ResourcePlan&                         program,
                                    uint32_t source_index,
                                    uint32_t resource_index, uint32_t dword_count, uint32_t read_dwords,
-                                   const SrtRuntime& runtime, SrtWalker& clean,
-                                   ResourceSnapshot&             snapshot,
+                                   const SrtRuntime& runtime, ResourceSnapshot& snapshot,
                                    std::vector<DescriptorValue>& descriptors,
                                    std::vector<Specialization>&  specializations,
                                    Normalize&& normalize) {
@@ -605,8 +576,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 		if (selector == nullptr) keys.clear();
 		DescriptorValue material_value;
 		if ((selector != nullptr && (byte_keys || !selector_keys->resolved) &&
-		     !clean.EvaluateDescriptor(selector->source, material_value)) ||
-		    !clean.EvaluateDescriptor(indirect.table_source, table_value)) {
+		     !WalkedDescriptor(program, selector->source, material_value)) ||
+		    !WalkedDescriptor(program, indirect.table_source, table_value)) {
 			return false;
 		}
 		if (table_value.dword_count == 2u) {
@@ -659,7 +630,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 			                                       : uint64_t {1} << 32u;
 			if (!indirect.key_count.IsEmpty()) {
 				uint32_t key_count = 0u;
-				if (!EvaluateIndirectCount(indirect, runtime, clean, key_count)) return false;
+				if (!EvaluateIndirectCount(program, indirect, runtime, key_count)) return false;
 				domain = std::min(domain, uint64_t {key_count});
 			}
 			if (indexed_pointer &&
@@ -771,8 +742,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 		if (selector == nullptr) {
 			uint32_t key_count = 0;
 			uint32_t first_key = 0;
-			if (!EvaluateIndirectCount(indirect, runtime, clean, key_count)) return false;
-			if (!indirect.selector_first.IsEmpty() && !clean.Evaluate(indirect.selector_first, first_key))
+			if (!EvaluateIndirectCount(program, indirect, runtime, key_count)) return false;
+			if (!indirect.selector_first.IsEmpty() && !WalkedValue(program, indirect.selector_first, first_key))
 				return false;
 			if (indirect.table_stride == 0u || key_count > MaxIndirectDescriptorProbes ||
 			    (indirect.selector_first.IsEmpty() && key_count != 0u &&
@@ -788,7 +759,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 		} else {
 			if (!indirect.selector_mask.IsEmpty() && table_value.dword_count != 2u) return false;
 			if (!selector_keys->resolved) {
-				if (!ResolveSelectorKeys(indirect, material_value, runtime, clean, keys)) return false;
+				if (!ResolveSelectorKeys(program, indirect, material_value, runtime, keys)) return false;
 				selector_keys->resolved = true;
 			}
 		}
@@ -827,7 +798,7 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 			                     std::span(candidate.dwords).first(read_dwords))) {
 				return read_failed("table descriptor read", key, 0u, table_base, table_offset, descriptor_bytes);
 			}
-		} else if (!clean.EvaluateDescriptor(sources[entry], candidate)) {
+		} else if (!WalkedDescriptor(program, sources[entry], candidate)) {
 			return false;
 		}
 		if (!normalize(candidate))
@@ -1341,6 +1312,17 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 	return result;
 }
 
+// State an extracted or loaded plan derives from its fields: one key set per selector group, which
+// extraction numbers densely in order of first use (false when they are not), and the walker.
+static bool DerivePlanState(ResourcePlan& plan) {
+	for (const auto group: plan.selector_groups) {
+		if (group == plan.selector_keys.size()) plan.selector_keys.emplace_back();
+		if (group != UINT32_MAX && group >= plan.selector_keys.size()) return false;
+	}
+	if (plan.srt_plan_complete && plan.resource_tracking_complete) CompileSrtWalker(plan);
+	return true;
+}
+
 ResourcePlan ExtractResourcePlan(const Program& program) {
 	ResourcePlan plan;
 	plan.stage                      = program.stage;
@@ -1380,16 +1362,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		if (const auto found = cloned.find(source); found != cloned.end()) {
 			return Value(found->second);
 		}
-		if (source->GetOpcode() == ValueOpcode::LoadBufferU32) {
-			plan.requires_specialization_memory = true;
-			plan.capture_specialization_reads   = true;
-		}
+		if (source->GetOpcode() == ValueOpcode::LoadBufferU32) plan.capture_specialization_reads = true;
 		auto& target =
 		    plan.value_storage.emplace_back(source->GetOpcode(), source->Flags<uint64_t>());
 		if (source->GetOpcode() == ValueOpcode::LoadAddressU32 ||
 		    source->GetOpcode() == ValueOpcode::ReadConstBuffer ||
 		    source->GetOpcode() == ValueOpcode::LoadBufferU32) {
-			target.SetFlags(SrtReadFlags {.index = source->Flags<MemoryFlags>().index});
+			target.SetFlags(MemoryFlags {.index = source->Flags<MemoryFlags>().index});
 		}
 		cloned.emplace(source, &target);
 		if (source->GetOpcode() == ValueOpcode::Phi) {
@@ -1411,7 +1390,6 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		target.dword_count    = source.dword_count;
 		target.indirect_descriptor = source.indirect_descriptor;
 		if (target.indirect_descriptor.has_value()) {
-			plan.requires_specialization_memory = true;
 			capture_indirect_reads = true;
 			target.indirect_descriptor->key_count = Clone(target.indirect_descriptor->key_count);
 			target.indirect_descriptor->selector_first =
@@ -1425,17 +1403,18 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	}
 	// Destination tables share keys when their complete selector domains agree.
 	// Resolve those domains lazily at dispatch time, without repeated guest reads.
+	std::vector<uint32_t> group_sources; // The first source of each group.
 	plan.selector_groups.resize(plan.descriptor_sources.size(), UINT32_MAX);
 	for (uint32_t source = 0; source < plan.descriptor_sources.size(); ++source) {
 		const auto& descriptor = program.descriptor_sources[source].indirect_descriptor;
 		if (!descriptor || !descriptor->selector) continue;
 		const auto& a = *descriptor;
 		uint32_t group = 0;
-		for (; group < plan.selector_keys.size(); ++group) {
-			const auto& b = *program.descriptor_sources[plan.selector_keys[group].source].indirect_descriptor;
+		for (; group < group_sources.size(); ++group) {
+			const auto& b = *program.descriptor_sources[group_sources[group]].indirect_descriptor;
 			if (EquivalentSelectorDomain(program, a, b)) break;
 		}
-		if (group == plan.selector_keys.size()) plan.selector_keys.push_back({.source = source});
+		if (group == group_sources.size()) group_sources.push_back(source);
 		plan.selector_groups[source] = group;
 	}
 	plan.srt_reads.reserve(program.srt_reads.size());
@@ -1523,57 +1502,16 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	plan.uniform_fill.element_count = Clone(plan.uniform_fill.element_count);
 	plan.capture_specialization_reads |=
 	    capture_indirect_reads || capture_control_reads || !plan.source_reads.empty();
-	if (plan.capture_specialization_reads) {
-		// Every former flat alias now targets its retained raw read. Mark those reads
-		// on the normalized graph, so discarded Phi/EXEC dependencies stay discarded.
-		std::unordered_set<const Inst*> planned_reads;
-		for (const auto& read: plan.srt_reads) {
-			auto* inst = read.value.ResolveInstruction();
-			EXIT_IF(inst == nullptr || (inst->GetOpcode() != ValueOpcode::LoadAddressU32 &&
-			                           inst->GetOpcode() != ValueOpcode::ReadConstBuffer));
-			planned_reads.insert(inst);
-		}
-		for (const auto& source: plan.descriptor_sources) {
-			if (!source.indirect_descriptor.has_value()) continue;
-			const auto& indirect = *source.indirect_descriptor;
-			if (indirect.selector) {
-				MarkCleanReads(planned_reads, Source(plan, indirect.selector->source));
-				if (indirect.selector->rows)
-					MarkCleanReads(planned_reads, Source(plan, indirect.selector->rows->source));
-			}
-			MarkCleanReads(planned_reads, nullptr, indirect.selector_mask);
-			MarkCleanReads(planned_reads, nullptr, indirect.selector_first);
-			MarkCleanReads(planned_reads, nullptr, indirect.key_count);
-			if (indirect.sources.empty()) {
-				MarkCleanReads(planned_reads, Source(plan, indirect.table_source));
-			} else {
-				for (const auto candidate: indirect.sources) {
-					MarkCleanReads(planned_reads, Source(plan, candidate));
-				}
-			}
-		}
-		// Clean writable addresses prove that shader writes cannot overlap captured reads.
-		for (const auto& buffer: plan.info.buffers) {
-			if (buffer.written) MarkCleanReads(planned_reads, Source(plan, buffer.source));
-		}
-		for (const auto& image: plan.info.images) {
-			if (image.written) MarkCleanReads(planned_reads, Source(plan, image.source));
-		}
-	}
 	if (capture_indirect_reads || !plan.source_reads.empty())
 		plan.resource_tracking_complete &= !program.has_address_writes;
-	CompileSrtWalker(plan);
+	DerivePlanState(plan);
 	return plan;
 }
 
 bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
                           ResourceSnapshot& snapshot, ResourceSpecialization& specialization) {
 	for (auto& keys: program.selector_keys) keys.resolved = false;
-	if (!program.resource_tracking_complete ||
-	    ((program.requires_specialization_memory || !program.source_reads.empty()) &&
-	     runtime.read_specialization_memory == nullptr)) {
-		return false;
-	}
+	if (!program.resource_tracking_complete) return false;
 	const auto uses_workgroup_count = [](const DescriptorSource& source) {
 		return source.indirect_descriptor && source.indirect_descriptor->workgroup_axis != UINT32_MAX;
 	};
@@ -1582,20 +1520,16 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	     std::ranges::any_of(program.descriptor_sources, uses_workgroup_count));
 	auto& reads = snapshot.specialization_reads;
 	reads = program.source_reads;
-	SrtReadCapture capture {runtime, reads};
-	auto observed = capture_reads ? capture.ObservedRuntime() : runtime;
-	SrtWalker clean(program, CleanRuntime(observed));
-	SrtWalker walker(program, observed,
-	                 capture_reads || program.requires_specialization_memory ? &clean : nullptr);
-	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
+	if (!WalkSrt(program, runtime, snapshot.flattened_srt, capture_reads ? &reads : nullptr)) {
 		return SpecializationFail(fmt::format("SRT refresh complete={} reads={} user_data={}",
 		    program.srt_plan_complete, program.srt_reads.size(), runtime.user_data.size()));
 	}
-	const auto active = std::span<const uint8_t>(program.active_sources);
+	SrtReadCapture capture {runtime, reads};
+	auto observed = capture_reads ? capture.ObservedRuntime() : runtime;
 	std::array<uint32_t, 3> workgroup_counts {};
 	if (runtime.indirect_args_address != 0u) {
 		for (uint32_t source = 0; source < program.descriptor_sources.size(); ++source) {
-			if ((!active.empty() && !active[source]) ||
+			if (!WalkedActive(program, source) ||
 			    !uses_workgroup_count(program.descriptor_sources[source])) continue;
 			if ((runtime.indirect_args_address & 3u) != 0u ||
 			    !ReadTable(runtime.indirect_args_address, sizeof(workgroup_counts), 0u,
@@ -1610,24 +1544,24 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	std::array<uint32_t, 4> stored {};
 	bool uniform_fill = words != 0;
 	if (uniform_fill && !fill.pattern_size.IsEmpty())
-		uniform_fill = clean.Evaluate(fill.pattern_size, words) && words >= 1 && words <= 4;
+		uniform_fill = WalkedValue(program, fill.pattern_size, words) && words >= 1 && words <= 4;
 	uint32_t elements = 0;
 	if (uniform_fill && !fill.element_count.IsEmpty())
-		uniform_fill = clean.Evaluate(fill.element_count, elements) && elements != 0;
+		uniform_fill = WalkedValue(program, fill.element_count, elements) && elements != 0;
 	for (uint32_t i = 0; i < words && uniform_fill; ++i) {
-		uniform_fill = clean.Evaluate(fill.values[i], stored[i]) && stored[i] == stored[0];
+		uniform_fill = WalkedValue(program, fill.values[i], stored[i]) && stored[i] == stored[0];
 	}
 	if (uniform_fill) {
 		snapshot.uniform_fill = fill.fill;
 		snapshot.uniform_fill.value = stored[0];
 		snapshot.uniform_fill.bounded_elements = elements;
 	}
-	const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
+	const auto evaluate = [&](uint32_t source, DescriptorValue& value) {
 		if (source >= program.descriptor_sources.size()) {
 			return false;
 		}
-		if (active.empty() || active[source]) {
-			return (capture_reads && written ? clean : walker).EvaluateDescriptor(source, value) ||
+		if (WalkedActive(program, source)) {
+			return WalkedDescriptor(program, source, value) ||
 			    SpecializationFail(fmt::format("descriptor evaluation source={}", source));
 		}
 		value = {};
@@ -1651,9 +1585,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			if (source == nullptr) return false;
 			if (source->indirect_descriptor.has_value()) {
 				snapshot.buffers[i] = {.dword_count = 4u};
-				if (active.empty() || active[base.source]) {
+				if (WalkedActive(program, base.source)) {
 					if (!MaterializeIndirectDescriptor(
-					        program, base.source, i, 4u, 4u, observed, clean, snapshot,
+					        program, base.source, i, 4u, 4u, observed, snapshot,
 					        snapshot.buffers, specialization.buffers,
 					        [&](DescriptorValue& value) {
 						        return NormalizeIndirectBuffer(
@@ -1661,7 +1595,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 					        }))
 						return resource_failed("buffer", i, base.source, snapshot.buffers.size());
 				}
-			} else if (!evaluate(base.source, snapshot.buffers[i], base.written)) {
+			} else if (!evaluate(base.source, snapshot.buffers[i])) {
 				return false;
 			}
 		}
@@ -1711,13 +1645,13 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		}
 		if (source->indirect_descriptor.has_value()) {
 			snapshot.images[i] = {.dword_count = 8u};
-			if (!active.empty() && !active[image.source]) {
+			if (!WalkedActive(program, image.source)) {
 				continue;
 			}
 			const auto& indirect = *source->indirect_descriptor;
 			const bool bounded_table = IsBoundedDescriptorTable(program, indirect);
 			if (!MaterializeIndirectDescriptor(
-			        program, image.source, i, 8u, image.r128 ? 4u : 8u, observed, clean, snapshot, snapshot.images,
+			        program, image.source, i, 8u, image.r128 ? 4u : 8u, observed, snapshot, snapshot.images,
 			        specialization.images, [&](DescriptorValue& value) {
 				        // A broad heap also contains resources for other typed image operations.
 				        if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128) ||
@@ -1728,7 +1662,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 				return resource_failed("image", i, image.source, snapshot.images.size());
 			}
 		} else {
-			if (!evaluate(image.source, snapshot.images[i], image.written)) {
+			if (!evaluate(image.source, snapshot.images[i])) {
 				return false;
 			}
 			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
@@ -1749,8 +1683,8 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			const auto* indirect = descriptor->indirect_descriptor ? &*descriptor->indirect_descriptor : nullptr;
 			if (indirect != nullptr && indirect->sources.empty()) {
 				snapshot.samplers[i] = {.dword_count = 4u};
-				if ((active.empty() || active[source]) && !MaterializeIndirectDescriptor(
-				        program, source, i, 4u, 4u, observed, clean, snapshot, snapshot.samplers,
+				if (WalkedActive(program, source) && !MaterializeIndirectDescriptor(
+				        program, source, i, 4u, 4u, observed, snapshot, snapshot.samplers,
 				        specialization.samplers,
 				        [](DescriptorValue&) { return true; }))
 					return resource_failed("sampler", i, source, snapshot.samplers.size());
@@ -2065,4 +1999,86 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 	image_remap.Apply(images);
 }
 
+namespace {
+
+using ControlPhis = decltype(ResourcePlan::control_phi_predecessors);
+
+// Instructions come first, so that any value can refer to them by index, then their arguments.
+// Plan Phis have no blocks; a control Phi lists the plan blocks of its arguments after them.
+void IoValues(Writer& writer, const std::list<Inst>& values, const ControlPhis& phis) {
+	IoAll(writer, static_cast<uint32_t>(values.size()));
+	for (const auto& inst: values) {
+		writer.index.emplace(&inst, static_cast<uint32_t>(writer.index.size()));
+		IoAll(writer, inst.GetOpcode(), inst.Flags<uint64_t>());
+	}
+	for (const auto& inst: values) {
+		if (inst.GetOpcode() != ValueOpcode::Phi) {
+			for (size_t index = 0; index < inst.NumArgs(); ++index) IoAll(writer, inst.Arg(index));
+			continue;
+		}
+		std::vector<Value> args;
+		for (size_t index = 0; index < inst.NumArgs(); ++index) {
+			EXIT_IF(inst.PhiBlock(index) != nullptr);
+			args.push_back(inst.Arg(index));
+		}
+		const auto found = phis.find(&inst);
+		IoAll(writer, args, found != phis.end() ? found->second : std::vector<uint32_t> {});
+	}
+}
+
+void IoValues(Reader& reader, std::list<Inst>& values, ControlPhis& phis) {
+	uint32_t count = 0;
+	Io(reader, count);
+	count = reader.Count(count);
+	for (uint32_t index = 0; index < count && reader.ok; ++index) {
+		ValueOpcode opcode {};
+		uint64_t    flags = 0;
+		IoAll(reader, opcode, flags);
+		reader.ok = reader.ok && opcode < ValueOpcode::Count;
+		if (reader.ok) reader.insts.push_back(&values.emplace_back(opcode, flags));
+	}
+	for (auto* inst: reader.insts) {
+		if (inst->GetOpcode() != ValueOpcode::Phi) {
+			for (size_t index = 0; index < inst->NumArgs(); ++index) {
+				Value arg;
+				Io(reader, arg);
+				inst->SetArg(index, arg);
+			}
+			continue;
+		}
+		std::vector<Value>    args;
+		std::vector<uint32_t> predecessors;
+		IoAll(reader, args, predecessors);
+		for (const auto arg: args) inst->AddPhiOperand(nullptr, arg);
+		if (!predecessors.empty()) phis.emplace(inst, std::move(predecessors));
+	}
+}
+
+// Binds every plan field: one added later fails to compile here until it is stored or derived.
+template <class S, class P> void IoPlan(S& io, P& plan) {
+	auto& [stage, shader_hash, user_data_base, user_data_count, values, memory_info, descriptor_sources,
+	       selector_groups, control_flow, control_phis, srt_reads, source_reads, capture_specialization_reads,
+	       srt_plan_complete, resource_tracking_complete, info, uniform_fill,
+	       // Derived on load.
+	       srt_code, srt_user_data, srt_captures, srt_scratch, material_keys, selector_keys] = plan;
+	IoValues(io, values, control_phis);
+	IoAll(io, stage, shader_hash, user_data_base, user_data_count, memory_info, descriptor_sources,
+	      selector_groups, control_flow, srt_reads, source_reads, capture_specialization_reads,
+	      srt_plan_complete, resource_tracking_complete, info, uniform_fill);
+}
+
+} // namespace
+
+std::vector<uint8_t> SaveResourcePlan(const ResourcePlan& plan) {
+	Writer writer;
+	IoPlan(writer, plan);
+	return std::move(writer.bytes);
+}
+
+bool LoadResourcePlan(std::span<const uint8_t> bytes, ResourcePlan& plan) {
+	Reader reader {bytes};
+	IoPlan(reader, plan);
+	return reader.ok && reader.bytes.empty() && DerivePlanState(plan);
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h

@@ -53,6 +53,11 @@ struct ResourceSpecialization {
 // its values and is independent of the translated shader CFG.
 ResourcePlan ExtractResourcePlan(const Program& program);
 
+// Shader cache record of a plan; loading it into a default-constructed plan rebuilds the walker
+// and scratch state like extraction. A failed load leaves a plan that must not be used.
+std::vector<uint8_t> SaveResourcePlan(const ResourcePlan& plan);
+bool                 LoadResourcePlan(std::span<const uint8_t> bytes, ResourcePlan& plan);
+
 // Refreshes cached resources and specialization in place. A failed refresh must not be used.
 bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
                           ResourceSnapshot& snapshot, ResourceSpecialization& specialization);

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -8,8 +8,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-class Value;
-
 using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span<uint32_t> values);
 
 // Image descriptors at base + first + k * step, k < count, within size bytes that a sampled
@@ -42,7 +40,6 @@ protected:
 struct SrtRuntime {
 	std::span<const uint32_t> user_data;
 	uint64_t                  shader_base                = 0;
-	SrtMemoryReader           read_memory                = nullptr;
 	void*                     userdata                   = nullptr;
 	SrtMemoryReader           read_specialization_memory = nullptr;
 	std::span<const uint32_t> workgroup_counts;
@@ -57,20 +54,12 @@ public:
 	SrtRuntime ObservedRuntime();
 
 private:
-	static bool ReadStrict(void* userdata, uint64_t address, std::span<uint32_t> values);
-	static bool ReadOrdinary(void* userdata, uint64_t address, std::span<uint32_t> values);
+	static bool Read(void* userdata, uint64_t address, std::span<uint32_t> values);
 
 	SrtRuntime                                  m_source;
 	std::vector<std::pair<uint64_t, uint64_t>>& m_ranges;
 };
 
-// Retained reads no longer need the guest instruction PC. A clean read evaluates
-// its address and bounds through the strict reader as well as the final DWORD.
-struct SrtReadFlags {
-	uint32_t index = 0;
-	uint32_t clean = 0;
-};
-
 enum class RuntimeValueType { Any, Integer, ImmutableInteger };
 
 bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
@@ -78,9 +67,20 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
 // Uniform across a compute wave, including values derived from its workgroup ID.
 // This does not imply that the host can evaluate the value.
 bool ValidateWaveUniformValue(const ResourcePlan& program, Value value);
-void CompileSrtWalker(ResourcePlan& program);
-// Uses the strict reader for values that affect shader specialization.
-SrtRuntime CleanRuntime(SrtRuntime runtime);
+// Compiles the plan's x86 walker. Without values it walks the plan's resources; otherwise it
+// only evaluates the values, with every block reachable.
+void CompileSrtWalker(ResourcePlan& plan, std::span<const Value> values = {});
+// Runs the walker. flat receives the scalar reads of reachable blocks and reads, when non-null,
+// one DWORD range per guest load. False when a reachable read is unknown.
+bool WalkSrt(const ResourcePlan& plan, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
+             std::vector<std::pair<uint64_t, uint64_t>>* reads);
+// Results of the last walk; false when unknown.
+bool WalkedValue(const ResourcePlan& plan, Value value, uint32_t& result);
+bool WalkedDescriptor(const ResourcePlan& plan, uint32_t source, DescriptorValue& result);
+bool WalkedActive(const ResourcePlan& plan, uint32_t source);
+// Length of the guest load at rip if it belongs to the walker running on this thread, else 0.
+// The exception handler completes such a load as a zero read.
+uint32_t SrtWalkerLoadLength(uint64_t rip);
 
 // Nonnegative affine coefficients: element zero is constant, remaining elements are
 // caller-selected leaves. Runtime range proofs must additionally exclude U32 wrap.
@@ -124,43 +124,6 @@ std::optional<std::array<uint64_t, Size>> AffineIndex(
 	return left;
 }
 
-// One memoized evaluation session shared by the entire shader resource refresh.
-class SrtWalker {
-public:
-	SrtWalker(const ResourcePlan& program, const SrtRuntime& runtime,
-	          SrtWalker* clean_evaluator = nullptr);
-	~SrtWalker();
-	SrtWalker(const SrtWalker&)            = delete;
-	SrtWalker& operator=(const SrtWalker&) = delete;
-
-	bool Evaluate(Value value, uint32_t& result);
-	bool EvaluateDescriptor(uint32_t source, DescriptorValue& result);
-	// Refreshes reachable scalar reads and active descriptor sources in one walk.
-	bool RefreshFlatBuffer(std::vector<uint32_t>& flat);
-
-private:
-	SrtWalker(SrtWalker& parent, Value active_mask, bool clean = false,
-	          SrtWalker* clean_evaluator = nullptr);
-	ResourcePlan::EvaluationContext& AcquireContext();
-	void ResetUnknowns();
-	static float Float32(uint64_t bits);
-	bool EvaluateWide(Value value, uint64_t& result);
-	bool Arg(const Inst& inst, size_t index, uint64_t& result);
-	bool EvaluatePhi(const Inst& inst, uint64_t& result);
-	bool EvaluateExtract(const Inst& inst, uint64_t& result);
-	bool EvaluateRawRead(const Inst& inst, uint64_t& result);
-	bool EvaluateInst(const Inst& inst, uint64_t& result);
-
-	const ResourcePlan&              m_program;
-	SrtRuntime                      m_runtime;
-	SrtWalker*                      m_clean_evaluator = nullptr;
-	Value                           m_active_mask;
-	const std::vector<uint8_t>*      m_reachable_blocks = nullptr;
-	const SrtWalker*                m_session;
-	uint8_t                         m_read_policy;
-	ResourcePlan::EvaluationContext& m_context;
-};
-
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ */

src/graphics/shader/rectListShader.cpp

@@ -18,45 +18,9 @@ using ShaderRecompiler::Spirv::Builder;
 
 constexpr uint32_t SpirvVersion15 = 0x00010500u;
 
-struct Parameter {
-	uint32_t input_location  = 0;
-	uint32_t output_location = 0;
-	bool     flat            = false;
-};
-
-std::vector<Parameter> GetParameters(const ShaderVertexInputInfo& vertex_info,
-                                     const ShaderPixelInputInfo*  pixel_info) {
-	if (pixel_info == nullptr) {
-		return {};
-	}
-	EXIT_IF(pixel_info->input_num > ShaderVertexInputInfo::RES_MAX);
-	EXIT_IF(pixel_info->stage.program == nullptr);
-
-	std::vector<uint32_t> active_inputs;
-	for (const auto& input: pixel_info->stage.program->info.inputs) {
-		if (input.kind == ShaderRecompiler::IR::StageInputKind::Parameter) {
-			active_inputs.push_back(input.location);
-		}
-	}
-
-	std::vector<Parameter> parameters;
-	std::array<bool, 32> output_locations {};
-	for (const auto input: active_inputs) {
-		const auto input_location = ShaderPixelParameterMappedLocation(*pixel_info, input);
-		const auto output_location = ShaderPixelParameterLocation(*pixel_info, active_inputs, input);
-		if ((vertex_info.stage.program->param_export_mask & (1u << input_location)) != 0 &&
-		    !output_locations[output_location]) {
-			parameters.push_back({input_location, output_location,
-			                      ShaderPixelParameterIsFlat(*pixel_info, input)});
-			output_locations[output_location] = true;
-		}
-	}
-	return parameters;
-}
-
 class RectListEmitter {
 public:
-	RectListEmitter(const std::vector<Parameter>& parameters_, spv::ExecutionModel model)
+	RectListEmitter(const std::vector<RectListParameter>& parameters_, spv::ExecutionModel model)
 	    : parameters(parameters_) {
 		builder.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450);
 
@@ -315,42 +279,68 @@ private:
 		              Result(spv::OpFAdd, vec4_float_type, p1, p2));
 	}
 
-	Builder                       builder {SpirvVersion15};
-	const std::vector<Parameter>& parameters;
-	std::vector<uint32_t>         interfaces;
-	std::vector<uint32_t>         inputs;
-	std::vector<uint32_t>         outputs;
-	uint32_t                      main                  = 0;
-	uint32_t                      void_type             = 0;
-	uint32_t                      bool_type             = 0;
-	uint32_t                      uint_type             = 0;
-	uint32_t                      int_type              = 0;
-	uint32_t                      float_type            = 0;
-	uint32_t                      vec2_bool_type        = 0;
-	uint32_t                      vec2_float_type       = 0;
-	uint32_t                      vec3_float_type       = 0;
-	uint32_t                      vec4_float_type       = 0;
-	uint32_t                      function_type         = 0;
-	uint32_t                      per_vertex_type       = 0;
-	uint32_t                      ptr_input_float       = 0;
-	uint32_t                      ptr_input_vec4_float  = 0;
-	uint32_t                      ptr_output_float      = 0;
-	uint32_t                      ptr_output_vec4_float = 0;
-	uint32_t                      gl_in                 = 0;
-	uint32_t                      gl_out                = 0;
-	uint32_t                      tess_inner            = 0;
-	uint32_t                      tess_outer            = 0;
-	uint32_t                      tess_coord            = 0;
-	uint32_t                      invocation_id         = 0;
+	Builder                               builder {SpirvVersion15};
+	const std::vector<RectListParameter>& parameters;
+	std::vector<uint32_t>                 interfaces;
+	std::vector<uint32_t>                 inputs;
+	std::vector<uint32_t>                 outputs;
+	uint32_t                              main                  = 0;
+	uint32_t                              void_type             = 0;
+	uint32_t                              bool_type             = 0;
+	uint32_t                              uint_type             = 0;
+	uint32_t                              int_type              = 0;
+	uint32_t                              float_type            = 0;
+	uint32_t                              vec2_bool_type        = 0;
+	uint32_t                              vec2_float_type       = 0;
+	uint32_t                              vec3_float_type       = 0;
+	uint32_t                              vec4_float_type       = 0;
+	uint32_t                              function_type         = 0;
+	uint32_t                              per_vertex_type       = 0;
+	uint32_t                              ptr_input_float       = 0;
+	uint32_t                              ptr_input_vec4_float  = 0;
+	uint32_t                              ptr_output_float      = 0;
+	uint32_t                              ptr_output_vec4_float = 0;
+	uint32_t                              gl_in                 = 0;
+	uint32_t                              gl_out                = 0;
+	uint32_t                              tess_inner            = 0;
+	uint32_t                              tess_outer            = 0;
+	uint32_t                              tess_coord            = 0;
+	uint32_t                              invocation_id         = 0;
 };
 
 } // namespace
 
-RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info,
-                                     const ShaderPixelInputInfo*  pixel_info) {
-	const auto      parameters = GetParameters(vertex_info, pixel_info);
-	RectListEmitter control(parameters, spv::ExecutionModelTessellationControl);
-	RectListEmitter evaluation(parameters, spv::ExecutionModelTessellationEvaluation);
+std::vector<RectListParameter> RectListParameters(
+    const ShaderPixelInputInfo& pixel_info, const ShaderRecompiler::IR::CompiledShaderInfo& program) {
+	EXIT_IF(pixel_info.input_num > ShaderVertexInputInfo::RES_MAX);
+	std::vector<uint32_t> active_inputs;
+	for (const auto& input: program.info.inputs) {
+		if (input.kind == ShaderRecompiler::IR::StageInputKind::Parameter) {
+			active_inputs.push_back(input.location);
+		}
+	}
+	std::vector<RectListParameter> parameters;
+	for (const auto input: active_inputs) {
+		parameters.push_back({ShaderPixelParameterMappedLocation(pixel_info, input),
+		                      ShaderPixelParameterLocation(pixel_info, active_inputs, input),
+		                      ShaderPixelParameterIsFlat(pixel_info, input)});
+	}
+	return parameters;
+}
+
+RectListShaders BuildRectListShaders(std::span<const RectListParameter> parameters,
+                                     uint32_t                           param_export_mask) {
+	std::vector<RectListParameter> exported;
+	std::array<bool, 32>           output_locations {};
+	for (const auto& parameter: parameters) {
+		if ((param_export_mask & (1u << parameter.input_location)) != 0 &&
+		    !output_locations[parameter.output_location]) {
+			exported.push_back(parameter);
+			output_locations[parameter.output_location] = true;
+		}
+	}
+	RectListEmitter control(exported, spv::ExecutionModelTessellationControl);
+	RectListEmitter evaluation(exported, spv::ExecutionModelTessellationEvaluation);
 	return {control.EmitControl(), evaluation.EmitEvaluation()};
 }
 

src/graphics/shader/rectListShader.h

@@ -2,20 +2,35 @@
 #define EMULATOR_SRC_GRAPHICS_SHADER_RECTLISTSHADER_H_
 
 #include <cstdint>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
 
 struct ShaderPixelInputInfo;
-struct ShaderVertexInputInfo;
+
+namespace ShaderRecompiler::IR {
+struct CompiledShaderInfo;
+} // namespace ShaderRecompiler::IR
+
+// A pixel parameter the generated stages forward from vertex export input_location.
+struct RectListParameter {
+	uint32_t input_location  = 0;
+	uint32_t output_location = 0;
+	bool     flat            = false;
+};
 
 struct RectListShaders {
 	std::vector<uint32_t> control;
 	std::vector<uint32_t> evaluation;
 };
 
-RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info,
-                                     const ShaderPixelInputInfo*  pixel_info);
+// Every parameter a rectangle-list pixel program reads.
+std::vector<RectListParameter> RectListParameters(
+    const ShaderPixelInputInfo& pixel_info, const ShaderRecompiler::IR::CompiledShaderInfo& program);
+// Forwards the parameters the vertex stage exports, one per output location.
+RectListShaders BuildRectListShaders(std::span<const RectListParameter> parameters,
+                                     uint32_t                           param_export_mask);
 
 } // namespace Libs::Graphics
 

src/graphics/shader/shader.cpp

@@ -11,7 +11,6 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
-#include "graphics/shader/recompiler/Tessellation.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/shaderCompiler.h"
 #include "graphics/shader/shaderVertexMetadata.h"
@@ -40,15 +39,19 @@
 
 namespace Libs::Graphics {
 
+// A mapped shader and its vertex metadata, parsed once when it is mapped.
+struct ShaderMapData: ShaderMappedData {
+	ShaderVertexMetadata vertex;
+	const char*          vertex_error = nullptr;
+};
+
 struct ShaderMapEntry {
-	ShaderMappedData data;
-	uint64_t hash;
+	ShaderMapData data;
+	uint64_t      hash;
 };
 
 static std::unique_ptr<std::unordered_map<uint64_t, ShaderMapEntry>> g_shader_map;
 static std::mutex                                                      g_shader_map_mutex;
-static std::unordered_map<uint64_t, ShaderTessellationInputInfo>       g_tess_layouts;
-static std::mutex                                                      g_tess_layouts_mutex;
 
 void ShaderInit() {
 	EXIT_IF(g_shader_map != nullptr);
@@ -64,8 +67,11 @@ void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 		     addr, data.code_size_bytes);
 	}
 	const auto hash = XXH3_64bits(reinterpret_cast<const void*>(addr), data.code_size_bytes);
+	ShaderMapData mapped {data};
+	mapped.vertex_error =
+	    ShaderReadVertexMetadata(data, HW::UserSgprInfo::SGPRS_MAX, mapped.vertex);
 	std::scoped_lock lock(g_shader_map_mutex);
-	(*g_shader_map)[addr] = {data, hash};
+	(*g_shader_map)[addr] = {mapped, hash};
 }
 
 static ShaderMapEntry ShaderGetMappedData(uint64_t addr, const char* label) {
@@ -261,8 +267,8 @@ static uint32_t ShaderCalcPsSystemInputBase(const HW::ShaderRegisters& regs) {
 
 static bool ShaderGetStaticVertexInputInfo(uint64_t shader_addr, const HW::UserSgprInfo& user_sgpr,
                                            uint32_t user_sgpr_num, const HW::ShaderRegisters& sh,
-                                           const ShaderMappedData& data,
-                                           ShaderVertexInputInfo&  info) {
+                                           const ShaderMapData&   data,
+                                           ShaderVertexInputInfo& info) {
 	KYTY_PROFILER_FUNCTION();
 
 	info = {};
@@ -277,13 +283,12 @@ static bool ShaderGetStaticVertexInputInfo(uint64_t shader_addr, const HW::UserS
 		     " user_sgpr_num=%u\n",
 		     shader_addr, user_sgpr_num);
 	}
-	ShaderVertexMetadata metadata;
-	std::string          metadata_error;
-	if (!ShaderReadVertexMetadata(data, HW::UserSgprInfo::SGPRS_MAX, metadata, &metadata_error)) {
+	if (data.vertex_error != nullptr) {
 		LOGF("ShaderGetVertexInputInfo(): invalid AGC metadata shader=0x%016" PRIx64 ": %s\n",
-		     shader_addr, metadata_error.c_str());
+		     shader_addr, data.vertex_error);
 		return false;
 	}
+	const auto& metadata = data.vertex;
 
 	if (metadata.vertex_attrib_reg >= 0) {
 		info.fetch_external   = false;
@@ -377,7 +382,6 @@ static void ShaderGetStaticInputInfoPS(
 }
 
 static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
-                                       const HW::ShaderRegisters& /*sh*/,
                                        const ShaderMappedData& data, ShaderComputeInputInfo& info) {
 	const bool async_compute             = info.async_compute;
 	const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions;
@@ -655,18 +659,6 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
 	};
 	EXIT_IF(tess.input_control_points == 0 || tess.input_control_points > 32 ||
 	        tess.output_control_points == 0 || tess.output_control_points > 32);
-	// Reflection decodes both programs; the key also covers the VGT fields cached in tess.
-	const uint64_t key[] = {local_hash, control_hash,
-	                        uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u | sh.m_vgtTfParam};
-	{
-		std::scoped_lock lock(g_tess_layouts_mutex);
-		auto [entry, inserted] = g_tess_layouts.try_emplace(XXH3_64bits(key, sizeof(key)), tess);
-		if (inserted) {
-			ShaderRecompiler::AnalyzeTessellationPrograms(params[0].code, params[1].code,
-			                                              entry->second);
-		}
-		tess = entry->second;
-	}
 	for (auto& stage: input_info) {
 		stage.tess = tess;
 	}
@@ -684,10 +676,9 @@ ShaderParams PrepareProgram(
 	    std::span<const uint32_t>(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), data);
 }
 
-ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
-                            ShaderComputeInputInfo& info) {
+ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, ShaderComputeInputInfo& info) {
 	const auto [data, hash] = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
-	ShaderGetStaticInputInfoCS(regs, sh, data, info);
+	ShaderGetStaticInputInfoCS(regs, data, info);
 	return GetShaderParams(
 	    regs.cs_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), data);

src/graphics/shader/shader.h

@@ -15,17 +15,6 @@
 
 namespace Libs::Graphics {
 
-namespace ShaderError {
-
-[[nodiscard]] inline bool Fail(std::string* error, std::string_view message) {
-	if (error != nullptr) {
-		error->assign(message.data(), message.size());
-	}
-	return false;
-}
-
-} // namespace ShaderError
-
 namespace HW {
 struct VertexShaderInfo;
 struct PixelShaderInfo;
@@ -45,16 +34,14 @@ enum class ShaderType {
 	TessellationEvaluation
 };
 
-namespace ShaderRecompiler::IR {
-struct CompiledShaderInfo;
-} // namespace ShaderRecompiler::IR
+struct ShaderModule;
 
 struct ShaderStageRuntime {
-	const ShaderRecompiler::IR::CompiledShaderInfo* program = nullptr;
-	const ShaderRecompiler::IR::ResourceSnapshot*   resources = nullptr;
+	const ShaderModule*                           module    = nullptr;
+	const ShaderRecompiler::IR::ResourceSnapshot* resources = nullptr;
 
 	[[nodiscard]] explicit operator bool() const {
-		return program != nullptr && resources != nullptr;
+		return module != nullptr && resources != nullptr;
 	}
 };
 

src/graphics/shader/shaderCompiler.h

@@ -32,6 +32,7 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& input_info, std::vector<u
 
 ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             const HW::UserConfig& user_config, ShaderVertexInputInfo& input_info);
+// Each stage's tess holds the VGT fields; the caller adds the layout reflected from the programs.
 std::array<ShaderParams, 3>
 PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             std::array<ShaderVertexInputInfo, 3>& input_info);
@@ -39,8 +40,7 @@ ShaderParams PrepareProgram(
     const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
     std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
     ShaderPixelInputInfo&                               input_info);
-ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
-                            ShaderComputeInputInfo& input_info);
+ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, ShaderComputeInputInfo& input_info);
 
 } // namespace Libs::Graphics
 

src/graphics/shader/shaderVertexMetadata.cpp

@@ -5,10 +5,10 @@
 
 namespace Libs::Graphics {
 
-bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
-                              ShaderVertexMetadata& metadata, std::string* error) {
+const char* ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
+                                     ShaderVertexMetadata& metadata) {
 	if (data.user_data == nullptr) {
-		return ShaderError::Fail(error, "missing AGC user-data header");
+		return "missing AGC user-data header";
 	}
 
 	ShaderUserData user_data {};
@@ -17,8 +17,7 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
 	constexpr uint32_t DirectResourceCount =
 	    static_cast<uint32_t>(AgcDirectResourceType::Last) + 1u;
 	if (user_data.direct_resource_count > DirectResourceCount) {
-		return ShaderError::Fail(error,
-		                         "AGC direct-resource count exceeds the known resource domain");
+		return "AGC direct-resource count exceeds the known resource domain";
 	}
 
 	std::array<uint16_t, DirectResourceCount> direct_offsets {};
@@ -26,7 +25,7 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
 	    static_cast<uint64_t>(user_data.direct_resource_count) * sizeof(uint16_t);
 	if (direct_size != 0) {
 		if (user_data.direct_resource_offset == nullptr) {
-			return ShaderError::Fail(error, "missing AGC direct-resource offsets");
+			return "missing AGC direct-resource offsets";
 		}
 		std::memcpy(direct_offsets.data(), user_data.direct_resource_offset, direct_size);
 	}
@@ -47,33 +46,33 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
 	}
 
 	if (next.vertex_attrib_reg >= 0 && next.vertex_buffer_reg < 0) {
-		return ShaderError::Fail(error, "vertex attribute table requires a vertex buffer table");
+		return "vertex attribute table requires a vertex buffer table";
 	}
 	if (next.vertex_buffer_reg < 0) {
 		metadata = next;
-		return true;
+		return nullptr;
 	}
 	if (static_cast<uint32_t>(next.vertex_buffer_reg) + 1u >= max_user_sgprs) {
-		return ShaderError::Fail(error, "vertex table pointer exceeds the user-SGPR domain");
+		return "vertex table pointer exceeds the user-SGPR domain";
 	}
 	if (next.vertex_attrib_reg < 0) {
 		metadata = next;
-		return true;
+		return nullptr;
 	}
 	if (static_cast<uint32_t>(next.vertex_attrib_reg) + 1u >= max_user_sgprs) {
-		return ShaderError::Fail(error, "vertex table pointer exceeds the user-SGPR domain");
+		return "vertex table pointer exceeds the user-SGPR domain";
 	}
 	if (data.num_input_semantics == 0 ||
 	    data.num_input_semantics > ShaderVertexInputInfo::RES_MAX) {
-		return ShaderError::Fail(error, "vertex semantic count is outside the supported domain");
+		return "vertex semantic count is outside the supported domain";
 	}
 
 	if (data.input_semantics == nullptr) {
-		return ShaderError::Fail(error, "missing vertex input semantics");
+		return "missing vertex input semantics";
 	}
 	next.input_semantics = {data.input_semantics, data.num_input_semantics};
 	metadata                   = next;
-	return true;
+	return nullptr;
 }
 
 } // namespace Libs::Graphics

src/graphics/shader/shaderVertexMetadata.h

@@ -4,7 +4,6 @@
 #include "graphics/shader/shader.h"
 
 #include <span>
-#include <string>
 
 namespace Libs::Graphics {
 
@@ -14,9 +13,10 @@ struct ShaderVertexMetadata {
 	std::span<const ShaderSemantic> input_semantics;
 };
 
-// Validates the vertex metadata and returns a view into the shader header.
-bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
-                              ShaderVertexMetadata& metadata, std::string* error);
+// Reads the vertex metadata as a view into the shader header. Returns nullptr if it is
+// valid, otherwise why it was rejected.
+const char* ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
+                                     ShaderVertexMetadata& metadata);
 
 } // namespace Libs::Graphics
 

src/kernel/memory.cpp

@@ -688,7 +688,8 @@ public:
 	bool Find(uint64_t phys_addr, bool next, PhysicalMemory::AllocatedBlock* out);
 	bool CanMapDirect(uint64_t phys_addr, size_t len);
 	bool ReleasePoolExpansion(uint64_t phys_addr, size_t len);
-	bool GetAllocatedSpan(uint64_t phys_addr, size_t len, std::vector<AllocatedBlock>* blocks);
+	bool GetAllocatedSpan(uint64_t phys_addr, size_t len, bool skip_gaps,
+	                      std::vector<AllocatedBlock>* blocks);
 	std::vector<AllocatedBlock> FindMappings(uint64_t phys_addr, size_t len);
 	void ProtectMapping(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode,
 	                    GpuAccessMode gpu_mode);
@@ -1353,87 +1354,31 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
                          int memory_type) {
 	Common::LockGuard lock(m_mutex);
 
-	if (len == 0 || UINT64_MAX - phys_addr < len) {
+	if (!CanMapDirect(phys_addr, len)) {
 		return false;
 	}
 
-	auto current = phys_addr;
-	auto next    = m_physical.upper_bound(current);
-	if (next == m_physical.begin()) {
-		return false;
-	}
-	auto first = std::prev(next);
-	bool has_automatic = false;
-	while (current < phys_addr + len) {
-		auto block = m_physical.upper_bound(current);
-		if (block == m_physical.begin()) {
-			return false;
-		}
-		--block;
-		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
-		    current >= block_end) {
-			return false;
-		}
-		has_automatic |= block->second.kind == AllocationKind::Automatic;
-		current = std::min<uint64_t>(phys_addr + len, block_end);
-		if (current < phys_addr + len) {
-			const auto following = std::next(block);
-			if (following == m_physical.end() || following->second.start_addr != current) {
-				return false;
-			}
-		}
-	}
-
-	AllocatedBlock mapping = first->second;
-	mapping.start_addr     = phys_addr;
-	mapping.size           = len;
-	mapping.map_vaddr      = vaddr;
-	mapping.map_size       = len;
-	mapping.host_vaddr     = vaddr;
-	mapping.host_size      = len;
-	mapping.prot           = prot;
-	mapping.mode           = mode;
-	mapping.gpu_mode       = gpu_mode;
-	mapping.memory_type    = memory_type;
+	AllocatedBlock mapping {};
+	mapping.start_addr  = phys_addr;
+	mapping.size        = len;
+	mapping.map_vaddr   = vaddr;
+	mapping.map_size    = len;
+	mapping.host_vaddr  = vaddr;
+	mapping.host_size   = len;
+	mapping.prot        = prot;
+	mapping.mode        = mode;
+	mapping.gpu_mode    = gpu_mode;
+	mapping.memory_type = memory_type;
 	CopyVirtualRangeName(mapping.name, name);
 	m_mappings.push_back(mapping);
-	if (has_automatic) {
-		RemoveFreeRange(m_automatic_free, phys_addr, len);
-	}
+	RemoveFreeRange(m_automatic_free, phys_addr, len);
 
 	return true;
 }
 
 bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
-	Common::LockGuard lock(m_mutex);
-
-	if (len == 0 || UINT64_MAX - phys_addr < len) {
-		return false;
-	}
-
-	const auto end     = phys_addr + len;
-	auto       current = phys_addr;
-	while (current < end) {
-		auto block = m_physical.upper_bound(current);
-		if (block == m_physical.begin()) {
-			return false;
-		}
-		--block;
-		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
-		    current >= block_end) {
-			return false;
-		}
-		current = std::min(end, block_end);
-		if (current < end) {
-			const auto following = std::next(block);
-			if (following == m_physical.end() || following->second.start_addr != current) {
-				return false;
-			}
-		}
-	}
-	return true;
+	std::vector<AllocatedBlock> blocks;
+	return GetAllocatedSpan(phys_addr, len, false, &blocks);
 }
 
 bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) {
@@ -1448,7 +1393,7 @@ bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) {
 	return true;
 }
 
-bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len,
+bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len, bool skip_gaps,
                                       std::vector<AllocatedBlock>* blocks) {
 	EXIT_IF(blocks == nullptr);
 	blocks->clear();
@@ -1459,34 +1404,26 @@ bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len,
 	Common::LockGuard lock(m_mutex);
 	const auto        end     = phys_addr + len;
 	auto              current = phys_addr;
-	while (current < end) {
-		auto block = m_physical.upper_bound(current);
-		if (block == m_physical.begin()) {
-			blocks->clear();
-			return false;
-		}
+	auto              block   = m_physical.upper_bound(phys_addr);
+	if (block != m_physical.begin()) {
 		--block;
+	}
+	for (; block != m_physical.end() && block->second.start_addr < end; ++block) {
 		const auto block_end = block->second.start_addr + block->second.size;
-		if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
-		    current >= block_end) {
-			blocks->clear();
+		if (block_end <= current) {
+			continue;
+		}
+		if (block->second.kind == AllocationKind::Pooled ||
+		    (!skip_gaps && current < block->second.start_addr)) {
 			return false;
 		}
-
 		auto part       = block->second;
-		part.start_addr = current;
-		part.size       = std::min(end, block_end) - current;
+		part.start_addr = std::max(current, part.start_addr);
+		current         = std::min(end, block_end);
+		part.size       = current - part.start_addr;
 		blocks->push_back(part);
-		current += part.size;
-		if (current < end) {
-			const auto following = std::next(block);
-			if (following == m_physical.end() || following->second.start_addr != current) {
-				blocks->clear();
-				return false;
-			}
-		}
 	}
-	return true;
+	return skip_gaps || current == end;
 }
 
 bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode,
@@ -2855,7 +2792,7 @@ int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, i
 	                                  alignment, memory_type, phys_addr_out);
 }
 
-static int ReleaseDirectMemoryInternal(int64_t start, size_t len) {
+static int ReleaseDirectMemoryInternal(int64_t start, size_t len, bool skip_gaps) {
 	std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
 
 	if (g_pooled_memory->ReleaseExpansion(static_cast<uint64_t>(start), len)) {
@@ -2866,7 +2803,8 @@ static int ReleaseDirectMemoryInternal(int64_t start, size_t len) {
 	}
 
 	std::vector<PhysicalMemory::AllocatedBlock> allocated_span;
-	if (!g_physical_memory->GetAllocatedSpan(static_cast<uint64_t>(start), len, &allocated_span)) {
+	if (!g_physical_memory->GetAllocatedSpan(static_cast<uint64_t>(start), len, skip_gaps,
+	                                         &allocated_span)) {
 		return KERNEL_ERROR_EACCES;
 	}
 
@@ -2965,7 +2903,8 @@ int KYTY_SYSV_ABI KernelReleaseDirectMemory(int64_t start, size_t len) {
 		return validation;
 	}
 	if (len != 0) {
-		(void)ReleaseDirectMemoryInternal(start, len);
+		// An unchecked release frees the allocated parts and skips the unallocated ones.
+		(void)ReleaseDirectMemoryInternal(start, len, true);
 	}
 	return OK;
 }
@@ -2982,7 +2921,7 @@ int KYTY_SYSV_ABI KernelCheckedReleaseDirectMemory(int64_t start, size_t len) {
 		return validation;
 	}
 
-	const int result = ReleaseDirectMemoryInternal(start, len);
+	const int result = ReleaseDirectMemoryInternal(start, len, false);
 	return result == KERNEL_ERROR_EACCES ? KERNEL_ERROR_ENOENT : result;
 }
 

src/kernel/memoryAddressSpace.inc

@@ -820,12 +820,12 @@ public:
 		return ProtectMappedUnlocked(vaddr, size, mode);
 	}
 
+	// Tracker protections only read m_mapped; the page tracker orders them per page.
 	bool ProtectTransient(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) {
 		if (size == 0 || UINT64_MAX - vaddr < size) {
 			return false;
 		}
-
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		return ProtectMappedUnlocked(vaddr, size, mode);
 	}
 

src/launcher/CMakeLists.txt

@@ -37,6 +37,16 @@ endif()
 target_include_directories(launcher PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/src")
 set_target_properties(launcher PROPERTIES AUTOUIC ON AUTOMOC ON AUTORCC ON)
 
+add_executable(launcher_game_views_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/LauncherGameViewsTests.cpp"
+	src/gameContent.cpp
+	src/launcherTheme.cpp
+	resources.qrc
+)
+target_link_libraries(launcher_game_views_tests Qt6::Widgets common)
+target_include_directories(launcher_game_views_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include")
+set_target_properties(launcher_game_views_tests PROPERTIES AUTOUIC ON AUTORCC ON)
+
 if(WIN32 AND NOT KYTY_CLANG_CL)
 	set_target_properties(launcher PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS}")
 elseif(LINUX)

src/launcher/forms/configuration_edit_dialog.ui

@@ -141,6 +141,7 @@
               <property name="value"><number>60</number></property>
              </widget>
             </item>
+            <!--
             <item row="3" column="0">
              <widget class="QLabel" name="label_shader_optimization">
               <property name="text"><string>Shader optimization:</string></property>
@@ -151,6 +152,7 @@
               <property name="toolTip"><string>Optimize shaders for code size or performance</string></property>
              </widget>
             </item>
+            -->
             <item row="4" column="0">
              <widget class="QLabel" name="label_gpu">
               <property name="text"><string>GPU:</string></property>
@@ -180,11 +182,23 @@
              </widget>
             </item>
             <item row="8" column="1">
+             <widget class="QCheckBox" name="checkBox_occlusion_queries">
+              <property name="toolTip"><string>Count the samples of occlusion queries on the GPU, so hidden lights and objects stay hidden. Costs some performance.</string></property>
+              <property name="text"><string>Enable occlusion queries</string></property>
+             </widget>
+            </item>
+            <item row="9" column="1">
              <widget class="QCheckBox" name="checkBox_hdr">
               <property name="toolTip"><string>Enable HDR10 output for supported games. Requires HDR enabled on the host display.</string></property>
               <property name="text"><string>Enable HDR display support</string></property>
              </widget>
             </item>
+            <item row="10" column="1">
+             <widget class="QCheckBox" name="checkBox_shader_cache">
+              <property name="toolTip"><string>Save compiled shaders and pipelines per game and load them at startup.</string></property>
+              <property name="text"><string>Cache shaders</string></property>
+             </widget>
+            </item>
            </layout>
           </widget>
          </item>

src/launcher/forms/configuration_list_widget.ui

@@ -304,8 +304,11 @@
         </layout>
        </item>
        <item>
-        <widget class="QStackedWidget" name="game_views">
-         <widget class="GameListTreeWidget" name="cfgs_list">
+        <widget class="GameViews" name="game_views">
+         <widget class="QTreeWidget" name="cfgs_list">
+          <property name="styleSheet">
+           <string notr="true">#cfgs_list { background: transparent; }</string>
+          </property>
           <property name="rootIsDecorated">
            <bool>false</bool>
           </property>
@@ -350,7 +353,11 @@
            </property>
           </column>
          </widget>
-         <widget class="QListView" name="cfgs_grid"/>
+         <widget class="QListView" name="cfgs_grid">
+          <property name="styleSheet">
+           <string notr="true">#cfgs_grid { background: transparent; }</string>
+          </property>
+         </widget>
         </widget>
        </item>
       </layout>
@@ -369,9 +376,10 @@
  </widget>
  <customwidgets>
   <customwidget>
-   <class>GameListTreeWidget</class>
-   <extends>QTreeWidget</extends>
-   <header>gameListTreeWidget.h</header>
+   <class>GameViews</class>
+   <extends>QStackedWidget</extends>
+   <header>gameViews.h</header>
+   <container>1</container>
   </customwidget>
  </customwidgets>
  <resources/>

src/launcher/forms/main_dialog.ui

@@ -112,6 +112,13 @@
          </property>
         </widget>
        </item>
+       <item>
+        <widget class="QCheckBox" name="close_terminal">
+         <property name="text">
+          <string>Close terminal automatically</string>
+         </property>
+        </widget>
+       </item>
        <item>
         <layout class="QHBoxLayout" name="themeLayout">
          <item>

src/launcher/include/additionalContentDialog.h

@@ -4,26 +4,39 @@
 #include <QDialog>
 #include <QIcon>
 #include <QString>
+#include <QStringList>
+
+#include <functional>
 
 class Configuration;
 class QLabel;
+class QLineEdit;
 class QListWidget;
 class QPushButton;
 
 class AdditionalContentDialog final: public QDialog {
 public:
-	AdditionalContentDialog(const Configuration& game, const QString& folder,
-	                        QWidget* parent = nullptr);
+	// entries are the game's user add-ons, "LABEL" or "LABEL=DIR"; save receives every edit.
+	AdditionalContentDialog(const Configuration& game, const QString& folder, QStringList entries,
+	                        std::function<void(const QStringList&)> save,
+	                        QWidget*                                parent = nullptr);
 
 private:
 	void Scan();
+	void Add();
+	void Remove();
 
-	QString      m_title_id;
-	QString      m_folder;
-	QIcon        m_game_icon;
-	QLabel*      m_status = nullptr;
-	QListWidget* m_list   = nullptr;
-	QPushButton* m_open   = nullptr;
+	QString                                 m_title_id;
+	QString                                 m_folder;
+	QStringList                             m_entries;
+	std::function<void(const QStringList&)> m_save;
+	QIcon                                   m_game_icon;
+	QLabel*                                 m_status     = nullptr;
+	QListWidget*                            m_list       = nullptr;
+	QPushButton*                            m_open       = nullptr;
+	QPushButton*                            m_remove     = nullptr;
+	QLineEdit*                              m_label_edit = nullptr;
+	QLineEdit*                              m_dir_edit   = nullptr;
 };
 
 #endif // ADDITIONAL_CONTENT_DIALOG_H

src/launcher/include/configuration.h

@@ -102,16 +102,17 @@ public:
 
 	Configuration() = default;
 
-	QString    name;
-	QString    title_id;    /* Serial / title id from sce_sys/param.json */
-	QString    gameVersion; /* appVersion / contentVersion from sce_sys/param.json */
-	QString    firmwareVer; /* requiredSystemSoftwareVersion from sce_sys/param.json */
-	QString    basedir;     /* Game base directory */
-	QString    game_path;   /* Launcher-unique game path */
-	QString    addons_dir;  /* Global setting; not saved per game. */
-	bool       custom_settings = false;
-	GameStatus game_status     = GameStatus::Unknown;
-	QString    game_comment;
+	QString     name;
+	QString     title_id;    /* Serial / title id from sce_sys/param.json */
+	QString     gameVersion; /* appVersion / contentVersion from sce_sys/param.json */
+	QString     firmwareVer; /* requiredSystemSoftwareVersion from sce_sys/param.json */
+	QString     basedir;     /* Game base directory */
+	QString     game_path;   /* Launcher-unique game path */
+	QString     addons_dir;  /* Global setting; not saved per game. */
+	QStringList addcont;     /* User add-ons ("LABEL" or "LABEL=DIR"), saved by title ID. */
+	bool        custom_settings = false;
+	GameStatus  game_status     = GameStatus::Unknown;
+	QString     game_comment;
 
 	// Controller preferences always come from the global configuration.
 	ControllerSettings controller;
@@ -129,6 +130,8 @@ public:
 	bool                   readback_tiled_images       = false;
 	bool                   sync_raw_image_buffers      = false;
 	bool                   tessellation_enabled        = false;
+	bool                   occlusion_queries_enabled   = false;
+	bool                   shader_cache_enabled        = true;
 	bool                   trophy_enabled              = true;
 	int                    trophy_duration_seconds     = 6;
 	bool                   perf_overlay_enabled        = false;
@@ -169,6 +172,8 @@ public:
 		readback_tiled_images       = other.readback_tiled_images;
 		sync_raw_image_buffers      = other.sync_raw_image_buffers;
 		tessellation_enabled        = other.tessellation_enabled;
+		occlusion_queries_enabled   = other.occlusion_queries_enabled;
+		shader_cache_enabled        = other.shader_cache_enabled;
 		trophy_enabled              = other.trophy_enabled;
 		trophy_duration_seconds     = other.trophy_duration_seconds;
 		perf_overlay_enabled        = other.perf_overlay_enabled;

src/launcher/include/configurationListWidget.h

@@ -87,6 +87,7 @@ private:
 	QString                       m_runtime_directory;
 	QStringList                   m_game_dirs;
 	QString                       m_addons_dir;
+	QMap<QString, QStringList>    m_addcont; // User add-ons by title ID.
 	Configuration                 m_global_info;
 	QMap<QString, Configuration*> m_custom_infos;
 	CompatibilityDatabase*        m_compatibility = nullptr;

src/launcher/include/gameListTreeWidget.h

@@ -1,77 +0,0 @@
-#ifndef GAME_LIST_TREE_WIDGET_H
-#define GAME_LIST_TREE_WIDGET_H
-
-#include "gameContent.h"
-
-#include <QPaintEvent>
-#include <QPainter>
-#include <QPalette>
-#include <QPixmap>
-#include <QPoint>
-#include <QResizeEvent>
-#include <QString>
-#include <QTreeWidget>
-
-class GameListTreeWidget: public QTreeWidget {
-public:
-	explicit GameListTreeWidget(QWidget* parent = nullptr): QTreeWidget(parent) {
-		setAutoFillBackground(false);
-		viewport()->setAutoFillBackground(false);
-	}
-
-	void SetBackgroundImage(const QString& key) {
-		if (m_path == key) {
-			return;
-		}
-		m_path   = key;
-		m_source = QPixmap();
-		if (!key.isEmpty()) {
-			m_source.loadFromData(GameContent::ReadFile(key, QStringLiteral("sce_sys/pic0.png"),
-			                                            GameContent::MaxImageSize));
-		}
-		UpdateScaledBackground();
-		viewport()->update();
-	}
-
-protected:
-	void resizeEvent(QResizeEvent* event) override {
-		QTreeWidget::resizeEvent(event);
-		UpdateScaledBackground();
-	}
-
-	void paintEvent(QPaintEvent* event) override {
-		{
-			QPainter painter(viewport());
-			if (m_scaled.isNull()) {
-				painter.fillRect(event->rect(), palette().brush(QPalette::Base));
-			} else {
-				painter.drawPixmap(m_scaled_pos, m_scaled);
-				auto overlay = palette().color(QPalette::Base);
-				overlay.setAlpha(185);
-				painter.fillRect(event->rect(), overlay);
-			}
-		}
-
-		QTreeWidget::paintEvent(event);
-	}
-
-private:
-	void UpdateScaledBackground() {
-		m_scaled = QPixmap();
-		if (m_source.isNull() || viewport()->size().isEmpty()) {
-			return;
-		}
-
-		m_scaled     = m_source.scaled(viewport()->size(), Qt::KeepAspectRatioByExpanding,
-		                               Qt::FastTransformation);
-		m_scaled_pos = QPoint((viewport()->width() - m_scaled.width()) / 2,
-		                      (viewport()->height() - m_scaled.height()) / 2);
-	}
-
-	QString m_path;
-	QPixmap m_source;
-	QPixmap m_scaled;
-	QPoint  m_scaled_pos;
-};
-
-#endif // GAME_LIST_TREE_WIDGET_H

src/launcher/include/gameViews.h

@@ -0,0 +1,148 @@
+#ifndef GAME_VIEWS_H
+#define GAME_VIEWS_H
+
+#include "gameContent.h"
+
+#include <QGuiApplication>
+#include <QIcon>
+#include <QListView>
+#include <QPaintEvent>
+#include <QPainter>
+#include <QPalette>
+#include <QPixmap>
+#include <QResizeEvent>
+#include <QStackedWidget>
+#include <QString>
+#include <QStyledItemDelegate>
+#include <QTreeWidget>
+
+inline constexpr int GAME_NAME_COLUMN             = 0;
+inline constexpr int GAME_SERIAL_COLUMN           = 1;
+inline constexpr int GAME_VERSION_COLUMN          = 2;
+inline constexpr int GAME_FIRMWARE_VERSION_COLUMN = 3;
+inline constexpr int GAME_SIZE_COLUMN             = 4;
+inline constexpr int GAME_PATH_COLUMN             = 5;
+inline constexpr int GAME_STATUS_COLUMN           = 6;
+inline constexpr int GAME_COMMENT_COLUMN          = 7;
+
+// Holds the transparent list and grid views and paints the selected game's pic0 behind both.
+class GameViews: public QStackedWidget {
+public:
+	using QStackedWidget::QStackedWidget;
+
+	void SetBackgroundImage(const QString& key) {
+		if (m_path == key) {
+			return;
+		}
+		m_path   = key;
+		m_source = QPixmap();
+		if (!key.isEmpty()) {
+			m_source.loadFromData(GameContent::ReadFile(key, QStringLiteral("sce_sys/pic0.png"),
+			                                            GameContent::MaxImageSize));
+		}
+		UpdateScaledBackground();
+		update();
+	}
+
+protected:
+	void resizeEvent(QResizeEvent* event) override {
+		QStackedWidget::resizeEvent(event);
+		UpdateScaledBackground();
+	}
+
+	void paintEvent(QPaintEvent* event) override {
+		QPainter painter(this);
+		auto     overlay = palette().color(QPalette::Base);
+		if (!m_scaled.isNull()) {
+			painter.drawPixmap((width() - m_scaled.width()) / 2, (height() - m_scaled.height()) / 2,
+			                   m_scaled);
+			overlay.setAlpha(160);
+		}
+		painter.fillRect(event->rect(), overlay);
+	}
+
+private:
+	void UpdateScaledBackground() {
+		m_scaled = m_source.isNull() ? QPixmap()
+		                             : m_source.scaled(size(), Qt::KeepAspectRatioByExpanding,
+		                                               Qt::FastTransformation);
+	}
+
+	QString m_path;
+	QPixmap m_source;
+	QPixmap m_scaled;
+};
+
+// A card on a translucent panel, so the text keeps its contrast over the game background.
+class GameGridDelegate: public QStyledItemDelegate {
+public:
+	using QStyledItemDelegate::QStyledItemDelegate;
+
+	QSize sizeHint(const QStyleOptionViewItem& option,
+	               const QModelIndex& /*index*/) const override {
+		return option.decorationSize +
+		       QSize(2 * CARD_PADDING, 3 * CARD_PADDING + 2 * option.fontMetrics.lineSpacing());
+	}
+
+	void paint(QPainter* painter, const QStyleOptionViewItem& option,
+	           const QModelIndex& index) const override {
+		// The stylesheet made the view's own palette transparent, so take the panel from the app's.
+		auto panel      = QGuiApplication::palette().color(QPalette::Window);
+		auto text_color = option.palette.color(QPalette::Text);
+		panel.setAlpha(210);
+		painter->save();
+		painter->setRenderHint(QPainter::Antialiasing);
+		painter->setPen(option.state.testFlag(QStyle::State_Selected) ? QPen(text_color)
+		                                                              : Qt::NoPen);
+		painter->setBrush(panel);
+		painter->drawRoundedRect(QRectF(option.rect).adjusted(0.5, 0.5, -0.5, -0.5), 4, 4);
+
+		const QRect cover(option.rect.topLeft() + QPoint(CARD_PADDING, CARD_PADDING),
+		                  option.decorationSize);
+		index.data(Qt::DecorationRole).value<QIcon>().paint(painter, cover);
+
+		QRect line(cover.left(), cover.bottom() + 1 + CARD_PADDING, cover.width(),
+		           option.fontMetrics.lineSpacing());
+		auto  title_font = option.font;
+		title_font.setBold(true);
+		painter->setFont(title_font);
+		painter->setPen(text_color);
+		painter->drawText(line, Qt::AlignLeft | Qt::AlignVCenter,
+		                  painter->fontMetrics().elidedText(index.data().toString(), Qt::ElideRight,
+		                                                    line.width()));
+
+		line.translate(0, line.height());
+		const auto size    = index.siblingAtColumn(GAME_SIZE_COLUMN).data().toString();
+		const auto details = QStringLiteral("%1  %2").arg(
+		    index.siblingAtColumn(GAME_SERIAL_COLUMN).data().toString(),
+		    index.siblingAtColumn(GAME_VERSION_COLUMN).data().toString());
+		text_color.setAlpha(200);
+		painter->setFont(option.font);
+		painter->setPen(text_color);
+		painter->drawText(line, Qt::AlignRight | Qt::AlignVCenter, size);
+		line.setRight(line.right() - option.fontMetrics.horizontalAdvance(size) - CARD_PADDING);
+		painter->drawText(line, Qt::AlignLeft | Qt::AlignVCenter,
+		                  option.fontMetrics.elidedText(details, Qt::ElideRight, line.width()));
+		painter->restore();
+	}
+
+private:
+	static constexpr int CARD_PADDING = 8;
+};
+
+// Shows the list's games as cards in the grid: both views share the items and the current game.
+inline void SetupGameGrid(QTreeWidget* list, QListView* grid) {
+	grid->setModel(list->model());
+	grid->setSelectionModel(list->selectionModel());
+	grid->setViewMode(QListView::IconMode);
+	grid->setMovement(QListView::Static);
+	grid->setResizeMode(QListView::Adjust);
+	grid->setSpacing(8);
+	grid->setUniformItemSizes(true);
+	grid->setItemDelegate(new GameGridDelegate(grid));
+	// A card also shows other columns (the size arrives late), which the grid does not repaint on.
+	QObject::connect(list->model(), &QAbstractItemModel::dataChanged, grid->viewport(),
+	                 qOverload<>(&QWidget::update));
+}
+
+#endif // GAME_VIEWS_H

src/launcher/include/mainDialog.h

@@ -22,7 +22,7 @@ public:
 	explicit MainDialog(QWidget* parent = nullptr);
 	~MainDialog() override = default;
 
-	void RunInterpreter(QProcess* process, const Configuration& info);
+	void RunInterpreter(QProcess* process, const Configuration& info, bool close_terminal);
 
 	static void WriteSettings(QSettings& s);
 	static void ReadSettings(QSettings& s);

src/launcher/src/additionalContentDialog.cpp

@@ -7,7 +7,10 @@
 #include <QDesktopServices>
 #include <QDialogButtonBox>
 #include <QDir>
+#include <QFileDialog>
+#include <QHBoxLayout>
 #include <QLabel>
+#include <QLineEdit>
 #include <QListWidget>
 #include <QPixmap>
 #include <QPushButton>
@@ -16,8 +19,11 @@
 #include <QVBoxLayout>
 
 AdditionalContentDialog::AdditionalContentDialog(const Configuration& game, const QString& folder,
-                                                 QWidget* parent)
-    : QDialog(parent), m_title_id(game.title_id.trimmed().toUpper()), m_folder(folder) {
+                                                 QStringList                             entries,
+                                                 std::function<void(const QStringList&)> save,
+                                                 QWidget*                                parent)
+    : QDialog(parent), m_title_id(game.title_id.trimmed().toUpper()), m_folder(folder),
+      m_entries(std::move(entries)), m_save(std::move(save)) {
 	setAttribute(Qt::WA_DeleteOnClose);
 	setWindowTitle(tr("Additional Content - %1").arg(game.name));
 	resize(720, 480);
@@ -39,6 +45,28 @@ AdditionalContentDialog::AdditionalContentDialog(const Configuration& game, cons
 	m_list->setVerticalScrollMode(QAbstractItemView::ScrollPerPixel);
 	layout->addWidget(m_list, 1);
 
+	auto* add_layout = new QHBoxLayout;
+	m_label_edit     = new QLineEdit(this);
+	m_label_edit->setPlaceholderText(tr("Label or content ID"));
+	m_dir_edit = new QLineEdit(this);
+	m_dir_edit->setPlaceholderText(tr("Add-on data folder (empty for a license only)"));
+	auto* browse = new QPushButton(tr("Browse..."), this);
+	auto* add    = new QPushButton(tr("Add"), this);
+	add->setDefault(true);
+	add_layout->addWidget(m_label_edit, 1);
+	add_layout->addWidget(m_dir_edit, 2);
+	add_layout->addWidget(browse);
+	add_layout->addWidget(add);
+	layout->addLayout(add_layout);
+	connect(browse, &QPushButton::clicked, this, [this]() {
+		const auto dir =
+		    QFileDialog::getExistingDirectory(this, tr("Add-on data folder"), m_dir_edit->text());
+		if (!dir.isEmpty()) {
+			m_dir_edit->setText(QDir::toNativeSeparators(dir));
+		}
+	});
+	connect(add, &QPushButton::clicked, this, &AdditionalContentDialog::Add);
+
 	m_status = new QLabel(this);
 	m_status->setTextFormat(Qt::PlainText);
 	m_status->setWordWrap(true);
@@ -47,10 +75,15 @@ AdditionalContentDialog::AdditionalContentDialog(const Configuration& game, cons
 	auto* buttons = new QDialogButtonBox(QDialogButtonBox::Close, this);
 	auto* refresh = buttons->addButton(tr("Refresh"), QDialogButtonBox::ActionRole);
 	m_open        = buttons->addButton(tr("Open folder"), QDialogButtonBox::ActionRole);
+	m_remove      = buttons->addButton(tr("Remove"), QDialogButtonBox::ActionRole);
 	layout->addWidget(buttons);
 	connect(refresh, &QPushButton::clicked, this, &AdditionalContentDialog::Scan);
 	connect(m_open, &QPushButton::clicked, this,
 	        [this]() { QDesktopServices::openUrl(QUrl::fromLocalFile(m_folder)); });
+	connect(m_remove, &QPushButton::clicked, this, &AdditionalContentDialog::Remove);
+	connect(m_list, &QListWidget::currentItemChanged, this, [this](const QListWidgetItem* item) {
+		m_remove->setEnabled(item != nullptr && item->data(Qt::UserRole).isValid());
+	});
 	connect(buttons, &QDialogButtonBox::rejected, this, &QDialog::close);
 
 	Scan();
@@ -73,18 +106,54 @@ void AdditionalContentDialog::Scan() {
 		                                 tr("%1\nInstalled").arg(title), m_list);
 		item->setToolTip(QString::fromStdString(entry.content_id));
 	}
+	for (const auto& entry: m_entries) {
+		const auto label = entry.section('=', 0, 0);
+		const auto dir   = entry.section('=', 1);
+		auto       text  = tr("%1\nLicense only").arg(label);
+		if (!dir.isEmpty()) {
+			text = (QDir(dir).exists() ? tr("%1\nData: %2") : tr("%1\nData folder not found: %2"))
+			           .arg(label, QDir::toNativeSeparators(dir));
+		}
+		auto* item = new QListWidgetItem(m_game_icon, text, m_list);
+		item->setData(Qt::UserRole, entry);
+	}
+	m_remove->setEnabled(false);
 
-	if (m_folder.isEmpty()) {
+	if (m_list->count() != 0) {
+		m_status->setText(tr("%n add-on(s)", nullptr, m_list->count()));
+	} else if (m_folder.isEmpty()) {
 		m_status->setText(tr("No add-ons folder is set. Open Global settings and set the "
 		                     "Add-ons folder to the location of your extracted add-on packages."));
 	} else if (!folder_exists) {
 		m_status->setText(tr("The add-ons folder does not exist:\n%1\nCheck the path in Global "
 		                     "settings.")
 		                      .arg(QDir::toNativeSeparators(m_folder)));
-	} else if (m_list->count() == 0) {
+	} else {
 		m_status->setText(
 		    tr("No add-ons found for this game in:\n%1").arg(QDir::toNativeSeparators(m_folder)));
-	} else {
-		m_status->setText(tr("%n installed add-on(s)", nullptr, m_list->count()));
 	}
 }
+
+void AdditionalContentDialog::Add() {
+	// A content ID ends with its entitlement label.
+	const auto label = m_label_edit->text().trimmed().section('-', -1);
+	const auto dir   = m_dir_edit->text().trimmed();
+	const auto entry = dir.isEmpty() ? label : label + '=' + QDir(dir).absolutePath();
+	if (!Common::ParseAddContent(entry.toStdString())) {
+		m_status->setText(tr("Enter a 16-character entitlement label (A-Z, 0-9) and an existing "
+		                     "data folder, or no folder for a license only."));
+		return;
+	}
+	m_entries.removeIf([&](const QString& e) { return e.section('=', 0, 0) == label; });
+	m_entries.append(entry);
+	m_save(m_entries);
+	m_label_edit->clear();
+	m_dir_edit->clear();
+	Scan();
+}
+
+void AdditionalContentDialog::Remove() {
+	m_entries.removeOne(m_list->currentItem()->data(Qt::UserRole).toString());
+	m_save(m_entries);
+	Scan();
+}

src/launcher/src/configuration.cpp

@@ -16,6 +16,8 @@ QVariantMap Configuration::GameSettings() const {
 	KYTY_CFG_SET(readback_tiled_images);
 	KYTY_CFG_SET(sync_raw_image_buffers);
 	KYTY_CFG_SET(tessellation_enabled);
+	KYTY_CFG_SET(occlusion_queries_enabled);
+	KYTY_CFG_SET(shader_cache_enabled);
 	KYTY_CFG_SET(trophy_enabled);
 	KYTY_CFG_SET(trophy_duration_seconds);
 	KYTY_CFG_SET(perf_overlay_enabled);
@@ -66,7 +68,9 @@ void Configuration::ReadGameSettingsValues(const Settings& s) {
 	KYTY_CFG_GET(readback_tiled_images);
 	KYTY_CFG_GET(sync_raw_image_buffers);
 	KYTY_CFG_GET(tessellation_enabled);
-	trophy_enabled   = s.value("trophy_enabled", trophy_enabled).toBool();
+	KYTY_CFG_GET(occlusion_queries_enabled);
+	shader_cache_enabled = s.value("shader_cache_enabled", shader_cache_enabled).toBool();
+	trophy_enabled       = s.value("trophy_enabled", trophy_enabled).toBool();
 	KYTY_CFG_GET(trophy_duration_seconds);
 	KYTY_CFG_GET(perf_overlay_enabled);
 	KYTY_CFG_GET(skip_notice_screen);

src/launcher/src/configurationEditDialog.cpp

@@ -289,6 +289,8 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
 	m_ui->checkBox_readback_tiled->setChecked(info.readback_tiled_images);
 	m_ui->checkBox_sync_raw_image_buffers->setChecked(info.sync_raw_image_buffers);
 	m_ui->checkBox_tessellation->setChecked(info.tessellation_enabled);
+	m_ui->checkBox_occlusion_queries->setChecked(info.occlusion_queries_enabled);
+	m_ui->checkBox_shader_cache->setChecked(info.shader_cache_enabled);
 	m_ui->checkBox_skip_notice_screen->setChecked(info.skip_notice_screen);
 	m_ui->checkBox_network_connected->setChecked(info.network_connected);
 	m_ui->checkBox_trophy_notifications->setChecked(info.trophy_enabled);
@@ -313,7 +315,7 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
 #else
 	m_ui->checkBox_red_zone_protection->setVisible(false);
 #endif
-	ListInit(m_ui->comboBox_shader_optimization_type, info.shader_optimization_type);
+	// ListInit(m_ui->comboBox_shader_optimization_type, info.shader_optimization_type);
 	ListInit(m_ui->comboBox_shader_log_direction, info.shader_log_direction);
 	m_ui->lineEdit_shader_log_folder->setText(info.shader_log_folder);
 	m_ui->lineEdit_shader_log_folder->setEnabled(info.shader_log_direction ==
@@ -483,6 +485,8 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui, boo
 	info.readback_tiled_images     = ui.checkBox_readback_tiled->isChecked();
 	info.sync_raw_image_buffers    = ui.checkBox_sync_raw_image_buffers->isChecked();
 	info.tessellation_enabled      = ui.checkBox_tessellation->isChecked();
+	info.occlusion_queries_enabled = ui.checkBox_occlusion_queries->isChecked();
+	info.shader_cache_enabled      = ui.checkBox_shader_cache->isChecked();
 	info.skip_notice_screen        = ui.checkBox_skip_notice_screen->isChecked();
 	info.network_connected         = ui.checkBox_network_connected->isChecked();
 	info.trophy_enabled            = ui.checkBox_trophy_notifications->isChecked();
@@ -497,8 +501,8 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui, boo
 #if defined(_WIN32)
 	info.red_zone_protection_enabled = ui.checkBox_red_zone_protection->isChecked();
 #endif
-	info.shader_optimization_type = TextToEnum<Configuration::ShaderOptimizationType>(
-	    ui.comboBox_shader_optimization_type->currentText());
+	// info.shader_optimization_type = TextToEnum<Configuration::ShaderOptimizationType>(
+	//     ui.comboBox_shader_optimization_type->currentText());
 	info.shader_log_direction = TextToEnum<Configuration::LogDirection>(
 	    ui.comboBox_shader_log_direction->currentText());
 	info.shader_log_folder           = ui.lineEdit_shader_log_folder->text();

src/launcher/src/configurationListWidget.cpp

@@ -8,7 +8,7 @@
 #include "configurationEditDialog.h"
 #include "configurationItem.h"
 #include "gameContent.h"
-#include "gameListTreeWidget.h"
+#include "gameViews.h"
 #include "inputMappingDialog.h"
 #include "mainDialog.h"
 #include "patchesDialog.h"
@@ -53,7 +53,6 @@
 #include <QSignalBlocker>
 #include <QSlider>
 #include <QStyle>
-#include <QStyledItemDelegate>
 #include <QToolButton>
 #include <QTreeWidget>
 #include <QUrl>
@@ -80,18 +79,10 @@ constexpr char CONF_LAUNCHER[]     = "Launcher";
 constexpr char CONF_GAME_DIR[]     = "game_dir";
 constexpr char CONF_GAME_DIRS[]    = "game_dirs";
 constexpr char CONF_ADDONS_DIR[]   = "addons_dir";
+constexpr char CONF_ADDCONT[]      = "AdditionalContent";
 constexpr char CONF_GLOBAL[]       = "GlobalConfiguration";
 constexpr char SAVE_DATA_DIR[]     = "_SaveData";
 
-constexpr int GAME_NAME_COLUMN             = 0;
-constexpr int GAME_SERIAL_COLUMN           = 1;
-constexpr int GAME_VERSION_COLUMN          = 2;
-constexpr int GAME_FIRMWARE_VERSION_COLUMN = 3;
-constexpr int GAME_SIZE_COLUMN             = 4;
-constexpr int GAME_PATH_COLUMN             = 5;
-constexpr int GAME_STATUS_COLUMN           = 6;
-constexpr int GAME_COMMENT_COLUMN          = 7;
-
 static QString NormalizeGameDirectory(const QString& dir) {
 	const auto trimmed = dir.trimmed();
 	if (trimmed.isEmpty()) {
@@ -151,54 +142,6 @@ static QStringList SettingsStringList(const QVariant& value) {
 	return text.isEmpty() ? QStringList() : QStringList({text});
 }
 
-class GameGridDelegate: public QStyledItemDelegate {
-public:
-	using QStyledItemDelegate::QStyledItemDelegate;
-
-	QSize sizeHint(const QStyleOptionViewItem& option, const QModelIndex&) const override {
-		return option.decorationSize + QSize(16, 24 + option.fontMetrics.lineSpacing() * 2);
-	}
-
-protected:
-	void initStyleOption(QStyleOptionViewItem* option, const QModelIndex& index) const override {
-		const auto image_size = option->decorationSize;
-		QStyledItemDelegate::initStyleOption(option, index);
-		option->decorationSize      = image_size;
-		option->decorationPosition  = QStyleOptionViewItem::Top;
-		option->decorationAlignment = Qt::AlignCenter;
-		option->displayAlignment    = Qt::AlignHCenter | Qt::AlignTop;
-		option->features |= QStyleOptionViewItem::WrapText;
-	}
-};
-
-static void ConfigureGameList(Ui::ConfigurationListWidget* ui) {
-	ui->cfgs_list->setAlternatingRowColors(false);
-	ui->cfgs_list->setAllColumnsShowFocus(true);
-	ui->cfgs_list->setIndentation(0);
-	ui->cfgs_list->setMouseTracking(false);
-	ui->cfgs_list->setSelectionBehavior(QAbstractItemView::SelectRows);
-	ui->cfgs_list->setSelectionMode(QAbstractItemView::SingleSelection);
-	ui->cfgs_list->setTextElideMode(Qt::ElideMiddle);
-	ui->cfgs_list->header()->setDefaultAlignment(Qt::AlignLeft | Qt::AlignVCenter);
-	ui->cfgs_list->header()->setHighlightSections(false);
-	ui->cfgs_list->header()->setStretchLastSection(true);
-
-	// Both presentations use the same items, icons and current game.
-	ui->cfgs_grid->setModel(ui->cfgs_list->model());
-	ui->cfgs_grid->setSelectionModel(ui->cfgs_list->selectionModel());
-	ui->cfgs_grid->setViewMode(QListView::IconMode);
-	ui->cfgs_grid->setMovement(QListView::Static);
-	ui->cfgs_grid->setResizeMode(QListView::Adjust);
-	ui->cfgs_grid->setSelectionMode(QAbstractItemView::SingleSelection);
-	ui->cfgs_grid->setSelectionBehavior(QAbstractItemView::SelectRows);
-	ui->cfgs_grid->setEditTriggers(QAbstractItemView::NoEditTriggers);
-	ui->cfgs_grid->setSpacing(8);
-	ui->cfgs_grid->setUniformItemSizes(true);
-	ui->cfgs_grid->setWordWrap(true);
-	ui->cfgs_grid->setItemDelegate(new GameGridDelegate(ui->cfgs_grid));
-	ui->cfgs_grid->setContextMenuPolicy(Qt::CustomContextMenu);
-}
-
 static void AddSaveDataDir(QStringList* dirs, QSet<QString>* seen, const QString& root,
                            const QString& title_id) {
 	const auto path = QDir(root).filePath(
@@ -252,7 +195,6 @@ ConfigurationListWidget::ConfigurationListWidget(QWidget* parent)
 	m_compatibility = new CompatibilityDatabase(
 	    QCoreApplication::arguments().contains(QStringLiteral("--local")), this);
 	m_ui->setupUi(this);
-	ConfigureGameList(m_ui);
 
 	m_ui->refresh_action->setShortcuts(QKeySequence::Refresh);
 	m_ui->refresh_button->setDefaultAction(m_ui->refresh_action);
@@ -266,8 +208,9 @@ ConfigurationListWidget::ConfigurationListWidget(QWidget* parent)
 	m_ui->edit_button->setEnabled(false);
 
 	m_ui->cfgs_list->setContextMenuPolicy(Qt::CustomContextMenu);
+	m_ui->cfgs_list->setAllColumnsShowFocus(true);
+	m_ui->cfgs_list->setTextElideMode(Qt::ElideMiddle);
 	m_ui->cfgs_list->setIconSize(QSize(48, 48));
-	m_ui->cfgs_list->setRootIsDecorated(false);
 	m_ui->cfgs_list->setUniformRowHeights(true);
 	m_ui->cfgs_list->setSortingEnabled(true);
 	m_ui->cfgs_list->setColumnWidth(GAME_NAME_COLUMN, 320);
@@ -279,6 +222,8 @@ ConfigurationListWidget::ConfigurationListWidget(QWidget* parent)
 	m_ui->cfgs_list->setColumnWidth(GAME_STATUS_COLUMN, 150);
 	m_ui->cfgs_list->setColumnWidth(GAME_COMMENT_COLUMN, 240);
 	m_ui->cfgs_list->sortItems(GAME_NAME_COLUMN, Qt::AscendingOrder);
+	SetupGameGrid(m_ui->cfgs_list, m_ui->cfgs_grid);
+	m_ui->cfgs_grid->setContextMenuPolicy(Qt::CustomContextMenu);
 
 	connect(m_ui->refresh_action, &QAction::triggered, this,
 	        &ConfigurationListWidget::ScanGameDirectory);
@@ -323,16 +268,12 @@ ConfigurationListWidget::ConfigurationListWidget(QWidget* parent)
 	connect(m_compatibility, &CompatibilityDatabase::Updated, this,
 	        &ConfigurationListWidget::ApplyCompatibility);
 
-	m_ui->cfgs_list->setDragDropMode(QAbstractItemView::NoDragDrop);
-
 	ReadSettings();
 	m_ui->grid_size_slider->setVisible(m_ui->grid_view_button->isChecked());
-	const auto resize_grid = [this](int width) {
-		m_ui->cfgs_grid->setIconSize(QSize(width, width * 9 / 16));
-	};
-	resize_grid(m_ui->grid_size_slider->value());
-	connect(m_ui->grid_size_slider, &QSlider::valueChanged, this, [this, resize_grid](int width) {
-		resize_grid(width);
+	const int grid_size = m_ui->grid_size_slider->value();
+	m_ui->cfgs_grid->setIconSize(QSize(grid_size, grid_size));
+	connect(m_ui->grid_size_slider, &QSlider::valueChanged, this, [this](int width) {
+		m_ui->cfgs_grid->setIconSize(QSize(width, width));
 		if (!m_ui->grid_size_slider->isSliderDown()) {
 			WriteSettings();
 		}
@@ -408,6 +349,13 @@ void ConfigurationListWidget::WriteSettings() {
 	s->remove(CONF_GAME_DIR);
 	s->endGroup();
 
+	s->remove(CONF_ADDCONT);
+	s->beginGroup(CONF_ADDCONT);
+	for (auto it = m_addcont.constBegin(); it != m_addcont.constEnd(); ++it) {
+		s->setValue(it.key(), it.value());
+	}
+	s->endGroup();
+
 	s->remove(CONF_GLOBAL);
 	s->beginGroup(CONF_GLOBAL);
 	m_global_info.WriteSettings(s.get());
@@ -455,6 +403,13 @@ void ConfigurationListWidget::ReadSettings() {
 	m_addons_dir = NormalizeGameDirectory(s->value(CONF_ADDONS_DIR).toString());
 	s->endGroup();
 
+	m_addcont.clear();
+	s->beginGroup(CONF_ADDCONT);
+	for (const auto& title_id: s->childKeys()) {
+		m_addcont.insert(title_id, SettingsStringList(s->value(title_id)));
+	}
+	s->endGroup();
+
 	s->beginGroup(CONF_GLOBAL);
 	if (!s->childKeys().isEmpty()) {
 		m_global_info.ReadSettings(s.get());
@@ -518,6 +473,7 @@ ConfigurationListWidget::CreateConfiguration(const ConfigurationItem& item) cons
 	}
 	info->host_input_mapping = m_global_info.host_input_mapping;
 	info->addons_dir         = m_addons_dir;
+	info->addcont            = m_addcont.value(item.GetInfo().title_id.trimmed().toUpper());
 	return info;
 }
 
@@ -1274,7 +1230,7 @@ void ConfigurationListWidget::SelectItem(QTreeWidgetItem* witem) {
 	auto* item = static_cast<ConfigurationItem*>(witem);
 	if (item == nullptr) {
 		m_selected_item = nullptr;
-		m_ui->cfgs_list->SetBackgroundImage({});
+		m_ui->game_views->SetBackgroundImage({});
 		m_ui->edit_button->setEnabled(false);
 		m_ui->delete_button->setEnabled(false);
 		emit Select();
@@ -1285,7 +1241,7 @@ void ConfigurationListWidget::SelectItem(QTreeWidgetItem* witem) {
 	m_ui->edit_button->setEnabled(!item->IsRunning());
 
 	m_selected_item = item;
-	m_ui->cfgs_list->SetBackgroundImage(item->GetInfo().basedir);
+	m_ui->game_views->SetBackgroundImage(item->GetInfo().basedir);
 
 	emit Select();
 }
@@ -1333,11 +1289,22 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 	connect(action_addcont, &QAction::triggered, this,
 	        [this, item = QPointer<ConfigurationItem>(item)]() {
 		        if (item != nullptr) {
-			        auto* dialog = new AdditionalContentDialog(item->GetInfo(), m_addons_dir, this);
+			        const auto title_id = item->GetInfo().title_id.trimmed().toUpper();
+			        auto*      dialog   = new AdditionalContentDialog(
+			            item->GetInfo(), m_addons_dir, m_addcont.value(title_id),
+			            [this, title_id](const QStringList& entries) {
+				            if (entries.isEmpty()) {
+					            m_addcont.remove(title_id);
+				            } else {
+					            m_addcont.insert(title_id, entries);
+				            }
+				            WriteSettings();
+			            },
+			            this);
 			        dialog->show();
 		        }
 	        });
-	action_addcont->setEnabled(item != nullptr);
+	action_addcont->setEnabled(item != nullptr && !item->GetInfo().title_id.trimmed().isEmpty());
 	QAction* action_compress =
 	    menu.addAction(tr("Compress to ZAR..."), this, SLOT(compress_game()));
 	QAction* action_remove_save_data =

src/launcher/src/mainDialog.cpp

@@ -17,7 +17,6 @@
 #include <QDir>
 #include <QFile>
 #include <QFileInfo>
-#include <QIODevice>
 #include <QLabel>
 #include <QMessageBox>
 #include <QObject>
@@ -28,7 +27,6 @@
 #include <QSettings>
 #include <QStringList>
 #include <QStyleHints>
-#include <QTextStream>
 #include <QVariant>
 #include <QtCore>
 
@@ -47,23 +45,17 @@
 class QWidget;
 
 #if defined(_WIN32)
-constexpr char EMULATOR_EXE[] = "kyty_emulator.exe";
+constexpr char  EMULATOR_EXE[] = "kyty_emulator.exe";
+constexpr char  CMD_EXE[]      = "cmd.exe";
+constexpr DWORD CMD_X_CHARS    = 175;
+constexpr DWORD CMD_Y_CHARS    = 1000;
 #else
 constexpr char EMULATOR_EXE[] = "kyty_emulator";
 #endif
-
-#if defined(_WIN32)
-constexpr char CMD_EXE[] = "cmd.exe";
-#elif defined(__linux__)
-constexpr char KYTY_BASH_FILE[] = "kyty_run.sh";
-#endif
-#if defined(_WIN32)
-constexpr DWORD CMD_X_CHARS = 175;
-constexpr DWORD CMD_Y_CHARS = 1000;
-#endif
 constexpr char SETTINGS_MAIN_DIALOG[]        = "MainDialog";
 constexpr char SETTINGS_MAIN_LAST_GEOMETRY[] = "geometry";
 constexpr char SETTINGS_CHECK_UPDATES[]      = "check_updates_on_startup";
+constexpr char SETTINGS_CLOSE_TERMINAL[]     = "close_terminal";
 constexpr char SETTINGS_THEME[]              = "theme";
 
 class MainDialogPrivate: public QObject {
@@ -89,6 +81,7 @@ public:
 private:
 	static QByteArray g_last_geometry;
 	static bool       g_check_updates_on_startup;
+	static bool       g_close_terminal;
 	static int        g_theme;
 
 	static void ApplyTheme(int theme) {
@@ -108,13 +101,12 @@ private:
 	TitleSound      m_title_sound;
 	QString         m_interpreter;
 
-	QProcess m_process;
-
-	QPointer<ConfigurationItem> m_running_item;
+	int m_running_games = 0;
 };
 
 QByteArray MainDialogPrivate::g_last_geometry;
 bool       MainDialogPrivate::g_check_updates_on_startup = true;
+bool       MainDialogPrivate::g_close_terminal           = false;
 int        MainDialogPrivate::g_theme                    = 0;
 
 MainDialog::MainDialog(QWidget* parent): QDialog(parent), m_p(new MainDialogPrivate(this)) {
@@ -122,7 +114,6 @@ MainDialog::MainDialog(QWidget* parent): QDialog(parent), m_p(new MainDialogPriv
 }
 
 MainDialogPrivate::~MainDialogPrivate() {
-	m_process.disconnect(this);
 	delete m_ui;
 }
 
@@ -134,10 +125,14 @@ void MainDialogPrivate::Setup(MainDialog* main_dialog) {
 	m_main_dialog = main_dialog;
 	m_update_checker = new UpdateChecker(main_dialog);
 	m_ui->check_updates_on_startup->setChecked(g_check_updates_on_startup);
+	m_ui->close_terminal->setChecked(g_close_terminal);
 	m_ui->theme_combo->setCurrentIndex(g_theme);
 	ApplyTheme(g_theme);
 	m_ui->check_updates_link->setVisible(UpdateChecker::IsSupported());
 	m_ui->check_updates_on_startup->setVisible(UpdateChecker::IsSupported());
+#if defined(__APPLE__)
+	m_ui->close_terminal->setVisible(false); // The emulator runs without a terminal.
+#endif
 
 	main_dialog->setWindowFlags(Qt::Dialog /*| Qt::MSWindowsFixedSizeDialogHint*/);
 
@@ -146,13 +141,12 @@ void MainDialogPrivate::Setup(MainDialog* main_dialog) {
 	connect(m_ui->widget, &ConfigurationListWidget::Select, this, [this]() {
 		Update();
 		const auto* item = m_ui->widget->GetSelectedItem();
-		m_title_sound.SetTitle(m_process.state() == QProcess::NotRunning && item != nullptr
-		                           ? item->GetInfo().basedir
-		                           : QString());
+		m_title_sound.SetTitle(m_running_games == 0 && item != nullptr ? item->GetInfo().basedir
+		                                                               : QString());
 	});
 	connect(m_ui->widget, &ConfigurationListWidget::PreviewControllerColor, this,
 	        [this](const QString& color) {
-		        if (m_process.state() == QProcess::NotRunning) {
+		        if (m_running_games == 0) {
 			        m_controller_preview.SetColor(color);
 		        }
 	        });
@@ -166,6 +160,10 @@ void MainDialogPrivate::Setup(MainDialog* main_dialog) {
 		g_check_updates_on_startup = checked;
 		m_ui->widget->WriteSettings();
 	});
+	connect(m_ui->close_terminal, &QCheckBox::toggled, this, [this](bool checked) {
+		g_close_terminal = checked;
+		m_ui->widget->WriteSettings();
+	});
 	connect(m_ui->theme_combo, &QComboBox::currentIndexChanged, this, [this](int index) {
 		g_theme = index;
 		ApplyTheme(index);
@@ -176,16 +174,6 @@ void MainDialogPrivate::Setup(MainDialog* main_dialog) {
 		m_ui->widget->WriteSettings();
 	});
 
-	connect(&m_process, &QProcess::stateChanged, this, [this](QProcess::ProcessState state) {
-		m_update_checker->SetGameRunning(state != QProcess::NotRunning);
-		if (state == QProcess::NotRunning) {
-			if (m_running_item != nullptr) {
-				m_running_item->SetRunning(false);
-			}
-			Update();
-		}
-	});
-
 	m_ui->label_settings_file->setText(tr("Settings file: ") + m_ui->widget->GetSettingsFile());
 
 	m_main_dialog->restoreGeometry(g_last_geometry);
@@ -287,6 +275,8 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	AddFlag(args, info.skip_notice_screen, "--skip-notice-screen");
 	AddFlag(args, info.network_connected, "--network-connected");
 	AddFlag(args, info.tessellation_enabled, "--tessellation");
+	AddFlag(args, info.occlusion_queries_enabled, "--occlusion-queries");
+	AddFlag(args, info.shader_cache_enabled, "--shader-cache");
 	args << "--vblank-frequency" << QString::number(info.vblank_frequency);
 	args << "--console-language" << QString::number(info.console_language);
 	AddFlag(args, info.vulkan_validation_enabled, "--vulkan-validation");
@@ -316,6 +306,9 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	if (!info.addons_dir.isEmpty()) {
 		args << "--addcont-dir" << info.addons_dir;
 	}
+	for (const auto& entry: info.addcont) {
+		args << "--addcont" << entry;
+	}
 
 	const auto patch_plan = Cheats::PlanPath(info.title_id);
 	if (QFileInfo::exists(patch_plan)) {
@@ -326,32 +319,14 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 }
 
 #ifdef __linux__
-static QString BashQuote(QString value) {
-	value.replace('\'', "'\\''");
-	return QStringLiteral("'") + value + QStringLiteral("'");
-}
-
-static bool CreateBashScript(const QString& interpreter, const QStringList& args,
-                             const QString& file_name) {
-	QFile file(file_name);
-	if (file.open(QIODevice::WriteOnly | QIODevice::Text)) {
-		QTextStream s(&file);
-
-		s << "#!/bin/bash\n";
-		s << BashQuote(interpreter);
-		for (const auto& arg: args) {
-			s << " " << BashQuote(arg);
-		}
-		s << "\n";
-		s << "echo Press any key...\n";
-		s << "read -n1\n";
-
-		file.close();
-
-		return file.setPermissions(file.permissions() | QFile::ExeUser | QFile::ExeOwner |
-		                           QFile::ExeGroup);
-	}
-	return false;
+// Runs the command in bash, then waits for a key (with close_on_clean_exit, only after an error).
+// The command goes in as positional parameters ("$@"), so it needs no quoting.
+static QStringList BashCommand(const QString& interpreter, const QStringList& args,
+                               bool close_on_clean_exit) {
+	const QString run = close_on_clean_exit ? "\"$@\" && exit" : "\"$@\"";
+	return QStringList {"bash", "-c", run + "; echo Press any key...; read -n1", "kyty",
+	                    interpreter} +
+	       args;
 }
 
 // Find a terminal and its command separator.
@@ -415,28 +390,23 @@ static bool FindTerminal(QString* program, QStringList* prefix) {
 #endif
 
 #if defined(_WIN32)
-// Quote one token for cmd.exe so paths with spaces survive /K parsing.
-static QString WinCmdQuote(QString value) {
-	value.replace(QLatin1Char('"'), QStringLiteral("\\\""));
-	return QLatin1Char('"') + value + QLatin1Char('"');
-}
-
-static QString BuildWinCmdKCommand(const QString& interpreter, const QStringList& args) {
-	QString command = WinCmdQuote(QDir::toNativeSeparators(interpreter));
-	for (const auto& arg: args) {
-		command += QLatin1Char(' ');
-		command += WinCmdQuote(arg);
-	}
-	return command;
+// cmd /K arguments that keep the console open (with close_on_clean_exit, only after an error).
+// A quote is written as "" rather than \" so cmd's quote count stays even and operators inside
+// arguments stay quoted.
+static QString CmdCommand(const QString& interpreter, const QStringList& args,
+                          bool close_on_clean_exit) {
+	static const QRegularExpression quote_or_end(QStringLiteral(R"((\\*)("|\z))"));
+	// The UCRT reads 2n backslashes before a quote as n.
+	auto quoted = QStringList {interpreter} + args;
+	quoted.replaceInStrings(quote_or_end, QStringLiteral(R"(\1\1\2\2)"));
+	return QStringLiteral("/K \"\"") + quoted.join(QStringLiteral("\" \"")) +
+	       (close_on_clean_exit ? "\" && exit\"" : "\"\"");
 }
 #endif
 
-void MainDialog::RunInterpreter(QProcess* process, const Configuration& info) {
+void MainDialog::RunInterpreter(QProcess* process, const Configuration& info, bool close_terminal) {
 	const auto& interpreter = m_p->GetInterpreter();
 
-	QFileInfo f(interpreter);
-	auto      dir = f.absoluteDir();
-
 	auto args = CreateEmulatorArgs(info);
 	if (args.isEmpty()) {
 		QMessageBox::critical(this, tr("Error"), tr("Invalid emulator configuration"));
@@ -444,40 +414,24 @@ void MainDialog::RunInterpreter(QProcess* process, const Configuration& info) {
 		return;
 	}
 
-#ifdef __linux__
-	auto bash_file_name = dir.filePath(KYTY_BASH_FILE);
-	if (!CreateBashScript(interpreter, args, bash_file_name)) {
-		QMessageBox::critical(this, tr("Error"), tr("Can't create file:\n") + bash_file_name);
-		QApplication::quit();
-		return;
-	}
-
-	{
-		QString     terminal;
-		QStringList terminal_prefix;
-		// Pass the script as a file argument (not bash -c) so paths with spaces work.
-		if (FindTerminal(&terminal, &terminal_prefix)) {
-			process->setProgram(terminal);
-			process->setArguments(terminal_prefix + QStringList {"bash", bash_file_name});
-		} else {
-			// Run without a terminal as a fallback.
-			process->setProgram(QStringLiteral("bash"));
-			process->setArguments({bash_file_name});
-		}
-	}
-#elif defined(_WIN32)
-	{
-		// Use nativeArguments so Qt does not re-quote the /K command string.
-		process->setProgram(CMD_EXE);
-		process->setArguments({});
-		process->setNativeArguments(QStringLiteral("/K \"") +
-		                            BuildWinCmdKCommand(interpreter, args) + QLatin1Char('"'));
-	}
-#else
+	// Without a terminal, the emulator runs directly.
 	process->setProgram(interpreter);
 	process->setArguments(args);
+#ifdef __linux__
+	QString     terminal;
+	QStringList terminal_prefix;
+	if (FindTerminal(&terminal, &terminal_prefix)) {
+		process->setProgram(terminal);
+		process->setArguments(terminal_prefix + BashCommand(interpreter, args, close_terminal));
+	}
+#elif defined(_WIN32)
+	// Use nativeArguments so Qt does not re-quote the /K command string.
+	process->setProgram(CMD_EXE);
+	process->setArguments({});
+	process->setNativeArguments(
+	    CmdCommand(QDir::toNativeSeparators(interpreter), args, close_terminal));
 #endif
-	process->setWorkingDirectory(dir.path());
+	process->setWorkingDirectory(QFileInfo(interpreter).absolutePath());
 #if defined(_WIN32)
 	process->setCreateProcessArgumentsModifier([](QProcess::CreateProcessArguments* args) {
 		args->flags |= static_cast<uint32_t>(CREATE_NEW_CONSOLE);
@@ -524,6 +478,7 @@ void MainDialogPrivate::WriteSettings(QSettings& s) {
 		s.setValue(SETTINGS_MAIN_LAST_GEOMETRY, g_last_geometry);
 	}
 	s.setValue(SETTINGS_CHECK_UPDATES, g_check_updates_on_startup);
+	s.setValue(SETTINGS_CLOSE_TERMINAL, g_close_terminal);
 	s.setValue(SETTINGS_THEME, g_theme);
 
 	s.endGroup();
@@ -534,6 +489,7 @@ void MainDialogPrivate::ReadSettings(QSettings& s) {
 
 	g_last_geometry = s.value(SETTINGS_MAIN_LAST_GEOMETRY, g_last_geometry).toByteArray();
 	g_check_updates_on_startup = s.value(SETTINGS_CHECK_UPDATES, true).toBool();
+	g_close_terminal           = s.value(SETTINGS_CLOSE_TERMINAL, false).toBool();
 	g_theme                    = s.value(SETTINGS_THEME, 0).toInt();
 	if (g_theme < 0 || g_theme > 2) {
 		g_theme = 0;
@@ -546,17 +502,30 @@ void MainDialogPrivate::Run() {
 	if (m_update_checker->IsInstalling()) {
 		return;
 	}
-	m_running_item = m_ui->widget->GetSelectedItem();
-	if (m_running_item == nullptr) {
+	QPointer<ConfigurationItem> item = m_ui->widget->GetSelectedItem();
+	if (item == nullptr) {
 		return;
 	}
 
-	m_running_item->SetRunning(true);
+	item->SetRunning(true);
+	m_running_games++;
 	m_controller_preview.SetSuspended(true);
 	m_title_sound.SetTitle({});
 
-	auto info = m_ui->widget->CreateConfiguration(*m_running_item);
-	m_main_dialog->RunInterpreter(&m_process, *info);
+	auto* process = new QProcess(this);
+	connect(process, &QProcess::stateChanged, this,
+	        [this, process, item](QProcess::ProcessState state) {
+		        if (state == QProcess::NotRunning) {
+			        if (item != nullptr) {
+				        item->SetRunning(false);
+			        }
+			        m_running_games--;
+			        process->deleteLater();
+			        Update();
+		        }
+	        });
+	auto info = m_ui->widget->CreateConfiguration(*item);
+	m_main_dialog->RunInterpreter(process, *info, g_close_terminal);
 
 	Update();
 }
@@ -564,8 +533,7 @@ void MainDialogPrivate::Run() {
 void MainDialogPrivate::Update() {
 	const auto* item = m_ui->widget->GetSelectedItem();
 
-	bool run_enabled = (m_process.state() == QProcess::NotRunning && item != nullptr &&
-	                    !m_update_checker->IsInstalling());
+	bool run_enabled = item != nullptr && !item->IsRunning() && !m_update_checker->IsInstalling();
 
 	if (run_enabled) {
 		const auto& info = item->GetInfo();
@@ -574,7 +542,8 @@ void MainDialogPrivate::Update() {
 	}
 
 	m_ui->widget->SetRunEnabled(run_enabled);
-	if (m_process.state() != QProcess::NotRunning) {
+	m_update_checker->SetGameRunning(m_running_games > 0);
+	if (m_running_games > 0) {
 		m_controller_preview.SetSuspended(true);
 		return;
 	}

src/launcher/styles/launcher.qss

@@ -1,32 +1,24 @@
+/* The views get their transparent background from their own styleSheet in
+   configuration_list_widget.ui: Breeze paints black behind the scrollbars of views without one. */
+QTreeWidget#cfgs_list,
 QListView#cfgs_grid {
-    background: palette(base);
-    border: 1px solid palette(mid);
-    color: palette(text);
-}
-
-QListView#cfgs_grid::item {
-    padding: 8px;
-}
-
-GameListTreeWidget#cfgs_list {
-    background: transparent;
     border: 1px solid palette(mid);
     color: palette(text);
     outline: 0;
 }
 
-GameListTreeWidget#cfgs_list::item {
+QTreeWidget#cfgs_list::item {
     background: transparent;
     border-bottom: 1px solid palette(mid);
     padding: 4px;
 }
 
-GameListTreeWidget#cfgs_list::item:selected {
+QTreeWidget#cfgs_list::item:selected {
     background: palette(highlight);
     color: palette(highlighted-text);
 }
 
-GameListTreeWidget#cfgs_list QHeaderView::section {
+QTreeWidget#cfgs_list QHeaderView::section {
     background: palette(button);
     color: palette(button-text);
     border: 0;

src/libs/agc.cpp

@@ -2027,7 +2027,7 @@ uint32_t* KYTY_SYSV_ABI AgcCbReleaseMem(CommandBuffer* buf, uint8_t action, uint
 	const uint32_t packet_action      = static_cast<uint32_t>(action) & 0x3fu;
 	const uint32_t packet_event_index = (action >= 0x2fu ? 6u : 5u);
 
-	cmd[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
+	cmd[0] = KYTY_PM4(8, Pm4::IT_RELEASE_MEM, 0);
 	cmd[1] = packet_action | (packet_event_index << 8u) | ((packet_gcr_cntl & 0xfffu) << 12u) |
 	         ((static_cast<uint32_t>(cache_policy) & 0x3u) << 25u);
 	cmd[2] = ((static_cast<uint32_t>(dst) & 0x3u) << 16u) |
@@ -3510,13 +3510,7 @@ int KYTY_SYSV_ABI AgcSetPacketPredication(uint32_t* packet, uint32_t predication
 int KYTY_SYSV_ABI AgcSetNop(uint32_t* packet) {
 	PRINT_NAME();
 
-	const auto size = AgcGetPacketSize(packet);
-
 	packet[0] = (packet[0] & 0xffff0003u) | (Pm4::IT_NOP << 8u);
-	// Prevent the disabled payload from being treated as an executable marker.
-	if (size > 1 && (packet[1] & 0xffff0000u) == 0x68750000u) {
-		packet[1] &= 0xffffu;
-	}
 
 	return OK;
 }
@@ -3778,27 +3772,21 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchAddress(uint32_t*             cmd,
                                                       const volatile Label* address) {
 	PRINT_NAME();
 
-	// Not sure
-
 	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
 	         "\t address = 0x%016" PRIx64 "\n",
 	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
-	auto vaddr = reinterpret_cast<uint64_t>(address);
-	auto op    = (cmd[0] >> 8u) & 0xffu;
+	if (((cmd[0] >> 8u) & 0xffu) != Pm4::IT_RELEASE_MEM) {
+		return GRAPHICS5_ERROR_INVALID_PACKET;
+	}
 
-	if ((op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM) ||
-	    op == Pm4::IT_RELEASE_MEM) {
-		cmd[3] = static_cast<uint32_t>(vaddr & 0xffffffffu);
-		cmd[4] = static_cast<uint32_t>((vaddr >> 32u) & 0xffffffffu);
-	} else if (op == Pm4::IT_EVENT_WRITE_EOP) {
-		cmd[2] = static_cast<uint32_t>(vaddr & 0xffffffffu);
-		cmd[3] = (cmd[3] & 0xffff0000u) | static_cast<uint32_t>((vaddr >> 32u) & 0xffffu);
-	} else {
-		EXIT("unsupported queueEndOfPipeAction packet for address patch: 0x%08" PRIx32 "\n",
-		     cmd[0]);
+	// A kSendIntCtx release has no address.
+	if (((cmd[2] >> 24u) & 0x7u) != 4u) {
+		const auto vaddr = reinterpret_cast<uint64_t>(address);
+		cmd[3]           = (cmd[3] & 0x3u) | (static_cast<uint32_t>(vaddr) & ~0x3u);
+		cmd[4]           = static_cast<uint32_t>(vaddr >> 32u);
 	}
 
 	return OK;
@@ -3813,10 +3801,7 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchData(uint32_t* cmd, uint64_t data)
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
-	const auto op             = (cmd[0] >> 8u) & 0xffu;
-	const bool is_release_mem = op == Pm4::IT_RELEASE_MEM ||
-	                            (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM);
-	if (!is_release_mem) {
+	if (((cmd[0] >> 8u) & 0xffu) != Pm4::IT_RELEASE_MEM) {
 		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 
@@ -4115,7 +4100,7 @@ static TessellationDriverState g_tessellation_driver_state {};
 static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) {
 	GraphicsDbgDumpDcb("d", size_in_dwords, dcb);
 	EXIT_IF(g_renderer == nullptr);
-	g_renderer->GetGpu().Submit(std::span {dcb, size_in_dwords}, {});
+	g_renderer->GetGpu().Submit(std::span {dcb, size_in_dwords});
 }
 
 int KYTY_SYSV_ABI AgcDriverSubmitDcb(const Packet* packet) {

src/libs/libC.cpp

@@ -771,10 +771,10 @@ LIB_DEFINE(InitLibcInternal_1) {
 	LIB_FUNC("Q3VBxCXhUHs", LibcInternal::memcpy);
 	LIB_FUNC("+P6FRGH4LfA", LibcInternal::memmove);
 	LIB_FUNC("DfivPArhucg", LibcInternal::memcmp);
-	LIB_FUNC("aesyjrHVWy4", LibcInternal::strcmp);
-	LIB_FUNC("Ovb2dSJOAuE", LibcInternal::strncmp);
+	LIB_FUNC("Ovb2dSJOAuE", LibcInternal::strcmp);
+	LIB_FUNC("aesyjrHVWy4", LibcInternal::strncmp);
 	LIB_FUNC("j4ViWNHEgww", LibcInternal::strlen);
-	LIB_FUNC("5Xa2ACNECdo", LibcInternal::strcpy);
+	LIB_FUNC("kiZSXIWd9vg", LibcInternal::strcpy);
 	LIB_FUNC("6sJWiWSRuqk", LibcInternal::strncpy);
 	LIB_FUNC("Ls4tzzhimqQ", LibcInternal::strcat);
 	LIB_FUNC("ob5xAW4ln-0", LibcInternal::strchr);

src/libs/libDeviceService.cpp

@@ -16,6 +16,14 @@ static int KYTY_SYSV_ABI DeviceServiceInitialize() {
 	return OK;
 }
 
+static int KYTY_SYSV_ABI DeviceServiceGetEventState(int32_t device_class) {
+	PRINT_NAME();
+
+	LOGF("\t device_class = %" PRId32 "\n", device_class);
+
+	return 0;
+}
+
 static int KYTY_SYSV_ABI DeviceServiceQueryDeviceInfo(uint32_t device_handle, void* out_info) {
 	PRINT_NAME();
 
@@ -30,6 +38,7 @@ static int KYTY_SYSV_ABI DeviceServiceQueryDeviceInfo(uint32_t device_handle, vo
 
 LIB_DEFINE(InitDeviceService_1) {
 	LIB_FUNC("84fDxStrG44", DeviceService::DeviceServiceInitialize);
+	LIB_FUNC("9ddRUOV8Q5A", DeviceService::DeviceServiceGetEventState);
 	LIB_FUNC("UNMEa+5lrUA", DeviceService::DeviceServiceQueryDeviceInfo);
 }
 

src/libs/libJpegDec.cpp

@@ -0,0 +1,402 @@
+#include "common/abi.h"
+#include "common/logging/log.h"
+#include "libs/libs.h"
+#include "loader/symbolDatabase.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdlib>
+
+// The emulator's stb_image implementation. Guest JPEG data reaches it here, and zeroed allocations
+// keep a scan that uses a Huffman table no DHT defined (an MJPEG frame) from reading uninitialised
+// tables.
+#define STB_IMAGE_IMPLEMENTATION
+#define STBI_NO_SIMD
+#define STBI_MALLOC(size)           calloc(1, size)
+#define STBI_REALLOC(pointer, size) realloc(pointer, size)
+#define STBI_FREE(pointer)          free(pointer)
+#include "stb_image.h"
+
+namespace Libs {
+
+LIB_VERSION("JpegDec", 1, "JpegDec", 1, 1);
+
+namespace JpegDec {
+
+constexpr int32_t JPEG_DEC_ERROR_INVALID_ADDR        = static_cast<int32_t>(0x80650001u);
+constexpr int32_t JPEG_DEC_ERROR_INVALID_SIZE        = static_cast<int32_t>(0x80650002u);
+constexpr int32_t JPEG_DEC_ERROR_INVALID_PARAM       = static_cast<int32_t>(0x80650003u);
+constexpr int32_t JPEG_DEC_ERROR_INVALID_HANDLE      = static_cast<int32_t>(0x80650004u);
+constexpr int32_t JPEG_DEC_ERROR_INVALID_COEF_MEMORY = static_cast<int32_t>(0x80650005u);
+constexpr int32_t JPEG_DEC_ERROR_INVALID_WORK_MEMORY = static_cast<int32_t>(0x80650006u);
+constexpr int32_t JPEG_DEC_ERROR_INVALID_DATA        = static_cast<int32_t>(0x80650010u);
+constexpr int32_t JPEG_DEC_ERROR_UNSUPPORT_DATA      = static_cast<int32_t>(0x80650011u);
+constexpr int32_t JPEG_DEC_ERROR_DECODE_ERROR        = static_cast<int32_t>(0x80650012u);
+
+constexpr uint32_t JPEG_DEC_ATTRIBUTE_CSC_STD_SAMPLING  = 1;
+constexpr uint32_t JPEG_DEC_ATTRIBUTE_CSC_FULL_SAMPLING = 2;
+constexpr uint16_t JPEG_DEC_DECODE_MODE_MJPEG           = 1;
+constexpr uint16_t JPEG_DEC_PIXEL_FORMAT_R8G8B8A8       = 0;
+constexpr uint16_t JPEG_DEC_PIXEL_FORMAT_B8G8R8A8       = 1;
+constexpr uint16_t JPEG_DEC_PIXEL_FORMAT_Y8U8Y8V8       = 10;
+constexpr uint16_t JPEG_DEC_PIXEL_FORMAT_Y8             = 11;
+constexpr uint16_t JPEG_DEC_COLOR_SPACE_UNKNOWN         = 0;
+constexpr uint16_t JPEG_DEC_COLOR_SPACE_YCC             = 1;
+constexpr uint16_t JPEG_DEC_COLOR_SPACE_GRAYSCALE       = 2;
+
+struct JpegDecCreateParam {
+	uint32_t this_size;
+	uint32_t attribute;
+	uint32_t max_image_width;
+};
+
+struct JpegDecDecodeParam {
+	const void* jpeg_mem_addr;
+	void*       image_mem_addr;
+	void*       coefficient_mem_addr;
+	uint32_t    jpeg_mem_size;
+	uint32_t    image_mem_size;
+	uint32_t    coefficient_mem_size;
+	uint16_t    decode_mode;
+	uint16_t    down_scale;
+	uint16_t    pixel_format;
+	uint16_t    alpha_value;
+	uint32_t    image_pitch;
+};
+
+struct JpegDecParseParam {
+	const void* jpeg_mem_addr;
+	uint32_t    jpeg_mem_size;
+	uint16_t    decode_mode;
+	uint16_t    down_scale;
+};
+
+struct JpegDecImageInfo {
+	uint32_t image_width;
+	uint32_t image_height;
+	uint16_t color_space;
+	uint16_t component_number;
+	uint8_t  component_id[4];
+	uint8_t  sampling_factor[4];
+	uint32_t coefficient_mem_size;
+	uint32_t suitable_csc_attribute;
+	uint32_t output_image_width;
+	uint32_t output_image_height;
+};
+
+static_assert(sizeof(JpegDecCreateParam) == 12 && sizeof(JpegDecDecodeParam) == 48);
+static_assert(sizeof(JpegDecParseParam) == 16 && sizeof(JpegDecImageInfo) == 36);
+
+struct JpegDecContext {
+	uint64_t magic;
+	uint32_t attribute;
+	uint32_t max_image_width;
+};
+
+constexpr uint64_t JPEG_DEC_CONTEXT_MAGIC = 0x4b5954594a504744ull; // KYTYJPGD
+
+static int32_t CheckCreateParam(const JpegDecCreateParam* param) {
+	if (param == nullptr) {
+		return JPEG_DEC_ERROR_INVALID_ADDR;
+	}
+	if (param->this_size != sizeof(JpegDecCreateParam)) {
+		return JPEG_DEC_ERROR_INVALID_SIZE;
+	}
+	if ((param->attribute != JPEG_DEC_ATTRIBUTE_CSC_STD_SAMPLING &&
+	     param->attribute != JPEG_DEC_ATTRIBUTE_CSC_FULL_SAMPLING) ||
+	    param->max_image_width == 0 || param->max_image_width > 0xffffu) {
+		return JPEG_DEC_ERROR_INVALID_PARAM;
+	}
+	return 0;
+}
+
+static bool CheckScale(uint16_t decode_mode, uint16_t down_scale) {
+	return decode_mode <= JPEG_DEC_DECODE_MODE_MJPEG &&
+	       (down_scale == 1 || down_scale == 2 || down_scale == 4 || down_scale == 8);
+}
+
+// Fills the image information from the frame header (T.81 B.2.2) that precedes the first scan.
+static int32_t ParseHeader(const void* jpeg, uint32_t size, uint16_t down_scale,
+                           JpegDecImageInfo* info) {
+	const auto* data = static_cast<const uint8_t*>(jpeg);
+	if (size < 4 || data[0] != 0xff || data[1] != 0xd8) {
+		return JPEG_DEC_ERROR_INVALID_DATA;
+	}
+	for (uint32_t offset = 2; offset + 4 <= size;) {
+		const uint8_t marker = data[offset + 1];
+		if (data[offset] != 0xff || marker == 0xda || marker == 0xd9) {
+			return JPEG_DEC_ERROR_INVALID_DATA;
+		}
+		if (marker == 0xff) {
+			offset++;
+			continue;
+		}
+		const uint32_t length  = (data[offset + 2] << 8u) | data[offset + 3];
+		const auto*    segment = data + offset + 4;
+		if (length < 2 || offset + 2 + length > size) {
+			return JPEG_DEC_ERROR_INVALID_DATA;
+		}
+		offset += 2 + length;
+		// SOF0-SOF15 except DHT, JPG and DAC.
+		if (marker < 0xc0 || marker > 0xcf || marker == 0xc4 || marker == 0xc8 || marker == 0xcc) {
+			continue;
+		}
+		const uint32_t components = length < 8 ? 0 : segment[5];
+		if (components == 0 || length < 8 + components * 3 || (segment[3] | segment[4]) == 0) {
+			return JPEG_DEC_ERROR_INVALID_DATA;
+		}
+		*info              = {};
+		info->image_height = (segment[1] << 8u) | segment[2];
+		info->image_width  = (segment[3] << 8u) | segment[4];
+		// Baseline, extended and progressive Huffman 8-bit frames with a known height.
+		if (marker > 0xc2 || segment[0] != 8 || components > 4 || info->image_height == 0) {
+			return JPEG_DEC_ERROR_UNSUPPORT_DATA;
+		}
+		uint32_t h_max = 1;
+		uint32_t v_max = 1;
+		for (uint32_t i = 0; i < components; i++) {
+			const uint8_t factor = segment[7 + i * 3];
+			if ((factor >> 4u) - 1u > 3u || (factor & 0xfu) - 1u > 3u) {
+				return JPEG_DEC_ERROR_INVALID_DATA;
+			}
+			info->component_id[i]    = segment[6 + i * 3];
+			info->sampling_factor[i] = factor;
+			h_max                    = std::max<uint32_t>(h_max, factor >> 4u);
+			v_max                    = std::max<uint32_t>(v_max, factor & 0xfu);
+		}
+		info->component_number = components;
+		if (marker == 0xc2) {
+			// Progressive scans keep every 16-bit DCT coefficient of the MCU-padded image.
+			const uint32_t mcu_x = (info->image_width + h_max * 8 - 1) / (h_max * 8);
+			const uint32_t mcu_y = (info->image_height + v_max * 8 - 1) / (v_max * 8);
+			for (uint32_t i = 0; i < components; i++) {
+				info->coefficient_mem_size += mcu_x * (info->sampling_factor[i] >> 4u) * mcu_y *
+				                              (info->sampling_factor[i] & 0xfu) * 128u;
+			}
+		}
+		switch (components) {
+			case 1: info->color_space = JPEG_DEC_COLOR_SPACE_GRAYSCALE; break;
+			case 3: info->color_space = JPEG_DEC_COLOR_SPACE_YCC; break;
+			default: info->color_space = JPEG_DEC_COLOR_SPACE_UNKNOWN; break;
+		}
+		// Standard sampling: grayscale, or 1x1 chroma with luma at most 2x2 (4:4:4/4:2:2/4:2:0).
+		const bool standard =
+		    components == 1 || (info->sampling_factor[1] == 0x11 &&
+			                    info->sampling_factor[2] == 0x11 && h_max <= 2 && v_max <= 2);
+		info->suitable_csc_attribute =
+		    standard ? JPEG_DEC_ATTRIBUTE_CSC_STD_SAMPLING : JPEG_DEC_ATTRIBUTE_CSC_FULL_SAMPLING;
+		info->output_image_width  = (info->image_width + down_scale - 1) / down_scale;
+		info->output_image_height = (info->image_height + down_scale - 1) / down_scale;
+		return 0;
+	}
+	return JPEG_DEC_ERROR_INVALID_DATA;
+}
+
+static uint8_t Luma(const uint8_t* p) {
+	return static_cast<uint8_t>((77 * p[0] + 150 * p[1] + 29 * p[2] + 128) >> 8);
+}
+
+// JFIF chroma of the average of two pixels, rounding half down so a saturated pair stays 255.
+static uint8_t Chroma(const uint8_t* p, const uint8_t* q, int r, int g, int b) {
+	return static_cast<uint8_t>(
+	    (r * (p[0] + q[0]) + g * (p[1] + q[1]) + b * (p[2] + q[2]) + 65791) >> 9);
+}
+
+static int32_t KYTY_SYSV_ABI JpegDecQueryMemorySize(const JpegDecCreateParam* param) {
+	PRINT_NAME();
+	const auto result = CheckCreateParam(param);
+	return result != 0 ? result : static_cast<int32_t>(sizeof(JpegDecContext));
+}
+
+static int32_t KYTY_SYSV_ABI JpegDecCreate(const JpegDecCreateParam* param, void* memory_address,
+                                           uint32_t memory_size, void** handle) {
+	PRINT_NAME();
+	if (const auto result = CheckCreateParam(param); result != 0) {
+		return result;
+	}
+	if (memory_address == nullptr || handle == nullptr) {
+		return JPEG_DEC_ERROR_INVALID_ADDR;
+	}
+	if (memory_size < sizeof(JpegDecContext)) {
+		return JPEG_DEC_ERROR_INVALID_SIZE;
+	}
+	*static_cast<JpegDecContext*>(memory_address) = {JPEG_DEC_CONTEXT_MAGIC, param->attribute,
+	                                                 param->max_image_width};
+	*handle                                       = memory_address;
+	return 0;
+}
+
+static int32_t KYTY_SYSV_ABI JpegDecDelete(void* handle) {
+	PRINT_NAME();
+	auto* ctx = static_cast<JpegDecContext*>(handle);
+	if (ctx == nullptr || ctx->magic != JPEG_DEC_CONTEXT_MAGIC) {
+		return JPEG_DEC_ERROR_INVALID_HANDLE;
+	}
+	ctx->magic = 0;
+	return 0;
+}
+
+static int32_t KYTY_SYSV_ABI JpegDecParseHeader(const JpegDecParseParam* param,
+                                                JpegDecImageInfo*        image_info) {
+	PRINT_NAME();
+	if (param == nullptr || image_info == nullptr || param->jpeg_mem_addr == nullptr) {
+		return JPEG_DEC_ERROR_INVALID_ADDR;
+	}
+	if (param->jpeg_mem_size == 0) {
+		return JPEG_DEC_ERROR_INVALID_SIZE;
+	}
+	if (!CheckScale(param->decode_mode, param->down_scale)) {
+		return JPEG_DEC_ERROR_INVALID_PARAM;
+	}
+	return ParseHeader(param->jpeg_mem_addr, param->jpeg_mem_size, param->down_scale, image_info);
+}
+
+static int32_t KYTY_SYSV_ABI JpegDecDecode(void* handle, const JpegDecDecodeParam* param,
+                                           JpegDecImageInfo* image_info) {
+	PRINT_NAME();
+	const auto* ctx = static_cast<const JpegDecContext*>(handle);
+	if (ctx == nullptr || ctx->magic != JPEG_DEC_CONTEXT_MAGIC) {
+		return JPEG_DEC_ERROR_INVALID_HANDLE;
+	}
+	if (param == nullptr || param->jpeg_mem_addr == nullptr || param->image_mem_addr == nullptr ||
+	    (param->pixel_format != JPEG_DEC_PIXEL_FORMAT_Y8 &&
+	     (reinterpret_cast<uintptr_t>(param->image_mem_addr) & 3u) != 0)) {
+		return JPEG_DEC_ERROR_INVALID_ADDR;
+	}
+	const auto format = param->pixel_format;
+	LOGF("\t jpeg_mem_size = %" PRIu32 ", image_mem_size = %" PRIu32 ", mode = %" PRIu16
+	     ", down_scale = %" PRIu16 ", pixel_format = %" PRIu16 ", image_pitch = %" PRIu32 "\n",
+	     param->jpeg_mem_size, param->image_mem_size, param->decode_mode, param->down_scale, format,
+	     param->image_pitch);
+	if (param->jpeg_mem_size == 0 || param->image_mem_size == 0) {
+		return JPEG_DEC_ERROR_INVALID_SIZE;
+	}
+	if (!CheckScale(param->decode_mode, param->down_scale) || param->alpha_value > 255 ||
+	    (format != JPEG_DEC_PIXEL_FORMAT_R8G8B8A8 && format != JPEG_DEC_PIXEL_FORMAT_B8G8R8A8 &&
+	     format != JPEG_DEC_PIXEL_FORMAT_Y8U8Y8V8 && format != JPEG_DEC_PIXEL_FORMAT_Y8) ||
+	    (format != JPEG_DEC_PIXEL_FORMAT_Y8 && (param->image_pitch & 3u) != 0)) {
+		return JPEG_DEC_ERROR_INVALID_PARAM;
+	}
+
+	JpegDecImageInfo info {};
+	if (const auto result =
+	        ParseHeader(param->jpeg_mem_addr, param->jpeg_mem_size, param->down_scale, &info);
+	    result != 0) {
+		return result;
+	}
+	if (info.color_space == JPEG_DEC_COLOR_SPACE_UNKNOWN) {
+		return JPEG_DEC_ERROR_UNSUPPORT_DATA;
+	}
+	if (info.output_image_width > ctx->max_image_width ||
+	    info.suitable_csc_attribute > ctx->attribute) {
+		return JPEG_DEC_ERROR_INVALID_WORK_MEMORY;
+	}
+	if (info.coefficient_mem_size > param->coefficient_mem_size ||
+	    (info.coefficient_mem_size != 0 && param->coefficient_mem_addr == nullptr)) {
+		return JPEG_DEC_ERROR_INVALID_COEF_MEMORY;
+	}
+	const uint32_t width  = info.output_image_width;
+	const uint32_t height = info.output_image_height;
+	const bool     large  = width > 32767 || height > 32767;
+	if (large && image_info == nullptr) {
+		return JPEG_DEC_ERROR_INVALID_ADDR;
+	}
+	uint32_t row = width * 4;
+	if (format == JPEG_DEC_PIXEL_FORMAT_Y8U8Y8V8) {
+		row = (width + 1) / 2 * 4;
+	} else if (format == JPEG_DEC_PIXEL_FORMAT_Y8) {
+		row = width;
+	}
+	const uint32_t pitch = param->image_pitch == 0 ? row : param->image_pitch;
+	if (pitch < row) {
+		return JPEG_DEC_ERROR_INVALID_PARAM;
+	}
+	if (static_cast<uint64_t>(pitch) * height > param->image_mem_size) {
+		return JPEG_DEC_ERROR_INVALID_SIZE;
+	}
+
+	int   src_width  = 0;
+	int   src_height = 0;
+	auto* rgb        = stbi_load_from_memory(static_cast<const stbi_uc*>(param->jpeg_mem_addr),
+	                                         static_cast<int>(param->jpeg_mem_size), &src_width,
+	                                         &src_height, nullptr, 3);
+	if (rgb == nullptr || static_cast<uint32_t>(src_width) != info.image_width ||
+	    static_cast<uint32_t>(src_height) != info.image_height) {
+		stbi_image_free(rgb);
+		return JPEG_DEC_ERROR_DECODE_ERROR;
+	}
+	// Each output pixel averages its down_scale x down_scale source block, packed in place: an
+	// output pixel never lands past a source pixel still to be read.
+	const uint32_t scale = param->down_scale;
+	if (scale > 1) {
+		auto* out = rgb;
+		for (uint32_t y = 0; y < height; y++) {
+			for (uint32_t x = 0; x < width; x++) {
+				const uint32_t x_end  = std::min((x + 1) * scale, info.image_width);
+				const uint32_t y_end  = std::min((y + 1) * scale, info.image_height);
+				const uint32_t count  = (x_end - x * scale) * (y_end - y * scale);
+				uint32_t       sum[3] = {};
+				for (uint32_t sy = y * scale; sy < y_end; sy++) {
+					for (uint32_t sx = x * scale; sx < x_end; sx++) {
+						const auto* p = rgb + (static_cast<size_t>(sy) * info.image_width + sx) * 3;
+						sum[0] += p[0];
+						sum[1] += p[1];
+						sum[2] += p[2];
+					}
+				}
+				for (const uint32_t c: sum) {
+					*out++ = static_cast<uint8_t>((c + count / 2) / count);
+				}
+			}
+		}
+	}
+	const auto  alpha = static_cast<uint8_t>(param->alpha_value);
+	const auto* p     = rgb;
+	for (uint32_t y = 0; y < height; y++) {
+		auto* dst = static_cast<uint8_t*>(param->image_mem_addr) + static_cast<size_t>(y) * pitch;
+		for (uint32_t x = 0; x < width; x++, p += 3) {
+			switch (format) {
+				case JPEG_DEC_PIXEL_FORMAT_R8G8B8A8:
+					dst    = std::copy_n(p, 3, dst);
+					*dst++ = alpha;
+					break;
+				case JPEG_DEC_PIXEL_FORMAT_B8G8R8A8:
+					*dst++ = p[2];
+					*dst++ = p[1];
+					*dst++ = p[0];
+					*dst++ = alpha;
+					break;
+				case JPEG_DEC_PIXEL_FORMAT_Y8: *dst++ = Luma(p); break;
+				default:
+					if (x % 2 == 0) {
+						// An odd width repeats the last pixel.
+						const auto* q = x + 1 < width ? p + 3 : p;
+						*dst++        = Luma(p);
+						*dst++        = Chroma(p, q, -43, -85, 128);
+						*dst++        = Luma(q);
+						*dst++        = Chroma(p, q, 128, -107, -21);
+					}
+					break;
+			}
+		}
+	}
+	stbi_image_free(rgb);
+	if (image_info != nullptr) {
+		*image_info = info;
+	}
+	return large ? 0 : static_cast<int32_t>((width << 16u) | height);
+}
+
+} // namespace JpegDec
+
+LIB_DEFINE(InitJpegDec_1) {
+	PRINT_NAME_ENABLE(true);
+
+	LIB_FUNC("uNAUmANZMEw", JpegDec::JpegDecQueryMemorySize);
+	LIB_FUNC("JPh3Zgg0Zwc", JpegDec::JpegDecCreate);
+	LIB_FUNC("1kzQRoWEgSA", JpegDec::JpegDecDecode);
+	LIB_FUNC("Hwh11+m5KoI", JpegDec::JpegDecDelete);
+	LIB_FUNC("LSinoSQH790", JpegDec::JpegDecParseHeader);
+}
+
+} // namespace Libs