libc: isolate guest setenv from host environment. Fix linux debugging

Co-authored-by: Techx3 <56268269+Techx3@users.noreply.github.com>

.vscode/launch.json

@@ -10,7 +10,14 @@
             "stopAtEntry": false,
             "cwd": "${workspaceFolder}\\_Build\\vscode-clang\\launcher",
             "environment": [],
-            "externalConsole": false
+            "externalConsole": false,
+            "linux": {
+                "type": "cppdbg",
+                "program": "${workspaceFolder}/_Build/vscode-clang/launcher/launcher",
+                "cwd": "${workspaceFolder}/_Build/vscode-clang/launcher",
+                "MIMode": "gdb",
+                "miDebuggerPath": "/usr/bin/gdb"
+            }
         },
         {
             "name": "Debug kyty_emulator",
@@ -25,13 +32,40 @@
             "stopAtEntry": false,
             "cwd": "${workspaceFolder}\\_Build\\vscode-clang",
             "environment": [],
-            "externalConsole": true
+            "externalConsole": true,
+            "linux": {
+                "type": "cppdbg",
+                "program": "${workspaceFolder}/_Build/vscode-clang/kyty_emulator",
+                "cwd": "${workspaceFolder}/_Build/vscode-clang",
+                "externalConsole": false,
+                "MIMode": "gdb",
+                "miDebuggerPath": "/usr/bin/gdb",
+                "setupCommands": [
+                    {
+                        "description": "Pass Kyty's internal signals to its handlers",
+                        "text": "handle SIG37 SIGSEGV SIGBUS SIGILL nostop noprint pass",
+                        "ignoreFailures": false
+                    }
+                ]
+            }
         },
         {
             "name": "Attach to running kyty_emulator",
             "type": "cppvsdbg",
             "request": "attach",
-            "processId": "${command:pickProcess}"
+            "processId": "${command:pickProcess}",
+            "linux": {
+                "type": "cppdbg",
+                "MIMode": "gdb",
+                "miDebuggerPath": "/usr/bin/gdb",
+                "setupCommands": [
+                    {
+                        "description": "Pass Kyty's internal signals to its handlers",
+                        "text": "handle SIG37 SIGSEGV SIGBUS SIGILL nostop noprint pass",
+                        "ignoreFailures": false
+                    }
+                ]
+            }
         }
     ]
 }

CMakeLists.txt

@@ -254,7 +254,7 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 	add_custom_command(
 		OUTPUT "${shader_header}"
 		COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}"
-		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
 		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
@@ -263,6 +263,22 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 	)
 	list(APPEND gpu_tiler_shader_headers "${shader_header}")
 endforeach()
+
+set(fault_buffer_shader_source "${gpu_tiler_shader_dir}/fault_buffer_process.comp")
+set(fault_buffer_shader_spv "${gpu_tiler_generated_dir}/fault_buffer_process.spv")
+set(fault_buffer_shader_header "${gpu_tiler_generated_dir}/fault_buffer_process_spv.h")
+add_custom_command(
+	OUTPUT "${fault_buffer_shader_header}"
+	COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}"
+	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
+	        -DCACHING_PAGEBITS=14 -DMAX_PAGE_FAULTS=1024
+	        -o "${fault_buffer_shader_spv}" "${fault_buffer_shader_source}"
+	COMMAND ${CMAKE_COMMAND} -DINPUT=${fault_buffer_shader_spv} -DOUTPUT=${fault_buffer_shader_header}
+	        -DSYMBOL=FAULT_BUFFER_PROCESS_SPV -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
+	DEPENDS "${fault_buffer_shader_source}"
+	VERBATIM
+)
+list(APPEND kyty_emulator_src "${fault_buffer_shader_header}")
 list(APPEND kyty_emulator_src ${gpu_tiler_shader_headers})
 
 set(gpu_blit_generated_dir "${PROJECT_BINARY_DIR}/gpu_blit_shaders")
@@ -278,7 +294,7 @@ foreach(shader_source IN LISTS gpu_blit_shader_sources)
 	add_custom_command(
 		OUTPUT "${shader_header}"
 		COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_blit_generated_dir}"
-		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
 		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
@@ -430,6 +446,7 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderPixelParameter.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
@@ -483,7 +500,7 @@ set(gpu_test_ms_depth_header
 add_custom_command(
 	OUTPUT "${gpu_test_ms_depth_header}"
 	COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_test_generated_dir}"
-	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 	        -o "${gpu_test_ms_depth_spv}" "${gpu_test_ms_depth_source}"
 	COMMAND ${CMAKE_COMMAND} -DINPUT=${gpu_test_ms_depth_spv}
 	        -DOUTPUT=${gpu_test_ms_depth_header} -DSYMBOL=GPU_TEST_MS_DEPTH_SPV
@@ -507,6 +524,16 @@ if(NOT KYTY_CLANG_CL)
 endif()
 
 if(BUILD_TESTING)
+	add_test(NAME emulator_user_name_cli
+	         COMMAND $<TARGET_FILE:kyty_emulator> --help --user-name 1234567890123456)
+	add_test(NAME emulator_user_name_too_long
+	         COMMAND $<TARGET_FILE:kyty_emulator> --help --user-name 12345678901234567)
+	set_tests_properties(emulator_user_name_too_long PROPERTIES WILL_FAIL TRUE)
+	add_test(NAME emulator_present_mode_cli
+	         COMMAND $<TARGET_FILE:kyty_emulator> --help --present-mode Immediate)
+	add_test(NAME emulator_present_mode_invalid
+	         COMMAND $<TARGET_FILE:kyty_emulator> --help --present-mode Invalid)
+	set_tests_properties(emulator_present_mode_invalid PROPERTIES WILL_FAIL TRUE)
 	add_test(NAME ime_dialog COMMAND $<TARGET_FILE:ime_dialog_tests>)
 	add_test(NAME shader_cfg COMMAND $<TARGET_FILE:shader_cfg_tests>)
 	add_test(NAME scalar_provenance COMMAND $<TARGET_FILE:scalar_provenance_tests>)

src/common/emulatorConfig.cpp

@@ -21,6 +21,7 @@ void Shutdown() {
 
 void Load(const ConfigOptions& cfg) {
 	EXIT_IF(g_config == nullptr);
+	EXIT_IF(cfg.user_name.empty() || cfg.user_name.size() > MAX_USER_NAME_LENGTH);
 
 	*g_config = cfg;
 }
@@ -33,6 +34,14 @@ uint32_t GetScreenHeight() {
 	return g_config->screen_height;
 }
 
+const std::string& GetUserName() {
+	return g_config->user_name;
+}
+
+PresentMode GetPresentMode() {
+	return g_config->present_mode;
+}
+
 bool FullscreenEnabled() {
 	return g_config->fullscreen_enabled;
 }
@@ -93,6 +102,10 @@ bool SpirvDebugPrintfEnabled() {
 	return g_config->spirv_debug_printf_enabled;
 }
 
+bool GpuAssistedValidationEnabled() {
+	return g_config->gpu_assisted_validation_enabled && g_config->vulkan_validation_enabled;
+}
+
 bool RenderDocEnabled() {
 	return g_config->renderdoc_enabled;
 }

src/common/emulatorConfig.h

@@ -3,6 +3,7 @@
 
 #include "common/common.h"
 
+#include <cstddef>
 #include <filesystem>
 #include <string>
 #include <vector>
@@ -26,14 +27,19 @@ enum class ProfilerDirection { None, Network };
 
 enum class OutputDirection { Silent, Console, File };
 
+enum class PresentMode { Fifo, Mailbox, Immediate };
+
 using Keymap = std::vector<std::string>;
 
 constexpr uint32_t DEFAULT_CONSOLE_LANGUAGE = 1;
 constexpr uint32_t MAX_CONSOLE_LANGUAGE     = 29;
+constexpr std::size_t MAX_USER_NAME_LENGTH = 16;
 
 struct ConfigOptions {
 	uint32_t               screen_width                = 1280;
 	uint32_t               screen_height               = 720;
+	std::string            user_name                   = "Kyty";
+	PresentMode            present_mode                = PresentMode::Fifo;
 	bool                   fullscreen_enabled          = false;
 	uint32_t               vblank_frequency            = 60;
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
@@ -49,6 +55,7 @@ struct ConfigOptions {
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   spirv_debug_printf_enabled  = false;
+	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
 	bool                   playgo_hack_enabled         = false;
@@ -62,6 +69,8 @@ void Load(const ConfigOptions& cfg);
 
 uint32_t GetScreenWidth();
 uint32_t GetScreenHeight();
+const std::string& GetUserName();
+PresentMode GetPresentMode();
 bool     FullscreenEnabled();
 uint32_t GetVblankFrequency();
 uint32_t GetConsoleLanguage();
@@ -84,6 +93,8 @@ ProfilerDirection GetProfilerDirection();
 
 bool SpirvDebugPrintfEnabled();
 
+bool GpuAssistedValidationEnabled();
+
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
 bool PlayGoHackEnabled();

src/common/slotVector.h

@@ -0,0 +1,132 @@
+#ifndef EMULATOR_SRC_COMMON_SLOTVECTOR_H_
+#define EMULATOR_SRC_COMMON_SLOTVECTOR_H_
+
+#include "common/assert.h"
+#include "common/abi.h"
+
+#include <compare>
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <limits>
+#include <optional>
+#include <utility>
+#include <vector>
+
+namespace Common {
+
+struct SlotId {
+	static constexpr uint32_t INVALID_INDEX = std::numeric_limits<uint32_t>::max();
+
+	constexpr SlotId() noexcept = default;
+	constexpr SlotId(uint32_t value) noexcept: index(value), generation(1) {}
+	constexpr SlotId(uint32_t value, uint32_t slot_generation) noexcept
+	    : index(value), generation(slot_generation) {}
+
+	[[nodiscard]] constexpr explicit operator bool() const noexcept { return index != INVALID_INDEX; }
+	constexpr auto operator<=>(const SlotId&) const noexcept = default;
+
+	uint32_t index = INVALID_INDEX;
+	uint32_t generation = 0;
+};
+
+// Stable-address slot storage for cache resources that are intentionally non-movable.
+template <typename T>
+class SlotVector {
+public:
+	SlotVector() = default;
+	KYTY_CLASS_NO_COPY(SlotVector);
+
+	[[nodiscard]] T& operator[](SlotId id) noexcept {
+		EXIT_IF(!is_allocated(id));
+		return *m_values[id.index].value;
+	}
+
+	[[nodiscard]] const T& operator[](SlotId id) const noexcept {
+		EXIT_IF(!is_allocated(id));
+		return *m_values[id.index].value;
+	}
+
+	[[nodiscard]] T* try_get(SlotId id) noexcept {
+		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+	}
+
+	[[nodiscard]] const T* try_get(SlotId id) const noexcept {
+		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+	}
+
+	[[nodiscard]] bool is_allocated(SlotId id) const noexcept {
+		return id && id.index < m_values.size() &&
+		       m_values[id.index].generation == id.generation && m_values[id.index].value.has_value();
+	}
+
+	template <typename... Args>
+	[[nodiscard]] SlotId insert(Args&&... args) {
+		uint32_t index = 0;
+		if (m_free_list.empty()) {
+			index = static_cast<uint32_t>(m_values.size());
+			m_values.emplace_back();
+			m_values.back().value.emplace(std::forward<Args>(args)...);
+		} else {
+			index = m_free_list.back();
+			m_free_list.pop_back();
+			EXIT_IF(m_values[index].value.has_value());
+			m_values[index].value.emplace(std::forward<Args>(args)...);
+		}
+		++m_size;
+		return SlotId {index, m_values[index].generation};
+	}
+
+	void erase(SlotId id) noexcept {
+		EXIT_IF(!is_allocated(id));
+		auto& slot = m_values[id.index];
+		slot.value.reset();
+		if (++slot.generation == 0) {
+			slot.generation = 1;
+		}
+		m_free_list.push_back(id.index);
+		--m_size;
+	}
+
+	[[nodiscard]] size_t size() const noexcept { return m_size; }
+	[[nodiscard]] size_t capacity() const noexcept { return m_values.size(); }
+
+	template <typename F>
+	void ForEach(F&& fn) {
+		for (uint32_t index = 0; index < m_values.size(); ++index) {
+			if (m_values[index].value) {
+				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+			}
+		}
+	}
+
+	template <typename F>
+	void ForEach(F&& fn) const {
+		for (uint32_t index = 0; index < m_values.size(); ++index) {
+			if (m_values[index].value) {
+				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+			}
+		}
+	}
+
+private:
+	struct Slot {
+		std::optional<T> value;
+		uint32_t         generation = 1;
+	};
+
+	std::deque<Slot>             m_values;
+	std::vector<uint32_t>        m_free_list;
+	size_t                       m_size = 0;
+};
+
+} // namespace Common
+
+template <>
+struct std::hash<Common::SlotId> {
+	[[nodiscard]] size_t operator()(Common::SlotId id) const noexcept {
+		return std::hash<uint64_t> {}((static_cast<uint64_t>(id.generation) << 32u) | id.index);
+	}
+};
+
+#endif // EMULATOR_SRC_COMMON_SLOTVECTOR_H_

src/emulator.cpp

@@ -15,6 +15,7 @@
 #include "kernel/fileSystem.h"
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
+#include "kytyGitVersion.h"
 #include "libs/agc.h"
 #include "libs/audio.h"
 #include "libs/controller.h"
@@ -26,13 +27,31 @@
 
 #include <cstdlib>
 #include <filesystem>
+#include <thread>
 
 namespace Emulator {
 
 static void PrintSystemInfo() {
-	Common::SystemInfo info = Common::GetSystemInfo();
+	const Common::SystemInfo info = Common::GetSystemInfo();
 
-	LOGF("ProcessorName = %s\n", info.ProcessorName.c_str());
+#if defined(__APPLE__)
+	static constexpr auto platform_name = "macOS";
+#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	static constexpr auto platform_name = "Windows";
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	static constexpr auto platform_name = "Linux";
+#else
+	static constexpr auto platform_name = "Unknown";
+#endif
+
+	LOGF("Build\n"
+	     "  version: %s\n\n"
+	     "Host\n"
+	     "  os:      %s\n"
+	     "  cpu:     %s\n"
+	     "  threads: %u\n\n",
+	     KYTY_BUILD_LABEL, platform_name, info.ProcessorName.c_str(),
+	     std::thread::hardware_concurrency());
 }
 
 static void KytyClose() {

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -7,6 +7,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 
 #include <cstdint>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -30,10 +31,9 @@ private:
 	friend class CommandProcessor;
 
 	struct BufferCursor {
-		uint32_t* next_packet         = nullptr;
-		uint32_t  remaining_dw        = 0;
-		uint32_t  total_dw            = 0;
-		uint32_t  deferred_advance_dw = 0;
+		std::span<const uint32_t> commands;
+		uint32_t                  offset_dw           = 0;
+		uint32_t                  deferred_advance_dw = 0;
 	};
 
 	std::vector<BufferCursor> m_buffer_stack;
@@ -50,7 +50,8 @@ public:
 		int64_t flip_arg  = 0;
 	};
 
-	explicit CommandProcessor(RenderContext& renderer): m_renderer(renderer) {}
+	CommandProcessor(RenderContext& renderer, int interrupt_event_id)
+	    : m_renderer(renderer), m_interrupt_event_id(interrupt_event_id) {}
 	~CommandProcessor() = default;
 
 	KYTY_CLASS_NO_COPY(CommandProcessor);
@@ -103,7 +104,7 @@ public:
 	                    uint32_t mode);
 	void DispatchIndirect(uint32_t data_offset, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
-	void TriggerEvent(uint32_t event_type, uint32_t event_index);
+	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);
 
 	void SetUserDataMarker(HW::UserSgprType type) { m_user_data_marker = type; }
 	[[nodiscard]] HW::UserSgprType GetUserDataMarker() const { return m_user_data_marker; }
@@ -131,13 +132,14 @@ public:
 	                    const volatile void* address, uint32_t count_in_dwords);
 	[[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; }
 
-	Pm4ProcessResult Process(Pm4Execution& execution, uint32_t* buffer, uint32_t size_dw);
-	void             ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw);
+	Pm4ProcessResult Process(Pm4Execution& execution, std::span<const uint32_t> commands);
+	void             ProcessIndirectBuffer(std::span<const uint32_t> commands);
 
 	void SetFlip(const FlipInfo& flip) { m_flip = flip; }
 
 	[[nodiscard]] uint64_t GetSubmitId() const { return m_submit_id; }
 	void                   SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; }
+	[[nodiscard]] bool     IsAsyncComputeQueue() const { return m_interrupt_event_id >= 0x20; }
 
 private:
 	template <typename T>
@@ -151,10 +153,10 @@ private:
 	                          uint32_t render_target_slice_offset, uint32_t first_vertex,
 	                          uint32_t first_instance);
 
-	CommandScheduler&    GetScheduler() const { return m_renderer.GetCommandScheduler(); }
-	RenderCommandBuffer& CurrentBuffer() { return GetScheduler().Current(); }
-	void                 CheckBuffer() const { GetScheduler().CheckActive(); }
-	GpuResourceManager&  GetGpuResources() const { return m_renderer.GetGpuResources(); }
+	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
+	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
+	void                CheckBuffer() const { GetScheduler().CheckActive(); }
+	GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); }
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;
@@ -177,9 +179,11 @@ private:
 
 	uint32_t m_const_ram[0x3000] = {0};
 
-	FlipInfo m_flip;
-	uint64_t m_submit_id      = 0;
-	bool     m_predicate_skip = false;
+	FlipInfo  m_flip;
+	const int m_interrupt_event_id;
+	uint64_t  m_submit_id                   = 0;
+	uint64_t  m_synthetic_occlusion_counter = 0;
+	bool      m_predicate_skip              = false;
 };
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -35,7 +35,6 @@ constexpr auto MakeContextDispatchTable() {
 	}
 	g_hw_ctx_func[Pm4::CB_TARGET_MASK]                 = HwCtxSetRenderTargetMask;
 	g_hw_ctx_func[Pm4::PA_SC_GENERIC_SCISSOR_TL]       = HwCtxSetGenericScissor;
-	g_hw_ctx_func[Pm4::CB_BLEND_RED]                   = HwCtxSetBlendColor;
 	g_hw_ctx_func[Pm4::CB_DCC_CONTROL]                 = HwCtxSetCbDccControl;
 	g_hw_ctx_func[Pm4::DB_STENCIL_CONTROL]             = HwCtxSetStencilControl;
 	g_hw_ctx_func[Pm4::DB_STENCILREFMASK]              = HwCtxSetStencilMask;
@@ -82,7 +81,6 @@ constexpr auto MakeContextDispatchTable() {
 	g_hw_ctx_func[Pm4::VGT_PRIMITIVEID_RESET]                 = HwCtxSetPrimitiveIdReset;
 	g_hw_ctx_func[Pm4::PA_CL_OBJPRIM_ID_CNTL]                 = HwCtxSetObjprimIdControl;
 	g_hw_ctx_func[Pm4::VGT_SHADER_STAGES_EN]                  = HwCtxSetShaderStages;
-	g_hw_ctx_func[Pm4::PA_CL_GB_VERT_CLIP_ADJ]                = HwCtxSetGuardBands;
 
 	for (uint32_t slot = 0; slot < 8; slot++) {
 		g_hw_ctx_func[Pm4::CB_COLOR0_BASE + slot * 15] = HwCtxSetRenderTarget;
@@ -217,6 +215,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_COND_EXEC]                 = CpOpCondExec;
 	g_cp_op_func[Pm4::IT_SET_PREDICATION]           = CpOpSetPredication;
 	g_cp_op_func[Pm4::IT_WRITE_DATA]                = CpOpWriteData;
+	g_cp_op_func[Pm4::IT_WAIT_REG_MEM]              = CpOpWaitRegMem32;
 	g_cp_op_func[Pm4::IT_INDIRECT_BUFFER]           = CpOpIndirectBuffer;
 	g_cp_op_func[Pm4::IT_COPY_DATA]                 = CpOpCopyData;
 	g_cp_op_func[Pm4::IT_EVENT_WRITE]               = CpOpEventWrite;
@@ -239,6 +238,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_WAIT_ON_CE_COUNTER]        = CpOpWaitOnCeCounter;
 	g_cp_op_func[Pm4::IT_WAIT_ON_DE_COUNTER_DIFF]   = CpOpWaitOnDeCounterDiff;
 	g_cp_op_func[Pm4::IT_GET_LOD_STATS]             = CpOpGetLodStats;
+	g_cp_op_func[Pm4::IT_WAIT_REG_MEM_64]           = CpOpWaitRegMem64;
 
 	return g_cp_op_func;
 }
@@ -247,12 +247,10 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	std::array<cp_op_parser_func_t, Pm4::R_NUM> g_cp_op_custom_func {};
 
 	g_cp_op_custom_func[Pm4::R_DISPATCH_RESET] = CpOpDispatchReset;
-	g_cp_op_custom_func[Pm4::R_WAIT_MEM_32]    = CpOpWaitRegMem32;
 	g_cp_op_custom_func[Pm4::R_WAIT_FLIP_DONE] = CpOpWaitFlipDone;
 	g_cp_op_custom_func[Pm4::R_PUSH_MARKER]    = CpOpPushMarker;
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_ACQUIRE_MEM]    = CpOpAcquireMem;
-	g_cp_op_custom_func[Pm4::R_WAIT_MEM_64]    = CpOpWaitRegMem64;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
 	g_cp_op_custom_func[Pm4::R_RELEASE_MEM]    = CpOpReleaseMem;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -56,7 +56,6 @@ uint32_t HwCtxSetWindowScissor(CommandProcessor&, uint32_t, uint32_t, const uint
 uint32_t HwCtxSetClipRect(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetRenderTargetMask(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetGenericScissor(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
-uint32_t HwCtxSetBlendColor(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetCbDccControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetStencilControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetStencilMask(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
@@ -90,7 +89,6 @@ uint32_t HwCtxSetDrawPayloadControl(CommandProcessor&, uint32_t, uint32_t, const
 uint32_t HwCtxSetPrimitiveIdReset(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetObjprimIdControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetShaderStages(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
-uint32_t HwCtxSetGuardBands(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetRenderTarget(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetColorInfo(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetBlendControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -71,19 +71,6 @@ constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
 	return raw_offset & ~RegisterSelectorMask;
 }
 
-// Indirect Cx descriptors retain their selector. Selector 1 offsets 0..31 address the
-// SPI_PS_INPUT_CNTL register bank; ordinary context-register offsets remain unchanged.
-constexpr uint32_t DecodeIndirectCxRegisterOffset(uint32_t raw_offset) {
-	const auto offset = NormalizeRegisterOffset(raw_offset);
-	return (raw_offset & RegisterSelectorMask) == Pm4::CX_PS_SHADER_USAGE_BASE && offset < 32u
-	           ? Pm4::SPI_PS_INPUT_CNTL_0 + offset
-	           : offset;
-}
-
-static_assert(DecodeIndirectCxRegisterOffset(Pm4::CX_PS_SHADER_USAGE_BASE + 2u) ==
-              Pm4::SPI_PS_INPUT_CNTL_0 + 2u);
-static_assert(DecodeIndirectCxRegisterOffset(Pm4::DB_Z_INFO) == Pm4::DB_Z_INFO);
-
 bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) {
 	return eop_event_type != 0x28u ||
 	       (gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate |
@@ -530,22 +517,6 @@ KYTY_HW_CTX_PARSER(HwCtxSetAaMask) {
 	return num_values;
 }
 
-KYTY_HW_CTX_PARSER(HwCtxSetBlendColor) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0046900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::CB_BLEND_RED);
-
-	HW::BlendColor r;
-
-	r.red   = *reinterpret_cast<const float*>(&buffer[0]);
-	r.green = *reinterpret_cast<const float*>(&buffer[1]);
-	r.blue  = *reinterpret_cast<const float*>(&buffer[2]);
-	r.alpha = *reinterpret_cast<const float*>(&buffer[3]);
-
-	cp.GetCtx().SetBlendColor(r);
-
-	return 4;
-}
-
 KYTY_HW_CTX_PARSER(HwCtxSetBlendControl) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900);
 
@@ -859,20 +830,6 @@ KYTY_HW_CTX_PARSER(HwCtxSetClipRect) {
 	return value_count;
 }
 
-KYTY_HW_CTX_PARSER(HwCtxSetGuardBands) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0046900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_CL_GB_VERT_CLIP_ADJ);
-
-	auto vert_clip    = *reinterpret_cast<const float*>(&buffer[0]); // PA_CL_GB_VERT_CLIP_ADJ
-	auto vert_discard = *reinterpret_cast<const float*>(&buffer[1]); // PA_CL_GB_VERT_DISC_ADJ
-	auto horz_clip    = *reinterpret_cast<const float*>(&buffer[2]); // PA_CL_GB_HORZ_CLIP_ADJ
-	auto horz_discard = *reinterpret_cast<const float*>(&buffer[3]); // PA_CL_GB_HORZ_DISC_ADJ
-
-	cp.GetCtx().SetGuardBands(horz_clip, vert_clip, horz_discard, vert_discard);
-
-	return 4;
-}
-
 KYTY_HW_CTX_PARSER(HwCtxSetHardwareScreenOffset) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900);
 	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SU_HARDWARE_SCREEN_OFFSET);
@@ -1248,23 +1205,16 @@ KYTY_HW_CTX_PARSER(HwCtxSetStencilInfo) {
 }
 
 KYTY_HW_CTX_PARSER(HwCtxSetStencilMask) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0026900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCILREFMASK);
+	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
+	EXIT_NOT_IMPLEMENTED(num_values == 0);
+	EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::DB_STENCILREFMASK);
+	EXIT_NOT_IMPLEMENTED(cmd_offset + num_values - 1u > Pm4::DB_STENCILREFMASK_BF);
 
-	HW::StencilMask r;
+	for (uint32_t i = 0; i < num_values; i++) {
+		g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, buffer[i]);
+	}
 
-	r.stencil_testval      = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILTESTVAL);
-	r.stencil_mask         = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILMASK);
-	r.stencil_writemask    = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILWRITEMASK);
-	r.stencil_opval        = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILOPVAL);
-	r.stencil_testval_bf   = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILTESTVAL_BF);
-	r.stencil_mask_bf      = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILMASK_BF);
-	r.stencil_writemask_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILWRITEMASK_BF);
-	r.stencil_opval_bf     = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILOPVAL_BF);
-
-	cp.GetCtx().SetStencilMask(r);
-
-	return 2;
+	return num_values;
 }
 
 KYTY_HW_CTX_PARSER(HwCtxSetViewportScaleOffset) {
@@ -1367,10 +1317,6 @@ static void HwShSetCsRegister(CommandProcessor& cp, uint32_t cmd_offset, uint32_
 			                         Pm4::COMPUTE_PGM_RSRC1_BULKY_MASK) != 0u;
 			cs_regs.threadgroup_configuration = ((value >> Pm4::COMPUTE_PGM_RSRC1_WGP_MODE_SHIFT) &
 			                                     Pm4::COMPUTE_PGM_RSRC1_WGP_MODE_MASK) != 0u;
-			cs_regs.wave_size                 = (((value >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) &
-			                                      Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u)
-			                                        ? 32u
-			                                        : 64u;
 			break;
 		case Pm4::COMPUTE_PGM_RSRC2:
 			cs_regs.scratch_en     = ((value >> Pm4::COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT) &
@@ -1913,11 +1859,11 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	uint64_t reference   = buffer[5] | (static_cast<uint64_t>(buffer[6]) << 32u);
 	uint32_t mode        = buffer[0] & 0x3u;
 	uint32_t function    = (buffer[0] >> 8u) & 0x7u;
-	auto*    then_buffer = reinterpret_cast<uint32_t*>((buffer[7] & 0xfffffffcu) |
-	                                                   (static_cast<uint64_t>(buffer[8]) << 32u));
+	auto*    then_buffer = reinterpret_cast<const uint32_t*>(
+	    (buffer[7] & 0xfffffffcu) | (static_cast<uint64_t>(buffer[8]) << 32u));
 	uint32_t then_num_dw = buffer[9] & 0xfffffu;
-	auto*    else_buffer = reinterpret_cast<uint32_t*>((buffer[10] & 0xfffffffcu) |
-	                                                   (static_cast<uint64_t>(buffer[11]) << 32u));
+	auto*    else_buffer = reinterpret_cast<const uint32_t*>(
+	    (buffer[10] & 0xfffffffcu) | (static_cast<uint64_t>(buffer[11]) << 32u));
 	uint32_t else_num_dw = buffer[12] & 0xfffffu;
 
 	EXIT_NOT_IMPLEMENTED(compare_addr == nullptr);
@@ -1931,10 +1877,10 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	     reinterpret_cast<uint64_t>(else_buffer), else_num_dw);
 
 	if (take_then) {
-		cp.ProcessIndirectBuffer(then_buffer, then_num_dw);
+		cp.ProcessIndirectBuffer({then_buffer, then_num_dw});
 	} else if (mode == 2 && else_num_dw != 0) {
 		EXIT_NOT_IMPLEMENTED(else_buffer == nullptr);
-		cp.ProcessIndirectBuffer(else_buffer, else_num_dw);
+		cp.ProcessIndirectBuffer({else_buffer, else_num_dw});
 	}
 
 	return payload_dw;
@@ -2201,12 +2147,19 @@ KYTY_CP_OP_PARSER(CpOpEventWrite) {
 
 	EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_EVENT_WRITE);
 
-	uint32_t event_index = (buffer[0] >> 8u) & 0x7u;
-	uint32_t event_type  = (buffer[0]) & 0x3fu;
+	const auto packet_size_dw = KYTY_PM4_LEN(cmd_id);
+	uint32_t   event_index    = (buffer[0] >> 8u) & 0x7u;
+	uint32_t   event_type     = (buffer[0]) & 0x3fu;
+	uint64_t   event_address  = 0;
 
-	cp.TriggerEvent(event_type, event_index);
+	if (event_type == 0x39u) {
+		EXIT_NOT_IMPLEMENTED(packet_size_dw != 4u);
+		event_address = buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u);
+	}
 
-	return KYTY_PM4_LEN(cmd_id) - 1u;
+	cp.TriggerEvent(event_type, event_index, event_address);
+
+	return packet_size_dw - 1u;
 }
 
 KYTY_CP_OP_PARSER(CpOpEventWriteEop) {
@@ -2368,7 +2321,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 	}
 
 	auto* indirect_buffer =
-	    reinterpret_cast<uint32_t*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
+	    reinterpret_cast<const uint32_t*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
 	uint32_t                     indirect_num_dw = control & 0xfffffu;
 	static std::atomic<uint32_t> indirect_log_count {0};
 	if (indirect_log_count.fetch_add(1) < 128) {
@@ -2388,7 +2341,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 
 	GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer);
 
-	cp.ProcessIndirectBuffer(indirect_buffer, indirect_num_dw);
+	cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw});
 
 	return 3;
 }
@@ -2411,10 +2364,10 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
 		EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
 	}
 	for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
-		// Keep the encoded offset for packet control values, and use the decoded offset only
-		// for register dispatch.
+		// Keep the encoded offset for packet control values, and use the normalized offset
+		// only for register dispatch.
 		auto raw_cmd_offset = indirect_buffer[0];
-		auto cmd_offset     = DecodeIndirectCxRegisterOffset(raw_cmd_offset);
+		auto cmd_offset     = NormalizeRegisterOffset(raw_cmd_offset);
 		auto value          = indirect_buffer[1];
 
 		if (HwCtxTrySetFakeRegister(cmd_offset, value)) {
@@ -2864,6 +2817,20 @@ KYTY_CP_OP_PARSER(CpOpSetShaderReg) {
 	auto pfunc = g_hw_sh_func[cmd_offset];
 
 	if (pfunc == nullptr) {
+		auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
+		bool handled    = num_values != 0;
+		for (uint32_t i = 0; i < num_values; i++) {
+			if (cmd_offset + i >= Pm4::SH_NUM || g_hw_sh_indirect_func[cmd_offset + i] == nullptr) {
+				handled = false;
+				break;
+			}
+		}
+		if (handled) {
+			for (uint32_t i = 0; i < num_values; i++) {
+				g_hw_sh_indirect_func[cmd_offset + i](cp, cmd_offset + i, buffer[1 + i]);
+			}
+			return num_values + 1u;
+		}
 		EXIT("unknown shader register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32
 		     "\n\tcmd_offset = %08" PRIx32 "\n",
 		     num_dw - dw, cmd_id, cmd_offset);
@@ -2877,6 +2844,10 @@ KYTY_CP_OP_PARSER(CpOpSetShaderReg) {
 KYTY_CP_OP_PARSER(CpOpSetUconfigReg) {
 	KYTY_PROFILER_FUNCTION();
 
+	if (Gen5::AgcIsInternalDataPacket(cmd_id, buffer)) {
+		return KYTY_PM4_LEN(cmd_id) - 1u;
+	}
+
 	auto raw_cmd_offset = buffer[0];
 	auto cmd_offset     = NormalizeRegisterOffset(raw_cmd_offset);
 
@@ -2967,19 +2938,21 @@ template <typename T>
 static uint32_t CpOpWaitRegMemSized(CommandProcessor& cp, uint32_t cmd_id, const uint32_t* buffer) {
 	static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
 
-	constexpr auto value_dw   = static_cast<uint32_t>(sizeof(T) / sizeof(uint32_t));
+	constexpr auto value_dw = static_cast<uint32_t>(sizeof(T) / sizeof(uint32_t));
 	constexpr auto payload_dw = 4u + value_dw * 2u;
 	constexpr auto packet_dw  = payload_dw + 1u;
-	constexpr auto register_id =
-	    (sizeof(T) == sizeof(uint32_t) ? Pm4::R_WAIT_MEM_32 : Pm4::R_WAIT_MEM_64);
+	constexpr auto opcode =
+	    (sizeof(T) == sizeof(uint32_t) ? Pm4::IT_WAIT_REG_MEM : Pm4::IT_WAIT_REG_MEM_64);
 
-	EXIT_NOT_IMPLEMENTED(KYTY_PM4_R(cmd_id) != register_id);
+	EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != opcode || KYTY_PM4_R(cmd_id) != 0u);
 	EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != packet_dw);
 
-	auto* addr = reinterpret_cast<const T*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
-	auto  mask = CpOpWaitRegMemReadValue<T>(buffer + 2u);
-	auto  ref  = CpOpWaitRegMemReadValue<T>(buffer + 2u + value_dw);
-	auto  ctrl = buffer[2u + value_dw * 2u];
+	constexpr auto address_align_mask = (sizeof(T) == sizeof(uint32_t) ? 0x3u : 0x7u);
+	auto  ctrl = buffer[0];
+	auto* addr = reinterpret_cast<const T*>((buffer[1] & ~address_align_mask) |
+	                                        (static_cast<uint64_t>(buffer[2] & 0x3ffffu) << 32u));
+	auto  ref  = CpOpWaitRegMemReadValue<T>(buffer + 3u);
+	auto  mask = CpOpWaitRegMemReadValue<T>(buffer + 3u + value_dw);
 	auto  poll = buffer[3u + value_dw * 2u];
 
 	EXIT_NOT_IMPLEMENTED((ctrl & 0x10u) == 0);

src/graphics/guest_gpu/gpu_format.cpp

@@ -9,79 +9,80 @@ namespace Libs::Graphics::Prospero {
 namespace {
 
 struct FormatInfo {
-	BufferFormat format;
-	uint32_t     bytes_per_element;
-	uint32_t     block_compressed_bytes_per_block;
-	uint32_t     render_target_bytes_per_element;
-	bool         supported_sampled_texture_format;
-	bool         unsigned_integer_texture_format;
+	BufferFormat       format;
+	uint32_t           bytes_per_element;
+	uint32_t           block_compressed_bytes_per_block;
+	uint32_t           render_target_bytes_per_element;
+	bool               sampled_texture;
+	bool               uint_texture;
 };
 
 constexpr FormatInfo kFormatInfo[] = {
-    {BufferFormat::k8UNorm, 1, 0, 1, true, false},
-    {BufferFormat::k8SNorm, 0, 0, 1, false, false},
-    {BufferFormat::k8UInt, 1, 0, 1, true, true},
-    {BufferFormat::k16UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k16SNorm, 2, 0, 2, true, false},
-    {BufferFormat::k16UInt, 2, 0, 2, true, true},
-    {BufferFormat::k16SInt, 2, 0, 2, false, false},
-    {BufferFormat::k16Float, 2, 0, 2, true, false},
-    {BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
-    {BufferFormat::k8_8UInt, 2, 0, 2, true, true},
-    {BufferFormat::k8_8SInt, 2, 0, 2, false, false},
-    {BufferFormat::k32UInt, 4, 0, 4, true, true},
-    {BufferFormat::k32SInt, 4, 0, 4, false, false},
-    {BufferFormat::k32Float, 4, 0, 4, true, false},
-    {BufferFormat::k16_16UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k16_16SNorm, 4, 0, 4, true, false},
-    {BufferFormat::k16_16UInt, 4, 0, 4, true, true},
-    {BufferFormat::k16_16SInt, 4, 0, 4, false, false},
-    {BufferFormat::k16_16Float, 4, 0, 4, true, false},
-    {BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
-    {BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
-    {BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k8_8_8_8SNorm, 4, 0, 4, true, false},
-    {BufferFormat::k8_8_8_8UInt, 4, 0, 4, true, true},
-    {BufferFormat::k8_8_8_8SInt, 4, 0, 4, false, false},
-    {BufferFormat::k32_32UInt, 8, 0, 8, true, true},
-    {BufferFormat::k32_32SInt, 8, 0, 8, false, false},
-    {BufferFormat::k32_32Float, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16UNorm, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16SNorm, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16UInt, 8, 0, 8, true, true},
-    {BufferFormat::k16_16_16_16SInt, 8, 0, 8, false, false},
-    {BufferFormat::k16_16_16_16Float, 8, 0, 8, true, false},
-    {BufferFormat::k32_32_32UInt, 12, 0, 12, true, true},
-    {BufferFormat::k32_32_32SInt, 12, 0, 12, false, false},
-    {BufferFormat::k32_32_32Float, 12, 0, 12, true, false},
-    {BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
-    {BufferFormat::k32_32_32_32SInt, 16, 0, 16, false, false},
-    {BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
-    {BufferFormat::k8Srgb, 1, 0, 0, true, false},
-    {BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
-    {BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
-    {BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
-    {BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
-    {BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
-    {BufferFormat::kFmask8_S4_F4, 1, 0, 1, true, false},
-    {BufferFormat::kBc1UNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc1Srgb, 0, 8, 0, true, false},
-    {BufferFormat::kBc2UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc2Srgb, 0, 16, 0, true, false},
-    {BufferFormat::kBc3UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc3Srgb, 0, 16, 0, true, false},
-    {BufferFormat::kBc4UNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc4SNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc5UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc5SNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc6UFloat, 0, 16, 0, true, false},
-    {BufferFormat::kBc6SFloat, 0, 16, 0, true, false},
-    {BufferFormat::kBc7UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc7Srgb, 0, 16, 0, true, false},
+	{BufferFormat::k8UNorm, 1, 0, 1, true, false},
+	{BufferFormat::k8SNorm, 0, 0, 1, false, false},
+	{BufferFormat::k8UInt, 1, 0, 1, true, true},
+	{BufferFormat::k16UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k16SNorm, 2, 0, 2, true, false},
+	{BufferFormat::k16UInt, 2, 0, 2, true, true},
+	{BufferFormat::k16SInt, 2, 0, 2, false, false},
+	{BufferFormat::k16Float, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UInt, 2, 0, 2, true, true},
+	{BufferFormat::k8_8SInt, 2, 0, 2, false, false},
+	{BufferFormat::k32UInt, 4, 0, 4, true, true},
+	{BufferFormat::k32SInt, 4, 0, 4, false, false},
+	{BufferFormat::k32Float, 4, 0, 4, true, false},
+	{BufferFormat::k16_16UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k16_16SNorm, 4, 0, 4, true, false},
+	{BufferFormat::k16_16UInt, 4, 0, 4, true, true},
+	{BufferFormat::k16_16SInt, 4, 0, 4, false, false},
+	{BufferFormat::k16_16Float, 4, 0, 4, true, false},
+	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
+	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
+	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
+	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k8_8_8_8SNorm, 4, 0, 4, true, false},
+	{BufferFormat::k8_8_8_8UInt, 4, 0, 4, true, true},
+	{BufferFormat::k8_8_8_8SInt, 4, 0, 4, false, false},
+	{BufferFormat::k32_32UInt, 8, 0, 8, true, true},
+	{BufferFormat::k32_32SInt, 8, 0, 8, false, false},
+	{BufferFormat::k32_32Float, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16UNorm, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16SNorm, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16UInt, 8, 0, 8, true, true},
+	{BufferFormat::k16_16_16_16SInt, 8, 0, 8, false, false},
+	{BufferFormat::k16_16_16_16Float, 8, 0, 8, true, false},
+	{BufferFormat::k32_32_32UInt, 12, 0, 12, true, true},
+	{BufferFormat::k32_32_32SInt, 12, 0, 12, false, false},
+	{BufferFormat::k32_32_32Float, 12, 0, 12, true, false},
+	{BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
+	{BufferFormat::k32_32_32_32SInt, 16, 0, 16, false, false},
+	{BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
+	{BufferFormat::k8Srgb, 1, 0, 0, true, false},
+	{BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
+	{BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
+	{BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
+	{BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
+	{BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
+	{BufferFormat::kFmask8_S4_F4, 1, 0, 1, true, false},
+	{BufferFormat::kBc1UNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc1Srgb, 0, 8, 0, true, false},
+	{BufferFormat::kBc2UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc2Srgb, 0, 16, 0, true, false},
+	{BufferFormat::kBc3UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc3Srgb, 0, 16, 0, true, false},
+	{BufferFormat::kBc4UNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc4SNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc5UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc5SNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc6UFloat, 0, 16, 0, true, false},
+	{BufferFormat::kBc6SFloat, 0, 16, 0, true, false},
+	{BufferFormat::kBc7UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc7Srgb, 0, 16, 0, true, false},
 };
 
 constexpr uint8_t RenderTargetTypeBit(ChannelType type) {
@@ -201,18 +202,22 @@ uint32_t RenderTargetBytesPerElement(BufferFormat format) {
 	return info != nullptr ? info->render_target_bytes_per_element : 0;
 }
 
-bool IsSupportedTextureFormat(BufferFormat format) {
-	const auto* info = FindFormatInfo(format);
-	return info != nullptr && info->supported_sampled_texture_format;
-}
-
-bool IsUintTextureFormat(BufferFormat format) {
-	const auto* info = FindFormatInfo(format);
-	return info != nullptr && info->unsigned_integer_texture_format;
-}
-
 bool IsFmaskTextureFormat(BufferFormat format) {
 	return format == BufferFormat::kFmask8_S4_F4;
 }
 
+bool IsSampledTextureFormat(BufferFormat format) {
+	const auto* info = FindFormatInfo(format);
+	return info != nullptr && info->sampled_texture;
+}
+
+bool IsUintTextureFormat(BufferFormat format) {
+	const auto* info = FindFormatInfo(format);
+	return info != nullptr && info->uint_texture;
+}
+
+BufferFormat RemapTextureFormat(BufferFormat format) {
+	return format == BufferFormat::k11_11_10UInt ? BufferFormat::k32UInt : format;
+}
+
 } // namespace Libs::Graphics::Prospero

src/graphics/guest_gpu/gpu_format.h

@@ -41,9 +41,10 @@ RenderTargetFormatEncoding ResolveRenderTargetFormat(ChannelLayout layout, Chann
 uint32_t                   NumBytesPerElement(BufferFormat format);
 uint32_t                   BlockCompressedBytesPerBlock(BufferFormat format);
 uint32_t                   RenderTargetBytesPerElement(BufferFormat format);
-bool                       IsSupportedTextureFormat(BufferFormat format);
-bool                       IsUintTextureFormat(BufferFormat format);
 bool                       IsFmaskTextureFormat(BufferFormat format);
+bool                       IsSampledTextureFormat(BufferFormat format);
+bool                       IsUintTextureFormat(BufferFormat format);
+BufferFormat               RemapTextureFormat(BufferFormat format);
 
 } // namespace Libs::Graphics::Prospero
 

src/graphics/guest_gpu/graphicsRun.cpp

@@ -27,6 +27,7 @@
 #include <cstdio>
 #include <deque>
 #include <memory>
+#include <mutex>
 #include <semaphore>
 #include <thread>
 #include <vector>
@@ -65,17 +66,10 @@ static bool GraphicsRunDebugDumpEnabled() {
 	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
 }
 
-GuestGpu::OwnedCmdBuffer::OwnedCmdBuffer(const uint32_t* data, uint32_t count) {
-	EXIT_IF(data == nullptr && count != 0);
-	if (count != 0) {
-		m_words.assign(data, data + count);
-	}
-}
-
 GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	GraphicsInitJmpTables();
-	m_gfx_cp = std::make_unique<CommandProcessor>(renderer);
+	m_gfx_cp = std::make_unique<CommandProcessor>(renderer, 0);
 	m_thread = std::jthread(ThreadRun, this);
 }
 
@@ -147,39 +141,33 @@ void GuestGpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	done.acquire();
 }
 
-void GuestGpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
-                      uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
-	if (num_draw_dw == 0) {
+void GuestGpu::Submit(std::span<const uint32_t> draw_commands,
+                      std::span<const uint32_t> constant_commands) {
+	if (draw_commands.empty()) {
 		return;
 	}
-	EXIT_IF(cmd_draw_buffer == nullptr);
 	GpuMutexLock lock(m_submission_mutex);
 	Submission   submission;
-	submission.type                          = SubmissionType::Graphics;
-	submission.queue_id                      = 0;
-	submission.commands                      = OwnedCmdBuffer(cmd_draw_buffer, num_draw_dw);
-	submission.constant_commands             = OwnedCmdBuffer(cmd_const_buffer, num_const_dw);
-	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
-	submission.reset_processor               = m_graphics_done;
-	m_graphics_done                          = false;
+	submission.type              = SubmissionType::Graphics;
+	submission.queue_id          = 0;
+	submission.commands          = draw_commands;
+	submission.constant_commands = constant_commands;
+	submission.reset_processor   = m_graphics_done;
+	m_graphics_done              = false;
 	Enqueue(std::move(submission));
 }
 
-void GuestGpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
-                             bool trigger_agc_interrupt_on_done) {
-	EXIT_IF(cmd_buffer == nullptr || num_dw == 0);
+void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands) {
+	EXIT_IF(commands.empty());
 	GpuMutexLock lock(m_submission_mutex);
 
-	constexpr uint32_t compute_queue_base = 0x20u;
-	EXIT_NOT_IMPLEMENTED(queue < compute_queue_base ||
-	                     queue >= compute_queue_base + ComputeQueueCount);
+	EXIT_NOT_IMPLEMENTED(queue < ComputeQueueBase || queue >= ComputeQueueBase + ComputeQueueCount);
 
-	const auto compute_queue = queue - compute_queue_base;
+	const auto compute_queue = queue - ComputeQueueBase;
 	Submission submission;
-	submission.type                          = SubmissionType::Compute;
-	submission.queue_id                      = 1 + compute_queue;
-	submission.commands                      = OwnedCmdBuffer(cmd_buffer, num_dw);
-	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
+	submission.type     = SubmissionType::Compute;
+	submission.queue_id = 1 + compute_queue;
+	submission.commands = commands;
 	Enqueue(std::move(submission));
 }
 
@@ -199,12 +187,6 @@ void GuestGpu::Done() {
 	if (!IsGpuThread()) {
 		WaitForIdle();
 	}
-	if (RenderDocCaptureInProgress()) {
-		SendCommandSync([this] {
-			Common::LockGuard render_lock(m_renderer.GetMutex());
-			RenderDocEndCapture();
-		});
-	}
 	m_graphics_done = true;
 	m_done_num++;
 }
@@ -220,7 +202,7 @@ CommandProcessor& GuestGpu::GetProcessor(uint32_t queue_id) {
 	}
 	auto& processor = m_compute_cp[queue_id - 1];
 	if (processor == nullptr) {
-		processor = std::make_unique<CommandProcessor>(m_renderer);
+		processor = std::make_unique<CommandProcessor>(m_renderer, ComputeQueueBase + queue_id - 1);
 	}
 	return *processor;
 }
@@ -273,8 +255,7 @@ void CommandProcessor::BufferFlush() {
 }
 
 void CommandProcessor::BufferFlushAndWait() {
-	auto& submitted = GetScheduler().FlushAndGetSubmitted();
-	submitted.WaitForFenceOnly();
+	GetScheduler().FlushAndWait();
 }
 
 void CommandProcessor::BufferWait() {
@@ -359,10 +340,8 @@ template void CommandProcessor::WaitRegMem<uint64_t>(uint32_t, const uint64_t*,
 
 void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num,
                                  uint32_t write_control) {
-	const uint32_t dst_sel      = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
-	const uint32_t cache_policy = (write_control >> 25u) & 0x3u;
-	const uint32_t increment    = (write_control >> 16u) & 0x1u;
-	const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
+	const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
+	const bool     write_one_address = ((write_control >> 16u) & 0x1u) != 0;
 
 	switch (dst_sel) {
 		case 0:
@@ -372,16 +351,17 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
 		case 6: break;
 		default: EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
 	}
-	EXIT_NOT_IMPLEMENTED(increment != 0);
-
-	if (cache_policy > 3 || write_confirm > 1) {
-		LOGF("\t warning: unexpected write_data control 0x%08" PRIx32 "\n", write_control);
-	}
 	if (dw_num == 0) {
 		return;
 	}
 
-	memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
+	if (write_one_address) {
+		for (uint32_t i = 0; i < dw_num; i++) {
+			dst[0] = src[i];
+		}
+	} else {
+		memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
+	}
 }
 
 void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) {
@@ -589,21 +569,20 @@ bool GuestGpu::Process(Submission& submission) {
 	switch (submission.type) {
 		case SubmissionType::Graphics: {
 			bool progressed = false;
-			submission.constant_complete |= submission.constant_commands.Empty();
+			submission.constant_complete |= submission.constant_commands.empty();
 			for (;;) {
 				bool round_progress = false;
 				if (!submission.constant_complete) {
 					submission.constant_complete =
-					    cp.Process(
-					        submission.constant_execution, submission.constant_commands.Data(),
-					        submission.constant_commands.Size()) == Pm4ProcessResult::Complete;
+					    cp.Process(submission.constant_execution, submission.constant_commands) ==
+					    Pm4ProcessResult::Complete;
 					round_progress |= submission.constant_execution.MadeProgress();
 				}
 				cp.SetCeComplete(submission.constant_complete);
 				if (!submission.command_complete) {
 					submission.command_complete =
-					    cp.Process(submission.command_execution, submission.commands.Data(),
-					               submission.commands.Size()) == Pm4ProcessResult::Complete;
+					    cp.Process(submission.command_execution, submission.commands) ==
+					    Pm4ProcessResult::Complete;
 					round_progress |= submission.command_execution.MadeProgress();
 				}
 				progressed |= round_progress;
@@ -620,14 +599,11 @@ bool GuestGpu::Process(Submission& submission) {
 			} else if (complete) {
 				m_renderer.GetGpuResources().RunGarbageCollector();
 			}
-			if (complete && submission.trigger_agc_interrupt_on_done) {
-				m_renderer.TriggerEopEvent(0);
-			}
 			break;
 		}
 		case SubmissionType::Compute: {
-			const auto      num_dw                  = submission.commands.Size();
-			auto*           buffer                  = submission.commands.Data();
+			const auto      num_dw = static_cast<uint32_t>(submission.commands.size());
+			const auto*     buffer = submission.commands.data();
 			static uint32_t compute_batch_log_count = 0;
 			if (first_slice && num_dw <= 128 && compute_batch_log_count++ < 32) {
 				LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n",
@@ -639,7 +615,7 @@ bool GuestGpu::Process(Submission& submission) {
 			if (first_slice) {
 				GraphicsDbgDumpDcb("cc", num_dw, buffer);
 			}
-			complete = cp.Process(submission.command_execution, buffer, num_dw) ==
+			complete = cp.Process(submission.command_execution, submission.commands) ==
 			           Pm4ProcessResult::Complete;
 			if (submission.command_execution.MadeProgress()) {
 				if (complete) {
@@ -649,9 +625,6 @@ bool GuestGpu::Process(Submission& submission) {
 			} else if (complete) {
 				m_renderer.GetGpuResources().RunGarbageCollector();
 			}
-			if (complete && submission.trigger_agc_interrupt_on_done) {
-				Sync::TriggerAgcUserInterrupt();
-			}
 			break;
 		}
 		case SubmissionType::FlipPreparation:
@@ -663,12 +636,13 @@ bool GuestGpu::Process(Submission& submission) {
 	return complete;
 }
 
-Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer,
-                                           uint32_t size_dw) {
+Pm4ProcessResult CommandProcessor::Process(Pm4Execution&             execution,
+                                           std::span<const uint32_t> commands) {
 	KYTY_PROFILER_BLOCK("CommandProcessor::Process");
 	EXIT_IF(g_current_execution != nullptr);
-	if (execution.m_buffer_stack.empty() && size_dw != 0) {
-		execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0});
+	EXIT_IF(commands.size() > UINT32_MAX);
+	if (execution.m_buffer_stack.empty() && !commands.empty()) {
+		execution.m_buffer_stack.push_back({commands});
 	}
 	execution.m_suspended     = false;
 	execution.m_made_progress = false;
@@ -693,14 +667,14 @@ Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* bu
 	                                        : Pm4ProcessResult::Blocked;
 }
 
-void CommandProcessor::ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw) {
+void CommandProcessor::ProcessIndirectBuffer(std::span<const uint32_t> commands) {
 	EXIT_IF(g_current_execution == nullptr);
-	if (size_dw == 0) {
+	if (commands.empty()) {
 		return;
 	}
 	auto&      execution  = *g_current_execution;
 	const auto stop_depth = execution.m_buffer_stack.size();
-	execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0});
+	execution.m_buffer_stack.push_back({commands});
 	ProcessPm4(execution, stop_depth);
 }
 
@@ -716,29 +690,28 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 		}
 		const auto buffer_index = execution.m_buffer_stack.size() - 1;
 		auto&      cursor       = execution.m_buffer_stack[buffer_index];
+		EXIT_IF(cursor.offset_dw > cursor.commands.size());
 		if (cursor.deferred_advance_dw != 0) {
-			EXIT_IF(cursor.deferred_advance_dw > cursor.remaining_dw);
-			cursor.next_packet += cursor.deferred_advance_dw;
-			cursor.remaining_dw -= cursor.deferred_advance_dw;
+			EXIT_IF(cursor.deferred_advance_dw > cursor.commands.size() - cursor.offset_dw);
+			cursor.offset_dw += cursor.deferred_advance_dw;
 			cursor.deferred_advance_dw = 0;
 			execution.m_made_progress  = true;
 			continue;
 		}
-		if (cursor.remaining_dw == 0) {
+		if (cursor.offset_dw == cursor.commands.size()) {
 			execution.m_buffer_stack.pop_back();
 			continue;
 		}
 
-		auto* const packet        = cursor.next_packet;
-		const auto  remaining_dw  = cursor.remaining_dw;
-		const auto  total_dw      = cursor.total_dw;
-		const auto  packet_header = packet[0];
-		const auto  opcode        = (packet_header >> 8u) & 0xffu;
+		const auto* const packet        = cursor.commands.data() + cursor.offset_dw;
+		const auto        total_dw      = static_cast<uint32_t>(cursor.commands.size());
+		const auto        remaining_dw  = total_dw - cursor.offset_dw;
+		const auto        packet_header = packet[0];
+		const auto        opcode        = (packet_header >> 8u) & 0xffu;
 		EXIT_NOT_IMPLEMENTED(remaining_dw > total_dw);
 
 		if (packet_header == 0x80000000u) {
-			cursor.next_packet++;
-			cursor.remaining_dw--;
+			cursor.offset_dw++;
 			execution.m_made_progress = true;
 			continue;
 		}
@@ -773,8 +746,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 					     dst, val, packet[1], packet[2]);
 				}
 			}
-			cursor.next_packet += packet_dw;
-			cursor.remaining_dw -= packet_dw;
+			cursor.offset_dw += packet_dw;
 			execution.m_made_progress = true;
 			continue;
 		}
@@ -807,8 +779,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 			return;
 		}
 		EXIT_IF(execution.m_buffer_stack.size() != buffer_index + 1);
-		execution.m_buffer_stack[buffer_index].next_packet += packet_dw;
-		execution.m_buffer_stack[buffer_index].remaining_dw -= packet_dw;
+		execution.m_buffer_stack[buffer_index].offset_dw += packet_dw;
 		execution.m_made_progress = true;
 	}
 }
@@ -1107,6 +1078,8 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y,
                                       uint32_t thread_group_z, uint32_t mode) {
+	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
+
 	uint32_t frame_num = 0;
 	// uint32_t local_x   = 1;
 	// uint32_t local_y   = 1;
@@ -1249,7 +1222,14 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 	switch (interrupt_selector) {
 		case 0x00:
 		case 0x03: with_interrupt = false; break;
-		case 0x01: Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); return;
+		case 0x01:
+			if (!IsAsyncComputeQueue()) {
+				Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id,
+				                                 interrupt_context_id);
+				return;
+			}
+			with_interrupt = true;
+			break;
 		case 0x02: with_interrupt = true; break;
 		default: EXIT("unknown interrupt_selector\n");
 	}
@@ -1262,10 +1242,11 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		if (with_interrupt) {
 			if (with_writeback) {
 				Sync::WriteAtEndOfPipeWithInterruptWriteBack32(m_submit_id, CurrentBuffer(), dst,
-				                                               data, interrupt_context_id);
+				                                               data, m_interrupt_event_id,
+				                                               interrupt_context_id);
 			} else {
 				Sync::WriteAtEndOfPipeWithInterrupt32(m_submit_id, CurrentBuffer(), dst, data,
-				                                      interrupt_context_id);
+				                                      m_interrupt_event_id, interrupt_context_id);
 			}
 		} else if (with_writeback) {
 			Sync::WriteAtEndOfPipeWithWriteBack32(m_submit_id, CurrentBuffer(), dst, data);
@@ -1280,12 +1261,12 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 				if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
 					auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
 					SynchronizeGpu();
-					Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
+					Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
 					              value >> 16u);
 					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
 					                            value >> 16u);
 					if (with_interrupt) {
-						m_renderer.TriggerEopEvent(interrupt_context_id);
+						m_renderer.TriggerInterrupt(m_interrupt_event_id, interrupt_context_id);
 					}
 					return;
 				}
@@ -1311,10 +1292,12 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					if (with_interrupt) {
 						if (with_writeback) {
 							Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
-							    m_submit_id, CurrentBuffer(), dst, value, interrupt_context_id);
+							    m_submit_id, CurrentBuffer(), dst, value, m_interrupt_event_id,
+							    interrupt_context_id);
 						} else {
 							Sync::WriteAtEndOfPipeWithInterrupt64(m_submit_id, CurrentBuffer(), dst,
-							                                      value, interrupt_context_id);
+							                                      value, m_interrupt_event_id,
+							                                      interrupt_context_id);
 						}
 					} else if (with_writeback) {
 						Sync::WriteAtEndOfPipeWithWriteBack64(m_submit_id, CurrentBuffer(), dst,
@@ -1393,26 +1376,35 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		case 0x04:
 			if constexpr (sizeof(T) == sizeof(uint64_t)) {
 				const auto clock = Sync::ReadReferenceClock();
-				std::memcpy(dst_gpu_addr, &clock, sizeof(clock));
+				auto*      dst   = static_cast<uint64_t*>(dst_gpu_addr);
+				std::memcpy(dst, &clock, sizeof(clock));
 				switch (cache_action) {
 					case 0x00:
-						if (((eop_event_type == 0x04 && event_index == 0x05) ||
-						     (eop_event_type == 0x28 && event_index == 0x00)) &&
-						    !with_interrupt) {
-							Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
-							                                   static_cast<uint64_t*>(dst_gpu_addr),
-							                                   clock);
+						if ((eop_event_type == 0x04 && event_index == 0x05) ||
+						    (eop_event_type == 0x28 && event_index == 0x00)) {
+							if (with_interrupt) {
+								Sync::WriteAtEndOfPipeWithInterrupt64(
+								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    interrupt_context_id);
+							} else {
+								Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
+								                                   dst, clock);
+							}
 							return;
 						}
 						break;
 					case 0x38:
-						if (((eop_event_type == 0x04 &&
-						      (event_index == 0x00 || event_index == 0x05)) ||
-						     (eop_event_type == 0x28 && event_index == 0x00)) &&
-						    !with_interrupt) {
-							Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
-							    m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
-							    clock);
+						if ((eop_event_type == 0x04 &&
+						     (event_index == 0x00 || event_index == 0x05)) ||
+						    (eop_event_type == 0x28 && event_index == 0x00)) {
+							if (with_interrupt) {
+								Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
+								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    interrupt_context_id);
+							} else {
+								Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
+								    m_submit_id, CurrentBuffer(), dst, clock);
+							}
 							return;
 						}
 						break;
@@ -1469,15 +1461,17 @@ void CommandProcessor::EmitGlobalBarrier() {
 void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) {
 	CheckBuffer();
 
-	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id);
+	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id, interrupt_context_id);
 }
 
-void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
+void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
+                                    uint64_t event_address) {
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::TriggerEvent()\n"
 		     "\t event_type  = 0x%08" PRIx32 "\n"
-		     "\t event_index = 0x%08" PRIx32 "\n",
-		     event_type, event_index);
+		     "\t event_index = 0x%08" PRIx32 "\n"
+		     "\t address     = 0x%016" PRIx64 "\n",
+		     event_type, event_index, event_address);
 	}
 
 	const auto valid_cache_event_index = event_index == 0x00000000 || event_index == 0x00000007;
@@ -1514,12 +1508,36 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
 		case 0x0000001a:
 		case 0x0000001b:
 		case 0x00000038:
-		case 0x00000039:
 		case 0x0000003a:
 			LOGF("\t temporary: ignoring unsupported event_write type 0x%08" PRIx32
 			     ", index 0x%08" PRIx32 "\n",
 			     event_type, event_index);
 			break;
+		case 0x00000039: {
+			if (event_index != 0x00000001 || event_address == 0 || (event_address & 0x7u) != 0) {
+				EXIT("invalid occlusion-counter dump: index=0x%08" PRIx32 ", address=0x%016" PRIx64
+				     "\n",
+				     event_index, event_address);
+			}
+			static std::once_flag warning_once;
+			std::call_once(warning_once, [] {
+				std::printf("Warning: game uses occlusion queries, which are currently treated as "
+				            "always visible; GPU usage may be higher and FPS may be lower.\n");
+			});
+
+			// Until host occlusion queries are implemented, publish an always-visible result. The
+			// PS5 layout contains one interleaved begin/end pair per DB, and bit 63 marks a result
+			// ready.
+			constexpr uint64_t ready_bit    = 1ull << 63u;
+			constexpr uint64_t counter_mask = ready_bit - 1u;
+			auto*              results      = reinterpret_cast<volatile uint64_t*>(event_address);
+			const auto         value        = ready_bit | m_synthetic_occlusion_counter;
+			for (uint32_t db = 0; db < 16u; db++) {
+				results[db * 2u] = value;
+			}
+			m_synthetic_occlusion_counter = (m_synthetic_occlusion_counter + 1u) & counter_mask;
+			break;
+		}
 		default:
 			EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, event_index);
 	}
@@ -1582,7 +1600,7 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
 	                                         m_flip.flip_arg);
 	Sync::WriteAtEndOfPipeWithInterruptWriteBackFlip32(
 	    m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
-	    m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
+	    m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request, m_interrupt_event_id);
 	GetScheduler().Flush();
 }
 
@@ -1603,7 +1621,7 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 }
 
 void CommandProcessor::SynchronizeGpu() {
-	GetScheduler().FinishCurrent();
+	GetScheduler().Finish();
 }
 
 bool GuestGpu::IsGpuThread() noexcept {

src/graphics/guest_gpu/graphicsRun.h

@@ -13,8 +13,8 @@
 #include <deque>
 #include <memory>
 #include <mutex>
+#include <span>
 #include <thread>
-#include <vector>
 
 namespace Libs::Graphics {
 
@@ -31,12 +31,12 @@ public:
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
 
-	void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
-	            uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false);
-	void SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw,
-	                   bool trigger_agc_interrupt_on_done = false);
-	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
+	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
+	void              Submit(std::span<const uint32_t> draw_commands,
+	                         std::span<const uint32_t> constant_commands);
+	void              SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
+	void              SubmitFlipPreparation(uint64_t request_id);
+	void              Done();
 	[[nodiscard]] int GetFrameNum() const;
 
 	[[nodiscard]] static bool IsGpuThread() noexcept;
@@ -45,39 +45,24 @@ private:
 	static constexpr uint32_t ComputePipeCount     = 7;
 	static constexpr uint32_t QueuesPerComputePipe = 8;
 	static constexpr uint32_t ComputeQueueCount    = ComputePipeCount * QueuesPerComputePipe;
+	static constexpr uint32_t ComputeQueueBase     = 0x20;
 	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
 
-	class OwnedCmdBuffer {
-	public:
-		OwnedCmdBuffer() = default;
-		explicit OwnedCmdBuffer(const uint32_t* data, uint32_t count);
-
-		[[nodiscard]] bool     Empty() const noexcept { return m_words.empty(); }
-		[[nodiscard]] uint32_t Size() const noexcept {
-			return static_cast<uint32_t>(m_words.size());
-		}
-		[[nodiscard]] uint32_t* Data() noexcept { return m_words.data(); }
-
-	private:
-		std::vector<uint32_t> m_words;
-	};
-
 	enum class SubmissionType { Graphics, Compute, FlipPreparation };
 
 	struct Submission {
-		SubmissionType type     = SubmissionType::Graphics;
-		uint32_t       queue_id = 0;
-		OwnedCmdBuffer commands;
-		OwnedCmdBuffer constant_commands;
-		Pm4Execution   command_execution;
-		Pm4Execution   constant_execution;
-		bool           trigger_agc_interrupt_on_done = false;
-		bool           reset_processor               = false;
-		bool           started                       = false;
-		bool           command_complete              = false;
-		bool           constant_complete             = false;
-		bool           blocked                       = false;
-		uint64_t       flip_request_id               = 0;
+		SubmissionType            type     = SubmissionType::Graphics;
+		uint32_t                  queue_id = 0;
+		std::span<const uint32_t> commands;
+		std::span<const uint32_t> constant_commands;
+		Pm4Execution              command_execution;
+		Pm4Execution              constant_execution;
+		bool                      reset_processor   = false;
+		bool                      started           = false;
+		bool                      command_complete  = false;
+		bool                      constant_complete = false;
+		bool                      blocked           = false;
+		uint64_t                  flip_request_id   = 0;
 	};
 
 	void              Enqueue(Submission submission);

src/graphics/guest_gpu/hardwareContext.h

@@ -1089,6 +1089,7 @@ public:
 	void SetPsShaderChksum(uint32_t value) { m_ps.ps_regs.chksum = value; }
 
 	void SetCsShader(const CsStageRegisters& cs_regs) { m_cs.cs_regs = cs_regs; }
+	void SetCsWaveSize(uint8_t wave_size) { m_cs.cs_regs.wave_size = wave_size; }
 
 	void SetPsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
 		m_ps.ps_user_sgpr.value[id] = value;

src/graphics/guest_gpu/pm4.cpp

@@ -21,12 +21,10 @@ constexpr auto MakeRegisterNames() {
 	names[R_DRAW_RESET]     = "R_DRAW_RESET";
 	names[R_WAIT_FLIP_DONE] = "R_WAIT_FLIP_DONE";
 	names[R_DISPATCH_RESET] = "R_DISPATCH_RESET";
-	names[R_WAIT_MEM_32]    = "R_WAIT_MEM_32";
 	names[R_PUSH_MARKER]    = "R_PUSH_MARKER";
 	names[R_POP_MARKER]     = "R_POP_MARKER";
 	names[R_ACQUIRE_MEM]    = "R_ACQUIRE_MEM";
 	names[R_WRITE_DATA]     = "R_WRITE_DATA";
-	names[R_WAIT_MEM_64]    = "R_WAIT_MEM_64";
 	names[R_FLIP]           = "R_FLIP";
 	names[R_RELEASE_MEM]    = "R_RELEASE_MEM";
 	names[R_DMA_DATA]       = "R_DMA_DATA";
@@ -62,6 +60,7 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_WRITE_DATA]                = "IT_WRITE_DATA";
 	names[IT_MEM_SEMAPHORE]             = "IT_MEM_SEMAPHORE";
 	names[IT_DRAW_INDEX_INDIRECT_MULTI] = "IT_DRAW_INDEX_INDIRECT_MULTI";
+	names[IT_WAIT_REG_MEM]               = "IT_WAIT_REG_MEM";
 	names[IT_INDIRECT_BUFFER]           = "IT_INDIRECT_BUFFER";
 	names[IT_COPY_DATA]                 = "IT_COPY_DATA";
 	names[IT_CP_DMA]                    = "IT_CP_DMA";
@@ -91,6 +90,7 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_DISPATCH_DRAW_PREAMBLE]    = "IT_DISPATCH_DRAW_PREAMBLE";
 	names[IT_DISPATCH_DRAW]             = "IT_DISPATCH_DRAW";
 	names[IT_GET_LOD_STATS]             = "IT_GET_LOD_STATS";
+	names[IT_WAIT_REG_MEM_64]           = "IT_WAIT_REG_MEM_64";
 	names[IT_SET_CONTEXT_REG_INDIRECT]  = "IT_SET_CONTEXT_REG_INDIRECT";
 	return names;
 }
@@ -100,7 +100,7 @@ constexpr auto g_opcode_names   = MakeOpcodeNames();
 
 } // namespace
 
-void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw,
+void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_t start_dw,
                          uint32_t num_dw) {
 	// db_dump();
 
@@ -143,9 +143,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 				if ((op == IT_SET_CONTEXT_REG_INDIRECT || op == IT_SET_SH_REG_INDIRECT ||
 				     op == IT_SET_UCONFIG_REG_INDIRECT) &&
 				    len == 4) {
-					auto* indirect_buffer =
-					    reinterpret_cast<uint32_t*>((static_cast<uint64_t>(cmd[0]) & 0xfffffffcu) |
-					                                (static_cast<uint64_t>(cmd[1]) << 32u));
+					auto* indirect_buffer = reinterpret_cast<const uint32_t*>(
+					    (static_cast<uint64_t>(cmd[0]) & 0xfffffffcu) |
+					    (static_cast<uint64_t>(cmd[1]) << 32u));
 					const uint32_t indirect_num_regs = cmd[3] & 0x3fffu;
 
 					for (uint32_t i = 0; indirect_buffer != nullptr && i < indirect_num_regs;

src/graphics/guest_gpu/pm4.h

@@ -42,6 +42,7 @@ constexpr uint32_t IT_WRITE_DATA                = 0x37;
 constexpr uint32_t IT_MEM_SEMAPHORE             = 0x39;
 constexpr uint32_t IT_DRAW_INDEX_INDIRECT_MULTI = 0x38;
 constexpr uint32_t IT_DISPATCH_DRAW_PREAMBLE    = 0x3A;
+constexpr uint32_t IT_WAIT_REG_MEM              = 0x3C;
 constexpr uint32_t IT_INDIRECT_BUFFER           = 0x3F;
 constexpr uint32_t IT_COPY_DATA                 = 0x40;
 constexpr uint32_t IT_CP_DMA                    = 0x41;
@@ -70,6 +71,7 @@ constexpr uint32_t IT_WAIT_ON_CE_COUNTER        = 0x86;
 constexpr uint32_t IT_WAIT_ON_DE_COUNTER_DIFF   = 0x88;
 constexpr uint32_t IT_DISPATCH_DRAW             = 0x8D;
 constexpr uint32_t IT_GET_LOD_STATS             = 0x8E;
+constexpr uint32_t IT_WAIT_REG_MEM_64           = 0x93;
 constexpr uint32_t IT_SET_CONTEXT_REG_INDIRECT  = 0x9F;
 
 /* Custom codes. Implemented via IT_NOP */
@@ -78,12 +80,10 @@ constexpr uint32_t R_ZERO           = 0x00;
 constexpr uint32_t R_DRAW_RESET     = 0x05;
 constexpr uint32_t R_WAIT_FLIP_DONE = 0x06;
 constexpr uint32_t R_DISPATCH_RESET = 0x09;
-constexpr uint32_t R_WAIT_MEM_32    = 0x0A;
 constexpr uint32_t R_PUSH_MARKER    = 0x0B;
 constexpr uint32_t R_POP_MARKER     = 0x0C;
 constexpr uint32_t R_ACQUIRE_MEM    = 0x14;
 constexpr uint32_t R_WRITE_DATA     = 0x15;
-constexpr uint32_t R_WAIT_MEM_64    = 0x16;
 constexpr uint32_t R_FLIP           = 0x17;
 constexpr uint32_t R_RELEASE_MEM    = 0x18;
 constexpr uint32_t R_DMA_DATA       = 0x19;
@@ -317,8 +317,8 @@ constexpr uint32_t FSR_RECURSIONS1              = 0xF2;
 constexpr uint32_t VGT_MULTI_PRIM_IB_RESET_INDX = 0x103;
 constexpr uint32_t CB_RMI_GL2_CACHE_CONTROL     = 0x104;
 constexpr uint32_t CB_BLEND_RED                 = 0x105;
-constexpr uint32_t CB_BLEND_GREEN               = 0x106;
-constexpr uint32_t CB_BLEND_BLUE                = 0x107;
+constexpr uint32_t CB_BLEND_BLUE                = 0x106;
+constexpr uint32_t CB_BLEND_GREEN               = 0x107;
 constexpr uint32_t CB_BLEND_ALPHA               = 0x108;
 constexpr uint32_t CB_DCC_CONTROL               = 0x109;
 
@@ -387,9 +387,6 @@ constexpr uint32_t SPI_SHADER_POS_FORMAT  = 0x1C3;
 constexpr uint32_t SPI_SHADER_Z_FORMAT    = 0x1C4;
 constexpr uint32_t SPI_SHADER_COL_FORMAT  = 0x1C5;
 
-// Indirect Cx descriptor selector for the 32-entry PS input-control register bank.
-constexpr uint32_t CX_PS_SHADER_USAGE_BASE = 0x10000000u;
-
 constexpr uint32_t CB_BLEND0_CONTROL                            = 0x1E0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT       = 0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK        = 0x1F;
@@ -984,8 +981,16 @@ constexpr uint32_t COMPUTE_PGM_RSRC1_FP16_OVFL_SHIFT  = 26;
 constexpr uint32_t COMPUTE_PGM_RSRC1_FP16_OVFL_MASK   = 0x1;
 constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_SHIFT   = 29;
 constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_MASK    = 0x1;
-constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_SHIFT     = 30;
-constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_MASK      = 0x1;
+
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT = 15;
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK  = 0x1;
+
+[[nodiscard]] constexpr uint8_t ComputeWaveSize(uint32_t dispatch_initiator) {
+	return ((dispatch_initiator >> COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT) &
+	        COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK) != 0u
+	           ? 32u
+	           : 64u;
+}
 
 constexpr uint32_t COMPUTE_PGM_RSRC2                      = 0x213;
 constexpr uint32_t COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT     = 0;
@@ -1083,7 +1088,7 @@ constexpr uint32_t UC_NOP                       = 0x80003FFF;
 
 constexpr uint32_t UC_NUM = 0x3FFF + 1;
 
-void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw,
+void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_t start_dw,
                          uint32_t num_dw);
 
 } // namespace Libs::Graphics::Pm4

src/graphics/host_gpu/graphicContext.h

@@ -3,20 +3,88 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/threads.h"
 #include "graphics/host_gpu/vulkanCommon.h" // IWYU pragma: export
-#include "graphics/host_gpu/vulkanInstance.h"
 
+#include <map>
+#include <mutex>
+#include <tuple>
 #include <vector>
 #include <vk_mem_alloc.h>
 
 namespace Libs::Graphics {
 
-class CommandBuffer;
 struct VulkanBuffer;
 struct VulkanImage;
 struct VulkanMemory;
 
-struct GraphicContext: public VulkanInstance {
+inline constexpr uint32_t VULKAN_TARGET_API_VERSION = VK_API_VERSION_1_3;
+
+struct GraphicContext {
+	vk::Instance                       instance                              = nullptr;
+	vk::DebugUtilsMessengerEXT         debug_messenger                       = nullptr;
+	vk::PhysicalDevice                 physical_device                       = nullptr;
+	vk::PhysicalDeviceProperties       physical_device_properties            = {};
+	vk::PhysicalDeviceMemoryProperties physical_device_memory_properties     = {};
+	vk::Device                         device                                = nullptr;
+	VmaAllocator                       allocator                             = nullptr;
+	bool                               memory_budget_ext_enabled             = false;
+	bool                               rt_extensions_enabled                 = false;
+	bool                               compute_subgroup_size_control_enabled = false;
+	bool                               compute_wave64_supported              = false;
+	bool                               sample_rate_shading_enabled           = false;
+	uint32_t                           subgroup_size                         = 0;
+	uint32_t                           min_subgroup_size                     = 0;
+	uint32_t                           max_subgroup_size                     = 0;
+	uint32_t                           max_push_descriptors                  = 0;
+	vk::ShaderStageFlags               required_subgroup_size_stages         = {};
+	Common::Mutex                      queue_mutex;
+	uint32_t                           queue_family = static_cast<uint32_t>(-1);
+	vk::Queue                          queue        = nullptr;
+
+	[[nodiscard]] const vk::PhysicalDeviceProperties& GetPhysicalDeviceProperties() const {
+		return physical_device_properties;
+	}
+
+	[[nodiscard]] const vk::PhysicalDeviceMemoryProperties&
+	GetPhysicalDeviceMemoryProperties() const {
+		return physical_device_memory_properties;
+	}
+
+	[[nodiscard]] vk::FormatProperties GetFormatProperties(vk::Format format) const {
+		std::scoped_lock lock(m_format_properties_mutex);
+		auto [it, inserted] = m_format_properties.try_emplace(format);
+		if (inserted) {
+			physical_device.getFormatProperties(format, &it->second);
+		}
+		return it->second;
+	}
+
+	[[nodiscard]] vk::Result GetImageFormatProperties(vk::Format format, vk::ImageType type,
+	                                                  vk::ImageTiling            tiling,
+	                                                  vk::ImageUsageFlags        usage,
+	                                                  vk::ImageCreateFlags       flags,
+	                                                  vk::ImageFormatProperties* properties) const {
+		using Key = std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
+		                       vk::ImageCreateFlags>;
+		std::scoped_lock lock(m_image_format_properties_mutex);
+		auto [it, inserted] =
+		    m_image_format_properties.try_emplace(Key {format, type, tiling, usage, flags});
+		if (inserted) {
+			it->second.first = physical_device.getImageFormatProperties(format, type, tiling, usage,
+			                                                            flags, &it->second.second);
+		}
+		if (properties != nullptr) {
+			*properties = it->second.second;
+		}
+		return it->second.first;
+	}
+
+	[[nodiscard]] vk::DeviceSize StorageMinAlignment() const {
+		const auto alignment = physical_device_properties.limits.minStorageBufferOffsetAlignment;
+		return alignment != 0 ? alignment : 1;
+	}
+
 	[[nodiscard]] bool CreateAllocator();
 	void               DestroyAllocator();
 	void               LogMemoryBudget() const;
@@ -24,7 +92,6 @@ struct GraphicContext: public VulkanInstance {
 	[[nodiscard]] uint64_t GetDeviceMemoryUsage() const;
 	[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
 	void                   CreateBuffer(uint64_t size, VulkanBuffer& buffer);
-	void                   DeleteBuffer(VulkanBuffer& buffer);
 	[[nodiscard]] bool     CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
 	void                   DeleteImage(VulkanImage& image);
 	void                   MapMemory(VulkanMemory& memory, void*& data);
@@ -36,6 +103,15 @@ struct GraphicContext: public VulkanInstance {
 
 	uint32_t screen_width  = 0;
 	uint32_t screen_height = 0;
+
+private:
+	mutable std::mutex                                 m_format_properties_mutex;
+	mutable std::map<vk::Format, vk::FormatProperties> m_format_properties;
+	mutable std::mutex                                 m_image_format_properties_mutex;
+	mutable std::map<std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
+	                            vk::ImageCreateFlags>,
+	                 std::pair<vk::Result, vk::ImageFormatProperties>>
+	    m_image_format_properties;
 };
 
 struct VulkanMemory {

src/graphics/host_gpu/rangeSet.h

@@ -53,12 +53,30 @@ public:
 		}
 	}
 
+	void Clear() { m_ranges.clear(); }
+
+	template <typename Func>
+	void ForEach(Func&& func) const {
+		for (const auto& [begin, end]: m_ranges) {
+			func(begin, end);
+		}
+	}
+
 	[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
 		std::vector<Range> result;
 		ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
 		return result;
 	}
 
+	[[nodiscard]] bool Intersects(uint64_t address, uint64_t size) const {
+		const auto end = End(address, size);
+		auto       it  = m_ranges.lower_bound(address);
+		if (it != m_ranges.begin() && std::prev(it)->second > address) {
+			return true;
+		}
+		return it != m_ranges.end() && it->first < end;
+	}
+
 	[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.upper_bound(address);
@@ -89,7 +107,7 @@ public:
 
 private:
 	static uint64_t End(uint64_t address, uint64_t size) {
-		if (address == 0 || size == 0 || size > UINT64_MAX - address) {
+		if (size == 0 || size > UINT64_MAX - address) {
 			EXIT("invalid range-set address or size\n");
 		}
 		return address + size;

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -3,12 +3,15 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/lruCache.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/renderer/cache/faultManager.h"
+#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
 #include <map>
-#include <memory>
 #include <span>
 #include <utility>
 #include <vector>
@@ -16,93 +19,95 @@
 namespace Libs::Graphics {
 
 struct GraphicContext;
-class CommandBuffer;
 class CommandScheduler;
 class TextureCache;
 
-struct BufferBinding {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	uint64_t              offset = 0;
-};
-
-struct ImageBufferSource {
-	Buffer*  buffer = nullptr;
-	uint64_t offset = 0;
-};
+using BufferId = Common::SlotId;
+inline constexpr BufferId NULL_BUFFER_ID {0};
 
 class BufferCache {
 public:
-	static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
-	static constexpr uint64_t GetBufferOffset(uint64_t vaddr) {
-		return vaddr & (CACHING_PAGE_SIZE - 1);
-	}
+	static constexpr uint32_t CACHING_PAGEBITS  = 14;
+	static constexpr uint64_t CACHING_PAGESIZE  = uint64_t {1} << CACHING_PAGEBITS;
+	static constexpr uint64_t CACHING_NUMPAGES  = uint64_t {1} << (40 - CACHING_PAGEBITS);
+	static constexpr uint64_t BDA_PAGETABLE_SIZE =
+	    CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
 
 	BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
 	            TextureCache& texture_cache);
 	~BufferCache();
 	KYTY_CLASS_NO_COPY(BufferCache);
 
-	void                        InvalidateMemory(uint64_t vaddr, uint64_t size);
-	void                        ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
-	void                        UnmapMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
-	                                         bool is_written = false, bool is_read = true,
-	                                         bool is_formatted = false);
-	[[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept;
-	[[nodiscard]] Buffer&       GetGdsBuffer() noexcept { return m_gds_buffer; }
-	[[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; }
-	[[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size,
-	                                            uint64_t alignment);
-	[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
-	[[nodiscard]] ImageBufferSource       ObtainBufferForImage(uint64_t vaddr, uint64_t size);
-	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
-	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
-	                bool src_gds = false);
+	void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
+	void                   ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
+	[[nodiscard]] Buffer&  GetBuffer(BufferId id) { return m_slot_buffers[id]; }
+	[[nodiscard]] BufferId FindBuffer(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBuffer(uint64_t vaddr, uint64_t size,
+	                                                        bool     is_written,
+	                                                        bool     is_texel_buffer = false,
+	                                                        BufferId id              = {});
+	[[nodiscard]] StreamBuffer&                GetUtilityBuffer(MemoryUsage usage) noexcept {
+		switch (usage) {
+			case MemoryUsage::Upload: return m_staging_buffer;
+			case MemoryUsage::Stream: return m_stream_buffer;
+			case MemoryUsage::Download: return m_download_buffer;
+			case MemoryUsage::DeviceLocal: return m_device_buffer;
+		}
+		EXIT("BufferCache: invalid utility-buffer usage\n");
+	}
+	[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
+	[[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept { return &m_bda_pagetable_buffer; }
+	[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return m_fault_manager.GetFaultBuffer(); }
+	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
+	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
+	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
+	                bool src_gds);
 	// Cache-index and exact dirty-range queries require GPU-thread serialization.
 	[[nodiscard]] bool IsRegionRegistered(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
+	void               ProcessFaultBuffer();
+	void               SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:
 	friend struct BufferCacheTestAccess;
 
-	struct CacheRange {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-	struct CachedBuffer;
 	struct DownloadCopy;
-	struct DownloadRange;
-	struct RetiredBuffer;
+	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
+	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
 	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static uint64_t           AlignDown(uint64_t value) noexcept;
-	[[nodiscard]] static uint64_t           AlignUp(uint64_t value);
 	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
 		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
 	}
 	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
-	[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
-	void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
-	            const void* source, uint64_t size);
-	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
-	                                              uint64_t size);
-	[[nodiscard]] bool SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
-	                                     uint64_t size, bool is_written, bool is_texel_buffer);
+	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
+	void TouchBuffer(const Buffer& buffer);
+	[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
+	void                   Register(BufferId id);
+	void Unregister(BufferId id);
+	template <bool insert>
+	void ChangeRegister(BufferId id);
+	void DeleteBuffer(BufferId id);
+	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
+	                                     bool is_written, bool is_texel_buffer);
+	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
+	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
-	void PublishDownloads(std::span<const DownloadRange> downloads);
-	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
+	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
+	FaultManager                                      m_fault_manager;
 	Buffer                                            m_gds_buffer;
-	std::shared_ptr<Buffer>                           m_null_buffer;
-	std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
+	Buffer                                            m_bda_pagetable_buffer;
+	Common::SlotVector<Buffer>                        m_slot_buffers;
+	Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
+	std::map<uint64_t, BufferId>                      m_buffers;
+	PageTable                                         m_page_table;
 	RangeSet                                          m_gpu_modified_ranges;
 	MemoryTracker                                     m_memory_tracker;
 	StreamBuffer                                      m_staging_buffer;
@@ -110,7 +115,7 @@ private:
 	StreamBuffer                                      m_download_buffer;
 	StreamBuffer                                      m_device_buffer;
 	TextureCache&                                     m_texture_cache;
-	uint64_t                                          m_total_used_memory = 0;
+	uint64_t                                          m_total_used_memory  = 0;
 	uint64_t m_trigger_gc_memory  = 1ull * 1024 * 1024 * 1024;
 	uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
 	uint64_t m_gc_tick            = 0;

src/graphics/host_gpu/renderer/cache/faultManager.cpp

@@ -0,0 +1,167 @@
+#include "graphics/host_gpu/renderer/cache/faultManager.h"
+
+#include "common/assert.h"
+#include "common/logging/log.h"
+#include "gpu_tiler_shaders/fault_buffer_process_spv.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+
+#include <bit>
+#include <cinttypes>
+#include <cstring>
+#include <limits>
+
+namespace Libs::Graphics {
+
+namespace {
+
+constexpr size_t MaxPageFaults    = 1024;
+constexpr size_t PageFaultAreaSize = MaxPageFaults * sizeof(uint64_t);
+
+} // namespace
+
+FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler,
+                           BufferCache& buffer_cache, uint32_t caching_pagebits,
+                           uint64_t caching_num_pages)
+    : m_graphics(graphics), m_scheduler(scheduler), m_buffer_cache(buffer_cache),
+      m_caching_pagesize(uint64_t {1} << caching_pagebits),
+      m_caching_num_pages(caching_num_pages), m_fault_buffer_size(caching_num_pages / 8),
+      m_fault_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
+                     m_fault_buffer_size),
+      m_download_buffer(graphics, scheduler, MemoryUsage::Download, 0, AllFlags,
+                        MaxPendingFaults * PageFaultAreaSize) {
+	SetVulkanObjectNameF(m_graphics.device, m_fault_buffer.Handle(), "Fault Buffer");
+
+	const vk::DescriptorSetLayoutBinding bindings[] {
+	    {0, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
+	    {1, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
+	};
+	vk::DescriptorSetLayoutCreateInfo layout_info {};
+	layout_info.sType        = vk::StructureType::eDescriptorSetLayoutCreateInfo;
+	layout_info.flags        = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
+	layout_info.bindingCount = std::size(bindings);
+	layout_info.pBindings    = bindings;
+	RequireVulkanSuccess(
+	    m_graphics.device.createDescriptorSetLayout(&layout_info, nullptr,
+	                                                &m_fault_process_desc_layout),
+	    "create fault-buffer descriptor layout");
+
+	vk::ShaderModuleCreateInfo module_info {};
+	module_info.sType    = vk::StructureType::eShaderModuleCreateInfo;
+	module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
+	module_info.pCode    = FAULT_BUFFER_PROCESS_SPV;
+	vk::ShaderModule module = nullptr;
+	RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
+	                     "create fault-buffer shader module");
+
+	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
+	pipeline_layout_info.sType          = vk::StructureType::ePipelineLayoutCreateInfo;
+	pipeline_layout_info.setLayoutCount = 1;
+	pipeline_layout_info.pSetLayouts    = &m_fault_process_desc_layout;
+	RequireVulkanSuccess(
+	    m_graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
+	                                           &m_fault_process_pipeline_layout),
+	    "create fault-buffer pipeline layout");
+
+	vk::PipelineShaderStageCreateInfo stage {};
+	stage.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
+	stage.stage  = vk::ShaderStageFlagBits::eCompute;
+	stage.module = module;
+	stage.pName  = "main";
+	vk::ComputePipelineCreateInfo pipeline_info {};
+	pipeline_info.sType  = vk::StructureType::eComputePipelineCreateInfo;
+	pipeline_info.stage  = stage;
+	pipeline_info.layout = m_fault_process_pipeline_layout;
+	const auto result = m_graphics.device.createComputePipelines(
+	    nullptr, 1, &pipeline_info, nullptr, &m_fault_process_pipeline);
+	m_graphics.device.destroyShaderModule(module, nullptr);
+	RequireVulkanSuccess(result, "create fault-buffer pipeline");
+	SetVulkanObjectNameF(m_graphics.device, m_fault_process_pipeline, "Fault Buffer Parser");
+}
+
+FaultManager::~FaultManager() {
+	m_graphics.device.destroyPipeline(m_fault_process_pipeline, nullptr);
+	m_graphics.device.destroyPipelineLayout(m_fault_process_pipeline_layout, nullptr);
+	m_graphics.device.destroyDescriptorSetLayout(m_fault_process_desc_layout, nullptr);
+}
+
+void FaultManager::ProcessFaultBuffer() {
+	if (const auto wait_tick = m_fault_areas[m_current_area]; wait_tick != 0) {
+		m_scheduler.Wait(wait_tick);
+		m_scheduler.PopPendingOperations();
+	}
+
+	const auto offset = m_current_area * PageFaultAreaSize;
+	auto*      mapped = m_download_buffer.Mapped().data() + offset;
+	std::memset(mapped, 0, PageFaultAreaSize);
+	m_download_buffer.Flush(offset, PageFaultAreaSize);
+
+	vk::BufferMemoryBarrier2 pre_barrier {};
+	pre_barrier.sType         = vk::StructureType::eBufferMemoryBarrier2;
+	pre_barrier.srcStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
+	pre_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
+	pre_barrier.dstStageMask  = vk::PipelineStageFlagBits2::eComputeShader;
+	pre_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderRead;
+	pre_barrier.buffer        = m_fault_buffer.Handle();
+	pre_barrier.offset        = 0;
+	pre_barrier.size          = m_fault_buffer_size;
+	auto post_barrier         = pre_barrier;
+	post_barrier.srcStageMask  = vk::PipelineStageFlagBits2::eComputeShader;
+	post_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
+	post_barrier.dstStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
+	post_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderWrite;
+
+	const vk::DescriptorBufferInfo infos[] {
+	    {m_fault_buffer.Handle(), 0, m_fault_buffer_size},
+	    {m_download_buffer.Handle(), offset, PageFaultAreaSize},
+	};
+	std::array<vk::WriteDescriptorSet, 2> writes {};
+	for (uint32_t index = 0; index < writes.size(); ++index) {
+		writes[index].sType           = vk::StructureType::eWriteDescriptorSet;
+		writes[index].dstBinding      = index;
+		writes[index].descriptorCount = 1;
+		writes[index].descriptorType  = vk::DescriptorType::eStorageBuffer;
+		writes[index].pBufferInfo     = &infos[index];
+	}
+
+	m_scheduler.EndRendering();
+	auto command = m_scheduler.Current().Handle();
+	vk::DependencyInfo dependency {};
+	dependency.sType                    = vk::StructureType::eDependencyInfo;
+	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
+	dependency.bufferMemoryBarrierCount = 1;
+	dependency.pBufferMemoryBarriers    = &pre_barrier;
+	command.pipelineBarrier2(dependency);
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_fault_process_pipeline);
+	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute,
+	                             m_fault_process_pipeline_layout, 0, writes);
+	const auto num_threads    = m_caching_num_pages / 32;
+	const auto num_workgroups = (num_threads + 63) / 64;
+	command.dispatch(static_cast<uint32_t>(num_workgroups), 1, 1);
+	dependency.pBufferMemoryBarriers = &post_barrier;
+	command.pipelineBarrier2(dependency);
+
+	const auto area = m_current_area;
+	m_scheduler.DeferOperation([this, mapped, offset, area] {
+		m_download_buffer.Invalidate(offset, PageFaultAreaSize);
+		m_fault_ranges.Clear();
+		const auto* faults = std::bit_cast<const uint64_t*>(mapped);
+		const auto  count  = static_cast<uint32_t>(faults[0]);
+		for (uint32_t index = 1; index <= count; ++index) {
+			m_fault_ranges.Add(faults[index], m_caching_pagesize);
+			LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
+		}
+		m_fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
+			EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());
+			(void)m_buffer_cache.FindBuffer(start, end - start);
+		});
+		m_fault_areas[area] = 0;
+	});
+
+	m_fault_areas[m_current_area++] = m_scheduler.CurrentTick();
+	m_current_area %= MaxPendingFaults;
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/faultManager.h

@@ -0,0 +1,46 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
+
+#include "common/abi.h"
+#include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
+
+#include <array>
+#include <cstdint>
+
+namespace Libs::Graphics {
+
+class BufferCache;
+
+class FaultManager {
+	static constexpr size_t MaxPendingFaults = 8;
+
+public:
+	FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, BufferCache& buffer_cache,
+	             uint32_t caching_pagebits, uint64_t caching_num_pages);
+	~FaultManager();
+	KYTY_CLASS_NO_COPY(FaultManager);
+
+	[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return &m_fault_buffer; }
+	void                  ProcessFaultBuffer();
+
+private:
+	GraphicContext&                            m_graphics;
+	CommandScheduler&                          m_scheduler;
+	BufferCache&                               m_buffer_cache;
+	RangeSet                                   m_fault_ranges;
+	uint64_t                                   m_caching_pagesize;
+	uint64_t                                   m_caching_num_pages;
+	uint64_t                                   m_fault_buffer_size;
+	Buffer                                     m_fault_buffer;
+	Buffer                                     m_download_buffer;
+	std::array<uint64_t, MaxPendingFaults>      m_fault_areas {};
+	uint32_t                                   m_current_area = 0;
+	vk::DescriptorSetLayout                    m_fault_process_desc_layout = nullptr;
+	vk::Pipeline                               m_fault_process_pipeline = nullptr;
+	vk::PipelineLayout                         m_fault_process_pipeline_layout = nullptr;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -60,10 +60,10 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	const auto unmap = [this, vaddr, size] {
 		if (m_scheduler.Active()) {
 			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.FinishCurrent();
+			m_scheduler.Finish();
 			m_scheduler.WaitPriorityOperations(tick);
 		}
-		m_buffer_cache.UnmapMemory(vaddr, size);
+		m_buffer_cache.InvalidateMemory(vaddr, size);
 		m_texture_cache.UnmapMemory(vaddr, size);
 		m_page_manager.OnGpuUnmap(vaddr, size);
 		std::lock_guard lock(m_mapped_ranges_mutex);
@@ -76,7 +76,19 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	m_gpu->SendCommandSync(unmap);
 }
 
+void GpuResourceManager::PrepareBda() {
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
+		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
+	});
+	m_fault_process_pending = true;
+}
+
 void GpuResourceManager::RunGarbageCollector() {
+	if (m_fault_process_pending) {
+		m_fault_process_pending = false;
+		m_buffer_cache.ProcessFaultBuffer();
+	}
 	m_texture_cache.ProcessDownloadImages();
 	m_texture_cache.RunGarbageCollector();
 	m_buffer_cache.RunGarbageCollector();

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -30,6 +30,7 @@ public:
 	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
 	void               MapMemory(uint64_t vaddr, uint64_t size);
 	void               UnmapMemory(uint64_t vaddr, uint64_t size);
+	void               PrepareBda();
 	void               RunGarbageCollector();
 
 private:
@@ -40,6 +41,7 @@ private:
 	mutable std::shared_mutex m_mapped_ranges_mutex;
 	RangeSet                  m_mapped_ranges;
 	GuestGpu*                 m_gpu = nullptr;
+	bool                      m_fault_process_pending = false;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -73,8 +73,12 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
 	buffer_info.usage       = flags;
 	buffer_info.sharingMode = vk::SharingMode::eExclusive;
 
+	const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
+	const VmaAllocationCreateFlags bda_flag =
+	    with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
 	VmaAllocationCreateInfo allocation_info {};
-	allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
+	allocation_info.flags =
+	    VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
 	allocation_info.usage = AllocationUsage(usage);
 	allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
 	                                     ? VkMemoryPropertyFlags {}
@@ -99,6 +103,13 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
 	m_buffer->memory.type            = allocation_result.memoryType;
 	m_buffer->memory.unique_id       = VulkanNextMemoryUniqueId();
 	graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
+	if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
+		vk::BufferDeviceAddressInfo address_info {};
+		address_info.sType  = vk::StructureType::eBufferDeviceAddressInfo;
+		address_info.buffer = m_buffer->buffer;
+		m_device_address    = graphics.device.getBufferAddress(address_info);
+		EXIT_IF(m_device_address == 0);
+	}
 
 	VkMemoryPropertyFlags properties = 0;
 	vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
@@ -122,6 +133,11 @@ vk::Buffer Buffer::Handle() const noexcept {
 	return m_buffer->buffer;
 }
 
+vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
+	EXIT_IF(m_device_address == 0);
+	return m_device_address;
+}
+
 bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
 	return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
 }
@@ -141,6 +157,15 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
 	}
 }
 
+void Buffer::Invalidate(uint64_t offset, uint64_t size) {
+	EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
+	if (!m_is_coherent && size != 0) {
+		const auto result =
+		    vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
+		EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
+	}
+}
+
 vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
                                         vk::AccessFlags destination) const {
 	if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
@@ -310,16 +335,6 @@ void StreamBuffer::Commit() {
 	watch.tick        = tick;
 }
 
-void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) {
-	EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset);
-	if (IsCoherent() || size == 0) {
-		return;
-	}
-	const auto result = vmaInvalidateAllocation(Graphics().allocator,
-	                                            NativeBuffer().memory.allocation, offset, size);
-	EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
-}
-
 uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
 	EXIT_IF(source == nullptr);
 	const auto [data, offset] = Map(size, alignment);

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -48,12 +48,14 @@ public:
 	[[nodiscard]] bool               IsCoherent() const noexcept { return m_is_coherent; }
 	[[nodiscard]] MemoryUsage        Usage() const noexcept { return m_usage; }
 	[[nodiscard]] uint64_t           CpuAddress() const noexcept { return m_cpu_address; }
+	[[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept;
 	[[nodiscard]] uint64_t           Offset(uint64_t address) const noexcept {
 		return address - m_cpu_address;
 	}
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
 	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
+	void               Invalidate(uint64_t offset, uint64_t size);
 	void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
 	              uint64_t destination_offset, uint64_t size,
 	              vk::AccessFlags source_before      = vk::AccessFlagBits::eMemoryWrite,
@@ -65,6 +67,10 @@ public:
 	                                                   vk::AccessFlagBits::eMemoryWrite);
 	void Fill(uint64_t offset, uint64_t size, uint32_t value);
 
+	// BufferCache state lives directly on the resource.
+	bool   is_deleted = false;
+	size_t lru_id     = 0;
+
 protected:
 	[[nodiscard]] GraphicContext&   Graphics() const noexcept { return *m_graphics; }
 	[[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; }
@@ -80,6 +86,7 @@ private:
 	MemoryUsage                   m_usage       = MemoryUsage::DeviceLocal;
 	uint64_t                      m_cpu_address = 0;
 	uint64_t                      m_size        = 0;
+	vk::DeviceAddress             m_device_address = 0;
 	std::unique_ptr<VulkanBuffer> m_buffer;
 	std::span<uint8_t>            m_mapped;
 	bool                          m_is_coherent = false;
@@ -93,9 +100,6 @@ public:
 	[[nodiscard]] std::pair<uint8_t*, uint64_t> Map(uint64_t size, uint64_t alignment = 0,
 	                                                bool allow_wait = true);
 	void                                        Commit();
-	// Download mappings become visible to the CPU only after their GPU completion tick is free.
-	// Call this from the scheduler's deferred completion operation before reading Mapped().
-	void                   Invalidate(uint64_t offset, uint64_t size);
 	[[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0);
 
 private:

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -4,17 +4,17 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "common/lruCache.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/pageManager.h"
 #include "graphics/host_gpu/regionManager.h"
 #include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/image/blitHelper.h"
 #include "graphics/host_gpu/renderer/image/image.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 
-#include <compare>
 #include <map>
-#include <memory>
-#include <set>
-#include <utility>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -25,8 +25,6 @@ class BufferCache;
 class CommandBuffer;
 class CommandScheduler;
 class RenderExecutor;
-class StreamBuffer;
-class TileManager;
 struct TextureCacheTestAccess;
 
 class TextureCache {
@@ -57,20 +55,26 @@ public:
 	[[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc);
 	[[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc);
 	[[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc);
-	[[nodiscard]] Image&        GetImage(ImageId id);
-	[[nodiscard]] const Image&  GetImage(ImageId id) const;
-	void                        MarkGpuWritten(ImageId id);
+	[[nodiscard]] Image&        GetImage(ImageId id) {
+		auto& image = m_slot_images[id];
+		TouchImage(image);
+		return image;
+	}
+	void MarkGpuWritten(ImageId id);
 
 	[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
-	[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
-	                                           bool formatted_buffer_write = false);
+	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
 	[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
-	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);
+	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice,
+	                                 uint32_t* fill_value = nullptr);
 	[[nodiscard]] bool ClearMeta(uint64_t address);
+	// Returns true when registered DCC absorbed the fill and the caller may skip the dispatch.
+	// False may still record PendingDcc state, but the guest dispatch must execute.
+	[[nodiscard]] bool TryConsumeDccFill(uint64_t address, uint64_t size, uint32_t fill_value);
 	[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
 
 	void UnmapMemory(uint64_t address, uint64_t size);
@@ -82,13 +86,18 @@ private:
 	struct ColorTransferPlan;
 	struct DownloadPlan;
 
-	struct Slot {
-		std::shared_ptr<Image> image;
-		uint32_t               generation = 1;
-	};
-
 	struct MetaDataInfo {
+		// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
+		// PendingDcc retains that exact fill until FindRenderTarget classifies the address,
+		// without exposing an unconfirmed buffer address to the normal metadata heuristics.
+		// Keep all surface metadata in one entry so CMask/FMask can be
+		// registered beside HTile and DCC without introducing parallel tracking paths.
+		enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc };
+
+		Type     type       = Type::PendingDcc;
 		uint32_t clear_mask = 0;
+		uint32_t fill_value = 0xffffffffu;
+		uint64_t fill_size  = 0;
 	};
 
 	struct OverlapResult {
@@ -100,26 +109,21 @@ private:
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
 	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
-	[[nodiscard]] Image&                 ResolveImage(ImageId id);
-	[[nodiscard]] const Image&           ResolveImage(ImageId id) const;
-	[[nodiscard]] std::shared_ptr<Image> ResolveOwner(ImageId id) const;
-	[[nodiscard]] ImageId                InsertImage(const ImageInfo& info);
-	[[nodiscard]] ImageId                GetNullImage(const ImageDesc& desc);
-	void                                 RegisterImage(ImageId id);
-	void                                 UnregisterImage(ImageId id);
-	void                                 DeleteImage(ImageId id);
-	void                                 FreeImage(ImageId id);
-	void                                 RetainImage(CommandBuffer& command, ImageId id);
-	void                                 TouchImage(Image& image);
-	void                                 TrackImage(ImageId id);
-	void                                 TrackImageHead(ImageId id);
-	void                                 TrackImageTail(ImageId id);
-	void                                 UntrackImage(ImageId id);
-	void                                 UntrackImageHead(ImageId id);
-	void                                 UntrackImageTail(ImageId id);
-	void                                 MarkAsMaybeDirty(ImageId id, Image& image);
-	void                                 TrackImageDownload(ImageId id);
-	void                                 TrackImageDownloadLocked(ImageId id, Image& image);
+	[[nodiscard]] ImageId     InsertImage(const ImageInfo& info);
+	[[nodiscard]] ImageId     GetNullImage(const ImageDesc& desc);
+	void                      RegisterImage(ImageId id);
+	void                      UnregisterImage(ImageId id);
+	void                      DeleteImage(ImageId id);
+	void                      FreeImage(ImageId id);
+	void                      TouchImage(Image& image);
+	void                      TrackImage(ImageId id);
+	void                      TrackImageHead(ImageId id);
+	void                      TrackImageTail(ImageId id);
+	void                      UntrackImage(ImageId id);
+	void                      UntrackImageHead(ImageId id);
+	void                      UntrackImageTail(ImageId id);
+	void                      MarkAsMaybeDirty(ImageId id, Image& image);
+	void                      TrackImageDownload(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
 	                                      bool exact_format);
 	[[nodiscard]] static BindingType UploadBinding(const Image& image);
@@ -148,31 +152,24 @@ private:
 	[[nodiscard]] bool CopyD16(Image& destination, Image& source);
 	void               CopyImage(ImageId destination, ImageId source);
 	void               AssociateStencil(ImageId depth, GuestRange stencil);
-	void               AssociateStencilLocked(ImageId depth, GuestRange stencil);
 	void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer);
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
-	void InvalidateCpuAliases(uint64_t address, uint64_t size);
-	void ClearGpuModified(ImageId id);
-
-	void                                        DownloadImage(ImageId id);
-	[[nodiscard]] bool                          TryDownloadImage(ImageId id);
-	[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
-	void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset);
+	void               InvalidateCpuAliases(uint64_t address, uint64_t size);
+	[[nodiscard]] bool TryDownloadImage(ImageId id);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
 	TrackingSpinLock                                  m_lock;
 	PageManager&                                      m_page_manager;
 	BlitHelper                                        m_blit_helper;
-	std::unique_ptr<TileManager>                      m_tiler;
+	TileManager                                       m_tiler;
 	BufferCache&                                      m_buffer_cache;
-	std::vector<Slot>                                 m_slots;
-	std::vector<uint32_t>                             m_free_slots;
+	Common::SlotVector<Image>                         m_slot_images;
 	ImagePageTable                                    m_image_page_table;
-	std::map<vk::Format, ImageId>                     m_null_images;
+	std::unordered_map<vk::Format, ImageId>           m_null_images;
 	Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
-	std::set<ImageId>                                 m_download_images;
+	std::unordered_set<ImageId>                       m_download_images;
 	std::map<uint64_t, MetaDataInfo>                  m_surface_metas;
 	uint64_t                                          m_total_used_memory  = 0;
 	uint64_t                                          m_trigger_gc_memory  = 0;

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -9,20 +9,42 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <algorithm>
+#include <array>
 #include <atomic>
 
 namespace Libs::Graphics {
 
 static std::atomic<uint32_t> g_render_color_log_count = 0;
 
+static void ResolveDccClearInfo(RenderColorInfo& info, vk::Format format, bool has_dcc,
+                                uint32_t packed_clear) {
+	// Register-backed DCC clears use the target's packed clear value. Decode one-word guest
+	// formats here; unsupported encodings remain tracked without unsafe materialization.
+	info.metadata_clear_supported =
+	    has_dcc && DecodePackedColorClear(format, packed_clear, info.color_clear_value);
+	switch (format) {
+		case vk::Format::eR8G8B8A8Unorm:
+		case vk::Format::eR8G8B8A8Srgb:
+		case vk::Format::eB8G8R8A8Unorm:
+		case vk::Format::eB8G8R8A8Srgb:
+		case vk::Format::eA2B10G10R10UnormPack32:
+		case vk::Format::eA2R10G10B10UnormPack32:
+			info.metadata_fixed_clear_supported = has_dcc;
+			break;
+		default: info.metadata_fixed_clear_supported = false; break;
+	}
+	if (!info.metadata_clear_supported) {
+		info.color_clear_value = {};
+	}
+}
+
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer& buffer,
                                               RenderColorInfo& r,
                                               uint32_t         render_target_slice_offset,
                                               uint32_t render_target_slot, bool ignore_target_mask,
@@ -55,20 +77,22 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		}
 
 		// No color output
-		r.type               = RenderColorType::NoColorOutput;
-		r.desc               = {};
-		r.base_addr          = 0;
-		r.image_id           = {};
-		r.image_view         = nullptr;
-		r.format             = vk::Format::eUndefined;
-		r.extent             = {};
-		r.base_mip_level     = 0;
-		r.base_array_layer   = 0;
-		r.buffer_size        = 0;
-		r.samples            = 1;
-		r.export_mapping     = {};
-		r.color_clear_enable = false;
-		r.color_clear_value  = {};
+		r.type                           = RenderColorType::NoColorOutput;
+		r.desc                           = {};
+		r.base_addr                      = 0;
+		r.image_id                       = {};
+		r.image_view                     = nullptr;
+		r.format                         = vk::Format::eUndefined;
+		r.extent                         = {};
+		r.base_mip_level                 = 0;
+		r.base_array_layer               = 0;
+		r.buffer_size                    = 0;
+		r.samples                        = 1;
+		r.export_mapping                 = {};
+		r.color_clear_enable             = false;
+		r.metadata_clear_supported       = false;
+		r.metadata_fixed_clear_supported = false;
+		r.color_clear_value              = {};
 		return;
 	}
 	const auto samples = render_sample_count(rt.attrib.num_fragments);
@@ -107,9 +131,9 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		}
 	}
 
-	// CB_COLOR_CONTROL describes the color-buffer operation / ROP3 logic op.
-	// ROP3 Copy is the normal color write path, not a render-target clear.
-	// SRGB clear words are still encoded as normalized component values.
+	// Color-control state selects the color-buffer operation and logical blend operation.
+	// The normal copy operation is a regular color write, not an attachment clear.
+	// Nonlinear clear values are still stored as normalized components.
 	// Fast color clears are metadata driven and must be handled explicitly when
 	// that metadata path is implemented; render-pass load must preserve contents.
 	r.color_clear_enable = false;
@@ -120,12 +144,23 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	uint32_t   pitch  = 0;
 	uint64_t   size   = 0;
 	bool       tile   = false;
-	const bool volume = rt.attrib3.dimension == 2;
-	if (rt.attrib3.dimension != 1 && !volume) {
+	static constexpr std::array image_types {Prospero::ImageType::kColor1D,
+	                                         Prospero::ImageType::kColor2D,
+	                                         Prospero::ImageType::kColor3D};
+	if (rt.attrib3.dimension >= image_types.size()) {
 		EXIT("unsupported render-target dimension: %u\n", rt.attrib3.dimension);
 	}
+	const auto image_type = image_types[rt.attrib3.dimension];
+	const bool is_1d      = image_type == Prospero::ImageType::kColor1D;
+	const bool volume     = image_type == Prospero::ImageType::kColor3D;
+	if (is_1d && rt.attrib2.height != 0) {
+		EXIT("1D render target has nonzero height: %u\n", rt.attrib2.height);
+	}
 	if (!volume && rt.attrib3.depth != 0) {
-		EXIT("2D render target has nonzero depth: %u\n", rt.attrib3.depth);
+		EXIT("non-3D render target has nonzero depth: %u\n", rt.attrib3.depth);
+	}
+	if (is_1d && samples != 1) {
+		EXIT("multisampled 1D render targets are unsupported\n");
 	}
 	if (volume && samples != 1) {
 		EXIT("multisampled 3D render targets are unsupported\n");
@@ -302,13 +337,20 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	desc.info.data         = {rt.base.addr, backing_size};
 	desc.info.pixel_format = target_format.format;
 	desc.info.guest_format = transfer_format;
-	desc.info.type         = volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D;
+	desc.info.type         = image_type;
 	desc.info.extent       = {width, height, depth};
 	desc.info.resources    = {levels, volume ? 1u : view.image_layers};
 	desc.info.pitch        = pitch;
 	desc.info.bytes_per_block = bytes_per_element;
 	desc.info.samples         = samples;
 	desc.info.tile_mode       = rt.attrib3.tile_mode;
+	const bool has_dcc        = rt.info.dcc_compression_enable && rt.dcc_addr.addr != 0;
+	if (has_dcc) {
+		// DCC uses a separate metadata allocation. Carry its address through ImageInfo so
+		// TextureCache can associate metadata fills observed before target registration.
+		desc.info.metadata.kind          = ImageMetadataKind::Dcc;
+		desc.info.metadata.range.address = rt.dcc_addr.addr;
+	}
 	for (uint32_t level = 0; level < levels; level++) {
 		if (volume) {
 			const auto& mip             = volume_layout.mips[level];
@@ -334,8 +376,13 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		};
 	}
 	desc.view_info.format = target_format.format;
-	desc.view_info.type =
-	    view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
+	if (is_1d) {
+		desc.view_info.type =
+		    view.layer_count == 1 ? vk::ImageViewType::e1D : vk::ImageViewType::e1DArray;
+	} else {
+		desc.view_info.type =
+		    view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
+	}
 	desc.view_info.aspect      = vk::ImageAspectFlagBits::eColor;
 	desc.view_info.base_level  = rt.view.current_mip_level;
 	desc.view_info.level_count = 1;
@@ -355,7 +402,7 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	r.samples                  = samples;
 	r.export_mapping           = target_format.export_mapping;
 	r.color_clear_enable       = false;
-	r.color_clear_value        = {};
+	ResolveDccClearInfo(r, target_format.format, has_dcc, rt.clear_word0.word0);
 	BindRenderTarget(r.image_id);
 }
 

src/graphics/host_gpu/renderer/colorRenderTarget.h

@@ -10,8 +10,6 @@
 
 namespace Libs::Graphics {
 
-class RenderCommandBuffer;
-
 enum class RenderColorType {
 	NoColorOutput,
 	RenderTexture,
@@ -31,7 +29,9 @@ struct RenderColorInfo {
 	uint32_t                        target_slot      = 0;
 	uint32_t                        samples          = 1;
 	Prospero::ColorComponentMapping export_mapping;
-	bool                            color_clear_enable = false;
+	bool                            color_clear_enable             = false;
+	bool                            metadata_clear_supported       = false;
+	bool                            metadata_fixed_clear_supported = false;
 	vk::ClearColorValue             color_clear_value {};
 };
 

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -1,96 +1,97 @@
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 
 #include "common/assert.h"
+#include "common/logging/log.h"
 #include "graphics/host_gpu/graphicContext.h"
 
 #include <algorithm>
+#include <cstdio>
+#include <optional>
 
 namespace Libs::Graphics {
 
 static thread_local CommandScheduler* g_deferred_callback_scheduler = nullptr;
 
-void CommandSlot::Reset() {
-	EXIT_IF(buffer == nullptr);
-	const auto result = buffer.reset(vk::CommandBufferResetFlagBits::eReleaseResources);
-	if (result != vk::Result::eSuccess) {
-		EXIT("failed to reset Vulkan command buffer: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
-	}
+namespace {
+
+void ReportVulkanFatal(const char* what, vk::Result result, uint64_t tick, uint32_t debug_op,
+                       uint64_t debug_submit, uint32_t arg0, uint32_t arg1, uint32_t arg2,
+                       uint32_t arg3, uint64_t arg4) {
+	LOGF("%s failed: %s (%d), tick=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	     what, VulkanToString(result).c_str(), static_cast<int>(result), tick, debug_op,
+	     debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::printf("%s failed: %s (%d), tick=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	            " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	            what, VulkanToString(result).c_str(), static_cast<int>(result), tick, debug_op,
+	            debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::fflush(stdout);
 }
 
-CommandScheduler::CommandPool::~CommandPool() {
-	Destroy();
-}
-
-void CommandScheduler::CommandPool::Create(GraphicContext& graphics) {
-	EXIT_IF(m_pool != nullptr || m_graphics != nullptr ||
-	        graphics.queue_family == static_cast<uint32_t>(-1));
-	m_graphics = &graphics;
+} // namespace
 
+CommandScheduler::CommandPool::CommandPool(GraphicContext& graphics, MasterSemaphore& master)
+    : m_graphics(graphics), m_master(master) {
+	EXIT_IF(graphics.queue_family == static_cast<uint32_t>(-1));
 	vk::CommandPoolCreateInfo create {};
 	create.sType            = vk::StructureType::eCommandPoolCreateInfo;
 	create.queueFamilyIndex = graphics.queue_family;
-	create.flags            = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
+	create.flags            = vk::CommandPoolCreateFlagBits::eTransient |
+	                          vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
 	const auto result       = graphics.device.createCommandPool(&create, nullptr, &m_pool);
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || m_pool == nullptr);
 }
 
-CommandSlot* CommandScheduler::CommandPool::CreateSlot() {
-	EXIT_IF(m_graphics == nullptr);
-	auto& graphics = *m_graphics;
+CommandScheduler::CommandPool::~CommandPool() {
+	m_graphics.device.destroyCommandPool(m_pool, nullptr);
+}
+
+size_t CommandScheduler::CommandPool::Grow() {
+	const auto first = m_ticks.size();
+	m_ticks.resize(first + GrowStep);
+	m_buffers.resize(first + GrowStep);
 
 	vk::CommandBufferAllocateInfo allocate {};
 	allocate.sType              = vk::StructureType::eCommandBufferAllocateInfo;
 	allocate.commandPool        = m_pool;
 	allocate.level              = vk::CommandBufferLevel::ePrimary;
-	allocate.commandBufferCount = 1;
-	vk::CommandBuffer buffer    = nullptr;
-	EXIT_IF(graphics.device.allocateCommandBuffers(&allocate, &buffer) != vk::Result::eSuccess);
-
-	vk::FenceCreateInfo fence_create {};
-	fence_create.sType = vk::StructureType::eFenceCreateInfo;
-	fence_create.flags = vk::FenceCreateFlagBits::eSignaled;
-	vk::Fence fence    = nullptr;
-	if (graphics.device.createFence(&fence_create, nullptr, &fence) != vk::Result::eSuccess) {
-		graphics.device.freeCommandBuffers(m_pool, 1, &buffer);
-		EXIT("failed to create command-buffer fence\n");
-	}
-
-	auto& slot      = m_slots.emplace_back();
-	slot.pool_mutex = &m_mutex;
-	slot.id         = static_cast<uint32_t>(m_slots.size() - 1);
-	slot.buffer     = buffer;
-	slot.fence      = fence;
-	return &slot;
+	allocate.commandBufferCount = static_cast<uint32_t>(GrowStep);
+	EXIT_IF(m_graphics.device.allocateCommandBuffers(&allocate, m_buffers.data() + first) !=
+	        vk::Result::eSuccess);
+	return first;
 }
 
-CommandSlot* CommandScheduler::CommandPool::Allocate(GraphicContext& graphics) {
-	Common::LockGuard lock(m_mutex);
-	if (m_pool == nullptr) {
-		Create(graphics);
-	}
-	EXIT_IF(m_graphics != &graphics);
-	auto  found = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; });
-	auto* slot  = found != m_slots.end() ? &*found : CreateSlot();
-	slot->busy  = true;
-	slot->Reset();
-	return slot;
-}
+vk::CommandBuffer CommandScheduler::CommandPool::Commit() {
+	auto       gpu_tick = m_master.KnownGpuTick();
+	const auto search   = [this, &gpu_tick](size_t begin, size_t end) -> std::optional<size_t> {
+		for (size_t index = begin; index < end; ++index) {
+			if (gpu_tick >= m_ticks[index]) {
+				m_ticks[index] = m_master.CurrentTick();
+				return index;
+			}
+		}
+		return std::nullopt;
+	};
 
-void CommandScheduler::CommandPool::Destroy() {
-	Common::LockGuard lock(m_mutex);
-	if (m_pool == nullptr) {
-		return;
+	auto found = search(m_hint, m_ticks.size());
+	if (!found) {
+		m_master.Refresh();
+		gpu_tick = m_master.KnownGpuTick();
+		found    = search(m_hint, m_ticks.size());
 	}
-	EXIT_IF(std::ranges::any_of(m_slots, [](const auto& slot) { return slot.busy; }));
-	EXIT_IF(m_graphics == nullptr);
-	for (const auto& slot: m_slots) {
-		m_graphics->device.destroyFence(slot.fence, nullptr);
+	if (!found) {
+		found = search(0, m_hint);
 	}
-	m_graphics->device.destroyCommandPool(m_pool, nullptr);
-	m_slots.clear();
-	m_pool     = nullptr;
-	m_graphics = nullptr;
+	if (!found) {
+		found           = Grow();
+		m_ticks[*found] = m_master.CurrentTick();
+	}
+
+	m_hint = *found + 1;
+	if (m_hint == m_ticks.size()) {
+		m_hint = 0;
+	}
+	return m_buffers[*found];
 }
 
 bool CommandScheduler::InDeferredOperation() noexcept {
@@ -99,10 +100,8 @@ bool CommandScheduler::InDeferredOperation() noexcept {
 
 CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics)
     : m_master(graphics), m_context(context), m_graphics(graphics),
-      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {
-	m_buffers.reserve(CommandBufferGrowStep);
-	m_buffer_ticks.reserve(CommandBufferGrowStep);
-}
+      m_command_pool(graphics, m_master), m_command(*this),
+      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {}
 
 CommandScheduler::~CommandScheduler() {
 	Shutdown();
@@ -128,9 +127,11 @@ void CommandScheduler::Shutdown() {
 		}
 		m_operation_state = OperationState::Draining;
 	}
-	if (Active() && m_recording) {
-		Finish();
+	if (!m_command.IsInvalid()) {
+		Submit();
 	}
+	m_master.Wait(CurrentTick() - 1);
+	PopPendingOperations();
 	DrainPriorityOperations();
 	m_priority_thread.request_stop();
 	m_operation_available.notify_all();
@@ -156,14 +157,10 @@ void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config
 	m_user_config = &user_config;
 	m_shaders     = &shaders;
 
-	if (!Active()) {
-		m_current = static_cast<int>(GrowCommandBuffers());
-	}
-
-	BindCurrent();
-	if (!m_recording) {
-		Current().Begin();
-		m_recording = true;
+	if (m_command.IsInvalid()) {
+		BeginNext();
+	} else {
+		BindCurrent();
 	}
 }
 
@@ -172,7 +169,7 @@ void CommandScheduler::BeginRendering(const RenderState& state) {
 }
 
 void CommandScheduler::EndRendering() {
-	if (Active() && m_recording) {
+	if (Active() && !m_command.IsInvalid()) {
 		Current().EndRendering();
 	}
 }
@@ -183,55 +180,40 @@ void CommandScheduler::Flush() {
 }
 
 void CommandScheduler::Flush(SubmitInfo& submit) {
-	SubmitCurrent(submit);
+	Submit(submit);
 	BeginNext();
 }
 
-CommandBuffer& CommandScheduler::FlushAndGetSubmitted() {
-	SubmitInfo submit;
-	auto&      submitted = SubmitCurrent(submit);
+void CommandScheduler::FlushAndWait() {
+	const auto tick = Submit();
+	m_master.Wait(tick);
 	BeginNext();
-	return submitted;
 }
 
 void CommandScheduler::Finish() {
 	CheckActive();
-	const auto tick = CurrentTick();
-	if (m_recording) {
-		SubmitInfo submit;
-		SubmitCurrent(submit);
+	if (!m_command.IsInvalid()) {
+		Submit();
 	}
-	for (auto& buffer: m_buffers) {
-		buffer->WaitForFenceAndReset();
-	}
-	m_master.Wait(tick);
+	m_master.Wait(CurrentTick() - 1);
+	BeginNext();
 	PopPendingOperations();
-	BindCurrent();
-	Current().Begin();
-	m_recording = true;
-}
-
-void CommandScheduler::FinishCurrent() {
-	SubmitInfo submit;
-	auto&      submitted = SubmitCurrent(submit);
-	submitted.WaitForFenceAndReset();
-	m_master.Refresh();
-	PopPendingOperations();
-	submitted.Begin();
-	m_recording = true;
 }
 
 void CommandScheduler::Wait(uint64_t tick) {
-	CheckActive();
 	EXIT_IF(tick > CurrentTick());
-	if (tick >= CurrentTick()) {
+	if (tick == CurrentTick()) {
+		CheckActive();
 		// A stream-buffer wrap can wait while a draw is being prepared through a reference to
-		// Current(). Recycle the same command object so that reference remains valid.
-		FinishCurrent();
-		return;
+		// Current(). The wrapper stays stable while its pooled Vulkan buffer is retired. Deferred
+		// resources are released only at the next GPU operation boundary.
+		const auto submitted_tick = Submit();
+		EXIT_IF(submitted_tick != tick);
+		m_master.Wait(tick);
+		BeginNext();
+	} else {
+		m_master.Wait(tick);
 	}
-	m_master.Wait(tick);
-	PopPendingOperations();
 }
 
 void CommandScheduler::PopPendingOperations() {
@@ -243,17 +225,18 @@ void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) {
 		m_master.Refresh();
 	}
 	for (;;) {
-		Common::UniqueFunction<void> callback;
+		PendingOperation operation;
 		{
 			std::lock_guard lock(m_operation_mutex);
 			if (m_pending_operations.empty() ||
 			    !m_master.IsFree(m_pending_operations.front().tick)) {
 				return;
 			}
-			callback = std::move(m_pending_operations.front().callback);
+			operation = std::move(m_pending_operations.front());
 			m_pending_operations.pop();
 		}
-		RunOperation(std::move(callback));
+		WaitPriorityOperations(operation.tick);
+		RunOperation(std::move(operation.callback));
 	}
 }
 
@@ -359,86 +342,81 @@ bool CommandScheduler::IsFree(uint64_t tick) {
 	return m_master.IsFree(tick);
 }
 
-CommandSlot* CommandScheduler::AllocateCommandBuffer() {
-	return m_command_pool.Allocate(m_graphics);
-}
-
-uint64_t CommandScheduler::NextSubmitSequence() noexcept {
-	return m_submit_sequence.fetch_add(1, std::memory_order_relaxed) + 1;
-}
-
 void CommandScheduler::CheckActive() const {
-	EXIT_IF(!Active() || static_cast<size_t>(m_current) >= m_buffers.size());
+	EXIT_IF(!Active());
 }
 
-RenderCommandBuffer& CommandScheduler::Current() const {
+CommandBuffer& CommandScheduler::Current() {
 	CheckActive();
-	EXIT_IF(m_buffers[m_current] == nullptr);
-	return *m_buffers[m_current];
+	return m_command;
 }
 
-void CommandScheduler::BindCurrent() const {
+void CommandScheduler::BindCurrent() {
 	EXIT_IF(m_registers == nullptr || m_user_config == nullptr || m_shaders == nullptr);
-	Current().Bind(*m_registers, *m_user_config, *m_shaders);
+	m_command.Bind(*m_registers, *m_user_config, *m_shaders);
 }
 
-CommandBuffer& CommandScheduler::SubmitCurrent(SubmitInfo& submit) {
-	CheckActive();
-	EXIT_IF(!m_recording);
-	auto& submitted = Current();
-	submitted.End();
-	const auto signal_tick = m_master.NextTick();
-	submit.AddSignal(m_master.Handle(), signal_tick);
-	submitted.Execute(submit);
-	m_buffer_ticks[static_cast<size_t>(m_current)] = signal_tick;
-	m_recording                                    = false;
-	return submitted;
+CommandBuffer& CommandScheduler::BeginCommand() {
+	EXIT_IF(!m_command.IsInvalid());
+	m_command.m_buffer = m_command_pool.Commit();
+	m_command.Begin();
+	return m_command;
 }
 
-int CommandScheduler::FindReusableBuffer(uint64_t gpu_tick) const {
-	EXIT_IF(m_buffer_ticks.size() != m_buffers.size());
-	const auto buffer_count = m_buffers.size();
-	for (size_t offset = 1; offset <= buffer_count; ++offset) {
-		const auto candidate = (static_cast<size_t>(m_current) + offset) % buffer_count;
-		if (gpu_tick >= m_buffer_ticks[candidate]) {
-			return static_cast<int>(candidate);
-		}
+uint64_t CommandScheduler::Submit(SubmitInfo submit) {
+	EXIT_IF(m_command.IsInvalid());
+	EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores ||
+	        submit.num_signal_semaphores >= SubmitInfo::MaxSemaphores);
+
+	m_command.End();
+	const auto buffer   = m_command.m_buffer;
+	auto&      graphics = m_graphics;
+	EXIT_IF(graphics.queue == nullptr);
+
+	vk::Result result;
+	uint64_t   tick;
+	{
+		Common::LockGuard lock(graphics.queue_mutex);
+		tick = m_master.NextTick();
+		submit.AddSignal(m_master.Handle(), tick);
+
+		vk::TimelineSemaphoreSubmitInfo timeline_info {};
+		timeline_info.sType                     = vk::StructureType::eTimelineSemaphoreSubmitInfo;
+		timeline_info.waitSemaphoreValueCount   = submit.num_wait_semaphores;
+		timeline_info.pWaitSemaphoreValues      = submit.wait_ticks.data();
+		timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores;
+		timeline_info.pSignalSemaphoreValues    = submit.signal_ticks.data();
+
+		vk::SubmitInfo submit_info {};
+		submit_info.sType                = vk::StructureType::eSubmitInfo;
+		submit_info.pNext                = &timeline_info;
+		submit_info.waitSemaphoreCount   = submit.num_wait_semaphores;
+		submit_info.pWaitSemaphores      = submit.wait_semaphores.data();
+		submit_info.pWaitDstStageMask    = submit.wait_stages.data();
+		submit_info.commandBufferCount   = 1;
+		submit_info.pCommandBuffers      = &buffer;
+		submit_info.signalSemaphoreCount = submit.num_signal_semaphores;
+		submit_info.pSignalSemaphores    = submit.signal_semaphores.data();
+
+		result = graphics.queue.submit(1, &submit_info, nullptr);
 	}
-	return -1;
-}
 
-size_t CommandScheduler::GrowCommandBuffers() {
-	const auto first = m_buffers.size();
-	const auto end   = first + CommandBufferGrowStep;
-	m_buffers.reserve(end);
-	m_buffer_ticks.reserve(end);
-	for (size_t i = first; i < end; ++i) {
-		m_buffers.emplace_back(std::make_unique<RenderCommandBuffer>(*this));
-		m_buffer_ticks.push_back(0);
+	if (result != vk::Result::eSuccess) {
+		ReportVulkanFatal("vkQueueSubmit", result, tick, m_command.m_debug_op,
+		                  m_command.m_debug_submit_id, m_command.m_debug_arg0,
+		                  m_command.m_debug_arg1, m_command.m_debug_arg2, m_command.m_debug_arg3,
+		                  m_command.m_debug_arg4);
 	}
-	return first;
+	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+
+	m_command.m_buffer = nullptr;
+	return tick;
 }
 
 void CommandScheduler::BeginNext() {
-	EXIT_IF(m_recording);
-
-	auto candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	bool refreshed = false;
-	if (candidate < 0) {
-		m_master.Refresh();
-		refreshed = true;
-		candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	}
-	if (candidate < 0) {
-		candidate = static_cast<int>(GrowCommandBuffers());
-	}
-
-	m_current = candidate;
-	Current().WaitForFenceAndReset();
-	PopPendingOperations(!refreshed);
+	EXIT_IF(!m_command.IsInvalid());
 	BindCurrent();
-	Current().Begin();
-	m_recording = true;
+	BeginCommand();
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -6,11 +6,7 @@
 #include "graphics/host_gpu/renderer/masterSemaphore.h"
 #include "graphics/host_gpu/renderer/render.h"
 
-#include <array>
-#include <atomic>
 #include <condition_variable>
-#include <deque>
-#include <memory>
 #include <mutex>
 
 #include <queue>
@@ -20,16 +16,6 @@
 
 namespace Libs::Graphics {
 
-struct CommandSlot {
-	Common::Mutex*    pool_mutex = nullptr;
-	uint32_t          id         = 0;
-	vk::CommandBuffer buffer     = nullptr;
-	vk::Fence         fence      = nullptr;
-	bool              busy       = false;
-
-	void Reset();
-};
-
 class CommandScheduler {
 public:
 	CommandScheduler(RenderContext& context, GraphicContext& graphics);
@@ -41,9 +27,10 @@ public:
 	void           EndRendering();
 	void           Flush();
 	void           Flush(SubmitInfo& submit);
-	CommandBuffer& FlushAndGetSubmitted();
+	void           FlushAndWait();
 	void           Finish();
-	void           FinishCurrent();
+	CommandBuffer& BeginCommand();
+	uint64_t       Submit(SubmitInfo submit = {});
 	// Deferred callbacks can observe an externally owned drain, but cannot initiate shutdown:
 	// the priority runner cannot join itself.
 	void                      Shutdown();
@@ -55,34 +42,35 @@ public:
 	void                      DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
 	[[nodiscard]] static bool InDeferredOperation() noexcept;
 
-	[[nodiscard]] bool            Active() const noexcept { return m_current >= 0; }
-	void                          CheckActive() const;
-	RenderCommandBuffer&          Current() const;
-	[[nodiscard]] uint64_t        CurrentTick() const noexcept { return m_master.CurrentTick(); }
-	[[nodiscard]] bool            IsFree(uint64_t tick);
-	[[nodiscard]] RenderContext&  Context() const noexcept { return m_context; }
-	[[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; }
+	[[nodiscard]] bool             Active() const noexcept { return m_registers != nullptr; }
+	void                           CheckActive() const;
+	CommandBuffer&                 Current();
+	[[nodiscard]] uint64_t         CurrentTick() const noexcept { return m_master.CurrentTick(); }
+	[[nodiscard]] bool             IsFree(uint64_t tick);
+	[[nodiscard]] MasterSemaphore& GetMasterSemaphore() noexcept { return m_master; }
+	[[nodiscard]] RenderContext&   Context() const noexcept { return m_context; }
+	[[nodiscard]] GraphicContext&  Graphics() const noexcept { return m_graphics; }
 
 private:
-	static constexpr size_t CommandBufferGrowStep = 4;
-
 	class CommandPool {
 	public:
-		CommandPool() = default;
+		CommandPool(GraphicContext& graphics, MasterSemaphore& master);
 		~CommandPool();
 		KYTY_CLASS_NO_COPY(CommandPool);
 
-		CommandSlot* Allocate(GraphicContext& graphics);
+		vk::CommandBuffer Commit();
 
 	private:
-		void         Create(GraphicContext& graphics);
-		CommandSlot* CreateSlot();
-		void         Destroy();
+		static constexpr size_t GrowStep = 4;
 
-		GraphicContext*         m_graphics = nullptr;
-		Common::Mutex           m_mutex;
-		vk::CommandPool         m_pool = nullptr;
-		std::deque<CommandSlot> m_slots;
+		size_t Grow();
+
+		GraphicContext&                m_graphics;
+		MasterSemaphore&               m_master;
+		vk::CommandPool                m_pool = nullptr;
+		std::vector<vk::CommandBuffer> m_buffers;
+		std::vector<uint64_t>          m_ticks;
+		size_t                         m_hint = 0;
 	};
 
 	enum class OperationState { Open, Draining, Closed };
@@ -92,39 +80,28 @@ private:
 		uint64_t                     tick = 0;
 	};
 
-	void                       BindCurrent() const;
-	CommandBuffer&             SubmitCurrent(SubmitInfo& submit);
-	void                       BeginNext();
-	void                       PopPendingOperations(bool refresh_gpu_tick);
-	[[nodiscard]] int          FindReusableBuffer(uint64_t gpu_tick) const;
-	[[nodiscard]] size_t       GrowCommandBuffers();
-	void                       PriorityOperationsThread(std::stop_token stop);
-	void                       RunOperation(Common::UniqueFunction<void>&& operation);
-	[[nodiscard]] CommandSlot* AllocateCommandBuffer();
-	[[nodiscard]] uint64_t     NextSubmitSequence() noexcept;
+	void BindCurrent();
+	void BeginNext();
+	void PopPendingOperations(bool refresh_gpu_tick);
+	void PriorityOperationsThread(std::stop_token stop);
+	void RunOperation(Common::UniqueFunction<void>&& operation);
 
-	MasterSemaphore                                   m_master;
-	RenderContext&                                    m_context;
-	GraphicContext&                                   m_graphics;
-	CommandPool                                       m_command_pool;
-	std::vector<std::unique_ptr<RenderCommandBuffer>> m_buffers;
-	std::vector<uint64_t>                             m_buffer_ticks;
-	std::queue<PendingOperation>                      m_pending_operations;
-	std::queue<PendingOperation>                      m_priority_operations;
-	std::mutex                                        m_operation_mutex;
-	std::condition_variable                           m_operation_available;
-	std::jthread                                      m_priority_thread;
-	bool                                              m_priority_active      = false;
-	uint64_t                                          m_priority_active_tick = 0;
-	OperationState                                    m_operation_state      = OperationState::Open;
-	int                                               m_current              = -1;
-	bool                                              m_recording            = false;
-	HW::Context*                                      m_registers            = nullptr;
-	HW::UserConfig*                                   m_user_config          = nullptr;
-	HW::Shader*                                       m_shaders              = nullptr;
-	std::atomic<uint64_t>                             m_submit_sequence      = 0;
-
-	friend class CommandBuffer;
+	MasterSemaphore              m_master;
+	RenderContext&               m_context;
+	GraphicContext&              m_graphics;
+	CommandPool                  m_command_pool;
+	CommandBuffer                m_command;
+	std::queue<PendingOperation> m_pending_operations;
+	std::queue<PendingOperation> m_priority_operations;
+	std::mutex                   m_operation_mutex;
+	std::condition_variable      m_operation_available;
+	std::jthread                 m_priority_thread;
+	bool                         m_priority_active      = false;
+	uint64_t                     m_priority_active_tick = 0;
+	OperationState               m_operation_state      = OperationState::Open;
+	HW::Context*                 m_registers            = nullptr;
+	HW::UserConfig*              m_user_config          = nullptr;
+	HW::Shader*                  m_shaders              = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/context.cpp

@@ -1,7 +1,5 @@
 #include "common/assert.h"
 #include "common/common.h"
-#include "common/emulatorConfig.h"
-#include "common/logging/log.h"
 #include "common/profiler.h"
 #include "common/threads.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -9,7 +7,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vma.h"
@@ -18,98 +15,28 @@
 #include <algorithm>
 #include <bit>
 #include <cstring>
-#include <memory>
 namespace Libs::Graphics {
 
-FenceResourceRetainer::~FenceResourceRetainer() {
-	if (!m_resources.empty()) {
-		EXIT("fence resource retainer destroyed before release\n");
-	}
-}
-
-void FenceResourceRetainer::Retain(std::shared_ptr<void> resource) {
-	if (resource == nullptr) {
-		EXIT("cannot retain a null fence resource\n");
-	}
-	if (std::ranges::none_of(m_resources, [&resource](const auto& retained) {
-		    return retained.get() == resource.get();
-	    })) {
-		m_resources.push_back(std::move(resource));
-	}
-}
-
-void FenceResourceRetainer::ReleaseAfterFence() noexcept {
-	m_resources.clear();
-}
-
 CommandBuffer::CommandBuffer(CommandScheduler& scheduler)
-    : m_context(scheduler.Context()), m_scheduler(scheduler), m_graphics(scheduler.Graphics()),
-      m_slot(scheduler.AllocateCommandBuffer()) {}
-
-CommandBuffer::~CommandBuffer() {
-	Release();
-}
+    : m_context(scheduler.Context()), m_graphics(scheduler.Graphics()) {}
 
 bool CommandBuffer::IsInvalid() const {
-	return m_slot == nullptr;
+	return m_buffer == nullptr;
 }
 
 vk::CommandBuffer CommandBuffer::Handle() const {
 	EXIT_IF(IsInvalid());
-
-	const auto handle = m_slot->buffer;
-	EXIT_IF(handle == nullptr);
-	return handle;
+	return m_buffer;
 }
 
-void CommandBuffer::Release() {
-	EXIT_IF(IsInvalid());
-
-	Common::LockGuard lock(*m_slot->pool_mutex);
-
-	WaitForFence();
-
-	m_slot->busy = false;
-	m_slot->Reset();
-	ReleaseResourcesAfterFence();
-	m_slot = nullptr;
-
-	EXIT_NOT_IMPLEMENTED(!IsInvalid());
-}
-
-void CommandBuffer::RetireBufferAfterFence(std::unique_ptr<VulkanBuffer> buffer) {
-	if (IsInvalid() || m_execute || buffer == nullptr || buffer->buffer == nullptr) {
-		EXIT("cannot retire a buffer on an invalid or submitted command buffer\n");
-	}
-	m_retired_buffers.push_back(std::move(buffer));
-}
-
-void CommandBuffer::RetainResourceUntilFence(std::shared_ptr<void> resource) {
-	if (IsInvalid() || m_execute) {
-		EXIT("cannot retain a resource on an invalid or submitted command buffer\n");
-	}
-	m_fence_resources.Retain(std::move(resource));
-}
-
-void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet& set) {
-	m_descriptor_sets_after_fence.push_back(&set);
-}
-
-void CommandBuffer::RecycleDescriptorsAfterFence() {
-	for (auto* set: m_descriptor_sets_after_fence) {
-		m_context.GetDescriptorCache().Recycle(*set);
-	}
-	m_descriptor_sets_after_fence.clear();
-}
-
-void CommandBuffer::Begin() const {
-	EXIT_IF(m_rendering);
+void CommandBuffer::Begin() {
+	EXIT_IF(m_rendering || IsInvalid());
 	auto buffer = Handle();
 
 	vk::CommandBufferBeginInfo begin_info {};
 	begin_info.sType            = vk::StructureType::eCommandBufferBeginInfo;
 	begin_info.pNext            = nullptr;
-	begin_info.flags            = {};
+	begin_info.flags            = vk::CommandBufferUsageFlagBits::eOneTimeSubmit;
 	begin_info.pInheritanceInfo = nullptr;
 
 	auto result = buffer.begin(&begin_info);
@@ -137,125 +64,6 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0,
 	m_debug_arg4      = arg4;
 }
 
-void CommandBuffer::Execute(const SubmitInfo& submit) {
-	EXIT_IF(IsInvalid());
-	EXIT_IF(m_execute);
-	EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores ||
-	        submit.num_signal_semaphores > SubmitInfo::MaxSemaphores);
-
-	auto buffer = Handle();
-	auto fence  = m_slot->fence;
-
-	vk::TimelineSemaphoreSubmitInfo timeline_info {};
-	timeline_info.sType                     = vk::StructureType::eTimelineSemaphoreSubmitInfo;
-	timeline_info.waitSemaphoreValueCount   = submit.num_wait_semaphores;
-	timeline_info.pWaitSemaphoreValues      = submit.wait_ticks.data();
-	timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores;
-	timeline_info.pSignalSemaphoreValues    = submit.signal_ticks.data();
-
-	vk::SubmitInfo submit_info {};
-	submit_info.sType                = vk::StructureType::eSubmitInfo;
-	submit_info.pNext                = &timeline_info;
-	submit_info.waitSemaphoreCount   = submit.num_wait_semaphores;
-	submit_info.pWaitSemaphores      = submit.wait_semaphores.data();
-	submit_info.pWaitDstStageMask    = submit.wait_stages.data();
-	submit_info.commandBufferCount   = 1;
-	submit_info.pCommandBuffers      = &buffer;
-	submit_info.signalSemaphoreCount = submit.num_signal_semaphores;
-	submit_info.pSignalSemaphores    = submit.signal_semaphores.data();
-
-	auto& graphics = m_graphics;
-	EXIT_IF(graphics.queue == nullptr);
-
-	auto result = graphics.device.resetFences(1, &fence);
-	if (result != vk::Result::eSuccess) {
-		LOGF("vkResetFences failed before submit: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	if (Config::GraphicsDebugDumpEnabled()) {
-		LOGF("vkQueueSubmit begin: slot=%u waits=%u signals=%u debug_op=%u debug_submit=%" PRIu64
-		     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     m_slot->id, submit.num_wait_semaphores, submit.num_signal_semaphores, m_debug_op,
-		     m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-
-	{
-		Common::LockGuard lock(graphics.queue_mutex);
-		m_submit_seq = m_scheduler.NextSubmitSequence();
-		result       = graphics.queue.submit(1, &submit_info, fence);
-	}
-
-	m_execute      = true;
-	m_fence_waited = false;
-
-	if (result != vk::Result::eSuccess) {
-		LOGF("vkQueueSubmit failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-}
-
-void CommandBuffer::WaitForFence() {
-	FinalizeFence(false);
-}
-
-void CommandBuffer::WaitForFenceOnly() {
-	EXIT_IF(IsInvalid());
-	if (!m_execute || m_fence_waited) {
-		return;
-	}
-	auto device = m_graphics.device;
-	auto result = device.waitForFences(1, &m_slot->fence, VK_TRUE, UINT64_MAX);
-	if (result != vk::Result::eSuccess) {
-		LOGF("vkWaitForFences failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-	m_fence_waited = true;
-}
-
-void CommandBuffer::WaitForFenceAndReset() {
-	FinalizeFence(true);
-}
-
-void CommandBuffer::FinalizeFence(bool reset_recording) {
-	const bool was_executed = m_execute;
-	WaitForFenceOnly();
-	if (was_executed) {
-		m_execute      = false;
-		m_fence_waited = false;
-		if (reset_recording) {
-			Common::LockGuard lock(*m_slot->pool_mutex);
-			m_slot->Reset();
-		}
-	}
-	if (was_executed) {
-		ReleaseResourcesAfterFence();
-	}
-	DeleteBuffersAfterFence();
-}
-
-void CommandBuffer::ReleaseResourcesAfterFence() {
-	RecycleDescriptorsAfterFence();
-	m_fence_resources.ReleaseAfterFence();
-}
-
-void CommandBuffer::DeleteBuffersAfterFence() {
-	for (const auto& buffer: m_retired_buffers) {
-		m_graphics.DeleteBuffer(*buffer);
-	}
-	m_retired_buffers.clear();
-}
-
 void CommandBuffer::BeginRendering(const RenderState& state) const {
 	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
 	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);

src/graphics/host_gpu/renderer/debug.cpp

@@ -27,7 +27,7 @@ static bool RenderTargetMaskHasMrt(uint32_t mask) {
 	return (mask & ~0x0fu) != 0;
 }
 
-static bool RenderTargetMaskHasBoundMrt(const RenderCommandBuffer& buffer) {
+static bool RenderTargetMaskHasBoundMrt(const CommandBuffer& buffer) {
 	const auto& hw   = buffer.GetRegisters();
 	const auto  mask = hw.GetRenderTargetMask();
 
@@ -45,7 +45,7 @@ static bool RenderTargetMaskHasBoundMrt(const RenderCommandBuffer& buffer) {
 	return bound_targets > 1;
 }
 
-uint32_t render_target_first_bound_slot(const RenderCommandBuffer& buffer) {
+uint32_t render_target_first_bound_slot(const CommandBuffer& buffer) {
 	const auto& hw   = buffer.GetRegisters();
 	const auto  mask = hw.GetRenderTargetMask();
 	for (uint32_t i = 0; i < 8; i++) {
@@ -266,20 +266,23 @@ static void RtCheck(const HW::RenderTarget& rt) {
 				logged = true;
 			}
 		}
+		const bool native_2x_one_frag =
+		    rt.info.fmask_compression_enable && !rt.info.fmask_data_compression_disable &&
+		    rt.info.cmask_fast_clear_enable && rt.fmask.addr != 0 && rt.cmask.addr != 0 &&
+		    rt.attrib.num_samples == 1 && rt.attrib.num_fragments == 1;
+		EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode && !native_2x_one_frag);
 		if (rt.info.fmask_compression_enable) {
 			EXIT_NOT_IMPLEMENTED(rt.attrib.num_samples == 0 && rt.attrib.num_fragments == 0);
 			static bool logged = false;
 			if (!logged) {
-				LOGF("RenderTarget: temporary: ignoring PS5 FMASK metadata for single-sample MSAA "
-				     "fallback, fmask=0x%016" PRIx64 "\n",
+				LOGF("RenderTarget: using native Vulkan MSAA without guest FMASK metadata, "
+				     "fmask=0x%016" PRIx64 "\n",
 				     rt.fmask.addr);
 				logged = true;
 			}
 		}
 
-		// EXIT_NOT_IMPLEMENTED(rt.info.fmask_compression_mode != 0x00000000);
 		EXIT_NOT_IMPLEMENTED(rt.info.fmask_data_compression_disable != false);
-		EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode != false);
 
 		if (rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable) {
 			static bool logged = false;
@@ -561,14 +564,11 @@ static void ZCheck(const HW::DepthRenderTarget& z, const HW::DepthControl& dc,
 				     z.depth_view.stencil_write_disable ? "true" : "false");
 			}
 		}
-		EXIT_NOT_IMPLEMENTED(z.z_read_base_addr != z.z_write_base_addr);
-		EXIT_NOT_IMPLEMENTED(z.stencil_read_base_addr != z.stencil_write_base_addr);
-		EXIT_NOT_IMPLEMENTED(z.z_write_base_addr == 0);
-		if (z.stencil_info.format != Prospero::StencilFormat::kInvalid &&
-		    z.stencil_write_base_addr == 0) {
-			LOGF("\t warning: stencil format is set without a stencil base address, continuing "
-			     "without stencil attachment\n");
-		}
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.depth_write_disable &&
+		                     z.z_read_base_addr != z.z_write_base_addr);
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.stencil_write_disable &&
+		                     z.stencil_read_base_addr != z.stencil_write_base_addr);
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.depth_write_disable && z.z_write_base_addr == 0);
 		// EXIT_NOT_IMPLEMENTED(z.pitch_div8_minus1 != 0x000000ff);
 		// EXIT_NOT_IMPLEMENTED(z.height_div8_minus1 != 0x0000008f);
 		// EXIT_NOT_IMPLEMENTED(z.slice_div64_minus1 != 0x00008fff);
@@ -888,12 +888,10 @@ void LogDrawPhase(const char* draw_name, const char* phase) {
 	}
 }
 
-void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
-                      uint32_t ps_crc32) {
+void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id) {
 	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: %s VS=0x%08" PRIx32 "/0x%08" PRIx32 " PS=0x%08" PRIx32 "/0x%08" PRIx32
-		     "\n",
-		     phase, vs_hash0, vs_crc32, ps_hash0, ps_crc32);
+		LOGF("PipelineTrace: %s VS=%" PRIu64 " PS=%" PRIu64 "\n", phase, vertex_program_id,
+		     pixel_program_id);
 	}
 }
 
@@ -1157,7 +1155,7 @@ ScissorRect calc_final_scissor(const HW::ScreenViewport& vp, const HW::ScanModeC
 	return ScissorRectClamp(final, extent.width, extent.height);
 }
 
-void hw_check(const RenderCommandBuffer& buffer) {
+void hw_check(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
@@ -1230,7 +1228,7 @@ void hw_check(const RenderCommandBuffer& buffer) {
 	// EXIT_NOT_IMPLEMENTED(hw.GetStencilClearValue() != 0);
 }
 
-void hw_print(const RenderCommandBuffer& buffer) {
+void hw_print(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);

src/graphics/host_gpu/renderer/debug.h

@@ -9,7 +9,7 @@
 
 namespace Libs::Graphics {
 
-class RenderCommandBuffer;
+class CommandBuffer;
 
 namespace HW {
 class Context;
@@ -28,15 +28,15 @@ struct ScissorRect {
 };
 
 uint32_t                 render_target_mask_slot(uint32_t mask, uint32_t slot);
-uint32_t                 render_target_first_bound_slot(const RenderCommandBuffer& buffer);
+uint32_t                 render_target_first_bound_slot(const CommandBuffer& buffer);
 bool                     graphics_debug_dump_enabled();
 void                     uc_print(const char* func, const HW::UserConfig& uc);
 void                     uc_check(const HW::UserConfig& uc);
 void                     sh_print(const char* func, const HW::Shader& uc);
 std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt);
 bool                     RenderIsColorTileModeLinear(Prospero::TileMode tile_mode);
-void                     hw_print(const RenderCommandBuffer& buffer);
-void                     hw_check(const RenderCommandBuffer& buffer);
+void                     hw_print(const CommandBuffer& buffer);
+void                     hw_check(const CommandBuffer& buffer);
 void                     LogDrawPhase(const char* draw_name, const char* phase);
 ScissorRect calc_final_scissor(const HW::ScreenViewport& vp, const HW::ScanModeControl& smc,
                                vk::Extent2D extent);

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -13,7 +13,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -71,8 +70,8 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
 	return fail == replace_op || pass == replace_op || depth_fail == replace_op;
 }
 
-[[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(const RenderCommandBuffer& buffer,
-                                                                 const DepthFormatPolicy&   policy,
+[[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(const CommandBuffer&     buffer,
+                                                                 const DepthFormatPolicy& policy,
                                                                  bool     has_stencil,
                                                                  uint32_t samples) {
 	auto&      graphics         = buffer.GetGraphics();
@@ -98,7 +97,7 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer& buffer,
                                               RenderDepthInfo& r) {
 	KYTY_PROFILER_FUNCTION();
 	(void)submit_id;
@@ -177,15 +176,17 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	    z.htile_surface.htile_uses_preload_win != 0 || z.htile_surface.preload != 0 ||
 	    z.htile_surface.prefetch_width != 0 || z.htile_surface.prefetch_height != 0 ||
 	    z.htile_surface.dst_outside_zero_to_one != 0 || z.z_read_base_addr == 0 ||
-	    z.z_write_base_addr != z.z_read_base_addr || (z.z_read_base_addr & 0xffffu) != 0 ||
+	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
+	    (z.z_read_base_addr & 0xffffu) != 0 ||
 	    dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways)) {
 		DepthFatal("unsupported depth register state");
 	}
 	if (has_stencil) {
 		if (z.stencil_info.format != Prospero::StencilFormat::k8UInt || !htile_stencil_compat ||
 		    z.stencil_read_base_addr == 0 ||
-		    z.stencil_write_base_addr != z.stencil_read_base_addr ||
-		    (z.stencil_read_base_addr & 0xffffu) != 0 || z.depth_view.stencil_write_disable) {
+		    (!z.depth_view.stencil_write_disable &&
+		     z.stencil_write_base_addr != z.stencil_read_base_addr) ||
+		    (z.stencil_read_base_addr & 0xffffu) != 0) {
 			DepthFatal("unsupported stencil attachment state");
 		}
 	} else if (z.stencil_read_base_addr != 0 || z.stencil_write_base_addr != 0 ||
@@ -286,18 +287,22 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	r.depth_min_bounds         = hw.GetDepthBoundsMin();
 	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
-	r.stencil_clear_enable = has_stencil && rc.stencil_clear_enable;
-	r.stencil_clear_value  = hw.GetStencilClearValue();
-	r.stencil_test_enable  = has_stencil && dc.stencil_enable;
+	r.stencil_clear_enable =
+	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;
+	r.stencil_clear_value = hw.GetStencilClearValue();
+	r.stencil_test_enable = has_stencil && dc.stencil_enable;
 	if (r.stencil_test_enable) {
-		const uint8_t front_write_mask = rc.stencil_clear_enable ? 0 : sm.stencil_writemask;
-		const uint8_t back_write_mask  = rc.stencil_clear_enable ? 0 : sm.stencil_writemask_bf;
+		const bool stencil_ops_disabled =
+		    rc.stencil_clear_enable || z.depth_view.stencil_write_disable;
+		const uint8_t front_write_mask = stencil_ops_disabled ? 0 : sm.stencil_writemask;
+		const uint8_t back_write_mask  = stencil_ops_disabled ? 0 : sm.stencil_writemask_bf;
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
 		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||
-		    (UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
+		    (front_write_mask != 0 &&
+		     UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
 		     sm.stencil_opval != sm.stencil_testval) ||
-		    (dc.backface_enable &&
+		    (dc.backface_enable && back_write_mask != 0 &&
 		     UsesStencilOpValue(sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf) &&
 		     sm.stencil_opval_bf != sm.stencil_testval_bf)) {
 			DepthFatal("unsupported stencil compare or replacement state");

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -11,8 +11,6 @@
 
 namespace Libs::Graphics {
 
-class RenderCommandBuffer;
-
 inline constexpr bool depth_htile_stencil_acceleration_compatible(bool has_stencil, bool has_htile,
                                                                   bool htile_stencil_disabled) {
 	return htile_stencil_disabled || (has_stencil && has_htile);

src/graphics/host_gpu/renderer/image/image.cpp

@@ -736,15 +736,10 @@ Image::~Image() {
 	if (m_graphics == nullptr) {
 		return;
 	}
-	{
-		std::lock_guard lock(views.mutex);
-		for (auto& cached: views.views) {
-			if (cached.view != nullptr) {
-				m_graphics->device.destroyImageView(cached.view, nullptr);
-				cached.view = nullptr;
-			}
+	for (const auto& cached: views) {
+		if (cached.view != nullptr) {
+			m_graphics->device.destroyImageView(cached.view, nullptr);
 		}
-		views.views.clear();
 	}
 	if (backing.image != nullptr) {
 		m_graphics->DeleteImage(backing);

src/graphics/host_gpu/renderer/image/image.h

@@ -2,12 +2,12 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 
 #include "common/assert.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
 
 #include <compare>
 #include <limits>
-#include <mutex>
 #include <optional>
 #include <span>
 #include <utility>
@@ -19,29 +19,13 @@ class Buffer;
 class CommandScheduler;
 struct ImageTestAccess;
 
-struct ImageId {
-	uint32_t index      = std::numeric_limits<uint32_t>::max();
-	uint32_t generation = 0;
-
-	[[nodiscard]] explicit operator bool() const noexcept {
-		return index != std::numeric_limits<uint32_t>::max();
-	}
-	auto operator<=>(const ImageId&) const = default;
-};
+using ImageId = Common::SlotId;
 
 struct CachedImageView {
 	ImageViewInfo info;
 	vk::ImageView view = nullptr;
 };
 
-struct ImageViewCache {
-	ImageViewCache() = default;
-	KYTY_CLASS_NO_COPY(ImageViewCache);
-
-	std::mutex                   mutex;
-	std::vector<CachedImageView> views;
-};
-
 struct ImageUsage {
 	bool texture       = false;
 	bool storage       = false;
@@ -157,7 +141,7 @@ public:
 
 	ImageInfo        info;
 	VulkanImage      backing;
-	ImageViewCache   views;
+	std::vector<CachedImageView> views;
 	ImageUsage       usage;
 	ImageBinding     binding;
 	bool             registered     = false;

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -4,8 +4,6 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/image.h"
 
-#include <mutex>
-
 namespace Libs::Graphics {
 
 namespace {
@@ -357,8 +355,7 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     image.layers);
 	}
 
-	std::lock_guard lock(views.mutex);
-	for (const auto& cached: views.views) {
+	for (const auto& cached: views) {
 		if (cached.info == normalized) {
 			return cached.view;
 		}
@@ -394,7 +391,7 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     view_info.level_count, view_info.base_layer, view_info.layer_count,
 		     static_cast<vk::ImageUsageFlags::MaskType>(view_info.usage));
 	}
-	views.views.push_back({normalized, view});
+	views.push_back({normalized, view});
 	return view;
 }
 

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -119,13 +119,16 @@ vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) {
 	return components;
 }
 
-vk::Format TextureGetFormat(Prospero::BufferFormat format) {
-	const auto vk_format = VulkanFormat(format);
+SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
+	const auto backing_format    = Prospero::RemapTextureFormat(format);
+	const auto vk_format         = VulkanFormat(backing_format);
+	const auto conversion_format =
+	    backing_format != format ? format : Prospero::BufferFormat::kInvalid;
 	if (vk_format != vk::Format::eUndefined) {
-		return vk_format;
+		return SurfaceFormatInfo(vk_format, conversion_format);
 	}
 	EXIT("unknown format: fmt = %u\n", static_cast<uint32_t>(format));
-	return vk::Format::eUndefined;
+	return SurfaceFormatInfo(vk::Format::eUndefined, Prospero::BufferFormat::kInvalid);
 }
 
 namespace {

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -3,7 +3,7 @@
 
 #include "common/abi.h"
 #include "common/common.h"
-#include "graphics/guest_gpu/gpu_defs.h"
+#include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
@@ -19,6 +19,15 @@ struct RenderTargetFormatInfo {
 	Prospero::ColorComponentMapping export_mapping;
 };
 
+struct SurfaceFormatInfo {
+	explicit constexpr SurfaceFormatInfo(vk::Format format,
+	                                     Prospero::BufferFormat conversion_format)
+	    : vk_format(format), conversion_format(conversion_format) {}
+
+	vk::Format             vk_format;
+	Prospero::BufferFormat conversion_format;
+};
+
 struct TextureUploadMipLayout {
 	uint64_t offset       = 0;
 	uint64_t size         = 0;
@@ -35,7 +44,7 @@ struct TextureUploadLayout {
 };
 
 vk::ComponentMapping   TextureGetComponentMapping(uint32_t swizzle);
-vk::Format             TextureGetFormat(Prospero::BufferFormat format);
+SurfaceFormatInfo      TextureGetSurfaceFormatInfo(Prospero::BufferFormat format);
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
                                                     Prospero::ChannelType   type,
                                                     Prospero::ChannelOrder  order);

src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp

@@ -1,313 +0,0 @@
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
-
-#include "common/assert.h"
-#include "common/profiler.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-#include <array>
-
-namespace Libs::Graphics {
-namespace {
-
-using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind;
-
-vk::ShaderStageFlags StageFlags(DescriptorCache::Stage stage) {
-	switch (stage) {
-		case DescriptorCache::Stage::Vertex: return vk::ShaderStageFlagBits::eVertex;
-		case DescriptorCache::Stage::Pixel: return vk::ShaderStageFlagBits::eFragment;
-		case DescriptorCache::Stage::Compute: return vk::ShaderStageFlagBits::eCompute;
-		default: EXIT("unknown descriptor stage\n");
-	}
-}
-
-bool IsSampledImage(BindingKind kind) {
-	switch (kind) {
-		case BindingKind::Sampled1D:
-		case BindingKind::Sampled1DArray:
-		case BindingKind::Sampled2D:
-		case BindingKind::Sampled2DArray:
-		case BindingKind::Sampled2DMsaa:
-		case BindingKind::Sampled2DMsaaArray:
-		case BindingKind::Sampled3D:
-		case BindingKind::SampledUint1D:
-		case BindingKind::SampledUint1DArray:
-		case BindingKind::SampledUint2D:
-		case BindingKind::SampledUint2DArray:
-		case BindingKind::SampledUint2DMsaa:
-		case BindingKind::SampledUint2DMsaaArray:
-		case BindingKind::SampledUint3D: return true;
-		default: return false;
-	}
-}
-
-bool IsStorageImage(BindingKind kind) {
-	switch (kind) {
-		case BindingKind::Storage1D:
-		case BindingKind::Storage1DArray:
-		case BindingKind::Storage2D:
-		case BindingKind::Storage2DArray:
-		case BindingKind::Storage3D:
-		case BindingKind::StorageUint1D:
-		case BindingKind::StorageUint1DArray:
-		case BindingKind::StorageUint2D:
-		case BindingKind::StorageUint2DArray:
-		case BindingKind::StorageUint3D: return true;
-		default: return false;
-	}
-}
-
-vk::DescriptorType DescriptorType(BindingKind kind) {
-	if (kind == BindingKind::Samplers) {
-		return vk::DescriptorType::eSampler;
-	}
-	if (IsSampledImage(kind)) {
-		return vk::DescriptorType::eSampledImage;
-	}
-	if (IsStorageImage(kind)) {
-		return vk::DescriptorType::eStorageImage;
-	}
-	return vk::DescriptorType::eStorageBuffer;
-}
-
-uint32_t DescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
-	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
-}
-
-std::vector<uint32_t> LayoutKey(DescriptorCache::Stage               stage,
-                                const ShaderRecompiler::IR::Program& program) {
-	std::vector<uint32_t> key;
-	key.reserve(1u + program.bindings.descriptors.size() * 4u);
-	key.push_back(static_cast<uint32_t>(stage));
-	for (const auto& binding: program.bindings.descriptors) {
-		key.push_back(static_cast<uint32_t>(binding.kind));
-		key.push_back(binding.binding);
-		key.push_back(DescriptorCount(binding));
-		key.push_back(static_cast<uint32_t>(DescriptorType(binding.kind)));
-	}
-	return key;
-}
-
-vk::DescriptorBufferInfo BufferInfo(const BufferView& view) {
-	EXIT_IF(view.buffer == nullptr);
-	return {view.buffer, view.offset, view.range};
-}
-
-} // namespace
-
-vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture,
-                                                       uint32_t              element) {
-	const auto view =
-	    texture.mip_views.empty()
-	        ? (element == 0u ? texture.image_view : vk::ImageView {})
-	        : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {});
-	EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
-	return {nullptr, view, texture.layout};
-}
-
-DescriptorCache::~DescriptorCache() {
-	for (auto& [layout, sets]: m_free_sets_by_layout) {
-		(void)layout;
-		for (auto* set: sets) {
-			delete set;
-		}
-	}
-	for (const auto& [key, layout]: m_descriptor_set_layouts) {
-		(void)key;
-		m_graphics.device.destroyDescriptorSetLayout(layout, nullptr);
-	}
-	for (const auto& pool: m_pools) {
-		m_graphics.device.destroyDescriptorPool(pool.pool, nullptr);
-	}
-}
-
-vk::DescriptorSetLayout
-DescriptorCache::GetDescriptorSetLayoutInternal(Stage                                stage,
-                                                const ShaderRecompiler::IR::Program& program) {
-	const auto key = LayoutKey(stage, program);
-	if (const auto found = m_descriptor_set_layouts.find(key);
-	    found != m_descriptor_set_layouts.end()) {
-		return found->second;
-	}
-
-	std::vector<vk::DescriptorSetLayoutBinding> bindings;
-	bindings.reserve(program.bindings.descriptors.size());
-	for (const auto& descriptor: program.bindings.descriptors) {
-		bindings.push_back({descriptor.binding, DescriptorType(descriptor.kind),
-		                    DescriptorCount(descriptor), StageFlags(stage), nullptr});
-	}
-	if (bindings.empty()) {
-		return nullptr;
-	}
-
-	vk::DescriptorSetLayoutCreateInfo info {};
-	info.sType                     = vk::StructureType::eDescriptorSetLayoutCreateInfo;
-	info.bindingCount              = static_cast<uint32_t>(bindings.size());
-	info.pBindings                 = bindings.data();
-	vk::DescriptorSetLayout layout = nullptr;
-	const auto result = m_graphics.device.createDescriptorSetLayout(&info, nullptr, &layout);
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || layout == nullptr);
-	m_descriptor_set_layouts.emplace(key, layout);
-	return layout;
-}
-
-void DescriptorCache::CreatePool() {
-	KYTY_PROFILER_FUNCTION();
-	constexpr uint32_t           MaxSets = 512;
-	const vk::DescriptorPoolSize sizes[] = {
-	    {vk::DescriptorType::eStorageBuffer,
-	     MaxSets * (ShaderRecompiler::IR::ShaderInfo::MaxBuffers +
-	                ShaderRecompiler::IR::ShaderInfo::MaxAddresses + 3u)},
-	    {vk::DescriptorType::eSampledImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
-	    {vk::DescriptorType::eStorageImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
-	    {vk::DescriptorType::eSampler, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxSamplers},
-	};
-	vk::DescriptorPoolCreateInfo info {};
-	info.sType         = vk::StructureType::eDescriptorPoolCreateInfo;
-	info.poolSizeCount = static_cast<uint32_t>(std::size(sizes));
-	info.pPoolSizes    = sizes;
-	info.maxSets       = MaxSets;
-	const auto pool_id = static_cast<int>(m_pools.size());
-	auto&      pool    = m_pools.emplace_back();
-	const auto result  = m_graphics.device.createDescriptorPool(&info, nullptr, &pool.pool);
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || pool.pool == nullptr);
-	pool.next_free_pool = m_first_free_pool;
-	m_first_free_pool   = pool_id;
-}
-
-VulkanDescriptorSet* DescriptorCache::Allocate(Stage                                stage,
-                                               const ShaderRecompiler::IR::Program& program) {
-	KYTY_PROFILER_FUNCTION();
-	Common::LockGuard lock(m_mutex);
-	const auto        layout = GetDescriptorSetLayoutInternal(stage, program);
-	EXIT_IF(layout == nullptr);
-	auto& free_sets = m_free_sets_by_layout[layout];
-	if (!free_sets.empty()) {
-		auto* result = free_sets.back();
-		free_sets.pop_back();
-		return result;
-	}
-
-	constexpr uint32_t Batch = 32;
-	for (int attempt = 0; attempt < 2; attempt++) {
-		for (int pool_id = m_first_free_pool; pool_id != -1;
-		     pool_id     = m_pools[pool_id].next_free_pool) {
-			auto&                                      pool = m_pools[pool_id];
-			std::array<vk::DescriptorSetLayout, Batch> layouts;
-			layouts.fill(layout);
-			std::array<vk::DescriptorSet, Batch> sets {};
-			vk::DescriptorSetAllocateInfo        info {};
-			info.sType              = vk::StructureType::eDescriptorSetAllocateInfo;
-			info.descriptorPool     = pool.pool;
-			info.descriptorSetCount = Batch;
-			info.pSetLayouts        = layouts.data();
-			if (m_graphics.device.allocateDescriptorSets(&info, sets.data()) ==
-			    vk::Result::eSuccess) {
-				free_sets.reserve(free_sets.size() + Batch - 1u);
-				for (uint32_t i = 0; i + 1u < Batch; i++) {
-					free_sets.push_back(new VulkanDescriptorSet {sets[i], layout, pool_id});
-				}
-				return new VulkanDescriptorSet {sets.back(), layout, pool_id};
-			}
-			m_first_free_pool = pool.next_free_pool;
-			break;
-		}
-		CreatePool();
-	}
-	return nullptr;
-}
-
-void DescriptorCache::Recycle(VulkanDescriptorSet& set) {
-	EXIT_IF(set.set == nullptr || set.layout == nullptr);
-	Common::LockGuard lock(m_mutex);
-	m_free_sets_by_layout[set.layout].push_back(&set);
-}
-
-VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage                                stage,
-                                                    const ShaderRecompiler::IR::Program& program,
-                                                    const NativeDescriptors&             data) {
-	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(data.buffers.size() != program.info.buffers.size() ||
-	        data.images.size() != program.info.images.size() ||
-	        data.samplers.size() != program.info.samplers.size() ||
-	        data.addresses.size() != program.info.addresses.size());
-	auto* set = Allocate(stage, program);
-	EXIT_NOT_IMPLEMENTED(set == nullptr);
-
-	size_t descriptor_count = 0;
-	for (const auto& binding: program.bindings.descriptors) {
-		descriptor_count += DescriptorCount(binding);
-	}
-	std::vector<vk::DescriptorBufferInfo> buffer_infos;
-	std::vector<vk::DescriptorImageInfo>  image_infos;
-	std::vector<vk::WriteDescriptorSet>   writes;
-	buffer_infos.reserve(descriptor_count);
-	image_infos.reserve(descriptor_count);
-	writes.reserve(program.bindings.descriptors.size());
-	std::vector<uint32_t> image_occurrences(data.images.size());
-
-	for (const auto& binding: program.bindings.descriptors) {
-		vk::WriteDescriptorSet write {};
-		write.sType             = vk::StructureType::eWriteDescriptorSet;
-		write.dstSet            = set->set;
-		write.dstBinding        = binding.binding;
-		write.descriptorType    = DescriptorType(binding.kind);
-		write.descriptorCount   = DescriptorCount(binding);
-		const auto buffer_start = buffer_infos.size();
-		const auto image_start  = image_infos.size();
-		switch (binding.kind) {
-			case BindingKind::Buffers:
-				for (const auto resource: binding.resources) {
-					buffer_infos.push_back(BufferInfo(data.buffers.at(resource)));
-				}
-				break;
-			case BindingKind::AddressMemory:
-				for (const auto resource: binding.resources) {
-					buffer_infos.push_back(BufferInfo(data.addresses.at(resource)));
-				}
-				break;
-			case BindingKind::FlattenedSrt:
-				buffer_infos.push_back(BufferInfo(data.flattened_srt));
-				break;
-			case BindingKind::UserData: buffer_infos.push_back(BufferInfo(data.user_data)); break;
-			case BindingKind::Gds: buffer_infos.push_back(BufferInfo(data.gds)); break;
-			case BindingKind::Samplers:
-				for (const auto resource: binding.resources) {
-					const auto sampler = data.samplers.at(resource);
-					EXIT_IF(sampler == nullptr);
-					image_infos.push_back({sampler, nullptr, vk::ImageLayout::eUndefined});
-				}
-				break;
-			default: {
-				for (const auto resource: binding.resources) {
-					const auto& texture = data.images.at(resource);
-					image_infos.push_back(MakeImageInfo(texture, image_occurrences.at(resource)++));
-				}
-				break;
-			}
-		}
-		if (buffer_infos.size() != buffer_start) {
-			write.pBufferInfo = buffer_infos.data() + buffer_start;
-		}
-		if (image_infos.size() != image_start) {
-			write.pImageInfo = image_infos.data() + image_start;
-		}
-		writes.push_back(write);
-	}
-	for (uint32_t i = 0; i < data.images.size(); i++) {
-		const auto expected = data.images[i].mip_views.empty()
-		                          ? 1u
-		                          : static_cast<uint32_t>(data.images[i].mip_views.size());
-		EXIT_IF(image_occurrences[i] != expected);
-	}
-	m_graphics.device.updateDescriptorSets(static_cast<uint32_t>(writes.size()), writes.data(), 0,
-	                                       nullptr);
-	return *set;
-}
-
-vk::DescriptorSetLayout
-DescriptorCache::GetDescriptorSetLayout(Stage stage, const ShaderRecompiler::IR::Program& program) {
-	Common::LockGuard lock(m_mutex);
-	return GetDescriptorSetLayoutInternal(stage, program);
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptorCache.h

@@ -1,116 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_
-
-#include "common/abi.h"
-#include "common/assert.h"
-#include "common/common.h"
-#include "common/threads.h"
-#include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/cache/textureCache.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/shaderBindings.h"
-
-#include <map>
-#include <memory>
-#include <span>
-#include <unordered_map>
-#include <vector>
-
-namespace Libs::Graphics {
-
-namespace ShaderRecompiler::IR {
-struct DescriptorValue;
-struct ImageResource;
-struct Program;
-struct ResourceSnapshot;
-} // namespace ShaderRecompiler::IR
-
-class CommandBuffer;
-struct DescriptorCacheTestAccess;
-struct ShaderStageRuntime;
-
-struct VulkanDescriptorSet {
-	vk::DescriptorSet       set     = nullptr;
-	vk::DescriptorSetLayout layout  = nullptr;
-	int                     pool_id = -1;
-};
-
-struct BufferView {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	vk::DeviceSize        offset = 0;
-	vk::DeviceSize        range  = VK_WHOLE_SIZE;
-};
-
-class DescriptorCache {
-public:
-	enum class Stage { Unknown, Vertex, Pixel, Compute };
-
-	struct TextureBinding {
-		ImageId                    image_id;
-		vk::ImageView              image_view = nullptr;
-		TextureCache::ImageDesc    desc;
-		vk::ImageLayout            layout = vk::ImageLayout::eUndefined;
-		std::vector<vk::ImageView> mip_views;
-	};
-
-	struct NativeDescriptors {
-		std::vector<BufferView>     buffers;
-		std::vector<TextureBinding> images;
-		std::vector<vk::Sampler>    samplers;
-		std::vector<BufferView>     addresses;
-		BufferView                  gds;
-		BufferView                  flattened_srt;
-		BufferView                  user_data;
-	};
-
-	struct PreparedBindings {
-		std::shared_ptr<const ShaderRecompiler::IR::Program>          program;
-		std::shared_ptr<const ShaderRecompiler::IR::ResourceSnapshot> snapshot;
-		NativeDescriptors                                             resources;
-		std::vector<uint32_t>                                         flattened_srt;
-		std::vector<uint32_t>                                         user_data;
-		vk::ShaderStageFlags                                          shader_stage;
-		Stage                                                         stage     = Stage::Unknown;
-		bool                                                          committed = false;
-	};
-
-	explicit DescriptorCache(GraphicContext& graphics): m_graphics(graphics) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	}
-	~DescriptorCache();
-	KYTY_CLASS_NO_COPY(DescriptorCache);
-
-	vk::DescriptorSetLayout GetDescriptorSetLayout(Stage                                stage,
-	                                               const ShaderRecompiler::IR::Program& program);
-	void                    Recycle(VulkanDescriptorSet& set);
-	VulkanDescriptorSet&    GetDescriptor(Stage stage, const ShaderRecompiler::IR::Program& program,
-	                                      const NativeDescriptors& descriptors);
-
-private:
-	friend struct DescriptorCacheTestAccess;
-
-	struct Pool {
-		vk::DescriptorPool pool           = nullptr;
-		int                next_free_pool = -1;
-	};
-
-	static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
-	                                             uint32_t              element = 0);
-	void                           CreatePool();
-	VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program);
-	vk::DescriptorSetLayout
-	GetDescriptorSetLayoutInternal(Stage stage, const ShaderRecompiler::IR::Program& program);
-
-	GraphicContext&   m_graphics;
-	Common::Mutex     m_mutex;
-	std::vector<Pool> m_pools;
-	int               m_first_free_pool = -1;
-	std::unordered_map<vk::DescriptorSetLayout, std::vector<VulkanDescriptorSet*>>
-	                                                         m_free_sets_by_layout;
-	std::map<std::vector<uint32_t>, vk::DescriptorSetLayout> m_descriptor_set_layouts;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp

@@ -0,0 +1,97 @@
+#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
+
+#include "common/assert.h"
+#include "common/profiler.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/masterSemaphore.h"
+
+namespace Libs::Graphics {
+namespace {
+
+constexpr uint32_t   DescriptorHeapCount = 1024;
+constexpr std::array DescriptorPoolSizes = {
+    vk::DescriptorPoolSize {vk::DescriptorType::eStorageBuffer, 8192},
+    vk::DescriptorPoolSize {vk::DescriptorType::eSampledImage, 8192},
+    vk::DescriptorPoolSize {vk::DescriptorType::eStorageImage, 1024},
+    vk::DescriptorPoolSize {vk::DescriptorType::eSampler, 1024},
+};
+
+} // namespace
+
+DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore)
+    : m_graphics(graphics), m_master_semaphore(master_semaphore) {
+	CreatePool();
+}
+
+DescriptorHeap::~DescriptorHeap() {
+	m_graphics.device.destroyDescriptorPool(m_current_pool, nullptr);
+	for (const auto& [pool, tick]: m_pending_pools) {
+		m_master_semaphore.Wait(tick);
+		m_graphics.device.destroyDescriptorPool(pool, nullptr);
+	}
+}
+
+vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) {
+	KYTY_PROFILER_FUNCTION();
+	EXIT_IF(layout == nullptr);
+
+	auto& batch = m_sets[layout];
+	if (batch.size != 0) {
+		return batch.sets[--batch.size];
+	}
+	if (Allocate(layout, batch)) {
+		return batch.sets[--batch.size];
+	}
+
+	m_pending_pools.emplace_back(m_current_pool, m_master_semaphore.CurrentTick());
+	if (const auto& [pool, tick] = m_pending_pools.front(); m_master_semaphore.IsFree(tick)) {
+		m_current_pool = pool;
+		m_pending_pools.pop_front();
+		EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess);
+	} else {
+		CreatePool();
+	}
+
+	m_sets.clear();
+	auto& fresh_batch = m_sets[layout];
+	EXIT_IF(!Allocate(layout, fresh_batch));
+	return fresh_batch.sets[--fresh_batch.size];
+}
+
+bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) {
+	std::array<vk::DescriptorSetLayout, DescriptorSetBatch> layouts;
+	layouts.fill(layout);
+
+	vk::DescriptorSetAllocateInfo allocate {};
+	allocate.sType          = vk::StructureType::eDescriptorSetAllocateInfo;
+	allocate.descriptorPool = m_current_pool;
+	allocate.pSetLayouts    = layouts.data();
+
+	for (;;) {
+		allocate.descriptorSetCount = batch.allocation;
+		const auto result = m_graphics.device.allocateDescriptorSets(&allocate, batch.sets.data());
+		if (result == vk::Result::eSuccess) {
+			batch.size = batch.allocation;
+			return true;
+		}
+		EXIT_IF(result != vk::Result::eErrorOutOfPoolMemory &&
+		        result != vk::Result::eErrorFragmentedPool);
+		if (batch.allocation == 1) {
+			return false;
+		}
+		batch.allocation /= 2;
+	}
+}
+
+void DescriptorHeap::CreatePool() {
+	vk::DescriptorPoolCreateInfo create {};
+	create.sType         = vk::StructureType::eDescriptorPoolCreateInfo;
+	create.flags         = {};
+	create.maxSets       = DescriptorHeapCount;
+	create.poolSizeCount = static_cast<uint32_t>(DescriptorPoolSizes.size());
+	create.pPoolSizes    = DescriptorPoolSizes.data();
+	EXIT_IF(m_graphics.device.createDescriptorPool(&create, nullptr, &m_current_pool) !=
+	        vk::Result::eSuccess);
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h

@@ -0,0 +1,45 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_
+
+#include "common/common.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+
+#include <array>
+#include <deque>
+#include <unordered_map>
+
+namespace Libs::Graphics {
+
+struct GraphicContext;
+class MasterSemaphore;
+
+class DescriptorHeap {
+public:
+	DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore);
+	~DescriptorHeap();
+	KYTY_CLASS_NO_COPY(DescriptorHeap);
+
+	[[nodiscard]] vk::DescriptorSet Commit(vk::DescriptorSetLayout layout);
+
+private:
+	static constexpr uint32_t DescriptorSetBatch = 32;
+
+	struct Batch {
+		std::array<vk::DescriptorSet, DescriptorSetBatch> sets {};
+		uint32_t                                          size       = 0;
+		uint32_t                                          allocation = DescriptorSetBatch;
+	};
+
+	[[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch);
+	void               CreatePool();
+
+	GraphicContext&                                     m_graphics;
+	MasterSemaphore&                                    m_master_semaphore;
+	vk::DescriptorPool                                  m_current_pool = nullptr;
+	std::deque<std::pair<vk::DescriptorPool, uint64_t>> m_pending_pools;
+	std::unordered_map<vk::DescriptorSetLayout, Batch>  m_sets;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -17,7 +17,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
@@ -45,10 +44,99 @@
 
 namespace Libs::Graphics {
 
+namespace {
+
+using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind;
+
+bool IsSampledImage(BindingKind kind) {
+	switch (kind) {
+		case BindingKind::Sampled1D:
+		case BindingKind::Sampled1DArray:
+		case BindingKind::Sampled2D:
+		case BindingKind::Sampled2DArray:
+		case BindingKind::Sampled2DMsaa:
+		case BindingKind::Sampled2DMsaaArray:
+		case BindingKind::Sampled3D:
+		case BindingKind::SampledUint1D:
+		case BindingKind::SampledUint1DArray:
+		case BindingKind::SampledUint2D:
+		case BindingKind::SampledUint2DArray:
+		case BindingKind::SampledUint2DMsaa:
+		case BindingKind::SampledUint2DMsaaArray:
+		case BindingKind::SampledUint3D: return true;
+		default: return false;
+	}
+}
+
+bool IsStorageImage(BindingKind kind) {
+	switch (kind) {
+		case BindingKind::Storage1D:
+		case BindingKind::Storage1DArray:
+		case BindingKind::Storage2D:
+		case BindingKind::Storage2DArray:
+		case BindingKind::Storage3D:
+		case BindingKind::StorageUint1D:
+		case BindingKind::StorageUint1DArray:
+		case BindingKind::StorageUint2D:
+		case BindingKind::StorageUint2DArray:
+		case BindingKind::StorageUint3D:
+		case BindingKind::StorageAtomic1D:
+		case BindingKind::StorageAtomic1DArray:
+		case BindingKind::StorageAtomic2D:
+		case BindingKind::StorageAtomic2DArray:
+		case BindingKind::StorageAtomic3D: return true;
+		default: return false;
+	}
+}
+
+} // namespace
+
+vk::DescriptorType NativeDescriptorType(BindingKind kind) {
+	if (kind == BindingKind::Samplers) {
+		return vk::DescriptorType::eSampler;
+	}
+	if (IsSampledImage(kind)) {
+		return vk::DescriptorType::eSampledImage;
+	}
+	if (IsStorageImage(kind)) {
+		return vk::DescriptorType::eStorageImage;
+	}
+	return vk::DescriptorType::eStorageBuffer;
+}
+
+uint32_t NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
+	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
+}
+
+vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element) {
+	const auto view =
+	    texture.mip_views.empty()
+	        ? (element == 0u ? texture.image_view : vk::ImageView {})
+	        : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {});
+	EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
+	return {nullptr, view, texture.layout};
+}
+
+static const char* ShaderStageResourceName(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return "Vertex";
+		case ShaderType::Pixel: return "Pixel";
+		case ShaderType::Compute: return "Compute";
+		default: return "Unknown";
+	}
+}
+
+static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
+		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
+		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
+		default: EXIT("unknown native shader stage\n");
+	}
+}
+
 static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) {
-	auto owner = context.GetBufferCache().ObtainNullBuffer();
-	dst.buffer = owner->Handle();
-	dst.owner  = std::move(owner);
+	dst.buffer = context.GetBufferCache().GetBuffer(NULL_BUFFER_ID).Handle();
 	dst.offset = 0;
 	dst.range  = 16;
 }
@@ -79,10 +167,11 @@ static bool IsMultisampledTexture(Prospero::ImageType type) {
 	       type == Prospero::ImageType::kColor2DMsaaArray;
 }
 
-static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& command_buffer,
+static BufferView NativeStorageBuffer(RenderContext&                              context,
                                       const ShaderBufferResource&                 descriptor,
                                       const ShaderRecompiler::IR::BufferResource& resource,
-                                      uint32_t&                                   buffer_offset) {
+                                      ShaderType stage, uint32_t slot, uint32_t& buffer_offset,
+                                      BufferId id) {
 	BufferView result;
 	buffer_offset = 0;
 
@@ -104,58 +193,26 @@ static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& com
 	    size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange) {
 		EXIT("storage buffer range or device alignment is unsupported\n");
 	}
-	auto binding = context.GetBufferCache().ObtainBuffer(
-	    command_buffer, address, size, resource.written, resource.read, resource.formatted);
-	const auto aligned_offset = binding.offset - binding.offset % alignment;
-	const auto adjustment     = binding.offset - aligned_offset;
+	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
+	                                                              resource.formatted, id);
+	const auto aligned_offset = offset - offset % alignment;
+	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
 	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
 		EXIT("storage buffer offset adjustment is unsupported\n");
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
-	result.owner  = std::move(binding.owner);
-	result.buffer = binding.buffer;
+	result.buffer = buffer->Handle();
 	result.offset = aligned_offset;
 	result.range  = static_cast<vk::DeviceSize>(size + adjustment);
-	return result;
-}
-
-static BufferView
-NativeAddressBuffer(RenderContext& context, CommandBuffer& command_buffer,
-                    const ShaderRecompiler::IR::AddressResource&           resource,
-                    const ShaderRecompiler::IR::ResourceSnapshot::Address& address) {
-	BufferView result;
-	if (address.binding_base == 0) {
-		BindNullStorageBuffer(context, result);
-		return result;
+	if (resource.formatted && resource.written) {
+		context.GetTextureCache().InvalidateMemoryFromGPU(address, size);
 	}
-	if (resource.written) {
-		EXIT("writable address resources are unsupported\n");
-	}
-	const auto limit =
-	    resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
-	        ? ShaderRecompiler::IR::FlatAddressWindowSize
-	        : static_cast<uint64_t>(
-	              context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
-	uint64_t   size   = 0;
-	const auto access = HostMemoryAccess::Mapped;
-	if (!HostMemoryQueryRange(address.binding_base, limit, access, size)) {
-		EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
-		     address.binding_base);
-	}
-	const auto& graphics  = context.GetGraphics();
-	const auto  alignment = graphics.StorageMinAlignment();
-	if (alignment == 0 ||
-	    size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange ||
-	    BufferCache::GetBufferOffset(address.binding_base) % alignment != 0) {
-		EXIT("address resource range or alignment is unsupported\n");
-	}
-	auto binding =
-	    context.GetBufferCache().ObtainBuffer(command_buffer, address.binding_base, size);
-	result.owner  = std::move(binding.owner);
-	result.buffer = binding.buffer;
-	result.offset = binding.offset;
-	result.range  = static_cast<vk::DeviceSize>(size);
+	SetVulkanObjectNameF(
+	    graphics.device, result.buffer,
+	    "Kyty.{}.StorageBuffer[slot={} guest=0x{:016x} size=0x{:x} access={} formatted={}]",
+	    ShaderStageResourceName(stage), slot, address, size,
+	    resource.written ? (resource.read ? "ReadWrite" : "Write") : "Read", resource.formatted);
 	return result;
 }
 
@@ -233,7 +290,8 @@ bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& r
 	       descriptor.BaseArray5() == 0;
 }
 
-bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image) {
+bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image,
+                                      bool r128) {
 	const auto width        = static_cast<uint32_t>(descriptor.Width5()) + 1u;
 	const auto height       = static_cast<uint32_t>(descriptor.Height5()) + 1u;
 	const auto type         = descriptor.Type();
@@ -260,12 +318,12 @@ bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, c
 	                                  descriptor.BaseArray5() <= descriptor.Depth() &&
 	                                  descriptor.Depth() < image.info.resources.layers;
 	const bool levels_ok =
-	    multisampled
-	        ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
-	              descriptor.LastLevel() <= 3 && descriptor.MaxMip() == descriptor.LastLevel() &&
-	              image.info.resources.levels == 1 && image.info.samples == samples
-	        : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
-	              descriptor.MaxMip() == 0 && image.info.samples == 1;
+	    multisampled ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
+	                       descriptor.LastLevel() <= 3 &&
+	                       (r128 || descriptor.MaxMip() == descriptor.LastLevel()) &&
+	                       image.info.resources.levels == 1 && image.info.samples == samples
+	                 : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
+	                       (r128 || descriptor.MaxMip() == 0) && image.info.samples == 1;
 	return image.info.IsDepth() && width == image.info.extent.width &&
 	       height == image.info.extent.height &&
 	       (supported_2d || supported_array || supported_cube || supported_msaa_2d ||
@@ -275,7 +333,8 @@ bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, c
 	       (!descriptor.MsaaDepth() || multisampled) && pitch >= width && pitch == image.info.pitch;
 }
 
-bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image) {
+bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image,
+                                     bool r128) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
 	const uint32_t     field3_expected = descriptor.DstSelXYZW() |
@@ -288,10 +347,14 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 	                                     (static_cast<uint32_t>(descriptor.MaxMip()) << 4u);
 	const bool         common          = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
 	                                     (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	                                     descriptor.fields[3] == field3_expected &&
-	                                     descriptor.fields[4] == field4_expected &&
-	                                     descriptor.fields[5] == field5_expected;
-	if (!common || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
+	                                     descriptor.fields[3] == field3_expected;
+	if (r128) {
+		return common && descriptor.fields[4] == 0 && descriptor.fields[5] == 0 &&
+		       descriptor.fields[6] == 0 && descriptor.fields[7] == 0;
+	}
+	const bool full = common && descriptor.fields[4] == field4_expected &&
+	                  descriptor.fields[5] == field5_expected;
+	if (!full || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
 		return false;
 	}
 	if (descriptor.fields[6] == 0) {
@@ -312,9 +375,9 @@ static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource
                                        vk::Format view_format, uint64_t size) {
 	const bool resource_ok = IsSupportedSampledDepthResource(resource);
 	const bool descriptor_ok =
-	    image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image);
+	    image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image, resource.r128);
 	const bool encoding_ok =
-	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image);
+	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image, resource.r128);
 	const bool format_ok =
 	    image != nullptr && IsSupportedSampledDepthFormat(image->info.pixel_format, view_format);
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
@@ -404,17 +467,19 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_swizzle =
 	    IsValidImageSwizzle(swizzle) &&
 	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
+	const auto max_mip = resource.r128 ? descriptor.LastLevel() : descriptor.MaxMip();
 	const auto view_last_level =
 	    resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage
 	        ? descriptor.LastLevel()
-	        : std::min(descriptor.LastLevel(), descriptor.MaxMip());
+	        : std::min(descriptor.LastLevel(), max_mip);
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       descriptor.BaseLevel() <= view_last_level && view_last_level <= descriptor.MaxMip() &&
+	       descriptor.BaseLevel() <= view_last_level && view_last_level <= max_mip &&
 	       descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
 	       !descriptor.MsaaDepth();
 }
 
-static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descriptor) {
+static bool IsSupportedStorageTextureEncoding(const ShaderRecompiler::IR::ImageResource& resource,
+                                              const ShaderTextureResource& descriptor) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
 	constexpr uint32_t field5_expected      = 0x00700000u;
@@ -426,9 +491,14 @@ static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descr
 	                                     (static_cast<uint32_t>(descriptor.Type()) << 28u);
 	const uint32_t     expected_field4 =
 	    descriptor.Depth() | (static_cast<uint32_t>(descriptor.BaseArray5()) << 16u);
-	return (descriptor.fields[1] & field1_reserved_mask) == 0 &&
-	       (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	       descriptor.fields[3] == expected_field3 && descriptor.fields[4] == expected_field4 &&
+	const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
+	                    (descriptor.fields[2] & field2_reserved_mask) == 0 &&
+	                    descriptor.fields[3] == expected_field3;
+	if (resource.r128) {
+		return common && descriptor.fields[4] == 0 && descriptor.fields[5] == 0 &&
+		       descriptor.fields[6] == 0 && descriptor.fields[7] == 0;
+	}
+	return common && descriptor.fields[4] == expected_field4 &&
 	       (descriptor.fields[5] & ~field5_max_mip_mask) == field5_expected;
 }
 
@@ -437,13 +507,13 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	const auto format        = descriptor.Format();
 	const bool resource_ok   = IsSupportedStorageImageResource(resource);
 	const bool descriptor_ok = IsSupportedStorageTextureDescriptor(resource, descriptor);
-	const bool encoding_ok   = IsSupportedStorageTextureEncoding(descriptor);
+	const bool encoding_ok   = IsSupportedStorageTextureEncoding(resource, descriptor);
 	const bool uint_resource =
 	    resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
 	const bool raw_sint_storage = format == Prospero::BufferFormat::k32SInt && uint_resource &&
 	                              resource.written && !resource.read && !resource.atomic;
 	const bool format_ok =
-	    raw_sint_storage || (Prospero::IsSupportedTextureFormat(format) &&
+	    raw_sint_storage || (Prospero::IsSampledTextureFormat(format) &&
 	                         uint_resource == Prospero::IsUintTextureFormat(format) &&
 	                         (!resource.atomic || format == Prospero::BufferFormat::k32UInt));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
@@ -552,15 +622,17 @@ static void PopulateTextureMipLayout(ImageInfo& info) {
 
 static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource,
                                      const ShaderTextureResource& descriptor, vk::Format format,
-                                     bool storage, uint32_t view_levels, uint32_t image_layers) {
+                                     bool shader_conversion, bool storage, uint32_t view_levels,
+                                     uint32_t image_layers) {
 	ImageViewInfo view {};
 	view.format      = format;
 	view.aspect      = vk::ImageAspectFlagBits::eColor;
 	view.base_level  = descriptor.BaseLevel();
 	view.level_count = view_levels;
-	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
-	view.mapping =
-	    storage ? vk::ComponentMapping {} : TextureGetComponentMapping(descriptor.DstSelXYZW());
+	view.usage   = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
+	view.mapping = storage || shader_conversion
+	                   ? vk::ComponentMapping {}
+	                   : TextureGetComponentMapping(descriptor.DstSelXYZW());
 	switch (resource.dimension) {
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
 			view.type       = vk::ImageViewType::e1D;
@@ -606,9 +678,8 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
 	return view;
 }
 
-DescriptorCache::TextureBinding
-RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
-                               const ShaderRecompiler::IR::DescriptorValue& value) {
+TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
+                                              const ShaderRecompiler::IR::DescriptorValue& value) {
 	ShaderTextureResource descriptor;
 	CopyNativeDescriptor(value, descriptor.fields);
 	const bool storage = resource.written;
@@ -631,20 +702,20 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto last_level   = descriptor.LastLevel();
 	const auto type         = TextureType(descriptor);
 	const bool multisampled = IsMultisampledTexture(type);
-	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
+	const auto max_mip      = resource.r128 ? last_level : descriptor.MaxMip();
+	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(max_mip) + 1u;
 	const bool dynamic_storage =
 	    storage && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
 	const auto view_last_level =
-	    !multisampled && !dynamic_storage ? std::min(last_level, descriptor.MaxMip()) : last_level;
+	    !multisampled && !dynamic_storage ? std::min(last_level, max_mip) : last_level;
 	const auto tile       = descriptor.TileMode();
 	const bool depth_tile = tile == Prospero::TileMode::kDepth;
 	const bool msaa_tile  = depth_tile || tile == Prospero::TileMode::kRenderTarget;
 	const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
 	if ((!multisampled && (base_level > view_last_level || view_last_level >= levels)) ||
 	    (multisampled &&
-	     (base_level != 0 || last_level == 0 || last_level > 3 ||
-	      descriptor.MaxMip() != last_level || !msaa_tile ||
-	      (descriptor.MsaaDepth() && !depth_tile) ||
+	     (base_level != 0 || last_level == 0 || last_level > 3 || max_mip != last_level ||
+	      !msaa_tile || (descriptor.MsaaDepth() && !depth_tile) ||
 	      (!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
 		EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
 		     "kind=%u dimension=%u mip_mode=%u read=%d written=%d "
@@ -660,11 +731,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto view_levels =
 	    multisampled ? 1u : static_cast<uint32_t>(view_last_level - base_level) + 1u;
-	const auto depth  = static_cast<uint32_t>(descriptor.Depth()) + 1u;
-	const auto format = descriptor.Format();
+	const auto depth          = static_cast<uint32_t>(descriptor.Depth()) + 1u;
+	const auto format         = descriptor.Format();
+	const auto surface_format = TextureGetSurfaceFormatInfo(format);
+	const bool shader_conversion =
+	    surface_format.conversion_format != Prospero::BufferFormat::kInvalid;
 	const bool sampled_numeric_class =
-	    storage || ((resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) ==
-	                Prospero::IsUintTextureFormat(format));
+	    storage || (Prospero::IsSampledTextureFormat(format) &&
+	                (resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) ==
+	                    Prospero::IsUintTextureFormat(format));
 	if (!storage &&
 	    (resource.kind == ShaderRecompiler::IR::ResourceKind::Image ||
 	     resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) &&
@@ -700,7 +775,7 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 		ValidateStorageTexture(resource, descriptor, size.size);
 	}
 
-	const auto pixel_format        = TextureGetFormat(format);
+	const auto pixel_format        = surface_format.vk_format;
 	const auto storage_view_format = storage && format == Prospero::BufferFormat::k32SInt
 	                                     ? vk::Format::eR32Uint
 	                                     : SrgbStorageViewFormat(pixel_format);
@@ -725,11 +800,11 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	} else {
 		PopulateTextureMipLayout(desc.info);
 	}
-	desc.view_info = TextureViewInfo(resource, descriptor, view_format, storage, view_levels,
-	                                 desc.info.resources.layers);
+	desc.view_info = TextureViewInfo(resource, descriptor, view_format, shader_conversion, storage,
+	                                 view_levels, desc.info.resources.layers);
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
 
-	auto       id                  = texture_cache.FindImage(desc);
+	auto       id                  = texture_cache.FindImage(desc, shader_conversion);
 	auto*      image               = &texture_cache.GetImage(id);
 	const bool stencil_association = static_cast<bool>(image->depth_id);
 	if (stencil_association) {
@@ -765,25 +840,25 @@ static vk::Sampler NativeSampler(RenderContext&                       context,
 	if (!depth_compare) {
 		descriptor.fields[0] &= ~(0x7u << 12u);
 	}
+	if (program.info.samplers[index].force_point_filtering) {
+		descriptor.SetPointFiltering();
+	}
 	return context.GetSamplerCache().GetSampler(descriptor);
 }
 
-static BufferView NativeUpload(RenderContext& context, CommandBuffer& command_buffer,
-                               std::span<const uint32_t> data) {
+static BufferView NativeUpload(RenderContext& context, std::span<const uint32_t> data) {
 	EXIT_IF(data.empty());
-	EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute());
-	auto binding = context.GetBufferCache().UploadTransient(data.data(), data.size_bytes(), 256);
-	return {.owner  = std::move(binding.owner),
-	        .buffer = binding.buffer,
-	        .offset = binding.offset,
-	        .range  = data.size_bytes()};
+	auto& command_buffer = context.GetCommandScheduler().Current();
+	EXIT_IF(command_buffer.IsInvalid());
+	auto&      buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+	const auto offset = buffer.Copy(data.data(), data.size_bytes(), 256);
+	return {.buffer = buffer.Handle(), .offset = offset, .range = data.size_bytes()};
 }
 
 void RenderExecutor::TrackImageBinding(ImageId id) {
-	auto image = m_context.GetTextureCache().ResolveOwner(id);
-	EXIT_IF(image == nullptr);
-	if (std::ranges::find(m_bound_images, image) == m_bound_images.end()) {
-		m_bound_images.push_back(std::move(image));
+	EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr);
+	if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) {
+		m_bound_images.push_back(id);
 	}
 }
 
@@ -807,16 +882,15 @@ void RenderExecutor::BindRenderTarget(ImageId id) {
 }
 
 void RenderExecutor::ResetBindings() {
-	for (const auto& image: m_bound_images) {
-		image->binding = {};
+	for (const auto id: m_bound_images) {
+		if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(id); image != nullptr) {
+			image->binding = {};
+		}
 	}
 	m_bound_images.clear();
 }
 
-DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&            buffer,
-                                                                  const ShaderStageRuntime& runtime,
-                                                                  vk::ShaderStageFlags shader_stage,
-                                                                  DescriptorCache::Stage stage) {
+PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(!runtime);
 	const auto& program  = *runtime.program;
@@ -826,12 +900,10 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&
 		EXIT("invalid native shader runtime snapshot: %s\n", error.c_str());
 	}
 
-	DescriptorCache::PreparedBindings prepared;
-	prepared.program      = runtime.program;
-	prepared.snapshot     = runtime.resources;
-	prepared.shader_stage = shader_stage;
-	prepared.stage        = stage;
-	auto& descriptors     = prepared.resources;
+	PreparedBindings prepared;
+	prepared.program  = runtime.program.get();
+	prepared.snapshot = runtime.resources.get();
+	auto& descriptors = prepared.resources;
 	descriptors.buffers.reserve(program.info.buffers.size());
 	descriptors.images.reserve(program.info.images.size());
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
@@ -856,44 +928,75 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&
 	prepared.user_data.resize(program.bindings.ShaderDataDwords());
 	if (ShaderRecompiler::IR::FindBinding(
 	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) {
-		descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer().Handle();
+		descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle();
 	}
 	return prepared;
 }
 
-void RenderExecutor::RebindBuffers(CommandBuffer&                     buffer,
-                                   DescriptorCache::PreparedBindings& prepared) {
+void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
+	KYTY_PROFILER_FUNCTION();
+	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
+	const auto& program  = *prepared.program;
+	const auto& snapshot = *prepared.snapshot;
+	auto&       cache    = m_context.GetBufferCache();
+
+	prepared.buffer_ids.clear();
+	prepared.buffer_ids.reserve(program.info.buffers.size());
+	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
+		ShaderBufferResource descriptor;
+		CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields);
+		const auto address = descriptor.Base48();
+		const auto stride  = descriptor.Stride();
+		const auto records = descriptor.NumRecords();
+		EXIT_IF(stride != 0 && records > UINT64_MAX / stride);
+		const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+		if (address == 0 || requested_size == 0) {
+			prepared.buffer_ids.emplace_back();
+			continue;
+		}
+		const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
+		prepared.buffer_ids.push_back(cache.FindBuffer(address, size));
+	}
+
+}
+
+void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
 	const auto& program   = *prepared.program;
 	const auto& snapshot  = *prepared.snapshot;
 	auto&       resources = prepared.resources;
 	const auto& layout    = program.bindings;
+	EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size());
 
 	resources.buffers.clear();
 	resources.buffers.reserve(program.info.buffers.size());
 	EXIT_IF(prepared.user_data.size() != layout.ShaderDataDwords());
-	std::fill(prepared.user_data.begin() + layout.buffer_offset_dword, prepared.user_data.end(), 0);
+	std::fill(prepared.user_data.begin() + layout.memory_offset_dword, prepared.user_data.end(), 0);
+	auto pack_memory_offset = [&](uint32_t index, uint32_t offset) {
+		const auto dword = layout.memory_offset_dword + index / 4u;
+		const auto shift = (index % 4u) * 8u;
+		prepared.user_data[dword] |= offset << shift;
+	};
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		ShaderBufferResource descriptor;
 		CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields);
 		uint32_t buffer_offset = 0;
-		resources.buffers.push_back(NativeStorageBuffer(m_context, buffer, descriptor,
-		                                                program.info.buffers[i], buffer_offset));
-		const auto dword = layout.buffer_offset_dword + i / 4u;
-		const auto shift = (i % 4u) * 8u;
-		prepared.user_data[dword] |= buffer_offset << shift;
+		resources.buffers.push_back(NativeStorageBuffer(m_context, descriptor,
+		                                                program.info.buffers[i], program.stage, i,
+		                                                buffer_offset, prepared.buffer_ids[i]));
+		pack_memory_offset(i, buffer_offset);
 	}
-	resources.addresses.clear();
-	resources.addresses.reserve(program.info.addresses.size());
-	for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
-		resources.addresses.push_back(NativeAddressBuffer(
-		    m_context, buffer, program.info.addresses[i], snapshot.addresses[i]));
+	if (!prepared.flattened_srt.empty()) {
+		resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt);
+	}
+	if (ShaderRecompiler::IR::FindBinding(
+	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) {
+		resources.user_data = NativeUpload(m_context, prepared.user_data);
 	}
 }
 
-void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
-                                  DescriptorCache::PreparedBindings& prepared) {
+void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
 	const auto& program  = *prepared.program;
@@ -902,7 +1005,7 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		const auto old_image = texture_cache.ResolveOwner(images[i].image_id);
+		const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
@@ -912,6 +1015,8 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
 		}
+	}
+	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		auto& binding = images[i];
 		binding.mip_views.clear();
 		const auto& resource = program.info.images[i];
@@ -941,121 +1046,221 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 }
 
 RenderExecutor::GraphicsBindings
-RenderExecutor::PrepareGraphicsBindings(CommandBuffer& buffer, const ShaderStageRuntime& vertex,
+RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
                                         const ShaderStageRuntime& pixel, bool pixel_active) {
 	GraphicsBindings bindings {
-	    .vertex = PrepareBindings(buffer, vertex, vk::ShaderStageFlagBits::eVertex,
-	                              DescriptorCache::Stage::Vertex),
+	    .vertex = PrepareBindings(vertex),
 	};
-	RebindBuffers(buffer, bindings.vertex);
-	RebindImages(buffer, bindings.vertex);
 	if (pixel_active) {
-		bindings.pixel.emplace(PrepareBindings(buffer, pixel, vk::ShaderStageFlagBits::eFragment,
-		                                       DescriptorCache::Stage::Pixel));
-		RebindBuffers(buffer, *bindings.pixel);
-		RebindImages(buffer, *bindings.pixel);
+		bindings.pixel.emplace(PrepareBindings(pixel));
+	}
+	FindBuffers(bindings.vertex);
+	if (bindings.pixel) {
+		FindBuffers(*bindings.pixel);
+	}
+	if (bindings.vertex.program->info.uses_dma ||
+	    (bindings.pixel && bindings.pixel->program->info.uses_dma)) {
+		m_context.GetGpuResources().PrepareBda();
+	}
+	RebindBuffers(bindings.vertex);
+	if (bindings.pixel) {
+		RebindBuffers(*bindings.pixel);
+	}
+	RebindImages(bindings.vertex);
+	if (bindings.pixel) {
+		RebindImages(*bindings.pixel);
 	}
 	return bindings;
 }
 
-static void RetainBindings(CommandBuffer& buffer, DescriptorCache::NativeDescriptors& resources) {
-	if (resources.flattened_srt.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(resources.flattened_srt.owner));
-	}
-	if (resources.user_data.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(resources.user_data.owner));
-	}
-	for (auto& view: resources.buffers) {
-		if (view.owner != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(view.owner));
-		}
-	}
-	for (auto& view: resources.addresses) {
-		if (view.owner != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(view.owner));
-		}
-	}
-}
-
 void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
                                     vk::PipelineBindPoint              pipeline_bind_point,
-                                    vk::PipelineLayout                 layout,
-                                    DescriptorCache::PreparedBindings& prepared) {
+                                    const PipelineCache::Pipeline&     pipeline,
+                                    std::span<PreparedBindings* const> prepared_bindings) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.program == nullptr || prepared.stage == DescriptorCache::Stage::Unknown ||
-	        prepared.committed);
-	const auto& program     = *prepared.program;
-	auto&       descriptors = prepared.resources;
-	if (!prepared.flattened_srt.empty()) {
-		descriptors.flattened_srt = NativeUpload(m_context, buffer, prepared.flattened_srt);
-	}
-	if (ShaderRecompiler::IR::FindBinding(
-	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) {
-		descriptors.user_data = NativeUpload(m_context, buffer, prepared.user_data);
-	}
-	RetainBindings(buffer, descriptors);
-
-	auto       vk_buffer     = buffer.Handle();
-	const auto shader_stages = ShaderPipelineStages(prepared.shader_stage);
-	if (descriptors.gds.buffer != nullptr) {
-		buffer.EndRendering();
-		const auto barrier = MakeGdsDependency(descriptors.gds.buffer);
-		vk_buffer.pipelineBarrier(
-		    vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer |
-		        vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader,
-		    shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr);
-	}
-
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		auto&       image   = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
-		auto&       binding = descriptors.images[i];
-		const auto& view    = binding.desc.view_info;
-		const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
-		                                   view.layer_count};
-		const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
-		if (image.info.data.Empty()) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              storage
-			                  ? vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite
-			                  : vk::AccessFlagBits2::eShaderRead,
-			              range, vk_buffer);
-		} else if ((image.binding.force_general || image.binding.is_target) &&
-		           !image.info.IsDepth()) {
-			const vk::AccessFlags2 storage_access = image.binding.shader_write
-			                                            ? vk::AccessFlagBits2::eShaderWrite
-			                                            : vk::AccessFlags2 {};
-			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead | storage_access |
-			                  vk::AccessFlagBits2::eColorAttachmentRead |
-			                  vk::AccessFlagBits2::eColorAttachmentWrite,
-			              {}, vk_buffer);
-		} else if (storage) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
-			              range, vk_buffer);
-		} else {
-			image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
-			                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
-			              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+	auto   vk_buffer        = buffer.Handle();
+	size_t descriptor_count = 0;
+	size_t write_count      = 0;
+	constexpr auto GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment;
+	const auto push_constant_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics
+	                                      ? vk::ShaderStageFlags {GraphicsStages}
+	                                      : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute};
+	for (const auto* prepared: prepared_bindings) {
+		EXIT_IF(prepared == nullptr || prepared->program == nullptr ||
+		        prepared->snapshot == nullptr || prepared->committed);
+		write_count += prepared->program->bindings.descriptors.size();
+		for (const auto& binding: prepared->program->bindings.descriptors) {
+			descriptor_count += NativeDescriptorCount(binding);
 		}
-		binding.layout = image.backing.state.layout;
+		const auto shader_stage = NativeShaderStage(prepared->program->stage);
+		EXIT_IF((pipeline_bind_point == vk::PipelineBindPoint::eGraphics &&
+		         (shader_stage & GraphicsStages) == vk::ShaderStageFlags {}) ||
+		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
+		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
+	m_descriptor_buffers.clear();
+	m_descriptor_images.clear();
+	m_descriptor_writes.clear();
+	m_push_constants.fill(0);
+	m_descriptor_buffers.reserve(descriptor_count);
+	m_descriptor_images.reserve(descriptor_count);
+	m_descriptor_writes.reserve(write_count);
 
-	if (!program.bindings.descriptors.empty()) {
-		auto& set =
-		    m_context.GetDescriptorCache().GetDescriptor(prepared.stage, program, descriptors);
-		vk_buffer.bindDescriptorSets(pipeline_bind_point, layout, program.bindings.descriptor_set,
-		                             1, &set.set, 0, nullptr);
-		buffer.RecycleDescriptorAfterFence(set);
+	for (auto* prepared: prepared_bindings) {
+		const auto& program       = *prepared->program;
+		auto&       descriptors   = prepared->resources;
+		const auto  shader_stage  = NativeShaderStage(program.stage);
+		const auto  shader_stages = ShaderPipelineStages(shader_stage);
+		if (descriptors.gds.buffer != nullptr) {
+			buffer.EndRendering();
+			const auto barrier = MakeGdsDependency(descriptors.gds.buffer);
+			vk_buffer.pipelineBarrier(
+			    vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer |
+			        vk::PipelineStageFlagBits::eAllGraphics |
+			        vk::PipelineStageFlagBits::eComputeShader,
+			    shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr);
+		}
+
+		for (uint32_t i = 0; i < program.info.images.size(); i++) {
+			auto& image   = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
+			auto& binding = descriptors.images[i];
+			const auto&                 view = binding.desc.view_info;
+			const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
+			                                   view.layer_count};
+			const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
+			if (image.info.data.Empty()) {
+				image.Transit(vk::ImageLayout::eGeneral,
+				              storage ? vk::AccessFlagBits2::eShaderRead |
+				                            vk::AccessFlagBits2::eShaderWrite
+				                      : vk::AccessFlagBits2::eShaderRead,
+				              range, vk_buffer);
+			} else if ((image.binding.force_general || image.binding.is_target) &&
+			           !image.info.IsDepth()) {
+				const vk::AccessFlags2 storage_access = image.binding.shader_write
+				                                            ? vk::AccessFlagBits2::eShaderWrite
+				                                            : vk::AccessFlags2 {};
+				image.Transit(vk::ImageLayout::eGeneral,
+				              vk::AccessFlagBits2::eShaderRead | storage_access |
+				                  vk::AccessFlagBits2::eColorAttachmentRead |
+				                  vk::AccessFlagBits2::eColorAttachmentWrite,
+				              {}, vk_buffer);
+			} else if (storage) {
+				image.Transit(vk::ImageLayout::eGeneral,
+				              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
+				              range, vk_buffer);
+			} else {
+				image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
+				                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
+				              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+			}
+			binding.layout = image.backing.state.layout;
+		}
+
+		m_image_occurrences.assign(descriptors.images.size(), 0);
+		for (const auto& binding: program.bindings.descriptors) {
+			vk::WriteDescriptorSet write {};
+			write.sType          = vk::StructureType::eWriteDescriptorSet;
+			write.dstBinding     = ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind);
+			write.descriptorType = NativeDescriptorType(binding.kind);
+			write.descriptorCount   = NativeDescriptorCount(binding);
+			const auto buffer_start = m_descriptor_buffers.size();
+			const auto image_start  = m_descriptor_images.size();
+			switch (binding.kind) {
+				case BindingKind::Buffers:
+					for (const auto resource: binding.resources) {
+						const auto& view = descriptors.buffers.at(resource);
+						EXIT_IF(view.buffer == nullptr);
+						m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					}
+					break;
+				case BindingKind::BdaPagetable:
+				case BindingKind::FaultBuffer: {
+					auto& cache = m_context.GetBufferCache();
+					const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable
+					                             ? cache.GetBdaPageTableBuffer()
+					                             : cache.GetFaultBuffer();
+					m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size());
+					break;
+				}
+				case BindingKind::FlattenedSrt:
+				case BindingKind::UserData:
+				case BindingKind::Gds: {
+					const auto& view =
+					    binding.kind == BindingKind::FlattenedSrt ? descriptors.flattened_srt
+					    : binding.kind == BindingKind::UserData   ? descriptors.user_data
+					                                              : descriptors.gds;
+					EXIT_IF(view.buffer == nullptr);
+					m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					break;
+				}
+				case BindingKind::Samplers:
+					for (const auto resource: binding.resources) {
+						const auto sampler = descriptors.samplers.at(resource);
+						EXIT_IF(sampler == nullptr);
+						m_descriptor_images.emplace_back(sampler, nullptr,
+						                                 vk::ImageLayout::eUndefined);
+					}
+					break;
+				default:
+					for (const auto resource: binding.resources) {
+						m_descriptor_images.push_back(MakeImageInfo(
+						    descriptors.images.at(resource), m_image_occurrences.at(resource)++));
+					}
+					break;
+			}
+			if (m_descriptor_buffers.size() != buffer_start) {
+				write.pBufferInfo = m_descriptor_buffers.data() + buffer_start;
+			}
+			if (m_descriptor_images.size() != image_start) {
+				write.pImageInfo = m_descriptor_images.data() + image_start;
+			}
+			m_descriptor_writes.push_back(write);
+		}
+		for (uint32_t i = 0; i < descriptors.images.size(); i++) {
+			const auto expected =
+			    descriptors.images[i].mip_views.empty()
+			        ? 1u
+			        : static_cast<uint32_t>(descriptors.images[i].mip_views.size());
+			EXIT_IF(m_image_occurrences[i] != expected);
+		}
+
+		if (program.bindings.push_constant_size != 0) {
+			const auto offset = program.bindings.push_constant_offset;
+			EXIT_IF(offset % sizeof(uint32_t) != 0 ||
+			        program.bindings.push_constant_size !=
+			            prepared->user_data.size() * sizeof(uint32_t) ||
+			        offset + program.bindings.push_constant_size >
+			            ShaderRecompiler::IR::NativePushConstantSize);
+			std::copy(prepared->user_data.begin(), prepared->user_data.end(),
+			          m_push_constants.begin() + offset / sizeof(uint32_t));
+		}
 	}
-	if (program.bindings.push_constant_size != 0) {
-		EXIT_IF(program.bindings.push_constant_size !=
-		        prepared.user_data.size() * sizeof(uint32_t));
-		vk_buffer.pushConstants(layout, prepared.shader_stage,
-		                        program.bindings.push_constant_offset,
-		                        program.bindings.push_constant_size, prepared.user_data.data());
+	vk_buffer.pushConstants(pipeline.pipeline_layout, push_constant_stages, 0,
+	                        ShaderRecompiler::IR::NativePushConstantSize,
+	                        m_push_constants.data());
+
+	if (!m_descriptor_writes.empty()) {
+		EXIT_IF(pipeline.descriptor_set_layout == nullptr);
+		if (pipeline.uses_push_descriptors) {
+			vk_buffer.pushDescriptorSetKHR(pipeline_bind_point, pipeline.pipeline_layout, 0,
+			                               static_cast<uint32_t>(m_descriptor_writes.size()),
+			                               m_descriptor_writes.data());
+		} else {
+			const auto set = m_context.GetDescriptorHeap().Commit(pipeline.descriptor_set_layout);
+			for (auto& write: m_descriptor_writes) {
+				write.dstSet = set;
+			}
+			m_context.GetGraphics().device.updateDescriptorSets(
+			    static_cast<uint32_t>(m_descriptor_writes.size()), m_descriptor_writes.data(), 0,
+			    nullptr);
+			vk_buffer.bindDescriptorSets(pipeline_bind_point, pipeline.pipeline_layout, 0, 1, &set,
+			                             0, nullptr);
+		}
+	}
+	for (auto* prepared: prepared_bindings) {
+		prepared->committed = true;
 	}
-	prepared.committed = true;
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -2,6 +2,8 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_
 
 #include "common/assert.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/image/image.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
@@ -10,9 +12,54 @@
 #include <cstdint>
 #include <cstring>
 #include <type_traits>
+#include <vector>
 
 namespace Libs::Graphics {
 
+namespace ShaderRecompiler::IR {
+struct ResourceSnapshot;
+}
+
+struct BufferView {
+	vk::Buffer     buffer = nullptr;
+	vk::DeviceSize offset = 0;
+	vk::DeviceSize range  = VK_WHOLE_SIZE;
+};
+
+struct TextureBinding {
+	ImageId                    image_id;
+	vk::ImageView              image_view = nullptr;
+	TextureCache::ImageDesc    desc;
+	vk::ImageLayout            layout = vk::ImageLayout::eUndefined;
+	std::vector<vk::ImageView> mip_views;
+};
+
+struct NativeDescriptors {
+	std::vector<BufferView>     buffers;
+	std::vector<TextureBinding> images;
+	std::vector<vk::Sampler>    samplers;
+	BufferView                  gds;
+	BufferView                  flattened_srt;
+	BufferView                  user_data;
+};
+
+struct PreparedBindings {
+	const ShaderRecompiler::IR::Program*          program  = nullptr;
+	const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
+	NativeDescriptors                             resources;
+	std::vector<BufferId>                         buffer_ids;
+	std::vector<uint32_t>                         flattened_srt;
+	std::vector<uint32_t>                         user_data;
+	bool                                          committed = false;
+};
+
+[[nodiscard]] vk::DescriptorType
+NativeDescriptorType(ShaderRecompiler::IR::DescriptorBindingKind kind);
+[[nodiscard]] uint32_t
+NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding);
+[[nodiscard]] vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
+                                                    uint32_t              element = 0);
+
 template <typename T>
 [[nodiscard]] T DecodeNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& value) {
 	static_assert(std::is_trivially_copyable_v<T>);
@@ -34,9 +81,9 @@ ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource,
                          Prospero::ImageType type, uint32_t base_layer, uint32_t image_layers);
 
 [[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor,
-                                                    const Image&                 image);
+                                                    const Image& image, bool r128 = false);
 [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
-                                                   const Image&                 image);
+                                                   const Image& image, bool r128 = false);
 [[nodiscard]] bool
 IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
                                const ShaderTextureResource& descriptor, const Image& image);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -10,11 +10,14 @@
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
+#include "graphics/shader/shaderCompiler.h"
 
 #include <atomic>
+#include <cstdio>
 #include <cstring>
 #include <span>
 #include <utility>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -36,16 +39,132 @@ void NormalizeStaticParamsForDynamicState(PipelineStaticParameters& static_param
 
 } // namespace
 
+struct PipelineCache::ProgramCache {
+	struct SourceKey {
+		ShaderType stage = ShaderType::Unknown;
+		uint64_t   hash  = 0;
+
+		bool operator==(const SourceKey&) const = default;
+	};
+
+	struct Permutation {
+		std::vector<uint32_t>                               static_key;
+		std::shared_ptr<const ShaderRecompiler::IR::Program> program;
+		ShaderProgram                                       handle;
+	};
+
+	struct SourceKeyHash {
+		std::size_t operator()(const SourceKey& key) const {
+			std::size_t hash = static_cast<std::size_t>(key.stage);
+			PipelineKeyHash::Mix(hash, static_cast<std::size_t>(key.hash));
+			if constexpr (sizeof(std::size_t) < sizeof(uint64_t)) {
+				PipelineKeyHash::Mix(hash, static_cast<std::size_t>(key.hash >> 32u));
+			}
+			return hash;
+		}
+	};
+
+	static uint64_t MixId(uint64_t hash, uint64_t value) {
+		return hash ^ (value + 0x9e3779b97f4a7c15ull + (hash << 6u) + (hash >> 2u));
+	}
+
+	static constexpr std::size_t MaxStaticKeyWords =
+	    5 + ShaderVertexInputInfo::RES_MAX * 17;
+
+	template <typename InputInfo>
+	ShaderProgram Get(const ShaderParams& params, InputInfo& input_info) {
+		constexpr ShaderType stage = [] {
+			if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
+				return ShaderType::Vertex;
+			} else if constexpr (std::is_same_v<InputInfo, ShaderPixelInputInfo>) {
+				return ShaderType::Pixel;
+			} else {
+				static_assert(std::is_same_v<InputInfo, ShaderComputeInputInfo>);
+				return ShaderType::Compute;
+			}
+		}();
+
+		BuildStageStaticKey(input_info, key_scratch);
+		auto& permutations = programs[{stage, params.hash}];
+		for (const auto& permutation: permutations) {
+			if (permutation.static_key == key_scratch &&
+			    MaterializeProgram(permutation.program, params, input_info)) {
+				return permutation.handle;
+			}
+		}
+
+		const auto module = CompileProgram(device, params, input_info);
+		EXIT_IF(module == nullptr || !input_info.stage);
+		uint64_t id = MixId(MixId(params.hash, static_cast<uint64_t>(stage)), permutations.size());
+		if (id == 0) {
+			id = 1;
+		}
+		const ShaderProgram handle {.id = id, .module = module};
+		permutations.push_back({key_scratch, input_info.stage.program, handle});
+
+		std::printf("Num compiled %u shaders\n", ++num_compiled);
+		return handle;
+	}
+
+	explicit ProgramCache(vk::Device device): device(device) {
+		key_scratch.reserve(MaxStaticKeyWords);
+	}
+
+	std::unordered_map<SourceKey, std::vector<Permutation>, SourceKeyHash> programs;
+	std::vector<uint32_t> key_scratch;
+	vk::Device device;
+	uint32_t   num_compiled = 0;
+};
+
+PipelineCache::PipelineCache(GraphicContext& graphics)
+    : m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics.device)) {
+	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
+}
+
 PipelineCache::~PipelineCache() {
 	auto destroy = [this](const auto& pipelines) {
 		for (const auto& [key, pipeline]: pipelines) {
 			(void)key;
 			m_graphics.device.destroyPipeline(pipeline->pipeline, nullptr);
 			m_graphics.device.destroyPipelineLayout(pipeline->pipeline_layout, nullptr);
+			m_graphics.device.destroyDescriptorSetLayout(pipeline->descriptor_set_layout, nullptr);
 		}
 	};
 	destroy(m_graphics_pipelines);
 	destroy(m_compute_pipelines);
+	for (const auto& [key, permutations]: m_program_cache->programs) {
+		(void)key;
+		for (const auto& permutation: permutations) {
+			m_graphics.device.destroyShaderModule(permutation.handle.module, nullptr);
+		}
+	}
+}
+
+ShaderProgram PipelineCache::GetVertexProgram(const HW::VertexShaderInfo& regs,
+                                              const HW::ShaderRegisters&  sh,
+                                              ShaderVertexInputInfo&      input_info) {
+	const auto params = PrepareProgram(regs, sh, input_info);
+	Common::LockGuard lock(m_mutex);
+	return m_program_cache->Get(params, input_info);
+}
+
+ShaderProgram PipelineCache::GetPixelProgram(
+    const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
+    const ShaderVertexInputInfo&                        vertex_info,
+    std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
+    ShaderPixelInputInfo&                               input_info) {
+	const auto params = PrepareProgram(regs, sh, vertex_info, target_export_mapping, input_info);
+	Common::LockGuard lock(m_mutex);
+	return m_program_cache->Get(params, input_info);
+}
+
+ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs,
+                                               const HW::ShaderRegisters&   sh,
+                                               ShaderComputeInputInfo&      input_info) {
+	input_info.needs_lds_barriers = !m_graphics.compute_wave64_supported;
+	const auto params             = PrepareProgram(regs, sh, input_info);
+	Common::LockGuard lock(m_mutex);
+	return m_program_cache->Get(params, input_info);
 }
 
 bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept {
@@ -53,24 +172,22 @@ bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other)
 }
 
 PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
-    RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
-    ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
-    ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
-    bool primitive_restart_enable, bool ps_active, std::span<const uint32_t> vs_spirv,
-    std::span<const uint32_t> ps_spirv) {
+    std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
+    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+    const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
+    bool primitive_restart_enable, const ShaderProgram& vertex_program,
+    const ShaderProgram& pixel_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200);
 
-	EXIT_IF(colors == nullptr);
-	EXIT_IF(color_count > RENDER_COLOR_ATTACHMENTS_MAX);
-	EXIT_IF(vs_spirv.empty());
-	EXIT_IF(ps_active && ps_spirv.empty());
+	EXIT_IF(colors.size() > RENDER_COLOR_ATTACHMENTS_MAX);
+	EXIT_IF(!vertex_program);
+	const bool ps_active = ps_input_info != nullptr;
+	EXIT_IF(ps_active && !pixel_program);
+	const auto color_count = static_cast<uint32_t>(colors.size());
 
 	Common::LockGuard lock(m_mutex);
-	auto&             ctx    = command.GetRegisters();
-	auto&             sh_ctx = command.GetShaders();
+	auto&             ctx = command.GetRegisters();
 
-	const auto&           vertex_info                              = sh_ctx.GetVs();
-	const auto&           ps_regs                                  = sh_ctx.GetPs();
 	const HW::BlendColor& bclr                                     = ctx.GetBlendColor();
 	uint32_t              color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	for (uint32_t i = 0; i < color_count; i++) {
@@ -81,11 +198,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 	}
 	const HW::ModeControl& mc = ctx.GetModeControl();
 
-	auto     vs_id = ShaderGetIdVS(vertex_info, vs_input_info, true);
-	ShaderId ps_id {};
-	if (ps_active) {
-		ps_id = ShaderGetIdPS(ps_regs, *ps_input_info, true);
-	}
+	const auto vs_id = vertex_program.id;
+	const auto ps_id = ps_active ? pixel_program.id : 0;
 
 	PipelineStaticParameters static_params {};
 	GraphicsPipeline         p {};
@@ -201,20 +315,17 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 		if (ps_active) {
 			ShaderDbgDumpInputInfo(*ps_input_info);
 		}
-		LOGF("PipelineTrace: shader binaries VS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64 "\n",
-		     vs_id.hash0, vs_id.crc32, static_cast<uint64_t>(vs_spirv.size()), ps_id.hash0,
-		     ps_id.crc32, static_cast<uint64_t>(ps_spirv.size()));
+		LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64
+		     " module=%p\n",
+		     vs_id, static_cast<void*>(vertex_program.module), ps_id,
+		     static_cast<void*>(pixel_program.module));
 	}
 
 	auto cached = std::make_unique<GraphicsPipeline>(p);
-	LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0,
-	                 ps_id.crc32);
-	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info,
-	                       vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0,
-	                       vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active);
-	LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0,
-	                 ps_id.crc32);
+	LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id);
+	CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vertex_program.module,
+	                       ps_input_info, pixel_program.module, static_params);
+	LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
 	EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr);
@@ -226,19 +337,16 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 }
 
 PipelineCache::ComputePipeline&
-PipelineCache::CreateComputePipeline(ShaderComputeInputInfo&      input_info,
-                                     const HW::ComputeShaderInfo& cs_regs,
-                                     std::span<const uint32_t>    cs_spirv) {
+PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info,
+                                     const ShaderProgram&    compute_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100);
 
-	EXIT_IF(cs_spirv.empty());
+	EXIT_IF(!compute_program);
 
 	Common::LockGuard lock(m_mutex);
 
-	auto cs_id = ShaderGetIdCS(cs_regs, input_info, true);
-
 	ComputePipeline p {};
-	p.cs_shader_id = cs_id;
+	p.cs_shader_id = compute_program.id;
 
 	ComputePipelineKey key {};
 	key.cs_shader_id = p.cs_shader_id;
@@ -252,7 +360,7 @@ PipelineCache::CreateComputePipeline(ShaderComputeInputInfo&      input_info,
 	}
 
 	auto cached = std::make_unique<ComputePipeline>(p);
-	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, input_info, cs_spirv);
+	CreatePipelineInternal(m_graphics, *cached, input_info, compute_program.module);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
 	EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -20,8 +20,7 @@ namespace Libs::Graphics {
 struct GraphicContext;
 struct RenderColorInfo;
 struct RenderDepthInfo;
-class RenderCommandBuffer;
-class DescriptorCache;
+class CommandBuffer;
 
 namespace HW {
 class Context;
@@ -96,44 +95,65 @@ struct PipelineRenderingState {
 	bool operator==(const PipelineRenderingState&) const = default;
 };
 
+struct ShaderProgram {
+	uint64_t         id     = 0;
+	vk::ShaderModule module = nullptr;
+
+	explicit operator bool() const {
+		return id != 0 && module != nullptr;
+	}
+};
+
 class PipelineCache {
 public:
-	PipelineCache(GraphicContext& graphics, DescriptorCache& descriptor_cache)
-	    : m_graphics(graphics), m_descriptor_cache(descriptor_cache) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	}
+	explicit PipelineCache(GraphicContext& graphics);
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 
 	struct Pipeline {
-		vk::PipelineLayout pipeline_layout = nullptr;
-		vk::Pipeline       pipeline        = nullptr;
+		vk::PipelineLayout      pipeline_layout       = nullptr;
+		vk::Pipeline            pipeline              = nullptr;
+		vk::DescriptorSetLayout descriptor_set_layout = nullptr;
+		bool                    uses_push_descriptors = false;
 	};
 
 	struct GraphicsPipeline: Pipeline {
-		ShaderId vs_shader_id;
-		ShaderId ps_shader_id;
+		uint64_t vs_shader_id = 0;
+		uint64_t ps_shader_id = 0;
 	};
 
 	struct ComputePipeline: Pipeline {
-		ShaderId cs_shader_id;
+		uint64_t cs_shader_id = 0;
 	};
 
-	GraphicsPipeline&
-	CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
-	                       ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
-	                       ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
-	                       bool primitive_restart_enable, bool ps_active,
-	                       std::span<const uint32_t> vs_spirv, std::span<const uint32_t> ps_spirv);
-	ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo&      input_info,
-	                                       const HW::ComputeShaderInfo& cs_regs,
-	                                       std::span<const uint32_t>    cs_spirv);
+	ShaderProgram GetVertexProgram(const HW::VertexShaderInfo& regs,
+	                               const HW::ShaderRegisters&  sh,
+	                               ShaderVertexInputInfo&      input_info);
+	ShaderProgram
+	GetPixelProgram(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
+	                const ShaderVertexInputInfo&                        vertex_info,
+	                std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
+	                ShaderPixelInputInfo&                               input_info);
+	ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs,
+	                                const HW::ShaderRegisters& sh,
+	                                ShaderComputeInputInfo& input_info);
+
+	GraphicsPipeline& CreateGraphicsPipeline(
+	    std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
+	    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+	    const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
+	    bool primitive_restart_enable, const ShaderProgram& vertex_program,
+	    const ShaderProgram& pixel_program);
+	ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info,
+	                                       const ShaderProgram&    compute_program);
 
 private:
+	struct ProgramCache;
+
 	struct GraphicsPipelineKey {
 		PipelineRenderingState   rendering;
-		ShaderId                 vs_shader_id;
-		ShaderId                 ps_shader_id;
+		uint64_t                 vs_shader_id = 0;
+		uint64_t                 ps_shader_id = 0;
 		PipelineStaticParameters static_params;
 
 		bool operator==(const GraphicsPipelineKey& other) const {
@@ -143,7 +163,7 @@ private:
 	};
 
 	struct ComputePipelineKey {
-		ShaderId cs_shader_id;
+		uint64_t cs_shader_id = 0;
 
 		bool operator==(const ComputePipelineKey& other) const {
 			return cs_shader_id == other.cs_shader_id;
@@ -156,15 +176,6 @@ private:
 			        (hash >> 2u);
 		}
 
-		static void MixShaderId(std::size_t& hash, const ShaderId& id) {
-			Mix(hash, id.hash0);
-			Mix(hash, id.crc32);
-			Mix(hash, id.ids.size());
-			for (auto value: id.ids) {
-				Mix(hash, value);
-			}
-		}
-
 		static void MixStaticParams(std::size_t& hash, const PipelineStaticParameters& params) {
 			const auto* bytes = reinterpret_cast<const uint8_t*>(&params);
 			for (std::size_t i = 0; i < sizeof(params); i++) {
@@ -186,8 +197,8 @@ private:
 		std::size_t operator()(const GraphicsPipelineKey& key) const {
 			std::size_t hash = 0;
 			PipelineKeyHash::MixRendering(hash, key.rendering);
-			PipelineKeyHash::MixShaderId(hash, key.vs_shader_id);
-			PipelineKeyHash::MixShaderId(hash, key.ps_shader_id);
+			PipelineKeyHash::Mix(hash, key.vs_shader_id);
+			PipelineKeyHash::Mix(hash, key.ps_shader_id);
 			PipelineKeyHash::MixStaticParams(hash, key.static_params);
 			return hash;
 		}
@@ -196,13 +207,13 @@ private:
 	struct ComputePipelineKeyHash {
 		std::size_t operator()(const ComputePipelineKey& key) const {
 			std::size_t hash = 0;
-			PipelineKeyHash::MixShaderId(hash, key.cs_shader_id);
+			PipelineKeyHash::Mix(hash, key.cs_shader_id);
 			return hash;
 		}
 	};
 
-	GraphicContext&  m_graphics;
-	DescriptorCache& m_descriptor_cache;
+	GraphicContext&                m_graphics;
+	std::unique_ptr<ProgramCache> m_program_cache;
 	std::unordered_map<GraphicsPipelineKey, std::unique_ptr<GraphicsPipeline>,
 	                   GraphicsPipelineKeyHash>
 	    m_graphics_pipelines;
@@ -211,19 +222,15 @@ private:
 	Common::Mutex m_mutex;
 };
 
-void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
-                      uint32_t ps_crc32);
+void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id);
 void CreatePipelineInternal(
-    GraphicContext& graphics, DescriptorCache& descriptor_cache,
-    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
-    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
-    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
-    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
-    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active);
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::ComputePipeline& pipeline,
-                            const ShaderComputeInputInfo&   input_info,
-                            std::span<const uint32_t>       cs_shader);
+    GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline,
+    const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info,
+    vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info,
+	vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline,
+                            const ShaderComputeInputInfo& input_info,
+                            vk::ShaderModule              compute_module);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp

@@ -1,60 +0,0 @@
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
-
-#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
-
-namespace Libs::Graphics {
-
-ShaderSubgroupCapabilities::ShaderSubgroupCapabilities(const GraphicContext& graphics)
-    : subgroup_size(graphics.subgroup_size), min_subgroup_size(graphics.min_subgroup_size),
-      max_subgroup_size(graphics.max_subgroup_size),
-      required_subgroup_size_stages(graphics.required_subgroup_size_stages),
-      subgroup_size_control_enabled(graphics.subgroup_size_control_enabled) {}
-
-ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size) {
-	// A 64-wide guest wave can't assume the host GPU's lane layout matches
-	// GCN's. AMD RDNA also reports a 64-wide subgroup but uses a different
-	// lane/pixel layout, which corrupted rendering under "native wave" mode.
-	// Per-invocation emulation works regardless of the host's lane layout.
-	if (guest_wave_size == 64u) {
-		return ShaderLaneMaskMode::PerInvocation;
-	}
-	return ShaderLaneMaskMode::NativeWave;
-}
-
-ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
-                                                    vk::ShaderStageFlagBits           stage,
-                                                    const ShaderRecompiler::IR::Program& program) {
-	const auto guest_wave_size = program.wave_size;
-	if (guest_wave_size != 32u && guest_wave_size != 64u) {
-		return {};
-	}
-	if (stage != vk::ShaderStageFlagBits::eCompute) {
-		const auto expected = SelectGraphicsLaneMaskMode(guest_wave_size);
-		if (program.lane_mask_mode != expected || (capabilities.subgroup_size != guest_wave_size &&
-		                                           expected != ShaderLaneMaskMode::PerInvocation)) {
-			return {};
-		}
-		return {expected == ShaderLaneMaskMode::NativeWave
-		            ? ShaderSubgroupMode::Natural
-		            : ShaderSubgroupMode::PerInvocationGraphics,
-		        0};
-	}
-	if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) {
-		return {};
-	}
-	if (capabilities.subgroup_size == guest_wave_size) {
-		return {ShaderSubgroupMode::Natural, 0};
-	}
-	if (capabilities.subgroup_size_control_enabled &&
-	    (capabilities.required_subgroup_size_stages & stage) &&
-	    guest_wave_size >= capabilities.min_subgroup_size &&
-	    guest_wave_size <= capabilities.max_subgroup_size) {
-		return {ShaderSubgroupMode::Controlled, guest_wave_size};
-	}
-	if (!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(program)) {
-		return {ShaderSubgroupMode::FlattenedMasks, 0};
-	}
-	return {};
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h

@@ -1,40 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
-
-#include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-namespace Libs::Graphics {
-
-enum class ShaderSubgroupMode {
-	Natural,
-	Controlled,
-	PerInvocationGraphics,
-	FlattenedMasks,
-	Unsupported
-};
-
-struct ShaderSubgroupConfiguration {
-	ShaderSubgroupMode mode          = ShaderSubgroupMode::Unsupported;
-	uint32_t           required_size = 0;
-};
-
-struct ShaderSubgroupCapabilities {
-	explicit ShaderSubgroupCapabilities(const GraphicContext& graphics);
-
-	uint32_t             subgroup_size     = 0;
-	uint32_t             min_subgroup_size = 0;
-	uint32_t             max_subgroup_size = 0;
-	vk::ShaderStageFlags required_subgroup_size_stages;
-	bool                 subgroup_size_control_enabled = false;
-};
-
-ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size);
-
-ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
-                                                    vk::ShaderStageFlagBits           stage,
-                                                    const ShaderRecompiler::IR::Program& program);
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -6,9 +6,8 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
@@ -20,6 +19,7 @@
 #include <algorithm>
 #include <limits>
 #include <span>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -389,116 +389,52 @@ static vk::BlendOp GetBlendOp(uint32_t op) {
 	return vk::BlendOp::eAdd;
 }
 
-static void CreateLayout(DescriptorCache&                   descriptor_cache,
-                         std::span<vk::DescriptorSetLayout> set_layouts, uint32_t& set_layouts_num,
-                         std::span<vk::PushConstantRange>     push_constant_info,
-                         uint32_t&                            push_constant_info_num,
-                         const ShaderRecompiler::IR::Program& program,
-                         vk::ShaderStageFlags vk_stage, DescriptorCache::Stage stage) {
-	const auto& bindings        = program.bindings;
-	const bool  need_descriptor = !bindings.descriptors.empty();
-
-	if (bindings.push_constant_size != 0) {
-		auto index = push_constant_info_num;
-
-		push_constant_info[index].stageFlags = vk_stage;
-		push_constant_info[index].offset     = bindings.push_constant_offset;
-		push_constant_info[index].size       = bindings.push_constant_size;
-		push_constant_info_num++;
-	}
-
-	if (need_descriptor) {
-		EXIT_IF(bindings.descriptor_set != set_layouts_num);
-
-		set_layouts[set_layouts_num] = descriptor_cache.GetDescriptorSetLayout(stage, program);
-		set_layouts_num++;
+static void AddLayoutBindings(std::vector<vk::DescriptorSetLayoutBinding>& descriptor_bindings,
+                              const ShaderRecompiler::IR::Program& program,
+                              vk::ShaderStageFlagBits              stage) {
+	for (const auto& binding: program.bindings.descriptors) {
+		descriptor_bindings.push_back(
+		    {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind),
+		     NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), stage, nullptr});
 	}
 }
 
-static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
-                                  const ShaderRecompiler::IR::Program&                   program,
-                                  vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
-                                  vk::PipelineShaderStageCreateInfo&                     stage) {
-	const auto config =
-	    ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program);
-	switch (config.mode) {
-		case ShaderSubgroupMode::Natural: return;
-		case ShaderSubgroupMode::PerInvocationGraphics: {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation "
-				     "EXEC/VCC on the host-native subgroup\n",
-				     program.wave_size, static_cast<uint32_t>(vk_stage));
-			}
-			return;
-		}
-		case ShaderSubgroupMode::FlattenedMasks: {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation "
-				     "EXEC/VCC\n",
-				     program.wave_size, static_cast<uint32_t>(vk_stage));
-			}
-			return;
-		}
-		case ShaderSubgroupMode::Controlled: break;
-		case ShaderSubgroupMode::Unsupported:
-		default:
-			EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u "
-			     "controlled_stages=0x%08x\n",
-			     program.wave_size, static_cast<uint32_t>(vk_stage), graphics.subgroup_size,
-			     graphics.min_subgroup_size, graphics.max_subgroup_size,
-			     static_cast<vk::ShaderStageFlags::MaskType>(
-			         graphics.required_subgroup_size_stages));
+static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                                   std::span<const vk::DescriptorSetLayoutBinding> bindings) {
+	uint32_t descriptor_count = 0;
+	for (const auto& binding: bindings) {
+		descriptor_count += binding.descriptorCount;
 	}
+	pipeline.uses_push_descriptors = descriptor_count <= graphics.max_push_descriptors;
 
-	required.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
-	required.requiredSubgroupSize = config.required_size;
-	stage.pNext                   = &required;
+	vk::DescriptorSetLayoutCreateInfo create {};
+	create.sType        = vk::StructureType::eDescriptorSetLayoutCreateInfo;
+	create.flags        = pipeline.uses_push_descriptors
+	                          ? vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR
+	                          : vk::DescriptorSetLayoutCreateFlags {};
+	create.bindingCount = static_cast<uint32_t>(bindings.size());
+	create.pBindings    = bindings.data();
+	EXIT_IF(graphics.device.createDescriptorSetLayout(
+	            &create, nullptr, &pipeline.descriptor_set_layout) != vk::Result::eSuccess);
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void CreatePipelineInternal(
-    GraphicContext& graphics, DescriptorCache& descriptor_cache,
-    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
-    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
-    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
-    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
-    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
-	EXIT_IF(ps_active && ps_input_info == nullptr);
+    GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline,
+    const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info,
+    vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info,
+	vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params) {
+	const bool ps_active = ps_input_info != nullptr;
+	EXIT_IF(vertex_module == nullptr || (ps_active && pixel_module == nullptr));
 
 	const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList;
 
-	vk::ShaderModule vert_shader_module         = nullptr;
 	vk::ShaderModule tess_control_shader_module = nullptr;
 	vk::ShaderModule tess_eval_shader_module    = nullptr;
-	vk::ShaderModule frag_shader_module         = nullptr;
 
 	vk::ShaderModuleCreateInfo create_info {};
-
 	create_info.sType = vk::StructureType::eShaderModuleCreateInfo;
-	create_info.pNext = nullptr;
-	create_info.flags = {};
-
-	create_info.codeSize = vs_shader.size() * 4;
-	create_info.pCode    = vs_shader.data();
-	auto result = graphics.device.createShaderModule(&create_info, nullptr, &vert_shader_module);
-	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: vkCreateShaderModule VS done result=%s module=%p\n",
-		     VulkanToString(result).c_str(), static_cast<void*>(vert_shader_module));
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	if (ps_active) {
-		create_info.codeSize = ps_shader.size() * 4;
-		create_info.pCode    = ps_shader.data();
-		result = graphics.device.createShaderModule(&create_info, nullptr, &frag_shader_module);
-		if (graphics_debug_dump_enabled()) {
-			LOGF("PipelineTrace: vkCreateShaderModule PS done result=%s module=%p\n",
-			     VulkanToString(result).c_str(), static_cast<void*>(frag_shader_module));
-		}
-		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-	}
+	vk::Result result {};
 	if (rect_list) {
 		const auto shaders =
 		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
@@ -523,40 +459,27 @@ void CreatePipelineInternal(
 		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	}
 
-	EXIT_NOT_IMPLEMENTED(vert_shader_module == nullptr);
 	EXIT_NOT_IMPLEMENTED(
 	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
-	EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
 
-	vk::PipelineShaderStageCreateInfo                     vert_shader_stage_info {};
-	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {};
+	vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
 
 	vert_shader_stage_info.sType               = vk::StructureType::ePipelineShaderStageCreateInfo;
 	vert_shader_stage_info.pNext               = nullptr;
 	vert_shader_stage_info.flags               = {};
 	vert_shader_stage_info.stage               = vk::ShaderStageFlagBits::eVertex;
-	vert_shader_stage_info.module              = vert_shader_module;
+	vert_shader_stage_info.module              = vertex_module;
 	vert_shader_stage_info.pName               = "main";
 	vert_shader_stage_info.pSpecializationInfo = nullptr;
-	EXIT_IF(!vs_input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
-	                      vert_subgroup_size, vert_shader_stage_info);
 
-	vk::PipelineShaderStageCreateInfo                     frag_shader_stage_info {};
-	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {};
+	vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
 	frag_shader_stage_info.sType               = vk::StructureType::ePipelineShaderStageCreateInfo;
 	frag_shader_stage_info.pNext               = nullptr;
 	frag_shader_stage_info.flags               = {};
 	frag_shader_stage_info.stage               = vk::ShaderStageFlagBits::eFragment;
-	frag_shader_stage_info.module              = frag_shader_module;
+	frag_shader_stage_info.module              = pixel_module;
 	frag_shader_stage_info.pName               = "main";
 	frag_shader_stage_info.pSpecializationInfo = nullptr;
-	if (ps_active) {
-		EXIT_IF(!ps_input_info->stage);
-		ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
-		                      *ps_input_info->stage.program, frag_subgroup_size,
-		                      frag_shader_stage_info);
-	}
 
 	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
 	tess_control_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
@@ -599,9 +522,10 @@ void CreatePipelineInternal(
 
 			uint32_t attr_size       = 4;
 			auto     registers_num   = vs_input_info.resources_dst[index].registers_num;
-			auto     used_components = (vs_input_info.resource_fetch_components[index] > 0
-			                                ? vs_input_info.resource_fetch_components[index]
-			                                : registers_num);
+			const auto compiled_components =
+			    vs_input_info.stage.program->info.vertex_fetch_components[index];
+			auto used_components =
+			    compiled_components > 0 ? static_cast<int>(compiled_components) : registers_num;
 			GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size,
 			               static_cast<uint32_t>(used_components));
 
@@ -877,40 +801,36 @@ void CreatePipelineInternal(
 	color_blending.blendConstants[2] = static_params.blend_color_blue;
 	color_blending.blendConstants[3] = static_params.blend_color_alpha;
 
-	vk::DescriptorSetLayout set_layouts[2]  = {};
-	uint32_t                set_layouts_num = 0;
-
-	vk::PushConstantRange push_constant_info[2];
-	uint32_t              push_constant_info_num = 0;
-
 	EXIT_IF(!vs_input_info.stage);
-	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num, *vs_input_info.stage.program,
-	             vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex);
+	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
+	AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program,
+	                  vk::ShaderStageFlagBits::eVertex);
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
-		CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-		             push_constant_info_num, *ps_input_info->stage.program,
-		             vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel);
+		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
+		                  vk::ShaderStageFlagBits::eFragment);
 	}
+	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
+	constexpr auto GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment;
+	const vk::PushConstantRange push_constants {
+	    GraphicsStages, 0, ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;
 	pipeline_layout_info.pNext                  = nullptr;
 	pipeline_layout_info.flags                  = {};
-	pipeline_layout_info.setLayoutCount         = set_layouts_num;
-	pipeline_layout_info.pSetLayouts            = (set_layouts_num > 0 ? set_layouts : nullptr);
-	pipeline_layout_info.pushConstantRangeCount = push_constant_info_num;
-	pipeline_layout_info.pPushConstantRanges =
-	    push_constant_info_num > 0 ? push_constant_info : nullptr;
+	pipeline_layout_info.setLayoutCount         = 1;
+	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.pushConstantRangeCount = 1;
+	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
 	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=0x%08" PRIx32 "/0x%08" PRIx32
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=%" PRIu32 " push_constants=%" PRIu32
-		     "\n",
-		     vs_hash0, vs_crc32, ps_hash0, ps_crc32, set_layouts_num, push_constant_info_num);
+		LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=%" PRIu64 " PS=%" PRIu64
+		     " set_layouts=1 push_constants=%" PRIu32 "\n",
+		     pipeline.vs_shader_id, pipeline.ps_shader_id, 1u);
 	}
 	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
 	                                              &pipeline.pipeline_layout);
@@ -1009,11 +929,12 @@ void CreatePipelineInternal(
 	EXIT_IF(pipeline.pipeline != nullptr);
 
 	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=0x%08" PRIx32 "/0x%08" PRIx32
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " topology=%" PRIu32 " color_mask=0x%08" PRIx32
+		LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=%" PRIu64 " PS=%" PRIu64
+		     " topology=%" PRIu32 " color_mask=0x%08" PRIx32
 		     " depth=%s blend=%s dyn_states=%" PRIu32 " viewport=%f,%f %fx%f"
 		     " scissor=%d,%d %ux%u\n",
-		     vs_hash0, vs_crc32, ps_hash0, ps_crc32, static_cast<uint32_t>(static_params.topology),
+		     pipeline.vs_shader_id, pipeline.ps_shader_id,
+		     static_cast<uint32_t>(static_params.topology),
 		     static_params.color_mask[0], (static_params.with_depth ? "true" : "false"),
 		     (static_params.blend_enable[0] ? "true" : "false"), dynamic_states_count, viewport.x,
 		     viewport.y, viewport.width, viewport.height, scissor.offset.x, scissor.offset.y,
@@ -1029,40 +950,19 @@ void CreatePipelineInternal(
 
 	EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr);
 
-	if (frag_shader_module != nullptr) {
-		graphics.device.destroyShaderModule(frag_shader_module, nullptr);
-	}
 	if (tess_control_shader_module != nullptr) {
 		graphics.device.destroyShaderModule(tess_control_shader_module, nullptr);
 	}
 	if (tess_eval_shader_module != nullptr) {
 		graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr);
 	}
-	graphics.device.destroyShaderModule(vert_shader_module, nullptr);
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::ComputePipeline& pipeline,
-                            const ShaderComputeInputInfo&   input_info,
-                            std::span<const uint32_t>       cs_shader) {
-	vk::ShaderModule comp_shader_module = nullptr;
-
-	vk::ShaderModuleCreateInfo create_info {};
-
-	create_info.sType    = vk::StructureType::eShaderModuleCreateInfo;
-	create_info.pNext    = nullptr;
-	create_info.flags    = {};
-	create_info.codeSize = cs_shader.size() * 4;
-	create_info.pCode    = cs_shader.data();
-	LOGF("PipelineTrace: vkCreateShaderModule CS begin words=%" PRIu64 "\n",
-	     static_cast<uint64_t>(cs_shader.size()));
-	auto result = graphics.device.createShaderModule(&create_info, nullptr, &comp_shader_module);
-	LOGF("PipelineTrace: vkCreateShaderModule CS done result=%s module=%p\n",
-	     VulkanToString(result).c_str(), static_cast<void*>(comp_shader_module));
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	EXIT_NOT_IMPLEMENTED(comp_shader_module == nullptr);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline,
+                            const ShaderComputeInputInfo& input_info,
+                            vk::ShaderModule              compute_module) {
+	EXIT_IF(compute_module == nullptr);
 
 	vk::PipelineShaderStageCreateInfo                     comp_shader_stage_info {};
 	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo comp_subgroup_size {};
@@ -1070,40 +970,41 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	comp_shader_stage_info.pNext               = nullptr;
 	comp_shader_stage_info.flags               = {};
 	comp_shader_stage_info.stage               = vk::ShaderStageFlagBits::eCompute;
-	comp_shader_stage_info.module              = comp_shader_module;
+	comp_shader_stage_info.module              = compute_module;
 	comp_shader_stage_info.pName               = "main";
 	comp_shader_stage_info.pSpecializationInfo = nullptr;
 	EXIT_IF(!input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
-	                      comp_subgroup_size, comp_shader_stage_info);
+	const auto wave_size = input_info.stage.program->wave_size;
+	if (graphics.compute_subgroup_size_control_enabled &&
+	    wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) {
+		comp_subgroup_size.sType =
+		    vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
+		comp_subgroup_size.requiredSubgroupSize = wave_size;
+		comp_shader_stage_info.pNext            = &comp_subgroup_size;
+	}
 
-	vk::DescriptorSetLayout set_layouts[1]  = {};
-	uint32_t                set_layouts_num = 0;
-
-	vk::PushConstantRange push_constant_info[1];
-	uint32_t              push_constant_info_num = 0;
-
-	EXIT_IF(!input_info.stage);
-	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num, *input_info.stage.program,
-	             vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);
+	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
+	AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
+	                  vk::ShaderStageFlagBits::eCompute);
+	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
+	const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
+	                                            ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;
 	pipeline_layout_info.pNext                  = nullptr;
 	pipeline_layout_info.flags                  = {};
-	pipeline_layout_info.setLayoutCount         = set_layouts_num;
-	pipeline_layout_info.pSetLayouts            = (set_layouts_num > 0 ? set_layouts : nullptr);
-	pipeline_layout_info.pushConstantRangeCount = push_constant_info_num;
-	pipeline_layout_info.pPushConstantRanges =
-	    push_constant_info_num > 0 ? push_constant_info : nullptr;
+	pipeline_layout_info.setLayoutCount         = 1;
+	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.pushConstantRangeCount = 1;
+	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
-	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=%u push_constants=%u\n",
-	     set_layouts_num, push_constant_info_num);
-	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
-	                                              &pipeline.pipeline_layout);
+	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n",
+	     1u);
+	auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
+	                                                  &pipeline.pipeline_layout);
 	LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n",
 	     VulkanToString(result).c_str(), static_cast<void*>(pipeline.pipeline_layout));
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
@@ -1129,8 +1030,6 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
 	EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr);
-
-	graphics.device.destroyShaderModule(comp_shader_module, nullptr);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/render.h

@@ -4,12 +4,14 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
-#include <memory>
+#include <optional>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -23,9 +25,6 @@ class Shader;
 struct GraphicContext;
 struct ShaderBufferResource;
 struct ShaderComputeInputInfo;
-struct CommandSlot;
-struct VulkanBuffer;
-struct VulkanDescriptorSet;
 struct RenderDepthInfo;
 struct RenderColorInfo;
 struct DrawCallInfo;
@@ -49,20 +48,6 @@ enum class CommandBufferDebugOp : uint32_t {
 	Unknown,
 };
 
-class FenceResourceRetainer {
-public:
-	FenceResourceRetainer() = default;
-	~FenceResourceRetainer();
-	KYTY_CLASS_NO_COPY(FenceResourceRetainer);
-
-	void               Retain(std::shared_ptr<void> resource);
-	void               ReleaseAfterFence() noexcept;
-	[[nodiscard]] bool Empty() const noexcept { return m_resources.empty(); }
-
-private:
-	std::vector<std::shared_ptr<void>> m_resources;
-};
-
 struct SubmitInfo {
 	static constexpr uint32_t MaxSemaphores = 3;
 
@@ -91,78 +76,52 @@ struct SubmitInfo {
 
 class CommandBuffer {
 public:
-	explicit CommandBuffer(CommandScheduler& scheduler);
-	~CommandBuffer();
+	~CommandBuffer() = default;
 
 	KYTY_CLASS_NO_COPY(CommandBuffer);
 
 	[[nodiscard]] bool IsInvalid() const;
 
-	void Begin() const;
-	void End() const;
-	void Execute(const SubmitInfo& submit = {});
 	void SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0 = 0, uint32_t arg1 = 0,
 	                  uint32_t arg2 = 0, uint32_t arg3 = 0, uint64_t arg4 = 0);
 	void BeginRendering(const RenderState& state) const;
 	void EndRendering() const;
-	void WaitForFenceOnly();
-	void WaitForFence();
-	void WaitForFenceAndReset();
-	void RetireBufferAfterFence(std::unique_ptr<VulkanBuffer> buffer);
-	void RetainResourceUntilFence(std::shared_ptr<void> resource);
-	void RecycleDescriptorAfterFence(VulkanDescriptorSet& set);
 
 	[[nodiscard]] vk::CommandBuffer Handle() const;
 	[[nodiscard]] GraphicContext&   GetGraphics() const noexcept { return m_graphics; }
 	[[nodiscard]] RenderContext&    GetContext() const noexcept { return m_context; }
-	[[nodiscard]] bool              IsExecute() const { return m_execute; }
+	[[nodiscard]] HW::Context&      GetRegisters() const noexcept { return *m_registers; }
+	[[nodiscard]] HW::UserConfig&   GetUserConfig() const noexcept { return *m_user_config; }
+	[[nodiscard]] HW::Shader&       GetShaders() const noexcept { return *m_shaders; }
 
 private:
-	void Release();
-	void FinalizeFence(bool reset_recording);
-	void ReleaseResourcesAfterFence();
-	void DeleteBuffersAfterFence();
-	void RecycleDescriptorsAfterFence();
-
-	RenderContext&                             m_context;
-	CommandScheduler&                          m_scheduler;
-	GraphicContext&                            m_graphics;
-	CommandSlot*                               m_slot            = nullptr;
-	bool                                       m_execute         = false;
-	bool                                       m_fence_waited    = false;
-	uint64_t                                   m_submit_seq      = 0;
-	uint32_t                                   m_debug_op        = 0;
-	uint64_t                                   m_debug_submit_id = 0;
-	uint32_t                                   m_debug_arg0      = 0;
-	uint32_t                                   m_debug_arg1      = 0;
-	uint32_t                                   m_debug_arg2      = 0;
-	uint32_t                                   m_debug_arg3      = 0;
-	uint64_t                                   m_debug_arg4      = 0;
-	std::vector<std::unique_ptr<VulkanBuffer>> m_retired_buffers;
-	FenceResourceRetainer                      m_fence_resources;
-	std::vector<VulkanDescriptorSet*>          m_descriptor_sets_after_fence;
-	mutable RenderState                        m_render_state;
-	mutable bool                               m_rendering = false;
-};
-
-class RenderCommandBuffer final: public CommandBuffer {
-public:
-	explicit RenderCommandBuffer(CommandScheduler& scheduler): CommandBuffer(scheduler) {}
-
+	explicit CommandBuffer(CommandScheduler& scheduler);
 	void Bind(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) noexcept {
 		m_registers   = &registers;
 		m_user_config = &user_config;
 		m_shaders     = &shaders;
 	}
 
-	[[nodiscard]] HW::Context&    GetRegisters() const noexcept { return *m_registers; }
-	[[nodiscard]] HW::UserConfig& GetUserConfig() const noexcept { return *m_user_config; }
-	[[nodiscard]] HW::Shader&     GetShaders() const noexcept { return *m_shaders; }
+	void Begin();
+	void End() const;
 
-private:
-	HW::Context*    m_registers   = nullptr;
-	HW::UserConfig* m_user_config = nullptr;
-	HW::Shader*     m_shaders     = nullptr;
+	RenderContext&      m_context;
+	GraphicContext&     m_graphics;
+	vk::CommandBuffer   m_buffer          = nullptr;
+	uint32_t            m_debug_op        = 0;
+	uint64_t            m_debug_submit_id = 0;
+	uint32_t            m_debug_arg0      = 0;
+	uint32_t            m_debug_arg1      = 0;
+	uint32_t            m_debug_arg2      = 0;
+	uint32_t            m_debug_arg3      = 0;
+	uint64_t            m_debug_arg4      = 0;
+	mutable RenderState m_render_state;
+	mutable bool        m_rendering   = false;
+	HW::Context*        m_registers   = nullptr;
+	HW::UserConfig*     m_user_config = nullptr;
+	HW::Shader*         m_shaders     = nullptr;
+
+	friend class CommandScheduler;
 };
 
 class RenderExecutor {
@@ -170,67 +129,69 @@ public:
 	explicit RenderExecutor(RenderContext& context): m_context(context) {}
 	KYTY_CLASS_NO_COPY(RenderExecutor);
 
-	void DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_type_and_size,
+	void DrawIndex(uint64_t submit_id, CommandBuffer& buffer, uint32_t index_type_and_size,
 	               uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type,
 	               uint32_t instance_count = 1, uint32_t render_target_slice_offset = 0,
 	               int32_t vertex_offset_add = 0, uint32_t first_instance = 0);
-	void DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
-	              uint32_t flags, uint32_t render_target_slice_offset = 0,
-	              uint32_t instance_count = 1, uint32_t first_vertex = 0,
-	              uint32_t first_instance = 0);
-	void DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x,
+	void DrawAuto(uint64_t submit_id, CommandBuffer& buffer, uint32_t index_count, uint32_t flags,
+	              uint32_t render_target_slice_offset = 0, uint32_t instance_count = 1,
+	              uint32_t first_vertex = 0, uint32_t first_instance = 0);
+	void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x,
 	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode);
 
-	[[nodiscard]] DescriptorCache::PreparedBindings
-	     PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime,
-	                     vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage);
-	void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings);
-	void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings);
+	[[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime);
+	void                           FindBuffers(PreparedBindings& bindings);
+	void                           RebindBuffers(PreparedBindings& bindings);
+	void                           RebindImages(PreparedBindings& bindings);
 	void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
-	                    vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings);
+	                    const PipelineCache::Pipeline&     pipeline,
+	                    std::span<PreparedBindings* const> bindings);
 
 private:
 	struct GraphicsBindings {
-		DescriptorCache::PreparedBindings                vertex;
-		std::optional<DescriptorCache::PreparedBindings> pixel;
+		PreparedBindings                vertex;
+		std::optional<PreparedBindings> pixel;
 	};
 
-	[[nodiscard]] DescriptorCache::TextureBinding
-	ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
-	               const ShaderRecompiler::IR::DescriptorValue& value);
-	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(CommandBuffer&            buffer,
-	                                                       const ShaderStageRuntime& vertex,
+	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
+	                                            const ShaderRecompiler::IR::DescriptorValue& value);
+	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
 	                                                       const ShaderStageRuntime& pixel,
 	                                                       bool                      pixel_active);
-	void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+	void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer& buffer,
 	                              RenderColorInfo& target, uint32_t render_target_slice_offset = 0,
 	                              uint32_t render_target_slot = UINT32_MAX,
 	                              bool ignore_target_mask = false, bool exact_format = false);
-	void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+	void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer& buffer,
 	                              RenderDepthInfo& target);
-	[[nodiscard]] bool PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
+	[[nodiscard]] bool PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& buffer,
 	                                          const DrawCallInfo& draw,
 	                                          uint32_t            render_target_slice_offset,
 	                                          bool log_setup_phases, DrawRenderState& state);
-	void               ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
-	                                       const DrawCallInfo& draw, DrawRenderState& state,
-	                                       vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
-	                                       const DrawIndexBufferSource& index_source,
-	                                       bool primitive_restart_enable, bool log_pipeline_phase,
-	                                       bool set_bind_debug, bool set_auto_debug);
+	void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer, const DrawCallInfo& draw,
+	                         DrawRenderState& state, vk::PrimitiveTopology topology,
+	                         const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
+	                         bool primitive_restart_enable, bool log_pipeline_phase,
+	                         bool set_bind_debug, bool set_auto_debug);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth);
-	[[nodiscard]] bool        ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
+	[[nodiscard]] bool        ResolveColorTargets(uint64_t submit_id, CommandBuffer& buffer,
 	                                              uint32_t render_target_slice_offset);
 	void                      BindImage(ImageId id, bool storage);
 	void                      BindRenderTarget(ImageId id);
 	void                      TrackImageBinding(ImageId id);
 	void                      ResetBindings();
 	[[nodiscard]] bool        TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
-	                                                     const RenderCommandBuffer&    buffer);
+	                                                     const CommandBuffer&          buffer);
 
-	RenderContext&                      m_context;
-	std::vector<std::shared_ptr<Image>> m_bound_images;
+	RenderContext&                        m_context;
+	std::vector<ImageId>                  m_bound_images;
+	std::vector<vk::DescriptorBufferInfo> m_descriptor_buffers;
+	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;
+	std::vector<vk::WriteDescriptorSet>   m_descriptor_writes;
+	std::vector<uint32_t>                 m_image_occurrences;
+	std::array<uint32_t, ShaderRecompiler::IR::NativePushConstantSize / sizeof(uint32_t)>
+	    m_push_constants {};
 
 	friend struct RenderExecutorTestAccess;
 };

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -11,11 +11,9 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -46,7 +44,7 @@ static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) {
 }
 
 bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
-                                                const RenderCommandBuffer&    buffer) {
+                                                const CommandBuffer&          buffer) {
 	const auto& program   = *input.stage.program;
 	const auto& resources = *input.stage.resources;
 	if (resources.buffers.size() != program.info.buffers.size()) {
@@ -83,9 +81,8 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
 	const auto& program   = *input.stage.program;
 	const auto& resources = *input.stage.resources;
 	if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 ||
-	    !program.info.images.empty() || !program.info.samplers.empty() ||
-	    !program.info.addresses.empty() || !resources.images.empty() ||
-	    !resources.samplers.empty() || !resources.addresses.empty()) {
+	    !program.info.images.empty() || !program.info.samplers.empty() || program.info.uses_dma ||
+	    !resources.images.empty() || !resources.samplers.empty()) {
 		return false;
 	}
 	const auto& resource   = program.info.buffers.front();
@@ -115,7 +112,7 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
 	    group_z == 1 && input.dispatch_threads_num[0] == group_x &&
 	    input.dispatch_threads_num[1] == 1 && input.dispatch_threads_num[2] == 1 &&
 	    input.group_id[0] && !input.group_id[1] && !input.group_id[2] &&
-	    input.thread_ids_num == 1 && input.wave_size == 32 && !input.tg_size_en && mode == 0x61u &&
+	    input.thread_ids_num == 1 && input.wave_size == 64 && !input.tg_size_en && mode == 0x61u &&
 	    group_x % input.threads_num[0] == 0 && descriptor.NumRecords() == group_x;
 	const auto size = BufferDescriptorSize(descriptor);
 	if (!full_dispatch || size == 0) {
@@ -139,6 +136,21 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com
 	}
 	auto& cache = command.GetContext().GetTextureCache();
 	if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) {
+		// Recognized metadata-fill shaders access DCC as an ordinary storage buffer and may run
+		// before the render target is bound. TryConsumeDccFill either consumes registered state
+		// or retains a PendingDcc fill while allowing the dispatch to run.
+		const bool registered_metadata =
+		    cache.TryConsumeDccFill(descriptor.Base48(), size, packed_clear);
+		static std::atomic<uint32_t> logged_metadata_clears {0};
+		if (logged_metadata_clears.fetch_add(1, std::memory_order_relaxed) < 32) {
+			LOGF("GraphicsRenderDispatchDirect: %s metadata clear shader=0x%016" PRIx64
+			     " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n",
+			     registered_metadata ? "tracked" : "deferred", input.stage.program->shader_hash,
+			     descriptor.Base48(), size, packed_clear);
+		}
+		if (registered_metadata) {
+			return true;
+		}
 		return false;
 	}
 	static std::atomic<uint32_t> logged_clears {0};
@@ -150,10 +162,11 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com
 	return true;
 }
 
-void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
                                     uint32_t thread_group_x, uint32_t thread_group_y,
                                     uint32_t thread_group_z, uint32_t mode) {
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -194,16 +207,12 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 	const auto& cs_regs = sh_ctx.GetCs();
 	const auto& sh_regs = ctx.GetShaderRegisters();
 
-	ShaderComputeInputInfo    input_info {};
-	std::span<const uint32_t> cs_shader;
-	if (!ShaderCompileInfoCS(cs_regs, sh_regs, input_info, cs_shader)) {
-		EXIT("ShaderCompileInfoCS failed for dispatch with CS shader 0x%016" PRIx64 "\n",
-		     cs_regs.cs_regs.data_addr);
-	}
-
+	ShaderComputeInputInfo input_info {};
 	const bool use_thread_dimensions = (mode & DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0;
+	input_info.dispatch_thread_dimensions = use_thread_dimensions;
+	const auto compute_program =
+	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
 	if (use_thread_dimensions) {
-		input_info.dispatch_thread_dimensions = true;
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
 		input_info.dispatch_threads_num[2]    = thread_group_z;
@@ -264,7 +273,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 			     r.Type() == Prospero::ImageType::kColor2DMsaa ||
 			             r.Type() == Prospero::ImageType::kColor2DMsaaArray
 			         ? 1u
-			         : static_cast<uint32_t>(r.MaxMip()) + 1u,
+			         : static_cast<uint32_t>(image.r128 ? r.LastLevel() : r.MaxMip()) + 1u,
 			     static_cast<uint32_t>(r.TileMode()));
 		}
 		for (uint32_t i = 0; i < program.info.samplers.size(); i++) {
@@ -318,14 +327,19 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 
 	buffer.EndRendering();
 	auto& pipeline =
-	    m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader);
-	auto bindings = PrepareBindings(buffer, input_info.stage, vk::ShaderStageFlagBits::eCompute,
-	                                DescriptorCache::Stage::Compute);
-	RebindBuffers(buffer, bindings);
-	RebindImages(buffer, bindings);
+	    m_context.GetPipelineCache().CreateComputePipeline(input_info, compute_program);
+	auto bindings = PrepareBindings(input_info.stage);
+	FindBuffers(bindings);
+	if (program.info.uses_dma) {
+		m_context.GetGpuResources().PrepareBda();
+	}
+	RebindBuffers(bindings);
+	RebindImages(bindings);
 
-	auto vk_buffer = buffer.Handle();
-	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline.pipeline_layout, bindings);
+	auto              vk_buffer        = buffer.Handle();
+	PreparedBindings* descriptor_stage = &bindings;
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
 	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
 	has_storage_writes =
 	    std::any_of(program.info.images.begin(), program.info.images.end(),

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -4,7 +4,6 @@
 #include "common/logging/log.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/presentation/videoOut.h"
-#include "kernel/pthread.h"
 #include "libs/errno.h"
 
 #include <algorithm>
@@ -13,8 +12,9 @@ namespace Libs::Graphics {
 
 RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
-      m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache),
-      m_sampler_cache(graphics), m_gpu_resources(graphics, m_command_scheduler) {
+      m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
+      m_pipeline_cache(graphics), m_sampler_cache(graphics),
+      m_gpu_resources(graphics, m_command_scheduler) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -38,7 +38,7 @@ void RenderContext::ShutdownGpu() {
 	}
 	if (m_video_out != nullptr) {
 		if (m_command_scheduler.Active()) {
-			m_command_scheduler.FinishCurrent();
+			m_command_scheduler.Finish();
 		}
 		m_command_scheduler.DrainPriorityOperations();
 		m_video_out = nullptr;
@@ -55,60 +55,54 @@ VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const {
 	return *m_video_out;
 }
 
-void RenderContext::AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) {
-	auto queue = LibKernel::EventQueue::KernelPinEqueue(eq);
-	if (!queue) {
+void RenderContext::AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
+	Common::LockGuard lock(m_interrupt_mutex);
+
+	auto it = std::find_if(
+	    m_interrupt_eqs.begin(), m_interrupt_eqs.end(),
+	    [eq, event_id](const auto& entry) { return entry.eq == eq && entry.event_id == event_id; });
+	if (it != m_interrupt_eqs.end()) {
 		return;
 	}
 
-	Common::LockGuard lock(m_eop_mutex);
-
-	auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) {
-		return entry.eq == eq && entry.id == id;
-	});
-	if (it != m_eop_eqs.end()) {
-		return;
-	}
-
-	m_eop_eqs.push_back({eq, std::move(queue), id});
+	m_interrupt_eqs.push_back({eq, event_id});
 }
 
-void RenderContext::DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) {
-	Common::LockGuard lock(m_eop_mutex);
+void RenderContext::DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
+	Common::LockGuard lock(m_interrupt_mutex);
 
-	auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) {
-		return entry.eq == eq && entry.id == id;
-	});
-	if (it == m_eop_eqs.end()) {
+	auto it = std::find_if(
+	    m_interrupt_eqs.begin(), m_interrupt_eqs.end(),
+	    [eq, event_id](const auto& entry) { return entry.eq == eq && entry.event_id == event_id; });
+	if (it == m_interrupt_eqs.end()) {
 		return;
 	}
 
-	m_eop_eqs.erase(it);
+	m_interrupt_eqs.erase(it);
 }
 
-void RenderContext::TriggerEopEvent(uint32_t context_id) {
-	std::vector<EopEqRegistration> registrations;
+void RenderContext::TriggerInterrupt(int event_id, uint32_t context_id) {
+	std::vector<InterruptEqRegistration> registrations;
 	{
-		Common::LockGuard lock(m_eop_mutex);
-		registrations = m_eop_eqs;
+		Common::LockGuard lock(m_interrupt_mutex);
+		for (const auto& registration: m_interrupt_eqs) {
+			if (registration.event_id == event_id) {
+				registrations.push_back(registration);
+			}
+		}
 	}
 
 	for (const auto& registration: registrations) {
 		const auto result = LibKernel::EventQueue::KernelTriggerEvent(
-		    registration.eq, static_cast<uintptr_t>(registration.id),
+		    registration.eq, static_cast<uintptr_t>(registration.event_id),
 		    LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS,
 		    reinterpret_cast<void*>(static_cast<uintptr_t>(context_id)));
 		if (result == LibKernel::KERNEL_ERROR_EBADF || result == LibKernel::KERNEL_ERROR_ENOENT) {
-			DeleteEopEq(registration.eq, registration.id);
+			DeleteInterruptEq(registration.eq, registration.event_id);
 			continue;
 		}
 		EXIT_NOT_IMPLEMENTED(result != OK);
 	}
-
-	auto tsc    = LibKernel::KernelReadTsc();
-	auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT,
-	                                                                  reinterpret_cast<void*>(tsc));
-	EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderContext.h

@@ -10,7 +10,7 @@
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "kernel/eventQueue.h"
 
@@ -23,7 +23,6 @@ class VideoOutDriver;
 
 namespace Libs::Graphics {
 
-constexpr int AGC_USER_INTERRUPT_EVENT = 0x1800;
 class GuestGpu;
 
 class RenderContext {
@@ -41,37 +40,36 @@ public:
 	Common::Mutex&      GetMutex() { return m_mutex; }
 	CommandScheduler&   GetCommandScheduler() { return m_command_scheduler; }
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
-	DescriptorCache&    GetDescriptorCache() { return m_descriptor_cache; }
+	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
 	GpuResourceManager& GetGpuResources() { return m_gpu_resources; }
 	BufferCache&        GetBufferCache() { return m_gpu_resources.GetBufferCache(); }
 	TextureCache&       GetTextureCache() { return m_gpu_resources.GetTextureCache(); }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
 
-	void AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id);
-	void DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id);
-	void TriggerEopEvent(uint32_t context_id);
+	void AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
+	void DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
+	void TriggerInterrupt(int event_id, uint32_t context_id);
 
 private:
-	struct EopEqRegistration {
-		LibKernel::EventQueue::KernelEqueue    eq = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
-		LibKernel::EventQueue::KernelEqueueRef queue;
-		int                                    id = 0;
+	struct InterruptEqRegistration {
+		LibKernel::EventQueue::KernelEqueue eq       = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
+		int                                 event_id = 0;
 	};
 
 	GraphicContext&           m_graphics;
 	Common::Mutex             m_mutex;
 	RenderExecutor            m_render_executor;
 	CommandScheduler          m_command_scheduler;
-	DescriptorCache           m_descriptor_cache;
+	DescriptorHeap            m_descriptor_heap;
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
 	GpuResourceManager        m_gpu_resources;
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 
-	Common::Mutex                  m_eop_mutex;
-	std::vector<EopEqRegistration> m_eop_eqs;
+	Common::Mutex                        m_interrupt_mutex;
+	std::vector<InterruptEqRegistration> m_interrupt_eqs;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -15,10 +15,8 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -105,7 +103,7 @@ static bool IsDualSourceBlendFactor(uint32_t factor) {
 }
 
 static void LogFramebufferSkip(const char* draw_name, const RenderColorInfo& color,
-                               const RenderDepthInfo& depth, const RenderCommandBuffer& buffer,
+                               const RenderDepthInfo& depth, const CommandBuffer& buffer,
                                uint32_t index_count, uint32_t flags) {
 	const auto& ctx  = buffer.GetRegisters();
 	const auto& ucfg = buffer.GetUserConfig();
@@ -128,7 +126,7 @@ static void LogFramebufferSkip(const char* draw_name, const RenderColorInfo& col
 	    static_cast<uint32_t>(ucfg.GetPrimType()), index_count, flags);
 }
 
-static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer,
+static void LogMrtState(const char* draw_name, const CommandBuffer& buffer,
                         const ShaderPixelInputInfo& ps_input_info) {
 	const auto& ctx            = buffer.GetRegisters();
 	const auto& sh_regs        = ctx.GetShaderRegisters();
@@ -142,18 +140,6 @@ static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer
 	                   (bc0.separate_alpha_blend && (IsDualSourceBlendFactor(bc0.alpha_srcblend) ||
 	                                                 IsDualSourceBlendFactor(bc0.alpha_destblend)));
 
-	for (uint32_t i = 1; i < 8; i++) {
-		const auto& rt = ctx.GetRenderTarget(i);
-		if (rt.base.addr != 0 || ps_input_info.target_output_mode[i] != 0 ||
-		    ((rt_mask >> (i * 4u)) & 0x0fu) != 0 || ((cb_shader_mask >> (i * 4u)) & 0x0fu) != 0) {
-			interesting = true;
-		}
-	}
-
-	if (!interesting) {
-		return;
-	}
-
 	auto log_id = g_mrt_state_log_count.fetch_add(1);
 	if (log_id >= 32) {
 		return;
@@ -191,7 +177,7 @@ static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer
 }
 
 static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& color,
-                               const RenderDepthInfo& depth, const RenderCommandBuffer& buffer,
+                               const RenderDepthInfo& depth, const CommandBuffer& buffer,
                                const ShaderPixelInputInfo& ps_input_info, uint32_t index_count,
                                uint32_t flags) {
 	const auto& ctx  = buffer.GetRegisters();
@@ -245,7 +231,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
 	LogMrtState(draw_name, buffer, ps_input_info);
 }
 
-static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
+static void LogDrawInputState(const CommandBuffer& buffer, const RenderColorInfo& color,
                               const ShaderVertexInputInfo& vs_input_info,
                               uint32_t index_type_and_size, uint32_t index_count,
                               const void* index_addr) {
@@ -323,17 +309,14 @@ static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderCol
 	}
 }
 
-static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandBuffer& buffer,
-                                                            const RenderColorInfo*     colors,
-                                                            uint32_t                   color_count,
-                                                            const RenderDepthInfo&     depth) {
+static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuffer vk_buffer,
+                                     const RenderColorInfo* colors, uint32_t color_count,
+                                     const RenderDepthInfo& depth) {
+	KYTY_PROFILER_FUNCTION();
+
 	EXIT_IF(colors == nullptr);
 	const auto& ctx = buffer.GetRegisters();
 
-	PipelineDynamicParameters ret {};
-	ret.color_write_count   = color_count;
-	ret.stencil_test_enable = depth.stencil_test_enable;
-
 	const auto&  vp = ctx.GetScreenViewport();
 	vk::Extent2D framebuffer_extent {};
 	if (color_count > 0 && colors[0].image_id) {
@@ -347,68 +330,22 @@ static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandB
 
 	const auto final_scissor = calc_final_scissor(vp, ctx.GetScanModeControl(), framebuffer_extent);
 
-	ret.viewport_scale[0]  = vp.viewports[0].xscale;
-	ret.viewport_scale[1]  = vp.viewports[0].yscale;
-	ret.viewport_scale[2]  = vp.viewports[0].zscale;
-	ret.viewport_offset[0] = vp.viewports[0].xoffset;
-	ret.viewport_offset[1] = vp.viewports[0].yoffset;
-	ret.viewport_offset[2] = vp.viewports[0].zoffset;
-	ret.scissor_ltrb[0]    = final_scissor.left;
-	ret.scissor_ltrb[1]    = final_scissor.top;
-	ret.scissor_ltrb[2]    = final_scissor.right;
-	ret.scissor_ltrb[3]    = final_scissor.bottom;
-	ret.line_width         = ctx.GetLineWidth();
-	ret.stencil_front      = depth.stencil_dynamic_front;
-	ret.stencil_back       = depth.stencil_dynamic_back;
-
-	const auto& mode        = ctx.GetModeControl();
-	const auto& poly_offset = ctx.GetPolyOffset();
-	const bool  use_front   = mode.poly_offset_front_enable && !mode.cull_front;
-	const bool  use_back    = mode.poly_offset_back_enable && !mode.cull_back;
-	ret.depth_bias_enable   = use_front || use_back;
-	if (ret.depth_bias_enable) {
-		// Vulkan has one bias for both faces. Prefer a visible front face when both are enabled.
-		const float guest_constant_factor =
-		    use_front ? poly_offset.front_offset : poly_offset.back_offset;
-		ret.depth_bias_constant_factor =
-		    ConvertPolygonOffsetConstantFactor(guest_constant_factor, poly_offset, depth.format);
-		ret.depth_bias_clamp = poly_offset.clamp;
-		ret.depth_bias_slope_factor =
-		    (use_front ? poly_offset.front_scale : poly_offset.back_scale) / 16.0f;
-	}
-
-	// CB_COLOR_CONTROL.operation controls special CB operations, not the normal color component
-	// write mask. Use CB_TARGET_MASK here so scanout passes with mode=Disable do not go black.
-	for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) {
-		ret.color_write_enable[i] =
-		    (i < color_count &&
-		     render_target_mask_slot(ctx.GetRenderTargetMask(), colors[i].target_slot) != 0);
-	}
-
-	return ret;
-}
-
-static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                             const PipelineDynamicParameters& dynamic_params) {
-	KYTY_PROFILER_FUNCTION();
-
 	vk::Viewport viewport {};
-	viewport.x        = dynamic_params.viewport_offset[0] - dynamic_params.viewport_scale[0];
-	viewport.y        = dynamic_params.viewport_offset[1] - dynamic_params.viewport_scale[1];
-	viewport.width    = dynamic_params.viewport_scale[0] * 2.0f;
-	viewport.height   = dynamic_params.viewport_scale[1] * 2.0f;
-	viewport.minDepth = dynamic_params.viewport_offset[2];
-	viewport.maxDepth = dynamic_params.viewport_scale[2] + dynamic_params.viewport_offset[2];
+	viewport.x        = vp.viewports[0].xoffset - vp.viewports[0].xscale;
+	viewport.y        = vp.viewports[0].yoffset - vp.viewports[0].yscale;
+	viewport.width    = vp.viewports[0].xscale * 2.0f;
+	viewport.height   = vp.viewports[0].yscale * 2.0f;
+	viewport.minDepth = vp.viewports[0].zoffset;
+	viewport.maxDepth = vp.viewports[0].zscale + vp.viewports[0].zoffset;
 	vk_buffer.setViewport(0, 1, &viewport);
 
 	vk::Rect2D scissor {};
-	scissor.offset = {dynamic_params.scissor_ltrb[0], dynamic_params.scissor_ltrb[1]};
-	scissor.extent = {
-	    static_cast<uint32_t>(dynamic_params.scissor_ltrb[2] - dynamic_params.scissor_ltrb[0]),
-	    static_cast<uint32_t>(dynamic_params.scissor_ltrb[3] - dynamic_params.scissor_ltrb[1])};
+	scissor.offset = {final_scissor.left, final_scissor.top};
+	scissor.extent = {static_cast<uint32_t>(final_scissor.right - final_scissor.left),
+	                  static_cast<uint32_t>(final_scissor.bottom - final_scissor.top)};
 	vk_buffer.setScissor(0, 1, &scissor);
 
-	float line_width = dynamic_params.line_width;
+	float line_width = ctx.GetLineWidth();
 	if (line_width != 1.0f) {
 		static bool logged = false;
 		if (!logged) {
@@ -421,26 +358,36 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe
 	}
 	vk_buffer.setLineWidth(line_width);
 
-	vk_buffer.setDepthBiasEnable(dynamic_params.depth_bias_enable ? VK_TRUE : VK_FALSE);
-	if (dynamic_params.depth_bias_enable) {
-		vk_buffer.setDepthBias(dynamic_params.depth_bias_constant_factor,
-		                       dynamic_params.depth_bias_clamp,
-		                       dynamic_params.depth_bias_slope_factor);
+	const auto& mode              = ctx.GetModeControl();
+	const auto& poly_offset       = ctx.GetPolyOffset();
+	const bool  use_front         = mode.poly_offset_front_enable && !mode.cull_front;
+	const bool  use_back          = mode.poly_offset_back_enable && !mode.cull_back;
+	const bool  depth_bias_enable = use_front || use_back;
+	vk_buffer.setDepthBiasEnable(depth_bias_enable ? VK_TRUE : VK_FALSE);
+	if (depth_bias_enable) {
+		// Vulkan has one bias for both faces. Prefer a visible front face when both are enabled.
+		const float guest_constant_factor =
+		    use_front ? poly_offset.front_offset : poly_offset.back_offset;
+		const float constant_factor =
+		    ConvertPolygonOffsetConstantFactor(guest_constant_factor, poly_offset, depth.format);
+		const float slope_factor =
+		    (use_front ? poly_offset.front_scale : poly_offset.back_scale) / 16.0f;
+		vk_buffer.setDepthBias(constant_factor, poly_offset.clamp, slope_factor);
 	}
 
-	if (dynamic_params.stencil_test_enable) {
+	if (depth.stencil_test_enable) {
 		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eFront,
-		                                dynamic_params.stencil_front.compareMask);
+		                                depth.stencil_dynamic_front.compareMask);
 		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eBack,
-		                                dynamic_params.stencil_back.compareMask);
+		                                depth.stencil_dynamic_back.compareMask);
 		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eFront,
-		                              dynamic_params.stencil_front.writeMask);
+		                              depth.stencil_dynamic_front.writeMask);
 		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eBack,
-		                              dynamic_params.stencil_back.writeMask);
+		                              depth.stencil_dynamic_back.writeMask);
 		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eFront,
-		                              dynamic_params.stencil_front.reference);
+		                              depth.stencil_dynamic_front.reference);
 		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eBack,
-		                              dynamic_params.stencil_back.reference);
+		                              depth.stencil_dynamic_back.reference);
 	}
 
 #if defined(__APPLE__)
@@ -448,11 +395,15 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe
 	// eColorWriteEnableEXT dynamic state and relies on the static colorWriteMask instead.
 #else
 	vk::Bool32 enable[RENDER_COLOR_ATTACHMENTS_MAX] = {};
-	for (uint32_t i = 0; i < dynamic_params.color_write_count; i++) {
-		enable[i] = (dynamic_params.color_write_enable[i] ? VK_TRUE : VK_FALSE);
+	// Color-control operation selects special color-buffer paths, not the normal component write
+	// mask. Attachment availability therefore follows the target write mask.
+	for (uint32_t i = 0; i < color_count; i++) {
+		enable[i] = render_target_mask_slot(ctx.GetRenderTargetMask(), colors[i].target_slot) != 0
+		                ? VK_TRUE
+		                : VK_FALSE;
 	}
-	if (dynamic_params.color_write_count != 0) {
-		vk_buffer.setColorWriteEnableEXT(dynamic_params.color_write_count, enable);
+	if (color_count != 0) {
+		vk_buffer.setColorWriteEnableEXT(color_count, enable);
 	}
 #endif
 }
@@ -470,7 +421,7 @@ static bool PixelShaderHasDepthOrCoverageSideEffects(const HW::ShaderRegisters&
 	       db.shader_execute_on_noop;
 }
 
-static bool ShouldSkipGeShader(const RenderCommandBuffer& buffer) {
+static bool ShouldSkipGeShader(const CommandBuffer& buffer) {
 	const auto& ctx         = buffer.GetRegisters();
 	const auto& ucfg        = buffer.GetUserConfig();
 	const auto& sh_ctx      = buffer.GetShaders();
@@ -532,15 +483,15 @@ static bool ShouldSkipGeShader(const RenderCommandBuffer& buffer) {
 }
 
 struct DrawRenderState {
-	RenderDepthInfo           depth_info;
-	RenderColorInfo           color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
-	uint32_t                  color_count                              = 0;
-	bool                      ps_active                                = true;
-	RenderState               rendering;
-	ShaderVertexInputInfo     vs_input_info;
-	ShaderPixelInputInfo      ps_input_info;
-	std::span<const uint32_t> vs_shader;
-	std::span<const uint32_t> ps_shader;
+	RenderDepthInfo       depth_info;
+	RenderColorInfo       color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
+	uint32_t              color_count                              = 0;
+	bool                  ps_active                                = true;
+	RenderState           rendering;
+	ShaderVertexInputInfo vs_input_info;
+	ShaderPixelInputInfo  ps_input_info;
+	ShaderProgram         vertex_program;
+	ShaderProgram         pixel_program;
 };
 
 struct DrawCallInfo {
@@ -552,6 +503,70 @@ struct DrawCallInfo {
 	uint32_t             first_instance = 0;
 };
 
+static bool ResolveDccAttachmentClear(TextureCache& cache, const RenderColorInfo& target,
+                                      const ImageViewInfo& view, vk::ClearColorValue& clear_value) {
+	if (target.desc.info.metadata.kind != ImageMetadataKind::Dcc) {
+		return false;
+	}
+
+	// A DCC fast clear may update metadata only and leave the color allocation stale. Vulkan has
+	// no guest DCC state, so materialize the deferred value when the surface is next bound.
+	const auto metadata_address = target.desc.info.metadata.range.address;
+	uint32_t   metadata_value   = 0xffffffffu;
+	if (!cache.IsMetaCleared(metadata_address, view.base_layer, &metadata_value)) {
+		return false;
+	}
+
+	// Translate recognized constant and register-backed DCC states into a host clear value.
+	clear_value = {};
+	switch (static_cast<uint8_t>(metadata_value)) {
+		case 0x00: break;
+		case 0x20:
+			if (!target.metadata_clear_supported) {
+				return false;
+			}
+			clear_value = target.color_clear_value;
+			break;
+		case 0x40:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[3] = 1.0f;
+			break;
+		case 0x80:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[0] = 1.0f;
+			clear_value.float32[1] = 1.0f;
+			clear_value.float32[2] = 1.0f;
+			break;
+		case 0xc0:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[0] = 1.0f;
+			clear_value.float32[1] = 1.0f;
+			clear_value.float32[2] = 1.0f;
+			clear_value.float32[3] = 1.0f;
+			break;
+		default: return false;
+	}
+
+	for (uint32_t layer = 1; layer < view.layer_count; layer++) {
+		if (!cache.IsMetaCleared(metadata_address, view.base_layer + layer)) {
+			return false;
+		}
+	}
+	// Consume only after every layer in this Vulkan view can be materialized together.
+	for (uint32_t layer = 0; layer < view.layer_count; layer++) {
+		if (!cache.TouchMeta(metadata_address, view.base_layer + layer, false)) {
+			EXIT("failed to consume DCC clear state\n");
+		}
+	}
+	return true;
+}
+
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
                                                  uint32_t color_count, RenderDepthInfo& depth) {
 	EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
@@ -565,7 +580,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
-		const auto old_image = cache.ResolveOwner(target.image_id);
+		const auto old_image = cache.m_slot_images.try_get(target.image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
@@ -576,6 +591,15 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		}
 		target.image_view = cache.FindRenderTarget(target.image_id, target.desc);
 		auto& image       = cache.GetImage(target.image_id);
+		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
+		                     "Kyty.MRT{}.Image[guest=0x{:016x} size=0x{:x} format={}]",
+		                     target.target_slot, image.info.data.address, image.info.data.size,
+		                     static_cast<uint32_t>(image.info.pixel_format));
+		SetVulkanObjectNameF(m_context.GetGraphics().device, target.image_view,
+		                     "Kyty.MRT{}.View[guest=0x{:016x} mip={} layer={}+{}]",
+		                     target.target_slot, image.info.data.address,
+		                     target.desc.view_info.base_level, target.desc.view_info.base_layer,
+		                     target.desc.view_info.layer_count);
 		EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr);
 		if (attachment_samples == 0) {
 			attachment_samples = target.samples;
@@ -599,10 +623,16 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.image_view   = target.image_view;
 		attachment.image_layout = layout;
 		attachment.clear_value  = target.color_clear_value.uint32;
-		attachment.is_clear     = target.color_clear_enable;
+		vk::ClearColorValue metadata_clear_value {};
+		const bool          metadata_clear =
+		    ResolveDccAttachmentClear(cache, target, view, metadata_clear_value);
+		if (metadata_clear) {
+			attachment.clear_value = metadata_clear_value.uint32;
+		}
+		attachment.is_clear = target.color_clear_enable || metadata_clear;
 	}
 	if (depth.image_id) {
-		const auto owner = cache.ResolveOwner(depth.image_id);
+		const auto owner = cache.m_slot_images.try_get(depth.image_id);
 		if (owner == nullptr || !owner->registered || owner->binding.needs_rebind) {
 			EXIT("depth target changed after render-state discovery\n");
 		}
@@ -619,6 +649,14 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			EXIT("failed to consume HTile clear state\n");
 		}
 		auto& image = cache.GetImage(depth.image_id);
+		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
+		                     "Kyty.DepthTarget.Image[guest=0x{:016x} size=0x{:x} format={}]",
+		                     image.info.data.address, image.info.data.size,
+		                     static_cast<uint32_t>(image.info.pixel_format));
+		SetVulkanObjectNameF(m_context.GetGraphics().device, depth.image_view,
+		                     "Kyty.DepthTarget.View[guest=0x{:016x} layer={}+{}]",
+		                     image.info.data.address, depth.desc.view_info.base_layer,
+		                     depth.desc.view_info.layer_count);
 		EXIT_IF(depth.image_view == nullptr || image.backing.samples != depth.samples);
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
@@ -668,11 +706,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	return state;
 }
 
-static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer) {
+static bool DrawHasActivePixelShader(const CommandBuffer& buffer) {
 	const auto& ctx              = buffer.GetRegisters();
 	const auto& sh_regs          = ctx.GetShaderRegisters();
 	const bool  has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
-	return has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs);
+	return ShaderAddressValid(buffer.GetShaders().GetPs().ps_regs.data_addr) &&
+	       (has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
 }
 
 enum class CbColorMode : uint8_t {
@@ -684,13 +723,14 @@ enum class CbColorMode : uint8_t {
 	DccDecompress      = 6,
 };
 
-static bool ConsumeMetadataColorOperation(const RenderCommandBuffer& buffer) {
+static bool ConsumeMetadataColorOperation(const CommandBuffer& buffer) {
 	const auto& ctx  = buffer.GetRegisters();
 	const auto  mode = ctx.GetColorControl().mode;
-	// These AGC CB modes run color-buffer metadata/decompression operations. The shader is a
-	// dummy vehicle for the CB, and its exported color must not be applied as a normal draw.
-	// Kyty currently stores host images as expanded Vulkan images and does not track CMASK/DCC
-	// metadata state, so the matching host operation is a no-op.
+	// These special modes run color-buffer metadata or decompression operations. The shader is a
+	// vehicle for that operation, and its exported color must not be applied as a normal draw.
+	// Kyty stores expanded Vulkan images rather than compressed guest surfaces, so no equivalent
+	// hardware pass is emitted. Tracked DCC clear state is materialized on attachment bind;
+	// future CMask/FMask support can consume its state through the same TextureCache path.
 	return mode == static_cast<uint8_t>(CbColorMode::EliminateFastClear) ||
 	       mode == static_cast<uint8_t>(CbColorMode::FmaskDecompress) ||
 	       mode == static_cast<uint8_t>(CbColorMode::DccDecompress);
@@ -711,11 +751,10 @@ struct DrawIndexBufferSource {
 };
 
 struct PreparedIndexBuffer {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	uint64_t              size   = 0;
-	vk::DeviceSize        offset = 0;
-	vk::IndexType         type   = vk::IndexType::eUint16;
+	vk::Buffer     buffer = nullptr;
+	uint64_t       size   = 0;
+	vk::DeviceSize offset = 0;
+	vk::IndexType  type   = vk::IndexType::eUint16;
 };
 
 static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) {
@@ -724,10 +763,10 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
 }
 
 struct VertexBufferRange {
-	uint64_t      base_address  = 0;
-	uint64_t      requested_end = 0;
-	uint64_t      acquired_end  = 0;
-	BufferBinding binding;
+	uint64_t                     base_address  = 0;
+	uint64_t                     requested_end = 0;
+	uint64_t                     acquired_end  = 0;
+	std::pair<Buffer*, uint64_t> binding;
 
 	[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
 };
@@ -735,14 +774,12 @@ struct VertexBufferRange {
 struct PreparedVertexBuffers {
 	static constexpr uint32_t MaxBuffers = ShaderVertexInputInfo::RES_MAX;
 
-	std::array<vk::Buffer, MaxBuffers>            buffers {};
-	std::array<vk::DeviceSize, MaxBuffers>        offsets {};
-	std::array<std::shared_ptr<void>, MaxBuffers> owners {};
-	uint32_t                                      count       = 0;
-	uint32_t                                      owner_count = 0;
+	std::array<vk::Buffer, MaxBuffers>     buffers {};
+	std::array<vk::DeviceSize, MaxBuffers> offsets {};
+	uint32_t                               count = 0;
 };
 
-static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         buffer,
+static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer&               buffer,
                                                   const ShaderVertexInputInfo& vs_input_info) {
 	EXIT_IF(vs_input_info.buffers_num < 0 ||
 	        vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX);
@@ -789,21 +826,22 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         b
 		const auto size =
 		    Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
 		range.acquired_end = range.base_address + size;
-		range.binding      = cache.ObtainBuffer(buffer, range.base_address, size);
+		range.binding      = cache.ObtainBuffer(range.base_address, size, false);
+		SetVulkanObjectNameF(
+		    buffer.GetContext().GetGraphics().device, range.binding.first->Handle(),
+		    "Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", range.base_address, size);
 	}
 
 	// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
 	PreparedVertexBuffers prepared;
-	prepared.count = static_cast<uint32_t>(vs_input_info.buffers_num);
-	std::shared_ptr<Buffer> null_owner;
-	vk::Buffer              null_buffer = nullptr;
+	prepared.count         = static_cast<uint32_t>(vs_input_info.buffers_num);
+	vk::Buffer null_buffer = nullptr;
 	for (int i = 0; i < vs_input_info.buffers_num; i++) {
 		const auto& vertex = vs_input_info.buffers[i];
 		const auto  size   = VertexBufferDescriptorSize(vertex);
 		if (size == 0) {
-			if (null_owner == nullptr) {
-				null_owner  = cache.ObtainNullBuffer();
-				null_buffer = null_owner->Handle();
+			if (null_buffer == nullptr) {
+				null_buffer = cache.GetBuffer(NULL_BUFFER_ID).Handle();
 			}
 			prepared.buffers[i] = null_buffer;
 			prepared.offsets[i] = 0;
@@ -820,24 +858,19 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         b
 			     vertex.addr);
 		}
 
-		prepared.buffers[i] = range->binding.buffer;
-		prepared.offsets[i] = range->binding.offset + vertex.addr - range->base_address;
+		prepared.buffers[i] = range->binding.first->Handle();
+		prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address;
+		SetVulkanObjectNameF(
+		    buffer.GetContext().GetGraphics().device, prepared.buffers[i],
+		    "Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i,
+		    vertex.addr, size, vertex.stride, vertex.num_records);
 	}
 
-	if (null_owner != nullptr) {
-		prepared.owners[prepared.owner_count++] = std::move(null_owner);
-	}
-	for (uint32_t i = 0; i < merged_count; i++) {
-		if (merged_ranges[i].binding.owner != nullptr) {
-			EXIT_IF(prepared.owner_count >= PreparedVertexBuffers::MaxBuffers);
-			prepared.owners[prepared.owner_count++] = std::move(merged_ranges[i].binding.owner);
-		}
-	}
 	return prepared;
 }
 
-static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
-                              const DrawCallInfo& draw, uint32_t phase) {
+static void SetDrawDebugPhase(CommandBuffer& buffer, uint64_t submit_id, const DrawCallInfo& draw,
+                              uint32_t phase) {
 	EXIT_IF(draw.name == nullptr);
 
 	buffer.SetDebugInfo(static_cast<uint32_t>(draw.debug_op), submit_id, phase, draw.index_count,
@@ -885,8 +918,8 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
 	return true;
 }
 
-static bool ResolvePrimitiveRestart(const RenderCommandBuffer& buffer,
-                                    vk::PrimitiveTopology topology, uint32_t index_type_and_size) {
+static bool ResolvePrimitiveRestart(const CommandBuffer& buffer, vk::PrimitiveTopology topology,
+                                    uint32_t index_type_and_size) {
 	const auto control = buffer.GetUserConfig().GetPrimitiveResetControl();
 	EXIT_NOT_IMPLEMENTED((control & ~0x3u) != 0);
 	if ((control & 0x1u) == 0) {
@@ -920,7 +953,7 @@ static bool ResolvePrimitiveRestart(const RenderCommandBuffer& buffer,
 	return true;
 }
 
-bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
+bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& buffer,
                                             const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
                                             bool log_setup_phases, DrawRenderState& state) {
@@ -948,7 +981,7 @@ bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuf
 	}
 	ResolveRenderDepthTarget(submit_id, buffer, state.depth_info);
 
-	state.ps_active = DrawHasActivePixelShader(buffer);
+	state.ps_active       = DrawHasActivePixelShader(buffer);
 	const bool with_depth = (state.depth_info.format != vk::Format::eUndefined &&
 	                         static_cast<bool>(state.depth_info.image_id));
 	if (state.color_count == 0 && !with_depth && !state.ps_active) {
@@ -960,7 +993,7 @@ bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuf
 	return true;
 }
 
-static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
+static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
                            DrawRenderState& state) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ctx    = buffer.GetRegisters();
@@ -970,37 +1003,33 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
 	const auto& pixel_shader_info  = sh_ctx.GetPs();
 	const auto& shader_regs        = ctx.GetShaderRegisters();
 
-	state.vs_shader     = {};
-	state.ps_shader     = {};
-	state.ps_input_info = {};
+	state.vertex_program = {};
+	state.pixel_program  = {};
+	state.ps_input_info  = {};
 	std::array<Prospero::ColorComponentMapping, RENDER_COLOR_ATTACHMENTS_MAX>
 	    target_export_mapping {};
 	for (uint32_t i = 0; i < state.color_count; i++) {
 		target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
 	}
-	const auto lane_mask_mode = SelectGraphicsLaneMaskMode(64u);
-
 	if (log_phases) {
-		LogDrawPhase(draw.name, "ShaderCompileInfoVS");
-	}
-	if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
-	                         state.vs_shader)) {
-		EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name);
+		LogDrawPhase(draw.name, "GetVertexProgram");
 	}
+	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
+	state.vertex_program =
+	    pipeline_cache.GetVertexProgram(vertex_shader_info, shader_regs, state.vs_input_info);
 
 	if (!state.ps_active) {
 		return;
 	}
 	if (log_phases) {
-		LogDrawPhase(draw.name, "ShaderCompileInfoPS");
-	}
-	if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
-	                         target_export_mapping, state.ps_input_info, state.ps_shader)) {
-		EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name);
+		LogDrawPhase(draw.name, "GetPixelProgram");
 	}
+	state.pixel_program =
+	    pipeline_cache.GetPixelProgram(pixel_shader_info, shader_regs, state.vs_input_info,
+	                                   target_export_mapping, state.ps_input_info);
 }
 
-static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, RenderCommandBuffer& buffer,
+static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, CommandBuffer& buffer,
                                                   const DrawCallInfo&          draw,
                                                   const ShaderVertexInputInfo& vs_input_info) {
 	EXIT_IF(draw.name == nullptr);
@@ -1010,7 +1039,7 @@ static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, RenderComm
 	return AcquireVertexBuffers(buffer, vs_input_info);
 }
 
-static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffer,
+static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffer,
                                               const DrawIndexBufferSource& source) {
 	PreparedIndexBuffer prepared;
 	if (!source.enabled) {
@@ -1020,28 +1049,29 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffe
 	prepared.size = source.size;
 	prepared.type = source.type;
 	if (source.host_data != nullptr) {
-		auto binding =
-		    buffer.GetContext().GetBufferCache().UploadTransient(source.host_data, source.size, 16);
-		prepared.owner  = std::move(binding.owner);
-		prepared.buffer = binding.buffer;
-		prepared.offset = binding.offset;
+		auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+		prepared.offset = stream.Copy(source.host_data, source.size, 16);
+		prepared.buffer = stream.Handle();
 	} else {
-		auto binding =
-		    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, source.address, source.size);
-		prepared.owner  = std::move(binding.owner);
-		prepared.buffer = binding.buffer;
-		prepared.offset = binding.offset;
+		auto [buffer_ptr, offset] =
+		    buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
+		prepared.buffer = buffer_ptr->Handle();
+		prepared.offset = offset;
+	}
+	if (source.host_data != nullptr) {
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
+		                     static_cast<uint32_t>(source.type));
+	} else {
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
+		                     source.address, source.size, static_cast<uint32_t>(source.type));
 	}
 	return prepared;
 }
 
-static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                                PreparedVertexBuffers& prepared) {
-	for (uint32_t i = 0; i < prepared.owner_count; i++) {
-		if (prepared.owners[i] != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(prepared.owners[i]));
-		}
-	}
+static void CommitVertexBuffers(vk::CommandBuffer            vk_buffer,
+                                const PreparedVertexBuffers& prepared) {
 	for (uint32_t i = 0; i < prepared.count; i++) {
 		EXIT_IF(prepared.buffers[i] == nullptr);
 	}
@@ -1051,19 +1081,15 @@ static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer v
 	}
 }
 
-static void CommitIndexBuffer(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                              PreparedIndexBuffer& prepared) {
+static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBuffer& prepared) {
 	if (prepared.size == 0) {
 		return;
 	}
-	if (prepared.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(prepared.owner));
-	}
 	EXIT_IF(prepared.buffer == nullptr);
 	vk_buffer.bindIndexBuffer(prepared.buffer, prepared.offset, prepared.type);
 }
 
-static void LogDrawStateIfNeeded(const RenderCommandBuffer& buffer, const DrawCallInfo& draw,
+static void LogDrawStateIfNeeded(const CommandBuffer& buffer, const DrawCallInfo& draw,
                                  const DrawRenderState& state, bool always_log,
                                  bool force_legacy_rect_log, uint32_t index_type_and_size,
                                  const void* index_addr) {
@@ -1131,7 +1157,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 	}
 }
 
-void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer,
                                          const DrawCallInfo& draw, DrawRenderState& state,
                                          vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
                                          const DrawIndexBufferSource& index_source,
@@ -1141,8 +1167,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	auto& ucfg = buffer.GetUserConfig();
 
 	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings        = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
-	                                               state.ps_input_info.stage, state.ps_active);
+	auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage,
+	                                        state.ps_active);
 	auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info);
 	auto index_binding   = PrepareIndexBuffer(buffer, index_source);
 	state.rendering =
@@ -1152,9 +1178,9 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 		LogDrawPhase(draw.name, "CreatePipeline");
 	}
 	auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline(
-	    state.color_info, state.color_count, state.depth_info, state.vs_input_info, buffer,
-	    &state.ps_input_info, topology, primitive_restart_enable, state.ps_active, state.vs_shader,
-	    state.ps_shader);
+	    std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer,
+	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
+	    state.vertex_program, state.pixel_program);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest
@@ -1166,21 +1192,23 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	if (set_auto_debug) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
 	}
-	CommitVertexBuffers(buffer, vk_buffer, vertex_bindings);
-	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout,
-	               bindings.vertex);
+	CommitVertexBuffers(vk_buffer, vertex_bindings);
 	if (bindings.pixel.has_value()) {
 		if (set_auto_debug) {
 			SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 		}
-		CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout,
-		               *bindings.pixel);
 	}
-	CommitIndexBuffer(buffer, vk_buffer, index_binding);
+	std::array<PreparedBindings*, 2> descriptor_stages {&bindings.vertex, nullptr};
+	const size_t                     descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u;
+	if (bindings.pixel) {
+		descriptor_stages[1] = &*bindings.pixel;
+	}
+	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
+	               std::span {descriptor_stages.data(), descriptor_stage_count});
+	CommitIndexBuffer(vk_buffer, index_binding);
 
-	const auto dynamic_params =
-	    BuildGraphicsDynamicParams(buffer, state.color_info, state.color_count, state.depth_info);
-	SetDynamicParams(buffer, vk_buffer, dynamic_params);
+	SetGraphicsDynamicParams(buffer, vk_buffer, state.color_info, state.color_count,
+	                         state.depth_info);
 
 	LogDrawPhase(draw.name, "BeginRendering");
 	if (set_auto_debug) {
@@ -1213,7 +1241,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	}
 }
 
-void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
                                uint32_t index_type_and_size, uint32_t index_count,
                                const void* index_addr, uint32_t flags, uint32_t type,
                                uint32_t instance_count, uint32_t render_target_slice_offset,
@@ -1221,6 +1249,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ucfg   = buffer.GetUserConfig();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -1345,13 +1374,14 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
+void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, uint32_t index_count,
                               uint32_t flags, uint32_t render_target_slice_offset,
                               uint32_t instance_count, uint32_t first_vertex,
                               uint32_t first_instance) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ucfg   = buffer.GetUserConfig();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -1443,7 +1473,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 	ResetBindings();
 }
 
-bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
+bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, CommandBuffer& buffer,
                                          uint32_t render_target_slice_offset) {
 	const auto& hw = buffer.GetRegisters();
 	if (hw.GetColorControl().mode != 3) {

src/graphics/host_gpu/renderer/renderDraw.h

@@ -1,43 +1,12 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 
-#include "graphics/host_gpu/renderer/renderTarget.h"
-
 #include <cstdint>
-#include <type_traits>
 
 namespace Libs::Graphics {
 
-struct RenderColorInfo;
 struct ShaderVertexInputInfo;
 
-#pragma pack(push, 1)
-
-struct PipelineDynamicParameters {
-	bool stencil_test_enable = false;
-	bool depth_bias_enable   = false;
-
-	float viewport_scale[3]  = {};
-	float viewport_offset[3] = {};
-	int   scissor_ltrb[4]    = {0};
-
-	float    line_width                                       = 1.0f;
-	float    depth_bias_constant_factor                       = 0.0f;
-	float    depth_bias_clamp                                 = 0.0f;
-	float    depth_bias_slope_factor                          = 0.0f;
-	uint32_t color_write_count                                = 1;
-	bool     color_write_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {true, true};
-
-	PipelineStencilDynamicState stencil_front;
-	PipelineStencilDynamicState stencil_back;
-};
-
-#pragma pack(pop)
-
-static_assert(std::is_trivially_copyable_v<PipelineDynamicParameters>);
-static_assert(std::is_standard_layout_v<PipelineDynamicParameters>);
-static_assert(alignof(PipelineDynamicParameters) == 1);
-
 [[nodiscard]] int32_t ResolveVertexOffset(uint32_t                     index_offset,
                                           const ShaderVertexInputInfo& vs_input_info);
 

src/graphics/host_gpu/renderer/sync.cpp

@@ -20,9 +20,7 @@
 
 namespace Libs::Graphics::Sync {
 
-constexpr int      GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
-constexpr int      GRAPHICS_EVENT_EOP                       = 0x40;
-constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY       = 100000000;
+constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
 
 bool ScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t& value) {
 	if (host_frequency == 0) {
@@ -66,8 +64,9 @@ struct EndOfPipeSignal {
 	std::array<uint32_t, 4> debug_args      = {};
 	uint64_t                debug_data      = 0;
 	std::optional<uint64_t> destination;
-	EndOfPipeCompletion     completion      = EndOfPipeCompletion::None;
-	uint64_t                completion_data = 0;
+	EndOfPipeCompletion     completion         = EndOfPipeCompletion::None;
+	uint64_t                completion_data    = 0;
+	int                     interrupt_event_id = 0;
 };
 
 enum class EndOfPipeWriteSize : uint32_t { Dword = 4, Qword = 8 };
@@ -96,8 +95,10 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) {
 		case EndOfPipeCompletion::None: return;
 		case EndOfPipeCompletion::Interrupt: {
 			const auto context_id = static_cast<uint32_t>(signal.completion_data);
-			scheduler.DeferPriorityOperation(
-			    [&renderer, context_id] { renderer.TriggerEopEvent(context_id); });
+			const auto event_id   = signal.interrupt_event_id;
+			scheduler.DeferPriorityOperation([&renderer, event_id, context_id] {
+				renderer.TriggerInterrupt(event_id, context_id);
+			});
 			return;
 		}
 		case EndOfPipeCompletion::Flip: {
@@ -108,9 +109,10 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) {
 		}
 		case EndOfPipeCompletion::FlipAndInterrupt: {
 			const auto request_id = signal.completion_data;
-			scheduler.DeferPriorityOperation([&renderer, request_id] {
+			const auto event_id   = signal.interrupt_event_id;
+			scheduler.DeferPriorityOperation([&renderer, event_id, request_id] {
 				renderer.GetVideoOut().CompleteFlip(request_id);
-				renderer.TriggerEopEvent(0);
+				renderer.TriggerInterrupt(event_id, 0);
 			});
 			return;
 		}
@@ -135,7 +137,8 @@ static bool TriggersInterrupt(EndOfPipeWriteAction action) {
 
 static void RecordEndOfPipeWrite(uint64_t submit_id, CommandBuffer& buffer, uint64_t destination,
                                  uint64_t value, EndOfPipeWriteSize size,
-                                 EndOfPipeWriteAction action, uint32_t context_id = 0) {
+                                 EndOfPipeWriteAction action, int interrupt_event_id = 0,
+                                 uint32_t context_id = 0) {
 	const auto width      = static_cast<uint32_t>(size);
 	const auto value_low  = static_cast<uint32_t>(value);
 	const auto value_high = static_cast<uint32_t>(value >> 32u);
@@ -150,18 +153,12 @@ static void RecordEndOfPipeWrite(uint64_t submit_id, CommandBuffer& buffer, uint
 	    .debug_data      = destination,
 	    .destination     = destination,
 	    .completion      = interrupt ? EndOfPipeCompletion::Interrupt : EndOfPipeCompletion::None,
-	    .completion_data = context_id != 0 ? context_id : value,
+	    .completion_data = interrupt ? context_id : value,
+	    .interrupt_event_id = interrupt_event_id,
 	};
 	RecordEndOfPipeSignal(signal);
 }
 
-void TriggerAgcUserInterrupt() {
-	auto tsc    = LibKernel::KernelReadTsc();
-	auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT,
-	                                                                  reinterpret_cast<void*>(tsc));
-	EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT);
-}
-
 void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                         uint32_t value) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
@@ -220,31 +217,35 @@ void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
 }
 
 void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint64_t* dst_gpu_addr, uint64_t value,
+                                              uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                               uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
 	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::InterruptWriteBack,
-	                     context_id);
+	                     event_id, context_id);
 }
 
 void WriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint32_t* dst_gpu_addr, uint32_t value,
+                                              uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                               uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
 	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::InterruptWriteBack,
-	                     context_id);
+	                     event_id, context_id);
 }
 
 void WriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint64_t* dst_gpu_addr, uint64_t value, uint32_t context_id) {
+                                     uint64_t* dst_gpu_addr, uint64_t value, int event_id,
+                                     uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
-	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Interrupt, context_id);
+	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Interrupt, event_id,
+	                     context_id);
 }
 
 void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint32_t* dst_gpu_addr, uint32_t value, uint32_t context_id) {
+                                     uint32_t* dst_gpu_addr, uint32_t value, int event_id,
+                                     uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
-	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, context_id);
+	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, event_id,
+	                     context_id);
 }
 
 uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int flip_mode,
@@ -270,18 +271,20 @@ uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int f
 void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuffer& buffer,
                                                   uint32_t* dst_gpu_addr, uint32_t value,
                                                   int handle, int index, int flip_mode,
-                                                  int64_t flip_arg, uint64_t request_id) {
+                                                  int64_t flip_arg, uint64_t request_id,
+                                                  int event_id) {
 	const auto destination = reinterpret_cast<uint64_t>(dst_gpu_addr);
 	RecordEndOfPipeSignal({
-	    .buffer          = &buffer,
-	    .submit_id       = submit_id,
-	    .debug_operation = CommandBufferDebugOp::EopWriteBackFlip,
-	    .debug_args      = {static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
-	                        static_cast<uint32_t>(flip_mode), value},
-	    .debug_data      = static_cast<uint64_t>(flip_arg),
-	    .destination     = destination,
-	    .completion      = EndOfPipeCompletion::FlipAndInterrupt,
-	    .completion_data = request_id,
+	    .buffer             = &buffer,
+	    .submit_id          = submit_id,
+	    .debug_operation    = CommandBufferDebugOp::EopWriteBackFlip,
+	    .debug_args         = {static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
+	                           static_cast<uint32_t>(flip_mode), value},
+	    .debug_data         = static_cast<uint64_t>(flip_arg),
+	    .destination        = destination,
+	    .completion         = EndOfPipeCompletion::FlipAndInterrupt,
+	    .completion_data    = request_id,
+	    .interrupt_event_id = event_id,
 	});
 }
 
@@ -316,36 +319,24 @@ void WriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer& buffer, int han
 	});
 }
 
-void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id) {
+void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, int event_id, uint32_t context_id) {
 	ValidateEndOfPipeSignal({.buffer = &buffer});
 	auto& renderer  = buffer.GetContext();
 	auto& scheduler = renderer.GetCommandScheduler();
 	EXIT_IF(!scheduler.Active() || &buffer != &scheduler.Current());
 	scheduler.DeferPriorityOperation(
-	    [&renderer, context_id] { renderer.TriggerEopEvent(context_id); });
+	    [&renderer, event_id, context_id] { renderer.TriggerInterrupt(event_id, context_id); });
 }
 
-static void EopEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) {
+static void InterruptEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) {
 	EXIT_IF(event == nullptr);
 	event->triggered    = false;
 	event->event.fflags = 0;
 	event->event.data   = 0;
 }
 
-static void EopEventDeleteFunc(LibKernel::EventQueue::KernelEqueue       eq,
-                               LibKernel::EventQueue::KernelEqueueEvent* event) {
-	EXIT_IF(event == nullptr);
-	EXIT_NOT_IMPLEMENTED(event->event.filter != LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS);
-	if (event->event.ident == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT ||
-	    event->event.ident == GRAPHICS_EVENT_EOP) {
-		auto* renderer = static_cast<RenderContext*>(event->filter.data);
-		EXIT_IF(renderer == nullptr);
-		renderer->DeleteEopEq(eq, static_cast<int>(event->event.ident));
-	}
-}
-
-static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
-                                void*                                     trigger_data) {
+static void InterruptEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
+                                      void*                                     trigger_data) {
 	EXIT_IF(event == nullptr);
 
 	auto triggered_event = event->event;
@@ -362,35 +353,35 @@ static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
 int AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                void* udata) {
 	LibKernel::EventQueue::KernelEqueueEvent event;
-	event.triggered                = false;
-	event.event.ident              = static_cast<uintptr_t>(id);
-	event.event.filter             = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS;
-	event.event.udata              = udata;
-	event.event.fflags             = 0;
-	event.event.data               = id;
-	event.filter.delete_event_func = EopEventDeleteFunc;
-	event.filter.reset_func        = EopEventResetFunc;
-	event.filter.trigger_func      = EopEventTriggerFunc;
-	event.filter.data              = &renderer;
+	event.triggered           = false;
+	event.event.ident         = static_cast<uintptr_t>(id);
+	event.event.filter        = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS;
+	event.event.udata         = udata;
+	event.event.fflags        = 0;
+	event.event.data          = id;
+	event.filter.reset_func   = InterruptEventResetFunc;
+	event.filter.trigger_func = InterruptEventTriggerFunc;
 
 	int result = LibKernel::EventQueue::KernelAddEvent(eq, event);
 
-	if (result == 0 &&
-	    (id == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || id == GRAPHICS_EVENT_EOP)) {
-		renderer.AddEopEq(eq, id);
+	if (result == 0) {
+		renderer.AddInterruptEq(eq, id);
 	}
 
 	return result;
 }
 
-int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) {
+int DeleteEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id) {
 	int result = LibKernel::EventQueue::KernelDeleteEvent(
 	    eq, static_cast<uintptr_t>(id), LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS);
+	if (result == OK || result == LibKernel::KERNEL_ERROR_ENOENT) {
+		renderer.DeleteInterruptEq(eq, id);
+	}
 
 	return result;
 }
 
-void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
+void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
 	const auto offset = uint64_t {dw_offset} * sizeof(uint32_t);
 	const auto size   = uint64_t {dw_size} * sizeof(uint32_t);
 	EXIT_IF(dst == nullptr || offset > gds.Size() || size > gds.Size() - offset ||

src/graphics/host_gpu/renderer/sync.h

@@ -17,8 +17,7 @@ namespace Sync {
                                            uint64_t& value);
 [[nodiscard]] uint64_t ReadReferenceClock();
 
-void TriggerAgcUserInterrupt();
-void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id);
+void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, int event_id, uint32_t context_id);
 
 void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                         uint32_t value);
@@ -35,16 +34,16 @@ void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
 void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
                                      uint64_t* dst_gpu_addr, uint64_t value);
 void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint32_t* dst_gpu_addr, uint32_t value,
+                                     uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                      uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint64_t* dst_gpu_addr, uint64_t value,
+                                     uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                      uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint32_t* dst_gpu_addr, uint32_t value,
+                                              uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                               uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint64_t* dst_gpu_addr, uint64_t value,
+                                              uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                               uint32_t context_id = 0);
 
 [[nodiscard]] uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index,
@@ -57,12 +56,13 @@ void WriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer& buffer, uint3
 void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuffer& buffer,
                                                   uint32_t* dst_gpu_addr, uint32_t value,
                                                   int handle, int index, int flip_mode,
-                                                  int64_t flip_arg, uint64_t request_id);
+                                                  int64_t flip_arg, uint64_t request_id,
+                                                  int event_id);
 
 int  AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                 void* udata);
-int  DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id);
-void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
+int  DeleteEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id);
+void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
 
 } // namespace Sync
 } // namespace Libs::Graphics

src/graphics/host_gpu/shaders/fault_buffer_process.comp

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#version 450
+#extension GL_ARB_gpu_shader_int64 : enable
+
+layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
+
+layout(std430, binding = 0) buffer input_buf {
+    uint fault_buffer[];
+};
+
+layout(std430, binding = 1) buffer output_buf {
+    uint64_t download_buffer[];
+};
+layout(std430, binding = 1) buffer output_buf32 {
+    uint download_buffer32[];
+};
+
+void main() {
+    const uint id = gl_GlobalInvocationID.x;
+    uint word = fault_buffer[id];
+    fault_buffer[id] = 0u;
+    const uint base_bit = id * 32u;
+    while (word != 0u) {
+        const uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u;
+        if (store_index >= MAX_PAGE_FAULTS) {
+            return;
+        }
+        const uint bit = findLSB(word);
+        word &= word - 1;
+        const uint page = base_bit + bit;
+        download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS;
+    }
+}

src/graphics/host_gpu/vma.cpp

@@ -72,7 +72,10 @@ bool GraphicContext::CreateAllocator() {
 	info.device           = device;
 	info.pVulkanFunctions = &functions;
 	info.vulkanApiVersion = VULKAN_TARGET_API_VERSION;
-	info.flags = memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0;
+	info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
+	if (memory_budget_ext_enabled) {
+		info.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
+	}
 
 	const auto result = static_cast<vk::Result>(vmaCreateAllocator(&info, &allocator));
 	if (result != vk::Result::eSuccess) {
@@ -199,20 +202,6 @@ void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) {
 	VulkanTrackAllocation(buffer.memory);
 }
 
-void GraphicContext::DeleteBuffer(VulkanBuffer& buffer) {
-	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(allocator == nullptr || buffer.buffer == nullptr ||
-	        buffer.memory.allocation == nullptr);
-
-	VulkanUntrackAllocation(buffer.memory);
-	vmaDestroyBuffer(allocator, buffer.buffer, buffer.memory.allocation);
-	buffer.buffer                 = nullptr;
-	buffer.memory.memory          = nullptr;
-	buffer.memory.allocation      = nullptr;
-	buffer.memory.allocation_info = {};
-	buffer.memory.offset          = 0;
-}
-
 bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanImage& image) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(allocator == nullptr || image.image != nullptr || image.memory.allocation != nullptr);

src/graphics/host_gpu/vulkanCommon.h

@@ -10,10 +10,13 @@
 #define VMA_STATIC_VULKAN_FUNCTIONS  0
 #define VMA_DYNAMIC_VULKAN_FUNCTIONS 1
 
+#include "common/emulatorConfig.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 
 #include <cstdint>
+#include <fmt/format.h>
 #include <string>
+#include <utility>
 #include <vector>
 #include <vulkan/vulkan.hpp>
 #include <vulkan/vulkan_hash.hpp>
@@ -29,6 +32,24 @@ std::string VulkanToString(vk::QueueFlags value);
 vk::Format  VulkanFormat(Prospero::BufferFormat guest_format);
 void        RequireVulkanSuccess(vk::Result result, const char* operation);
 
+template <typename Handle, typename... Args>
+void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<Args...> format,
+                          Args&&... args) {
+	if (!Config::GraphicsDebugDumpEnabled() || device == nullptr || handle == nullptr ||
+	    VULKAN_HPP_DEFAULT_DISPATCHER.vkSetDebugUtilsObjectNameEXT == nullptr) {
+		return;
+	}
+
+	const auto                      name = fmt::format(format, std::forward<Args>(args)...);
+	vk::DebugUtilsObjectNameInfoEXT info {};
+	info.sType        = vk::StructureType::eDebugUtilsObjectNameInfoEXT;
+	info.objectType   = Handle::objectType;
+	info.objectHandle = static_cast<uint64_t>(
+	    reinterpret_cast<uintptr_t>(static_cast<typename Handle::CType>(handle)));
+	info.pObjectName = name.c_str();
+	(void)device.setDebugUtilsObjectNameEXT(&info);
+}
+
 template <typename T, typename Enumerator>
 [[nodiscard]] std::vector<T> EnumerateVulkan(const char* operation, Enumerator&& enumerate) {
 	for (;;) {

src/graphics/host_gpu/vulkanInstance.h

@@ -1,93 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_
-
-#include "common/abi.h"
-#include "common/common.h"
-#include "common/threads.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-
-#include <map>
-#include <mutex>
-#include <tuple>
-#include <vk_mem_alloc.h>
-
-namespace Libs::Graphics {
-
-inline constexpr uint32_t VULKAN_TARGET_API_VERSION = VK_API_VERSION_1_3;
-
-struct VulkanInstance {
-	vk::Instance                       instance                          = nullptr;
-	vk::DebugUtilsMessengerEXT         debug_messenger                   = nullptr;
-	vk::PhysicalDevice                 physical_device                   = nullptr;
-	vk::PhysicalDeviceProperties       physical_device_properties        = {};
-	vk::PhysicalDeviceMemoryProperties physical_device_memory_properties = {};
-	vk::Device                         device                            = nullptr;
-	VmaAllocator                       allocator                         = nullptr;
-	bool                               memory_budget_ext_enabled         = false;
-	bool                               rt_extensions_enabled             = false;
-	bool                               subgroup_size_control_enabled     = false;
-	bool                               sample_rate_shading_enabled       = false;
-	uint32_t                           subgroup_size                     = 0;
-	uint32_t                           min_subgroup_size                 = 0;
-	uint32_t                           max_subgroup_size                 = 0;
-	vk::ShaderStageFlags               required_subgroup_size_stages     = {};
-	Common::Mutex                      queue_mutex;
-	uint32_t                           queue_family = static_cast<uint32_t>(-1);
-	vk::Queue                          queue        = nullptr;
-
-	[[nodiscard]] const vk::PhysicalDeviceProperties& GetPhysicalDeviceProperties() const {
-		return physical_device_properties;
-	}
-
-	[[nodiscard]] const vk::PhysicalDeviceMemoryProperties&
-	GetPhysicalDeviceMemoryProperties() const {
-		return physical_device_memory_properties;
-	}
-
-	[[nodiscard]] vk::FormatProperties GetFormatProperties(vk::Format format) const {
-		std::scoped_lock lock(m_format_properties_mutex);
-		auto [it, inserted] = m_format_properties.try_emplace(format);
-		if (inserted) {
-			physical_device.getFormatProperties(format, &it->second);
-		}
-		return it->second;
-	}
-
-	[[nodiscard]] vk::Result GetImageFormatProperties(vk::Format format, vk::ImageType type,
-	                                                  vk::ImageTiling            tiling,
-	                                                  vk::ImageUsageFlags        usage,
-	                                                  vk::ImageCreateFlags       flags,
-	                                                  vk::ImageFormatProperties* properties) const {
-		using Key = std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
-		                       vk::ImageCreateFlags>;
-		std::scoped_lock lock(m_image_format_properties_mutex);
-		auto [it, inserted] =
-		    m_image_format_properties.try_emplace(Key {format, type, tiling, usage, flags});
-		if (inserted) {
-			it->second.first = physical_device.getImageFormatProperties(format, type, tiling, usage,
-			                                                            flags, &it->second.second);
-		}
-		if (properties != nullptr) {
-			*properties = it->second.second;
-		}
-		return it->second.first;
-	}
-
-	[[nodiscard]] vk::DeviceSize StorageMinAlignment() const {
-		const auto alignment = physical_device_properties.limits.minStorageBufferOffsetAlignment;
-		return alignment != 0 ? alignment : 1;
-	}
-
-private:
-	mutable std::mutex                                 m_format_properties_mutex;
-	mutable std::map<vk::Format, vk::FormatProperties> m_format_properties;
-	mutable std::mutex                                 m_image_format_properties_mutex;
-	mutable std::map<std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
-	                            vk::ImageCreateFlags>,
-	                 std::pair<vk::Result, vk::ImageFormatProperties>>
-	    m_image_format_properties;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_

src/graphics/presentation/renderDoc.cpp

@@ -25,11 +25,13 @@ namespace Libs::Graphics {
 enum class RenderDocState : uint32_t {
 	Idle,
 	Requested,
+	Starting,
 	Capturing,
 };
 
 static RENDERDOC_API_1_6_0*        g_api             = nullptr;
 static std::atomic<RenderDocState> g_state           = RenderDocState::Idle;
+static std::atomic_uint32_t        g_captured_flips  = 0;
 static std::atomic_bool            g_unavailable_log = false;
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -140,7 +142,7 @@ bool RenderDocCaptureInProgress() {
 
 void RenderDocStartCapture() {
 	RenderDocState expected = RenderDocState::Requested;
-	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Capturing,
+	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Starting,
 	                                                         std::memory_order_acq_rel)) {
 		return;
 	}
@@ -162,6 +164,8 @@ void RenderDocStartCapture() {
 		LOGF("RenderDoc: capture failed to start\n");
 		return;
 	}
+	g_captured_flips.store(0, std::memory_order_release);
+	g_state.store(RenderDocState::Capturing, std::memory_order_release);
 	LOGF("RenderDoc: capture started\n");
 }
 
@@ -175,4 +179,15 @@ void RenderDocEndCapture() {
 	LOGF(ok != 0 ? "RenderDoc: capture finished\n" : "RenderDoc: capture failed\n");
 }
 
+void RenderDocOnGuestFlip() {
+	if (!RenderDocCaptureInProgress()) {
+		return;
+	}
+	const auto flip = g_captured_flips.fetch_add(1, std::memory_order_acq_rel) + 1;
+	LOGF("RenderDoc: captured guest flip %u/2\n", flip);
+	if (flip >= 2) {
+		RenderDocEndCapture();
+	}
+}
+
 } // namespace Libs::Graphics

src/graphics/presentation/renderDoc.h

@@ -7,6 +7,7 @@ void RenderDocInit();
 void RenderDocRequestCapture();
 void RenderDocStartCapture();
 void RenderDocEndCapture();
+void RenderDocOnGuestFlip();
 
 [[nodiscard]] bool RenderDocCaptureRequested();
 [[nodiscard]] bool RenderDocCaptureInProgress();

src/graphics/presentation/videoOut.cpp

@@ -16,6 +16,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/presentation/presenter.h"
+#include "graphics/presentation/renderDoc.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "libs/libs.h"
@@ -42,6 +43,9 @@ constexpr int      VIDEO_OUT_EVENT_PRE_VBLANK_START                     = 2;
 constexpr int      VIDEO_OUT_EVENT_SET_MODE                             = 8;
 constexpr int      VIDEO_OUT_TRUE                                       = 1;
 constexpr int      VIDEO_OUT_FALSE                                      = 0;
+constexpr int      VIDEO_OUT_BUS_TYPE_MAIN                              = 0;
+constexpr int      VIDEO_OUT_BUS_TYPE_OVERLAY                           = 1;
+constexpr int      VIDEO_OUT_BUS_TYPE_SUB                               = 2;
 constexpr int      VIDEO_OUT_FLIP_MODE_VSYNC                            = 1;
 constexpr int      VIDEO_OUT_FLIP_MODE_VSYNC_MULTI                      = 4;
 constexpr int      VIDEO_OUT_BUFFER_INDEX_BLACK                         = -2;
@@ -233,7 +237,7 @@ private:
 
 struct VideoOutDriver::Impl {
 public:
-	static constexpr int VIDEO_OUT_NUM_MAX = 2;
+	static constexpr int VIDEO_OUT_NUM_MAX = 4;
 
 	Impl(uint32_t width, uint32_t height, Graphics::Presenter& presenter)
 	    : m_renderer(presenter.Renderer()), m_presenter(presenter), m_flip_queue(presenter) {
@@ -244,7 +248,7 @@ public:
 	~Impl();
 	KYTY_CLASS_NO_COPY(Impl);
 
-	int             Open();
+	int             Open(int bus_type);
 	bool            Close(int handle);
 	VideoOutConfig* Get(int handle);
 	VideoOutConfig* Get(int handle, uint64_t& generation);
@@ -606,19 +610,11 @@ void VideoOutDriver::Impl::Init(uint32_t width, uint32_t height) {
 	}
 }
 
-int VideoOutDriver::Impl::Open() {
+int VideoOutDriver::Impl::Open(int bus_type) {
 	Common::LockGuard lock(m_mutex);
 
-	int handle = -1;
-
-	for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) {
-		if (!m_video_out_ctx[i].opened) {
-			handle = i;
-			break;
-		}
-	}
-
-	if (handle < 0) {
+	const int handle = bus_type + 1;
+	if (m_video_out_ctx[handle].opened) {
 		return -1;
 	}
 	auto&             config = m_video_out_ctx[handle];
@@ -1127,6 +1123,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 	m_mutex.Unlock();
 
 	m_presenter.Present(*r.frame);
+	Graphics::RenderDocOnGuestFlip();
 
 	m_mutex.Lock();
 	if (m_requests.empty() || m_requests.front().id != r.id ||
@@ -1171,12 +1168,15 @@ KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void*
 	PRINT_NAME();
 
 	EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0);
-	EXIT_NOT_IMPLEMENTED(bus_type != 0);
+	if (bus_type != VIDEO_OUT_BUS_TYPE_MAIN && bus_type != VIDEO_OUT_BUS_TYPE_OVERLAY &&
+	    bus_type != VIDEO_OUT_BUS_TYPE_SUB) {
+		return VIDEO_OUT_ERROR_INVALID_VALUE;
+	}
 	EXIT_NOT_IMPLEMENTED(index != 0);
 
 	LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast<uint64_t>(param));
 
-	int handle = DriverState().Open();
+	int handle = DriverState().Open(bus_type);
 
 	if (handle < 0) {
 		return VIDEO_OUT_ERROR_RESOURCE_BUSY;

src/graphics/presentation/window/swapchain.cpp

@@ -1,30 +1,9 @@
-#include "SDL.h"
-#include "SDL_error.h"
-#include "SDL_events.h"
-#include "SDL_gamecontroller.h"
-#include "SDL_hints.h"
-#include "SDL_joystick.h"
-#include "SDL_keyboard.h"
-#include "SDL_keycode.h"
-#include "SDL_mouse.h"
-#include "SDL_pixels.h"
-#include "SDL_rwops.h"
-#include "SDL_stdinc.h"
-#include "SDL_surface.h"
-#include "SDL_thread.h"
-#include "SDL_touch.h"
-#include "SDL_video.h"
-#include "SDL_vulkan.h"
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/emulatorConfig.h"
-#include "common/file.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
-#include "common/stringUtils.h"
-#include "common/systemInfo.h"
 #include "common/threads.h"
-#include "common/timer.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
@@ -34,19 +13,11 @@
 #include "graphics/presentation/presenter.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window/windowInternal.h"
-#include "libs/controller.h"
-#include "loader/systemContent.h"
 
 #include <algorithm>
-#include <array>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
 #include <deque>
-#include <fmt/format.h>
 #include <limits>
 #include <memory>
-#include <string>
 #include <vector>
 #include <vulkan/vk_platform.h>
 
@@ -58,10 +29,10 @@
 namespace Libs::Graphics {
 
 struct Presenter::Frame {
-	VulkanImage                    image;
-	std::unique_ptr<CommandBuffer> present_commands;
-	bool                           busy         = false;
-	bool                           reusing_last = false;
+	VulkanImage image;
+	uint64_t    present_tick = 0;
+	bool        busy         = false;
+	bool        reusing_last = false;
 
 	void Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format);
 	void Transit(vk::CommandBuffer command, vk::ImageLayout layout, vk::AccessFlags2 access);
@@ -71,12 +42,11 @@ struct Presenter::Frame {
 
 class FramePool final {
 public:
-	explicit FramePool(WindowContext& window): m_window(window) {}
+	FramePool(WindowContext& window, CommandScheduler& scheduler)
+	    : m_window(window), m_scheduler(scheduler) {}
 	~FramePool() {
+		m_scheduler.Wait(m_scheduler.CurrentTick() - 1);
 		for (auto& frame: m_frames) {
-			if (frame->present_commands != nullptr) {
-				frame->present_commands->WaitForFenceOnly();
-			}
 			if (frame->image.image != nullptr) {
 				m_window.graphic_ctx.DeleteImage(frame->image);
 			}
@@ -188,15 +158,10 @@ public:
 	}
 
 private:
-	static void WaitForFrame(Presenter::Frame& frame) {
-		// The producer only waits here. Reset stays on the presentation thread that owns the
-		// allocating Vulkan command pool.
-		if (frame.present_commands != nullptr) {
-			frame.present_commands->WaitForFenceOnly();
-		}
-	}
+	void WaitForFrame(Presenter::Frame& frame) { m_scheduler.Wait(frame.present_tick); }
 
 	WindowContext&                                 m_window;
+	CommandScheduler&                              m_scheduler;
 	Common::Mutex                                  m_mutex;
 	Common::CondVar                                m_available;
 	std::vector<std::unique_ptr<Presenter::Frame>> m_frames;
@@ -330,7 +295,7 @@ public:
 	[[nodiscard]] Status AcquireNextImage();
 	[[nodiscard]] bool   PrepareImeOverlay();
 	void RecordPresentCommands(CommandBuffer& command, VulkanImage& source, bool draw_ime_overlay);
-	void Submit(CommandBuffer& command);
+	uint64_t             Submit(CommandScheduler& scheduler);
 	[[nodiscard]] Status Present();
 
 	[[nodiscard]] uint32_t ImageCount() const noexcept {
@@ -340,7 +305,6 @@ public:
 
 private:
 	void Destroy();
-	void RefreshSurfaceSize();
 
 	WindowContext&              m_window;
 	vk::SwapchainKHR            m_handle = nullptr;
@@ -358,7 +322,7 @@ private:
 struct Presenter::Impl {
 	explicit Impl(WindowContext& owner)
 	    : renderer(*owner.render_context), window(owner), swapchain(owner),
-	      present_scheduler(renderer, owner.graphic_ctx), frames(owner) {
+	      present_scheduler(renderer, owner.graphic_ctx), frames(owner, present_scheduler) {
 		EXIT_IF(owner.render_context == nullptr);
 		swapchain.Create();
 		frames.Initialize(swapchain.ImageCount(), swapchain.Format());
@@ -402,12 +366,12 @@ struct Presenter::Impl {
 
 void Swapchain::Create() {
 	auto& graphics = m_window.graphic_ctx;
-	EXIT_IF(graphics.screen_width == 0);
-	EXIT_IF(graphics.screen_height == 0);
 	EXIT_IF(graphics.device == nullptr);
 	EXIT_IF(m_window.surface == nullptr);
 
 	Common::LockGuard lock(m_window.mutex);
+	EXIT_IF(graphics.screen_width == 0);
+	EXIT_IF(graphics.screen_height == 0);
 	const auto&       surface = m_window.surface_capabilities;
 	EXIT_NOT_IMPLEMENTED(surface.formats.empty());
 
@@ -465,7 +429,21 @@ void Swapchain::Create() {
 	create_info.imageSharingMode = vk::SharingMode::eExclusive;
 	create_info.preTransform     = transform;
 	create_info.compositeAlpha   = composite;
-	create_info.presentMode      = vk::PresentModeKHR::eFifo;
+	switch (Config::GetPresentMode()) {
+		case Config::PresentMode::Mailbox:
+			create_info.presentMode = vk::PresentModeKHR::eMailbox;
+			break;
+		case Config::PresentMode::Immediate:
+			create_info.presentMode = vk::PresentModeKHR::eImmediate;
+			break;
+		case Config::PresentMode::Fifo:
+		default: create_info.presentMode = vk::PresentModeKHR::eFifo; break;
+	}
+	if (std::find(surface.present_modes.begin(), surface.present_modes.end(),
+	              create_info.presentMode) == surface.present_modes.end()) {
+		LOGF("warning: requested present mode is unavailable; falling back to Fifo\n");
+		create_info.presentMode = vk::PresentModeKHR::eFifo;
+	}
 	create_info.clipped          = VK_TRUE;
 	RequireVulkanSuccess(graphics.device.createSwapchainKHR(&create_info, nullptr, &m_handle),
 	                     "vkCreateSwapchainKHR");
@@ -560,30 +538,18 @@ void Swapchain::Destroy() {
 	m_render_complete.clear();
 }
 
-void Swapchain::RefreshSurfaceSize() {
-	int width  = 0;
-	int height = 0;
-	SDL_Vulkan_GetDrawableSize(m_window.window, &width, &height);
-	if (width > 0 && height > 0) {
-		m_window.graphic_ctx.screen_width  = static_cast<uint32_t>(width);
-		m_window.graphic_ctx.screen_height = static_cast<uint32_t>(height);
-	}
-
-	m_window.RefreshSurfaceCapabilities();
-}
-
 void Swapchain::Recreate(bool surface_lost) {
 	Destroy();
 	if (surface_lost) {
 #if defined(__APPLE__)
 		// Surface recreation goes through SDL_Vulkan_CreateSurface, which touches the
 		// window's view/layer and must run on the main thread on macOS.
-		m_window.RunOnMainThread([this] { m_window.RecreateSurface(); }, true);
+		m_window.RunOnMainThread([this] { m_window.RecreateSurface(); });
 #else
 		m_window.RecreateSurface();
 #endif
 	}
-	RefreshSurfaceSize();
+	m_window.RefreshSurfaceCapabilities();
 	Create();
 }
 
@@ -628,7 +594,6 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 	}
 	EXIT_IF(m_image_index >= m_images.size());
 	auto vk_command = command.Handle();
-	command.Begin();
 
 	vk::ImageMemoryBarrier to_transfer {};
 	to_transfer.sType                           = vk::StructureType::eImageMemoryBarrier;
@@ -699,15 +664,14 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 		                           vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1,
 		                           &to_present);
 	}
-	command.End();
 }
 
-void Swapchain::Submit(CommandBuffer& command) {
+uint64_t Swapchain::Submit(CommandScheduler& scheduler) {
 	EXIT_IF(m_frame_index >= m_image_acquired.size() || m_image_index >= m_render_complete.size());
 	SubmitInfo submit;
 	submit.AddWait(m_image_acquired[m_frame_index], 1, vk::PipelineStageFlagBits::eTransfer);
 	submit.AddSignal(m_render_complete[m_image_index]);
-	command.Execute(submit);
+	return scheduler.Submit(submit);
 }
 
 Swapchain::Status Swapchain::Present() {
@@ -782,15 +746,9 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 		EXIT_IF(producer->IsInvalid());
 		frame->Clear(*producer, clear);
 	} else {
-		if (frame->present_commands == nullptr) {
-			frame->present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
-		}
-		auto& command = *frame->present_commands;
-		command.WaitForFenceAndReset();
-		command.Begin();
+		auto& command = m_impl->present_scheduler.BeginCommand();
 		frame->Clear(command, clear);
-		command.End();
-		command.Execute();
+		frame->present_tick = m_impl->present_scheduler.Submit();
 	}
 	return *frame;
 }
@@ -817,29 +775,6 @@ void Presenter::Present(Frame& frame, bool reuse) {
 	KYTY_PROFILER_FUNCTION();
 	m_impl->frames.ValidateForPresent(&frame, reuse);
 
-	auto& window = m_impl->window;
-	if (window.window_hidden) {
-#if defined(__APPLE__)
-		// AppKit traps if a window is shown off the main thread; marshal and wait so the
-		// swapchain below is recreated against a visible window.
-		window.RunOnMainThread(
-		    [&window] {
-			    window.UpdateIcon();
-			    SDL_ShowWindow(window.window);
-			    SDL_RaiseWindow(window.window);
-		    },
-		    true);
-#else
-		window.UpdateIcon();
-
-		SDL_ShowWindow(window.window);
-		SDL_RaiseWindow(window.window);
-#endif
-
-		window.window_hidden = false;
-		m_impl->RecoverSwapchain(Swapchain::Status::Recreate);
-	}
-
 	const auto ime_visual = GetImeVisualState();
 	auto&      swapchain  = m_impl->swapchain;
 	for (uint32_t attempt = 0; attempt < 2; attempt++) {
@@ -848,16 +783,12 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			m_impl->RecoverSwapchain(status);
 			continue;
 		}
-		if (frame.present_commands == nullptr) {
-			frame.present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
-		}
 		{
 			Common::LockGuard render_lock(m_impl->renderer.GetMutex());
-			frame.present_commands->WaitForFenceAndReset();
-			auto&      command          = *frame.present_commands;
-			const bool draw_ime_overlay = ime_visual.active && swapchain.PrepareImeOverlay();
+			auto&             command          = m_impl->present_scheduler.BeginCommand();
+			const bool        draw_ime_overlay = ime_visual.active && swapchain.PrepareImeOverlay();
 			swapchain.RecordPresentCommands(command, frame.image, draw_ime_overlay);
-			swapchain.Submit(command);
+			frame.present_tick = swapchain.Submit(m_impl->present_scheduler);
 		}
 		status = swapchain.Present();
 		if (status != Swapchain::Status::Success) {
@@ -866,7 +797,7 @@ void Presenter::Present(Frame& frame, bool reuse) {
 		}
 
 		m_impl->presented_ime_revision.store(ime_visual.revision, std::memory_order_release);
-		window.UpdateTitle();
+		m_impl->window.UpdateTitle();
 		m_impl->frames.Release(&frame, true);
 		return;
 	}

src/graphics/presentation/window/vulkanWindow.cpp

@@ -22,7 +22,6 @@
 #include "common/logging/log.h"
 #include "common/profiler.h"
 #include "common/stringUtils.h"
-#include "common/systemInfo.h"
 #include "common/threads.h"
 #include "common/timer.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -199,7 +198,15 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
 		features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
+#if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
+#else
+		vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
+		fragment_barycentric.sType =
+		    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+		fragment_barycentric.pNext = &depth_clip_control;
+		features12.pNext           = &fragment_barycentric;
+#endif
 		features13.pNext = &features12;
 
 		device_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
@@ -231,6 +238,12 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			skip_device = true;
 #endif
 		}
+#if !defined(__APPLE__)
+		if (fragment_barycentric.fragmentShaderBarycentric != VK_TRUE) {
+			LOGF("fragmentShaderBarycentric is not supported\n");
+			skip_device = true;
+		}
+#endif
 
 		if (features12.samplerMirrorClampToEdge != VK_TRUE) {
 			LOGF("samplerMirrorClampToEdge is not supported\n");
@@ -262,6 +275,22 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			LOGF("depthBiasClamp is not supported\n");
 			skip_device = true;
 		}
+		if (device_features2.features.shaderClipDistance != VK_TRUE) {
+			LOGF("shaderClipDistance is not supported\n");
+			skip_device = true;
+		}
+		if (device_features2.features.shaderCullDistance != VK_TRUE) {
+			LOGF("shaderCullDistance is not supported\n");
+			skip_device = true;
+		}
+		if (device_features2.features.largePoints != VK_TRUE) {
+			LOGF("largePoints is not supported\n");
+			skip_device = true;
+		}
+		if (features12.shaderOutputLayer != VK_TRUE) {
+			LOGF("shaderOutputLayer is not supported\n");
+			skip_device = true;
+		}
 
 		if (device_features2.features.fragmentStoresAndAtomics != VK_TRUE) {
 			LOGF("fragmentStoresAndAtomics is not supported\n");
@@ -286,10 +315,6 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			LOGF("shaderStorageImageWriteWithoutFormat is not supported\n");
 			skip_device = true;
 		}
-		if (device_features2.features.shaderStorageImageReadWithoutFormat != VK_TRUE) {
-			LOGF("shaderStorageImageReadWithoutFormat is not supported\n");
-			skip_device = true;
-		}
 
 		if (device_features2.features.shaderImageGatherExtended != VK_TRUE) {
 			LOGF("shaderImageGatherExtended is not supported\n");
@@ -467,14 +492,19 @@ static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device,
 	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
 	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
 	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
-	graphics.subgroup_size_control_enabled = features13.subgroupSizeControl == VK_TRUE &&
-	                                         subgroup_size_control.minSubgroupSize <= 64 &&
-	                                         subgroup_size_control.maxSubgroupSize >= 64;
+	graphics.compute_subgroup_size_control_enabled =
+	    features13.subgroupSizeControl == VK_TRUE &&
+	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
+	    subgroup_size_control.minSubgroupSize <= 64 &&
+	    subgroup_size_control.maxSubgroupSize >= 64;
+	graphics.compute_wave64_supported =
+	    graphics.subgroup_size == 64u || graphics.compute_subgroup_size_control_enabled;
 
-	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s\n",
+	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
 	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
 	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
-	     graphics.subgroup_size_control_enabled ? "true" : "false");
+	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
+	     graphics.compute_wave64_supported ? "true" : "false");
 }
 
 static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r,
@@ -518,10 +548,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	features12.pNext                    = &depth_clip_control;
 	features12.samplerMirrorClampToEdge = VK_TRUE;
 
-	vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {};
-	subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures;
-	subgroup_size_control.pNext = &features12;
-
 	vk::PhysicalDeviceVulkan13Features supported_features13 {};
 	supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
 
@@ -530,6 +556,14 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	supported_features12.pNext = nullptr;
 	supported_features13.pNext = &supported_features12;
 
+#if !defined(__APPLE__)
+	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_fragment_barycentric {};
+	supported_fragment_barycentric.sType =
+	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+	supported_fragment_barycentric.pNext = nullptr;
+	supported_features12.pNext           = &supported_fragment_barycentric;
+#endif
+
 	const auto robustness2_ext_enabled =
 	    HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
 
@@ -537,7 +571,11 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
 	supported_robustness2.pNext = nullptr;
 	if (robustness2_ext_enabled) {
+#if defined(__APPLE__)
 		supported_features12.pNext = &supported_robustness2;
+#else
+		supported_fragment_barycentric.pNext = &supported_robustness2;
+#endif
 	}
 
 	vk::PhysicalDeviceFeatures2 supported_features2 {};
@@ -552,7 +590,19 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	                     supported_features13.synchronization2 != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features12.bufferDeviceAddress != VK_TRUE);
+#if !defined(__APPLE__)
+	EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
+#endif
 	features12.timelineSemaphore = VK_TRUE;
+	features12.shaderOutputLayer = VK_TRUE;
+	features12.bufferDeviceAddress = VK_TRUE;
 
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
@@ -562,38 +612,50 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	device_features.depthBounds = VK_TRUE; // unsupported by MoltenVK
 #endif
 	device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
-	device_features.shaderStorageImageReadWithoutFormat  = VK_TRUE;
 	device_features.shaderImageGatherExtended            = VK_TRUE;
 	device_features.independentBlend                     = VK_TRUE;
 	device_features.tessellationShader                   = VK_TRUE;
 	device_features.sampleRateShading                    = VK_TRUE;
 	device_features.depthBiasClamp                       = VK_TRUE;
+	device_features.shaderClipDistance                   = VK_TRUE;
+	device_features.shaderCullDistance                   = VK_TRUE;
+	device_features.largePoints                          = VK_TRUE;
+	device_features.vertexPipelineStoresAndAtomics       = VK_TRUE;
 	graphics.sample_rate_shading_enabled                 = true;
-	device_features.vertexPipelineStoresAndAtomics =
-	    supported_features2.features.vertexPipelineStoresAndAtomics;
-
-	if (graphics.subgroup_size_control_enabled &&
-	    supported_features13.subgroupSizeControl == VK_TRUE) {
-		subgroup_size_control.subgroupSizeControl = VK_TRUE;
-	}
-
-	const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE
-	                                      ? static_cast<const void*>(&subgroup_size_control)
-	                                      : static_cast<const void*>(&features12));
+	device_features.shaderInt64 = VK_TRUE;
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
 	robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
-	robustness2.pNext = const_cast<void*>(base_feature_chain);
+#if defined(__APPLE__)
+	robustness2.pNext = &features12;
+#else
+	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
+	fragment_barycentric.sType =
+	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+	fragment_barycentric.pNext                     = &features12;
+	fragment_barycentric.fragmentShaderBarycentric = VK_TRUE;
+	robustness2.pNext                              = &fragment_barycentric;
+#endif
 	if (robustness2_ext_enabled) {
 		robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2;
 		robustness2.robustImageAccess2  = supported_robustness2.robustImageAccess2;
 		robustness2.nullDescriptor      = supported_robustness2.nullDescriptor;
 	}
 
+	const bool subgroup_size_control_enabled =
+	    graphics.compute_subgroup_size_control_enabled &&
+	    supported_features13.subgroupSizeControl == VK_TRUE;
+
 	auto features13 = required_features13;
-	features13.pNext =
-	    robustness2_ext_enabled ? &robustness2 : const_cast<void*>(base_feature_chain);
-	features13.robustImageAccess = supported_features13.robustImageAccess;
+#if defined(__APPLE__)
+	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
+	                                           : static_cast<void*>(&features12);
+#else
+	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
+	                                           : static_cast<void*>(&fragment_barycentric);
+#endif
+	features13.robustImageAccess   = supported_features13.robustImageAccess;
+	features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
 
 	LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
 	     features13.robustImageAccess == VK_TRUE ? "true" : "false",
@@ -833,23 +895,26 @@ void WindowContext::CreateVulkan() {
 	app_info.engineVersion      = 1;
 	app_info.apiVersion         = VULKAN_TARGET_API_VERSION; // NOLINT
 
-	vk::ValidationFeatureEnableEXT enabled_features[] = {
+	if (Config::SpirvDebugPrintfEnabled() && Config::GpuAssistedValidationEnabled()) {
+		EXIT("--spirv-debug-printf and --gpu-assisted-validation are mutually exclusive\n");
+	}
+
+	vk::ValidationFeatureEnableEXT enabled_features[3]    = {};
+	uint32_t                       enabled_features_count = 0;
 #ifdef KYTY_ENABLE_BEST_PRACTICES
-	    vk::ValidationFeatureEnableEXT::eBestPractices,
+	enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eBestPractices;
 #endif
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
-	    vk::ValidationFeatureEnableEXT::eDebugPrintf,
-#endif
-	};
-
-	uint32_t enabled_features_count =
-	    sizeof(enabled_features) / sizeof(vk::ValidationFeatureEnableEXT);
-
-#ifdef KYTY_ENABLE_DEBUG_PRINTF
-	if (!Config::SpirvDebugPrintfEnabled()) {
-		enabled_features_count--;
+	if (Config::SpirvDebugPrintfEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eDebugPrintf;
 	}
 #endif
+	if (Config::GpuAssistedValidationEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eGpuAssisted;
+		LOGF("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::printf("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::fflush(stdout);
+	}
 
 	vk::ValidationFeaturesEXT validation_features {};
 	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
@@ -922,6 +987,7 @@ void WindowContext::CreateVulkan() {
 #else
 	device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
 	device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
+	device_extensions.push_back(VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME);
 #endif
 
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
@@ -939,7 +1005,15 @@ void WindowContext::CreateVulkan() {
 		EXIT("Could not find suitable device");
 	}
 
-	graphic_ctx.physical_device.getProperties(&graphic_ctx.physical_device_properties);
+	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
+	vk::PhysicalDeviceProperties2              physical_device_properties {};
+	push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties;
+	push_descriptor_properties.pNext = nullptr;
+	physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2;
+	physical_device_properties.pNext = &push_descriptor_properties;
+	graphic_ctx.physical_device.getProperties2(&physical_device_properties);
+	graphic_ctx.physical_device_properties = physical_device_properties.properties;
+	graphic_ctx.max_push_descriptors       = push_descriptor_properties.maxPushDescriptors;
 	graphic_ctx.physical_device.getMemoryProperties(&graphic_ctx.physical_device_memory_properties);
 	const auto& device_properties = graphic_ctx.GetPhysicalDeviceProperties();
 
@@ -962,10 +1036,6 @@ void WindowContext::CreateVulkan() {
 		}
 	}
 
-	memcpy(device_name, device_properties.deviceName, sizeof(device_name));
-	std::snprintf(processor_name, sizeof(processor_name), "%s",
-	              Common::GetSystemInfo().ProcessorName.c_str());
-
 	VulkanInitSubgroupSizeControl(graphic_ctx.physical_device, graphic_ctx);
 
 	graphic_ctx.device = VulkanCreateDevice(graphic_ctx.physical_device, r, queue_family,
@@ -1004,7 +1074,6 @@ void WindowContext::RecreateSurface() {
 		EXIT("Could not recreate the Vulkan surface: %s\n", SDL_GetError());
 	}
 	surface = native_surface;
-	RefreshSurfaceCapabilities();
 }
 
 WindowContext::~WindowContext() {

src/graphics/presentation/window/window.cpp

@@ -23,7 +23,6 @@
 #include "common/file.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
-#include "common/stringUtils.h"
 #include "common/systemInfo.h"
 #include "common/threads.h"
 #include "common/timer.h"
@@ -39,21 +38,18 @@
 #include "libs/controller.h"
 #include "loader/systemContent.h"
 
-#include <algorithm>
-#include <cstdio>
 #include <cstdlib>
-#include <cstring>
 #include <memory>
 #include <string>
 #include <vector>
 #include <vulkan/vk_platform.h>
 
+#include <fmt/format.h>
+
 #define STB_IMAGE_IMPLEMENTATION
 #define STBI_NO_SIMD
 #include "stb_image.h"
 
-#include <fmt/format.h>
-
 // IWYU pragma: no_include <intrin.h>
 
 #define KYTY_ENABLE_DEBUG_PRINTF
@@ -197,8 +193,6 @@ std::unique_ptr<WindowContext> g_window;
 } // namespace
 
 constexpr const char* KYTY_SDL_WINDOW_CAPTION = "Game";
-constexpr uint32_t    KYTY_SDL_WINDOW_FLAGS =
-    (static_cast<uint32_t>(SDL_WINDOW_HIDDEN) | static_cast<uint32_t>(SDL_WINDOW_VULKAN));
 constexpr int KYTY_SDL_WINDOWPOS_CENTERED = SDL_WINDOWPOS_CENTERED; /*NOLINT(hicpp-signed-bitwise)*/
 
 static void SetPause(WindowLoopState& game, bool flag) {
@@ -407,6 +401,7 @@ static void GameEventDidEnterForeground(WindowLoopState& game) {
 
 void WindowContext::Resize(uint32_t new_width, uint32_t new_height) {
 	EXIT_IF(new_width == 0 || new_height == 0);
+	Common::LockGuard lock(mutex);
 	graphic_ctx.screen_width  = new_width;
 	graphic_ctx.screen_height = new_height;
 }
@@ -728,8 +723,7 @@ void WindowContext::ProcessEvent(double time_s) {
 	}
 }
 
-#if defined(__APPLE__)
-void WindowContext::RunOnMainThread(std::function<void()> task, bool wait) {
+void WindowContext::RunOnMainThread(std::function<void()> task) {
 	if (Common::Thread::IsMainThread()) {
 		task();
 		return;
@@ -747,9 +741,6 @@ void WindowContext::RunOnMainThread(std::function<void()> task, bool wait) {
 	event.type = SDL_USEREVENT;
 	SDL_PushEvent(&event);
 
-	if (!wait) {
-		return;
-	}
 	Common::LockGuard lock(main_task_mutex);
 	while (main_tasks_run < ticket) {
 		main_task_done.Wait(&main_task_mutex);
@@ -772,7 +763,6 @@ void WindowContext::DrainMainThreadTasks() {
 	main_tasks_run += tasks.size();
 	main_task_done.SignalAll();
 }
-#endif
 
 void WindowContext::Run() {
 	Common::Timer timer;
@@ -783,9 +773,7 @@ void WindowContext::Run() {
 	loop.paused.store(false, std::memory_order_release);
 
 	while (!loop.need_exit) {
-#if defined(__APPLE__)
 		DrainMainThreadTasks();
-#endif
 		if (loop.paused.load(std::memory_order_acquire)) {
 			if (!timer.IsPaused()) {
 				timer.Pause();
@@ -821,10 +809,7 @@ static void WindowCreate(WindowContext& context) {
 
 	LOGF("WindowCreate(): width = %d, height = %d\n", width, height);
 
-	uint32_t window_flags = KYTY_SDL_WINDOW_FLAGS;
-	if (Config::FullscreenEnabled()) {
-		window_flags |= static_cast<uint32_t>(SDL_WINDOW_FULLSCREEN_DESKTOP);
-	}
+	uint32_t window_flags = WindowContext::InitialWindowFlags(Config::FullscreenEnabled());
 #if defined(__APPLE__)
 	// macOS 26 window chrome (CoreUI asset decode, SwiftUI titlebar) has been observed
 	// throwing NSExceptions under Rosetta during the first CATransaction commit. A
@@ -836,13 +821,20 @@ static void WindowCreate(WindowContext& context) {
 	context.window = SDL_CreateWindow(KYTY_SDL_WINDOW_CAPTION, KYTY_SDL_WINDOWPOS_CENTERED,
 	                                  KYTY_SDL_WINDOWPOS_CENTERED, width, height, window_flags);
 
-	context.window_hidden = true;
-
 	if (context.window == nullptr) {
 		EXIT("%s\n", SDL_GetError());
 	}
 
 	SDL_SetWindowResizable(context.window, SDL_FALSE);
+	context.UpdateIcon();
+}
+
+uint32_t WindowContext::InitialWindowFlags(bool fullscreen) noexcept {
+	auto flags = static_cast<uint32_t>(SDL_WINDOW_VULKAN);
+	if (fullscreen) {
+		flags |= static_cast<uint32_t>(SDL_WINDOW_FULLSCREEN_DESKTOP);
+	}
+	return flags;
 }
 
 Presenter& WindowInit(uint32_t width, uint32_t height) {
@@ -959,6 +951,7 @@ void WindowContext::UpdateTitle() {
 	    Loader::SystemContentParamSfoGetString("TITLE_ID", title_id, sizeof(title_id));
 	static bool has_app_ver =
 	    Loader::SystemContentParamSfoGetString("APP_VER", app_ver, sizeof(app_ver));
+	static const std::string processor_name = Common::GetSystemInfo().ProcessorName;
 	static uint64_t fps_start   = Common::Timer::QueryPerformanceCounter();
 	static uint64_t frame_num   = 0;
 	static uint64_t fps_frames  = 0;
@@ -983,19 +976,14 @@ void WindowContext::UpdateTitle() {
 		fps_frames  = 0;
 	}
 
-	auto fps = fmt::format(
+	const auto* device_name = graphic_ctx.GetPhysicalDeviceProperties().deviceName.data();
+	auto text = fmt::format(
 	    "[{} | {}] {}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", KYTY_BUILD_LABEL, build_type,
 	    (has_title ? title : ""), (has_title ? ", " : ""), (has_title_id ? title_id : ""),
 	    (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""), (has_app_ver ? " " : ""),
 	    device_name, processor_name, frame_num, current_fps);
 
-#if defined(__APPLE__)
-	// AppKit traps on title changes off the main thread; fire-and-forget keeps present pacing.
-	RunOnMainThread([this, fps = std::move(fps)] { SDL_SetWindowTitle(window, fps.c_str()); },
-	                false);
-#else
-	SDL_SetWindowTitle(window, fps.c_str());
-#endif
+	RunOnMainThread([this, text = std::move(text)] { SDL_SetWindowTitle(window, text.c_str()); });
 }
 
 } // namespace Libs::Graphics

src/graphics/presentation/window/windowInternal.h

@@ -9,13 +9,10 @@
 
 #include <atomic>
 #include <cstdint>
+#include <functional>
 #include <memory>
 #include <vector>
 
-#if defined(__APPLE__)
-#include <functional>
-#endif
-
 namespace Libs::Graphics {
 
 class Presenter;
@@ -39,6 +36,7 @@ struct WindowContext {
 	KYTY_CLASS_NO_COPY(WindowContext);
 
 	[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
+	[[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept;
 	void                                                    CreateVulkan();
 	void                                                    RecreateSurface();
 	void                                                    RefreshSurfaceCapabilities();
@@ -49,36 +47,25 @@ struct WindowContext {
 	void ProcessDisplayEvent(const SDL_DisplayEvent& event);
 	void ProcessEvent(double time_seconds);
 	void Run();
-
-#if defined(__APPLE__)
-	// AppKit only allows window operations (show, icon, title, view/layer changes) on the
-	// main thread; the present thread marshals them through the SDL main loop with these.
-	// wait=true blocks until the task has run on the main thread.
-	void RunOnMainThread(std::function<void()> task, bool wait);
+	// SDL window operations must complete on the main thread.
+	void RunOnMainThread(std::function<void()> task);
 	void DrainMainThreadTasks();
-#endif
 
 	GraphicContext                 graphic_ctx;
 	SDL_Window*                    window        = nullptr;
-	bool                           window_hidden = true;
 	vk::SurfaceKHR                 surface       = nullptr;
 	SurfaceCapabilities            surface_capabilities;
 	std::unique_ptr<RenderContext> render_context;
 	std::unique_ptr<Presenter>     presenter;
 	WindowLoopState                loop;
 
-	char device_name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE] = {0};
-	char processor_name[64]                            = {0};
-
 	Common::Mutex mutex;
 
-#if defined(__APPLE__)
 	Common::Mutex                      main_task_mutex;
 	Common::CondVar                    main_task_done;
 	std::vector<std::function<void()>> main_tasks;            // guarded by main_task_mutex
 	uint64_t                           main_tasks_queued = 0; // guarded by main_task_mutex
 	uint64_t                           main_tasks_run    = 0; // guarded by main_task_mutex
-#endif
 };
 
 } // namespace Libs::Graphics

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -7,7 +7,6 @@
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/recompiler/frontend/translate/Translate.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
@@ -15,6 +14,7 @@
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
 #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
+#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h"
 #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 #include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
 
@@ -46,7 +46,9 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
 	std::string dump = "CFG:\n";
 	dump += CFG::GraphToString(cfg);
 	dump += "\nIR:\n";
-	dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir);
+	dump += fmt::format("mode={} scratch_dwords={}\n",
+	                    ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords);
+	dump += IR::ProgramToString(ir);
 	return dump;
 }
 
@@ -77,85 +79,6 @@ void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg,
 	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
 }
 
-bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
-	switch (inst.op) {
-		case IR::Opcode::SLoadDword:
-		case IR::Opcode::SBufferLoadDword:
-		case IR::Opcode::BufferLoadUbyte:
-		case IR::Opcode::BufferLoadSbyte:
-		case IR::Opcode::BufferLoadUshort:
-		case IR::Opcode::BufferLoadSshort:
-		case IR::Opcode::BufferLoadDword:
-		case IR::Opcode::BufferStoreByte:
-		case IR::Opcode::BufferStoreShort:
-		case IR::Opcode::BufferStoreDword:
-		case IR::Opcode::AtomicSwapU32:
-		case IR::Opcode::AtomicAddU32:
-		case IR::Opcode::AtomicSubU32:
-		case IR::Opcode::AtomicSMinI32:
-		case IR::Opcode::AtomicUMinU32:
-		case IR::Opcode::AtomicSMaxI32:
-		case IR::Opcode::AtomicUMaxU32:
-		case IR::Opcode::AtomicAndU32:
-		case IR::Opcode::AtomicOrU32:
-		case IR::Opcode::AtomicXorU32:
-		case IR::Opcode::AtomicFMinF32:
-		case IR::Opcode::AtomicFMaxF32:
-		case IR::Opcode::FlatLoadUbyte:
-		case IR::Opcode::FlatLoadSbyte:
-		case IR::Opcode::FlatLoadUshort:
-		case IR::Opcode::FlatLoadSshort:
-		case IR::Opcode::FlatLoadDword:
-		case IR::Opcode::FlatStoreByte:
-		case IR::Opcode::FlatStoreShort:
-		case IR::Opcode::FlatStoreDword:
-		case IR::Opcode::DsReadUbyte:
-		case IR::Opcode::DsReadSbyte:
-		case IR::Opcode::DsReadUshort:
-		case IR::Opcode::DsReadSshort:
-		case IR::Opcode::DsReadB32:
-		case IR::Opcode::DsMinF32:
-		case IR::Opcode::DsMaxF32:
-		case IR::Opcode::DsWriteByte:
-		case IR::Opcode::DsWriteShort:
-		case IR::Opcode::DsWriteB32:
-		case IR::Opcode::DsWriteAddtidB32:
-		case IR::Opcode::DsReadAddtidB32:
-		case IR::Opcode::DsAppend:
-		case IR::Opcode::DsConsume:
-		case IR::Opcode::ImageLoad:
-		case IR::Opcode::ImageStore:
-		case IR::Opcode::ImageSample:
-		case IR::Opcode::ImageGather4:
-		case IR::Opcode::Export: return true;
-		default: return false;
-	}
-}
-
-bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, uint32_t* failure_block,
-                                            std::string* reason) {
-	for (const auto& block: ir.blocks) {
-		if (!block.terminator.loop_header) {
-			continue;
-		}
-		for (const auto& inst: block.instructions) {
-			if (!InstructionMaySplitSpirvBlock(inst)) {
-				continue;
-			}
-			if (reason != nullptr) {
-				*reason = fmt::format("loop header block {} contains an instruction whose SPIR-V "
-				                      "lowering emits internal control flow: {}",
-				                      block.id, IR::InstructionToString(inst).c_str());
-			}
-			if (failure_block != nullptr) {
-				*failure_block = block.id;
-			}
-			return true;
-		}
-	}
-	return false;
-}
-
 enum class EmbeddedFetchValueType {
 	Unknown,
 	Constant,
@@ -193,17 +116,8 @@ void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg
 	lanes->erase(first, last);
 }
 
-struct EmbeddedFetchLoad {
-	uint32_t              pc         = 0;
-	int                   attrib_id  = -1;
-	uint32_t              components = 0;
-	std::vector<uint32_t> prolog_loads;
-};
-
-struct EmbeddedFetchData {
-	std::vector<EmbeddedFetchLoad> loads;
-	int32_t                        vertex_offset_sgpr = -1;
-};
+using EmbeddedFetchLoad = Frontend::EmbeddedFetchLoad;
+using EmbeddedFetchData = Frontend::EmbeddedFetchPlan;
 
 bool IsDecodedSgpr(const Decoder::Operand& op) {
 	return op.kind == Decoder::OperandKind::Sgpr || op.kind == Decoder::OperandKind::VccLo ||
@@ -322,15 +236,6 @@ int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) {
 	return static_cast<int>((raw_offset + static_cast<uint32_t>(dword) * 4u) / 16u);
 }
 
-void AppendUniquePcs(std::vector<uint32_t>& dst, const std::vector<uint32_t>& src) {
-	dst.reserve(dst.size() + src.size());
-	for (auto pc: src) {
-		if (std::find(dst.begin(), dst.end(), pc) == dst.end()) {
-			dst.push_back(pc);
-		}
-	}
-}
-
 EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded,
                                             const ShaderVertexInputInfo* input_info,
                                             uint32_t user_data_base, uint32_t user_data_count,
@@ -565,101 +470,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 	return data;
 }
 
-const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const std::vector<EmbeddedFetchLoad>& loads,
-                                               uint32_t                              pc) {
-	for (const auto& load: loads) {
-		if (load.pc == pc) {
-			return &load;
-		}
-	}
-	return nullptr;
-}
-
-bool EmbeddedFetchPcInList(const std::vector<uint32_t>& pcs, uint32_t pc) {
-	return std::find(pcs.begin(), pcs.end(), pc) != pcs.end();
-}
-
-bool IsIrFetchPrologLoad(const IR::Instruction& inst) {
-	return inst.op == IR::Opcode::SLoadDword || inst.op == IR::Opcode::SBufferLoadDword;
-}
-
-int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
-                                 const EmbeddedFetchLoad&     load) {
-	if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num &&
-	    input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) {
-		return load.attrib_id;
-	}
-	for (int i = 0; i < input_info->resources_num; i++) {
-		const auto& dst = input_info->resources_dst[i];
-		if (dst.attr_id == load.attrib_id &&
-		    load.components <= static_cast<uint32_t>(std::max(dst.registers_num, 1))) {
-			return i;
-		}
-	}
-	for (int i = 0; i < input_info->resources_num; i++) {
-		if (input_info->resources_dst[i].attr_id == load.attrib_id) {
-			return i;
-		}
-	}
-	return -1;
-}
-
-uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputInfo* input_info,
-                                      const std::vector<EmbeddedFetchLoad>& loads) {
-	if (loads.empty()) {
-		return 0;
-	}
-
-	std::vector<uint32_t> prolog_pcs;
-	prolog_pcs.reserve(loads.size());
-	for (const auto& load: loads) {
-		AppendUniquePcs(prolog_pcs, load.prolog_loads);
-	}
-
-	auto*    mutable_input_info = const_cast<ShaderVertexInputInfo*>(input_info);
-	uint32_t rewritten          = 0;
-	for (auto& block: ir.blocks) {
-		for (auto& inst: block.instructions) {
-			if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) {
-				auto pc = inst.pc;
-				inst    = {};
-				inst.pc = pc;
-				inst.op = IR::Opcode::ControlNop;
-				continue;
-			}
-
-			if (inst.op != IR::Opcode::BufferLoadDword) {
-				continue;
-			}
-			const auto* load = FindEmbeddedFetchLoad(loads, inst.pc);
-			if (load == nullptr || inst.memory.component_index >= load->components) {
-				continue;
-			}
-
-			const auto resource_id = ResolveEmbeddedFetchResource(input_info, *load);
-			if (resource_id < 0 || resource_id >= input_info->resources_num) {
-				LOGF("ShaderRecompiler VS embedded fetch remap failed: pc=0x%08" PRIx32
-				     " attrib=%d resources=%d\n",
-				     inst.pc, load->attrib_id, input_info->resources_num);
-				continue;
-			}
-
-			inst.op              = IR::Opcode::LoadInputF32;
-			inst.input_info.attr = static_cast<uint32_t>(resource_id);
-			inst.input_info.chan = inst.memory.component_index;
-			inst.memory          = {};
-			inst.src_count       = 0;
-
-			mutable_input_info->resource_fetch_components[resource_id] =
-			    std::max(mutable_input_info->resource_fetch_components[resource_id],
-			             static_cast<int>(inst.input_info.chan + 1u));
-			rewritten++;
-		}
-	}
-
-	return rewritten;
-}
-
 } // namespace
 
 bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
@@ -751,90 +561,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	}
 
 	IR::Program ir;
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
-	     StageName(options.stage), options.shader_hash);
-	const auto lower_program = [&](IR::Program& output) {
-		output = {};
-		if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
-			return false;
-		}
-		output.lane_mask_mode  = options.lane_mask_mode;
-		output.shader_hash     = options.shader_hash;
-		output.user_data_base  = options.user_data_base;
-		output.user_data_count = options.user_data_count;
-		return true;
-	};
-	if (!lower_program(ir)) {
-		return false;
-	}
-
-	if (!dispatcher_fallback) {
-		const auto select_dispatcher = [&](uint32_t failure_block, const std::string& reason,
-		                                   const CFG::Graph& diagnostic_cfg) {
-			dispatcher_fallback    = true;
-			dispatcher_reason      = reason;
-			cfg.failure_kind       = CFG::FailureKind::StructuredControlFlow;
-			cfg.failure_block      = failure_block;
-			cfg.unsupported_reason = reason;
-			ir.cfg_failure_kind    = CFG::FailureKind::StructuredControlFlow;
-			ir.fallback_reason     = reason;
-			LogDispatcherFallback(options, diagnostic_cfg, "emit", reason);
-		};
-		const auto retry_budget = cfg.natural_loops.size();
-		for (size_t retry = 0;; retry++) {
-			std::string emitter_reason;
-			uint32_t    failure_block = UINT32_MAX;
-			if (!NeedsDispatcherForStructuredLoopHeader(ir, &failure_block, &emitter_reason)) {
-				break;
-			}
-			if (retry >= retry_budget) {
-				select_dispatcher(failure_block,
-				                  emitter_reason + "; no-clone loop-header isolation exhausted",
-				                  cfg);
-				break;
-			}
-
-			const auto* source_block = cfg.FindBlock(failure_block);
-			const auto  source_pc = source_block != nullptr ? source_block->start_pc : UINT32_MAX;
-			const auto  previous_cfg  = cfg;
-			const auto  blocks_before = cfg.blocks.size();
-			std::string isolation_error;
-			if (!CFG::IsolateLoopHeader(cfg, failure_block, &isolation_error)) {
-				const auto failed_cfg = cfg;
-				cfg                   = previous_cfg;
-				select_dispatcher(failure_block,
-				                  fmt::format("{}; no-clone loop-header isolation failed: {}",
-				                              emitter_reason, isolation_error),
-				                  failed_cfg);
-				break;
-			}
-
-			IR::Program isolated_ir;
-			if (!lower_program(isolated_ir)) {
-				const auto lower_error = error != nullptr ? *error : "IR lowering failed";
-				if (error != nullptr) {
-					error->clear();
-				}
-				cfg = previous_cfg;
-				select_dispatcher(failure_block,
-				                  fmt::format("{}; isolated loop-header lowering failed: {}",
-				                              emitter_reason, lower_error),
-				                  cfg);
-				break;
-			}
-			ir = std::move(isolated_ir);
-			LOGF("%s CFG isolated semantic loop header: stage=%s hash=0x%016" PRIx64
-			     " source_block=%" PRIu32 " pc=0x%08" PRIx32 " blocks=%" PRIu64 "->%" PRIu64
-			     " reason=%s\n",
-			     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-			     failure_block, source_pc, static_cast<uint64_t>(blocks_before),
-			     static_cast<uint64_t>(cfg.blocks.size()), emitter_reason.c_str());
-		}
-	}
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
-	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
 	const ShaderVertexInputInfo*  vertex  = nullptr;
 	const ShaderPixelInputInfo*   pixel   = nullptr;
 	const ShaderComputeInputInfo* compute = nullptr;
@@ -846,40 +572,59 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	}
 	EmbeddedFetchData embedded_fetch;
 	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
-		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, ir.user_data_base,
-		                                           ir.user_data_count, options.wave_size);
-		auto rewritten = RewriteEmbeddedVertexFetches(ir, vertex, embedded_fetch.loads);
-		if (rewritten > 0 || !embedded_fetch.loads.empty()) {
-			LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n",
-			     GetDumpLabel(options), static_cast<uint64_t>(embedded_fetch.loads.size()),
-			     rewritten);
+		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base,
+		                                           options.user_data_count, options.wave_size);
+		if (!embedded_fetch.loads.empty()) {
+			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
+			     static_cast<uint64_t>(embedded_fetch.loads.size()));
 		}
 	}
-	ir.dispatcher_fallback = dispatcher_fallback;
-	if (!dispatcher_reason.empty()) {
-		ir.fallback_reason = dispatcher_reason;
-	}
-
-	ir.values = std::make_shared<IR::ValueProgram>();
-	if (!Frontend::TranslateProgram(ir, *ir.values, vertex, pixel, compute, error)) {
+	Frontend::TranslateOptions translate_options {
+	    .stage               = options.stage,
+	    .wave_size           = options.wave_size,
+	    .shader_hash         = options.shader_hash,
+	    .user_data_base      = options.user_data_base,
+	    .user_data_count     = options.user_data_count,
+	    .scratch_dwords      = options.scratch_dwords,
+	    .dispatcher_fallback = dispatcher_fallback,
+	    .cfg_failure_kind    = cfg.failure_kind,
+	    .fallback_reason     = dispatcher_reason.empty() ? cfg.unsupported_reason
+	                                                    : dispatcher_reason,
+	    .vertex              = vertex,
+	    .pixel               = pixel,
+	    .compute             = compute,
+	    .embedded_fetch      = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
+	};
+	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	if (!Frontend::TranslateProgram(decoded, cfg, translate_options, ir, error)) {
 		return false;
 	}
-	ir.blocks.clear();
-	IR::RewriteToSsa(ir.values->blocks);
-	IR::ConstantPropagationPass(ir.values->blocks);
-	for (auto& info: ir.values->block_info) {
-		info.condition       = info.condition.Resolve();
-		info.indirect_target = info.indirect_target.Resolve();
-	}
-	IR::RemoveIdentities(ir.values->blocks);
-	IR::EliminateDeadCode(ir.values->blocks);
-	const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size);
+	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
+	     " elapsed_ms=%" PRIu64 "\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
+	IR::RewriteToSsa(ir.blocks);
+	IR::ConstantPropagationPass(ir.blocks);
+	IR::ResolveControlFlowIdentities(ir);
+	IR::RemoveIdentities(ir.blocks);
+	IR::EliminateDeadCode(ir.blocks);
+	const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size);
 	if (read_lane_stats.rewritten_reads != 0) {
 		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
 		     read_lane_stats.rewritten_reads);
-		IR::ConstantPropagationPass(ir.values->blocks);
-		IR::RemoveIdentities(ir.values->blocks);
-		IR::EliminateDeadCode(ir.values->blocks);
+		IR::ConstantPropagationPass(ir.blocks);
+		IR::ResolveControlFlowIdentities(ir);
+		IR::RemoveIdentities(ir.blocks);
+		IR::EliminateDeadCode(ir.blocks);
+	}
+	if (options.stage == ShaderType::Compute) {
+		const auto lds_barriers =
+		    IR::InsertSharedMemoryBarriers(ir, ir.wave_size, *compute);
+		if (lds_barriers.inserted_barriers != 0) {
+			LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options),
+			     lds_barriers.inserted_barriers);
+		}
 	}
 	std::string srt_error;
 	if (!IR::BuildSrtPlan(ir, &srt_error)) {
@@ -889,11 +634,11 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		}
 		return false;
 	}
-	IR::EliminateDeadCode(ir.values->blocks);
+	IR::EliminateDeadCode(ir.blocks);
 	if (!IR::TrackResources(ir, error)) {
 		return false;
 	}
-	IR::EliminateDeadCode(ir.values->blocks);
+	IR::EliminateDeadCode(ir.blocks);
 	if (options.stage == ShaderType::Vertex) {
 		ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
 	}
@@ -920,7 +665,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 			runtime.read_memory = ReadZeroMemory;
 		}
 		runtime.userdata         = options.read_memory_data;
-		runtime.flat_memory_base = options.flat_memory_base;
 		if (!IR::MaterializeResources(ir, runtime, resources, error)) {
 			return false;
 		}
@@ -936,10 +680,10 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	if (!IR::CollectShaderInfo(ir, info_options, error)) {
 		return false;
 	}
-	IR::BindingLayoutOptions layout_options;
-	layout_options.descriptor_set       = options.descriptor_set;
-	layout_options.push_constant_offset = options.push_constant_offset;
-	if (!IR::AllocateBindings(ir, layout_options, error)) {
+	if (!IR::AllocateBindings(ir, options.push_constant_offset, error)) {
+		return false;
+	}
+	if (!Spirv::AnalyzeProgramRequirements(ir, error)) {
 		return false;
 	}
 	std::string ir_dump;

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -6,7 +6,6 @@
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 
-#include <optional>
 #include <span>
 #include <vector>
 
@@ -14,15 +13,13 @@ namespace Libs::Graphics::ShaderRecompiler {
 
 struct CompileOptions {
 	ShaderType                  stage           = ShaderType::Compute;
-	ShaderLaneMaskMode          lane_mask_mode  = ShaderLaneMaskMode::NativeWave;
 	uint32_t                    wave_size       = 64;
 	uint32_t                    user_data_base  = 0;
 	uint32_t                    user_data_count = 64;
+	uint32_t                    scratch_dwords  = 0;
 	uint64_t                    shader_hash     = 0;
 	uint64_t                    shader_base     = 0;
-	std::optional<uint64_t>     flat_memory_base;
-	uint32_t                    descriptor_set             = 0;
-	uint32_t                    push_constant_offset       = 0;
+	uint32_t                    push_constant_offset      = 0;
 	bool                        dump_ir                    = true;
 	bool                        early_dump                 = false;
 	const char*                 dump_label                 = nullptr;

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -1,12 +1,13 @@
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 
-#include <cstdio>
+#include "common/debug.h"
+
 #include <cstring>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
 static constexpr size_t InitialSpirvSectionReserve         = 4096;
-static constexpr size_t InitialSpirvFunctionSectionReserve = 450000;
+static constexpr size_t InitialSpirvFunctionSectionReserve = 32768;
 
 static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_t* words,
                                    size_t words_num) {
@@ -22,7 +23,7 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
 Builder::Builder(uint32_t version): m_version(version) {
 	m_debug.reserve(InitialSpirvSectionReserve);
 	m_annotations.reserve(InitialSpirvSectionReserve);
-	m_types.reserve(InitialSpirvSectionReserve);
+	m_declarations.reserve(InitialSpirvSectionReserve);
 	m_functions.reserve(InitialSpirvFunctionSectionReserve);
 }
 
@@ -30,6 +31,110 @@ uint32_t Builder::AllocateId() {
 	return m_next_id++;
 }
 
+void Builder::RequireCapability(uint32_t capability) {
+	if (m_required_capabilities.insert(capability).second) {
+		AddCapability({capability});
+	}
+}
+
+void Builder::RequireExtension(const char* name) {
+	if (m_required_extensions.emplace(name).second) {
+		AddExtension(name);
+	}
+}
+
+uint32_t Builder::Import(const char* name) {
+	if (const auto it = m_import_ids.find(name); it != m_import_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_import_ids.emplace(name, id);
+	AddExtInstImport(id, name);
+	return id;
+}
+
+uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) {
+	return Type(opcode, std::vector<uint32_t>(operands));
+}
+
+uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
+	std::vector<uint32_t> key;
+	key.reserve(operands.size() + 2u);
+	key.push_back(opcode);
+	key.push_back(static_cast<uint32_t>(operands.size()));
+	key.insert(key.end(), operands.begin(), operands.end());
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	return id;
+}
+
+uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+                                std::initializer_list<TypeAnnotation> annotations) {
+	if (annotations.size() == 0) {
+		return Type(opcode, operands);
+	}
+	std::vector<uint32_t> key {opcode, static_cast<uint32_t>(operands.size())};
+	key.insert(key.end(), operands.begin(), operands.end());
+	key.push_back(static_cast<uint32_t>(annotations.size()));
+	for (const auto& annotation: annotations) {
+		key.push_back(annotation.opcode);
+		key.push_back(static_cast<uint32_t>(annotation.operands.size()));
+		key.insert(key.end(), annotation.operands.begin(), annotation.operands.end());
+	}
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	for (const auto& annotation: annotations) {
+		words = {annotation.opcode, id};
+		words.insert(words.end(), annotation.operands.begin(), annotation.operands.end());
+		AppendInstructionWords(m_annotations, words.data(), words.size());
+	}
+	return id;
+}
+
+uint32_t Builder::Constant(uint32_t opcode, uint32_t type,
+                           std::initializer_list<uint32_t> operands) {
+	return Constant(opcode, type, std::vector<uint32_t>(operands));
+}
+
+uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) {
+	std::vector<uint32_t> key;
+	key.reserve(operands.size() + 2u);
+	key.push_back(opcode);
+	key.push_back(type);
+	key.insert(key.end(), operands.begin(), operands.end());
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, type, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	return id;
+}
+
+uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) {
+	const auto id = AllocateId();
+	DefineGlobalVariable(id, pointer_type, storage_class);
+	return id;
+}
+
+void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) {
+	AddType({59u, pointer_type, id, storage_class});
+}
+
 void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
 	const auto len        = text != nullptr ? std::strlen(text) : 0;
 	const auto word_count = (len + 1u + 3u) / 4u;
@@ -103,7 +208,7 @@ void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
 }
 
 void Builder::AddType(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_types, words.begin(), words.size());
+	AppendInstructionWords(m_declarations, words.begin(), words.size());
 }
 
 void Builder::AddFunction(std::initializer_list<uint32_t> words) {
@@ -114,28 +219,33 @@ void Builder::AddFunction(const std::vector<uint32_t>& words) {
 	AppendInstructionWords(m_functions, words.data(), words.size());
 }
 
+DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
+	std::vector<uint32_t> words {245u, type, result};
+	words.resize(words.size() + incoming_count * 2u);
+	const DeferredPhi phi {m_functions.size(), incoming_count};
+	AddFunction(words);
+	m_unpatched_phi_incomings += incoming_count;
+	return phi;
+}
+
+void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
+	EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0);
+	const auto value_word  = phi.word_offset + 3u + incoming * 2u;
+	const auto parent_word = value_word + 1u;
+	EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
+	m_functions[value_word]  = value;
+	m_functions[parent_word] = parent;
+	m_unpatched_phi_incomings--;
+}
+
 std::vector<uint32_t> Builder::Build() const {
-	if (m_debug.size() > InitialSpirvSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=debug size=%zu reserve=%zu\n",
-		            m_debug.size(), InitialSpirvSectionReserve);
-	}
-	if (m_annotations.size() > InitialSpirvSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=annotations size=%zu reserve=%zu\n",
-		            m_annotations.size(), InitialSpirvSectionReserve);
-	}
-	if (m_types.size() > InitialSpirvSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=types size=%zu reserve=%zu\n",
-		            m_types.size(), InitialSpirvSectionReserve);
-	}
-	if (m_functions.size() > InitialSpirvFunctionSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=functions size=%zu reserve=%zu\n",
-		            m_functions.size(), InitialSpirvFunctionSectionReserve);
-	}
+	EXIT_IF(m_unpatched_phi_incomings != 0);
 
 	std::vector<uint32_t> module;
 	module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() +
 	               m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() +
-	               m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size());
+	               m_debug.size() + m_annotations.size() + m_declarations.size() +
+	               m_functions.size());
 
 	module.push_back(0x07230203u);
 	module.push_back(m_version);
@@ -151,7 +261,7 @@ std::vector<uint32_t> Builder::Build() const {
 	module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end());
 	module.insert(module.end(), m_debug.begin(), m_debug.end());
 	module.insert(module.end(), m_annotations.begin(), m_annotations.end());
-	module.insert(module.end(), m_types.begin(), m_types.end());
+	module.insert(module.end(), m_declarations.begin(), m_declarations.end());
 	module.insert(module.end(), m_functions.begin(), m_functions.end());
 
 	return module;

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -4,10 +4,23 @@
 #include "common/common.h"
 
 #include <initializer_list>
+#include <map>
+#include <set>
+#include <string>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
+struct TypeAnnotation {
+	uint32_t              opcode = 0;
+	std::vector<uint32_t> operands;
+};
+
+struct DeferredPhi {
+	size_t word_offset    = 0;
+	size_t incoming_count = 0;
+};
+
 class Builder {
 public:
 	explicit Builder(uint32_t version = 0x00010300u);
@@ -15,19 +28,29 @@ public:
 	KYTY_CLASS_DEFAULT_COPY(Builder);
 
 	uint32_t AllocateId();
+	void     RequireCapability(uint32_t capability);
+	void     RequireExtension(const char* name);
+	uint32_t Import(const char* name);
+	uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {});
+	uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands);
+	uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+	                       std::initializer_list<TypeAnnotation> annotations);
+	uint32_t Constant(uint32_t opcode, uint32_t type,
+	                  std::initializer_list<uint32_t> operands = {});
+	uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands);
+	uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class);
+	void     DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
 
-	void AddCapability(std::initializer_list<uint32_t> operands);
-	void AddExtension(const char* name);
-	void AddExtInstImport(uint32_t id, const char* name);
-	void AddMemoryModel(std::initializer_list<uint32_t> operands);
-	void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-	                   const std::vector<uint32_t>& interfaces);
-	void AddExecutionMode(std::initializer_list<uint32_t> operands);
-	void AddName(uint32_t target, const char* name);
-	void AddAnnotation(std::initializer_list<uint32_t> words);
-	void AddType(std::initializer_list<uint32_t> words);
-	void AddFunction(std::initializer_list<uint32_t> words);
-	void AddFunction(const std::vector<uint32_t>& words);
+	void        AddMemoryModel(std::initializer_list<uint32_t> operands);
+	void        AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
+	                          const std::vector<uint32_t>& interfaces);
+	void        AddExecutionMode(std::initializer_list<uint32_t> operands);
+	void        AddName(uint32_t target, const char* name);
+	void        AddAnnotation(std::initializer_list<uint32_t> words);
+	void        AddFunction(std::initializer_list<uint32_t> words);
+	void        AddFunction(const std::vector<uint32_t>& words);
+	DeferredPhi AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count);
+	void        PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent);
 
 	[[nodiscard]] std::vector<uint32_t> Build() const;
 
@@ -37,19 +60,28 @@ private:
 	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
 	                              std::initializer_list<uint32_t> operands);
 	static void AppendString(std::vector<uint32_t>& words, const char* text);
+	void        AddCapability(std::initializer_list<uint32_t> operands);
+	void        AddExtension(const char* name);
+	void        AddExtInstImport(uint32_t id, const char* name);
+	void        AddType(std::initializer_list<uint32_t> words);
 
-	uint32_t              m_next_id = 1;
-	uint32_t              m_version = 0;
-	std::vector<uint32_t> m_capabilities;
-	std::vector<uint32_t> m_extensions;
-	std::vector<uint32_t> m_ext_inst_imports;
-	std::vector<uint32_t> m_memory_model;
-	std::vector<uint32_t> m_entry_points;
-	std::vector<uint32_t> m_execution_modes;
-	std::vector<uint32_t> m_debug;
-	std::vector<uint32_t> m_annotations;
-	std::vector<uint32_t> m_types;
-	std::vector<uint32_t> m_functions;
+	uint32_t                                  m_next_id = 1;
+	uint32_t                                  m_version = 0;
+	std::vector<uint32_t>                     m_capabilities;
+	std::vector<uint32_t>                     m_extensions;
+	std::vector<uint32_t>                     m_ext_inst_imports;
+	std::vector<uint32_t>                     m_memory_model;
+	std::vector<uint32_t>                     m_entry_points;
+	std::vector<uint32_t>                     m_execution_modes;
+	std::vector<uint32_t>                     m_debug;
+	std::vector<uint32_t>                     m_annotations;
+	std::vector<uint32_t>                     m_declarations;
+	std::vector<uint32_t>                     m_functions;
+	std::set<uint32_t>                        m_required_capabilities;
+	std::set<std::string>                     m_required_extensions;
+	std::map<std::string, uint32_t>           m_import_ids;
+	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
+	size_t                                    m_unpatched_phi_incomings = 0;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -1,13 +1,11 @@
 #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
 #include <array>
-#include <cinttypes>
-#include <cstdio>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
@@ -15,51 +13,6 @@ using ShaderError::Fail;
 
 namespace {
 
-bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
-	using Kind = IR::DescriptorBindingKind;
-	using Dim  = Decoder::ImageDimension;
-	if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
-		const bool integer = image.kind == IR::ResourceKind::ImageUint;
-		switch (image.dimension) {
-			case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
-			case Dim::Dim1DArray:
-				kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
-				return true;
-			case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
-			case Dim::Dim2DMsaa:
-				kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
-				return true;
-			case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
-			case Dim::Dim2DArray:
-				kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
-				return true;
-			case Dim::Dim2DMsaaArray:
-				kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
-				return true;
-			case Dim::Unknown: return false;
-		}
-	}
-	const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
-	if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
-		return false;
-	}
-	switch (image.dimension) {
-		case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
-		case Dim::Dim1DArray:
-			kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
-			return true;
-		case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
-		case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
-		case Dim::Dim2DArray:
-			kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
-			return true;
-		case Dim::Dim2DMsaa:
-		case Dim::Dim2DMsaaArray: return false;
-		case Dim::Unknown: return false;
-	}
-	return false;
-}
-
 bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 	using Kind                                             = IR::DescriptorBindingKind;
 	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
@@ -81,40 +34,51 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		Expect(Kind::Buffers, Dense(program.info.buffers.size()));
 	}
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		Kind kind;
-		if (!ImageBinding(program.info.images[i], kind)) {
+		const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
+		if (!kind.has_value()) {
 			return Fail(error, "native shader plan has an invalid image class");
 		}
-		present[static_cast<size_t>(kind)] = true;
+		present[static_cast<size_t>(*kind)] = true;
 		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage;
 		const auto count   = dynamic ? program.info.images[i].mip_count : 1u;
 		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
 			return Fail(error, "native shader plan has an invalid image mip descriptor count");
 		}
-		expected[static_cast<size_t>(kind)].insert(expected[static_cast<size_t>(kind)].end(), count,
-		                                           i);
+		expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(),
+		                                            count, i);
 	}
 	if (!program.info.samplers.empty()) {
 		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
 	}
-	const auto uses_gds = program.values != nullptr &&
-	                      std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
-		                      return std::ranges::any_of(*block, [](const IR::Inst& inst) {
-			                      return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
-		                      });
-	                      });
+	bool uses_gds = false;
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
+				continue;
+			}
+			const auto index = inst.Flags<IR::MemoryFlags>().index;
+			if (index >= program.memory_info.size()) {
+				return Fail(error, "shared operation has invalid memory metadata");
+			}
+			const auto kind = program.memory_info[index].kind;
+			if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+				return Fail(error, "shared operation has invalid resource kind");
+			}
+			uses_gds |= kind == IR::ResourceKind::Gds;
+		}
+	}
 	if (uses_gds) {
 		Expect(Kind::Gds);
 	}
-	if (!program.info.addresses.empty()) {
-		Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
+	if (program.info.uses_dma) {
+		Expect(Kind::BdaPagetable);
+		Expect(Kind::FaultBuffer);
 	}
 	const bool uses_flattened_runtime =
-	    program.values != nullptr &&
-	    (!program.values->srt_reads.empty() ||
+	    !program.srt_reads.empty() ||
 	     std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
 		     return image.indirect_mapping_capacity != 0u;
-	     }));
+	     });
 	if (uses_flattened_runtime) {
 		Expect(Kind::FlattenedSrt);
 	}
@@ -123,18 +87,12 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 	}
 
 	std::array<bool, KindCount> seen {};
-	for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
-		const auto& binding = program.bindings.descriptors[i];
-		const auto  kind    = static_cast<size_t>(binding.kind);
+	for (const auto& binding: program.bindings.descriptors) {
+		const auto kind = static_cast<size_t>(binding.kind);
 		if (kind >= KindCount || seen[kind] || !present[kind] ||
 		    binding.resources != expected[kind]) {
 			return Fail(error, "native descriptor groups do not match shader topology");
 		}
-		for (uint32_t j = 0; j < i; j++) {
-			if (program.bindings.descriptors[j].binding == binding.binding) {
-				return Fail(error, "native descriptor binding numbers are not unique");
-			}
-		}
 		seen[kind] = true;
 	}
 	for (size_t i = 0; i < KindCount; i++) {
@@ -143,8 +101,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		}
 	}
 	const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
-	if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
-	    program.bindings.buffer_offset_count != program.info.buffers.size() ||
+	if (program.bindings.push_constant_offset % sizeof(uint32_t) != 0 ||
+	    program.bindings.push_constant_offset + program.bindings.push_constant_size >
+	        IR::NativePushConstantSize ||
+	    program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
+	    program.bindings.memory_offset_count != program.info.buffers.size() ||
 	    (!has_user_storage &&
 	     program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
 	    (has_user_storage && program.bindings.push_constant_size != 0) ||
@@ -156,9 +117,6 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		return Fail(error, "native user-data layout is inconsistent");
 	}
 
-	if (program.values == nullptr) {
-		return Fail(error, "native shader plan has no typed SSA");
-	}
 	const auto planning_only_handle = [&](const IR::Inst& handle) {
 		return !handle.Uses().empty() &&
 		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
@@ -168,11 +126,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 				       return false;
 			       }
 			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
-			       return index < program.values->memory_info.size() &&
-			              program.values->memory_info[index].planning_only;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].planning_only;
 		       });
 	};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto dense = inst.Flags<uint32_t>();
 			switch (inst.GetOpcode()) {
@@ -188,8 +146,13 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 					if (planning_only_handle(inst)) {
 						break;
 					}
-					if (dense >= program.info.addresses.size()) {
-						return Fail(error, "typed address handle has an invalid dense resource");
+					if (inst.NumArgs() != 2 || !program.info.uses_dma) {
+						return Fail(error, "typed address handle has invalid DMA metadata");
+					}
+					break;
+				case IR::ValueOpcode::GetScratchResource:
+					if (inst.NumArgs() != 0 || program.scratch_dwords == 0) {
+						return Fail(error, "typed scratch handle has invalid shader metadata");
 					}
 					break;
 				case IR::ValueOpcode::GetImageResource:
@@ -205,7 +168,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 				case IR::ValueOpcode::ReadConst: {
 					const auto slot = inst.Arg(1).Resolve();
 					if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
-					    slot.U32() >= program.values->srt_reads.size()) {
+					    slot.U32() >= program.srt_reads.size()) {
 						return Fail(error, "flattened SRT read has an invalid dense slot");
 					}
 					break;
@@ -219,96 +182,120 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 
 } // namespace
 
-static constexpr size_t InitialEmitterVectorReserve = 4096;
-
-static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
-                                  size_t size) {
-	if (size <= InitialEmitterVectorReserve) {
-		return;
-	}
-	std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
-	            " stage=%u vector=%s size=%zu reserve=%zu\n",
-	            program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
-	            InitialEmitterVectorReserve);
-}
-
-void CollectValueRegisterMirrors(const IR::ValueProgram&                program,
-                                 std::vector<Emitter::RegisterBinding>& registers) {
+bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
+	program.spirv_requirements.reset();
+	IR::SpirvRequirements requirements {};
+	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::SetScalarRegister:
-				case IR::ValueOpcode::SetThreadBitScalarRegister:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(
-						    registers,
-						    {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())});
+			const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
+			if (address_access != IR::AddressAccess::None) {
+				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
+				if (memory_index >= program.memory_info.size()) {
+					return Fail(error, "address operation has invalid memory metadata");
+				}
+				if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+					if (program.scratch_dwords == 0) {
+						return Fail(error, "scratch operation has no per-thread storage");
 					}
-					break;
-				case IR::ValueOpcode::SetVectorRegister:
-					Emitter::CollectRegister(
-					    registers,
-					    {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())});
-					break;
-				case IR::ValueOpcode::SetM0:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(registers, {IR::RegisterFile::M0, 0});
+					requirements.function_scratch = true;
+				} else if (address_access == IR::AddressAccess::Write) {
+					return Fail(error, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
+				}
+			}
+			if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
+				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
+				if (memory_index >= program.memory_info.size()) {
+					return Fail(error, "buffer operation has invalid memory metadata");
+				}
+				const auto& memory = program.memory_info[memory_index];
+				if (memory.kind == IR::ResourceKind::Buffer) {
+					if (memory.resource >= program.info.buffers.size()) {
+						return Fail(error, "buffer operation has invalid resource metadata");
 					}
-					break;
-				case IR::ValueOpcode::SetExecLo:
-				case IR::ValueOpcode::SetExecHi:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(
-						    registers, {IR::RegisterFile::Exec,
-						                inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u});
+					if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
+						if (program.stage != ShaderType::Compute) {
+							return Fail(error, "buffer ADD_TID is only valid for compute shaders");
+						}
+						requirements.subgroup_local_invocation_id = true;
 					}
-					break;
-				case IR::ValueOpcode::SetVccLo:
-				case IR::ValueOpcode::SetVccHi:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(
-						    registers, {IR::RegisterFile::Vcc,
-						                inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u});
-					}
-					break;
-				default: break;
+				}
+			}
+			const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
+			if (shared_access != IR::SharedAccess::None) {
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.memory_info.size()) {
+					return Fail(error, "shared operation has invalid memory metadata");
+				}
+				const auto kind = program.memory_info[index].kind;
+				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+					return Fail(error, "shared operation has invalid resource kind");
+				}
+				if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
+					requirements.function_lds = true;
+				}
+				if (shared_access == IR::SharedAccess::Append ||
+				    shared_access == IR::SharedAccess::Consume) {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+				}
 			}
-		}
-	}
-}
-
-bool ValueProgramHasOpcode(const IR::ValueProgram& program, IR::ValueOpcode opcode) {
-	return std::ranges::any_of(program.blocks, [opcode](const IR::Block* block) {
-		return std::ranges::any_of(
-		    *block, [opcode](const IR::Inst& inst) { return inst.GetOpcode() == opcode; });
-	});
-}
-
-bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
 			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::Ballot:
+				case IR::ValueOpcode::Ballot: MarkBallot(); break;
 				case IR::ValueOpcode::DppMoveU32:
+				case IR::ValueOpcode::ReadFirstLane:
+				case IR::ValueOpcode::ReadLane: {
+					MarkBallot();
+					requirements.subgroup_shuffle = true;
+					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
+						requirements.subgroup_local_invocation_id = true;
+					}
+					break;
+				}
 				case IR::ValueOpcode::DppUpdateU32:
 				case IR::ValueOpcode::WqmMask:
-				case IR::ValueOpcode::ReadFirstLane:
-				case IR::ValueOpcode::ReadLane:
-				case IR::ValueOpcode::WriteLane:
-				case IR::ValueOpcode::Permlane16U32:
-				case IR::ValueOpcode::SwizzleU32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume:
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume: return true;
+				case IR::ValueOpcode::WriteLane: {
+					MarkBallot();
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::Permlane16U32: {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::SwizzleU32: {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::LaneId:
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
+				case IR::ValueOpcode::ImageGatherRaw:
+					requirements.image_gather_extended = true;
+					break;
+				case IR::ValueOpcode::SetAttribute: {
+					const auto index = inst.Flags<IR::ExportFlags>().index;
+					if (index >= program.export_info.size()) {
+						return Fail(error, "attribute export has invalid metadata");
+					}
+					if (program.stage == ShaderType::Pixel &&
+					    program.export_info[index].vm) {
+						requirements.pixel_valid_mask = true;
+					}
+					break;
+				}
 				default: break;
 			}
 		}
 	}
-	return false;
+	program.spirv_requirements.emplace(requirements);
+	return true;
 }
 
 bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
@@ -322,7 +309,8 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
 		return false;
 	}
 	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete) {
+	    !program.shader_info_complete || !program.binding_layout_complete ||
+	    !program.spirv_requirements.has_value()) {
 		SetError(error, "SPIR-V emitter requires a fully planned native shader program");
 		return false;
 	}
@@ -335,51 +323,23 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
 	if (!ValidateNativeProgram(program, error)) {
 		return false;
 	}
-	if (program.values == nullptr) {
-		SetError(error, "SPIR-V emitter requires planned typed SSA");
-		return false;
-	}
-	const auto& value_program = *program.values;
-	if (!IR::ValidateValueProgram(value_program, true, error)) {
+	if (!IR::ValidateProgram(program, true, error)) {
 		return false;
 	}
 	EmitterState state(program, resources, input_info);
-	state.stage                = program.stage;
-	state.wave_size            = program.wave_size;
-	state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
-	state.exact_subgroup_operations =
-	    !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
-	state.registers.reserve(InitialEmitterVectorReserve);
-	state.inputs.reserve(InitialEmitterVectorReserve);
-	state.outputs.reserve(InitialEmitterVectorReserve);
-	state.interface_variables.reserve(InitialEmitterVectorReserve);
-	state.reachable_blocks.reserve(InitialEmitterVectorReserve);
-	CollectValueRegisterMirrors(value_program, state.registers);
+	state.stage     = program.stage;
+	state.wave_size = program.wave_size;
+	state.inputs.reserve(program.info.inputs.size());
+	state.outputs.reserve(program.info.outputs.size());
+	state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
 	CopyProgramInputsAndOutputs(state, program);
-	state.needs_subgroup_ballot  = ProgramNeedsSubgroupBallot(program) ||
-	                               (!state.per_invocation_masks &&
-	                                ValueProgramHasOpcode(value_program, IR::ValueOpcode::Ballot));
-	state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
-	state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
-	state.needs_compute_derivatives          = ProgramNeedsComputeDerivatives(program);
-	state.needs_image_gather_extended        = ProgramNeedsImageGatherExtended(program);
-	state.needs_function_lds                 = ProgramNeedsFunctionLds(program);
-	state.needs_pixel_valid_mask             = ProgramNeedsPixelValidMask(program);
 	AllocateInputVariables(state);
 	AllocateOutputVariables(state);
-	AllocateDescriptorVariables(state);
-	EmitHeaderAndTypes(state);
-	AllocateRegisterVariables(state);
-	if (!EmitValueProgram(state, value_program, error)) {
+	DefineModule(state);
+	if (!EmitProgram(state, program, error)) {
 		return false;
 	}
 
-	ReportReserveExceeded(program, "registers", state.registers.size());
-	ReportReserveExceeded(program, "inputs", state.inputs.size());
-	ReportReserveExceeded(program, "outputs", state.outputs.size());
-	ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
-	ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
-
 	auto binary = state.builder.Build();
 	if (binary.empty()) {
 		SetError(error, "SPIR-V builder returned an empty module");

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,7 +9,7 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
+bool AnalyzeProgramRequirements(IR::Program& program, std::string* error);
 
 bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
                  ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -32,92 +32,88 @@ uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint3
 
 Pair ExtractPair(EmitterState& state, uint32_t value) {
 	Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.low, value, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.high, value, 1});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.low, value, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.high, value, 1});
 	return result;
 }
 
 uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, state.uint_pair_type, result, low, high});
+	state.builder.AddFunction({OpCompositeConstruct, TypeU64(state), result, low, high});
 	return result;
 }
 
-uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                   uint32_t high_compare, uint32_t low_compare, bool equal_compare = false,
-                   bool not_equal = false) {
-	const auto lhs        = ExtractPair(state, lhs_value);
-	const auto rhs        = ExtractPair(state, rhs_value);
-	const auto high_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.high, rhs.high);
-	const auto low_equal  = NewBinary(state, OpIEqual, state.bool_type, lhs.low, rhs.low);
-	if (equal_compare) {
-		return NewBinary(
-		    state, not_equal ? OpLogicalOr : OpLogicalAnd, state.bool_type,
-		    not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, high_equal) : high_equal,
-		    not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, low_equal) : low_equal);
-	}
-	const auto high_result = NewBinary(state, high_compare, state.bool_type, lhs.high, rhs.high);
-	const auto low_result  = NewBinary(state, low_compare, state.bool_type, lhs.low, rhs.low);
-	const auto low_path = NewBinary(state, OpLogicalAnd, state.bool_type, high_equal, low_result);
-	return NewBinary(state, OpLogicalOr, state.bool_type, high_result, low_path);
+uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                        bool not_equal) {
+	const auto compare = NewBinary(state, not_equal ? OpINotEqual : OpIEqual,
+	                               TypeBoolVector(state, 2), lhs_value, rhs_value);
+	return NewUnary(state, not_equal ? OpAny : OpAll, TypeBool(state), compare);
+}
+
+uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                          uint32_t high_compare, uint32_t low_compare) {
+	const auto lhs         = ExtractPair(state, lhs_value);
+	const auto rhs         = ExtractPair(state, rhs_value);
+	const auto high_equal  = NewBinary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high);
+	const auto high_result = NewBinary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
+	const auto low_result  = NewBinary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
+	const auto low_path = NewBinary(state, OpLogicalAnd, TypeBool(state), high_equal, low_result);
+	return NewBinary(state, OpLogicalOr, TypeBool(state), high_result, low_path);
 }
 
 uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs   = ExtractPair(state, lhs_value);
-	const auto rhs   = ExtractPair(state, rhs_value);
-	const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0));
-	const auto high0 = NewBinary(state, OpIAdd, state.uint_type, lhs.high, rhs.high);
-	const auto high  = NewBinary(state, OpIAdd, state.uint_type, high0, carry.carry);
-	return MakePair(state, carry.sum, high);
+	const auto lhs      = ExtractPair(state, lhs_value);
+	const auto rhs      = ExtractPair(state, rhs_value);
+	const auto low_pair = state.builder.AllocateId();
+	const auto low      = state.builder.AllocateId();
+	const auto carry    = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, low_pair, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
+	const auto high0 = NewBinary(state, OpIAdd, TypeU32(state), lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpIAdd, TypeU32(state), high0, carry);
+	return MakePair(state, low, high);
 }
 
 uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
 	const auto lhs    = ExtractPair(state, lhs_value);
 	const auto rhs    = ExtractPair(state, rhs_value);
-	const auto low    = NewBinary(state, OpISub, state.uint_type, lhs.low, rhs.low);
-	const auto borrow = NewBinary(state, OpULessThan, state.bool_type, lhs.low, rhs.low);
+	const auto low    = NewBinary(state, OpISub, TypeU32(state), lhs.low, rhs.low);
+	const auto borrow = NewBinary(state, OpULessThan, TypeBool(state), lhs.low, rhs.low);
 	const auto borrow_u32 =
-	    NewSelect(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0));
-	const auto high0 = NewBinary(state, OpISub, state.uint_type, lhs.high, rhs.high);
-	const auto high  = NewBinary(state, OpISub, state.uint_type, high0, borrow_u32);
+	    NewSelect(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
+	const auto high0 = NewBinary(state, OpISub, TypeU32(state), lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpISub, TypeU32(state), high0, borrow_u32);
 	return MakePair(state, low, high);
 }
 
 uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
-	const auto operand_type = signed_value ? state.int_type : state.uint_type;
-	const auto pair_type    = signed_value ? state.int_pair_type : state.uint_pair_type;
+	const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
+	const auto pair_type    = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
 	uint32_t   lhs_operand  = lhs;
 	uint32_t   rhs_operand  = rhs;
 	if (signed_value) {
-		lhs_operand = NewUnary(state, OpBitcast, state.int_type, lhs);
-		rhs_operand = NewUnary(state, OpBitcast, state.int_type, rhs);
+		lhs_operand = NewUnary(state, OpBitcast, TypeI32(state), lhs);
+		rhs_operand = NewUnary(state, OpBitcast, TypeI32(state), rhs);
 	}
 	const auto extended = state.builder.AllocateId();
 	state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended,
 	                           lhs_operand, rhs_operand});
 	const auto high = state.builder.AllocateId();
 	state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1});
-	return signed_value ? NewUnary(state, OpBitcast, state.uint_type, high) : high;
+	return signed_value ? NewUnary(state, OpBitcast, TypeU32(state), high) : high;
 }
 
 uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
 	const auto lhs   = ExtractPair(state, lhs_value);
 	const auto rhs   = ExtractPair(state, rhs_value);
-	const auto low   = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.low);
+	const auto low   = NewBinary(state, OpIMul, TypeU32(state), lhs.low, rhs.low);
 	const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
-	const auto high1 = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.high);
-	const auto high2 = NewBinary(state, OpIMul, state.uint_type, lhs.high, rhs.low);
+	const auto high1 = NewBinary(state, OpIMul, TypeU32(state), lhs.low, rhs.high);
+	const auto high2 = NewBinary(state, OpIMul, TypeU32(state), lhs.high, rhs.low);
 	return MakePair(state, low,
-	                NewBinary(state, OpIAdd, state.uint_type,
-	                          NewBinary(state, OpIAdd, state.uint_type, high0, high1), high2));
-}
-
-uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value,
-                       uint32_t rhs_value) {
-	const auto lhs = ExtractPair(state, lhs_value);
-	const auto rhs = ExtractPair(state, rhs_value);
-	return MakePair(state, NewBinary(state, opcode, state.uint_type, lhs.low, rhs.low),
-	                NewBinary(state, opcode, state.uint_type, lhs.high, rhs.high));
+	                NewBinary(state, OpIAdd, TypeU32(state),
+	                          NewBinary(state, OpIAdd, TypeU32(state), high0, high1), high2));
 }
 
 uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
@@ -129,52 +125,94 @@ uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint3
 	} else if (opcode == OpShiftRightLogical) {
 		EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
 	} else {
-		const auto logical      = EmitShift64(state, OpShiftRightLogical, value, shift);
-		const auto logical_pair = ExtractPair(state, logical);
-		const auto sign =
-		    NewBinary(state, OpSLessThan, state.bool_type, pair.high, ConstantU32(state, 0));
-		const auto inverse =
-		    NewBinary(state, OpISub, state.uint_type, ConstantU32(state, 64), shift);
-		const auto fill = NewBinary(state, OpShiftLeftLogical, state.uint_type,
-		                            ConstantU32(state, 0xffffffffu), inverse);
-		const auto high_shift =
-		    NewBinary(state, OpISub, state.uint_type, shift, ConstantU32(state, 32));
-		const auto high_fill = NewSelect(
-		    state, state.uint_type,
-		    NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, ConstantU32(state, 32)),
-		    ConstantU32(state, 0xffffffffu), fill);
-		const auto fill_low =
-		    NewSelect(state, state.uint_type, sign,
-		              NewSelect(state, state.uint_type,
-		                        NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift,
-		                                  ConstantU32(state, 32)),
-		                        NewBinary(state, OpShiftLeftLogical, state.uint_type,
-		                                  ConstantU32(state, 0xffffffffu), high_shift),
-		                        ConstantU32(state, 0)),
+		const auto amount =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
+		const auto word_shift =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
+		const auto at_least_32 =
+		    NewBinary(state, OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
+		const auto nonzero =
+		    NewBinary(state, OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
+		const auto high_shifted =
+		    NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
+		const auto low_shifted =
+		    NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
+		const auto carry_count =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state),
+		              NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
+		              ConstantU32(state, 31));
+		const auto carry =
+		    NewSelect(state, TypeU32(state), nonzero,
+		              NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
 		              ConstantU32(state, 0));
-		const auto fill_high =
-		    NewSelect(state, state.uint_type, sign, high_fill, ConstantU32(state, 0));
-		return MakePair(
-		    state, NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.low, fill_low),
-		    NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.high, fill_high));
+		const auto low_below_32 = NewBinary(state, OpBitwiseOr, TypeU32(state), low_shifted, carry);
+		const auto negative =
+		    NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
+		const auto sign_fill = NewSelect(state, TypeU32(state), negative,
+		                                 ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
+		return MakePair(state,
+		                NewSelect(state, TypeU32(state), at_least_32, high_shifted, low_below_32),
+		                NewSelect(state, TypeU32(state), at_least_32, sign_fill, high_shifted));
 	}
 	return MakePair(state, low, high);
 }
 
+uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
+	shift &= 63u;
+	if (shift == 0u) {
+		return value;
+	}
+	const auto pair = ExtractPair(state, value);
+	if (opcode == OpShiftLeftLogical) {
+		if (shift < 32u) {
+			const auto low  = NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.low,
+			                            ConstantU32(state, shift));
+			const auto high = NewBinary(state, OpBitwiseOr, TypeU32(state),
+			                            NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+			                                      pair.high, ConstantU32(state, shift)),
+			                            NewBinary(state, OpShiftRightLogical, TypeU32(state),
+			                                      pair.low, ConstantU32(state, 32u - shift)));
+			return MakePair(state, low, high);
+		}
+		return MakePair(state, ConstantU32(state, 0),
+		                shift == 32u ? pair.low
+		                             : NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+		                                         pair.low, ConstantU32(state, shift - 32u)));
+	}
+	if (shift < 32u) {
+		const auto low = NewBinary(state, OpBitwiseOr, TypeU32(state),
+		                           NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low,
+		                                     ConstantU32(state, shift)),
+		                           NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high,
+		                                     ConstantU32(state, 32u - shift)));
+		const auto high =
+		    NewBinary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
+		return MakePair(state, low, high);
+	}
+	const auto high = opcode == OpShiftRightArithmetic
+	                      ? NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high,
+	                                  ConstantU32(state, 31u))
+	                      : ConstantU32(state, 0);
+	const auto low  = shift == 32u ? pair.high
+	                               : NewBinary(state, opcode, TypeU32(state), pair.high,
+	                                           ConstantU32(state, shift - 32u));
+	return MakePair(state, low, high);
+}
+
 uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
 	const auto pair   = ExtractPair(state, value);
 	const auto high_i = state.builder.AllocateId();
 	const auto low_i  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.int_type, high_i, state.glsl_std450, GlslFindUMsb, pair.high});
+	    {OpExtInst, TypeI32(state), high_i, GlslStd450(state), GlslFindUMsb, pair.high});
 	state.builder.AddFunction(
-	    {OpExtInst, state.int_type, low_i, state.glsl_std450, GlslFindUMsb, pair.low});
-	const auto high = NewUnary(state, OpBitcast, state.uint_type, high_i);
-	const auto low  = NewUnary(state, OpBitcast, state.uint_type, low_i);
+	    {OpExtInst, TypeI32(state), low_i, GlslStd450(state), GlslFindUMsb, pair.low});
+	const auto high = NewUnary(state, OpBitcast, TypeU32(state), high_i);
+	const auto low  = NewUnary(state, OpBitcast, TypeU32(state), low_i);
 	const auto high_nonzero =
-	    NewBinary(state, OpINotEqual, state.bool_type, pair.high, ConstantU32(state, 0));
-	return NewSelect(state, state.uint_type, high_nonzero,
-	                 NewBinary(state, OpIAdd, state.uint_type, high, ConstantU32(state, 32)), low);
+	    NewBinary(state, OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
+	return NewSelect(state, TypeU32(state), high_nonzero,
+	                 NewBinary(state, OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
 }
 
 uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
@@ -188,10 +226,10 @@ uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bo
 uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
 	const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
 	const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
-	const auto ab      = NewBinary(state, OpIAdd, state.uint_type, a, b);
-	const auto abc     = NewBinary(state, OpIAdd, state.uint_type, ab, c);
-	return NewBinary(state, OpISub, state.uint_type,
-	                 NewBinary(state, OpISub, state.uint_type, abc, minimum), maximum);
+	const auto ab      = NewBinary(state, OpIAdd, TypeU32(state), a, b);
+	const auto abc     = NewBinary(state, OpIAdd, TypeU32(state), ab, c);
+	return NewBinary(state, OpISub, TypeU32(state),
+	                 NewBinary(state, OpISub, TypeU32(state), abc, minimum), maximum);
 }
 
 uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
@@ -204,17 +242,17 @@ uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
 	const auto max_ab   = EmitMinMaxF32Value(state, a, b, true);
 	const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
 	const auto median   = EmitMinMaxF32Value(state, min_ab, high_min, true);
-	const auto nan_ab   = NewBinary(state, OpLogicalOr, state.bool_type,
+	const auto nan_ab   = NewBinary(state, OpLogicalOr, TypeBool(state),
 	                                EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
 	const auto any_nan =
-	    NewBinary(state, OpLogicalOr, state.bool_type, nan_ab, EmitClassifyF32(state, c).nan);
-	return NewSelect(state, state.float_type, any_nan, min3, median);
+	    NewBinary(state, OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
+	return NewSelect(state, TypeF32(state), any_nan, min3, median);
 }
 
 uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
                  std::initializer_list<uint32_t> args) {
 	const auto            result = state.builder.AllocateId();
-	std::vector<uint32_t> words {OpExtInst, type, result, state.glsl_std450, opcode};
+	std::vector<uint32_t> words {OpExtInst, type, result, GlslStd450(state), opcode};
 	words.insert(words.end(), args.begin(), args.end());
 	state.builder.AddFunction(words);
 	return result;
@@ -225,41 +263,41 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
 	const auto converted_raw = state.builder.AllocateId();
 	if (signed_value) {
 		const auto converted_i = state.builder.AllocateId();
-		state.builder.AddFunction({OpConvertFToS, state.int_type, converted_i, trunc});
-		state.builder.AddFunction({OpBitcast, state.uint_type, converted_raw, converted_i});
+		state.builder.AddFunction({OpConvertFToS, TypeI32(state), converted_i, trunc});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), converted_raw, converted_i});
 	} else {
-		state.builder.AddFunction({OpConvertFToU, state.uint_type, converted_raw, trunc});
+		state.builder.AddFunction({OpConvertFToU, TypeU32(state), converted_raw, trunc});
 	}
 	const auto nan = EmitClassifyF32(state, src).nan;
 	if (signed_value) {
-		const auto below = NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src,
+		const auto below = NewBinary(state, OpFOrdLessThanEqual, TypeBool(state), src,
 		                             ConstantF32(state, 0xcf000000u));
-		const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
+		const auto above = NewBinary(state, OpFOrdGreaterThanEqual, TypeBool(state), src,
 		                             ConstantF32(state, 0x4f000000u));
-		const auto high  = NewSelect(state, state.uint_type, above, ConstantU32(state, 0x7fffffffu),
-		                             converted_raw);
+		const auto high =
+		    NewSelect(state, TypeU32(state), above, ConstantU32(state, 0x7fffffffu), converted_raw);
 		const auto low =
-		    NewSelect(state, state.uint_type, below, ConstantU32(state, 0x80000000u), high);
-		return NewSelect(state, state.uint_type, nan, ConstantU32(state, 0), low);
+		    NewSelect(state, TypeU32(state), below, ConstantU32(state, 0x80000000u), high);
+		return NewSelect(state, TypeU32(state), nan, ConstantU32(state, 0), low);
 	}
 	const auto below =
-	    NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, ConstantF32(state, 0));
-	const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
+	    NewBinary(state, OpFOrdLessThanEqual, TypeBool(state), src, ConstantF32(state, 0));
+	const auto above = NewBinary(state, OpFOrdGreaterThanEqual, TypeBool(state), src,
 	                             ConstantF32(state, 0x4f800000u));
-	const auto zero  = NewBinary(state, OpLogicalOr, state.bool_type, nan, below);
+	const auto zero  = NewBinary(state, OpLogicalOr, TypeBool(state), nan, below);
 	const auto high =
-	    NewSelect(state, state.uint_type, above, ConstantU32(state, 0xffffffffu), converted_raw);
-	return NewSelect(state, state.uint_type, zero, ConstantU32(state, 0), high);
+	    NewSelect(state, TypeU32(state), above, ConstantU32(state, 0xffffffffu), converted_raw);
+	return NewSelect(state, TypeU32(state), zero, ConstantU32(state, 0), high);
 }
 
 uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
-	return EmitExt(state, state.uint_type, GlslPackHalf2x16, {src});
+	return EmitExt(state, TypeU32(state), GlslPackHalf2x16, {src});
 }
 
 uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) {
-	const auto pair   = EmitExt(state, state.vec2_float_type, GlslUnpackHalf2x16, {bits});
+	const auto pair   = EmitExt(state, TypeF32Vector(state, 2), GlslUnpackHalf2x16, {bits});
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, result, pair, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), result, pair, 0});
 	return result;
 }
 
@@ -277,7 +315,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		return true;
 	};
 	const auto ext_unary = [&](uint32_t ext) {
-		ctx.Define(inst, EmitExt(state, state.float_type, ext, {ctx.Arg(inst, 0)}));
+		ctx.Define(inst, EmitExt(state, TypeF32(state), ext, {ctx.Arg(inst, 0)}));
 		return true;
 	};
 	switch (op) {
@@ -285,8 +323,6 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::BitCastF16U16:
 		case IR::ValueOpcode::ConvertU32U16:
 		case IR::ValueOpcode::ConvertU32U8:
-		case IR::ValueOpcode::PackUint2x32:
-		case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
 		case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
 		case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
 		case IR::ValueOpcode::ConvertU16U32:
@@ -299,7 +335,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::ConvertF16F32: {
 			const auto pair = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair,
+			state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 2), pair,
 			                           ctx.Arg(inst, 0), ConstantF32(state, 0)});
 			ctx.Define(inst, EmitPackHalf(state, pair));
 			return true;
@@ -314,15 +350,23 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
 			return true;
 		case IR::ValueOpcode::ConvertF32S32: {
-			const auto signed_value = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0));
+			const auto signed_value = NewUnary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 0));
 			ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value});
 			return true;
 		}
 		case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
+		case IR::ValueOpcode::CompositeConstructU64:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U64,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
 		case IR::ValueOpcode::CompositeConstructU32x2:
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
 			return true;
+		case IR::ValueOpcode::CompositeConstructU32x3:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x3,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
 		case IR::ValueOpcode::CompositeConstructF32x2:
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
@@ -331,28 +375,28 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
 			return true;
+		case IR::ValueOpcode::CompositeExtractU64:
 		case IR::ValueOpcode::CompositeExtractU32x2:
+		case IR::ValueOpcode::CompositeExtractU32x3:
 		case IR::ValueOpcode::CompositeExtractU32x4:
 			ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
 			         {ctx.Arg(inst, 0), inst.Arg(1).U32()});
 			return true;
 		case IR::ValueOpcode::PackHalf2x16:
-			ctx.Define(inst, EmitExt(state, state.uint_type, GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
 			return true;
 		case IR::ValueOpcode::PackSnorm2x16:
-			ctx.Define(inst,
-			           EmitExt(state, state.uint_type, GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
 			return true;
 		case IR::ValueOpcode::PackUnorm2x16:
-			ctx.Define(inst,
-			           EmitExt(state, state.uint_type, GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
 			return true;
 		case IR::ValueOpcode::PackFloat2x16Rtz: {
 			const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
 			const auto high =
-			    NewBinary(state, OpShiftLeftLogical, state.uint_type,
+			    NewBinary(state, OpShiftLeftLogical, TypeU32(state),
 			              EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
-			ctx.Define(inst, NewBinary(state, OpBitwiseOr, state.uint_type, low, high));
+			ctx.Define(inst, NewBinary(state, OpBitwiseOr, TypeU32(state), low, high));
 			return true;
 		}
 		case IR::ValueOpcode::FPAbs32:
@@ -362,7 +406,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
 			return true;
 		case IR::ValueOpcode::FPSaturate32:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslFClamp,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFClamp,
 			                         {ctx.Arg(inst, 0), ConstantF32(state, 0),
 			                          ConstantF32(state, 0x3f800000u)}));
 			return true;
@@ -414,10 +458,10 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::IAbs32: {
 			const auto value = ctx.Arg(inst, 0);
-			const auto neg   = NewUnary(state, OpSNegate, state.uint_type, value);
+			const auto neg   = NewUnary(state, OpSNegate, TypeU32(state), value);
 			const auto negative =
-			    NewBinary(state, OpSLessThan, state.bool_type, value, ConstantU32(state, 0));
-			ctx.Define(inst, NewSelect(state, state.uint_type, negative, neg, value));
+			    NewBinary(state, OpSLessThan, TypeBool(state), value, ConstantU32(state, 0));
+			ctx.Define(inst, NewSelect(state, TypeU32(state), negative, neg, value));
 			return true;
 		}
 		case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32);
@@ -426,15 +470,25 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::ShiftRightArithmetic32:
 			return binary(OpShiftRightArithmetic, IR::Type::U32);
 		case IR::ValueOpcode::ShiftLeftLogical64:
-			ctx.Define(inst,
-			           EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
+			                     ? EmitConstantShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::ShiftRightLogical64:
-			ctx.Define(inst,
-			           EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
+			                     ? EmitConstantShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::ShiftRightArithmetic64:
-			ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			ctx.Define(inst,
+			           inst.Arg(1).Resolve().IsImmediate()
+			               ? EmitConstantShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			                                     inst.Arg(1).Resolve().U32())
+			               : EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
 			                             ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
@@ -442,41 +496,30 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
 		case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
 		case IR::ValueOpcode::BitwiseAnd64:
-			ctx.Define(inst,
-			           EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::BitwiseOr64:
-			ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			ctx.Define(inst, NewBinary(state, OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0),
+			                           ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
 		case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
 		case IR::ValueOpcode::BitCount64: {
-			const auto pair = ExtractPair(state, ctx.Arg(inst, 0));
-			ctx.Define(inst, NewBinary(state, OpIAdd, state.uint_type,
-			                           NewUnary(state, OpBitCount, state.uint_type, pair.low),
-			                           NewUnary(state, OpBitCount, state.uint_type, pair.high)));
+			const auto pair =
+			    ExtractPair(state, NewUnary(state, OpBitCount, TypeU64(state), ctx.Arg(inst, 0)));
+			ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state), pair.low, pair.high));
 			return true;
 		}
 		case IR::ValueOpcode::FindILsb32: {
-			const auto value = EmitExt(state, state.int_type, GlslFindILsb, {ctx.Arg(inst, 0)});
-			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			const auto value = EmitExt(state, TypeI32(state), GlslFindILsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, TypeU32(state), value));
 			return true;
 		}
 		case IR::ValueOpcode::FindUMsb32: {
-			const auto value = EmitExt(state, state.int_type, GlslFindUMsb, {ctx.Arg(inst, 0)});
-			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			const auto value = EmitExt(state, TypeI32(state), GlslFindUMsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, TypeU32(state), value));
 			return true;
 		}
 		case IR::ValueOpcode::FindUMsb64:
 			ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0)));
 			return true;
-		case IR::ValueOpcode::FindSMsb32: {
-			const auto value =
-			    EmitExt(state, state.int_type, 74u,
-			            {NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0))});
-			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
-			return true;
-		}
 		case IR::ValueOpcode::SMin32:
 			ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
 			return true;
@@ -524,53 +567,23 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
 		case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
 		case IR::ValueOpcode::IEqual64:
-			ctx.Define(inst,
-			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false));
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
 			return true;
 		case IR::ValueOpcode::INotEqual64:
-			ctx.Define(inst,
-			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true));
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
 			return true;
 		case IR::ValueOpcode::ULessThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan,
-			                           OpULessThan));
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpULessThan, OpULessThan));
 			return true;
 		case IR::ValueOpcode::SLessThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan,
-			                           OpULessThan));
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpSLessThan, OpULessThan));
 			return true;
 		case IR::ValueOpcode::UGreaterThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan,
-			                           OpUGreaterThan));
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpUGreaterThan, OpUGreaterThan));
 			return true;
-		case IR::ValueOpcode::SGreaterThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan,
-			                           OpUGreaterThan));
-			return true;
-		case IR::ValueOpcode::ULessThanEqual64: {
-			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                               OpUGreaterThan, OpUGreaterThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
-			return true;
-		}
-		case IR::ValueOpcode::SLessThanEqual64: {
-			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                               OpSGreaterThan, OpUGreaterThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
-			return true;
-		}
-		case IR::ValueOpcode::UGreaterThanEqual64: {
-			const auto less =
-			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
-			return true;
-		}
-		case IR::ValueOpcode::SGreaterThanEqual64: {
-			const auto less =
-			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
-			return true;
-		}
 		case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
 		case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
 		case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);
@@ -602,7 +615,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32);
 		case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32);
 		case IR::ValueOpcode::FPFma32:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslFma,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFma,
 			                         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
 			return true;
 		case IR::ValueOpcode::FPMin32:
@@ -625,44 +638,44 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::FPRecip32: {
 			const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
-			ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
+			ctx.Define(inst, NewBinary(state, OpFDiv, TypeF32(state),
 			                           ConstantF32(state, 0x3f800000u), source));
 			return true;
 		}
 		case IR::ValueOpcode::FPRecipIFlag32:
 			// Integer-to-float inputs used by IFLAG cannot be denormal.
-			ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
+			ctx.Define(inst, NewBinary(state, OpFDiv, TypeF32(state),
 			                           ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
 			return true;
 		case IR::ValueOpcode::FPRecipSqrt32:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslInverseSqrt,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslInverseSqrt,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPSqrt:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslSqrt,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslSqrt,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPSin:
 		case IR::ValueOpcode::FPCos: {
 			auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
 			source =
-			    NewBinary(state, OpFMul, state.float_type, source, ConstantF32(state, 0x40c90fdbu));
-			ctx.Define(inst, EmitExt(state, state.float_type,
+			    NewBinary(state, OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
+			ctx.Define(inst, EmitExt(state, TypeF32(state),
 			                         op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source}));
 			return true;
 		}
 		case IR::ValueOpcode::FPExp2:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslExp2,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslExp2,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPLog2:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslLog2,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslLog2,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPLdexp: {
-			const auto exponent = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 1));
+			const auto exponent = NewUnary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 1));
 			ctx.Define(inst,
-			           EmitExt(state, state.float_type, GlslLdexp, {ctx.Arg(inst, 0), exponent}));
+			           EmitExt(state, TypeF32(state), GlslLdexp, {ctx.Arg(inst, 0), exponent}));
 			return true;
 		}
 		case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -5,7 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
                             uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, true_value, false_value});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value});
 	return ret;
 }
 
@@ -26,20 +26,20 @@ void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t h
 	const auto high_less        = state.builder.AllocateId();
 	const auto high_ge          = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
+	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
+	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, carry_part, low, carry_count});
+	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
 	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, high_less, high_part, carry});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_ge, low, dword_shift});
+	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
 	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
 	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
 }
@@ -59,79 +59,59 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t
 	const auto low_less         = state.builder.AllocateId();
 	const auto low_ge           = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
+	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
+	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, carry_part, high, carry_count});
+	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
 	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, low_less, low_part, carry});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_ge, high, dword_shift});
+	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
 	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
 	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
 }
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, mask)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
 	return ret;
 }
 
-AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
-                                  uint32_t carry_in) {
-	const auto pair0  = state.builder.AllocateId();
-	const auto sum0   = state.builder.AllocateId();
-	const auto carry0 = state.builder.AllocateId();
-	const auto pair1  = state.builder.AllocateId();
-	const auto sum1   = state.builder.AllocateId();
-	const auto carry1 = state.builder.AllocateId();
-	const auto carry  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair0, lhs, rhs});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum0, pair0, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry0, pair0, 1});
-	state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair1, sum0, carry_in});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum1, pair1, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry1, pair1, 1});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry0, carry1});
-	return {sum1, carry};
-}
-
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, ret, value, ConstantU32(state, shift)});
+	    {OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
 	return ret;
 }
 
 uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                 uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
 uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, state.uint_type, ret, ConstantU32(state, constant), value});
+	state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
 	return ret;
 }
 
 uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, f32});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32});
 
 	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
 	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
@@ -142,12 +122,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
 	const auto normal_payload = state.builder.AllocateId();
 	const auto normal         = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, state.uint_type, half_exp, exp, ConstantU32(state, 112)});
+	state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, normal_exp, half_exp, ConstantU32(state, 10)});
+	    {OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, state.uint_type, normal_payload, normal_exp, normal_mant});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, normal, sign, normal_payload});
+	    {OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
 
 	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
 	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
@@ -160,8 +140,8 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto sub_mant  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, sub_mant, mant_with_hidden, sub_shift});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, subnormal, sign, sub_mant});
+	    {OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
 
 	const auto nan_payload =
 	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
@@ -185,8 +165,8 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {max_value ? OpUGreaterThan : OpULessThan, state.bool_type, cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
+	    {max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
 	return ret;
 }
 
@@ -194,44 +174,44 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {max_value ? OpSGreaterThan : OpSLessThan, state.bool_type, cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
+	    {max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalOr, state.bool_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value});
+	state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value});
 	return ret;
 }
 
@@ -243,12 +223,8 @@ F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
 	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
 	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
-	const auto quiet_bit_clear =
-	    EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0);
-	cls.nan        = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
-	cls.snan       = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear);
-	cls.zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
-	cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u));
+	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
+	cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
 	return cls;
 }
 
@@ -316,7 +292,7 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto rhs_class = EmitClassifyF32(state, rhs);
 
 	const auto numeric_cond = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type,
+	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state),
 	                           numeric_cond, lhs, rhs});
 	const auto ordered_bits =
 	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
@@ -328,19 +304,14 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 
 	const auto rhs_nan_bits =
 	    EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits);
-	const auto non_snan_bits =
-	    EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
-	const auto rhs_snan_bits =
-	    EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits);
-	const auto result_bits =
-	    EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits);
+	const auto result_bits = EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
 	return EmitBitcastU32ToF32(state, result_bits);
 }
 
 uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.float_type, ret, state.glsl_std450, GlslTrunc, value});
+	    {OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value});
 	return ret;
 }
 
@@ -353,18 +324,18 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)});
+	    {OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, non_zero, abs_bits, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, flush, non_zero, subnormal});
-	state.builder.AddFunction({OpSelect, state.uint_type, selected, flush, sign_bits, bits});
-	state.builder.AddFunction({OpBitcast, state.float_type, ret, selected});
+	    {OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
+	state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected});
 	return ret;
 }
 
@@ -377,46 +348,45 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
 	const auto large_finite = state.builder.AllocateId();
 	const auto reduced      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.float_type, fract, state.glsl_std450, GlslFract, src});
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, src});
+	    {OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)});
+	    {OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)});
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, large_finite, large, finite});
+	    {OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
 	state.builder.AddFunction(
-	    {OpSelect, state.float_type, reduced, large_finite, ConstantF32(state, 0), fract});
+	    {OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
 	if (!preserve_signed_zero) {
 		return reduced;
 	}
 	const auto zero = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, state.bool_type, zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, state.float_type, ret, zero, src, reduced});
+	state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced});
 	return ret;
 }
 
 uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpFNegate, state.float_type, ret, value});
+	state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, state.float_type, ret, state.glsl_std450, GlslFAbs, value});
+	state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value});
 	return ret;
 }
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, bits});
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, ret, unpacked, 0});
+	state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
+	                           GlslUnpackHalf2x16, bits});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -2,7 +2,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
@@ -32,24 +32,17 @@ bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
 	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
 }
 
+bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
+	return state.stage == ShaderType::Pixel &&
+	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
+}
+
 void SetError(std::string* error, const char* message) {
 	if (error != nullptr) {
 		*error = message;
 	}
 }
 
-void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg) {
-	if (std::any_of(registers.begin(), registers.end(),
-	                [reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
-		return;
-	}
-	registers.push_back({reg, 0});
-}
-
-bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
-	return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
-}
-
 bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
                uint32_t index) {
 	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
@@ -59,8 +52,8 @@ bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind ki
 
 void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
 	for (const auto& input: program.info.inputs) {
-		state.inputs.push_back(
-		    {input.kind, input.location, input.component_count, 0, input.debug_name});
+		state.inputs.push_back({input.kind, input.location, input.component_count, 0,
+		                        input.debug_name, input.per_vertex});
 	}
 	for (const auto& output: program.info.outputs) {
 		if (HasOutput(state.outputs, output.kind, output.index)) {
@@ -71,7 +64,7 @@ void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program
 }
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
-	if (exp.kind == IR::ExportTargetKind::Position) {
+	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
 		return state.per_vertex_variable;
 	}
 	if (exp.kind == IR::ExportTargetKind::MrtZ) {
@@ -88,170 +81,6 @@ uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo
 	return 0;
 }
 
-bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() == IR::ValueOpcode::ImageQueryLod) {
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() == IR::ValueOpcode::ImageGatherRaw) {
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsFunctionLds(const IR::Program& program) {
-	if (program.stage == ShaderType::Compute || program.values == nullptr) {
-		return false;
-	}
-
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::SwizzleU32: return true;
-				case IR::ValueOpcode::LoadSharedU8:
-				case IR::ValueOpcode::LoadSharedU16:
-				case IR::ValueOpcode::LoadSharedU32:
-				case IR::ValueOpcode::WriteSharedU8:
-				case IR::ValueOpcode::WriteSharedU16:
-				case IR::ValueOpcode::WriteSharedU32:
-				case IR::ValueOpcode::SharedAtomicFMin32:
-				case IR::ValueOpcode::SharedAtomicFMax32:
-				case IR::ValueOpcode::SharedAtomicSwap32:
-				case IR::ValueOpcode::SharedAtomicIAdd32:
-				case IR::ValueOpcode::SharedAtomicISub32:
-				case IR::ValueOpcode::SharedAtomicSMin32:
-				case IR::ValueOpcode::SharedAtomicUMin32:
-				case IR::ValueOpcode::SharedAtomicSMax32:
-				case IR::ValueOpcode::SharedAtomicUMax32:
-				case IR::ValueOpcode::SharedAtomicAnd32:
-				case IR::ValueOpcode::SharedAtomicOr32:
-				case IR::ValueOpcode::SharedAtomicXor32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume: {
-					const auto index = inst.Flags<IR::MemoryFlags>().index;
-					if (index < program.values->memory_info.size() &&
-					    program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
-						return true;
-					}
-					break;
-				}
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsPixelValidMask(const IR::Program& program) {
-	if (program.stage != ShaderType::Pixel || program.values == nullptr) {
-		return false;
-	}
-
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != IR::ValueOpcode::SetAttribute) {
-				continue;
-			}
-			const auto index = inst.Flags<IR::ExportFlags>().index;
-			if (index < program.values->export_info.size() &&
-			    program.values->export_info[index].vm) {
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
-		for (const auto* block: program.values->blocks) {
-			for (const auto& inst: *block) {
-				if (inst.GetOpcode() == IR::ValueOpcode::WqmMask) {
-					return true;
-				}
-			}
-		}
-	}
-	return ProgramRequiresExactSubgroupSize(program);
-}
-
-bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::DppMoveU32:
-				case IR::ValueOpcode::ReadFirstLane:
-				case IR::ValueOpcode::ReadLane:
-				case IR::ValueOpcode::Permlane16U32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume:
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume:
-				case IR::ValueOpcode::SwizzleU32: return true;
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::DppMoveU32:
-				case IR::ValueOpcode::DppUpdateU32:
-				case IR::ValueOpcode::WqmMask:
-				case IR::ValueOpcode::LaneId:
-				case IR::ValueOpcode::WriteLane:
-				case IR::ValueOpcode::Permlane16U32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume:
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume:
-				case IR::ValueOpcode::SwizzleU32: return true;
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
-	for (const auto& binding: state.registers) {
-		if (binding.reg == reg) {
-			return binding.pointer_id;
-		}
-	}
-	return 0;
-}
-
 uint32_t          ConstantU32(EmitterState& state, uint32_t value);
 [[noreturn]] void ExitDescriptorBindingFailure(const EmitterState&       state,
                                                IR::DescriptorBindingKind kind, uint32_t resource,
@@ -310,11 +139,7 @@ ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryIn
 	return ImageViewKindFromDimension(mem.image_dimension);
 }
 
-ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   bool uint_image, uint32_t use_pc) {
-	(void)state;
-	(void)uint_image;
-	(void)use_pc;
+ImageViewKind StorageImageViewKind(const IR::MemoryInfo& mem) {
 	return ImageViewKindFromDimension(mem.image_dimension);
 }
 
@@ -344,45 +169,70 @@ uint32_t ImageViewSpatialComponents(ImageViewKind view) {
 	}
 }
 
-uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) {
-	return state.sampled_images[SampledImageIndex(integer, view)].image_type;
+uint32_t ImageViewImageType(EmitterState& state, ImageViewKind view, bool integer) {
+	const auto component = integer ? TypeU32(state) : TypeF32(state);
+	return state.builder.Type(OpTypeImage,
+	                          {component, ImageSpirvDimension(view), 0, ImageSpirvArrayed(view),
+	                           ImageSpirvMultisampled(view), 1, ImageFormatUnknown});
 }
 
-uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) {
-	return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type;
+uint32_t ImageViewSampledImageType(EmitterState& state, ImageViewKind view, bool integer) {
+	return state.builder.Type(OpTypeSampledImage, {ImageViewImageType(state, view, integer)});
 }
 
-uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) {
+uint32_t ImageViewSizeType(EmitterState& state, ImageViewKind view) {
 	switch (ImageViewCoordinateComponents(view)) {
-		case 1u: return state.uint_type;
-		case 2u: return state.vec2_uint_type;
-		case 3u: return state.vec3_uint_type;
+		case 1u: return TypeU32(state);
+		case 2u: return TypeU32Vector(state, 2);
+		case 3u: return TypeU32Vector(state, 3);
 		default: return 0;
 	}
 }
 
-uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) {
-	return state.storage_images[StorageImageIndex(uint_image, view)].image_type;
+uint32_t StorageImageType(EmitterState& state, StorageImageClass image_class, ImageViewKind view) {
+	uint32_t component = TypeF32(state);
+	uint32_t format    = ImageFormatUnknown;
+	if (image_class == StorageImageClass::FormatlessUint) {
+		component = TypeU32(state);
+	} else if (image_class == StorageImageClass::AtomicUint) {
+		component = TypeU32(state);
+		format    = ImageFormatR32ui;
+	}
+	return state.builder.Type(OpTypeImage, {component, ImageSpirvDimension(view), 0,
+	                                        ImageSpirvArrayed(view), 0, 2, format});
 }
 
-uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) {
-	return state.storage_images[StorageImageIndex(uint_image, view)].pointer_type;
+uint32_t StorageImagePointerType(EmitterState& state, StorageImageClass image_class,
+                                 ImageViewKind view) {
+	return state.builder.Type(
+	    OpTypePointer, {StorageClassUniformConstant, StorageImageType(state, image_class, view)});
 }
 
-uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) {
-	return state.storage_images[StorageImageIndex(uint_image, view)].variable;
+StorageImageClass StorageImageClassForResource(const EmitterState& state, uint32_t resource) {
+	const auto& image      = state.program.info.images.at(resource);
+	const bool  uint_image = image.kind == IR::ResourceKind::StorageImageUint;
+	EXIT_IF(image.kind != IR::ResourceKind::StorageImage && !uint_image);
+	EXIT_IF(image.atomic && !uint_image);
+	return StorageImageClassFor(uint_image, image.atomic);
+}
+
+uint32_t StorageImageVariable(const EmitterState& state, StorageImageClass image_class,
+                              ImageViewKind view) {
+	return state.storage_image_variables[StorageImageIndex(image_class, view)];
 }
 
 uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
                                     ImageViewKind view) {
 	(void)use_pc;
-	const bool  integer     = mem.kind == IR::ResourceKind::ImageUint;
-	const auto  kind        = SampledBindingKind(integer, view);
-	const auto  binding     = ResourceForDescriptor(state, kind, mem.resource);
-	const auto& descriptors = state.sampled_images[SampledImageIndex(integer, view)];
-	const auto  pointer     = DescriptorElementPointer(
-	    state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
-	    mem.resource, "sampled image descriptor array was not emitted");
+	const bool integer      = mem.kind == IR::ResourceKind::ImageUint;
+	const auto kind         = SampledBindingKind(integer, view);
+	const auto binding      = ResourceForDescriptor(state, kind, mem.resource);
+	const auto variable     = state.sampled_image_variables[SampledImageIndex(integer, view)];
+	const auto pointer_type = state.builder.Type(
+	    OpTypePointer, {StorageClassUniformConstant, ImageViewImageType(state, view, integer)});
+	const auto pointer =
+	    DescriptorElementPointer(state, pointer_type, variable, binding.array_index, kind,
+	                             mem.resource, "sampled image descriptor array was not emitted");
 	const auto image = state.builder.AllocateId();
 	state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
 	return image;
@@ -391,11 +241,14 @@ uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& m
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
 	(void)use_pc;
 	const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
+	const auto sampler_type = state.builder.Type(OpTypeSampler);
+	const auto pointer_type =
+	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type});
 	const auto pointer = DescriptorElementPointer(
-	    state, state.ptr_uniform_sampler, state.sampler_variable, binding.array_index,
+	    state, pointer_type, state.sampler_variable, binding.array_index,
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.sampler_type, sampler_id, pointer});
+	state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer});
 	return sampler_id;
 }
 
@@ -423,43 +276,33 @@ uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32
 	return MakeSampledImage(state, selected, use_pc, view);
 }
 
-uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
-                                       uint32_t use_pc, ImageViewKind view) {
-	(void)use_pc;
-	const auto kind     = StorageBindingKind(uint_image, view);
+uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, ImageViewKind view) {
+	const auto image_class = StorageImageClassForResource(state, resource);
+	const auto kind        = StorageBindingKind(image_class, view);
 	const auto binding  = ResourceForDescriptor(state, kind, resource);
-	const auto ptr_type = StorageImagePointerType(state, uint_image, view);
-	const auto variable = StorageImageVariable(state, uint_image, view);
+	const auto ptr_type    = StorageImagePointerType(state, image_class, view);
+	const auto variable    = StorageImageVariable(state, image_class, view);
 	return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
 	                                "storage image descriptor array was not emitted");
 }
 
-uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
-                                    uint32_t use_pc, ImageViewKind view) {
-	const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
-	if (pointer == 0) {
-		ExitDescriptorBindingFailure(state, StorageBindingKind(uint_image, view), resource,
-		                             "storage image descriptor pointer creation failed");
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, ImageViewKind view,
+                           uint32_t mip_lod, uint32_t coord, uint32_t texel) {
+	const auto image_class = StorageImageClassForResource(state, resource);
+	if (image_class != StorageImageClass::AtomicUint) {
+		state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat);
 	}
-	const auto type  = StorageImageType(state, uint_image, view);
-	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, type, image, pointer});
-	return image;
-}
-
-void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_image,
-                           ImageViewKind view, uint32_t mip_lod, uint32_t coord, uint32_t texel) {
-	const auto  kind    = StorageBindingKind(uint_image, view);
+	const auto  kind    = StorageBindingKind(image_class, view);
 	const auto  binding = ResourceForDescriptor(state, kind, resource);
 	const auto& image   = state.program.info.images.at(resource);
 	const auto  LoadAt  = [&](uint32_t array_index) {
 		const auto pointer = DescriptorElementPointer(
-		    state, StorageImagePointerType(state, uint_image, view),
-		    StorageImageVariable(state, uint_image, view), array_index, kind, resource,
+		    state, StorageImagePointerType(state, image_class, view),
+		    StorageImageVariable(state, image_class, view), array_index, kind, resource,
 		    "storage image descriptor array was not emitted");
 		const auto descriptor = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpLoad, StorageImageType(state, uint_image, view), descriptor, pointer});
+		    {OpLoad, StorageImageType(state, image_class, view), descriptor, pointer});
 		return descriptor;
 	};
 	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
@@ -482,11 +325,11 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_ima
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction(words);
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
-		state.builder.AddFunction({OpLabel, labels[mip]});
+		EmitLabel(state, labels[mip]);
 		state.builder.AddFunction({OpImageWrite, LoadAt(binding.array_index + mip), coord, texel});
 		state.builder.AddFunction({OpBranch, merge_label});
 	}
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 }
 
 uint32_t ExecutionModelForStage(ShaderType stage) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp

@@ -1,52 +0,0 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
-
-#include <algorithm>
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-
-bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
-	if (reg.file != IR::RegisterFile::Scalar) {
-		return false;
-	}
-	const auto& registers = state.program.bindings.user_data_registers;
-	const auto  found     = std::lower_bound(registers.begin(), registers.end(), reg.index);
-	if (found == registers.end() || *found != reg.index) {
-		return false;
-	}
-	dword_index = static_cast<uint32_t>(found - registers.begin());
-	return true;
-}
-
-uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
-	if (state.push_constant_variable != 0) {
-		const auto pointer = state.builder.AllocateId();
-		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
-		                           state.push_constant_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-		return value;
-	}
-	if (state.vsharp_storage_variable != 0) {
-		const auto pointer = state.builder.AllocateId();
-		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-		                           state.vsharp_storage_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-		return value;
-	}
-	return ConstantU32(state, 0);
-}
-
-uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
-	if (reg.file != IR::RegisterFile::Exec) {
-		return 0;
-	}
-	if (state.per_invocation_masks) {
-		return reg.index == 0 ? 1u : 0u;
-	}
-	return reg.index == 0 || state.wave_size == 64u ? 0xffffffffu : 0u;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -5,44 +5,55 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
 
-uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
-	uint32_t ret = ConstantU32(state, 0);
-	for (uint32_t i = 0; i < 8u; i++) {
-		const auto mask     = 0x0fu << (i * 4u);
-		const auto masked   = state.builder.AllocateId();
-		const auto non_zero = state.builder.AllocateId();
-		const auto expanded = state.builder.AllocateId();
-		const auto combined = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
-		state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
-		                           ConstantU32(state, mask), ConstantU32(state, 0)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
-		ret = combined;
+bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) {
+	const auto register_index = IR::RegIndex(reg);
+	const auto& registers = state.program.bindings.user_data_registers;
+	const auto  found     = std::lower_bound(registers.begin(), registers.end(), register_index);
+	if (found == registers.end() || *found != register_index) {
+		return false;
 	}
-	return ret;
+	dword_index = static_cast<uint32_t>(found - registers.begin());
+	return true;
 }
 
-uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
-	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value,
-	                               ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
+uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
 	return result;
 }
 
-void StoreRegisterMirror(ValueEmitContext& ctx, IR::Register reg, uint32_t value) {
-	if (const auto pointer = PointerForRegister(ctx.state, reg); pointer != 0) {
-		if (IsInactiveWave32ExecHigh(ctx.state, reg)) {
-			value = ConstantU32(ctx.state, 0);
-		}
-		ctx.state.builder.AddFunction({OpStore, pointer, value});
-	}
-}
-
-bool IsPrologue(const ValueEmitContext& ctx) {
-	return !ctx.program.blocks.empty() && ctx.current_block == ctx.program.blocks.front();
+uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
+	                           ConstantU64(state, 0x0000000100000001ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                           ConstantU64(state, 0x0000000200000002ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                           ConstantU64(state, 0x1111111111111111ull)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
+	return result;
 }
 
 uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
@@ -57,26 +68,39 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 	if (kind == IR::StageInputKind::FrontFacing) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, state.bool_type, value, variable});
+		state.builder.AddFunction({OpLoad, TypeBool(state), value, variable});
 		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		    {OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, state.int_type, value, variable});
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+		state.builder.AddFunction({OpLoad, TypeI32(state), value, variable});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::FragCoord) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_input_float, pointer, variable,
-		                           ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+		state.builder.AddFunction({OpAccessChain,
+		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		                           variable, ConstantU32(state, component)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::BaryCoordSmooth ||
+	    kind == IR::StageInputKind::BaryCoordNoPerspective) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		const auto bits    = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain,
+		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		                           variable, ConstantU32(state, component + 1u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	return EmitInputComponentU32(state, kind, component);
@@ -85,30 +109,34 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
 	auto&      state  = ctx.state;
 	const auto ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 	                           ConstantU32(state, ScopeSubgroup), active});
-	const auto low  = state.builder.AllocateId();
-	const auto high = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
-	const auto wqm_low  = EmitWqmLaneU32(state, low);
-	const auto wqm_high = EmitWqmLaneU32(state, high);
+	const auto low = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
+	const auto wqm_low  = EmitWqmWordU32(state, low);
 	const auto lane     = EmitSubgroupLocalInvocationId(state);
-	const auto upper    = state.builder.AllocateId();
-	const auto mask     = state.builder.AllocateId();
-	const auto bit_lane = state.builder.AllocateId();
-	const auto bit      = state.builder.AllocateId();
-	const auto hit      = state.builder.AllocateId();
-	const auto result   = state.builder.AllocateId();
+	uint32_t   mask     = wqm_low;
+	uint32_t   bit_lane = lane;
+	if (state.wave_size == 64u) {
+		const auto high = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
+		const auto wqm_high = EmitWqmWordU32(state, high);
+		const auto upper    = state.builder.AllocateId();
+		mask                = state.builder.AllocateId();
+		bit_lane            = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, TypeU32(state), mask, upper, wqm_high, wqm_low});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)});
+	}
+	const auto bit    = state.builder.AllocateId();
+	const auto hit    = state.builder.AllocateId();
+	const auto result = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 32)});
-	state.builder.AddFunction({OpSelect, state.uint_type, mask, upper, wqm_high, wqm_low});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bit_lane, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_lane});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, result, hit, ConstantU32(state, 0)});
+	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, TypeBool(state), result, hit, ConstantU32(state, 0)});
 	return result;
 }
 
@@ -128,35 +156,35 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	const auto row_ok     = state.builder.AllocateId();
 	const auto masks_ok   = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, bank_shift, lane, ConstantU32(state, 2)});
+	    {OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, row_shift, lane, ConstantU32(state, 4)});
+	    {OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)});
+	    {OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)});
+	    {OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
+	    {OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
+	    {OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
+	    {OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, flags.row_mask), row_bit});
+	    {OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, bank_ok, bank_hit, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, row_ok, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, masks_ok, bank_ok, row_ok});
+	    {OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
 	uint32_t writable = masks_ok;
 	if (!flags.bound_control) {
 		const auto target  = EmitDppTargetLane(state, flags.control);
 		const auto bounded = state.builder.AllocateId();
-		state.builder.AddFunction({OpLogicalAnd, state.bool_type, bounded, writable, target.valid});
+		state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
 		writable = bounded;
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, result, exec, writable});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable});
 	return result;
 }
 
@@ -169,25 +197,93 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
 	if (state.stage == ShaderType::Vertex) {
 		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
 	}
+	const auto load_per_vertex = [&](uint32_t vertex) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		return value;
+	};
+	if (input->per_vertex) {
+		const auto barycentric_kind = state.input_info.pixel->ps_no_perspective
+		                                  ? IR::StageInputKind::BaryCoordNoPerspective
+		                                  : IR::StageInputKind::BaryCoordSmooth;
+		const auto barycentric      = InputVariableForKind(state, barycentric_kind);
+		uint32_t   sum              = 0;
+		for (uint32_t vertex = 0; vertex < 3u; vertex++) {
+			const auto pointer = state.builder.AllocateId();
+			const auto weight  = state.builder.AllocateId();
+			const auto product = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain,
+			                           TypePointer(state, StorageClassInput, TypeF32(state)),
+			                           pointer, barycentric, ConstantU32(state, vertex)});
+			state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer});
+			state.builder.AddFunction(
+			    {OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
+			if (vertex == 0u) {
+				sum = product;
+			} else {
+				const auto next = state.builder.AllocateId();
+				state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product});
+				sum = next;
+			}
+		}
+		const auto bits = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum});
+		return bits;
+	}
 	const auto vector    = state.builder.AllocateId();
 	const auto component = state.builder.AllocateId();
 	const auto bits      = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.vec4_float_type, vector, input->variable_id});
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, component, vector, chan & 3u});
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
+	state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component});
 	return bits;
 }
 
-bool MrtUsesUint(const EmitterState& state, const IR::ExportInfo& exp) {
-	return state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-	       exp.index < std::size(state.input_info.pixel->target_output_mode) &&
-	       state.input_info.pixel->target_output_mode[exp.index] == 7u;
+uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32_t chan,
+                                    uint32_t mode) {
+	auto&       state = ctx.state;
+	const auto* input = InputBindingForParameter(state, attr);
+	if (!input->per_vertex) {
+		return EmitAttribute(ctx, attr, chan);
+	}
+	const auto load_vertex = [&](uint32_t vertex) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		return value;
+	};
+
+	const auto selected_vertex = (mode + 1u) % 3u;
+	uint32_t   value           = load_vertex(selected_vertex);
+	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
+		const auto delta = state.builder.AllocateId();
+		state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)});
+		value = delta;
+	}
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	return bits;
+}
+
+uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
+	if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
+	    exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
+		return 0;
+	}
+	return state.input_info.pixel->target_output_mode[exp.index];
 }
 
 uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
 	const auto value = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(
-	    {OpCompositeExtract, ctx.state.uint_type, value, vector, component});
+	    {OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
 	return value;
 }
 
@@ -195,6 +291,8 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
                       bool uint_output) {
 	auto& state = ctx.state;
 	if (exp.compr && !uint_output) {
+		const auto unpack =
+		    MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
 		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
 		                   ConstantF32(state, 0x3f800000u)};
 		for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -203,21 +301,21 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 			}
 			const auto packed   = state.builder.AllocateId();
 			const auto unpacked = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, state.uint_type, packed, data, pair});
-			state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
-			                           state.glsl_std450, GlslUnpackHalf2x16, packed});
+			state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
+			state.builder.AddFunction(
+			    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed});
 			for (uint32_t lane = 0; lane < 2u; lane++) {
 				const auto component = pair * 2u + lane;
 				if (((exp.en >> component) & 1u) != 0u) {
 					f32[component] = state.builder.AllocateId();
 					state.builder.AddFunction(
-					    {OpCompositeExtract, state.float_type, f32[component], unpacked, lane});
+					    {OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
 				}
 			}
 		}
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
+		state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
+		                           f32[1], f32[2], f32[3]});
 		return vector;
 	}
 	uint32_t raw[4] = {
@@ -238,7 +336,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 					continue;
 				}
 				raw[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, raw[component],
+				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component],
 				                           packed, ConstantU32(state, lane * 16u),
 				                           ConstantU32(state, 16)});
 			}
@@ -252,30 +350,73 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	}
 	if (uint_output) {
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeConstruct, state.vec4_uint_type, vector, raw[0], raw[1], raw[2], raw[3]});
+		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0],
+		                           raw[1], raw[2], raw[3]});
 		return vector;
 	}
 	uint32_t f32[4] {};
 	for (uint32_t component = 0; component < 4u; component++) {
 		f32[component] = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, state.float_type, f32[component], raw[component]});
+		state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]});
 	}
 	const auto vector = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
+	    {OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]});
 	return vector;
 }
 
+void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp) {
+	auto& state = ctx.state;
+	for (uint32_t component = 0; component < 4; component++) {
+		if ((exp.en & (1u << component)) == 0) {
+			continue;
+		}
+		const auto output = IR::DecodePositionExportComponent(
+		    state.input_info.vertex->pa_cl_vs_out_cntl, exp.index, component);
+		if (output.layer) {
+			const auto raw   = ExportRawComponent(ctx, data, component);
+			const auto layer = state.builder.AllocateId();
+			state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
+			                           ConstantU32(state, 0x7ffu)});
+			state.builder.AddFunction({OpStore, state.layer_variable, layer});
+			continue;
+		}
+		if (!output.point_size && output.clip_distance == UINT32_MAX &&
+		    output.cull_distance == UINT32_MAX) {
+			continue;
+		}
+
+		const auto raw = ExportRawComponent(ctx, data, component);
+		const auto f32 = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+		if (output.point_size) {
+			state.builder.AddFunction({OpStore, state.point_size_variable, f32});
+			continue;
+		}
+		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
+			if (index == UINT32_MAX) {
+				return;
+			}
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer,
+			     variable, ConstantU32(state, index)});
+			state.builder.AddFunction({OpStore, pointer, f32});
+		};
+		StoreDistance(state.clip_distance_variable, output.clip_distance);
+		StoreDistance(state.cull_distance_variable, output.cull_distance);
+	}
+}
+
 void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&       state = ctx.state;
 	const auto& exp   = ctx.Export(inst);
 	const auto  exec  = ctx.Arg(inst, 1);
-	if (state.stage == ShaderType::Pixel && exp.vm && state.needs_pixel_valid_mask &&
+	if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
 	    state.pixel_valid_mask_variable != 0) {
 		const auto value = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
+		    {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
 		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
 	}
 	if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
@@ -284,20 +425,25 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	EmitIfCondition(state, exec, [&]() {
 		const auto data = ctx.Arg(inst, 0);
+		if (exp.kind == IR::ExportTargetKind::Position && exp.index != 0) {
+			EmitAuxPositionExport(ctx, data, exp);
+			return;
+		}
 		if (exp.kind == IR::ExportTargetKind::MrtZ) {
 			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
 				const auto raw = ExportRawComponent(ctx, data, 0);
 				const auto f32 = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, state.float_type, f32, raw});
+				state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
 				state.builder.AddFunction({OpStore, state.depth_variable, f32});
 			}
 			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
 				const auto raw     = ExportRawComponent(ctx, data, 2);
 				const auto value   = state.builder.AllocateId();
 				const auto pointer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, state.int_type, value, raw});
-				state.builder.AddFunction({OpAccessChain, state.ptr_output_int, pointer,
-				                           state.sample_mask_variable, ConstantU32(state, 0)});
+				state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw});
+				state.builder.AddFunction(
+				    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer,
+				     state.sample_mask_variable, ConstantU32(state, 0)});
 				state.builder.AddFunction({OpStore, pointer, value});
 			}
 			return;
@@ -306,8 +452,8 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (variable == 0) {
 			return;
 		}
-		const bool uint_output = MrtUsesUint(state, exp);
-		const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
+		const bool uint_output = MrtOutputMode(state, exp) == 7u;
+		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
 		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
 		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
@@ -322,8 +468,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		}
 		if (exp.kind == IR::ExportTargetKind::Position) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer,
-			                           variable, ConstantU32(state, 0)});
+			state.builder.AddFunction(
+			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
+			     pointer, variable, ConstantU32(state, 0)});
 			state.builder.AddFunction({OpStore, pointer, value});
 		} else {
 			state.builder.AddFunction({OpStore, variable, value});
@@ -336,63 +483,10 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	switch (inst.GetOpcode()) {
-		case IR::ValueOpcode::Identity:
-		case IR::ValueOpcode::ConditionRef: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
-		case IR::ValueOpcode::SetScalarRegister:
-			StoreRegisterMirror(
-			    ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
-			    ctx.Arg(inst, 1));
-			return true;
-		case IR::ValueOpcode::SetThreadBitScalarRegister:
-			StoreRegisterMirror(
-			    ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
-			    BoolAsU32(ctx, ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::SetVectorRegister:
-			StoreRegisterMirror(
-			    ctx, {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())},
-			    ctx.Arg(inst, 1));
-			return true;
-		case IR::ValueOpcode::SetScc: ctx.Arg(inst, 0); return true;
-		case IR::ValueOpcode::SetExec: ctx.Arg(inst, 0); return true;
-		case IR::ValueOpcode::SetExecLo:
-		case IR::ValueOpcode::SetExecHi:
-			if (!IsPrologue(ctx)) {
-				StoreRegisterMirror(ctx,
-				                    {IR::RegisterFile::Exec,
-				                     inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u},
-				                    ctx.Arg(inst, 0));
-			}
-			return true;
-		case IR::ValueOpcode::SetVcc: ctx.Arg(inst, 0); return true;
-		case IR::ValueOpcode::SetVccLo:
-		case IR::ValueOpcode::SetVccHi:
-			if (!IsPrologue(ctx)) {
-				StoreRegisterMirror(ctx,
-				                    {IR::RegisterFile::Vcc,
-				                     inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u},
-				                    ctx.Arg(inst, 0));
-			}
-			return true;
-		case IR::ValueOpcode::SetM0:
-			StoreRegisterMirror(ctx, {IR::RegisterFile::M0, 0}, ctx.Arg(inst, 0));
-			return true;
-		case IR::ValueOpcode::GetScalarRegister:
-		case IR::ValueOpcode::GetThreadBitScalarRegister:
-		case IR::ValueOpcode::GetVectorRegister:
-		case IR::ValueOpcode::GetScc:
-		case IR::ValueOpcode::GetExec:
-		case IR::ValueOpcode::GetExecLo:
-		case IR::ValueOpcode::GetExecHi:
-		case IR::ValueOpcode::GetVcc:
-		case IR::ValueOpcode::GetVccLo:
-		case IR::ValueOpcode::GetVccHi:
-		case IR::ValueOpcode::GetM0: return true;
+		case IR::ValueOpcode::Identity: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
 		case IR::ValueOpcode::Void:
 		case IR::ValueOpcode::Reference:
 		case IR::ValueOpcode::ReferenceU32:
-		case IR::ValueOpcode::Prologue:
-		case IR::ValueOpcode::Epilogue:
 		case IR::ValueOpcode::ControlNop:
 		case IR::ValueOpcode::Waitcnt:
 		case IR::ValueOpcode::Sendmsg:
@@ -408,7 +502,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetUserData: {
 			const auto reg   = inst.Arg(0).ScalarRegister();
 			uint32_t   dword = 0;
-			if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) {
+			if (!UserDataDwordIndex(state, reg, dword)) {
 				ctx.Define(inst, ConstantU32(state, 0));
 			} else {
 				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
@@ -430,7 +524,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto flags    = inst.Flags<IR::DppMoveFlags>();
 			const auto target   = EmitDppTargetLane(state, flags.control);
 			const auto shuffled = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+			state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
 			                           target.lane});
 			if (flags.fetch_inactive) {
@@ -438,12 +532,12 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 				return true;
 			}
 			const auto ballot = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+			state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
 			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
 			const auto can_fetch     = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
+			    {OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
 			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
 			return true;
 		}
@@ -453,6 +547,9 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
 			return true;
 		}
+		case IR::ValueOpcode::WqmU64:
+			ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
+			return true;
 		case IR::ValueOpcode::WqmMask:
 			ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
 			return true;
@@ -460,23 +557,15 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
 			return true;
 		case IR::ValueOpcode::Ballot:
-			if (state.per_invocation_masks) {
-				const auto result = state.builder.AllocateId();
-				state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result,
-				                           BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
-				                           ConstantU32(state, 0), ConstantU32(state, 0)});
-				ctx.Define(inst, result);
-			} else {
-				ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
-				         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
-			}
+			ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
 			return true;
 		case IR::ValueOpcode::ReadFirstLane: {
 			const auto ballot = state.builder.AllocateId();
 			const auto lane   = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+			state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
-			state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, lane,
+			state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), lane,
 			                           ConstantU32(state, ScopeSubgroup), ballot});
 			ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
 			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
@@ -488,7 +577,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::WriteLane: {
 			const auto hit = state.builder.AllocateId();
-			state.builder.AddFunction({OpIEqual, state.bool_type, hit,
+			state.builder.AddFunction({OpIEqual, TypeBool(state), hit,
 			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
 			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
 			return true;
@@ -507,40 +596,40 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto index     = state.builder.AllocateId();
 			const auto target    = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
+			    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
 			if (flags.x16) {
 				state.builder.AddFunction(
-				    {OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
+				    {OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
 			} else {
-				state.builder.AddFunction({OpCopyObject, state.uint_type, row_value, row});
+				state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row});
 			}
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+			    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
+			    {OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
 			state.builder.AddFunction(
-			    {OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
+			    {OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
 			state.builder.AddFunction(
-			    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
+			    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
+			    {OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
 			state.builder.AddFunction(
-			    {OpShiftRightLogical, state.uint_type, shifted, selected, shift});
+			    {OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, index, shifted, ConstantU32(state, 15)});
-			state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index});
+			    {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
+			state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
 			const auto shuffled = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+			state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
 			                           target});
 			uint32_t result = shuffled;
 			if (!flags.fetch_inactive) {
 				const auto source_exec = state.builder.AllocateId();
-				state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
+				state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec,
 				                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
 				                           target});
 				result = state.builder.AllocateId();
-				state.builder.AddFunction({OpSelect, state.uint_type, result, source_exec, shuffled,
+				state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
 				                           ConstantU32(state, 0)});
 			}
 			ctx.Define(inst, result);
@@ -549,6 +638,10 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetAttribute:
 			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
 			return true;
+		case IR::ValueOpcode::GetInterpolationParameter:
+			ctx.Define(inst, EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(),
+			                                            inst.Arg(2).U32()));
+			return true;
 		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
 		case IR::ValueOpcode::GetShaderBase:
 			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
@@ -558,10 +651,9 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetSrtResource:
 		case IR::ValueOpcode::GetBufferResource:
 		case IR::ValueOpcode::GetAddressResource:
+		case IR::ValueOpcode::GetScratchResource:
 		case IR::ValueOpcode::GetImageResource:
 		case IR::ValueOpcode::GetSamplerResource:
-		case IR::ValueOpcode::GetLdsResource:
-		case IR::ValueOpcode::GetGdsResource:
 		case IR::ValueOpcode::MakeImageAddress: return true;
 		default: return false;
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -3,10 +3,7 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t EmitTrueBool(EmitterState& state) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
-	return ret;
+	return ConstantBool(state, true);
 }
 
 DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
@@ -17,15 +14,15 @@ DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto selected  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
+	    {OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
+	    {OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
+	    {OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
+	    {OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected});
 	return {target, EmitTrueBool(state)};
 }
 
@@ -37,20 +34,20 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto target       = state.builder.AllocateId();
 	const auto valid        = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 	if (left) {
 		state.builder.AddFunction(
-		    {OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
+		    {OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
 		state.builder.AddFunction(
-		    {OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
+		    {OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
 	} else {
 		state.builder.AddFunction(
-		    {OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
+		    {OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
 		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
+		    {OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
 	}
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
 	return {target, valid};
 }
 
@@ -64,15 +61,15 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
 	const auto selected = state.builder.AllocateId();
 	const auto target   = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
+	    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
 	state.builder.AddFunction(
-	    {OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
-	state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
+	    {OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
+	state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected});
 	return {target, EmitTrueBool(state)};
 }
 
@@ -84,12 +81,12 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
 	const auto mirrored  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
+	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
+	    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
 	state.builder.AddFunction(
-	    {OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
+	    {OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored});
 	return {target, EmitTrueBool(state)};
 }
 
@@ -118,11 +115,11 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
 
 uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
 	if (state.subgroup_local_invocation_id_variable == 0) {
-		return ConstantU32(state, 0);
+		EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n");
 	}
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
+	    {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
 	return value;
 }
 
@@ -151,9 +148,9 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
 	}
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
-	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+	state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)),
+	                           pointer, variable, ConstantU32(state, component)});
+	state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
 	return value;
 }
 
@@ -163,7 +160,7 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
 		return ConstantU32(state, 0);
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
+	state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
 	return value;
 }
 
@@ -200,47 +197,47 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	}
 
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw});
 	return bits;
 }
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
 	const auto active_ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
+	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
 	                           ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
 	return EmitBallotLaneActiveBool(state, active_ballot, lane);
 }
 uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
 	const auto low = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
 	uint32_t mask = low;
 	if (state.wave_size == 64u) {
 		const auto high     = state.builder.AllocateId();
 		const auto in_high  = state.builder.AllocateId();
 		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
+		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
 		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
+		    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low});
 		mask = selected;
 	}
 
 	const auto lane_low = state.builder.AllocateId();
 	const auto bit      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
+	    {OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
+	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
 
 	const auto hit      = state.builder.AllocateId();
 	const auto active   = state.builder.AllocateId();
 	const auto in_range = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
+	    {OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.wave_size)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range});
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -1,3 +1,4 @@
+#include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
 
 #include <algorithm>
@@ -14,6 +15,13 @@ uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
 	return index;
 }
 
+uint32_t DmaskComponent(uint32_t dmask, uint32_t index) {
+	for (uint32_t component = 0; component < 4u; component++) {
+		if (((dmask >> component) & 1u) != 0u && index-- == 0u) return component;
+	}
+	return 0u;
+}
+
 uint32_t ImageGatherComponent(uint32_t dmask) {
 	switch (dmask) {
 		case 0x2u: return 1;
@@ -78,10 +86,10 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	auto value = ctx.Def(address.Arg(packed));
 	if (ComponentUsesA16(mem, component)) {
 		if ((half & 1u) != 0u) {
-			value = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, value,
+			value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value,
 			               ConstantU32(ctx.state, 16));
 		}
-		value = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, value,
+		value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value,
 		               ConstantU32(ctx.state, 0xffffu));
 	}
 	return value;
@@ -92,7 +100,7 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	const auto value = AddressU32(ctx, mem, address, component);
 	return ComponentUsesA16(mem, component)
 	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, OpBitcast, ctx.state.float_type, value);
+	           : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value);
 }
 
 ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) {
@@ -117,28 +125,20 @@ uint32_t ZeroF32(EmitterState& state) {
 	return ConstantF32(state, 0);
 }
 
-uint32_t ZeroU32x4(EmitterState& state) {
-	const auto zero  = ConstantU32(state, 0);
-	const auto value = state.builder.AllocateId();
-	state.builder.AddType(
-	    {OpConstantComposite, state.vec4_uint_type, value, zero, zero, zero, zero});
-	return value;
-}
-
 uint32_t CubeAxis(EmitterState& state, uint32_t value) {
-	return Binary(state, OpFSub, state.float_type, value, ConstantF32(state, 0x3f800000u));
+	return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
 }
 
 uint32_t CubeLayer(EmitterState& state, uint32_t value) {
 	const auto guest = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertFToU, state.uint_type, guest, value});
+	state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value});
 	const auto padding =
-	    Binary(state, OpShiftLeftLogical, state.uint_type,
-	           Binary(state, OpShiftRightLogical, state.uint_type, guest, ConstantU32(state, 3)),
+	    Binary(state, OpShiftLeftLogical, TypeU32(state),
+	           Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
 	           ConstantU32(state, 1));
-	const auto host   = Binary(state, OpISub, state.uint_type, guest, padding);
+	const auto host   = Binary(state, OpISub, TypeU32(state), guest, padding);
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, state.float_type, result, host});
+	state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host});
 	return result;
 }
 
@@ -159,10 +159,10 @@ uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		             : ZeroF32(ctx.state);
 		if (mem.image_cube) z = CubeLayer(ctx.state, z);
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec3_float_type, result, x, y, z});
+		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec2_float_type, result, x, y});
+		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
 	}
 	return result;
 }
@@ -179,10 +179,10 @@ uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
 		                                                 : ConstantU32(ctx.state, 0);
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec3_uint_type, result, x, y, z});
+		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec2_uint_type, result, x, y});
+		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
 	}
 	return result;
 }
@@ -196,10 +196,52 @@ uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst
 }
 
 uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
-	return Unary(ctx.state, OpBitcast, ctx.state.uint_type, value);
+	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
 }
 
-uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) {
+uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref,
+                      const IR::MemoryInfo& mem, bool gather = false) {
+	if (mem.data_bits == 16u) {
+		uint32_t   packed[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+		                        ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+		const auto scalar    = [&](uint32_t index) {
+			if (dref) return value;
+			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
+			const auto result    = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction({OpCompositeExtract,
+			                               integer ? TypeU32(ctx.state) : TypeF32(ctx.state),
+			                               result, value, component});
+			return result;
+		};
+		for (uint32_t word = 0; word < mem.data_dwords; word++) {
+			const auto low_index  = word * 2u;
+			const auto high_index = low_index + 1u;
+			const auto low        = scalar(low_index);
+			const auto high = high_index < mem.component_count
+			                      ? scalar(high_index)
+			                      : (integer ? ConstantU32(ctx.state, 0) : ZeroF32(ctx.state));
+			if (integer) {
+				const auto mask = ConstantU32(ctx.state, 0xffffu);
+				packed[word] =
+				    Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
+				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low, mask),
+				           Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
+				                  Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high, mask),
+				                  ConstantU32(ctx.state, 16u)));
+			} else {
+				const auto pair = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction(
+				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				packed[word] = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
+				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+			}
+		}
+		const auto result = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+		                               packed[0], packed[1], packed[2], packed[3]});
+		return result;
+	}
 	uint32_t component[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		if (dref) {
@@ -208,12 +250,12 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool
 		}
 		const auto scalar = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction({OpCompositeExtract,
-		                               integer ? ctx.state.uint_type : ctx.state.float_type, scalar,
+		                               integer ? TypeU32(ctx.state) : TypeF32(ctx.state), scalar,
 		                               value, index});
 		component[index] = integer ? scalar : FloatBits(ctx, scalar);
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, result,
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
 	                               component[0], component[1], component[2], component[3]});
 	return result;
 }
@@ -240,15 +282,15 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::
 		for (uint32_t index = 0; index < components; index++) {
 			result[index] = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, ctx.state.uint_type, result[index], size, index});
+			    {OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
 		}
 	}
 	if (ImageSpirvMultisampled(view) == 0u) {
 		result[3] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpImageQueryLevels, ctx.state.uint_type, result[3], image});
+		ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
 	}
 	const auto vector = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, vector,
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
 	                               result[0], result[1], result[2], result[3]});
 	return vector;
 }
@@ -262,55 +304,51 @@ uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR
 		const auto result = ctx.state.builder.AllocateId();
 		if (components == 3u) {
 			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, ctx.state.vec3_int_type, result, zero, zero, zero});
+			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
 		} else {
 			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, ctx.state.vec2_int_type, result, zero, zero});
+			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
 		}
 		return result;
 	}
-	const auto packed    = Unary(ctx.state, OpBitcast, ctx.state.int_type,
+	const auto packed    = Unary(ctx.state, OpBitcast, TypeI32(ctx.state),
 	                             AddressU32(ctx, mem, address, layout.offset));
 	uint32_t   values[3] = {zero, zero, zero};
 	for (uint32_t index = 0; index < components; index++) {
 		values[index] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldSExtract, ctx.state.int_type, values[index],
+		ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index],
 		                               packed, ConstantU32(ctx.state, index * 8u),
 		                               ConstantU32(ctx.state, 6)});
 	}
 	if (components == 1u) return values[0];
 	const auto result = ctx.state.builder.AllocateId();
 	if (components == 3u) {
-		ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec3_int_type, result,
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result,
 		                               values[0], values[1], values[2]});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec2_int_type, result, values[0], values[1]});
+		    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
 	}
 	return result;
 }
 
 uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) {
 	const auto components   = ImageViewSpatialComponents(view);
-	const auto array_type   = state.builder.AllocateId();
 	const auto count        = ConstantU32(state, 4);
-	const auto element_type = components == 1u ? state.int_type : state.vec2_int_type;
-	state.builder.AddType({OpTypeArray, array_type, element_type, count});
-	uint32_t offsets[4] {};
+	const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
+	const auto array_type   = state.builder.Type(OpTypeArray, {element_type, count});
+	uint32_t   offsets[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
 		if (components == 1u) {
 			offsets[index] = x;
 		} else {
-			offsets[index] = state.builder.AllocateId();
-			state.builder.AddType({OpConstantComposite, state.vec2_int_type, offsets[index], x,
-			                       ConstantI32(state, 0)});
+			offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2),
+			                                        {x, ConstantI32(state, 0)});
 		}
 	}
-	const auto value = state.builder.AllocateId();
-	state.builder.AddType(
-	    {OpConstantComposite, array_type, value, offsets[0], offsets[1], offsets[2], offsets[3]});
-	return value;
+	return state.builder.Constant(OpConstantComposite, array_type,
+	                              {offsets[0], offsets[1], offsets[2], offsets[3]});
 }
 
 uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
@@ -320,8 +358,120 @@ uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
 	return UINT32_MAX;
 }
 
+Format::BufferFormatInfo ImageConversionFormat(const EmitterState&   state,
+                                               const IR::MemoryInfo& mem) {
+	const auto format = state.program.info.images[mem.resource].conversion_format;
+	if (format == Prospero::BufferFormat::kInvalid) return {};
+	const auto info = Format::GetFormatInfo(format);
+	EXIT_IF(!Prospero::IsSampledTextureFormat(format) ||
+	        !Prospero::IsUintTextureFormat(format) ||
+	        Prospero::RemapTextureFormat(format) == format ||
+	        info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
+	        info.byte_size != sizeof(uint32_t) || info.component_count == 0u ||
+	        info.component_count > 4u);
+	return info;
+}
+
+uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+
+	const auto packed = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
+	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	for (uint32_t component = 0; component < info.component_count; component++) {
+		components[component] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state),
+		                               components[component], packed,
+		                               ConstantU32(ctx.state, info.component_bit_offset[component]),
+		                               ConstantU32(ctx.state, info.component_bits[component])});
+	}
+	for (uint32_t component = info.component_count; component < 4u; component++) {
+		components[component] = components[component % info.component_count];
+	}
+
+	const auto swizzle = ctx.state.program.info.images[mem.resource].shader_swizzle;
+	uint32_t   selected[4] {};
+	for (uint32_t component = 0; component < 4u; component++) {
+		const auto selector = (swizzle >> (component * 3u)) & 7u;
+		if (selector == 1u) {
+			selected[component] = ConstantU32(ctx.state, 1u);
+		} else if (selector >= 4u) {
+			selected[component] = components[selector - 4u];
+		} else {
+			selected[component] = ConstantU32(ctx.state, 0u);
+		}
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               selected[0], selected[1], selected[2], selected[3]});
+	return result;
+}
+
+uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t gathered) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return gathered;
+
+	const auto component = ImageGatherComponent(mem.dmask);
+	const auto selector =
+	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	if (selector < 4u) {
+		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
+		return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4),
+		                                  {value, value, value, value});
+	}
+
+	uint32_t values[4] {};
+	for (uint32_t lane = 0; lane < 4u; lane++) {
+		const auto packed = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
+		values[lane]        = ctx.state.builder.AllocateId();
+		const auto physical = (selector - 4u) % info.component_count;
+		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed,
+		                               ConstantU32(ctx.state, info.component_bit_offset[physical]),
+		                               ConstantU32(ctx.state, info.component_bits[physical])});
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               values[0], values[1], values[2], values[3]});
+	return result;
+}
+
+uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+
+	auto packed = ConstantU32(ctx.state, 0u);
+	for (uint32_t component = 0; component < info.component_count; component++) {
+		const auto value = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
+		const auto maximum =
+		    ConstantU32(ctx.state, info.component_bits[component] == 32u
+		                               ? UINT32_MAX
+		                               : (1u << info.component_bits[component]) - 1u);
+		const auto within  = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum);
+		const auto clamped = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
+		const auto shifted =
+		    info.component_bit_offset[component] == 0u
+		        ? clamped
+		        : Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped,
+		                 ConstantU32(ctx.state, info.component_bit_offset[component]));
+		packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
+	}
+	const auto zero   = ConstantU32(ctx.state, 0u);
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(
+	    {OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
+	return result;
+}
+
 uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) {
-	const auto swizzle = ctx.state.program.info.images[mem.resource].storage_swizzle;
+	const auto swizzle = ctx.state.program.info.images[mem.resource].shader_swizzle;
 	uint32_t   values[4] {};
 	const auto dmask = mem.dmask != 0u ? mem.dmask : 1u;
 	for (uint32_t component = 0; component < 4u; component++) {
@@ -331,15 +481,27 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 			const auto packed_index = DmaskComponentIndex(dmask, source);
 			raw                     = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, ctx.state.uint_type, raw, data, packed_index});
+			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
+			if (mem.data_bits == 16u) {
+				if ((packed_index & 1u) != 0u) {
+					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					             ConstantU32(ctx.state, 16u));
+				}
+				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				             ConstantU32(ctx.state, 0xffffu));
+			}
 		}
-		values[component] = integer ? raw : Unary(ctx.state, OpBitcast, ctx.state.float_type, raw);
+		values[component] = integer ? raw
+		                    : mem.data_bits == 16u
+		                        ? EmitF16BitsToF32(ctx.state, raw)
+		                        : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
 	}
 	const auto texel = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct,
-	                               integer ? ctx.state.vec4_uint_type : ctx.state.vec4_float_type,
-	                               texel, values[0], values[1], values[2], values[3]});
-	return texel;
+	ctx.state.builder.AddFunction(
+	    {OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
+	     texel, values[0], values[1], values[2], values[3]});
+	return PackImageTexel(ctx, mem, texel);
 }
 
 uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
@@ -357,8 +519,9 @@ uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
 } // namespace
 
 bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
-	const auto op = inst.GetOpcode();
-	if (op < IR::ValueOpcode::ImageQueryDimensions || op > IR::ValueOpcode::ImageAtomicXor32) {
+	const auto op         = inst.GetOpcode();
+	const auto image_info = IR::ImageOpcodeInfoOf(op);
+	if (image_info.access == IR::ImageAccess::None) {
 		return false;
 	}
 	auto&       state     = ctx.state;
@@ -366,32 +529,30 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto  pc        = inst.Flags<IR::MemoryFlags>().pc;
 	const auto  image_arg = inst.Arg(0);
 	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
-	const bool  sampled = op == IR::ValueOpcode::ImageQueryLod ||
-	                      op == IR::ValueOpcode::ImageSampleRaw ||
-	                      op == IR::ValueOpcode::ImageGatherRaw;
-	const auto* address = ctx.ImageAddress(inst.Arg(sampled ? 2 : 1));
+	const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
 	if (address == nullptr) return true;
 	if (op == IR::ValueOpcode::ImageQueryDimensions) {
+		state.builder.RequireCapability(CapabilityImageQuery);
 		ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address));
 		return true;
 	}
 	if (op == IR::ValueOpcode::ImageQueryLod) {
+		state.builder.RequireCapability(CapabilityImageQuery);
 		const auto view    = SampledImageViewKind(state, mem, pc);
 		const auto sampled = MakeSampledImage(state, mem, pc, view);
 		const auto lod     = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpImageQueryLod, state.vec2_float_type, lod, sampled,
+		    {OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
 		     CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))});
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
 			const auto component = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpCompositeExtract, state.float_type, component, lod, index});
+			state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index});
 			values[index] = FloatBits(ctx, component);
 		}
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, values[0],
+		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0],
 		                           values[1], values[2], values[3]});
 		ctx.Define(inst, result);
 		return true;
@@ -403,28 +564,31 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(
 		    inst,
 		    EmitValueOrDefaultIfCondition(
-		        state, condition, state.vec4_uint_type, ZeroU32x4(state), [&]() {
+		        state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4),
+		        [&]() {
 			        const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
 			        const auto color = state.builder.AllocateId();
 			        const auto coord = CoordU32(ctx, mem, *address, view);
 			        if (ImageSpirvMultisampled(view) != 0u) {
 				        state.builder.AddFunction(
-				            {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
-				             color, image, coord, ImageOperandsSampleMask,
+				            {OpImageFetch,
+				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
+				             image, coord, ImageOperandsSampleMask,
 				             AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))});
 			        } else {
 				        state.builder.AddFunction(
-				            {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
-				             color, image, coord, ImageOperandsLodMask,
-				             LodU32(ctx, mem, *address, view)});
+				            {OpImageFetch,
+				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
+				             image, coord, ImageOperandsLodMask, LodU32(ctx, mem, *address, view)});
 			        }
-			        return ResultVector(ctx, color, integer, false);
+			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), integer, false,
+			                            mem);
 		        }));
 		return true;
 	}
 	if (op == IR::ValueOpcode::ImageWrite) {
 		const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint;
-		const auto view       = StorageImageViewKind(state, mem, uint_image, pc);
+		const auto view       = StorageImageViewKind(mem);
 		EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
 			const auto mip_lod =
 			    state.program.info.images[mem.resource].mip_mode == IR::ImageMipMode::DynamicStorage
@@ -432,7 +596,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			        : 0u;
 			const auto coord = CoordU32(ctx, mem, *address, view);
 			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
-			EmitStorageImageWrite(state, mem.resource, uint_image, view, mip_lod, coord, texel);
+			EmitStorageImageWrite(state, mem.resource, view, mip_lod, coord, texel);
 		});
 		return true;
 	}
@@ -441,6 +605,11 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto layout  = Layout(mem, view);
 		const bool integer = mem.kind == IR::ResourceKind::ImageUint;
 		const bool dref    = HasFlag(mem, Decoder::ImageSampleFlagCompare);
+		if (dref && state.program.info.images[mem.resource].conversion_format !=
+		                Prospero::BufferFormat::kInvalid) {
+			ctx.Fail(inst, "uses depth comparison with a packed integer image");
+			return true;
+		}
 		const auto coord =
 		    CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view));
 		if (op == IR::ValueOpcode::ImageGatherRaw) {
@@ -448,7 +617,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto            sample  = state.builder.AllocateId();
 			std::vector<uint32_t> words =
 			    dref ? std::vector<uint32_t> {OpImageDrefGather,
-			                                  state.vec4_float_type,
+			                                  TypeF32Vector(state, 4),
 			                                  sample,
 			                                  sampled,
 			                                  coord,
@@ -456,9 +625,15 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			                                      ? AddressF32(ctx, mem, *address, layout.dref)
 			                                      : ZeroF32(state)}
 			         : std::vector<uint32_t> {
-			               OpImageGather, integer ? state.vec4_uint_type : state.vec4_float_type,
-			               sample,        sampled,
-			               coord,         ConstantU32(state, ImageGatherComponent(mem.dmask))};
+			               OpImageGather,
+			               integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4),
+			               sample,
+			               sampled,
+			               coord,
+			               ConstantU32(state, ImageConversionFormat(state, mem).format ==
+			                                          Prospero::BufferFormat::kInvalid
+			                                      ? ImageGatherComponent(mem.dmask)
+			                                      : 0u)};
 			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
 				words.push_back(ImageOperandsConstOffsetsMask);
 				words.push_back(HorizontalOffsets(state, view));
@@ -467,7 +642,8 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				words.push_back(PackedOffset(ctx, mem, *address, layout, view));
 			}
 			state.builder.AddFunction(words);
-			ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false));
+			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
+			                              integer && !dref, false, mem, true));
 			return true;
 		}
 		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
@@ -478,7 +654,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		                        ? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod)
 		                        : (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod);
 		const auto result_type =
-		    dref ? state.float_type : (integer ? state.vec4_uint_type : state.vec4_float_type);
+		    dref ? TypeF32(state) : (integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4));
 		const auto dref_value =
 		    dref ? (layout.dref != NoImageComponent ? AddressF32(ctx, mem, *address, layout.dref)
 		                                            : ZeroF32(state))
@@ -517,7 +693,9 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		};
 		const auto& image = state.program.info.images[mem.resource];
 		if (image.indirect_root != mem.resource) {
-			ctx.Define(inst, ResultVector(ctx, EmitSample(mem.resource), integer, dref));
+			const auto sample = EmitSample(mem.resource);
+			ctx.Define(inst, ResultVector(ctx, dref ? sample : UnpackImageTexel(ctx, mem, sample),
+			                              integer, dref, mem));
 			return true;
 		}
 		const auto* handle = image_arg.ResolveInstruction();
@@ -537,10 +715,11 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		}
 		const auto LoadMapping = [&](uint32_t index) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-			                           state.flattened_srt_variable, ConstantU32(state, 0), index});
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state),
+			                           pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+			                           index});
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+			state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
 			return value;
 		};
 		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
@@ -549,39 +728,38 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		auto       selected = ConstantU32(state, 0u);
 		for (uint32_t iteration = 0; iteration < std::bit_width(image.indirect_mapping_capacity);
 		     iteration++) {
-			const auto active = Binary(state, OpULessThan, state.bool_type, low, high);
+			const auto active = Binary(state, OpULessThan, TypeBool(state), low, high);
 			const auto mid =
-			    Binary(state, OpShiftRightLogical, state.uint_type,
-			           Binary(state, OpIAdd, state.uint_type, low, high), ConstantU32(state, 1u));
+			    Binary(state, OpShiftRightLogical, TypeU32(state),
+			           Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u));
 			const auto probe = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, probe, active, mid, ConstantU32(state, 0u)});
-			const auto entry      = Binary(state, OpIAdd, state.uint_type, mapping,
-			                               Binary(state, OpIAdd, state.uint_type,
-			                                      Binary(state, OpShiftLeftLogical, state.uint_type,
+			    {OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
+			const auto entry      = Binary(state, OpIAdd, TypeU32(state), mapping,
+			                               Binary(state, OpIAdd, TypeU32(state),
+			                                      Binary(state, OpShiftLeftLogical, TypeU32(state),
 			                                             probe, ConstantU32(state, 1u)),
 			                                      ConstantU32(state, 1u)));
 			const auto mapped_key = LoadMapping(entry);
 			const auto candidate =
-			    LoadMapping(Binary(state, OpIAdd, state.uint_type, entry, ConstantU32(state, 1u)));
-			const auto equal         = Binary(state, OpIEqual, state.bool_type, mapped_key, key);
-			const auto match         = Binary(state, OpLogicalAnd, state.bool_type, active, equal);
+			    LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
+			const auto equal         = Binary(state, OpIEqual, TypeBool(state), mapped_key, key);
+			const auto match         = Binary(state, OpLogicalAnd, TypeBool(state), active, equal);
 			const auto next_selected = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, next_selected, match, candidate, selected});
+			    {OpSelect, TypeU32(state), next_selected, match, candidate, selected});
 			selected              = next_selected;
-			const auto less       = Binary(state, OpULessThan, state.bool_type, mapped_key, key);
-			const auto take_upper = Binary(state, OpLogicalAnd, state.bool_type, active, less);
-			const auto take_lower = Binary(state, OpLogicalAnd, state.bool_type, active,
-			                               Unary(state, OpLogicalNot, state.bool_type, less));
+			const auto less       = Binary(state, OpULessThan, TypeBool(state), mapped_key, key);
+			const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less);
+			const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active,
+			                               Unary(state, OpLogicalNot, TypeBool(state), less));
 			const auto next_low   = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, next_low, take_upper,
-			     Binary(state, OpIAdd, state.uint_type, mid, ConstantU32(state, 1u)), low});
+			    {OpSelect, TypeU32(state), next_low, take_upper,
+			     Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
 			low                  = next_low;
 			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, next_high, take_lower, mid, high});
+			state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high});
 			high = next_high;
 		}
 		const auto            default_label = state.builder.AllocateId();
@@ -596,32 +774,36 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 		state.builder.AddFunction(switch_words);
 		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
-		state.builder.AddFunction({OpLabel, default_label});
+		EmitLabel(state, default_label);
 		phi_words.push_back(EmitSample(image.indirect_resources[0]));
 		phi_words.push_back(default_label);
 		state.builder.AddFunction({OpBranch, merge_label});
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
-			state.builder.AddFunction({OpLabel, labels[candidate - 1u]});
+			EmitLabel(state, labels[candidate - 1u]);
 			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
 			phi_words.push_back(labels[candidate - 1u]);
 			state.builder.AddFunction({OpBranch, merge_label});
 		}
-		state.builder.AddFunction({OpLabel, merge_label});
+		EmitLabel(state, merge_label);
 		state.builder.AddFunction(phi_words);
-		ctx.Define(inst, ResultVector(ctx, phi_words[2], integer, dref));
+		ctx.Define(inst,
+		           ResultVector(ctx, dref ? phi_words[2] : UnpackImageTexel(ctx, mem, phi_words[2]),
+		                        integer, dref, mem));
 		return true;
 	}
 	const auto atomic_opcode = ImageAtomicOpcode(op);
 	if (atomic_opcode != 0u) {
-		const auto view = StorageImageViewKind(state, mem, true, pc);
+		const auto view = StorageImageViewKind(mem);
 		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
 			           const auto pointer = state.builder.AllocateId();
+			           const auto pointer_type =
+			               state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)});
 			           state.builder.AddFunction(
-			               {OpImageTexelPointer, state.ptr_image_uint, pointer,
-			                StorageImageDescriptorPointer(state, mem.resource, true, pc, view),
+			               {OpImageTexelPointer, pointer_type, pointer,
+			                StorageImageDescriptorPointer(state, mem.resource, view),
 			                CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)});
 			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction({atomic_opcode, state.uint_type, old, pointer,
+			           state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
 			                                      ConstantU32(state, ScopeDevice),
 			                                      ConstantU32(state, MemorySemanticsNone),
 			                                      ctx.Arg(inst, 2)});

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -6,7 +6,7 @@
 #include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
@@ -36,19 +36,25 @@ enum : uint32_t {
 	ExecutionModeSignedZeroInfNanPreserve    = 4461,
 	ExecutionModeDerivativeGroupQuadsKHR     = 5289,
 	AddressingModelLogical                   = 0,
+	AddressingModelPhysicalStorageBuffer64   = 5348,
 	MemoryModelGLSL450                       = 1,
 	CapabilityShader                         = 1,
+	CapabilityInt64                          = 11,
 	CapabilityImageGatherExtended            = 25,
+	CapabilityClipDistance                   = 32,
+	CapabilityCullDistance                   = 33,
 	CapabilitySampled1D                      = 43,
 	CapabilityImage1D                        = 44,
 	CapabilityImageQuery                     = 50,
-	CapabilityStorageImageReadWithoutFormat  = 55,
 	CapabilityStorageImageWriteWithoutFormat = 56,
 	CapabilityGroupNonUniform                = 61,
 	CapabilityGroupNonUniformBallot          = 64,
 	CapabilityGroupNonUniformShuffle         = 65,
 	CapabilitySignedZeroInfNanPreserve       = 4466,
+	CapabilityShaderViewportIndexLayerEXT    = 5254,
+	CapabilityFragmentBarycentricKHR         = 5284,
 	CapabilityComputeDerivativeGroupQuadsKHR = 5288,
+	CapabilityPhysicalStorageBufferAddresses = 5347,
 	StorageClassUniformConstant              = 0,
 	StorageClassInput                        = 1,
 	StorageClassOutput                       = 3,
@@ -57,6 +63,7 @@ enum : uint32_t {
 	StorageClassPushConstant                 = 9,
 	StorageClassImage                        = 11,
 	StorageClassStorageBuffer                = 12,
+	StorageClassPhysicalStorageBuffer        = 5349,
 	FunctionControlNone                      = 0,
 	SelectionControlNone                     = 0,
 	LoopControlNone                          = 0,
@@ -72,10 +79,15 @@ enum : uint32_t {
 	DecorationBinding       = 33,
 	DecorationDescriptorSet = 34,
 	DecorationOffset        = 35,
+	DecorationPerVertexKHR  = 5285,
 };
 
 enum : uint32_t {
 	BuiltInPosition                  = 0,
+	BuiltInPointSize                 = 1,
+	BuiltInClipDistance              = 3,
+	BuiltInCullDistance              = 4,
+	BuiltInLayer                     = 9,
 	BuiltInFragCoord                 = 15,
 	BuiltInFrontFacing               = 17,
 	BuiltInSampleMask                = 20,
@@ -87,6 +99,8 @@ enum : uint32_t {
 	BuiltInSubgroupLocalInvocationId = 41,
 	BuiltInVertexIndex               = 42,
 	BuiltInInstanceIndex             = 43,
+	BuiltInBaryCoordKHR              = 5286,
+	BuiltInBaryCoordNoPerspKHR       = 5287,
 };
 
 enum : uint32_t {
@@ -132,11 +146,15 @@ enum : uint32_t {
 	OpTypeStruct                   = 30,
 	OpTypePointer                  = 32,
 	OpTypeFunction                 = 33,
+	OpConstantTrue                 = 41,
+	OpConstantFalse                = 42,
 	OpConstant                     = 43,
 	OpConstantComposite            = 44,
 	OpUndef                        = 1,
 	OpFunction                     = 54,
+	OpFunctionParameter            = 55,
 	OpFunctionEnd                  = 56,
+	OpFunctionCall                 = 57,
 	OpVariable                     = 59,
 	OpImageTexelPointer            = 60,
 	OpLoad                         = 61,
@@ -166,6 +184,8 @@ enum : uint32_t {
 	OpConvertFToS                  = 110,
 	OpConvertSToF                  = 111,
 	OpConvertUToF                  = 112,
+	OpUConvert                     = 113,
+	OpConvertUToPtr                = 120,
 	OpBitcast                      = 124,
 	OpSNegate                      = 126,
 	OpFNegate                      = 127,
@@ -180,6 +200,8 @@ enum : uint32_t {
 	OpIAddCarry                    = 149,
 	OpUMulExtended                 = 151,
 	OpSMulExtended                 = 152,
+	OpAny                          = 154,
+	OpAll                          = 155,
 	OpLogicalNotEqual              = 165,
 	OpLogicalOr                    = 166,
 	OpLogicalAnd                   = 167,
@@ -242,40 +264,41 @@ enum : uint32_t {
 	OpSwitch                       = 251,
 	OpKill                         = 252,
 	OpReturn                       = 253,
+	OpReturnValue                  = 254,
 	OpGroupNonUniformBallot        = 339,
 	OpGroupNonUniformBallotFindLSB = 343,
 	OpGroupNonUniformShuffle       = 345,
 };
 
 enum : uint32_t {
-	GlslRoundEven      = 2,
-	GlslTrunc          = 3,
-	GlslFAbs           = 4,
-	GlslFloor          = 8,
-	GlslCeil           = 9,
-	GlslFract          = 10,
-	GlslSin            = 13,
-	GlslCos            = 14,
-	GlslExp2           = 29,
-	GlslLog2           = 30,
-	GlslSqrt           = 31,
-	GlslInverseSqrt    = 32,
-	GlslFMin           = 37,
-	GlslFMax           = 40,
-	GlslFClamp         = 43,
-	GlslLdexp          = 53,
-	GlslFma            = 50,
-	GlslPackSnorm2x16  = 56,
-	GlslPackUnorm2x16  = 57,
-	GlslPackHalf2x16   = 58,
-	GlslUnpackHalf2x16 = 62,
-	GlslFindILsb       = 73,
-	GlslFindUMsb       = 75,
+	MemoryAccessAlignedMask = 0x2,
 };
 
-struct RegisterBinding {
-	IR::Register reg;
-	uint32_t     pointer_id = 0;
+enum : uint32_t {
+	GlslRoundEven       = 2,
+	GlslTrunc           = 3,
+	GlslFAbs            = 4,
+	GlslFloor           = 8,
+	GlslCeil            = 9,
+	GlslFract           = 10,
+	GlslSin             = 13,
+	GlslCos             = 14,
+	GlslExp2            = 29,
+	GlslLog2            = 30,
+	GlslSqrt            = 31,
+	GlslInverseSqrt     = 32,
+	GlslFMin            = 37,
+	GlslFMax            = 40,
+	GlslFClamp          = 43,
+	GlslLdexp           = 53,
+	GlslFma             = 50,
+	GlslPackSnorm2x16   = 56,
+	GlslPackUnorm2x16   = 57,
+	GlslPackHalf2x16    = 58,
+	GlslUnpackUnorm2x16 = 61,
+	GlslUnpackHalf2x16  = 62,
+	GlslFindILsb        = 73,
+	GlslFindUMsb        = 75,
 };
 
 struct InputBinding {
@@ -284,6 +307,7 @@ struct InputBinding {
 	uint32_t           component_count = 1;
 	uint32_t           variable_id     = 0;
 	std::string        debug_name;
+	bool               per_vertex = false;
 };
 
 struct OutputBinding {
@@ -299,149 +323,108 @@ struct DescriptorResourceBinding {
 	uint32_t                     array_index = 0;
 };
 
-struct SampledImageDescriptors {
-	uint32_t image_type         = 0;
-	uint32_t sampled_image_type = 0;
-	uint32_t pointer_type       = 0;
-	uint32_t array_type         = 0;
-	uint32_t array_pointer_type = 0;
-	uint32_t variable           = 0;
+enum class ImageViewKind {
+	Dim1D,
+	Dim1DArray,
+	Dim2D,
+	Dim2DArray,
+	Dim3D,
+	Dim2DMsaa,
+	Dim2DMsaaArray,
+	Count,
 };
 
-struct StorageImageDescriptors {
-	uint32_t image_type         = 0;
-	uint32_t pointer_type       = 0;
-	uint32_t array_type         = 0;
-	uint32_t array_pointer_type = 0;
-	uint32_t variable           = 0;
+constexpr uint32_t SampledImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Count);
+constexpr uint32_t StorageImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Dim2DMsaa);
+
+enum class StorageImageClass : uint32_t {
+	FormatlessFloat,
+	FormatlessUint,
+	AtomicUint,
+	Count,
 };
 
 struct EmitterState {
 	EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_,
 	             ShaderStageInputInfo input_info_)
-	    : program(program_), resources(resources_), input_info(input_info_) {}
+	    : program(program_), resources(resources_), input_info(input_info_),
+	      requirements(*program_.spirv_requirements) {}
 
-	Builder                                          builder;
-	const IR::Program&                               program;
-	const IR::ResourceSnapshot&                      resources;
-	ShaderStageInputInfo                             input_info;
-	ShaderType                                       stage     = ShaderType::Unknown;
-	uint32_t                                         wave_size = 64;
-	bool                                             exact_subgroup_operations    = false;
-	bool                                             per_invocation_masks         = false;
-	uint32_t                                         void_type                    = 0;
-	uint32_t                                         bool_type                    = 0;
-	uint32_t                                         uint_type                    = 0;
-	uint32_t                                         uint_pair_type               = 0;
-	uint32_t                                         int_pair_type                = 0;
-	uint32_t                                         int_type                     = 0;
-	uint32_t                                         float_type                   = 0;
-	uint32_t                                         vec2_uint_type               = 0;
-	uint32_t                                         vec3_uint_type               = 0;
-	uint32_t                                         vec4_uint_type               = 0;
-	uint32_t                                         vec2_int_type                = 0;
-	uint32_t                                         vec3_int_type                = 0;
-	uint32_t                                         vec4_int_type                = 0;
-	uint32_t                                         vec2_float_type              = 0;
-	uint32_t                                         vec3_float_type              = 0;
-	uint32_t                                         vec4_float_type              = 0;
-	uint32_t                                         ptr_func_uint                = 0;
-	uint32_t                                         ptr_func_bool                = 0;
-	uint32_t                                         ptr_func_float               = 0;
-	uint32_t                                         ptr_func_uint_pair           = 0;
-	uint32_t                                         ptr_func_vec4_uint           = 0;
-	uint32_t                                         ptr_func_vec2_float          = 0;
-	uint32_t                                         ptr_input_float              = 0;
-	uint32_t                                         ptr_input_bool               = 0;
-	uint32_t                                         ptr_input_int                = 0;
-	uint32_t                                         ptr_input_uint               = 0;
-	uint32_t                                         ptr_input_vec2_float         = 0;
-	uint32_t                                         ptr_input_vec3_float         = 0;
-	uint32_t                                         ptr_input_vec2_int           = 0;
-	uint32_t                                         ptr_input_vec3_int           = 0;
-	uint32_t                                         ptr_input_vec4_int           = 0;
-	uint32_t                                         ptr_input_vec2_uint          = 0;
-	uint32_t                                         ptr_input_vec3_uint          = 0;
-	uint32_t                                         ptr_input_vec4_uint          = 0;
-	uint32_t                                         ptr_input_vec4_float         = 0;
-	uint32_t                                         sample_mask_array_type       = 0;
-	uint32_t                                         ptr_output_int               = 0;
-	uint32_t                                         ptr_output_sample_mask_array = 0;
-	uint32_t                                         ptr_output_float             = 0;
-	uint32_t                                         ptr_output_vec4_float        = 0;
-	uint32_t                                         per_vertex_type              = 0;
-	uint32_t                                         ptr_output_per_vertex        = 0;
-	uint32_t                                         storage_runtime_array_type   = 0;
-	uint32_t                                         storage_buffer_type          = 0;
-	uint32_t                                         ptr_storage_buffer           = 0;
-	uint32_t                                         ptr_storage_buffer_uint      = 0;
-	uint32_t                                         storage_buffer_array_type    = 0;
-	uint32_t                                         ptr_storage_buffer_array     = 0;
-	uint32_t                                         storage_buffer_variable      = 0;
-	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
-	uint32_t                                         address_memory_array_type = 0;
-	uint32_t                                         ptr_address_memory_array  = 0;
-	uint32_t                                         address_memory_variable   = 0;
-	uint32_t                                         gds_variable              = 0;
-	uint32_t                                         push_constant_array_type  = 0;
-	uint32_t                                         push_constant_block_type  = 0;
-	uint32_t                                         ptr_push_constant_block   = 0;
-	uint32_t                                         ptr_push_constant_uint    = 0;
-	uint32_t                                         push_constant_variable    = 0;
-	uint32_t                                         vsharp_storage_variable   = 0;
-	uint32_t                                         flattened_srt_variable    = 0;
-	uint32_t                                         lds_array_type            = 0;
-	uint32_t                                         ptr_workgroup_array       = 0;
-	uint32_t                                         ptr_workgroup_uint        = 0;
-	uint32_t                                         lds_variable              = 0;
-	std::array<SampledImageDescriptors, 14>          sampled_images;
-	std::array<StorageImageDescriptors, 10>          storage_images;
-	uint32_t                                         sampler_type                          = 0;
-	uint32_t                                         sampler_array_type                    = 0;
-	uint32_t                                         ptr_uniform_sampler                   = 0;
-	uint32_t                                         ptr_uniform_sampler_array             = 0;
-	uint32_t                                         sampler_variable                      = 0;
-	uint32_t                                         ptr_image_uint                        = 0;
-	uint32_t                                         func_type                             = 0;
-	uint32_t                                         main_func                             = 0;
-	uint32_t                                         entry_label                           = 0;
-	uint32_t                                         pixel_valid_mask_variable             = 0;
-	bool                                             dispatcher_fallback                   = false;
-	uint32_t                                         dispatch_pc_variable                  = 0;
-	uint32_t                                         dispatch_header_label                 = 0;
-	uint32_t                                         dispatch_select_label                 = 0;
-	uint32_t                                         dispatch_default_label                = 0;
-	uint32_t                                         dispatch_after_switch_label           = 0;
-	uint32_t                                         dispatch_continue_label               = 0;
-	uint32_t                                         dispatch_merge_label                  = 0;
-	uint32_t                                         glsl_std450                           = 0;
-	uint32_t                                         subgroup_local_invocation_id_variable = 0;
-	uint32_t                                         per_vertex_variable                   = 0;
-	uint32_t                                         depth_variable                        = 0;
-	uint32_t                                         sample_mask_variable                  = 0;
-	bool                                             needs_subgroup_ballot                 = false;
-	bool                                             needs_subgroup_shuffle                = false;
-	bool                                             needs_subgroup_local_invocation_id    = false;
-	bool                                             needs_compute_derivatives             = false;
-	bool                                             needs_image_gather_extended           = false;
-	bool                                             needs_function_lds                    = false;
-	bool                                             needs_pixel_valid_mask                = false;
-	bool                                             unsupported_ir_instruction            = false;
-	IR::Opcode                                       unsupported_ir_opcode = IR::Opcode::Count;
-	uint32_t                                         unsupported_ir_pc     = 0;
-	std::vector<RegisterBinding>                     registers;
-	std::vector<InputBinding>                        inputs;
-	std::vector<OutputBinding>                       outputs;
-	std::vector<uint32_t>                            interface_variables;
-	std::vector<bool>                                reachable_blocks;
-	std::map<uint32_t, uint32_t>                     block_labels;
-	std::map<uint32_t, uint32_t>                     constants;
-	std::map<uint32_t, uint32_t>                     signed_constants;
-	std::map<uint32_t, uint32_t>                     float_constants;
+	Builder                      builder;
+	const IR::Program&           program;
+	const IR::ResourceSnapshot&  resources;
+	ShaderStageInputInfo         input_info;
+	const IR::SpirvRequirements& requirements;
+	ShaderType                   stage                   = ShaderType::Unknown;
+	uint32_t                     wave_size               = 64;
+	uint32_t                     storage_buffer_variable = 0;
+	std::array<uint32_t, IR::ShaderInfo::MaxBuffers>      memory_byte_offsets {};
+	uint32_t                                             bda_pagetable_variable = 0;
+	uint32_t                                             fault_buffer_variable  = 0;
+	uint32_t                                             bda_pointer_function   = 0;
+	uint32_t                                             gds_variable            = 0;
+	uint32_t                                             gds_length              = 0;
+	uint32_t                                             push_constant_variable  = 0;
+	uint32_t                                             vsharp_storage_variable = 0;
+	uint32_t                                             flattened_srt_variable  = 0;
+	uint32_t                                             lds_variable            = 0;
+	uint32_t                                             scratch_variable        = 0;
+	std::array<uint32_t, 2u * SampledImageViewKindCount> sampled_image_variables {};
+	std::array<uint32_t,
+	           static_cast<uint32_t>(StorageImageClass::Count) * StorageImageViewKindCount>
+	                           storage_image_variables {};
+	uint32_t                   sampler_variable                      = 0;
+	uint32_t                   main_func                             = 0;
+	uint32_t                   entry_label                           = 0;
+	uint32_t                   current_label                         = 0;
+	uint32_t                   pixel_valid_mask_variable             = 0;
+	uint32_t                   subgroup_local_invocation_id_variable = 0;
+	uint32_t                   per_vertex_variable                   = 0;
+	uint32_t                   point_size_variable                   = 0;
+	uint32_t                   clip_distance_variable                = 0;
+	uint32_t                   cull_distance_variable                = 0;
+	uint32_t                   layer_variable                        = 0;
+	uint32_t                   clip_distance_count                   = 0;
+	uint32_t                   cull_distance_count                   = 0;
+	uint32_t                   depth_variable                        = 0;
+	uint32_t                   sample_mask_variable                  = 0;
+	std::vector<InputBinding>  inputs;
+	std::vector<OutputBinding> outputs;
+	std::vector<uint32_t>      interface_variables;
 };
 
+uint32_t TypeVoid(EmitterState& state);
+uint32_t TypeBool(EmitterState& state);
+uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
+uint32_t TypeU32(EmitterState& state);
+uint32_t TypeU64(EmitterState& state);
+uint32_t TypeDeviceAddress(EmitterState& state);
+uint32_t TypeU32Pair(EmitterState& state);
+uint32_t TypeI32(EmitterState& state);
+uint32_t TypeI32Pair(EmitterState& state);
+uint32_t TypeF32(EmitterState& state);
+uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
+
+uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
+uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
+uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
+uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee);
+uint32_t TypeFunction(EmitterState& state);
+uint32_t TypeStorageBufferPointer(EmitterState& state);
+uint32_t TypeStorageBufferElementPointer(EmitterState& state);
+uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
+uint32_t TypePhysicalU32Pointer(EmitterState& state);
+uint32_t TypePushConstantElementPointer(EmitterState& state);
+uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
+uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
+
+inline void EmitLabel(EmitterState& state, uint32_t label) {
+	state.current_label = label;
+	state.builder.AddFunction({OpLabel, label});
+}
+
 struct ValueEmitContext {
-	ValueEmitContext(EmitterState& state_, const IR::ValueProgram& program_)
+	ValueEmitContext(EmitterState& state_, const IR::Program& program_)
 	    : state(state_), program(program_) {}
 
 	uint32_t              Def(IR::Value value);
@@ -456,19 +439,18 @@ struct ValueEmitContext {
 	const IR::MemoryInfo& Memory(const IR::Inst& inst) const;
 	const IR::ExportInfo& Export(const IR::Inst& inst) const;
 	uint32_t              Label(const IR::Block* block) const;
-	size_t                BlockIndex(const IR::Block* block) const;
 	void                  Fail(const IR::Inst& inst, const char* reason);
 
-	EmitterState&                                  state;
-	const IR::ValueProgram&                        program;
-	std::unordered_map<const IR::Inst*, uint32_t>  definitions;
-	std::unordered_map<const IR::Block*, uint32_t> labels;
-	std::unordered_map<const IR::Inst*, uint32_t>  phi_variables;
-	std::unordered_map<const IR::Inst*, uint32_t>  dispatcher_variables;
-	const IR::Block*                               current_block        = nullptr;
-	uint32_t                                       scratch_u32_variable = 0;
-	bool                                           failed               = false;
-	std::string                                    error;
+	EmitterState&                                                      state;
+	const IR::Program&                                            program;
+	std::unordered_map<const IR::Inst*, uint32_t>                      definitions;
+	std::unordered_map<const IR::Block*, uint32_t>                     labels;
+	const std::unordered_map<const IR::Inst*, uint32_t>*               dispatcher_spills = nullptr;
+	std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
+	const IR::Block*                                                   current_block = nullptr;
+	uint32_t                                                           scratch_u32_variable = 0;
+	bool                                                               failed               = false;
+	std::string                                                        error;
 };
 
 enum class VertexInputScalarKind { Float, Sint, Uint };
@@ -490,26 +472,23 @@ struct ImageSampleLayout {
 	uint32_t grad_y = NoImageComponent;
 };
 
-enum class ImageViewKind {
-	Dim1D,
-	Dim1DArray,
-	Dim2D,
-	Dim2DArray,
-	Dim3D,
-	Dim2DMsaa,
-	Dim2DMsaaArray,
-	Count,
-};
-
-constexpr uint32_t SampledImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Count);
-constexpr uint32_t StorageImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Dim2DMsaa);
-
 constexpr uint32_t SampledImageIndex(bool integer, ImageViewKind view) {
 	return static_cast<uint32_t>(view) + (integer ? SampledImageViewKindCount : 0u);
 }
 
-constexpr uint32_t StorageImageIndex(bool integer, ImageViewKind view) {
-	return static_cast<uint32_t>(view) + (integer ? StorageImageViewKindCount : 0u);
+constexpr StorageImageClass StorageImageClassFor(bool uint_image, bool atomic) {
+	if (atomic) {
+		return StorageImageClass::AtomicUint;
+	}
+	if (uint_image) {
+		return StorageImageClass::FormatlessUint;
+	}
+	return StorageImageClass::FormatlessFloat;
+}
+
+constexpr uint32_t StorageImageIndex(StorageImageClass image_class, ImageViewKind view) {
+	return static_cast<uint32_t>(view) +
+	       static_cast<uint32_t>(image_class) * StorageImageViewKindCount;
 }
 
 constexpr IR::DescriptorBindingKind SampledBindingKind(bool integer, ImageViewKind view) {
@@ -539,24 +518,44 @@ constexpr IR::DescriptorBindingKind SampledBindingKind(bool integer, ImageViewKi
 	return IR::DescriptorBindingKind::Count;
 }
 
-constexpr IR::DescriptorBindingKind StorageBindingKind(bool integer, ImageViewKind view) {
-	if (integer) {
-		switch (view) {
-			case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageUint1D;
-			case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::StorageUint1DArray;
-			case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageUint2D;
-			case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::StorageUint2DArray;
-			case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
-			default: break;
-		}
-	}
-	switch (view) {
-		case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::Storage1D;
-		case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::Storage1DArray;
-		case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::Storage2D;
-		case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
-		case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
-		default: break;
+constexpr IR::DescriptorBindingKind StorageBindingKind(StorageImageClass image_class,
+                                                       ImageViewKind     view) {
+	switch (image_class) {
+		case StorageImageClass::FormatlessFloat:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::Storage1D;
+				case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::Storage1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::Storage2D;
+				case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::FormatlessUint:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageUint1D;
+				case ImageViewKind::Dim1DArray:
+					return IR::DescriptorBindingKind::StorageUint1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageUint2D;
+				case ImageViewKind::Dim2DArray:
+					return IR::DescriptorBindingKind::StorageUint2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::AtomicUint:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageAtomic1D;
+				case ImageViewKind::Dim1DArray:
+					return IR::DescriptorBindingKind::StorageAtomic1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageAtomic2D;
+				case ImageViewKind::Dim2DArray:
+					return IR::DescriptorBindingKind::StorageAtomic2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageAtomic3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::Count: break;
 	}
 	return IR::DescriptorBindingKind::Count;
 }
@@ -586,17 +585,10 @@ constexpr uint32_t ImageSpirvMultisampled(ImageViewKind view) {
 	return view == ImageViewKind::Dim2DMsaa || view == ImageViewKind::Dim2DMsaaArray ? 1u : 0u;
 }
 
-struct AddCarryResult {
-	uint32_t sum   = 0;
-	uint32_t carry = 0;
-};
-
 struct F32Class {
-	uint32_t bits       = 0;
-	uint32_t nan        = 0;
-	uint32_t snan       = 0;
-	uint32_t zero       = 0;
-	uint32_t quiet_bits = 0;
+	uint32_t bits = 0;
+	uint32_t nan  = 0;
+	uint32_t zero = 0;
 };
 
 uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr);
@@ -605,48 +597,30 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr);
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr);
 
+bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr);
+
 VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location);
 
 uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input);
 
-uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind);
+uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind);
 
-uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind);
+uint32_t VertexParameterScalarPointerType(EmitterState& state, VertexInputScalarKind kind);
 
-uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind,
+uint32_t VertexParameterVectorOrScalarType(EmitterState& state, VertexInputScalarKind kind,
                                            uint32_t components);
 
-uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind,
+uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
                                          uint32_t components);
 
 void SetError(std::string* error, const char* message);
 
-void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg);
-
-bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg);
-
 bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
 
 void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
 
-bool ProgramNeedsComputeDerivatives(const IR::Program& program);
-
-bool ProgramNeedsImageGatherExtended(const IR::Program& program);
-
-bool ProgramNeedsFunctionLds(const IR::Program& program);
-
-bool ProgramNeedsPixelValidMask(const IR::Program& program);
-
-bool ProgramNeedsSubgroupBallot(const IR::Program& program);
-
-bool ProgramNeedsSubgroupShuffle(const IR::Program& program);
-
-bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program);
-
-uint32_t PointerForRegister(const EmitterState& state, IR::Register reg);
-
 uint32_t ConstantU32(EmitterState& state, uint32_t value);
 
 uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
@@ -670,11 +644,16 @@ uint32_t ImageViewCoordinateComponents(ImageViewKind view);
 
 uint32_t ImageViewSpatialComponents(ImageViewKind view);
 
-uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer);
+uint32_t ImageViewImageType(EmitterState& state, ImageViewKind view, bool integer);
 
-uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer);
+uint32_t ImageViewSampledImageType(EmitterState& state, ImageViewKind view, bool integer);
 
-uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view);
+uint32_t ImageViewSizeType(EmitterState& state, ImageViewKind view);
+
+uint32_t StorageImageType(EmitterState& state, StorageImageClass image_class, ImageViewKind view);
+
+uint32_t StorageImagePointerType(EmitterState& state, StorageImageClass image_class,
+                                 ImageViewKind view);
 
 uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
                                     ImageViewKind view);
@@ -686,18 +665,12 @@ uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32
 uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
                           ImageViewKind view, uint32_t image_resource);
 
-ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   bool uint_image, uint32_t use_pc);
+ImageViewKind StorageImageViewKind(const IR::MemoryInfo& mem);
 
-uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
-                                       uint32_t      use_pc = UINT32_MAX,
-                                       ImageViewKind view   = ImageViewKind::Dim2D);
+uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, ImageViewKind view);
 
-uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
-                                    uint32_t use_pc, ImageViewKind view = ImageViewKind::Dim2D);
-
-void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_image,
-                           ImageViewKind view, uint32_t mip_lod, uint32_t coord, uint32_t texel);
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, ImageViewKind view,
+                           uint32_t mip_lod, uint32_t coord, uint32_t texel);
 
 uint32_t ExecutionModelForStage(ShaderType stage);
 
@@ -711,6 +684,14 @@ uint32_t FloatBits(float value);
 
 uint32_t ConstantF32Value(EmitterState& state, float value);
 
+uint32_t ConstantBool(EmitterState& state, bool value);
+
+uint32_t ConstantU64(EmitterState& state, uint64_t value);
+
+uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
+
+uint32_t GlslStd450(EmitterState& state);
+
 void AllocateInputVariables(EmitterState& state);
 
 void AllocateOutputVariables(EmitterState& state);
@@ -721,16 +702,12 @@ void AddInputAnnotationsAndNames(EmitterState& state);
 
 void AddOutputAnnotationsAndNames(EmitterState& state);
 
-void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, uint32_t set,
-                        uint32_t binding);
+void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name,
+                        IR::DescriptorBindingKind kind);
 
 void AddDescriptorAnnotationsAndNames(EmitterState& state);
 
-void EmitHeaderAndTypes(EmitterState& state);
-
-void AllocateRegisterVariables(EmitterState& state);
-
-void AllocateDescriptorVariables(EmitterState& state);
+void DefineModule(EmitterState& state);
 
 uint32_t EmitTrueBool(EmitterState& state);
 
@@ -768,40 +745,32 @@ uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint3
 
 uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
 
-bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index);
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
 
-uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   uint32_t use_pc);
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
 
-Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t use_pc);
-
-void EmitStorageBufferOffsets(EmitterState& state);
-
-DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState&         state,
-                                                        const IR::MemoryInfo& mem, uint32_t use_pc);
-
-uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                        uint32_t use_pc);
-
-uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc);
-
-uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                         uint32_t index, uint32_t use_pc);
-
-uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index);
+void EmitMemoryOffsets(EmitterState& state);
 
 uint32_t LdsDwordCount(const EmitterState& state);
 
-uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index);
+struct MemoryResourceAccess {
+	IR::ResourceKind kind             = IR::ResourceKind::None;
+	uint32_t         object_pointer   = 0;
+	uint32_t         length           = 0;
+	uint32_t         index_offset     = 0;
+	bool             add_index_offset = false;
+};
 
-uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index);
+MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
-uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index);
+uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
+                                uint32_t raw_index);
 
-uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
-                                  uint32_t use_pc);
+uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
+                                   uint32_t index);
+
+uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                  uint32_t index);
 
 uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value);
 
@@ -831,8 +800,6 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
                             uint32_t false_value);
 
-uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
-
 void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
                                    uint32_t& out_low, uint32_t& out_high);
 
@@ -841,9 +808,6 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
 
-AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
-                                  uint32_t carry_in);
-
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
 
 uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
@@ -892,8 +856,6 @@ uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 
-uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg);
-
 bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
 
 bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -902,47 +864,40 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
 
 bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
 
-bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error);
+bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error);
+
+void DefineGetBdaPointer(EmitterState& state);
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
 void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
-	if (condition == 0) {
-		fn();
-		return;
-	}
-
 	const auto then_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label});
-	state.builder.AddFunction({OpLabel, then_label});
+	EmitLabel(state, then_label);
 	fn();
 	state.builder.AddFunction({OpBranch, merge_label});
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 }
 
 template <typename Fn>
 uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
                                        uint32_t default_value, Fn&& fn) {
-	if (condition == 0) {
-		return fn();
-	}
-
 	const auto then_label  = state.builder.AllocateId();
 	const auto then_exit   = state.builder.AllocateId();
 	const auto else_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
-	state.builder.AddFunction({OpLabel, then_label});
+	EmitLabel(state, then_label);
 	const auto then_value = fn();
 	state.builder.AddFunction({OpBranch, then_exit});
-	state.builder.AddFunction({OpLabel, then_exit});
+	EmitLabel(state, then_exit);
 	state.builder.AddFunction({OpBranch, merge_label});
-	state.builder.AddFunction({OpLabel, else_label});
+	EmitLabel(state, else_label);
 	state.builder.AddFunction({OpBranch, merge_label});
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(
 	    {OpPhi, type, value, then_value, then_exit, default_value, else_label});
@@ -951,7 +906,7 @@ uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition,
 
 template <typename Fn>
 uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
-	return EmitValueOrDefaultIfCondition(state, condition, state.uint_type, ConstantU32(state, 0),
+	return EmitValueOrDefaultIfCondition(state, condition, TypeU32(state), ConstantU32(state, 0),
 	                                     std::forward<Fn>(fn));
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -2,21 +2,41 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
+uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
+	if (state.push_constant_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer,
+		                           state.push_constant_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		return value;
+	}
+	if (state.vsharp_storage_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+		                           state.vsharp_storage_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		return value;
+	}
+	return ConstantU32(state, 0);
+}
+
 uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpIAdd, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
-uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   uint32_t use_pc) {
-	(void)use_pc;
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
 		                             "buffer specialization is missing");
@@ -24,9 +44,7 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn
 	return state.program.info.buffers[mem.resource].packed_stride;
 }
 
-Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t use_pc) {
-	(void)use_pc;
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
 		                             "buffer specialization is missing");
@@ -34,160 +52,148 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
 	return state.program.info.buffers[mem.resource].descriptor_format;
 }
 
-void EmitStorageBufferOffsets(EmitterState& state) {
-	for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
+void EmitMemoryOffsets(EmitterState& state) {
+	for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
 		const auto word =
-		    EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
-		const auto shift                = ConstantU32(state, (i % 4u) * 8u + 2u);
-		state.storage_buffer_offsets[i] = EmitBinaryU32(
+		    EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
+		const auto shift             = ConstantU32(state, (i % 4u) * 8u);
+		state.memory_byte_offsets[i] = EmitBinaryU32(
 		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
-		    ConstantU32(state, 0x3fu));
+		    ConstantU32(state, 0xffu));
 	}
 }
 
-bool IsAddressMemoryKind(IR::ResourceKind kind) {
-	return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat ||
-	       kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch;
-}
-
-DescriptorResourceBinding
-StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) {
-	(void)use_pc;
-	const auto binding =
-	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
-	if (state.storage_buffer_variable == 0) {
-		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
-		                             "storage buffer descriptor array was not emitted");
-	}
-	return binding;
-}
-
-uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
-                                uint32_t use_pc) {
-	if (IsAddressMemoryKind(mem.kind)) {
-		return index;
-	}
-	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
-	return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]);
-}
-
-uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                        uint32_t use_pc) {
-	if (IsAddressMemoryKind(mem.kind)) {
-		if (state.address_memory_variable == 0) {
-			ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
-			                             mem.resource, "address memory binding was not emitted");
-		}
-		const auto binding =
-		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
-		const auto pointer = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-		                           state.address_memory_variable,
-		                           ConstantU32(state, binding.array_index)});
-		return pointer;
-	}
-	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-	                           state.storage_buffer_variable,
-	                           ConstantU32(state, binding.array_index)});
-	return pointer;
-}
-
-uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc) {
-	index                = EmitStorageBufferIndex(state, mem, index, use_pc);
-	const auto object    = EmitStorageBufferObjectPointer(state, mem, use_pc);
-	const auto length    = state.builder.AllocateId();
-	const auto in_bounds = state.builder.AllocateId();
-	state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
-	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
-	return in_bounds;
-}
-
-uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                         uint32_t index, uint32_t use_pc) {
-	index = EmitStorageBufferIndex(state, mem, index, use_pc);
-	if (IsAddressMemoryKind(mem.kind)) {
-		if (state.address_memory_variable == 0) {
-			ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
-			                             mem.resource, "address memory binding was not emitted");
-		}
-		const auto binding =
-		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
-		const auto pointer = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable,
-		     ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
-		return pointer;
-	}
-	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable,
-	     ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
-	return pointer;
-}
-
-uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) {
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index});
-	return pointer;
-}
-
 uint32_t LdsDwordCount(const EmitterState& state) {
 	return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
 }
 
-uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) {
-	const auto in_bounds = state.builder.AllocateId();
+static void EnsureLdsStorage(EmitterState& state) {
+	if (state.lds_variable != 0) {
+		return;
+	}
+	if (state.stage != ShaderType::Compute) {
+		EXIT("function LDS was not prepared before SPIR-V function emission\n");
+	}
+	state.lds_variable = state.builder.DefineGlobalVariable(
+	    TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)),
+	    StorageClassWorkgroup);
+	state.builder.AddName(state.lds_variable, "lds_dwords");
+}
+
+MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) {
+	MemoryResourceAccess access {.kind = mem.kind};
+	switch (mem.kind) {
+		case IR::ResourceKind::Lds:
+			EnsureLdsStorage(state);
+			access.object_pointer = state.lds_variable;
+			access.length         = ConstantU32(state, LdsDwordCount(state));
+			return access;
+		case IR::ResourceKind::Gds:
+			if (state.gds_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Gds, mem.resource,
+				                             "GDS binding was not emitted");
+			}
+			access.object_pointer = state.gds_variable;
+			if (state.gds_length == 0) {
+				EXIT("GDS length was not prepared at function entry\n");
+			}
+			access.length = state.gds_length;
+			return access;
+		case IR::ResourceKind::Scratch:
+			if (state.scratch_variable == 0) {
+				EXIT("scratch storage was not prepared before SPIR-V function emission\n");
+			}
+			access.object_pointer = state.scratch_variable;
+			access.length         = ConstantU32(state, state.program.scratch_dwords);
+			return access;
+		case IR::ResourceKind::ScalarAddress:
+		case IR::ResourceKind::Flat:
+		case IR::ResourceKind::Global:
+			EXIT("physical address memory must use the BDA emitter\n");
+		case IR::ResourceKind::ScalarBuffer:
+		case IR::ResourceKind::Buffer: {
+			if (state.storage_buffer_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers,
+				                             mem.resource,
+				                             "storage buffer descriptor array was not emitted");
+			}
+			const auto binding =
+			    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
+			access.object_pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
+			                           access.object_pointer, state.storage_buffer_variable,
+			                           ConstantU32(state, binding.array_index)});
+			access.index_offset     = EmitBinaryU32(state, OpShiftRightLogical,
+			                                        state.memory_byte_offsets[binding.array_index],
+			                                        ConstantU32(state, 2u));
+			access.add_index_offset = true;
+			break;
+		}
+		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
+	}
+	access.length = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))});
+	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	return access;
+}
+
+uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
+                                uint32_t raw_index) {
+	return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
+}
+
+uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
+                                   uint32_t index) {
+	const auto in_bounds = state.builder.AllocateId();
+	state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length});
 	return in_bounds;
 }
 
-uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
-                                  uint32_t use_pc) {
-	if (mem.kind == IR::ResourceKind::Lds) {
-		return EmitLdsElementPointer(state, index);
+uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                  uint32_t index) {
+	const auto pointer = state.builder.AllocateId();
+	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
+		const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
+		                           : state.stage == ShaderType::Compute     ? StorageClassWorkgroup
+		                                                                    : StorageClassFunction;
+		state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
+		                           pointer, access.object_pointer, index});
+	} else {
+		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+		                           access.object_pointer, ConstantU32(state, 0), index});
 	}
-	if (mem.kind == IR::ResourceKind::Gds) {
-		return EmitGdsElementPointer(state, index);
-	}
-	return EmitStorageBufferElementPointer(state, mem, index, use_pc);
+	return pointer;
 }
 
 uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.int_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeI32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                        uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
 uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
                               uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpSelect, state.float_type, ret, condition, true_value, false_value});
+	state.builder.AddFunction({OpSelect, TypeF32(state), ret, condition, true_value, false_value});
 	return ret;
 }
 
@@ -200,8 +206,8 @@ uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto value    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw});
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0});
+	    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), GlslUnpackHalf2x16, raw});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), value, unpacked, 0});
 	return EmitTBufferBitcastF32ToU32(state, value);
 }
 
@@ -229,9 +235,9 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
 
 	const auto mantissa_f32 = state.builder.AllocateId();
 	const auto subnormal    = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa});
+	state.builder.AddFunction({OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
 	state.builder.AddFunction(
-	    {OpFMul, state.float_type, subnormal, mantissa_f32,
+	    {OpFMul, TypeF32(state), subnormal, mantissa_f32,
 	     ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
 
 	const auto zero_exp    = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0);
@@ -249,22 +255,22 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 		case Format::ComponentType::Sint: return raw;
 		case Format::ComponentType::Uscaled: {
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
+			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
 			return EmitTBufferBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Sscaled: {
 			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
 			const auto value      = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
+			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
 			return EmitTBufferBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Unorm: {
 			const auto value      = state.builder.AllocateId();
 			const auto normalized = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << bits) - 1u);
-			state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
+			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
 			state.builder.AddFunction(
-			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
+			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
 			return EmitTBufferBitcastF32ToU32(state, normalized);
 		}
 		case Format::ComponentType::Snorm: {
@@ -273,10 +279,10 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			const auto normalized = state.builder.AllocateId();
 			const auto clamped    = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << (bits - 1u)) - 1u);
-			state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
+			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
 			state.builder.AddFunction(
-			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
-			state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
+			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
+			state.builder.AddFunction({OpExtInst, TypeF32(state), clamped, GlslStd450(state),
 			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
 			return EmitTBufferBitcastF32ToU32(state, clamped);
 		}
@@ -298,21 +304,6 @@ void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
 	    {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
 }
 
-uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
-	const auto length    = state.builder.AllocateId();
-	const auto in_bounds = state.builder.AllocateId();
-	state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0});
-	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
-	return in_bounds;
-}
-
-uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                           state.gds_variable, ConstantU32(state, 0), index});
-	return pointer;
-}
-
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
 	if ((control & 0xc000u) == 0xc000u) {
 		const uint32_t mask         = control & 0x1fu;
@@ -336,16 +327,16 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 		const auto base   = state.builder.AllocateId();
 		const auto target = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)});
+		    {OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
 		state.builder.AddFunction(
-		    {OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)});
+		    {OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
 		state.builder.AddFunction(
-		    {OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift});
+		    {OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)});
+		    {OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm});
+		    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
+		state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, perm});
 		return target;
 	}
 
@@ -355,16 +346,16 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
+	    {OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
+	    {OpBitwiseOr, TypeU32(state), ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
+	    {OpBitwiseXor, TypeU32(state), xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
+	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, xored});
 	return target;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -4,6 +4,7 @@
 #include <bit>
 #include <fmt/format.h>
 #include <functional>
+#include <optional>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -13,12 +14,12 @@ void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
 	const auto kill_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	const auto inactive    = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
+	state.builder.AddFunction({OpLogicalNot, TypeBool(state), inactive, active});
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
-	state.builder.AddFunction({OpLabel, kill_label});
+	EmitLabel(state, kill_label);
 	state.builder.AddFunction({OpKill});
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 }
 
 void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
@@ -29,63 +30,58 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
 	const auto mask_value = state.builder.AllocateId();
 	const auto active     = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
+	    {OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
 	EmitKillIfBoolFalse(state, active);
 }
 
-uint32_t BoolConstant(EmitterState& state, bool value) {
-	const auto id = state.builder.AllocateId();
-	state.builder.AddType({value ? 41u : 42u, state.bool_type, id});
-	return id;
+uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
+	const auto value_type = ctx.TypeId(type);
+	return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type);
 }
 
-uint32_t ConstantPair(EmitterState& state, uint64_t value) {
-	const auto id = state.builder.AllocateId();
-	state.builder.AddType({OpConstantComposite, state.uint_pair_type, id,
-	                       ConstantU32(state, static_cast<uint32_t>(value)),
-	                       ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
-	return id;
-}
+struct DeferredPhiPatch {
+	DeferredPhi     phi;
+	const IR::Inst* instruction = nullptr;
+};
 
-uint32_t PhiPointerType(const EmitterState& state, IR::Type type) {
-	switch (type) {
-		case IR::Type::U1: return state.ptr_func_bool;
-		case IR::Type::U8:
-		case IR::Type::U16:
-		case IR::Type::U32:
-		case IR::Type::F16: return state.ptr_func_uint;
-		case IR::Type::F32: return state.ptr_func_float;
-		case IR::Type::U64:
-		case IR::Type::U32x2: return state.ptr_func_uint_pair;
-		case IR::Type::U32x4: return state.ptr_func_vec4_uint;
-		case IR::Type::F32x2: return state.ptr_func_vec2_float;
-		default: return 0;
-	}
-}
+struct StructuredFunctionState {
+	std::unordered_map<const IR::Block*, uint32_t> block_exit_labels;
+	std::vector<DeferredPhiPatch>                  deferred_phis;
+};
 
-void StorePhiEdge(ValueEmitContext& ctx, const IR::Block* from, const IR::Block* to) {
+struct DispatcherFunctionState {
+	std::unordered_map<const IR::Inst*, uint32_t> spills;
+	uint32_t                                      header_label       = 0;
+	uint32_t                                      select_label       = 0;
+	uint32_t                                      after_switch_label = 0;
+	uint32_t                                      continue_label     = 0;
+	uint32_t                                      merge_label        = 0;
+};
+
+void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                            const IR::Block* from, const IR::Block* to) {
 	if (to == nullptr) {
 		return;
 	}
 	for (const auto& phi: *to) {
 		if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
-			continue;
+			break;
 		}
 		for (size_t index = 0; index < phi.NumArgs(); index++) {
 			if (phi.PhiBlock(index) == from) {
 				ctx.state.builder.AddFunction(
-				    {OpStore, ctx.phi_variables.at(&phi), ctx.Def(phi.Arg(index))});
+				    {OpStore, dispatcher.spills.at(&phi), ctx.Def(phi.Arg(index))});
 				break;
 			}
 		}
 	}
 }
 
-const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) {
+const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
 	const auto found = std::ranges::find_if(
-	    program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; });
+	    program.block_info, [&](const IR::BlockInfo& info) { return info.id == id; });
 	if (found == program.block_info.end()) {
 		return nullptr;
 	}
@@ -98,7 +94,7 @@ void EmitReturn(ValueEmitContext& ctx) {
 }
 
 void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
-                              const IR::ValueBlockInfo& info) {
+                              const IR::BlockInfo& info) {
 	const auto& term       = info.terminator;
 	const auto  emit_merge = [&]() {
 		if (term.loop_header) {
@@ -124,7 +120,6 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				EmitReturn(ctx);
 				return;
 			}
-			StorePhiEdge(ctx, block, target);
 			emit_merge();
 			ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
 			return;
@@ -136,8 +131,6 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				EmitReturn(ctx);
 				return;
 			}
-			StorePhiEdge(ctx, block, true_block);
-			StorePhiEdge(ctx, block, false_block);
 			const auto condition = ctx.Def(info.condition);
 			emit_merge();
 			ctx.state.builder.AddFunction(
@@ -148,39 +141,34 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 	}
 }
 
-void EmitDispatcherTarget(ValueEmitContext& ctx, const IR::Block* from, uint32_t target) {
+void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                          const IR::Block* from, uint32_t target) {
 	const auto* block = TargetBlock(ctx.program, target);
 	if (block != nullptr) {
-		StorePhiEdge(ctx, from, block);
+		StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
 	}
 }
 
-void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
-                              const IR::ValueBlockInfo& info, uint32_t exit_label) {
+uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                              const IR::Block* block, const IR::BlockInfo& info) {
 	const auto& term = info.terminator;
 	switch (term.kind) {
 		case CFG::TerminatorKind::Branch:
-			EmitDispatcherTarget(ctx, block, term.true_block);
-			ctx.state.builder.AddFunction(
-			    {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, term.true_block)});
-			break;
+			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
+			return ConstantU32(ctx.state, term.true_block);
 		case CFG::TerminatorKind::ConditionalBranch: {
-			EmitDispatcherTarget(ctx, block, term.true_block);
-			EmitDispatcherTarget(ctx, block, term.false_block);
+			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
+			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, selected,
+			ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
 			                               ctx.Def(info.condition),
 			                               ConstantU32(ctx.state, term.true_block),
 			                               ConstantU32(ctx.state, term.false_block)});
-			ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
-			break;
+			return selected;
 		}
 		case CFG::TerminatorKind::IndirectBranch: {
 			for (const auto target: term.indirect_targets) {
-				EmitDispatcherTarget(ctx, block, target);
-			}
-			for (const auto target: term.indirect_selector_targets) {
-				EmitDispatcherTarget(ctx, block, target);
+				EmitDispatcherTarget(ctx, dispatcher, block, target);
 			}
 			uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
 			if (!info.indirect_target.IsEmpty()) {
@@ -194,108 +182,193 @@ void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
 					const auto match = ctx.state.builder.AllocateId();
 					const auto next  = ctx.state.builder.AllocateId();
-					ctx.state.builder.AddFunction({OpIEqual, ctx.state.bool_type, match, selector,
+					ctx.state.builder.AddFunction({OpIEqual, TypeBool(ctx.state), match, selector,
 					                               ConstantU32(ctx.state, values[index])});
-					ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, next, match,
+					ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), next, match,
 					                               ConstantU32(ctx.state, targets[index]),
 					                               selected});
 					selected = next;
 				}
 			}
-			ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
-			break;
+			return selected;
 		}
-		default:
-			ctx.state.builder.AddFunction(
-			    {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
-			break;
+		default: return ConstantU32(ctx.state, UINT32_MAX);
 	}
-	ctx.state.builder.AddFunction({OpBranch, exit_label});
 }
 
-bool EmitInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
-	const auto finish = [&]() {
-		if (const auto found = ctx.dispatcher_variables.find(&inst);
-		    found != ctx.dispatcher_variables.end()) {
-			ctx.state.builder.AddFunction(
-			    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
-		}
-		return true;
-	};
-	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
-		ctx.state.builder.AddFunction(
-		    {OpLoad, ctx.TypeId(inst.GetType()), ctx.Result(inst), ctx.phi_variables.at(&inst)});
-		return finish();
-	}
+bool EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
 	    EmitValueImage(ctx, inst)) {
-		return finish();
+		return true;
 	}
 	ctx.Fail(inst, "has no direct SPIR-V emitter");
 	return false;
 }
 
-void EmitBlock(ValueEmitContext& ctx, const IR::Block* block) {
+bool EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
+                               const IR::Inst& inst) {
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		const auto type = ctx.TypeId(inst.GetType());
+		if (type == 0 || inst.NumArgs() == 0) {
+			ctx.Fail(inst, "has no native SPIR-V representation");
+			return false;
+		}
+		for (size_t index = 0; index < inst.NumArgs(); index++) {
+			const auto* predecessor = inst.PhiBlock(index);
+			if (predecessor == nullptr || !ctx.labels.contains(predecessor)) {
+				ctx.Fail(inst, "has a predecessor outside the structured function");
+				return false;
+			}
+		}
+		structured.deferred_phis.push_back(
+		    {ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst});
+		return true;
+	}
+	return EmitDirectInstruction(ctx, inst);
+}
+
+bool EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                               const IR::Inst& inst) {
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		const auto type = ctx.TypeId(inst.GetType());
+		if (type == 0) {
+			ctx.Fail(inst, "cannot be loaded by the dispatcher");
+			return false;
+		}
+		ctx.state.builder.AddFunction(
+		    {OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)});
+		return true;
+	}
+	if (!EmitDirectInstruction(ctx, inst)) {
+		return false;
+	}
+	if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) {
+		ctx.state.builder.AddFunction(
+		    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
+	}
+	return true;
+}
+
+template <typename EmitInstruction>
+void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
 	ctx.current_block = block;
-	ctx.state.builder.AddFunction({OpLabel, ctx.Label(block)});
+	EmitLabel(ctx.state, ctx.Label(block));
+	bool emitted_non_phi = false;
 	for (const auto& inst: *block) {
-		if (!EmitInstruction(ctx, inst)) {
+		if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+			if (emitted_non_phi) {
+				ctx.Fail(inst, "appears after a non-Phi instruction");
+				return;
+			}
+		} else {
+			emitted_non_phi = true;
+		}
+		if (!emit_instruction(inst)) {
 			return;
 		}
 	}
 }
 
-void EmitStructuredFunction(ValueEmitContext& ctx) {
-	ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
-	for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
-		EmitBlock(ctx, ctx.program.blocks[index]);
-		EmitStructuredTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index]);
+void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) {
+	for (const auto& deferred: structured.deferred_phis) {
+		for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) {
+			const auto* predecessor = deferred.instruction->PhiBlock(index);
+			const auto  found       = structured.block_exit_labels.find(predecessor);
+			if (found == structured.block_exit_labels.end()) {
+				ctx.failed = true;
+				ctx.error  = "typed Phi predecessor was not emitted";
+				return;
+			}
+			ctx.state.builder.PatchDeferredPhi(
+			    deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second);
+			if (ctx.failed) {
+				return;
+			}
+		}
 	}
 }
 
-void EmitDispatcherFunction(ValueEmitContext& ctx) {
+void EmitStructuredFunction(ValueEmitContext& ctx) {
+	StructuredFunctionState structured;
+	ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
+	for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
+		const auto* block = ctx.program.blocks[index];
+		EmitBlock(ctx, block, [&](const IR::Inst& inst) {
+			return EmitStructuredInstruction(ctx, structured, inst);
+		});
+		if (ctx.failed) {
+			break;
+		}
+		structured.block_exit_labels.emplace(block, ctx.state.current_label);
+		EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]);
+	}
+	if (!ctx.failed) {
+		PatchStructuredPhis(ctx, structured);
+	}
+}
+
+void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
 	auto&       state = ctx.state;
 	const auto* entry = ctx.program.blocks.front();
 	state.builder.AddFunction({OpBranch, ctx.Label(entry)});
-	EmitBlock(ctx, entry);
-	EmitDispatcherTerminator(ctx, entry, ctx.program.block_info.front(),
-	                         state.dispatch_header_label);
+	EmitBlock(ctx, entry, [&](const IR::Inst& inst) {
+		return EmitDispatcherInstruction(ctx, dispatcher, inst);
+	});
+	if (ctx.failed) {
+		return;
+	}
+	const auto initial_pc =
+	    EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front());
+	const auto initial_parent = state.current_label;
+	state.builder.AddFunction({OpBranch, dispatcher.header_label});
 
-	state.builder.AddFunction({OpLabel, state.dispatch_header_label});
-	const auto pc = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.uint_type, pc, state.dispatch_pc_variable});
+	EmitLabel(state, dispatcher.header_label);
+	const auto pc      = state.builder.AllocateId();
+	const auto next_pc = state.builder.AllocateId();
+	state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
+	                           dispatcher.continue_label});
 	const auto done = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpIEqual, state.bool_type, done, pc, ConstantU32(ctx.state, UINT32_MAX)});
+	    {OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
 	state.builder.AddFunction(
-	    {OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone});
+	    {OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label, LoopControlNone});
 	state.builder.AddFunction(
-	    {OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
+	    {OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
 
-	state.builder.AddFunction({OpLabel, state.dispatch_select_label});
+	EmitLabel(state, dispatcher.select_label);
 	state.builder.AddFunction(
-	    {OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
-	std::vector<uint32_t> words {OpSwitch, pc, state.dispatch_default_label};
+	    {OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone});
+	std::vector<uint32_t> words {OpSwitch, pc, dispatcher.after_switch_label};
 	for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
 		words.push_back(ctx.program.block_info[index].id);
 		words.push_back(ctx.Label(ctx.program.blocks[index]));
 	}
 	state.builder.AddFunction(words);
-	state.builder.AddFunction({OpLabel, state.dispatch_default_label});
-	state.builder.AddFunction(
-	    {OpStore, state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
-	state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
+	std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc,
+	                                     ConstantU32(state, UINT32_MAX), dispatcher.select_label};
 
 	for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) {
-		EmitBlock(ctx, ctx.program.blocks[index]);
-		EmitDispatcherTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index],
-		                         state.dispatch_after_switch_label);
+		EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) {
+			return EmitDispatcherInstruction(ctx, dispatcher, inst);
+		});
+		if (ctx.failed) {
+			break;
+		}
+		const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index],
+		                                           ctx.program.block_info[index]);
+		next_pc_words.push_back(selected);
+		next_pc_words.push_back(state.current_label);
+		state.builder.AddFunction({OpBranch, dispatcher.after_switch_label});
 	}
-	state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
-	state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
-	state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
-	state.builder.AddFunction({OpBranch, state.dispatch_header_label});
-	state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
+	if (ctx.failed) {
+		return;
+	}
+	EmitLabel(state, dispatcher.after_switch_label);
+	state.builder.AddFunction(next_pc_words);
+	state.builder.AddFunction({OpBranch, dispatcher.continue_label});
+	EmitLabel(state, dispatcher.continue_label);
+	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	EmitLabel(state, dispatcher.merge_label);
 	EmitReturn(ctx);
 }
 
@@ -303,16 +376,17 @@ void EmitDispatcherFunction(ValueEmitContext& ctx) {
 
 uint32_t ValueEmitContext::TypeId(IR::Type type) const {
 	switch (type) {
-		case IR::Type::U1: return state.bool_type;
+		case IR::Type::U1: return TypeBool(state);
 		case IR::Type::U8:
 		case IR::Type::U16:
 		case IR::Type::U32:
-		case IR::Type::F16: return state.uint_type;
-		case IR::Type::U64:
-		case IR::Type::U32x2: return state.uint_pair_type;
-		case IR::Type::F32: return state.float_type;
-		case IR::Type::U32x4: return state.vec4_uint_type;
-		case IR::Type::F32x2: return state.vec2_float_type;
+		case IR::Type::F16: return TypeU32(state);
+		case IR::Type::U64: return TypeU64(state);
+		case IR::Type::U32x2: return TypeU32Pair(state);
+		case IR::Type::F32: return TypeF32(state);
+		case IR::Type::U32x3: return TypeU32Vector(state, 3);
+		case IR::Type::U32x4: return TypeU32Vector(state, 4);
+		case IR::Type::F32x2: return TypeF32Vector(state, 2);
 		default: return 0;
 	}
 }
@@ -321,11 +395,11 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
 	value = value.Resolve();
 	if (value.IsImmediate()) {
 		switch (value.GetType()) {
-			case IR::Type::U1: return BoolConstant(state, value.U1());
+			case IR::Type::U1: return ConstantBool(state, value.U1());
 			case IR::Type::U8: return ConstantU32(state, value.U8());
 			case IR::Type::U16: return ConstantU32(state, value.U16());
 			case IR::Type::U32: return ConstantU32(state, value.U32());
-			case IR::Type::U64: return ConstantPair(state, value.U64());
+			case IR::Type::U64: return ConstantU64(state, value.U64());
 			case IR::Type::F16: return ConstantU32(state, value.F16Bits());
 			case IR::Type::F32:
 				return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
@@ -338,12 +412,17 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
 		error  = "direct SPIR-V emitter received a non-value argument";
 		return ConstantU32(state, 0);
 	}
-	if (program.dispatcher_fallback && current_block != nullptr &&
+	if (dispatcher_spills != nullptr && current_block != nullptr &&
 	    inst->Parent() != current_block) {
-		if (const auto found = dispatcher_variables.find(inst);
-		    found != dispatcher_variables.end()) {
+		if (const auto found = dispatcher_spills->find(inst); found != dispatcher_spills->end()) {
+			if (const auto loaded = dispatcher_block_loads.find(inst);
+			    loaded != dispatcher_block_loads.end() &&
+			    loaded->second.first == state.current_label) {
+				return loaded->second.second;
+			}
 			const auto id = state.builder.AllocateId();
 			state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
+			dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
 			return id;
 		}
 	}
@@ -419,48 +498,48 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const {
 	return labels.at(block);
 }
 
-size_t ValueEmitContext::BlockIndex(const IR::Block* block) const {
-	return static_cast<size_t>(std::ranges::find(program.blocks, block) - program.blocks.begin());
-}
-
 void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
 	failed = true;
 	error  = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
 }
 
-bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) {
-	ValueEmitContext ctx(state, program);
+bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error) {
+	ValueEmitContext                       ctx(state, program);
+	std::optional<DispatcherFunctionState> dispatcher;
+	if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
+		state.pixel_valid_mask_variable = state.builder.AllocateId();
+		state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
+	}
 	for (const auto* block: program.blocks) {
 		ctx.labels.emplace(block, state.builder.AllocateId());
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
-				continue;
+	}
+	if (state.program.dispatcher_fallback) {
+		auto& dispatch = dispatcher.emplace();
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
+					continue;
+				}
+				if (SpillPointerType(ctx, inst.GetType()) == 0) {
+					ctx.Fail(inst, "cannot be stored by the dispatcher");
+					break;
+				}
+				dispatch.spills.emplace(&inst, state.builder.AllocateId());
 			}
-			const auto ptr_type = PhiPointerType(state, inst.GetType());
-			if (ptr_type == 0) {
-				ctx.failed = true;
-				ctx.error  = fmt::format("typed opcode Phi of type {} cannot be spilled across a "
-				                         "control-flow edge",
-				                         IR::TypeName(inst.GetType()));
-				break;
-			}
-			ctx.phi_variables.emplace(&inst, state.builder.AllocateId());
 		}
-	}
-	if (ctx.failed) {
-		SetError(error, ctx.error.c_str());
-		return false;
-	}
-	if (program.dispatcher_fallback) {
 		const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
 			value                  = value.Resolve();
 			const auto* definition = value.TryInstruction();
 			if (definition == nullptr || definition->Parent() == consumer ||
-			    PhiPointerType(state, definition->GetType()) == 0) {
+			    definition->Parent() == program.blocks.front()) {
 				return;
 			}
-			if (!ctx.dispatcher_variables.contains(definition)) {
-				ctx.dispatcher_variables.emplace(definition, state.builder.AllocateId());
+			if (SpillPointerType(ctx, definition->GetType()) == 0) {
+				ctx.Fail(*definition, "cannot be stored by the dispatcher");
+				return;
+			}
+			if (!dispatch.spills.contains(definition)) {
+				dispatch.spills.emplace(definition, state.builder.AllocateId());
 			}
 		};
 		for (const auto* block: program.blocks) {
@@ -476,83 +555,77 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
 			mark_cross_block(program.block_info[index].condition, program.blocks[index]);
 			mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
 		}
-		state.dispatcher_fallback         = true;
-		state.dispatch_pc_variable        = state.builder.AllocateId();
-		state.dispatch_header_label       = state.builder.AllocateId();
-		state.dispatch_select_label       = state.builder.AllocateId();
-		state.dispatch_default_label      = state.builder.AllocateId();
-		state.dispatch_after_switch_label = state.builder.AllocateId();
-		state.dispatch_continue_label     = state.builder.AllocateId();
-		state.dispatch_merge_label        = state.builder.AllocateId();
+		dispatch.header_label       = state.builder.AllocateId();
+		dispatch.select_label       = state.builder.AllocateId();
+		dispatch.after_switch_label = state.builder.AllocateId();
+		dispatch.continue_label     = state.builder.AllocateId();
+		dispatch.merge_label        = state.builder.AllocateId();
+		ctx.dispatcher_spills       = &dispatch.spills;
 	}
+	if (ctx.failed) {
+		SetError(error, ctx.error.c_str());
+		return false;
+	}
+	DefineGetBdaPointer(state);
 	for (const auto* block: program.blocks) {
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
 			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
 			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
 		    })) {
 			ctx.scratch_u32_variable = state.builder.AllocateId();
 			break;
 		}
 	}
 	state.builder.AddFunction(
-	    {OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
-	state.builder.AddFunction({OpLabel, state.entry_label});
-	if (state.needs_function_lds) {
+	    {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
+	EmitLabel(state, state.entry_label);
+	if (state.requirements.function_lds) {
 		state.builder.AddFunction(
-		    {OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
+		    {OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)),
+		     state.lds_variable, StorageClassFunction});
+	}
+	if (state.requirements.function_scratch) {
+		state.builder.AddFunction(
+		    {OpVariable,
+		     TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
+		     state.scratch_variable, StorageClassFunction});
 	}
 	if (state.pixel_valid_mask_variable != 0) {
-		state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable,
-		                           StorageClassFunction});
+		state.builder.AddFunction({OpVariable,
+		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
+		                           state.pixel_valid_mask_variable, StorageClassFunction});
 	}
-	if (program.dispatcher_fallback) {
-		state.builder.AddFunction(
-		    {OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
-	}
-	for (const auto& binding: state.registers) {
-		state.builder.AddFunction(
-		    {OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
-	}
-	for (const auto* block: program.blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
-				continue;
-			}
-			state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
-			                           ctx.phi_variables.at(&inst), StorageClassFunction});
-		}
-	}
-	if (program.dispatcher_fallback) {
+	if (state.program.dispatcher_fallback) {
 		for (const auto* block: program.blocks) {
 			for (const auto& inst: *block) {
-				if (const auto found = ctx.dispatcher_variables.find(&inst);
-				    found != ctx.dispatcher_variables.end()) {
-					state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
+				if (const auto found = dispatcher->spills.find(&inst);
+				    found != dispatcher->spills.end()) {
+					state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()),
 					                           found->second, StorageClassFunction});
 				}
 			}
 		}
 	}
 	if (ctx.scratch_u32_variable != 0) {
-		state.builder.AddFunction(
-		    {OpVariable, state.ptr_func_uint, ctx.scratch_u32_variable, StorageClassFunction});
+		state.builder.AddFunction({OpVariable,
+		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
+		                           ctx.scratch_u32_variable, StorageClassFunction});
 	}
-	for (const auto& binding: state.registers) {
-		state.builder.AddFunction({OpStore, binding.pointer_id,
-		                           ConstantU32(state, InitialRegisterValue(state, binding.reg))});
+	if (state.gds_variable != 0) {
+		state.gds_length = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
 	}
 	if (state.pixel_valid_mask_variable != 0) {
 		state.builder.AddFunction(
 		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
 	}
-	EmitStorageBufferOffsets(state);
+	EmitMemoryOffsets(state);
 	if (program.blocks.empty()) {
 		EmitReturn(ctx);
-	} else if (program.dispatcher_fallback) {
-		EmitDispatcherFunction(ctx);
+	} else if (state.program.dispatcher_fallback) {
+		EmitDispatcherFunction(ctx, *dispatcher);
 	} else {
 		EmitStructuredFunction(ctx);
 	}

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -1,7 +1,6 @@
 #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 
 #include <algorithm>
-#include <array>
 #include <fmt/format.h>
 #include <iterator>
 #include <map>
@@ -683,6 +682,66 @@ void RebuildPredecessors(Graph& graph) {
 	}
 }
 
+void PruneUnreachableBlocks(Graph& graph) {
+	if (graph.entry_block >= graph.blocks.size()) {
+		return;
+	}
+
+	std::vector<bool>     reachable(graph.blocks.size(), false);
+	std::vector<uint32_t> pending {graph.entry_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id >= graph.blocks.size() || reachable[block_id]) {
+			continue;
+		}
+		reachable[block_id] = true;
+		for (const auto successor: graph.blocks[block_id].successors) {
+			pending.push_back(successor);
+		}
+	}
+	if (std::ranges::all_of(reachable, [](bool value) { return value; })) {
+		return;
+	}
+
+	std::vector<uint32_t>   id_map(graph.blocks.size(), UINT32_MAX);
+	std::vector<BasicBlock> blocks;
+	blocks.reserve(std::ranges::count(reachable, true));
+	for (uint32_t old_id = 0; old_id < graph.blocks.size(); old_id++) {
+		if (!reachable[old_id]) {
+			continue;
+		}
+		id_map[old_id] = static_cast<uint32_t>(blocks.size());
+		blocks.push_back(std::move(graph.blocks[old_id]));
+	}
+
+	graph.entry_block   = RemapId(graph.entry_block, id_map);
+	graph.failure_block = RemapId(graph.failure_block, id_map);
+	graph.blocks        = std::move(blocks);
+	for (auto& block: graph.blocks) {
+		block.id = RemapId(block.id, id_map);
+		RemapIds(block.successors, id_map);
+		block.predecessors.clear();
+		block.dominators.clear();
+		block.post_dominators.clear();
+		auto& terminator          = block.terminator;
+		terminator.true_block     = RemapId(terminator.true_block, id_map);
+		terminator.false_block    = RemapId(terminator.false_block, id_map);
+		terminator.merge_block    = RemapId(terminator.merge_block, id_map);
+		terminator.continue_block = RemapId(terminator.continue_block, id_map);
+		for (auto& target: terminator.indirect_targets) {
+			target = RemapId(target, id_map);
+		}
+		for (auto& target: terminator.indirect_selector_targets) {
+			target = RemapId(target, id_map);
+		}
+	}
+	graph.back_edges.clear();
+	graph.natural_loops.clear();
+	graph.components.clear();
+	RebuildPredecessors(graph);
+}
+
 void ComputeDominators(Graph& graph) {
 	const auto count = static_cast<uint32_t>(graph.blocks.size());
 	const auto all   = AllBlockIds(count);
@@ -1079,11 +1138,75 @@ bool IsLoopControlGateway(const Graph& graph, const NaturalLoop& loop, uint32_t
 	       is_control_target(block->terminator.false_block);
 }
 
+bool HasLinearPathToTerminal(const Graph& graph, uint32_t start) {
+	std::vector<bool> visited(graph.blocks.size(), false);
+	for (auto block_id = start;;) {
+		const auto* block = graph.FindBlock(block_id);
+		if (block == nullptr || block_id >= visited.size() || visited[block_id]) {
+			return false;
+		}
+		if (block->successors.empty()) {
+			return true;
+		}
+		if (block->successors.size() != 1) {
+			return false;
+		}
+		visited[block_id] = true;
+		block_id          = block->successors.front();
+	}
+}
+
+bool IsEnclosingLinearExit(const Graph& graph, uint32_t header, uint32_t block_id) {
+	// AGC commonly lowers nested early returns through a terminal epilogue shared with an
+	// enclosing conditional. Such a path is an exit boundary, not part of the inner selection.
+	const auto* block = graph.FindBlock(block_id);
+	if (block == nullptr || graph.Dominates(header, block_id) ||
+	    !HasLinearPathToTerminal(graph, block_id) || block->predecessors.empty()) {
+		return false;
+	}
+	return std::ranges::all_of(block->predecessors, [&](uint32_t predecessor) {
+		return graph.Dominates(header, predecessor) || graph.Dominates(predecessor, header);
+	});
+}
+
+bool CanReachBefore(const Graph& graph, uint32_t start, uint32_t target, uint32_t stop) {
+	std::vector<uint32_t> pending = {start};
+	std::vector<bool>     visited(graph.blocks.size(), false);
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id == target) {
+			return true;
+		}
+		if (block_id == stop || block_id >= visited.size() || visited[block_id]) {
+			continue;
+		}
+		visited[block_id] = true;
+		const auto* block = graph.FindBlock(block_id);
+		if (block != nullptr) {
+			pending.insert(pending.end(), block->successors.begin(), block->successors.end());
+		}
+	}
+	return false;
+}
+
 uint32_t FindSelectionMerge(const Graph& graph, const BasicBlock& block) {
 	const auto  global_merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
 	                                                                block.terminator.false_block);
 	const auto* loop         = FindInnermostContainingLoop(graph, block.id);
 	if (loop == nullptr) {
+		const auto* global_block = graph.FindBlock(global_merge);
+		const auto  true_target  = block.terminator.true_block;
+		const auto  false_target = block.terminator.false_block;
+		if (global_block != nullptr && global_block->successors.empty()) {
+			const bool false_reaches_true =
+			    CanReachBefore(graph, false_target, true_target, global_merge);
+			const bool true_reaches_false =
+			    CanReachBefore(graph, true_target, false_target, global_merge);
+			if (false_reaches_true != true_reaches_false) {
+				return false_reaches_true ? true_target : false_target;
+			}
+		}
 		return global_merge;
 	}
 
@@ -1120,6 +1243,11 @@ bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& blo
 		};
 		return is_repeat_target(true_target) && is_repeat_target(false_target);
 	}
+	const bool true_in_body  = Contains(loop->body_blocks, true_target);
+	const bool false_in_body = Contains(loop->body_blocks, false_target);
+	if (true_in_body != false_in_body) {
+		return true;
+	}
 	const auto is_control_target = [&](uint32_t target) {
 		return target == loop->merge || target == loop->continue_block;
 	};
@@ -1198,6 +1326,37 @@ bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
 	return false;
 }
 
+bool IsolateSemanticLoopHeaders(Graph& graph, std::string* error) {
+	const auto isolation_budget = graph.natural_loops.size() + 1u;
+	for (size_t isolation = 0; isolation < isolation_budget; isolation++) {
+		const auto loop = std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(),
+		                               [&](const auto& value) {
+			                               const auto* header = graph.FindBlock(value.header);
+			                               return header != nullptr &&
+			                                      header->inst_begin != header->inst_end;
+		                               });
+		if (loop == graph.natural_loops.end()) {
+			return true;
+		}
+
+		// SPIR-V requires OpLoopMerge and its branch to remain in the loop header's
+		// physical block. Keep that header as a dedicated control node, exactly like
+		// the separate Loop node in shadPS4's structured AST. Guest instructions live
+		// in the body so later translation may introduce bounds/EXEC control flow without
+		// displacing OpLoopMerge into a helper-created block.
+		if (!IsolateSemanticLoopHeader(graph, loop->header)) {
+			SetFailure(graph, FailureKind::StructuredControlFlow, loop->header,
+			           fmt::format("failed to isolate semantic loop header {}", loop->header),
+			           error);
+			return false;
+		}
+	}
+
+	SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
+	           "CFG semantic loop-header isolation exceeded rewrite budget", error);
+	return false;
+}
+
 bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
                            const std::vector<uint32_t>& construct_blocks,
                            bool                         force_split = false) {
@@ -1269,17 +1428,24 @@ std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& head
 	std::vector<uint32_t> region;
 	std::vector<uint32_t> pending = {header.terminator.true_block, header.terminator.false_block};
 	const auto*           loop    = FindInnermostContainingLoop(graph, header.id);
+	const auto global_merge = graph.FindNearestCommonPostDominator(header.terminator.true_block,
+	                                                               header.terminator.false_block);
 	while (!pending.empty()) {
 		const auto block_id = pending.back();
 		pending.pop_back();
 		if (block_id == merge || Contains(region, block_id) ||
-		    (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block))) {
+		    (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block)) ||
+		    IsEnclosingLinearExit(graph, header.id, block_id) ||
+		    (block_id == global_merge && HasLinearPathToTerminal(graph, block_id))) {
 			continue;
 		}
 		const auto* block = graph.FindBlock(block_id);
 		if (block == nullptr) {
 			continue;
 		}
+		if (block->successors.empty()) {
+			continue;
+		}
 		AddUnique(region, block_id);
 		pending.insert(pending.end(), block->successors.begin(), block->successors.end());
 	}
@@ -1324,7 +1490,11 @@ bool SplitOneSelectionMerge(Graph& graph, std::string* error) {
 		}
 		const auto region   = SelectionRegion(graph, *block, merge);
 		const auto external = std::find_if(region.begin(), region.end(), [&](uint32_t member) {
-			return !graph.Dominates(block_id, member);
+			const auto* member_block = graph.FindBlock(member);
+			return member_block != nullptr &&
+			       std::ranges::any_of(member_block->predecessors, [&](uint32_t predecessor) {
+				       return predecessor != block_id && !Contains(region, predecessor);
+			       });
 		});
 		if (external != region.end()) {
 			SetFailure(
@@ -1449,8 +1619,8 @@ uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_
 namespace {
 
 struct GotoRouteBlocks {
-	uint32_t                before = UINT32_MAX;
-	std::array<uint32_t, 5> blocks {};
+	uint32_t              before = UINT32_MAX;
+	std::vector<uint32_t> blocks;
 };
 
 void PlaceGotoRouteBlocks(Graph& graph, uint32_t original_block_count,
@@ -1514,6 +1684,48 @@ uint32_t AppendGotoSetBlock(Graph& graph, uint32_t route_variable, bool value, u
 	return block_id;
 }
 
+void AppendEnclosingRouteBlocks(Graph& graph, uint32_t original_block_count,
+                                uint32_t route_variable, uint32_t shared, uint32_t other,
+                                uint32_t route_select, uint32_t route_root,
+                                std::vector<uint32_t>& route_blocks) {
+	for (uint32_t depth = 0; depth < original_block_count; depth++) {
+		uint32_t predecessor = UINT32_MAX;
+		uint32_t leaf        = UINT32_MAX;
+		for (uint32_t candidate = 0; candidate < original_block_count; candidate++) {
+			const auto* candidate_block = graph.FindBlock(candidate);
+			if (candidate_block == nullptr ||
+			    candidate_block->terminator.kind != TerminatorKind::ConditionalBranch) {
+				continue;
+			}
+			const auto candidate_true  = candidate_block->terminator.true_block;
+			const auto candidate_false = candidate_block->terminator.false_block;
+			if (candidate_true == route_root &&
+			    (candidate_false == shared || candidate_false == other)) {
+				predecessor = candidate;
+				leaf        = candidate_false;
+				break;
+			}
+			if (candidate_false == route_root &&
+			    (candidate_true == shared || candidate_true == other)) {
+				predecessor = candidate;
+				leaf        = candidate_true;
+				break;
+			}
+		}
+		if (predecessor == UINT32_MAX) {
+			break;
+		}
+
+		const auto routed_leaf =
+		    AppendGotoSetBlock(graph, route_variable, leaf == other, route_select);
+		auto* predecessor_block = graph.FindBlock(predecessor);
+		ReplaceValue(predecessor_block->successors, leaf, routed_leaf);
+		ReplaceTerminatorTarget(predecessor_block->terminator, leaf, routed_leaf);
+		route_blocks.push_back(routed_leaf);
+		route_root = predecessor;
+	}
+}
+
 // Turn H0 -> shared/H1, H1 -> shared/other into nested selections whose empty
 // forwarding blocks set one typed SSA route value. Guest semantic blocks remain unique.
 bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t outer_id,
@@ -1538,66 +1750,139 @@ bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t out
 
 		const bool true_is_shared  = inner->terminator.true_block == shared;
 		const bool false_is_shared = inner->terminator.false_block == shared;
-		if (true_is_shared == false_is_shared) {
-			continue;
-		}
-		const auto other =
-		    true_is_shared ? inner->terminator.false_block : inner->terminator.true_block;
-		if (other == outer_id || other == inner_id || other == shared) {
-			continue;
-		}
-		const auto first_arm = std::min(shared, other);
-		if (first_arm >= original_block_count || outer_id >= inner_id || inner_id >= first_arm) {
-			continue;
-		}
-		if (graph.FindNearestCommonPostDominator(shared, other) == UINT32_MAX) {
-			continue;
+		if (true_is_shared != false_is_shared) {
+			const auto other =
+			    true_is_shared ? inner->terminator.false_block : inner->terminator.true_block;
+			if (other == outer_id || other == inner_id || other == shared) {
+				continue;
+			}
+			const auto first_arm = std::min(shared, other);
+			if (first_arm >= original_block_count || outer_id >= inner_id ||
+			    inner_id >= first_arm ||
+			    graph.FindNearestCommonPostDominator(shared, other) == UINT32_MAX) {
+				continue;
+			}
+
+			const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared);
+			const auto inner_merge  = AppendSyntheticBranchBlock(graph, route_select);
+			const auto outer_shared =
+			    AppendGotoSetBlock(graph, route_variable, false, route_select);
+			const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge);
+			const auto inner_other  = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
+
+			outer                         = graph.FindBlock(outer_id);
+			auto* mutable_inner           = graph.FindBlock(inner_id);
+			outer->terminator.true_block  = inner_is_true ? inner_id : outer_shared;
+			outer->terminator.false_block = inner_is_true ? outer_shared : inner_id;
+			outer->successors = {outer->terminator.true_block, outer->terminator.false_block};
+			mutable_inner->terminator.true_block  = true_is_shared ? inner_shared : inner_other;
+			mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared;
+			mutable_inner->successors             = {mutable_inner->terminator.true_block,
+			                                         mutable_inner->terminator.false_block};
+
+			std::vector<uint32_t> route_blocks = {inner_shared, inner_other, inner_merge,
+			                                      outer_shared};
+			AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared, other,
+			                           route_select, outer_id, route_blocks);
+			route_blocks.push_back(route_select);
+			route = {first_arm, std::move(route_blocks)};
+			return true;
 		}
 
-		const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared);
-		const auto inner_merge  = AppendSyntheticBranchBlock(graph, route_select);
-		const auto outer_shared = AppendGotoSetBlock(graph, route_variable, false, route_select);
-		const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge);
-		const auto inner_other  = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
+		// H0 -> arm/H1, H1 -> exit/body, and both arms reach a common continuation.
+		// Route exit/continuation after joining H0 and H1 so neither has a nonlocal edge.
+		for (const bool exit_is_true: {true, false}) {
+			const auto other =
+			    exit_is_true ? inner->terminator.true_block : inner->terminator.false_block;
+			const auto body =
+			    exit_is_true ? inner->terminator.false_block : inner->terminator.true_block;
+			if (other == outer_id || other == inner_id || other == shared || body == outer_id ||
+			    body == inner_id || body == shared || !graph.Dominates(inner_id, body)) {
+				continue;
+			}
 
-		outer                         = graph.FindBlock(outer_id);
-		auto* mutable_inner           = graph.FindBlock(inner_id);
-		outer->terminator.true_block  = inner_is_true ? inner_id : outer_shared;
-		outer->terminator.false_block = inner_is_true ? outer_shared : inner_id;
-		outer->successors = {outer->terminator.true_block, outer->terminator.false_block};
-		mutable_inner->terminator.true_block  = true_is_shared ? inner_shared : inner_other;
-		mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared;
-		mutable_inner->successors             = {mutable_inner->terminator.true_block,
-		                                         mutable_inner->terminator.false_block};
-		route = {first_arm, {inner_shared, inner_other, inner_merge, outer_shared, route_select}};
-		return true;
+			const auto continuation = graph.FindNearestCommonPostDominator(shared, body);
+			const auto* continuation_block = graph.FindBlock(continuation);
+			if (continuation_block == nullptr || continuation == other ||
+			    CanReachBefore(graph, other, continuation, UINT32_MAX)) {
+				continue;
+			}
+			std::vector<uint32_t> outer_predecessors;
+			std::vector<uint32_t> inner_predecessors;
+			bool                  external_predecessor = false;
+			for (const auto predecessor: continuation_block->predecessors) {
+				if (predecessor == outer_id || graph.Dominates(shared, predecessor)) {
+					outer_predecessors.push_back(predecessor);
+				} else if (predecessor == inner_id || graph.Dominates(body, predecessor)) {
+					inner_predecessors.push_back(predecessor);
+				} else {
+					external_predecessor = true;
+				}
+			}
+			const auto first_arm = std::min(continuation, other);
+			if (outer_predecessors.empty() || inner_predecessors.empty() ||
+			    external_predecessor || first_arm >= original_block_count ||
+			    outer_id >= inner_id || inner_id >= first_arm ||
+			    graph.FindNearestCommonPostDominator(continuation, other) == UINT32_MAX) {
+				continue;
+			}
+
+			const auto route_select =
+			    AppendGotoSelectBlock(graph, route_variable, other, continuation);
+			const auto inner_merge  = AppendSyntheticBranchBlock(graph, route_select);
+			const auto outer_continue =
+			    AppendGotoSetBlock(graph, route_variable, false, route_select);
+			const auto inner_continue =
+			    AppendGotoSetBlock(graph, route_variable, false, inner_merge);
+			const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
+
+			auto* mutable_inner = graph.FindBlock(inner_id);
+			if (exit_is_true) {
+				mutable_inner->terminator.true_block = inner_other;
+			} else {
+				mutable_inner->terminator.false_block = inner_other;
+			}
+			ReplaceValue(mutable_inner->successors, other, inner_other);
+			for (const auto predecessor: outer_predecessors) {
+				auto* predecessor_block = graph.FindBlock(predecessor);
+				ReplaceValue(predecessor_block->successors, continuation, outer_continue);
+				ReplaceTerminatorTarget(predecessor_block->terminator, continuation,
+				                        outer_continue);
+			}
+			for (const auto predecessor: inner_predecessors) {
+				auto* predecessor_block = graph.FindBlock(predecessor);
+				ReplaceValue(predecessor_block->successors, continuation, inner_continue);
+				ReplaceTerminatorTarget(predecessor_block->terminator, continuation,
+				                        inner_continue);
+			}
+
+			std::vector<uint32_t> route_blocks = {inner_continue, inner_other, inner_merge,
+			                                      outer_continue};
+			if (continuation == shared) {
+				AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared,
+				                           other, route_select, outer_id, route_blocks);
+			}
+			route_blocks.push_back(route_select);
+			route = {first_arm, std::move(route_blocks)};
+			return true;
+		}
 	}
 	return false;
 }
 
-bool RouteSharedSelectionArms(Graph& graph) {
+bool RouteSharedSelectionArm(Graph& graph, uint32_t route_variable) {
 	if (graph.irreducible) {
 		return false;
 	}
-	const auto route_budget = static_cast<uint32_t>(graph.blocks.size());
-	bool       found        = false;
-	for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) {
-		const auto block_count = static_cast<uint32_t>(graph.blocks.size());
-		bool       routed      = false;
-		for (uint32_t block_id = 0; block_id < block_count; block_id++) {
-			GotoRouteBlocks route;
-			if (!RouteOneSharedArm(graph, block_count, block_id, route_variable, route)) {
-				continue;
-			}
+	const auto block_count = static_cast<uint32_t>(graph.blocks.size());
+	// Route later/deeper candidates first so earlier shared arms remain structured.
+	for (uint32_t block_id = block_count; block_id-- > 0;) {
+		GotoRouteBlocks route;
+		if (RouteOneSharedArm(graph, block_count, block_id, route_variable, route)) {
 			PlaceGotoRouteBlocks(graph, block_count, route);
 			RebuildPredecessors(graph);
 			RecomputeAnalyses(graph);
-			found  = true;
-			routed = true;
-			break;
-		}
-		if (!routed) {
-			return found;
+			return true;
 		}
 	}
 	return false;
@@ -1633,7 +1918,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 	labels.insert(end_pc);
 
 	std::map<uint32_t, SetpcTargetInfo> setpc_targets;
-	bool                                indirect_setpc = false;
 	for (uint32_t i = 0; i < program.instructions.size(); i++) {
 		const auto& inst    = program.instructions[i];
 		const auto  next_pc = InstructionEndPc(inst);
@@ -1670,7 +1954,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 				}
 				labels.insert(target);
 			}
-			indirect_setpc = indirect_setpc || target_info.indirect;
 			setpc_targets.emplace(inst.pc, std::move(target_info));
 			if (next_pc <= end_pc) {
 				labels.insert(next_pc);
@@ -1748,9 +2031,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 					block.terminator.indirect_selector_targets.push_back(
 					    pc_to_block.at(target_info.selector_target_pcs[i]));
 				}
-				if (target_info.table_load_pc != UINT32_MAX) {
-					AddUnique(graph.code_table_load_pcs, target_info.table_load_pc);
-				}
 			} else {
 				block.terminator.kind       = TerminatorKind::Branch;
 				block.terminator.condition  = BranchCondition::Always;
@@ -1811,6 +2091,22 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 	for (auto& block: graph.blocks) {
 		SortUnique(block.predecessors);
 	}
+	PruneUnreachableBlocks(graph);
+	graph.code_table_load_pcs.clear();
+	bool indirect_setpc = false;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.kind != TerminatorKind::IndirectBranch) {
+			continue;
+		}
+		indirect_setpc = true;
+		if (block.inst_begin >= block.inst_end || block.inst_end > program.instructions.size()) {
+			continue;
+		}
+		const auto found = setpc_targets.find(program.instructions[block.inst_end - 1u].pc);
+		if (found != setpc_targets.end() && found->second.table_load_pc != UINT32_MAX) {
+			AddUnique(graph.code_table_load_pcs, found->second.table_load_pc);
+		}
+	}
 	SortUnique(graph.code_table_load_pcs);
 
 	ComputeDominators(graph);
@@ -1855,6 +2151,9 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
 	if (!SplitSharedMergeBlocks(graph, error)) {
 		return false;
 	}
+	if (!IsolateSemanticLoopHeaders(graph, error)) {
+		return false;
+	}
 	ClearStructuredTerminators(graph);
 
 	std::map<uint32_t, uint32_t> merge_headers;
@@ -1881,6 +2180,14 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
 			    error);
 			return false;
 		}
+		if (header->inst_begin != header->inst_end ||
+		    header->terminator.kind != TerminatorKind::Branch) {
+			SetFailure(graph, FailureKind::StructuredControlFlow, loop.header,
+			           fmt::format("loop header {} is not a dedicated empty control block",
+			                       loop.header),
+			           error);
+			return false;
+		}
 		if (!reserve_merge_block(loop.header, loop.merge)) {
 			return false;
 		}
@@ -1922,17 +2229,25 @@ bool Structurize(Graph& graph, std::string* error) {
 	if (StructurizeImpl(graph, error)) {
 		return true;
 	}
-	if (!RouteSharedSelectionArms(original)) {
-		return false;
-	}
+
 	Graph       failed_graph = std::move(graph);
 	std::string failed_error = error != nullptr ? *error : std::string {};
 	graph                    = std::move(original);
-	if (error != nullptr) {
-		error->clear();
-	}
-	if (StructurizeImpl(graph, error)) {
-		return true;
+	const auto route_budget  = static_cast<uint32_t>(graph.blocks.size());
+	// Apply one route at a time and retry. Eagerly routing every matching diamond can
+	// rewrite unrelated selections that were already structurally valid.
+	for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) {
+		if (!RouteSharedSelectionArm(graph, route_variable)) {
+			break;
+		}
+		Graph routed = graph;
+		if (error != nullptr) {
+			error->clear();
+		}
+		if (StructurizeImpl(routed, error)) {
+			graph = std::move(routed);
+			return true;
+		}
 	}
 	graph = std::move(failed_graph);
 	if (error != nullptr) {
@@ -1941,31 +2256,6 @@ bool Structurize(Graph& graph, std::string* error) {
 	return false;
 }
 
-bool IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error) {
-	const auto loop =
-	    std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(),
-	                 [header_id](const auto& value) { return value.header == header_id; });
-	if (loop == graph.natural_loops.end()) {
-		SetFailure(graph, FailureKind::StructuredControlFlow, header_id,
-		           fmt::format("block {} is not a natural loop header", header_id), error);
-		return false;
-	}
-
-	ClearStructuredTerminators(graph);
-	if (!IsolateSemanticLoopHeader(graph, header_id)) {
-		SetFailure(graph, FailureKind::StructuredControlFlow, header_id,
-		           fmt::format("loop header block {} has no semantic instructions", header_id),
-		           error);
-		return false;
-	}
-
-	graph.unsupported   = false;
-	graph.failure_kind  = FailureKind::None;
-	graph.failure_block = UINT32_MAX;
-	graph.unsupported_reason.clear();
-	return Structurize(graph, error);
-}
-
 std::string BranchConditionToString(BranchCondition condition) {
 	switch (condition) {
 		case BranchCondition::Always: return "always";

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -110,7 +110,6 @@ struct Graph {
 
 bool        BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error);
 bool        Structurize(Graph& graph, std::string* error);
-bool        IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error);
 std::string BranchConditionToString(BranchCondition condition);
 std::string FailureKindToString(FailureKind kind);
 std::string GraphToString(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -153,28 +153,38 @@ constexpr MimgSampleInfo MIMG_SAMPLE_OPCODE_LIST[] = {
     SampleInfo(0x6fu, "image_sample_c_cd_cl_o",
                ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd |
                    ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xa0u, "image_sample_a", 0),
-    SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp),
-    SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias),
-    SampleInfo(0xa6u, "image_sample_b_cl_a", ImageSampleFlagBias | ImageSampleFlagLodClamp),
-    SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare),
-    SampleInfo(0xa9u, "image_sample_c_cl_a", ImageSampleFlagCompare | ImageSampleFlagLodClamp),
-    SampleInfo(0xadu, "image_sample_c_b_a", ImageSampleFlagCompare | ImageSampleFlagBias),
+    SampleInfo(0xa0u, "image_sample_a", ImageSampleFlagAdjust),
+    SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias | ImageSampleFlagAdjust),
+    SampleInfo(0xa6u, "image_sample_b_cl_a",
+               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare | ImageSampleFlagAdjust),
+    SampleInfo(0xa9u, "image_sample_c_cl_a",
+               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xadu, "image_sample_c_b_a",
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagAdjust),
     SampleInfo(0xaeu, "image_sample_c_b_cl_a",
-               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp),
-    SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset),
-    SampleInfo(0xb1u, "image_sample_cl_o_a", ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xb5u, "image_sample_b_o_a", ImageSampleFlagBias | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+                   ImageSampleFlagAdjust),
+    SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset | ImageSampleFlagAdjust),
+    SampleInfo(0xb1u, "image_sample_cl_o_a",
+               ImageSampleFlagLodClamp | ImageSampleFlagOffset | ImageSampleFlagAdjust),
+    SampleInfo(0xb5u, "image_sample_b_o_a",
+               ImageSampleFlagBias | ImageSampleFlagOffset | ImageSampleFlagAdjust),
     SampleInfo(0xb6u, "image_sample_b_cl_o_a",
-               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xb8u, "image_sample_c_o_a", ImageSampleFlagCompare | ImageSampleFlagOffset),
+               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
+    SampleInfo(0xb8u, "image_sample_c_o_a",
+               ImageSampleFlagCompare | ImageSampleFlagOffset | ImageSampleFlagAdjust),
     SampleInfo(0xb9u, "image_sample_c_cl_o_a",
-               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
     SampleInfo(0xbdu, "image_sample_c_b_o_a",
-               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
     SampleInfo(0xbeu, "image_sample_c_b_cl_o_a",
                ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
-                   ImageSampleFlagOffset),
+                   ImageSampleFlagOffset | ImageSampleFlagAdjust),
 };
 
 constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
@@ -299,7 +309,9 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const uint32_t vaddr  = word1 & 0xffu;
 	const uint32_t srsrc  = (word1 >> 16u) & 0x1fu;
 	const uint32_t ssamp  = (word1 >> 21u) & 0x1fu;
+	const bool     r128   = ((word0 >> 15u) & 0x1u) != 0u;
 	const bool     a16    = ((word1 >> 30u) & 0x1u) != 0u;
+	const bool     d16    = ((word1 >> 31u) & 0x1u) != 0u;
 	const auto*    sample = LookupSample(opcode);
 	const auto*    gather = LookupGather(opcode);
 	const auto*    atomic = LookupAtomic(opcode);
@@ -311,7 +323,9 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.opcode_id          = opcode;
 	inst.opcode             = DecodeMimgOpcode(opcode, sample, gather, atomic);
 	inst.dmask              = (word0 >> 8u) & 0xfu;
-	inst.data_dwords        = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
+	inst.data_components    = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
+	inst.data_bits          = d16 ? 16u : 32u;
+	inst.data_dwords        = d16 ? (inst.data_components + 1u) / 2u : inst.data_components;
 	inst.glc                = ((word0 >> 13u) & 1u) != 0;
 	inst.slc                = ((word0 >> 25u) & 1u) != 0;
 	inst.image_sample_flags = DecodeMimgSampleFlags(sample, gather);
@@ -319,6 +333,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		inst.image_sample_flags |= ImageSampleFlagA16;
 	}
 	inst.image_dimension  = dimension;
+	inst.image_r128       = r128;
 	inst.image_nsa_dwords = nsa_dwords;
 	for (uint32_t i = 0; i < nsa_dwords * 4u; i++) {
 		inst.image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
@@ -334,6 +349,11 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		SetUnsupported(inst, Family::MIMG, opcode,
 		               "MIMG image gather requires exactly one dmask bit");
 	}
+	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
+	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
+	if (d16 && !supports_d16) {
+		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
+	}
 
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -96,13 +96,14 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
     {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32},      {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
     {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32},      {0x36u, Opcode::DS_READ_B32, 1, 32},
-    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2_B32, 2, 32},
+    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2ST64_B32, 2, 32},
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
     {0x77u, Opcode::DS_READ2_B64, 4, 32},        {0x78u, Opcode::DS_READ2ST64_B64, 4, 32},
+    {0xa6u, Opcode::DS_READ_U16_D16, 1, 16},
     {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
     {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
     {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
@@ -403,10 +404,11 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 		SetUnsupported(inst, Family::DS, opcode,
 		               "DS read addtid data operands are not implemented");
 	}
-	if (inst.opcode == Opcode::DS_WRITE2_B32 || opcode == 0x37u) {
+	if (inst.opcode == Opcode::DS_WRITE2_B32 || inst.opcode == Opcode::DS_READ2_B32) {
 		inst.offset           = offset0 * 4u;
 		inst.secondary_offset = offset1 * 4u;
-	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || opcode == 0x38u) {
+	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 ||
+	           inst.opcode == Opcode::DS_READ2ST64_B32) {
 		inst.offset           = offset0 * 256u;
 		inst.secondary_offset = offset1 * 256u;
 	} else if (inst.opcode == Opcode::DS_WRITE2_B64 || inst.opcode == Opcode::DS_READ2_B64) {
@@ -419,6 +421,11 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	}
 
 	DecodeVectorGpr(vdst, inst.dst);
+	if (inst.opcode == Opcode::DS_READ_U16_D16) {
+		// Native D16 reads update only the selected destination half. Reuse the partial
+		// destination representation so typed IR translation preserves the upper word.
+		inst.dst.sdwa_sel = 4u;
+	}
 	DecodeVectorGpr(addr, inst.src0);
 	DecodeVectorGpr(data0, inst.src1);
 	DecodeVectorGpr(data1, inst.src2);

src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
 
+#include "common/assert.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <array>
@@ -27,7 +28,7 @@ consteval auto MakeOpcodeTable(const Entry (&entries)[EntryCount]) {
 	for (size_t i = 0; i < EntryCount; i++) {
 		const auto encoding = entries[i].encoding;
 		if (encoding >= EncodingCount || table.indices[encoding] != 0) {
-			throw "invalid opcode table";
+			EXIT("invalid opcode table\n");
 		}
 		table.entries[i]        = entries[i];
 		table.indices[encoding] = static_cast<uint16_t>(i + 1);

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -52,6 +52,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x20u, Opcode::S_SETPC_B64},
     {0x24u, Opcode::S_AND_SAVEEXEC_B64},
     {0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
+    {0x2du, Opcode::S_QUADMASK_B64},
     {0x34u, Opcode::S_ABS_I32},
     {0x37u, Opcode::S_ANDN1_SAVEEXEC_B64},
     {0x3bu, Opcode::S_BITREPLICATE_B64_B32},

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -140,6 +140,7 @@ std::string ImageSampleFlagsToString(uint32_t flags) {
 	AppendFlag(&text, &first, flags, ImageSampleFlagA16, "a16");
 	AppendFlag(&text, &first, flags, ImageSampleFlagCd, "cd");
 	AppendFlag(&text, &first, flags, ImageSampleFlagGatherHorizontal, "gather_horizontal");
+	AppendFlag(&text, &first, flags, ImageSampleFlagAdjust, "adjust");
 	return text;
 }
 
@@ -153,6 +154,12 @@ std::string FormatMimg(const Instruction& inst) {
 	                OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(),
 	                OperandToString(inst.src1).c_str(), OperandToString(inst.src2).c_str(),
 	                inst.dmask, ImageDimensionToString(inst.image_dimension));
+	if (inst.data_bits == 16u) {
+		text += " d16=1";
+	}
+	if (inst.image_r128) {
+		text += " r128=1";
+	}
 	switch (inst.opcode) {
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_GATHER4_LZ:
@@ -498,6 +505,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_FF1_I32_B64:
 		case Opcode::S_NOT_B64:
 		case Opcode::S_WQM_B64:
+		case Opcode::S_QUADMASK_B64:
 		case Opcode::S_AND_SAVEEXEC_B32:
 		case Opcode::S_ANDN1_SAVEEXEC_B32:
 		case Opcode::S_AND_SAVEEXEC_B64:
@@ -658,6 +666,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_READ_I16:
 		case Opcode::DS_READ_U16:
 		case Opcode::DS_READ2_B32:
+		case Opcode::DS_READ2ST64_B32:
 		case Opcode::DS_READ_B32:
 		case Opcode::DS_READ_B64:
 		case Opcode::DS_READ2_B64:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -47,6 +47,7 @@ enum class Opcode {
 	S_FF1_I32_B64,
 	S_FLBIT_I32_B64,
 	S_BITREPLICATE_B64_B32,
+	S_QUADMASK_B64,
 	S_GETPC_B64,
 	S_SETPC_B64,
 	S_AND_SAVEEXEC_B32,
@@ -298,6 +299,8 @@ enum class Opcode {
 	V_LSHRREV_B32,
 	V_ASHR_I32,
 	V_ASHRREV_I32,
+	V_LSHLREV_B64,
+	V_LSHRREV_B64,
 	V_LSHLREV_B16,
 	V_LSHRREV_B16,
 	V_ASHRREV_I16,
@@ -363,6 +366,7 @@ enum class Opcode {
 	V_CMPX_LE_F16,
 	V_CMPX_GT_F16,
 	V_CMPX_GE_F16,
+	V_CMPX_NGT_F16,
 	V_CMPX_NEQ_F16,
 	V_CMPX_NLT_F16,
 	V_CMPX_LT_I32,
@@ -375,6 +379,7 @@ enum class Opcode {
 	V_CMP_EQ_U16,
 	V_CMP_LE_U16,
 	V_CMP_GT_U16,
+	V_CMPX_GT_U16,
 	V_CMP_NE_U16,
 	V_CMP_GE_U16,
 	V_CMP_F_U32,
@@ -388,6 +393,8 @@ enum class Opcode {
 	V_CMP_EQ_I64,
 	V_CMP_GT_U64,
 	V_CMP_NE_U64,
+	V_CMPX_NE_I64,
+	V_CMPX_NE_U64,
 	V_CMPX_LT_U32,
 	V_CMPX_EQ_U32,
 	V_CMPX_LE_U32,
@@ -489,7 +496,9 @@ enum class Opcode {
 	DS_READ_U8,
 	DS_READ_I16,
 	DS_READ_U16,
+	DS_READ_U16_D16,
 	DS_READ2_B32,
+	DS_READ2ST64_B32,
 	DS_READ_B32,
 	DS_READ_B64,
 	DS_READ2_B64,
@@ -584,6 +593,7 @@ enum ImageSampleFlag : uint32_t {
 	ImageSampleFlagA16              = 1u << 7u,
 	ImageSampleFlagCd               = 1u << 8u,
 	ImageSampleFlagGatherHorizontal = 1u << 9u,
+	ImageSampleFlagAdjust           = 1u << 10u,
 };
 
 enum class ImageDimension : uint32_t {
@@ -645,6 +655,7 @@ struct Instruction {
 	uint32_t       offset                                       = 0;
 	uint32_t       secondary_offset                             = 0;
 	uint32_t       dmask                                        = 0;
+	uint32_t       data_components                              = 0;
 	uint32_t       data_dwords                                  = 1;
 	uint32_t       data_bits                                    = 32;
 	uint32_t       data_format                                  = 0;
@@ -663,6 +674,7 @@ struct Instruction {
 	bool           slc                                          = false;
 	bool           idxen                                        = false;
 	bool           offen                                        = false;
+	bool           image_r128                                   = false;
 	int32_t        branch_offset                                = 0;
 	uint32_t       branch_target                                = 0;
 	struct {

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -7,11 +7,10 @@ namespace {
 
 using Detail::OpcodeMap;
 
-// These profiles describe the selectors that the current lowering implements. They are not
+// These profiles describe the selectors that the current translator implements. They are not
 // architectural SDWA legality classes.
 enum class Vop2SdwaProfile {
 	None,
-	Cndmask,
 	Float32,
 	Float16,
 	IntegerFullDestination,
@@ -29,7 +28,7 @@ struct Vop2OpcodeInfo {
 };
 
 constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
-    {0x01u, Opcode::V_CNDMASK_B32, Vop2SdwaProfile::Cndmask},
+    {0x01u, Opcode::V_CNDMASK_B32, Vop2SdwaProfile::IntegerPartialDestination},
     {0x02u, Opcode::V_DOT2C_F32_F16},
     {0x03u, Opcode::V_ADD_F32, Vop2SdwaProfile::Float32},
     {0x04u, Opcode::V_SUB_F32, Vop2SdwaProfile::Float32},
@@ -225,22 +224,26 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x96u, Opcode::V_CMPX_GE_I32},       {0xa9u, Opcode::V_CMP_LT_U16},
     {0xaau, Opcode::V_CMP_EQ_U16},        {0xabu, Opcode::V_CMP_LE_U16},
     {0xacu, Opcode::V_CMP_GT_U16},        {0xadu, Opcode::V_CMP_NE_U16},
-    {0xaeu, Opcode::V_CMP_GE_U16},        {0xc0u, Opcode::V_CMP_F_U32},
+    {0xaeu, Opcode::V_CMP_GE_U16},        {0xbcu, Opcode::V_CMPX_GT_U16},
+    {0xc0u, Opcode::V_CMP_F_U32},
     {0xc1u, Opcode::V_CMP_LT_U32},        {0xc2u, Opcode::V_CMP_EQ_U32},
     {0xc3u, Opcode::V_CMP_LE_U32},        {0xc4u, Opcode::V_CMP_GT_U32},
     {0xc5u, Opcode::V_CMP_NE_U32},        {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},         {0xa2u, Opcode::V_CMP_EQ_I64, false},
+    {0xb5u, Opcode::V_CMPX_NE_I64, false},
     {0xd1u, Opcode::V_CMPX_LT_U32},       {0xd2u, Opcode::V_CMPX_EQ_U32},
     {0xd3u, Opcode::V_CMPX_LE_U32},       {0xd4u, Opcode::V_CMPX_GT_U32},
     {0xd5u, Opcode::V_CMPX_NE_U32},       {0xd6u, Opcode::V_CMPX_GE_U32},
     {0xe4u, Opcode::V_CMP_GT_U64, false}, {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xf5u, Opcode::V_CMPX_NE_U64, false},
     {0xc9u, Opcode::V_CMP_LT_F16},        {0xcau, Opcode::V_CMP_EQ_F16},
     {0xcbu, Opcode::V_CMP_LE_F16},        {0xccu, Opcode::V_CMP_GT_F16},
     {0xcdu, Opcode::V_CMP_LG_F16},        {0xceu, Opcode::V_CMP_GE_F16},
     {0xedu, Opcode::V_CMP_NEQ_F16},       {0xd9u, Opcode::V_CMPX_LT_F16},
     {0xdau, Opcode::V_CMPX_EQ_F16},       {0xdbu, Opcode::V_CMPX_LE_F16},
     {0xdcu, Opcode::V_CMPX_GT_F16},       {0xdeu, Opcode::V_CMPX_GE_F16},
-    {0xfdu, Opcode::V_CMPX_NEQ_F16},      {0xfeu, Opcode::V_CMPX_NLT_F16},
+    {0xfbu, Opcode::V_CMPX_NGT_F16},      {0xfdu, Opcode::V_CMPX_NEQ_F16},
+    {0xfeu, Opcode::V_CMPX_NLT_F16},
 };
 
 constexpr auto VOPC_OPS = Detail::MakeOpcodeTable<0x100>(VOPC_OPCODE_LIST);
@@ -280,6 +283,8 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x16au, Opcode::V_MUL_HI_U32},
     {0x16bu, Opcode::V_MUL_LO_I32},
     {0x16cu, Opcode::V_MUL_HI_I32},
+    {0x2ffu, Opcode::V_LSHLREV_B64},
+    {0x300u, Opcode::V_LSHRREV_B64},
     {0x303u, Opcode::V_ADD_NC_U16},
     {0x304u, Opcode::V_SUB_NC_U16},
     {0x307u, Opcode::V_LSHRREV_B16},
@@ -870,6 +875,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_LE_F16:
 		case Opcode::V_CMPX_GT_F16:
 		case Opcode::V_CMPX_GE_F16:
+		case Opcode::V_CMPX_NGT_F16:
 		case Opcode::V_CMPX_NEQ_F16:
 		case Opcode::V_CMPX_NLT_F16:
 		case Opcode::V_CMP_CLASS_F32: return true;
@@ -929,8 +935,6 @@ constexpr uint32_t SdwaSelAll() {
 
 constexpr Vop2SdwaRule VOP2_SDWA_RULES[] = {
     {},
-    {SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(),
-     true, false},
     {SdwaSelFull(), SdwaSelFull(), SdwaSelFull(), false, true},
     {SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(),
      true, true},
@@ -1248,6 +1252,8 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 		case Opcode::V_MIN_I16:
 		case Opcode::V_ADD_NC_I16:
 		case Opcode::V_SUB_NC_I16:
+		case Opcode::V_LSHLREV_B64:
+		case Opcode::V_LSHRREV_B64:
 		case Opcode::V_LSHLREV_B16:
 		case Opcode::V_LSHRREV_B16:
 		case Opcode::V_ASHRREV_I16:
@@ -1501,11 +1507,15 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_GT_U32:
 		case Opcode::V_CMPX_NE_U32:
 		case Opcode::V_CMPX_GE_U32:
+		case Opcode::V_CMPX_NE_I64:
+		case Opcode::V_CMPX_NE_U64:
+		case Opcode::V_CMPX_GT_U16:
 		case Opcode::V_CMPX_LT_F16:
 		case Opcode::V_CMPX_EQ_F16:
 		case Opcode::V_CMPX_LE_F16:
 		case Opcode::V_CMPX_GT_F16:
 		case Opcode::V_CMPX_GE_F16:
+		case Opcode::V_CMPX_NGT_F16:
 		case Opcode::V_CMPX_NEQ_F16:
 		case Opcode::V_CMPX_NLT_F16: return true;
 		default: return false;