graphics: replace command fences with timeline reuse

This reverts commit 7012634f32.

CMakeLists.txt

@@ -254,7 +254,7 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 	add_custom_command(
 		OUTPUT "${shader_header}"
 		COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}"
-		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
 		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
@@ -278,7 +278,7 @@ foreach(shader_source IN LISTS gpu_blit_shader_sources)
 	add_custom_command(
 		OUTPUT "${shader_header}"
 		COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_blit_generated_dir}"
-		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
 		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
@@ -430,6 +430,7 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderPixelParameter.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
@@ -483,7 +484,7 @@ set(gpu_test_ms_depth_header
 add_custom_command(
 	OUTPUT "${gpu_test_ms_depth_header}"
 	COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_test_generated_dir}"
-	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 	        -o "${gpu_test_ms_depth_spv}" "${gpu_test_ms_depth_source}"
 	COMMAND ${CMAKE_COMMAND} -DINPUT=${gpu_test_ms_depth_spv}
 	        -DOUTPUT=${gpu_test_ms_depth_header} -DSYMBOL=GPU_TEST_MS_DEPTH_SPV

src/emulator.cpp

@@ -15,6 +15,7 @@
 #include "kernel/fileSystem.h"
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
+#include "kytyGitVersion.h"
 #include "libs/agc.h"
 #include "libs/audio.h"
 #include "libs/controller.h"
@@ -26,13 +27,31 @@
 
 #include <cstdlib>
 #include <filesystem>
+#include <thread>
 
 namespace Emulator {
 
 static void PrintSystemInfo() {
-	Common::SystemInfo info = Common::GetSystemInfo();
+	const Common::SystemInfo info = Common::GetSystemInfo();
 
-	LOGF("ProcessorName = %s\n", info.ProcessorName.c_str());
+#if defined(__APPLE__)
+	static constexpr auto platform_name = "macOS";
+#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	static constexpr auto platform_name = "Windows";
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	static constexpr auto platform_name = "Linux";
+#else
+	static constexpr auto platform_name = "Unknown";
+#endif
+
+	LOGF("Build\n"
+	     "  version: %s\n\n"
+	     "Host\n"
+	     "  os:      %s\n"
+	     "  cpu:     %s\n"
+	     "  threads: %u\n\n",
+	     KYTY_BUILD_LABEL, platform_name, info.ProcessorName.c_str(),
+	     std::thread::hardware_concurrency());
 }
 
 static void KytyClose() {

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -7,6 +7,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 
 #include <cstdint>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -30,10 +31,9 @@ private:
 	friend class CommandProcessor;
 
 	struct BufferCursor {
-		uint32_t* next_packet         = nullptr;
-		uint32_t  remaining_dw        = 0;
-		uint32_t  total_dw            = 0;
-		uint32_t  deferred_advance_dw = 0;
+		std::span<const uint32_t> commands;
+		uint32_t                  offset_dw           = 0;
+		uint32_t                  deferred_advance_dw = 0;
 	};
 
 	std::vector<BufferCursor> m_buffer_stack;
@@ -50,7 +50,8 @@ public:
 		int64_t flip_arg  = 0;
 	};
 
-	explicit CommandProcessor(RenderContext& renderer): m_renderer(renderer) {}
+	CommandProcessor(RenderContext& renderer, int interrupt_event_id)
+	    : m_renderer(renderer), m_interrupt_event_id(interrupt_event_id) {}
 	~CommandProcessor() = default;
 
 	KYTY_CLASS_NO_COPY(CommandProcessor);
@@ -131,13 +132,14 @@ public:
 	                    const volatile void* address, uint32_t count_in_dwords);
 	[[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; }
 
-	Pm4ProcessResult Process(Pm4Execution& execution, uint32_t* buffer, uint32_t size_dw);
-	void             ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw);
+	Pm4ProcessResult Process(Pm4Execution& execution, std::span<const uint32_t> commands);
+	void             ProcessIndirectBuffer(std::span<const uint32_t> commands);
 
 	void SetFlip(const FlipInfo& flip) { m_flip = flip; }
 
 	[[nodiscard]] uint64_t GetSubmitId() const { return m_submit_id; }
 	void                   SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; }
+	[[nodiscard]] bool     IsAsyncComputeQueue() const { return m_interrupt_event_id >= 0x20; }
 
 private:
 	template <typename T>
@@ -177,10 +179,11 @@ private:
 
 	uint32_t m_const_ram[0x3000] = {0};
 
-	FlipInfo m_flip;
-	uint64_t m_submit_id                   = 0;
-	uint64_t m_synthetic_occlusion_counter = 0;
-	bool     m_predicate_skip              = false;
+	FlipInfo  m_flip;
+	const int m_interrupt_event_id;
+	uint64_t  m_submit_id                   = 0;
+	uint64_t  m_synthetic_occlusion_counter = 0;
+	bool      m_predicate_skip              = false;
 };
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -1859,11 +1859,11 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	uint64_t reference   = buffer[5] | (static_cast<uint64_t>(buffer[6]) << 32u);
 	uint32_t mode        = buffer[0] & 0x3u;
 	uint32_t function    = (buffer[0] >> 8u) & 0x7u;
-	auto*    then_buffer = reinterpret_cast<uint32_t*>((buffer[7] & 0xfffffffcu) |
-	                                                   (static_cast<uint64_t>(buffer[8]) << 32u));
+	auto*    then_buffer = reinterpret_cast<const uint32_t*>(
+	    (buffer[7] & 0xfffffffcu) | (static_cast<uint64_t>(buffer[8]) << 32u));
 	uint32_t then_num_dw = buffer[9] & 0xfffffu;
-	auto*    else_buffer = reinterpret_cast<uint32_t*>((buffer[10] & 0xfffffffcu) |
-	                                                   (static_cast<uint64_t>(buffer[11]) << 32u));
+	auto*    else_buffer = reinterpret_cast<const uint32_t*>(
+	    (buffer[10] & 0xfffffffcu) | (static_cast<uint64_t>(buffer[11]) << 32u));
 	uint32_t else_num_dw = buffer[12] & 0xfffffu;
 
 	EXIT_NOT_IMPLEMENTED(compare_addr == nullptr);
@@ -1877,10 +1877,10 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	     reinterpret_cast<uint64_t>(else_buffer), else_num_dw);
 
 	if (take_then) {
-		cp.ProcessIndirectBuffer(then_buffer, then_num_dw);
+		cp.ProcessIndirectBuffer({then_buffer, then_num_dw});
 	} else if (mode == 2 && else_num_dw != 0) {
 		EXIT_NOT_IMPLEMENTED(else_buffer == nullptr);
-		cp.ProcessIndirectBuffer(else_buffer, else_num_dw);
+		cp.ProcessIndirectBuffer({else_buffer, else_num_dw});
 	}
 
 	return payload_dw;
@@ -2321,7 +2321,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 	}
 
 	auto* indirect_buffer =
-	    reinterpret_cast<uint32_t*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
+	    reinterpret_cast<const uint32_t*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
 	uint32_t                     indirect_num_dw = control & 0xfffffu;
 	static std::atomic<uint32_t> indirect_log_count {0};
 	if (indirect_log_count.fetch_add(1) < 128) {
@@ -2341,7 +2341,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 
 	GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer);
 
-	cp.ProcessIndirectBuffer(indirect_buffer, indirect_num_dw);
+	cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw});
 
 	return 3;
 }

src/graphics/guest_gpu/graphicsRun.cpp

@@ -66,17 +66,10 @@ static bool GraphicsRunDebugDumpEnabled() {
 	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
 }
 
-GuestGpu::OwnedCmdBuffer::OwnedCmdBuffer(const uint32_t* data, uint32_t count) {
-	EXIT_IF(data == nullptr && count != 0);
-	if (count != 0) {
-		m_words.assign(data, data + count);
-	}
-}
-
 GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	GraphicsInitJmpTables();
-	m_gfx_cp = std::make_unique<CommandProcessor>(renderer);
+	m_gfx_cp = std::make_unique<CommandProcessor>(renderer, 0);
 	m_thread = std::jthread(ThreadRun, this);
 }
 
@@ -148,39 +141,33 @@ void GuestGpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	done.acquire();
 }
 
-void GuestGpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
-                      uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
-	if (num_draw_dw == 0) {
+void GuestGpu::Submit(std::span<const uint32_t> draw_commands,
+                      std::span<const uint32_t> constant_commands) {
+	if (draw_commands.empty()) {
 		return;
 	}
-	EXIT_IF(cmd_draw_buffer == nullptr);
 	GpuMutexLock lock(m_submission_mutex);
 	Submission   submission;
-	submission.type                          = SubmissionType::Graphics;
-	submission.queue_id                      = 0;
-	submission.commands                      = OwnedCmdBuffer(cmd_draw_buffer, num_draw_dw);
-	submission.constant_commands             = OwnedCmdBuffer(cmd_const_buffer, num_const_dw);
-	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
-	submission.reset_processor               = m_graphics_done;
-	m_graphics_done                          = false;
+	submission.type              = SubmissionType::Graphics;
+	submission.queue_id          = 0;
+	submission.commands          = draw_commands;
+	submission.constant_commands = constant_commands;
+	submission.reset_processor   = m_graphics_done;
+	m_graphics_done              = false;
 	Enqueue(std::move(submission));
 }
 
-void GuestGpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
-                             bool trigger_agc_interrupt_on_done) {
-	EXIT_IF(cmd_buffer == nullptr || num_dw == 0);
+void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands) {
+	EXIT_IF(commands.empty());
 	GpuMutexLock lock(m_submission_mutex);
 
-	constexpr uint32_t compute_queue_base = 0x20u;
-	EXIT_NOT_IMPLEMENTED(queue < compute_queue_base ||
-	                     queue >= compute_queue_base + ComputeQueueCount);
+	EXIT_NOT_IMPLEMENTED(queue < ComputeQueueBase || queue >= ComputeQueueBase + ComputeQueueCount);
 
-	const auto compute_queue = queue - compute_queue_base;
+	const auto compute_queue = queue - ComputeQueueBase;
 	Submission submission;
-	submission.type                          = SubmissionType::Compute;
-	submission.queue_id                      = 1 + compute_queue;
-	submission.commands                      = OwnedCmdBuffer(cmd_buffer, num_dw);
-	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
+	submission.type     = SubmissionType::Compute;
+	submission.queue_id = 1 + compute_queue;
+	submission.commands = commands;
 	Enqueue(std::move(submission));
 }
 
@@ -215,7 +202,7 @@ CommandProcessor& GuestGpu::GetProcessor(uint32_t queue_id) {
 	}
 	auto& processor = m_compute_cp[queue_id - 1];
 	if (processor == nullptr) {
-		processor = std::make_unique<CommandProcessor>(m_renderer);
+		processor = std::make_unique<CommandProcessor>(m_renderer, ComputeQueueBase + queue_id - 1);
 	}
 	return *processor;
 }
@@ -268,8 +255,7 @@ void CommandProcessor::BufferFlush() {
 }
 
 void CommandProcessor::BufferFlushAndWait() {
-	auto& submitted = GetScheduler().FlushAndGetSubmitted();
-	submitted.WaitForFenceOnly();
+	GetScheduler().FlushAndWait();
 }
 
 void CommandProcessor::BufferWait() {
@@ -583,21 +569,20 @@ bool GuestGpu::Process(Submission& submission) {
 	switch (submission.type) {
 		case SubmissionType::Graphics: {
 			bool progressed = false;
-			submission.constant_complete |= submission.constant_commands.Empty();
+			submission.constant_complete |= submission.constant_commands.empty();
 			for (;;) {
 				bool round_progress = false;
 				if (!submission.constant_complete) {
 					submission.constant_complete =
-					    cp.Process(
-					        submission.constant_execution, submission.constant_commands.Data(),
-					        submission.constant_commands.Size()) == Pm4ProcessResult::Complete;
+					    cp.Process(submission.constant_execution, submission.constant_commands) ==
+					    Pm4ProcessResult::Complete;
 					round_progress |= submission.constant_execution.MadeProgress();
 				}
 				cp.SetCeComplete(submission.constant_complete);
 				if (!submission.command_complete) {
 					submission.command_complete =
-					    cp.Process(submission.command_execution, submission.commands.Data(),
-					               submission.commands.Size()) == Pm4ProcessResult::Complete;
+					    cp.Process(submission.command_execution, submission.commands) ==
+					    Pm4ProcessResult::Complete;
 					round_progress |= submission.command_execution.MadeProgress();
 				}
 				progressed |= round_progress;
@@ -614,14 +599,11 @@ bool GuestGpu::Process(Submission& submission) {
 			} else if (complete) {
 				m_renderer.GetGpuResources().RunGarbageCollector();
 			}
-			if (complete && submission.trigger_agc_interrupt_on_done) {
-				m_renderer.TriggerEopEvent(0);
-			}
 			break;
 		}
 		case SubmissionType::Compute: {
-			const auto      num_dw                  = submission.commands.Size();
-			auto*           buffer                  = submission.commands.Data();
+			const auto      num_dw = static_cast<uint32_t>(submission.commands.size());
+			const auto*     buffer = submission.commands.data();
 			static uint32_t compute_batch_log_count = 0;
 			if (first_slice && num_dw <= 128 && compute_batch_log_count++ < 32) {
 				LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n",
@@ -633,7 +615,7 @@ bool GuestGpu::Process(Submission& submission) {
 			if (first_slice) {
 				GraphicsDbgDumpDcb("cc", num_dw, buffer);
 			}
-			complete = cp.Process(submission.command_execution, buffer, num_dw) ==
+			complete = cp.Process(submission.command_execution, submission.commands) ==
 			           Pm4ProcessResult::Complete;
 			if (submission.command_execution.MadeProgress()) {
 				if (complete) {
@@ -643,9 +625,6 @@ bool GuestGpu::Process(Submission& submission) {
 			} else if (complete) {
 				m_renderer.GetGpuResources().RunGarbageCollector();
 			}
-			if (complete && submission.trigger_agc_interrupt_on_done) {
-				Sync::TriggerAgcUserInterrupt();
-			}
 			break;
 		}
 		case SubmissionType::FlipPreparation:
@@ -657,12 +636,13 @@ bool GuestGpu::Process(Submission& submission) {
 	return complete;
 }
 
-Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer,
-                                           uint32_t size_dw) {
+Pm4ProcessResult CommandProcessor::Process(Pm4Execution&             execution,
+                                           std::span<const uint32_t> commands) {
 	KYTY_PROFILER_BLOCK("CommandProcessor::Process");
 	EXIT_IF(g_current_execution != nullptr);
-	if (execution.m_buffer_stack.empty() && size_dw != 0) {
-		execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0});
+	EXIT_IF(commands.size() > UINT32_MAX);
+	if (execution.m_buffer_stack.empty() && !commands.empty()) {
+		execution.m_buffer_stack.push_back({commands});
 	}
 	execution.m_suspended     = false;
 	execution.m_made_progress = false;
@@ -687,14 +667,14 @@ Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* bu
 	                                        : Pm4ProcessResult::Blocked;
 }
 
-void CommandProcessor::ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw) {
+void CommandProcessor::ProcessIndirectBuffer(std::span<const uint32_t> commands) {
 	EXIT_IF(g_current_execution == nullptr);
-	if (size_dw == 0) {
+	if (commands.empty()) {
 		return;
 	}
 	auto&      execution  = *g_current_execution;
 	const auto stop_depth = execution.m_buffer_stack.size();
-	execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0});
+	execution.m_buffer_stack.push_back({commands});
 	ProcessPm4(execution, stop_depth);
 }
 
@@ -710,29 +690,28 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 		}
 		const auto buffer_index = execution.m_buffer_stack.size() - 1;
 		auto&      cursor       = execution.m_buffer_stack[buffer_index];
+		EXIT_IF(cursor.offset_dw > cursor.commands.size());
 		if (cursor.deferred_advance_dw != 0) {
-			EXIT_IF(cursor.deferred_advance_dw > cursor.remaining_dw);
-			cursor.next_packet += cursor.deferred_advance_dw;
-			cursor.remaining_dw -= cursor.deferred_advance_dw;
+			EXIT_IF(cursor.deferred_advance_dw > cursor.commands.size() - cursor.offset_dw);
+			cursor.offset_dw += cursor.deferred_advance_dw;
 			cursor.deferred_advance_dw = 0;
 			execution.m_made_progress  = true;
 			continue;
 		}
-		if (cursor.remaining_dw == 0) {
+		if (cursor.offset_dw == cursor.commands.size()) {
 			execution.m_buffer_stack.pop_back();
 			continue;
 		}
 
-		auto* const packet        = cursor.next_packet;
-		const auto  remaining_dw  = cursor.remaining_dw;
-		const auto  total_dw      = cursor.total_dw;
-		const auto  packet_header = packet[0];
-		const auto  opcode        = (packet_header >> 8u) & 0xffu;
+		const auto* const packet        = cursor.commands.data() + cursor.offset_dw;
+		const auto        total_dw      = static_cast<uint32_t>(cursor.commands.size());
+		const auto        remaining_dw  = total_dw - cursor.offset_dw;
+		const auto        packet_header = packet[0];
+		const auto        opcode        = (packet_header >> 8u) & 0xffu;
 		EXIT_NOT_IMPLEMENTED(remaining_dw > total_dw);
 
 		if (packet_header == 0x80000000u) {
-			cursor.next_packet++;
-			cursor.remaining_dw--;
+			cursor.offset_dw++;
 			execution.m_made_progress = true;
 			continue;
 		}
@@ -767,8 +746,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 					     dst, val, packet[1], packet[2]);
 				}
 			}
-			cursor.next_packet += packet_dw;
-			cursor.remaining_dw -= packet_dw;
+			cursor.offset_dw += packet_dw;
 			execution.m_made_progress = true;
 			continue;
 		}
@@ -801,8 +779,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 			return;
 		}
 		EXIT_IF(execution.m_buffer_stack.size() != buffer_index + 1);
-		execution.m_buffer_stack[buffer_index].next_packet += packet_dw;
-		execution.m_buffer_stack[buffer_index].remaining_dw -= packet_dw;
+		execution.m_buffer_stack[buffer_index].offset_dw += packet_dw;
 		execution.m_made_progress = true;
 	}
 }
@@ -1245,7 +1222,14 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 	switch (interrupt_selector) {
 		case 0x00:
 		case 0x03: with_interrupt = false; break;
-		case 0x01: Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); return;
+		case 0x01:
+			if (!IsAsyncComputeQueue()) {
+				Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id,
+				                                 interrupt_context_id);
+				return;
+			}
+			with_interrupt = true;
+			break;
 		case 0x02: with_interrupt = true; break;
 		default: EXIT("unknown interrupt_selector\n");
 	}
@@ -1258,10 +1242,11 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		if (with_interrupt) {
 			if (with_writeback) {
 				Sync::WriteAtEndOfPipeWithInterruptWriteBack32(m_submit_id, CurrentBuffer(), dst,
-				                                               data, interrupt_context_id);
+				                                               data, m_interrupt_event_id,
+				                                               interrupt_context_id);
 			} else {
 				Sync::WriteAtEndOfPipeWithInterrupt32(m_submit_id, CurrentBuffer(), dst, data,
-				                                      interrupt_context_id);
+				                                      m_interrupt_event_id, interrupt_context_id);
 			}
 		} else if (with_writeback) {
 			Sync::WriteAtEndOfPipeWithWriteBack32(m_submit_id, CurrentBuffer(), dst, data);
@@ -1281,7 +1266,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
 					                            value >> 16u);
 					if (with_interrupt) {
-						m_renderer.TriggerEopEvent(interrupt_context_id);
+						m_renderer.TriggerInterrupt(m_interrupt_event_id, interrupt_context_id);
 					}
 					return;
 				}
@@ -1307,10 +1292,12 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					if (with_interrupt) {
 						if (with_writeback) {
 							Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
-							    m_submit_id, CurrentBuffer(), dst, value, interrupt_context_id);
+							    m_submit_id, CurrentBuffer(), dst, value, m_interrupt_event_id,
+							    interrupt_context_id);
 						} else {
 							Sync::WriteAtEndOfPipeWithInterrupt64(m_submit_id, CurrentBuffer(), dst,
-							                                      value, interrupt_context_id);
+							                                      value, m_interrupt_event_id,
+							                                      interrupt_context_id);
 						}
 					} else if (with_writeback) {
 						Sync::WriteAtEndOfPipeWithWriteBack64(m_submit_id, CurrentBuffer(), dst,
@@ -1389,26 +1376,35 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		case 0x04:
 			if constexpr (sizeof(T) == sizeof(uint64_t)) {
 				const auto clock = Sync::ReadReferenceClock();
-				std::memcpy(dst_gpu_addr, &clock, sizeof(clock));
+				auto*      dst   = static_cast<uint64_t*>(dst_gpu_addr);
+				std::memcpy(dst, &clock, sizeof(clock));
 				switch (cache_action) {
 					case 0x00:
-						if (((eop_event_type == 0x04 && event_index == 0x05) ||
-						     (eop_event_type == 0x28 && event_index == 0x00)) &&
-						    !with_interrupt) {
-							Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
-							                                   static_cast<uint64_t*>(dst_gpu_addr),
-							                                   clock);
+						if ((eop_event_type == 0x04 && event_index == 0x05) ||
+						    (eop_event_type == 0x28 && event_index == 0x00)) {
+							if (with_interrupt) {
+								Sync::WriteAtEndOfPipeWithInterrupt64(
+								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    interrupt_context_id);
+							} else {
+								Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
+								                                   dst, clock);
+							}
 							return;
 						}
 						break;
 					case 0x38:
-						if (((eop_event_type == 0x04 &&
-						      (event_index == 0x00 || event_index == 0x05)) ||
-						     (eop_event_type == 0x28 && event_index == 0x00)) &&
-						    !with_interrupt) {
-							Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
-							    m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
-							    clock);
+						if ((eop_event_type == 0x04 &&
+						     (event_index == 0x00 || event_index == 0x05)) ||
+						    (eop_event_type == 0x28 && event_index == 0x00)) {
+							if (with_interrupt) {
+								Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
+								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    interrupt_context_id);
+							} else {
+								Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
+								    m_submit_id, CurrentBuffer(), dst, clock);
+							}
 							return;
 						}
 						break;
@@ -1465,7 +1461,7 @@ void CommandProcessor::EmitGlobalBarrier() {
 void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) {
 	CheckBuffer();
 
-	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id);
+	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id, interrupt_context_id);
 }
 
 void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
@@ -1604,7 +1600,7 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
 	                                         m_flip.flip_arg);
 	Sync::WriteAtEndOfPipeWithInterruptWriteBackFlip32(
 	    m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
-	    m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
+	    m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request, m_interrupt_event_id);
 	GetScheduler().Flush();
 }
 
@@ -1625,7 +1621,7 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 }
 
 void CommandProcessor::SynchronizeGpu() {
-	GetScheduler().FinishCurrent();
+	GetScheduler().Finish();
 }
 
 bool GuestGpu::IsGpuThread() noexcept {

src/graphics/guest_gpu/graphicsRun.h

@@ -13,8 +13,8 @@
 #include <deque>
 #include <memory>
 #include <mutex>
+#include <span>
 #include <thread>
-#include <vector>
 
 namespace Libs::Graphics {
 
@@ -31,12 +31,12 @@ public:
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
 
-	void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
-	            uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false);
-	void SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw,
-	                   bool trigger_agc_interrupt_on_done = false);
-	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
+	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
+	void              Submit(std::span<const uint32_t> draw_commands,
+	                         std::span<const uint32_t> constant_commands);
+	void              SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
+	void              SubmitFlipPreparation(uint64_t request_id);
+	void              Done();
 	[[nodiscard]] int GetFrameNum() const;
 
 	[[nodiscard]] static bool IsGpuThread() noexcept;
@@ -45,39 +45,24 @@ private:
 	static constexpr uint32_t ComputePipeCount     = 7;
 	static constexpr uint32_t QueuesPerComputePipe = 8;
 	static constexpr uint32_t ComputeQueueCount    = ComputePipeCount * QueuesPerComputePipe;
+	static constexpr uint32_t ComputeQueueBase     = 0x20;
 	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
 
-	class OwnedCmdBuffer {
-	public:
-		OwnedCmdBuffer() = default;
-		explicit OwnedCmdBuffer(const uint32_t* data, uint32_t count);
-
-		[[nodiscard]] bool     Empty() const noexcept { return m_words.empty(); }
-		[[nodiscard]] uint32_t Size() const noexcept {
-			return static_cast<uint32_t>(m_words.size());
-		}
-		[[nodiscard]] uint32_t* Data() noexcept { return m_words.data(); }
-
-	private:
-		std::vector<uint32_t> m_words;
-	};
-
 	enum class SubmissionType { Graphics, Compute, FlipPreparation };
 
 	struct Submission {
-		SubmissionType type     = SubmissionType::Graphics;
-		uint32_t       queue_id = 0;
-		OwnedCmdBuffer commands;
-		OwnedCmdBuffer constant_commands;
-		Pm4Execution   command_execution;
-		Pm4Execution   constant_execution;
-		bool           trigger_agc_interrupt_on_done = false;
-		bool           reset_processor               = false;
-		bool           started                       = false;
-		bool           command_complete              = false;
-		bool           constant_complete             = false;
-		bool           blocked                       = false;
-		uint64_t       flip_request_id               = 0;
+		SubmissionType            type     = SubmissionType::Graphics;
+		uint32_t                  queue_id = 0;
+		std::span<const uint32_t> commands;
+		std::span<const uint32_t> constant_commands;
+		Pm4Execution              command_execution;
+		Pm4Execution              constant_execution;
+		bool                      reset_processor   = false;
+		bool                      started           = false;
+		bool                      command_complete  = false;
+		bool                      constant_complete = false;
+		bool                      blocked           = false;
+		uint64_t                  flip_request_id   = 0;
 	};
 
 	void              Enqueue(Submission submission);

src/graphics/guest_gpu/pm4.cpp

@@ -100,7 +100,7 @@ constexpr auto g_opcode_names   = MakeOpcodeNames();
 
 } // namespace
 
-void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw,
+void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_t start_dw,
                          uint32_t num_dw) {
 	// db_dump();
 
@@ -143,9 +143,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 				if ((op == IT_SET_CONTEXT_REG_INDIRECT || op == IT_SET_SH_REG_INDIRECT ||
 				     op == IT_SET_UCONFIG_REG_INDIRECT) &&
 				    len == 4) {
-					auto* indirect_buffer =
-					    reinterpret_cast<uint32_t*>((static_cast<uint64_t>(cmd[0]) & 0xfffffffcu) |
-					                                (static_cast<uint64_t>(cmd[1]) << 32u));
+					auto* indirect_buffer = reinterpret_cast<const uint32_t*>(
+					    (static_cast<uint64_t>(cmd[0]) & 0xfffffffcu) |
+					    (static_cast<uint64_t>(cmd[1]) << 32u));
 					const uint32_t indirect_num_regs = cmd[3] & 0x3fffu;
 
 					for (uint32_t i = 0; indirect_buffer != nullptr && i < indirect_num_regs;

src/graphics/guest_gpu/pm4.h

@@ -1088,7 +1088,7 @@ constexpr uint32_t UC_NOP                       = 0x80003FFF;
 
 constexpr uint32_t UC_NUM = 0x3FFF + 1;
 
-void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw,
+void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_t start_dw,
                          uint32_t num_dw);
 
 } // namespace Libs::Graphics::Pm4

src/graphics/host_gpu/rangeSet.h

@@ -59,6 +59,15 @@ public:
 		return result;
 	}
 
+	[[nodiscard]] bool Intersects(uint64_t address, uint64_t size) const {
+		const auto end = End(address, size);
+		auto       it  = m_ranges.lower_bound(address);
+		if (it != m_ranges.begin() && std::prev(it)->second > address) {
+			return true;
+		}
+		return it != m_ranges.end() && it->first < end;
+	}
+
 	[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.upper_bound(address);

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -16,6 +16,7 @@
 #include <array>
 #include <cinttypes>
 #include <cstring>
+#include <memory>
 #include <utility>
 #include <vector>
 
@@ -49,17 +50,33 @@ struct BufferCache::DownloadCopy {
 	uint64_t size          = 0;
 };
 
-struct BufferCache::DownloadRange {
-	uint64_t address = 0;
-	uint64_t size    = 0;
-	uint64_t offset  = 0;
-};
+void BufferCache::Register(BufferId id) {
+	auto& buffer = m_slot_buffers[id];
+	PageTable::PageRange pages {};
+	EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		m_page_table[page] = id;
+	}
+	const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
+	(void)it;
+	EXIT_IF(!inserted);
+	buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
+	m_total_used_memory += buffer.Size();
+}
 
 void BufferCache::Unregister(BufferId id) {
 	auto& buffer = m_slot_buffers[id];
 	if (buffer.is_deleted) {
 		return;
 	}
+	PageTable::PageRange pages {};
+	EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
+	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+		auto* owner = m_page_table.Find(page);
+		if (owner != nullptr && *owner == id) {
+			*owner = {};
+		}
+	}
 	const auto found = m_buffers.find(buffer.CpuAddress());
 	EXIT_IF(found == m_buffers.end() || found->second != id);
 	m_buffers.erase(found);
@@ -67,9 +84,16 @@ void BufferCache::Unregister(BufferId id) {
 		EXIT("BufferCache: allocation accounting underflow\n");
 	}
 	m_total_used_memory -= buffer.Size();
+	m_lru_cache.Free(buffer.lru_id);
 	buffer.is_deleted = true;
 }
 
+void BufferCache::TouchBuffer(const Buffer& buffer) {
+	if (!buffer.is_deleted) {
+		m_lru_cache.Touch(buffer.lru_id, m_gc_tick);
+	}
+}
+
 void BufferCache::DeleteBuffer(BufferId id) {
 	auto* buffer = m_slot_buffers.try_get(id);
 	if (buffer == nullptr || buffer->is_deleted) {
@@ -100,79 +124,62 @@ std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy&
 	return {begin, end - begin};
 }
 
-std::vector<BufferCache::DownloadRange>
-BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
-	uint64_t reservation_size = 0;
+void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
+	std::vector<DownloadCopy> batch;
+	batch.reserve(copies.size());
+	uint64_t                  packed_size = 0;
+	auto&                     download    = m_download_buffer;
+	const auto flush = [&] {
+		const auto [mapped, base_offset] = download.Map(packed_size, DOWNLOAD_ALIGNMENT);
+		EXIT_IF(mapped == nullptr);
+		uint64_t cursor = 0;
+		for (const auto& copy: batch) {
+			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
+			download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
+			                  envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
+			                  vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
+			                  vk::AccessFlagBits::eHostRead);
+			cursor += AlignDownload(envelope_size);
+		}
+		download.Commit();
+		const auto completion_tick = m_scheduler.CurrentTick();
+		m_scheduler.Finish();
+		m_scheduler.WaitPriorityOperations(completion_tick);
+		cursor = 0;
+		for (const auto& copy: batch) {
+			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
+			const auto offset = cursor + copy.source_offset - source_begin;
+			download.Invalidate(base_offset + offset, copy.size);
+			Libs::LibKernel::Memory::WriteBacking(copy.address, mapped + offset, copy.size);
+			cursor += AlignDownload(envelope_size);
+		}
+		batch.clear();
+		packed_size = 0;
+	};
+	for (auto copy: copies) {
+		while (copy.size != 0) {
+			const auto available = download.Size() - packed_size;
+			const auto prefix    = copy.source_offset & 3u;
+			const auto bytes     = std::min(copy.size, available - prefix);
+			DownloadCopy part {copy.buffer, copy.source_offset, copy.address, bytes};
+			const auto [source_begin, envelope_size] = DownloadEnvelope(part);
+			(void)source_begin;
+			packed_size += AlignDownload(envelope_size);
+			batch.push_back(part);
+			copy.source_offset += bytes;
+			copy.address += bytes;
+			copy.size -= bytes;
+			if (packed_size == download.Size()) {
+				flush();
+			}
+		}
+	}
+	if (!batch.empty()) {
+		flush();
+	}
 	for (const auto& copy: copies) {
-		const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-		(void)source_begin;
-		if (envelope_size > UINT64_MAX - (DOWNLOAD_ALIGNMENT - 1)) {
-			EXIT("BufferCache: download batch alignment overflow\n");
-		}
-		const auto aligned_size = AlignDownload(envelope_size);
-		if (aligned_size > UINT64_MAX - reservation_size) {
-			EXIT("BufferCache: download batch overflow\n");
-		}
-		reservation_size += aligned_size;
+		m_gpu_modified_ranges.Subtract(copy.address, copy.size);
 	}
-	if (reservation_size == 0) {
-		return {};
-	}
-
-	auto& download                   = m_download_buffer;
-	const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
-	if (mapped == nullptr) {
-		EXIT("BufferCache: download batch could not reserve the shared stream\n");
-	}
-
-	std::vector<DownloadRange> downloads;
-	downloads.reserve(copies.size());
-	uint64_t cursor = 0;
-	for (const auto& copy: copies) {
-		const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-		const auto prefix                        = copy.source_offset - source_begin;
-		download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
-		                  envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
-		                  vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
-		                  vk::AccessFlagBits::eHostRead);
-		downloads.push_back({copy.address, copy.size, base_offset + cursor + prefix});
-		cursor += AlignDownload(envelope_size);
-	}
-	download.Commit();
-	return downloads;
-}
-
-void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
-	for (const auto& range: downloads) {
-		m_download_buffer.Invalidate(range.offset, range.size);
-		Libs::LibKernel::Memory::WriteBacking(
-		    range.address, m_download_buffer.Mapped().data() + range.offset, range.size);
-	}
-}
-
-void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, BufferId id,
-                                       uint64_t address, uint64_t size) {
-	EXIT_IF(copies.empty());
-	auto downloads = RecordDownloads(copies);
-	auto retired   = downloads;
-	m_scheduler.DeferPriorityOperation(
-	    [this, downloads = std::move(downloads)]() mutable { PublishDownloads(downloads); });
-	m_scheduler.DeferOperation([this, retired = std::move(retired), id, address, size]() mutable {
-		for (const auto& range: retired) {
-			m_gpu_modified_ranges.Subtract(range.address, range.size);
-		}
-		// ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
-		// interval on those pages was downloaded and removed. Clearing the original
-		// query therefore cannot orphan a dirty sibling on an edge page.
-		m_memory_tracker.UnmarkRegionAsGpuModified(address, size);
-		if (m_memory_tracker.IsRegionGpuModified(address, size) ||
-		    !m_gpu_modified_ranges.Intersections(address, size).empty()) {
-			EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
-		}
-		m_memory_tracker.UntrackMemory(address, size);
-		EXIT_IF(!m_slot_buffers[id].is_deleted);
-		m_slot_buffers.erase(id);
-	});
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -251,15 +258,15 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
 		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
 		                                           "memory invalidation");
 	    },
-	    [&](uint64_t address, uint64_t bytes) noexcept {
+		[&](uint64_t address, uint64_t bytes) noexcept {
 		    for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
 			    for (uint64_t copied = 0; copied < range.size;) {
 				    const auto copy_address = range.address + copied;
-				    auto       owner        = m_buffers.upper_bound(copy_address);
-				    if (owner == m_buffers.begin()) {
+				    const auto* owner = m_page_table.Find(copy_address >> PageTable::kPageBits);
+				    if (owner == nullptr || !*owner) {
 					    EXIT("BufferCache: invalidation readback has no buffer owner\n");
 				    }
-				    auto& buffer = m_slot_buffers[std::prev(owner)->second];
+				    auto& buffer = m_slot_buffers[*owner];
 				    if (!buffer.IsInBounds(copy_address, 1)) {
 					    EXIT("BufferCache: invalidation readback is outside its buffer owner\n");
 				    }
@@ -281,12 +288,7 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
 		m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
 		return;
 	}
-	auto downloads = RecordDownloads(copies);
-	m_scheduler.FinishCurrent();
-	PublishDownloads(downloads);
-	for (const auto& range: downloads) {
-		m_gpu_modified_ranges.Subtract(range.address, range.size);
-	}
+	DownloadBufferMemory(copies);
 	// The enumeration above covered whole dirty pages and every exact interval on them.
 	m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
 	if (is_write) {
@@ -294,78 +296,6 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
 	}
 }
 
-void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
-		EXIT("BufferCache: invalid unmap range\n");
-	}
-	std::vector<DownloadCopy>                  copies;
-	std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
-	std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
-	for (const auto& [begin, id]: m_buffers) {
-		const auto& buffer = m_slot_buffers[id];
-		if (vaddr < begin + buffer.Size() && begin < vaddr + size) {
-			retired_buffers.emplace_back(begin, buffer.Size());
-		}
-	}
-	for (const auto& [begin, id]: m_buffers) {
-		const auto& buffer = m_slot_buffers[id];
-		if (vaddr >= begin + buffer.Size() || begin >= vaddr + size ||
-		    !m_memory_tracker.IsRegionGpuModified(begin, buffer.Size())) {
-			continue;
-		}
-		const auto dirty = m_gpu_modified_ranges.Intersections(begin, buffer.Size());
-		if (dirty.empty()) {
-			EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n");
-		}
-		modified_buffers.emplace_back(begin, buffer.Size());
-	}
-	for (const auto& [begin, bytes]: modified_buffers) {
-		auto owner = m_buffers.find(begin);
-		if (owner == m_buffers.end() || m_slot_buffers[owner->second].Size() != bytes) {
-			EXIT("BufferCache: unmap owner changed during collection\n");
-		}
-		auto& buffer = m_slot_buffers[owner->second];
-		m_memory_tracker.ForEachDownloadRange<false>(
-		    begin, buffer.Size(),
-		    [&](uint64_t address, uint64_t bytes) noexcept {
-			    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
-			                                           "unmap");
-		    },
-		    [&](uint64_t address, uint64_t bytes) noexcept {
-			    for (const auto& range: m_gpu_modified_ranges.Intersections(address, bytes)) {
-				    copies.push_back({&buffer, range.address - begin, range.address, range.size});
-			    }
-		    });
-	}
-	if (!copies.empty()) {
-		auto downloads = RecordDownloads(copies);
-		m_scheduler.FinishCurrent();
-		PublishDownloads(downloads);
-	} else if (!retired_buffers.empty()) {
-		// Image uploads can reference a clean cached buffer without owning it. Submit the active
-		// command stream before removing such backing.
-		m_scheduler.FinishCurrent();
-	}
-	for (const auto& [begin, bytes]: modified_buffers) {
-		m_gpu_modified_ranges.Subtract(begin, bytes);
-		m_memory_tracker.UnmarkRegionAsGpuModified(begin, bytes);
-	}
-	for (const auto& [begin, bytes]: retired_buffers) {
-		m_memory_tracker.MarkRegionAsCpuModified(begin, bytes);
-	}
-	if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
-		EXIT("BufferCache: unmap retained dirty byte ranges\n");
-	}
-	m_memory_tracker.UntrackMemory(vaddr, size);
-	for (const auto& [begin, bytes]: retired_buffers) {
-		(void)bytes;
-		const auto found = m_buffers.find(begin);
-		if (found != m_buffers.end()) {
-			DeleteBuffer(found->second);
-		}
-	}
-}
-
 BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0) {
 		return NULL_BUFFER_ID;
@@ -373,89 +303,132 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
 	if (size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: invalid buffer discovery request\n");
 	}
-	auto& command = m_scheduler.Current();
-	EXIT_IF(command.IsInvalid() || command.IsExecute());
-	auto       begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
-	auto       end   = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
-	const auto next  = m_buffers.upper_bound(vaddr);
-	if (next != m_buffers.begin()) {
-		const auto owner  = std::prev(next);
-		auto&      buffer = m_slot_buffers[owner->second];
+	const auto* owner = m_page_table.Find(vaddr >> PageTable::kPageBits);
+	if (owner != nullptr && *owner) {
+		auto& buffer = m_slot_buffers[*owner];
 		if (buffer.IsInBounds(vaddr, size)) {
-			buffer.tick_accessed_last = m_gc_tick;
-			return owner->second;
+			return *owner;
 		}
 	}
+	return CreateBuffer(vaddr, size);
+}
 
-	auto merge = [&](const Buffer& buffer) {
-		const auto buffer_begin = buffer.CpuAddress();
-		const auto buffer_end   = buffer_begin + buffer.Size();
-		if (begin >= buffer_end || buffer_begin >= end) {
-			return false;
-		}
-		begin = std::min(begin, buffer_begin);
-		end   = std::max(end, buffer_end);
-		return true;
-	};
-	std::vector<BufferId> overlaps;
-	auto                  first = m_buffers.lower_bound(begin);
+BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
+	auto& command = m_scheduler.Current();
+	EXIT_IF(command.IsInvalid());
+	auto       begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
+	auto       end   = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
+	auto       first = m_buffers.lower_bound(begin);
 	if (first != m_buffers.begin()) {
 		const auto previous = std::prev(first);
-		if (merge(m_slot_buffers[previous->second])) {
+		if (const auto& buffer = m_slot_buffers[previous->second];
+		    buffer.CpuAddress() + buffer.Size() > begin) {
 			first = previous;
 		}
 	}
-	for (auto candidate = first; candidate != m_buffers.end(); ++candidate) {
-		if (candidate->first >= end) {
-			break;
-		}
-		if (merge(m_slot_buffers[candidate->second])) {
-			overlaps.push_back(candidate->second);
-		}
+	auto last = first;
+	for (; last != m_buffers.end() && last->first < end; ++last) {
+		const auto& buffer = m_slot_buffers[last->second];
+		begin              = std::min(begin, buffer.CpuAddress());
+		end                = std::max(end, buffer.CpuAddress() + buffer.Size());
 	}
 
 	const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
 	                                      AllFlags, end - begin);
-	auto&      buffer         = m_slot_buffers[id];
-	buffer.tick_accessed_last = m_gc_tick;
+	auto&      buffer = m_slot_buffers[id];
 	SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
 	                     "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
-	for (const auto overlap: overlaps) {
-		const auto& old = m_slot_buffers[overlap];
+	for (auto overlap = first; overlap != last;) {
+		const auto current = overlap++;
+		const auto old_id  = current->second;
+		const auto& old    = m_slot_buffers[old_id];
 		buffer.CopyFrom(command, old, 0, old.CpuAddress() - begin, old.Size());
+		DeleteBuffer(old_id);
 	}
-	for (const auto overlap: overlaps) {
-		DeleteBuffer(overlap);
-	}
-	m_total_used_memory += buffer.Size();
-	const auto [it, inserted] = m_buffers.emplace(begin, id);
-	(void)it;
-	EXIT_IF(!inserted);
+	Register(id);
 	return id;
 }
 
 bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, bool is_written,
                                     bool is_texel_buffer) {
-	std::vector<std::pair<uint64_t, uint64_t>> uploads;
+	std::vector<vk::BufferCopy> copies;
+	uint64_t                    total_size = 0;
+	vk::Buffer                  source;
 	m_memory_tracker.ForEachUploadRange(
 	    vaddr, size, is_written,
-	    [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); },
-	    [&]() noexcept {
-		    for (const auto& [address, bytes]: uploads) {
-			    WriteDataBuffer(buffer, address, reinterpret_cast<const void*>(address), bytes);
-		    }
-	    });
+	    [&](uint64_t address, uint64_t bytes) noexcept {
+		    copies.emplace_back(total_size, buffer.Offset(address), bytes);
+		    total_size += bytes;
+	    },
+	    [&]() noexcept { source = UploadCopies(buffer, copies, total_size); });
+	if (source) {
+		auto& command = m_scheduler.Current();
+		command.EndRendering();
+		const auto native = command.Handle();
+		vk::BufferMemoryBarrier before {};
+		before.sType         = vk::StructureType::eBufferMemoryBarrier;
+		before.srcAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite |
+		                       vk::AccessFlagBits::eTransferRead |
+		                       vk::AccessFlagBits::eTransferWrite;
+		before.dstAccessMask       = vk::AccessFlagBits::eTransferWrite;
+		before.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+		before.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+		before.buffer              = buffer.Handle();
+		before.offset              = 0;
+		before.size                = buffer.Size();
+		native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
+		                       vk::PipelineStageFlagBits::eTransfer,
+		                       vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &before, 0, nullptr);
+		native.copyBuffer(source, buffer.Handle(), static_cast<uint32_t>(copies.size()),
+		                  copies.data());
+		auto after          = before;
+		after.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+		after.dstAccessMask = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite;
+		native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+		                       vk::PipelineStageFlagBits::eAllCommands,
+		                       vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr);
+	}
 	if (is_texel_buffer && !is_written) {
 		return SynchronizeBufferFromImage(buffer, vaddr, size);
 	}
 	return false;
 }
 
+vk::Buffer BufferCache::UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
+                                     uint64_t total_size) {
+	if (copies.empty()) {
+		return nullptr;
+	}
+
+	auto [mapped, base_offset] = m_staging_buffer.Map(total_size, 4);
+	if (mapped != nullptr) {
+		for (auto& copy: copies) {
+			const auto address = buffer.CpuAddress() + copy.dstOffset;
+			std::memcpy(mapped + copy.srcOffset, reinterpret_cast<const void*>(address), copy.size);
+			copy.srcOffset += base_offset;
+		}
+		m_staging_buffer.Commit();
+		return m_staging_buffer.Handle();
+	}
+
+	auto temporary = std::make_unique<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0,
+	                                         vk::BufferUsageFlagBits::eTransferSrc, total_size);
+	for (const auto& copy: copies) {
+		const auto address = buffer.CpuAddress() + copy.dstOffset;
+		std::memcpy(temporary->Mapped().data() + copy.srcOffset,
+		            reinterpret_cast<const void*>(address), copy.size);
+	}
+	temporary->Flush(0, total_size);
+	const auto handle = temporary->Handle();
+	m_scheduler.DeferOperation([owner = std::move(temporary)]() mutable { owner.reset(); });
+	return handle;
+}
+
 std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t size,
                                                        bool is_written, bool is_texel_buffer,
                                                        BufferId id) {
 	auto& command = m_scheduler.Current();
-	if (command.IsInvalid() || command.IsExecute() || vaddr == 0 || size == 0 ||
+	if (command.IsInvalid() || vaddr == 0 || size == 0 ||
 	    vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
@@ -465,7 +438,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
 	    m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
 		const auto alignment = std::max<uint64_t>(
 		    m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1);
-		auto [mapped, offset] = m_stream_buffer.Map(size, alignment);
+		auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false);
 		if (mapped != nullptr && Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) {
 			m_stream_buffer.Commit();
 			return {&m_stream_buffer, offset};
@@ -477,6 +450,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
 		id     = FindBuffer(vaddr, size);
 		buffer = &m_slot_buffers[id];
 	}
+	TouchBuffer(*buffer);
 	(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
@@ -490,20 +464,18 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
 		EXIT("BufferCache: invalid image source\n");
 	}
 	auto find_owner = [&]() -> Buffer* {
-		auto owner = m_buffers.upper_bound(vaddr);
-		if (owner == m_buffers.begin()) {
+		const auto* owner = m_page_table.Find(vaddr >> PageTable::kPageBits);
+		if (owner == nullptr || !*owner) {
 			return nullptr;
 		}
-		--owner;
-		auto* buffer = m_slot_buffers.try_get(owner->second);
-		return buffer != nullptr && buffer->IsInBounds(vaddr, size) ? buffer : nullptr;
+		auto& buffer = m_slot_buffers[*owner];
+		return buffer.IsInBounds(vaddr, size) ? &buffer : nullptr;
 	};
 
 	{
 		const bool cpu_modified            = m_memory_tracker.IsRegionCpuModified(vaddr, size);
 		const bool gpu_modified            = m_memory_tracker.IsRegionGpuModified(vaddr, size);
-		const auto dirty                   = m_gpu_modified_ranges.Intersections(vaddr, size);
-		const bool has_dirty_buffer_source = !dirty.empty();
+		const bool has_dirty_buffer_source = m_gpu_modified_ranges.Intersects(vaddr, size);
 		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
 		                                           "image source");
 
@@ -512,17 +484,10 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
 			if (!IsRegionRegistered(vaddr, size)) {
 				EXIT("BufferCache: GPU-dirty image source has no native buffer\n");
 			}
-			const auto id       = FindBuffer(vaddr, size);
-			auto&      buffer   = m_slot_buffers[id];
-			owner               = &buffer;
-			const auto resolved = m_buffers.find(buffer.CpuAddress());
-			if (resolved == m_buffers.end() || resolved->second != id ||
-			    !buffer.IsInBounds(vaddr, size)) {
-				EXIT("BufferCache: merged image source does not contain the requested range\n");
-			}
+			owner = &m_slot_buffers[FindBuffer(vaddr, size)];
 		}
 		if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) {
-			owner->tick_accessed_last = m_gc_tick;
+			TouchBuffer(*owner);
 			return {owner, owner->Offset(vaddr)};
 		}
 		if (has_dirty_buffer_source && owner == nullptr) {
@@ -537,8 +502,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
 	}
 	m_staging_buffer.Commit();
 
-	const auto dirty                   = m_gpu_modified_ranges.Intersections(vaddr, size);
-	const bool has_dirty_buffer_source = !dirty.empty();
+	const bool has_dirty_buffer_source = m_gpu_modified_ranges.Intersects(vaddr, size);
 	auto*      owner                   = find_owner();
 	if (has_dirty_buffer_source && owner == nullptr) {
 		EXIT("BufferCache: GPU-dirty image source lost its native owner\n");
@@ -548,7 +512,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
 		return {&m_staging_buffer, stage_offset};
 	}
 
-	owner->tick_accessed_last = m_gc_tick;
+	TouchBuffer(*owner);
 	std::vector<std::pair<uint64_t, uint64_t>> uploads;
 	m_memory_tracker.ForEachUploadRange(
 	    vaddr, size, false,
@@ -581,7 +545,7 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data)
 			continue;
 		}
 		WriteDataBuffer(buffer, begin, data.data() + begin - vaddr, range_end - begin);
-		buffer.tick_accessed_last = m_gc_tick;
+		TouchBuffer(buffer);
 	}
 }
 
@@ -701,7 +665,7 @@ bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 }
 
 bool BufferCache::HasGpuDirtyBytes(uint64_t vaddr, uint64_t size) {
-	return !m_gpu_modified_ranges.Intersections(vaddr, size).empty();
+	return m_gpu_modified_ranges.Intersects(vaddr, size);
 }
 
 bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
@@ -721,30 +685,18 @@ void BufferCache::RunGarbageCollector() {
 	const uint64_t age        = std::min<uint64_t>(aggressive ? 80 : 160, tick);
 	const size_t   limit      = aggressive ? 64 : 32;
 
-	std::vector<uint64_t> candidates;
-	for (const auto& [address, id]: m_buffers) {
-		const auto& buffer = m_slot_buffers[id];
-		if (tick - std::min(tick, buffer.tick_accessed_last) >= age) {
-			candidates.push_back(address);
-		}
-	}
-	std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) {
-		return m_slot_buffers[m_buffers.at(left)].tick_accessed_last <
-		       m_slot_buffers[m_buffers.at(right)].tick_accessed_last;
-	});
-	if (candidates.size() > limit) {
-		candidates.resize(limit);
-	}
-
-	for (const auto address: candidates) {
-		auto found = m_buffers.find(address);
-		EXIT_IF(found == m_buffers.end());
-		const auto id     = found->second;
-		auto&      buffer = m_slot_buffers[id];
+	std::vector<BufferId> dirty_buffers;
+	std::vector<DownloadCopy> copies;
+	size_t                    retire_count = 0;
+	m_lru_cache.ForEachItemBelow(tick - age, [&](BufferId id) {
+		auto& buffer = m_slot_buffers[id];
+		EXIT_IF(buffer.is_deleted);
 		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, buffer.CpuAddress(),
 		                                           buffer.Size(), "garbage collection");
 		const bool dirty = m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size());
-		std::vector<DownloadCopy> copies;
+		if (dirty && !aggressive) {
+			return false;
+		}
 		if (dirty) {
 			m_memory_tracker.ForEachDownloadRange<false>(
 			    buffer.CpuAddress(), buffer.Size(),
@@ -753,24 +705,35 @@ void BufferCache::RunGarbageCollector() {
 				                                           dirty_size, "garbage collection");
 			    },
 			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    for (const auto range:
-				         m_gpu_modified_ranges.Intersections(dirty_address, dirty_size)) {
+				    m_gpu_modified_ranges.ForEachIntersection(
+				        dirty_address, dirty_size, [&](RangeSet::Range range) {
 					    copies.push_back({&buffer, range.address - buffer.CpuAddress(),
 					                      range.address, range.size});
-				    }
-			    });
-		}
-
-		const auto retire_address = buffer.CpuAddress();
-		const auto retire_size    = buffer.Size();
-		if (dirty) {
-			EXIT_IF(copies.empty());
-			Unregister(id);
-			QueueGarbageDownload(copies, id, retire_address, retire_size);
+				        });
+				});
+			dirty_buffers.push_back(id);
 		} else {
-			m_memory_tracker.UntrackMemory(retire_address, retire_size);
+			m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());
 			DeleteBuffer(id);
 		}
+		return ++retire_count == limit;
+	});
+	if (dirty_buffers.empty()) {
+		return;
+	}
+
+	EXIT_IF(copies.empty());
+	DownloadBufferMemory(copies);
+	for (const auto id: dirty_buffers) {
+		auto& buffer = m_slot_buffers[id];
+		m_memory_tracker.UnmarkRegionAsGpuModified(buffer.CpuAddress(), buffer.Size());
+		if (m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size()) ||
+		    m_gpu_modified_ranges.Intersects(buffer.CpuAddress(), buffer.Size())) {
+			EXIT("BufferCache: garbage collection retained GPU ownership\n");
+		}
+		m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());
+		Unregister(id);
+		m_slot_buffers.erase(id);
 	}
 }
 

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -3,9 +3,11 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/lruCache.h"
 #include "common/slotVector.h"
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
 #include <map>
@@ -33,7 +35,6 @@ public:
 
 	void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
 	void                   ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
-	void                   UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] Buffer&  GetBuffer(BufferId id) { return m_slot_buffers[id]; }
 	[[nodiscard]] BufferId FindBuffer(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBuffer(uint64_t vaddr, uint64_t size,
@@ -65,41 +66,46 @@ private:
 	friend struct BufferCacheTestAccess;
 
 	struct DownloadCopy;
-	struct DownloadRange;
+	using PageTable = MultiLevelPageTable<BufferId, 14, 40, 16>;
+	static_assert(CACHING_PAGE_SIZE == (uint64_t {1} << PageTable::kPageBits));
 	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
 	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
 		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
 	}
 	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
+	void TouchBuffer(const Buffer& buffer);
+	[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
+	void                   Register(BufferId id);
 	void Unregister(BufferId id);
 	void DeleteBuffer(BufferId id);
 	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
 	                                     bool is_written, bool is_texel_buffer);
+	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
+	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
-	void PublishDownloads(std::span<const DownloadRange> downloads);
-	void QueueGarbageDownload(std::span<const DownloadCopy> copies, BufferId id, uint64_t address,
-	                          uint64_t size);
+	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
 
-	GraphicContext&              m_graphics;
-	CommandScheduler&            m_scheduler;
-	Buffer                       m_gds_buffer;
-	Common::SlotVector<Buffer>   m_slot_buffers;
-	std::map<uint64_t, BufferId> m_buffers;
-	RangeSet                     m_gpu_modified_ranges;
-	MemoryTracker                m_memory_tracker;
-	StreamBuffer                 m_staging_buffer;
-	StreamBuffer                 m_stream_buffer;
-	StreamBuffer                 m_download_buffer;
-	StreamBuffer                 m_device_buffer;
-	TextureCache&                m_texture_cache;
-	uint64_t                     m_total_used_memory  = 0;
-	uint64_t                     m_trigger_gc_memory  = 1ull * 1024 * 1024 * 1024;
-	uint64_t                     m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
-	uint64_t                     m_gc_tick            = 0;
+	GraphicContext&                                   m_graphics;
+	CommandScheduler&                                 m_scheduler;
+	Buffer                                            m_gds_buffer;
+	Common::SlotVector<Buffer>                        m_slot_buffers;
+	Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
+	std::map<uint64_t, BufferId>                      m_buffers;
+	PageTable                                         m_page_table;
+	RangeSet                                          m_gpu_modified_ranges;
+	MemoryTracker                                     m_memory_tracker;
+	StreamBuffer                                      m_staging_buffer;
+	StreamBuffer                                      m_stream_buffer;
+	StreamBuffer                                      m_download_buffer;
+	StreamBuffer                                      m_device_buffer;
+	TextureCache&                                     m_texture_cache;
+	uint64_t                                          m_total_used_memory  = 0;
+	uint64_t m_trigger_gc_memory  = 1ull * 1024 * 1024 * 1024;
+	uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
+	uint64_t m_gc_tick            = 0;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -60,10 +60,10 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	const auto unmap = [this, vaddr, size] {
 		if (m_scheduler.Active()) {
 			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.FinishCurrent();
+			m_scheduler.Finish();
 			m_scheduler.WaitPriorityOperations(tick);
 		}
-		m_buffer_cache.UnmapMemory(vaddr, size);
+		m_buffer_cache.InvalidateMemory(vaddr, size);
 		m_texture_cache.UnmapMemory(vaddr, size);
 		m_page_manager.OnGpuUnmap(vaddr, size);
 		std::lock_guard lock(m_mapped_ranges_mutex);

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -66,8 +66,8 @@ public:
 	void Fill(uint64_t offset, uint64_t size, uint32_t value);
 
 	// BufferCache state lives directly on the resource.
-	bool     is_deleted         = false;
-	uint64_t tick_accessed_last = 0;
+	bool   is_deleted = false;
+	size_t lru_id     = 0;
 
 protected:
 	[[nodiscard]] GraphicContext&   Graphics() const noexcept { return *m_graphics; }

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1209,7 +1209,7 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e
 		if (ensure_valid && owner->depth_id) {
 			owner = m_slot_images.try_get(owner->depth_id);
 		}
-		if (owner == nullptr || (ensure_valid && !owner->SafeToDownload())) {
+		if (owner == nullptr || (ensure_valid && !SafeToDownload(*owner))) {
 			continue;
 		}
 		matches.push_back(id);
@@ -1580,7 +1580,7 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 		if (owner->depth_id) {
 			owner = m_texture_cache.m_slot_images.try_get(owner->depth_id);
 		}
-		if (owner != nullptr && owner->SafeToDownload()) {
+		if (owner != nullptr && m_texture_cache.SafeToDownload(*owner)) {
 			matches.push_back(id);
 		}
 	}
@@ -1871,9 +1871,6 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 		metadata = m_surface_metas.erase(metadata);
 	}
 	auto images = FindImagesInRegion(address, size, false);
-	if (!images.empty()) {
-		m_scheduler.Finish();
-	}
 	for (const auto id: images) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -1,96 +1,97 @@
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 
 #include "common/assert.h"
+#include "common/logging/log.h"
 #include "graphics/host_gpu/graphicContext.h"
 
 #include <algorithm>
+#include <cstdio>
+#include <optional>
 
 namespace Libs::Graphics {
 
 static thread_local CommandScheduler* g_deferred_callback_scheduler = nullptr;
 
-void CommandSlot::Reset() {
-	EXIT_IF(buffer == nullptr);
-	const auto result = buffer.reset(vk::CommandBufferResetFlagBits::eReleaseResources);
-	if (result != vk::Result::eSuccess) {
-		EXIT("failed to reset Vulkan command buffer: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
-	}
+namespace {
+
+void ReportVulkanFatal(const char* what, vk::Result result, uint64_t tick, uint32_t debug_op,
+                       uint64_t debug_submit, uint32_t arg0, uint32_t arg1, uint32_t arg2,
+                       uint32_t arg3, uint64_t arg4) {
+	LOGF("%s failed: %s (%d), tick=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	     what, VulkanToString(result).c_str(), static_cast<int>(result), tick, debug_op,
+	     debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::printf("%s failed: %s (%d), tick=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	            " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	            what, VulkanToString(result).c_str(), static_cast<int>(result), tick, debug_op,
+	            debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::fflush(stdout);
 }
 
-CommandScheduler::CommandPool::~CommandPool() {
-	Destroy();
-}
-
-void CommandScheduler::CommandPool::Create(GraphicContext& graphics) {
-	EXIT_IF(m_pool != nullptr || m_graphics != nullptr ||
-	        graphics.queue_family == static_cast<uint32_t>(-1));
-	m_graphics = &graphics;
+} // namespace
 
+CommandScheduler::CommandPool::CommandPool(GraphicContext& graphics, MasterSemaphore& master)
+    : m_graphics(graphics), m_master(master) {
+	EXIT_IF(graphics.queue_family == static_cast<uint32_t>(-1));
 	vk::CommandPoolCreateInfo create {};
 	create.sType            = vk::StructureType::eCommandPoolCreateInfo;
 	create.queueFamilyIndex = graphics.queue_family;
-	create.flags            = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
+	create.flags            = vk::CommandPoolCreateFlagBits::eTransient |
+	                          vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
 	const auto result       = graphics.device.createCommandPool(&create, nullptr, &m_pool);
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || m_pool == nullptr);
 }
 
-CommandSlot* CommandScheduler::CommandPool::CreateSlot() {
-	EXIT_IF(m_graphics == nullptr);
-	auto& graphics = *m_graphics;
+CommandScheduler::CommandPool::~CommandPool() {
+	m_graphics.device.destroyCommandPool(m_pool, nullptr);
+}
+
+size_t CommandScheduler::CommandPool::Grow() {
+	const auto first = m_ticks.size();
+	m_ticks.resize(first + GrowStep);
+	m_buffers.resize(first + GrowStep);
 
 	vk::CommandBufferAllocateInfo allocate {};
 	allocate.sType              = vk::StructureType::eCommandBufferAllocateInfo;
 	allocate.commandPool        = m_pool;
 	allocate.level              = vk::CommandBufferLevel::ePrimary;
-	allocate.commandBufferCount = 1;
-	vk::CommandBuffer buffer    = nullptr;
-	EXIT_IF(graphics.device.allocateCommandBuffers(&allocate, &buffer) != vk::Result::eSuccess);
-
-	vk::FenceCreateInfo fence_create {};
-	fence_create.sType = vk::StructureType::eFenceCreateInfo;
-	fence_create.flags = vk::FenceCreateFlagBits::eSignaled;
-	vk::Fence fence    = nullptr;
-	if (graphics.device.createFence(&fence_create, nullptr, &fence) != vk::Result::eSuccess) {
-		graphics.device.freeCommandBuffers(m_pool, 1, &buffer);
-		EXIT("failed to create command-buffer fence\n");
-	}
-
-	auto& slot      = m_slots.emplace_back();
-	slot.pool_mutex = &m_mutex;
-	slot.id         = static_cast<uint32_t>(m_slots.size() - 1);
-	slot.buffer     = buffer;
-	slot.fence      = fence;
-	return &slot;
+	allocate.commandBufferCount = static_cast<uint32_t>(GrowStep);
+	EXIT_IF(m_graphics.device.allocateCommandBuffers(&allocate, m_buffers.data() + first) !=
+	        vk::Result::eSuccess);
+	return first;
 }
 
-CommandSlot* CommandScheduler::CommandPool::Allocate(GraphicContext& graphics) {
-	Common::LockGuard lock(m_mutex);
-	if (m_pool == nullptr) {
-		Create(graphics);
-	}
-	EXIT_IF(m_graphics != &graphics);
-	auto  found = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; });
-	auto* slot  = found != m_slots.end() ? &*found : CreateSlot();
-	slot->busy  = true;
-	slot->Reset();
-	return slot;
-}
+vk::CommandBuffer CommandScheduler::CommandPool::Commit() {
+	auto       gpu_tick = m_master.KnownGpuTick();
+	const auto search   = [this, &gpu_tick](size_t begin, size_t end) -> std::optional<size_t> {
+		for (size_t index = begin; index < end; ++index) {
+			if (gpu_tick >= m_ticks[index]) {
+				m_ticks[index] = m_master.CurrentTick();
+				return index;
+			}
+		}
+		return std::nullopt;
+	};
 
-void CommandScheduler::CommandPool::Destroy() {
-	Common::LockGuard lock(m_mutex);
-	if (m_pool == nullptr) {
-		return;
+	auto found = search(m_hint, m_ticks.size());
+	if (!found) {
+		m_master.Refresh();
+		gpu_tick = m_master.KnownGpuTick();
+		found    = search(m_hint, m_ticks.size());
 	}
-	EXIT_IF(std::ranges::any_of(m_slots, [](const auto& slot) { return slot.busy; }));
-	EXIT_IF(m_graphics == nullptr);
-	for (const auto& slot: m_slots) {
-		m_graphics->device.destroyFence(slot.fence, nullptr);
+	if (!found) {
+		found = search(0, m_hint);
 	}
-	m_graphics->device.destroyCommandPool(m_pool, nullptr);
-	m_slots.clear();
-	m_pool     = nullptr;
-	m_graphics = nullptr;
+	if (!found) {
+		found           = Grow();
+		m_ticks[*found] = m_master.CurrentTick();
+	}
+
+	m_hint = *found + 1;
+	if (m_hint == m_ticks.size()) {
+		m_hint = 0;
+	}
+	return m_buffers[*found];
 }
 
 bool CommandScheduler::InDeferredOperation() noexcept {
@@ -99,10 +100,8 @@ bool CommandScheduler::InDeferredOperation() noexcept {
 
 CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics)
     : m_master(graphics), m_context(context), m_graphics(graphics),
-      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {
-	m_buffers.reserve(CommandBufferGrowStep);
-	m_buffer_ticks.reserve(CommandBufferGrowStep);
-}
+      m_command_pool(graphics, m_master), m_command(*this),
+      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {}
 
 CommandScheduler::~CommandScheduler() {
 	Shutdown();
@@ -128,9 +127,11 @@ void CommandScheduler::Shutdown() {
 		}
 		m_operation_state = OperationState::Draining;
 	}
-	if (Active() && m_recording) {
-		Finish();
+	if (!m_command.IsInvalid()) {
+		Submit();
 	}
+	m_master.Wait(CurrentTick() - 1);
+	PopPendingOperations();
 	DrainPriorityOperations();
 	m_priority_thread.request_stop();
 	m_operation_available.notify_all();
@@ -156,14 +157,10 @@ void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config
 	m_user_config = &user_config;
 	m_shaders     = &shaders;
 
-	if (!Active()) {
-		m_current = static_cast<int>(GrowCommandBuffers());
-	}
-
-	BindCurrent();
-	if (!m_recording) {
-		Current().Begin();
-		m_recording = true;
+	if (m_command.IsInvalid()) {
+		BeginNext();
+	} else {
+		BindCurrent();
 	}
 }
 
@@ -172,7 +169,7 @@ void CommandScheduler::BeginRendering(const RenderState& state) {
 }
 
 void CommandScheduler::EndRendering() {
-	if (Active() && m_recording) {
+	if (Active() && !m_command.IsInvalid()) {
 		Current().EndRendering();
 	}
 }
@@ -183,60 +180,40 @@ void CommandScheduler::Flush() {
 }
 
 void CommandScheduler::Flush(SubmitInfo& submit) {
-	SubmitCurrent(submit);
+	Submit(submit);
 	BeginNext();
 }
 
-CommandBuffer& CommandScheduler::FlushAndGetSubmitted() {
-	SubmitInfo submit;
-	auto&      submitted = SubmitCurrent(submit);
+void CommandScheduler::FlushAndWait() {
+	const auto tick = Submit();
+	m_master.Wait(tick);
 	BeginNext();
-	return submitted;
 }
 
 void CommandScheduler::Finish() {
 	CheckActive();
-	const auto tick = CurrentTick();
-	if (m_recording) {
-		SubmitInfo submit;
-		SubmitCurrent(submit);
+	if (!m_command.IsInvalid()) {
+		Submit();
 	}
-	for (auto& buffer: m_buffers) {
-		buffer->WaitForFenceAndReset();
-	}
-	m_master.Wait(tick);
+	m_master.Wait(CurrentTick() - 1);
 	PopPendingOperations();
-	BindCurrent();
-	Current().Begin();
-	m_recording = true;
-}
-
-void CommandScheduler::FinishCurrent() {
-	SubmitInfo submit;
-	auto&      submitted = SubmitCurrent(submit);
-	submitted.WaitForFenceAndReset();
-	m_master.Refresh();
-	PopPendingOperations();
-	submitted.Begin();
-	m_recording = true;
+	BeginNext();
 }
 
 void CommandScheduler::Wait(uint64_t tick) {
-	CheckActive();
 	EXIT_IF(tick > CurrentTick());
-	if (tick >= CurrentTick()) {
+	if (tick == CurrentTick()) {
+		CheckActive();
 		// A stream-buffer wrap can wait while a draw is being prepared through a reference to
-		// Current(). Recycle the same command object so that reference remains valid. Deferred
+		// Current(). The wrapper stays stable while its pooled Vulkan buffer is retired. Deferred
 		// resources are released only at the next GPU operation boundary.
-		SubmitInfo submit;
-		auto&      submitted = SubmitCurrent(submit);
-		submitted.WaitForFenceAndReset();
-		m_master.Refresh();
-		submitted.Begin();
-		m_recording = true;
-		return;
+		const auto submitted_tick = Submit();
+		EXIT_IF(submitted_tick != tick);
+		m_master.Wait(tick);
+		BeginNext();
+	} else {
+		m_master.Wait(tick);
 	}
-	m_master.Wait(tick);
 }
 
 void CommandScheduler::PopPendingOperations() {
@@ -365,83 +342,81 @@ bool CommandScheduler::IsFree(uint64_t tick) {
 	return m_master.IsFree(tick);
 }
 
-CommandSlot* CommandScheduler::AllocateCommandBuffer() {
-	return m_command_pool.Allocate(m_graphics);
-}
-
-uint64_t CommandScheduler::NextSubmitSequence() noexcept {
-	return m_submit_sequence.fetch_add(1, std::memory_order_relaxed) + 1;
-}
-
 void CommandScheduler::CheckActive() const {
-	EXIT_IF(!Active() || static_cast<size_t>(m_current) >= m_buffers.size());
+	EXIT_IF(!Active());
 }
 
-RenderCommandBuffer& CommandScheduler::Current() const {
+RenderCommandBuffer& CommandScheduler::Current() {
 	CheckActive();
-	EXIT_IF(m_buffers[m_current] == nullptr);
-	return *m_buffers[m_current];
+	return m_command;
 }
 
-void CommandScheduler::BindCurrent() const {
+void CommandScheduler::BindCurrent() {
 	EXIT_IF(m_registers == nullptr || m_user_config == nullptr || m_shaders == nullptr);
-	Current().Bind(*m_registers, *m_user_config, *m_shaders);
+	m_command.Bind(*m_registers, *m_user_config, *m_shaders);
 }
 
-CommandBuffer& CommandScheduler::SubmitCurrent(SubmitInfo& submit) {
-	CheckActive();
-	EXIT_IF(!m_recording);
-	auto& submitted = Current();
-	submitted.End();
-	const auto signal_tick = m_master.NextTick();
-	submit.AddSignal(m_master.Handle(), signal_tick);
-	submitted.Execute(submit);
-	m_buffer_ticks[static_cast<size_t>(m_current)] = signal_tick;
-	m_recording                                    = false;
-	return submitted;
+CommandBuffer& CommandScheduler::BeginCommand() {
+	EXIT_IF(!m_command.IsInvalid());
+	m_command.m_buffer = m_command_pool.Commit();
+	m_command.Begin();
+	return m_command;
 }
 
-int CommandScheduler::FindReusableBuffer(uint64_t gpu_tick) const {
-	EXIT_IF(m_buffer_ticks.size() != m_buffers.size());
-	const auto buffer_count = m_buffers.size();
-	for (size_t offset = 1; offset <= buffer_count; ++offset) {
-		const auto candidate = (static_cast<size_t>(m_current) + offset) % buffer_count;
-		if (gpu_tick >= m_buffer_ticks[candidate]) {
-			return static_cast<int>(candidate);
-		}
+uint64_t CommandScheduler::Submit(SubmitInfo submit) {
+	EXIT_IF(m_command.IsInvalid());
+	EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores ||
+	        submit.num_signal_semaphores >= SubmitInfo::MaxSemaphores);
+
+	m_command.End();
+	const auto buffer   = m_command.m_buffer;
+	auto&      graphics = m_graphics;
+	EXIT_IF(graphics.queue == nullptr);
+
+	vk::Result result;
+	uint64_t   tick;
+	{
+		Common::LockGuard lock(graphics.queue_mutex);
+		tick = m_master.NextTick();
+		submit.AddSignal(m_master.Handle(), tick);
+
+		vk::TimelineSemaphoreSubmitInfo timeline_info {};
+		timeline_info.sType                     = vk::StructureType::eTimelineSemaphoreSubmitInfo;
+		timeline_info.waitSemaphoreValueCount   = submit.num_wait_semaphores;
+		timeline_info.pWaitSemaphoreValues      = submit.wait_ticks.data();
+		timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores;
+		timeline_info.pSignalSemaphoreValues    = submit.signal_ticks.data();
+
+		vk::SubmitInfo submit_info {};
+		submit_info.sType                = vk::StructureType::eSubmitInfo;
+		submit_info.pNext                = &timeline_info;
+		submit_info.waitSemaphoreCount   = submit.num_wait_semaphores;
+		submit_info.pWaitSemaphores      = submit.wait_semaphores.data();
+		submit_info.pWaitDstStageMask    = submit.wait_stages.data();
+		submit_info.commandBufferCount   = 1;
+		submit_info.pCommandBuffers      = &buffer;
+		submit_info.signalSemaphoreCount = submit.num_signal_semaphores;
+		submit_info.pSignalSemaphores    = submit.signal_semaphores.data();
+
+		result = graphics.queue.submit(1, &submit_info, nullptr);
 	}
-	return -1;
-}
 
-size_t CommandScheduler::GrowCommandBuffers() {
-	const auto first = m_buffers.size();
-	const auto end   = first + CommandBufferGrowStep;
-	m_buffers.reserve(end);
-	m_buffer_ticks.reserve(end);
-	for (size_t i = first; i < end; ++i) {
-		m_buffers.emplace_back(std::make_unique<RenderCommandBuffer>(*this));
-		m_buffer_ticks.push_back(0);
+	if (result != vk::Result::eSuccess) {
+		ReportVulkanFatal("vkQueueSubmit", result, tick, m_command.m_debug_op,
+		                  m_command.m_debug_submit_id, m_command.m_debug_arg0,
+		                  m_command.m_debug_arg1, m_command.m_debug_arg2, m_command.m_debug_arg3,
+		                  m_command.m_debug_arg4);
 	}
-	return first;
+	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+
+	m_command.m_buffer = nullptr;
+	return tick;
 }
 
 void CommandScheduler::BeginNext() {
-	EXIT_IF(m_recording);
-
-	auto candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	if (candidate < 0) {
-		m_master.Refresh();
-		candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	}
-	if (candidate < 0) {
-		candidate = static_cast<int>(GrowCommandBuffers());
-	}
-
-	m_current = candidate;
-	Current().WaitForFenceAndReset();
+	EXIT_IF(!m_command.IsInvalid());
 	BindCurrent();
-	Current().Begin();
-	m_recording = true;
+	BeginCommand();
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -6,11 +6,7 @@
 #include "graphics/host_gpu/renderer/masterSemaphore.h"
 #include "graphics/host_gpu/renderer/render.h"
 
-#include <array>
-#include <atomic>
 #include <condition_variable>
-#include <deque>
-#include <memory>
 #include <mutex>
 
 #include <queue>
@@ -20,16 +16,6 @@
 
 namespace Libs::Graphics {
 
-struct CommandSlot {
-	Common::Mutex*    pool_mutex = nullptr;
-	uint32_t          id         = 0;
-	vk::CommandBuffer buffer     = nullptr;
-	vk::Fence         fence      = nullptr;
-	bool              busy       = false;
-
-	void Reset();
-};
-
 class CommandScheduler {
 public:
 	CommandScheduler(RenderContext& context, GraphicContext& graphics);
@@ -41,9 +27,10 @@ public:
 	void           EndRendering();
 	void           Flush();
 	void           Flush(SubmitInfo& submit);
-	CommandBuffer& FlushAndGetSubmitted();
+	void           FlushAndWait();
 	void           Finish();
-	void           FinishCurrent();
+	CommandBuffer& BeginCommand();
+	uint64_t       Submit(SubmitInfo submit = {});
 	// Deferred callbacks can observe an externally owned drain, but cannot initiate shutdown:
 	// the priority runner cannot join itself.
 	void                      Shutdown();
@@ -55,9 +42,9 @@ public:
 	void                      DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
 	[[nodiscard]] static bool InDeferredOperation() noexcept;
 
-	[[nodiscard]] bool             Active() const noexcept { return m_current >= 0; }
+	[[nodiscard]] bool             Active() const noexcept { return m_registers != nullptr; }
 	void                           CheckActive() const;
-	RenderCommandBuffer&           Current() const;
+	RenderCommandBuffer&           Current();
 	[[nodiscard]] uint64_t         CurrentTick() const noexcept { return m_master.CurrentTick(); }
 	[[nodiscard]] bool             IsFree(uint64_t tick);
 	[[nodiscard]] MasterSemaphore& GetMasterSemaphore() noexcept { return m_master; }
@@ -65,25 +52,25 @@ public:
 	[[nodiscard]] GraphicContext&  Graphics() const noexcept { return m_graphics; }
 
 private:
-	static constexpr size_t CommandBufferGrowStep = 4;
-
 	class CommandPool {
 	public:
-		CommandPool() = default;
+		CommandPool(GraphicContext& graphics, MasterSemaphore& master);
 		~CommandPool();
 		KYTY_CLASS_NO_COPY(CommandPool);
 
-		CommandSlot* Allocate(GraphicContext& graphics);
+		vk::CommandBuffer Commit();
 
 	private:
-		void         Create(GraphicContext& graphics);
-		CommandSlot* CreateSlot();
-		void         Destroy();
+		static constexpr size_t GrowStep = 4;
 
-		GraphicContext*         m_graphics = nullptr;
-		Common::Mutex           m_mutex;
-		vk::CommandPool         m_pool = nullptr;
-		std::deque<CommandSlot> m_slots;
+		size_t Grow();
+
+		GraphicContext&                m_graphics;
+		MasterSemaphore&               m_master;
+		vk::CommandPool                m_pool = nullptr;
+		std::vector<vk::CommandBuffer> m_buffers;
+		std::vector<uint64_t>          m_ticks;
+		size_t                         m_hint = 0;
 	};
 
 	enum class OperationState { Open, Draining, Closed };
@@ -93,39 +80,28 @@ private:
 		uint64_t                     tick = 0;
 	};
 
-	void                       BindCurrent() const;
-	CommandBuffer&             SubmitCurrent(SubmitInfo& submit);
-	void                       BeginNext();
-	void                       PopPendingOperations(bool refresh_gpu_tick);
-	[[nodiscard]] int          FindReusableBuffer(uint64_t gpu_tick) const;
-	[[nodiscard]] size_t       GrowCommandBuffers();
-	void                       PriorityOperationsThread(std::stop_token stop);
-	void                       RunOperation(Common::UniqueFunction<void>&& operation);
-	[[nodiscard]] CommandSlot* AllocateCommandBuffer();
-	[[nodiscard]] uint64_t     NextSubmitSequence() noexcept;
+	void BindCurrent();
+	void BeginNext();
+	void PopPendingOperations(bool refresh_gpu_tick);
+	void PriorityOperationsThread(std::stop_token stop);
+	void RunOperation(Common::UniqueFunction<void>&& operation);
 
-	MasterSemaphore                                   m_master;
-	RenderContext&                                    m_context;
-	GraphicContext&                                   m_graphics;
-	CommandPool                                       m_command_pool;
-	std::vector<std::unique_ptr<RenderCommandBuffer>> m_buffers;
-	std::vector<uint64_t>                             m_buffer_ticks;
-	std::queue<PendingOperation>                      m_pending_operations;
-	std::queue<PendingOperation>                      m_priority_operations;
-	std::mutex                                        m_operation_mutex;
-	std::condition_variable                           m_operation_available;
-	std::jthread                                      m_priority_thread;
-	bool                                              m_priority_active      = false;
-	uint64_t                                          m_priority_active_tick = 0;
-	OperationState                                    m_operation_state      = OperationState::Open;
-	int                                               m_current              = -1;
-	bool                                              m_recording            = false;
-	HW::Context*                                      m_registers            = nullptr;
-	HW::UserConfig*                                   m_user_config          = nullptr;
-	HW::Shader*                                       m_shaders              = nullptr;
-	std::atomic<uint64_t>                             m_submit_sequence      = 0;
-
-	friend class CommandBuffer;
+	MasterSemaphore              m_master;
+	RenderContext&               m_context;
+	GraphicContext&              m_graphics;
+	CommandPool                  m_command_pool;
+	RenderCommandBuffer          m_command;
+	std::queue<PendingOperation> m_pending_operations;
+	std::queue<PendingOperation> m_priority_operations;
+	std::mutex                   m_operation_mutex;
+	std::condition_variable      m_operation_available;
+	std::jthread                 m_priority_thread;
+	bool                         m_priority_active      = false;
+	uint64_t                     m_priority_active_tick = 0;
+	OperationState               m_operation_state      = OperationState::Open;
+	HW::Context*                 m_registers            = nullptr;
+	HW::UserConfig*              m_user_config          = nullptr;
+	HW::Shader*                  m_shaders              = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/context.cpp

@@ -1,7 +1,5 @@
 #include "common/assert.h"
 #include "common/common.h"
-#include "common/emulatorConfig.h"
-#include "common/logging/log.h"
 #include "common/profiler.h"
 #include "common/threads.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -16,70 +14,29 @@
 
 #include <algorithm>
 #include <bit>
-#include <cstdio>
 #include <cstring>
 namespace Libs::Graphics {
 
-namespace {
-
-void ReportVulkanFatal(const char* what, vk::Result result, uint32_t slot, uint64_t submit_seq,
-                       uint32_t debug_op, uint64_t debug_submit, uint32_t arg0, uint32_t arg1,
-                       uint32_t arg2, uint32_t arg3, uint64_t arg4) {
-	LOGF("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
-	     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-	     what, VulkanToString(result).c_str(), static_cast<int>(result), slot, submit_seq, debug_op,
-	     debug_submit, arg0, arg1, arg2, arg3, arg4);
-	std::printf("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64
-	            " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-	            what, VulkanToString(result).c_str(), static_cast<int>(result), slot, submit_seq,
-	            debug_op, debug_submit, arg0, arg1, arg2, arg3, arg4);
-	std::fflush(stdout);
-}
-
-} // namespace
-
 CommandBuffer::CommandBuffer(CommandScheduler& scheduler)
-    : m_context(scheduler.Context()), m_scheduler(scheduler), m_graphics(scheduler.Graphics()),
-      m_slot(scheduler.AllocateCommandBuffer()) {}
-
-CommandBuffer::~CommandBuffer() {
-	Release();
-}
+    : m_context(scheduler.Context()), m_graphics(scheduler.Graphics()) {}
 
 bool CommandBuffer::IsInvalid() const {
-	return m_slot == nullptr;
+	return m_buffer == nullptr;
 }
 
 vk::CommandBuffer CommandBuffer::Handle() const {
 	EXIT_IF(IsInvalid());
-
-	const auto handle = m_slot->buffer;
-	EXIT_IF(handle == nullptr);
-	return handle;
+	return m_buffer;
 }
 
-void CommandBuffer::Release() {
-	EXIT_IF(IsInvalid());
-
-	Common::LockGuard lock(*m_slot->pool_mutex);
-
-	WaitForFence();
-
-	m_slot->busy = false;
-	m_slot->Reset();
-	m_slot = nullptr;
-
-	EXIT_NOT_IMPLEMENTED(!IsInvalid());
-}
-
-void CommandBuffer::Begin() const {
-	EXIT_IF(m_rendering);
+void CommandBuffer::Begin() {
+	EXIT_IF(m_rendering || IsInvalid());
 	auto buffer = Handle();
 
 	vk::CommandBufferBeginInfo begin_info {};
 	begin_info.sType            = vk::StructureType::eCommandBufferBeginInfo;
 	begin_info.pNext            = nullptr;
-	begin_info.flags            = {};
+	begin_info.flags            = vk::CommandBufferUsageFlagBits::eOneTimeSubmit;
 	begin_info.pInheritanceInfo = nullptr;
 
 	auto result = buffer.begin(&begin_info);
@@ -107,106 +64,6 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0,
 	m_debug_arg4      = arg4;
 }
 
-void CommandBuffer::Execute(const SubmitInfo& submit) {
-	EXIT_IF(IsInvalid());
-	EXIT_IF(m_execute);
-	EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores ||
-	        submit.num_signal_semaphores > SubmitInfo::MaxSemaphores);
-
-	auto buffer = Handle();
-	auto fence  = m_slot->fence;
-
-	vk::TimelineSemaphoreSubmitInfo timeline_info {};
-	timeline_info.sType                     = vk::StructureType::eTimelineSemaphoreSubmitInfo;
-	timeline_info.waitSemaphoreValueCount   = submit.num_wait_semaphores;
-	timeline_info.pWaitSemaphoreValues      = submit.wait_ticks.data();
-	timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores;
-	timeline_info.pSignalSemaphoreValues    = submit.signal_ticks.data();
-
-	vk::SubmitInfo submit_info {};
-	submit_info.sType                = vk::StructureType::eSubmitInfo;
-	submit_info.pNext                = &timeline_info;
-	submit_info.waitSemaphoreCount   = submit.num_wait_semaphores;
-	submit_info.pWaitSemaphores      = submit.wait_semaphores.data();
-	submit_info.pWaitDstStageMask    = submit.wait_stages.data();
-	submit_info.commandBufferCount   = 1;
-	submit_info.pCommandBuffers      = &buffer;
-	submit_info.signalSemaphoreCount = submit.num_signal_semaphores;
-	submit_info.pSignalSemaphores    = submit.signal_semaphores.data();
-
-	auto& graphics = m_graphics;
-	EXIT_IF(graphics.queue == nullptr);
-
-	auto result = graphics.device.resetFences(1, &fence);
-	if (result != vk::Result::eSuccess) {
-		ReportVulkanFatal("vkResetFences (before submit)", result, m_slot->id, m_submit_seq,
-		                  m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2,
-		                  m_debug_arg3, m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	if (Config::GraphicsDebugDumpEnabled()) {
-		LOGF("vkQueueSubmit begin: slot=%u waits=%u signals=%u debug_op=%u debug_submit=%" PRIu64
-		     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     m_slot->id, submit.num_wait_semaphores, submit.num_signal_semaphores, m_debug_op,
-		     m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-
-	{
-		Common::LockGuard lock(graphics.queue_mutex);
-		m_submit_seq = m_scheduler.NextSubmitSequence();
-		result       = graphics.queue.submit(1, &submit_info, fence);
-	}
-
-	m_execute      = true;
-	m_fence_waited = false;
-
-	if (result != vk::Result::eSuccess) {
-		ReportVulkanFatal("vkQueueSubmit", result, m_slot->id, m_submit_seq, m_debug_op,
-		                  m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		                  m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-}
-
-void CommandBuffer::WaitForFence() {
-	FinalizeFence(false);
-}
-
-void CommandBuffer::WaitForFenceOnly() {
-	EXIT_IF(IsInvalid());
-	if (!m_execute || m_fence_waited) {
-		return;
-	}
-	auto device = m_graphics.device;
-	auto result = device.waitForFences(1, &m_slot->fence, VK_TRUE, UINT64_MAX);
-	if (result != vk::Result::eSuccess) {
-		ReportVulkanFatal("vkWaitForFences", result, m_slot->id, m_submit_seq, m_debug_op,
-		                  m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		                  m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-	m_fence_waited = true;
-}
-
-void CommandBuffer::WaitForFenceAndReset() {
-	FinalizeFence(true);
-}
-
-void CommandBuffer::FinalizeFence(bool reset_recording) {
-	const bool was_executed = m_execute;
-	WaitForFenceOnly();
-	if (was_executed) {
-		m_execute      = false;
-		m_fence_waited = false;
-		if (reset_recording) {
-			Common::LockGuard lock(*m_slot->pool_mutex);
-			m_slot->Reset();
-		}
-	}
-}
-
 void CommandBuffer::BeginRendering(const RenderState& state) const {
 	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
 	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);

src/graphics/host_gpu/renderer/image/image.cpp

@@ -736,15 +736,10 @@ Image::~Image() {
 	if (m_graphics == nullptr) {
 		return;
 	}
-	{
-		std::lock_guard lock(views.mutex);
-		for (auto& cached: views.views) {
-			if (cached.view != nullptr) {
-				m_graphics->device.destroyImageView(cached.view, nullptr);
-				cached.view = nullptr;
-			}
+	for (const auto& cached: views) {
+		if (cached.view != nullptr) {
+			m_graphics->device.destroyImageView(cached.view, nullptr);
 		}
-		views.views.clear();
 	}
 	if (backing.image != nullptr) {
 		m_graphics->DeleteImage(backing);

src/graphics/host_gpu/renderer/image/image.h

@@ -8,7 +8,6 @@
 
 #include <compare>
 #include <limits>
-#include <mutex>
 #include <optional>
 #include <span>
 #include <utility>
@@ -27,14 +26,6 @@ struct CachedImageView {
 	vk::ImageView view = nullptr;
 };
 
-struct ImageViewCache {
-	ImageViewCache() = default;
-	KYTY_CLASS_NO_COPY(ImageViewCache);
-
-	std::mutex                   mutex;
-	std::vector<CachedImageView> views;
-};
-
 struct ImageUsage {
 	bool texture       = false;
 	bool storage       = false;
@@ -150,7 +141,7 @@ public:
 
 	ImageInfo        info;
 	VulkanImage      backing;
-	ImageViewCache   views;
+	std::vector<CachedImageView> views;
 	ImageUsage       usage;
 	ImageBinding     binding;
 	bool             registered     = false;

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -4,8 +4,6 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/image.h"
 
-#include <mutex>
-
 namespace Libs::Graphics {
 
 namespace {
@@ -357,8 +355,7 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     image.layers);
 	}
 
-	std::lock_guard lock(views.mutex);
-	for (const auto& cached: views.views) {
+	for (const auto& cached: views) {
 		if (cached.info == normalized) {
 			return cached.view;
 		}
@@ -394,7 +391,7 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     view_info.level_count, view_info.base_layer, view_info.layer_count,
 		     static_cast<vk::ImageUsageFlags::MaskType>(view_info.usage));
 	}
-	views.views.push_back({normalized, view});
+	views.push_back({normalized, view});
 	return view;
 }
 

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -891,7 +891,7 @@ static vk::Sampler NativeSampler(RenderContext&                       context,
 static BufferView NativeUpload(RenderContext& context, std::span<const uint32_t> data) {
 	EXIT_IF(data.empty());
 	auto& command_buffer = context.GetCommandScheduler().Current();
-	EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute());
+	EXIT_IF(command_buffer.IsInvalid());
 	auto&      buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
 	const auto offset = buffer.Copy(data.data(), data.size_bytes(), 256);
 	return {.buffer = buffer.Handle(), .offset = offset, .range = data.size_bytes()};

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -548,9 +548,10 @@ void CreatePipelineInternal(
 
 			uint32_t attr_size       = 4;
 			auto     registers_num   = vs_input_info.resources_dst[index].registers_num;
-			auto     used_components = (vs_input_info.resource_fetch_components[index] > 0
-			                                ? vs_input_info.resource_fetch_components[index]
-			                                : registers_num);
+			const auto compiled_components =
+			    vs_input_info.stage.program->info.vertex_fetch_components[index];
+			auto used_components =
+			    compiled_components > 0 ? static_cast<int>(compiled_components) : registers_num;
 			GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size,
 			               static_cast<uint32_t>(used_components));
 

src/graphics/host_gpu/renderer/render.h

@@ -25,7 +25,6 @@ class Shader;
 struct GraphicContext;
 struct ShaderBufferResource;
 struct ShaderComputeInputInfo;
-struct CommandSlot;
 struct RenderDepthInfo;
 struct RenderColorInfo;
 struct DrawCallInfo;
@@ -77,40 +76,31 @@ struct SubmitInfo {
 
 class CommandBuffer {
 public:
-	explicit CommandBuffer(CommandScheduler& scheduler);
-	~CommandBuffer();
+	~CommandBuffer() = default;
 
 	KYTY_CLASS_NO_COPY(CommandBuffer);
 
 	[[nodiscard]] bool IsInvalid() const;
 
-	void Begin() const;
-	void End() const;
-	void Execute(const SubmitInfo& submit = {});
 	void SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0 = 0, uint32_t arg1 = 0,
 	                  uint32_t arg2 = 0, uint32_t arg3 = 0, uint64_t arg4 = 0);
 	void BeginRendering(const RenderState& state) const;
 	void EndRendering() const;
-	void WaitForFenceOnly();
-	void WaitForFence();
-	void WaitForFenceAndReset();
 
 	[[nodiscard]] vk::CommandBuffer Handle() const;
 	[[nodiscard]] GraphicContext&   GetGraphics() const noexcept { return m_graphics; }
 	[[nodiscard]] RenderContext&    GetContext() const noexcept { return m_context; }
-	[[nodiscard]] bool              IsExecute() const { return m_execute; }
+
+protected:
+	explicit CommandBuffer(CommandScheduler& scheduler);
 
 private:
-	void Release();
-	void FinalizeFence(bool reset_recording);
+	void Begin();
+	void End() const;
 
 	RenderContext&      m_context;
-	CommandScheduler&   m_scheduler;
 	GraphicContext&     m_graphics;
-	CommandSlot*        m_slot            = nullptr;
-	bool                m_execute         = false;
-	bool                m_fence_waited    = false;
-	uint64_t            m_submit_seq      = 0;
+	vk::CommandBuffer   m_buffer          = nullptr;
 	uint32_t            m_debug_op        = 0;
 	uint64_t            m_debug_submit_id = 0;
 	uint32_t            m_debug_arg0      = 0;
@@ -120,12 +110,12 @@ private:
 	uint64_t            m_debug_arg4      = 0;
 	mutable RenderState m_render_state;
 	mutable bool        m_rendering = false;
+
+	friend class CommandScheduler;
 };
 
 class RenderCommandBuffer final: public CommandBuffer {
 public:
-	explicit RenderCommandBuffer(CommandScheduler& scheduler): CommandBuffer(scheduler) {}
-
 	void Bind(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) noexcept {
 		m_registers   = &registers;
 		m_user_config = &user_config;
@@ -137,9 +127,13 @@ public:
 	[[nodiscard]] HW::Shader&     GetShaders() const noexcept { return *m_shaders; }
 
 private:
+	explicit RenderCommandBuffer(CommandScheduler& scheduler): CommandBuffer(scheduler) {}
+
 	HW::Context*    m_registers   = nullptr;
 	HW::UserConfig* m_user_config = nullptr;
 	HW::Shader*     m_shaders     = nullptr;
+
+	friend class CommandScheduler;
 };
 
 class RenderExecutor {

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -4,7 +4,6 @@
 #include "common/logging/log.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/presentation/videoOut.h"
-#include "kernel/pthread.h"
 #include "libs/errno.h"
 
 #include <algorithm>
@@ -39,7 +38,7 @@ void RenderContext::ShutdownGpu() {
 	}
 	if (m_video_out != nullptr) {
 		if (m_command_scheduler.Active()) {
-			m_command_scheduler.FinishCurrent();
+			m_command_scheduler.Finish();
 		}
 		m_command_scheduler.DrainPriorityOperations();
 		m_video_out = nullptr;
@@ -56,60 +55,54 @@ VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const {
 	return *m_video_out;
 }
 
-void RenderContext::AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) {
-	auto queue = LibKernel::EventQueue::KernelPinEqueue(eq);
-	if (!queue) {
+void RenderContext::AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
+	Common::LockGuard lock(m_interrupt_mutex);
+
+	auto it = std::find_if(
+	    m_interrupt_eqs.begin(), m_interrupt_eqs.end(),
+	    [eq, event_id](const auto& entry) { return entry.eq == eq && entry.event_id == event_id; });
+	if (it != m_interrupt_eqs.end()) {
 		return;
 	}
 
-	Common::LockGuard lock(m_eop_mutex);
-
-	auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) {
-		return entry.eq == eq && entry.id == id;
-	});
-	if (it != m_eop_eqs.end()) {
-		return;
-	}
-
-	m_eop_eqs.push_back({eq, std::move(queue), id});
+	m_interrupt_eqs.push_back({eq, event_id});
 }
 
-void RenderContext::DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) {
-	Common::LockGuard lock(m_eop_mutex);
+void RenderContext::DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
+	Common::LockGuard lock(m_interrupt_mutex);
 
-	auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) {
-		return entry.eq == eq && entry.id == id;
-	});
-	if (it == m_eop_eqs.end()) {
+	auto it = std::find_if(
+	    m_interrupt_eqs.begin(), m_interrupt_eqs.end(),
+	    [eq, event_id](const auto& entry) { return entry.eq == eq && entry.event_id == event_id; });
+	if (it == m_interrupt_eqs.end()) {
 		return;
 	}
 
-	m_eop_eqs.erase(it);
+	m_interrupt_eqs.erase(it);
 }
 
-void RenderContext::TriggerEopEvent(uint32_t context_id) {
-	std::vector<EopEqRegistration> registrations;
+void RenderContext::TriggerInterrupt(int event_id, uint32_t context_id) {
+	std::vector<InterruptEqRegistration> registrations;
 	{
-		Common::LockGuard lock(m_eop_mutex);
-		registrations = m_eop_eqs;
+		Common::LockGuard lock(m_interrupt_mutex);
+		for (const auto& registration: m_interrupt_eqs) {
+			if (registration.event_id == event_id) {
+				registrations.push_back(registration);
+			}
+		}
 	}
 
 	for (const auto& registration: registrations) {
 		const auto result = LibKernel::EventQueue::KernelTriggerEvent(
-		    registration.eq, static_cast<uintptr_t>(registration.id),
+		    registration.eq, static_cast<uintptr_t>(registration.event_id),
 		    LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS,
 		    reinterpret_cast<void*>(static_cast<uintptr_t>(context_id)));
 		if (result == LibKernel::KERNEL_ERROR_EBADF || result == LibKernel::KERNEL_ERROR_ENOENT) {
-			DeleteEopEq(registration.eq, registration.id);
+			DeleteInterruptEq(registration.eq, registration.event_id);
 			continue;
 		}
 		EXIT_NOT_IMPLEMENTED(result != OK);
 	}
-
-	auto tsc    = LibKernel::KernelReadTsc();
-	auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT,
-	                                                                  reinterpret_cast<void*>(tsc));
-	EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderContext.h

@@ -23,7 +23,6 @@ class VideoOutDriver;
 
 namespace Libs::Graphics {
 
-constexpr int AGC_USER_INTERRUPT_EVENT = 0x1800;
 class GuestGpu;
 
 class RenderContext {
@@ -48,15 +47,14 @@ public:
 	TextureCache&       GetTextureCache() { return m_gpu_resources.GetTextureCache(); }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
 
-	void AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id);
-	void DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id);
-	void TriggerEopEvent(uint32_t context_id);
+	void AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
+	void DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
+	void TriggerInterrupt(int event_id, uint32_t context_id);
 
 private:
-	struct EopEqRegistration {
-		LibKernel::EventQueue::KernelEqueue    eq = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
-		LibKernel::EventQueue::KernelEqueueRef queue;
-		int                                    id = 0;
+	struct InterruptEqRegistration {
+		LibKernel::EventQueue::KernelEqueue eq       = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
+		int                                 event_id = 0;
 	};
 
 	GraphicContext&           m_graphics;
@@ -70,8 +68,8 @@ private:
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 
-	Common::Mutex                  m_eop_mutex;
-	std::vector<EopEqRegistration> m_eop_eqs;
+	Common::Mutex                        m_interrupt_mutex;
+	std::vector<InterruptEqRegistration> m_interrupt_eqs;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -710,7 +710,8 @@ static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer) {
 	const auto& ctx              = buffer.GetRegisters();
 	const auto& sh_regs          = ctx.GetShaderRegisters();
 	const bool  has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
-	return has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs);
+	return ShaderAddressValid(buffer.GetShaders().GetPs().ps_regs.data_addr) &&
+	       (has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
 }
 
 enum class CbColorMode : uint8_t {

src/graphics/host_gpu/renderer/sync.cpp

@@ -20,9 +20,7 @@
 
 namespace Libs::Graphics::Sync {
 
-constexpr int      GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
-constexpr int      GRAPHICS_EVENT_EOP                       = 0x40;
-constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY       = 100000000;
+constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
 
 bool ScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t& value) {
 	if (host_frequency == 0) {
@@ -66,8 +64,9 @@ struct EndOfPipeSignal {
 	std::array<uint32_t, 4> debug_args      = {};
 	uint64_t                debug_data      = 0;
 	std::optional<uint64_t> destination;
-	EndOfPipeCompletion     completion      = EndOfPipeCompletion::None;
-	uint64_t                completion_data = 0;
+	EndOfPipeCompletion     completion         = EndOfPipeCompletion::None;
+	uint64_t                completion_data    = 0;
+	int                     interrupt_event_id = 0;
 };
 
 enum class EndOfPipeWriteSize : uint32_t { Dword = 4, Qword = 8 };
@@ -96,8 +95,10 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) {
 		case EndOfPipeCompletion::None: return;
 		case EndOfPipeCompletion::Interrupt: {
 			const auto context_id = static_cast<uint32_t>(signal.completion_data);
-			scheduler.DeferPriorityOperation(
-			    [&renderer, context_id] { renderer.TriggerEopEvent(context_id); });
+			const auto event_id   = signal.interrupt_event_id;
+			scheduler.DeferPriorityOperation([&renderer, event_id, context_id] {
+				renderer.TriggerInterrupt(event_id, context_id);
+			});
 			return;
 		}
 		case EndOfPipeCompletion::Flip: {
@@ -108,9 +109,10 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) {
 		}
 		case EndOfPipeCompletion::FlipAndInterrupt: {
 			const auto request_id = signal.completion_data;
-			scheduler.DeferPriorityOperation([&renderer, request_id] {
+			const auto event_id   = signal.interrupt_event_id;
+			scheduler.DeferPriorityOperation([&renderer, event_id, request_id] {
 				renderer.GetVideoOut().CompleteFlip(request_id);
-				renderer.TriggerEopEvent(0);
+				renderer.TriggerInterrupt(event_id, 0);
 			});
 			return;
 		}
@@ -135,7 +137,8 @@ static bool TriggersInterrupt(EndOfPipeWriteAction action) {
 
 static void RecordEndOfPipeWrite(uint64_t submit_id, CommandBuffer& buffer, uint64_t destination,
                                  uint64_t value, EndOfPipeWriteSize size,
-                                 EndOfPipeWriteAction action, uint32_t context_id = 0) {
+                                 EndOfPipeWriteAction action, int interrupt_event_id = 0,
+                                 uint32_t context_id = 0) {
 	const auto width      = static_cast<uint32_t>(size);
 	const auto value_low  = static_cast<uint32_t>(value);
 	const auto value_high = static_cast<uint32_t>(value >> 32u);
@@ -150,18 +153,12 @@ static void RecordEndOfPipeWrite(uint64_t submit_id, CommandBuffer& buffer, uint
 	    .debug_data      = destination,
 	    .destination     = destination,
 	    .completion      = interrupt ? EndOfPipeCompletion::Interrupt : EndOfPipeCompletion::None,
-	    .completion_data = context_id != 0 ? context_id : value,
+	    .completion_data = interrupt ? context_id : value,
+	    .interrupt_event_id = interrupt_event_id,
 	};
 	RecordEndOfPipeSignal(signal);
 }
 
-void TriggerAgcUserInterrupt() {
-	auto tsc    = LibKernel::KernelReadTsc();
-	auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT,
-	                                                                  reinterpret_cast<void*>(tsc));
-	EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT);
-}
-
 void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                         uint32_t value) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
@@ -220,31 +217,35 @@ void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
 }
 
 void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint64_t* dst_gpu_addr, uint64_t value,
+                                              uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                               uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
 	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::InterruptWriteBack,
-	                     context_id);
+	                     event_id, context_id);
 }
 
 void WriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint32_t* dst_gpu_addr, uint32_t value,
+                                              uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                               uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
 	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::InterruptWriteBack,
-	                     context_id);
+	                     event_id, context_id);
 }
 
 void WriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint64_t* dst_gpu_addr, uint64_t value, uint32_t context_id) {
+                                     uint64_t* dst_gpu_addr, uint64_t value, int event_id,
+                                     uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
-	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Interrupt, context_id);
+	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Interrupt, event_id,
+	                     context_id);
 }
 
 void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint32_t* dst_gpu_addr, uint32_t value, uint32_t context_id) {
+                                     uint32_t* dst_gpu_addr, uint32_t value, int event_id,
+                                     uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
-	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, context_id);
+	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, event_id,
+	                     context_id);
 }
 
 uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int flip_mode,
@@ -270,18 +271,20 @@ uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int f
 void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuffer& buffer,
                                                   uint32_t* dst_gpu_addr, uint32_t value,
                                                   int handle, int index, int flip_mode,
-                                                  int64_t flip_arg, uint64_t request_id) {
+                                                  int64_t flip_arg, uint64_t request_id,
+                                                  int event_id) {
 	const auto destination = reinterpret_cast<uint64_t>(dst_gpu_addr);
 	RecordEndOfPipeSignal({
-	    .buffer          = &buffer,
-	    .submit_id       = submit_id,
-	    .debug_operation = CommandBufferDebugOp::EopWriteBackFlip,
-	    .debug_args      = {static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
-	                        static_cast<uint32_t>(flip_mode), value},
-	    .debug_data      = static_cast<uint64_t>(flip_arg),
-	    .destination     = destination,
-	    .completion      = EndOfPipeCompletion::FlipAndInterrupt,
-	    .completion_data = request_id,
+	    .buffer             = &buffer,
+	    .submit_id          = submit_id,
+	    .debug_operation    = CommandBufferDebugOp::EopWriteBackFlip,
+	    .debug_args         = {static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
+	                           static_cast<uint32_t>(flip_mode), value},
+	    .debug_data         = static_cast<uint64_t>(flip_arg),
+	    .destination        = destination,
+	    .completion         = EndOfPipeCompletion::FlipAndInterrupt,
+	    .completion_data    = request_id,
+	    .interrupt_event_id = event_id,
 	});
 }
 
@@ -316,36 +319,24 @@ void WriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer& buffer, int han
 	});
 }
 
-void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id) {
+void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, int event_id, uint32_t context_id) {
 	ValidateEndOfPipeSignal({.buffer = &buffer});
 	auto& renderer  = buffer.GetContext();
 	auto& scheduler = renderer.GetCommandScheduler();
 	EXIT_IF(!scheduler.Active() || &buffer != &scheduler.Current());
 	scheduler.DeferPriorityOperation(
-	    [&renderer, context_id] { renderer.TriggerEopEvent(context_id); });
+	    [&renderer, event_id, context_id] { renderer.TriggerInterrupt(event_id, context_id); });
 }
 
-static void EopEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) {
+static void InterruptEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) {
 	EXIT_IF(event == nullptr);
 	event->triggered    = false;
 	event->event.fflags = 0;
 	event->event.data   = 0;
 }
 
-static void EopEventDeleteFunc(LibKernel::EventQueue::KernelEqueue       eq,
-                               LibKernel::EventQueue::KernelEqueueEvent* event) {
-	EXIT_IF(event == nullptr);
-	EXIT_NOT_IMPLEMENTED(event->event.filter != LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS);
-	if (event->event.ident == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT ||
-	    event->event.ident == GRAPHICS_EVENT_EOP) {
-		auto* renderer = static_cast<RenderContext*>(event->filter.data);
-		EXIT_IF(renderer == nullptr);
-		renderer->DeleteEopEq(eq, static_cast<int>(event->event.ident));
-	}
-}
-
-static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
-                                void*                                     trigger_data) {
+static void InterruptEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
+                                      void*                                     trigger_data) {
 	EXIT_IF(event == nullptr);
 
 	auto triggered_event = event->event;
@@ -362,30 +353,30 @@ static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
 int AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                void* udata) {
 	LibKernel::EventQueue::KernelEqueueEvent event;
-	event.triggered                = false;
-	event.event.ident              = static_cast<uintptr_t>(id);
-	event.event.filter             = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS;
-	event.event.udata              = udata;
-	event.event.fflags             = 0;
-	event.event.data               = id;
-	event.filter.delete_event_func = EopEventDeleteFunc;
-	event.filter.reset_func        = EopEventResetFunc;
-	event.filter.trigger_func      = EopEventTriggerFunc;
-	event.filter.data              = &renderer;
+	event.triggered           = false;
+	event.event.ident         = static_cast<uintptr_t>(id);
+	event.event.filter        = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS;
+	event.event.udata         = udata;
+	event.event.fflags        = 0;
+	event.event.data          = id;
+	event.filter.reset_func   = InterruptEventResetFunc;
+	event.filter.trigger_func = InterruptEventTriggerFunc;
 
 	int result = LibKernel::EventQueue::KernelAddEvent(eq, event);
 
-	if (result == 0 &&
-	    (id == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || id == GRAPHICS_EVENT_EOP)) {
-		renderer.AddEopEq(eq, id);
+	if (result == 0) {
+		renderer.AddInterruptEq(eq, id);
 	}
 
 	return result;
 }
 
-int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) {
+int DeleteEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id) {
 	int result = LibKernel::EventQueue::KernelDeleteEvent(
 	    eq, static_cast<uintptr_t>(id), LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS);
+	if (result == OK || result == LibKernel::KERNEL_ERROR_ENOENT) {
+		renderer.DeleteInterruptEq(eq, id);
+	}
 
 	return result;
 }

src/graphics/host_gpu/renderer/sync.h

@@ -17,8 +17,7 @@ namespace Sync {
                                            uint64_t& value);
 [[nodiscard]] uint64_t ReadReferenceClock();
 
-void TriggerAgcUserInterrupt();
-void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id);
+void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, int event_id, uint32_t context_id);
 
 void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                         uint32_t value);
@@ -35,16 +34,16 @@ void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
 void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
                                      uint64_t* dst_gpu_addr, uint64_t value);
 void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint32_t* dst_gpu_addr, uint32_t value,
+                                     uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                      uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint64_t* dst_gpu_addr, uint64_t value,
+                                     uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                      uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint32_t* dst_gpu_addr, uint32_t value,
+                                              uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                               uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint64_t* dst_gpu_addr, uint64_t value,
+                                              uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                               uint32_t context_id = 0);
 
 [[nodiscard]] uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index,
@@ -57,11 +56,12 @@ void WriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer& buffer, uint3
 void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuffer& buffer,
                                                   uint32_t* dst_gpu_addr, uint32_t value,
                                                   int handle, int index, int flip_mode,
-                                                  int64_t flip_arg, uint64_t request_id);
+                                                  int64_t flip_arg, uint64_t request_id,
+                                                  int event_id);
 
 int  AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                 void* udata);
-int  DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id);
+int  DeleteEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id);
 void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
 
 } // namespace Sync

src/graphics/presentation/window/swapchain.cpp

@@ -58,10 +58,10 @@
 namespace Libs::Graphics {
 
 struct Presenter::Frame {
-	VulkanImage                    image;
-	std::unique_ptr<CommandBuffer> present_commands;
-	bool                           busy         = false;
-	bool                           reusing_last = false;
+	VulkanImage image;
+	uint64_t    present_tick = 0;
+	bool        busy         = false;
+	bool        reusing_last = false;
 
 	void Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format);
 	void Transit(vk::CommandBuffer command, vk::ImageLayout layout, vk::AccessFlags2 access);
@@ -71,12 +71,11 @@ struct Presenter::Frame {
 
 class FramePool final {
 public:
-	explicit FramePool(WindowContext& window): m_window(window) {}
+	FramePool(WindowContext& window, CommandScheduler& scheduler)
+	    : m_window(window), m_scheduler(scheduler) {}
 	~FramePool() {
+		m_scheduler.Wait(m_scheduler.CurrentTick() - 1);
 		for (auto& frame: m_frames) {
-			if (frame->present_commands != nullptr) {
-				frame->present_commands->WaitForFenceOnly();
-			}
 			if (frame->image.image != nullptr) {
 				m_window.graphic_ctx.DeleteImage(frame->image);
 			}
@@ -188,15 +187,10 @@ public:
 	}
 
 private:
-	static void WaitForFrame(Presenter::Frame& frame) {
-		// The producer only waits here. Reset stays on the presentation thread that owns the
-		// allocating Vulkan command pool.
-		if (frame.present_commands != nullptr) {
-			frame.present_commands->WaitForFenceOnly();
-		}
-	}
+	void WaitForFrame(Presenter::Frame& frame) { m_scheduler.Wait(frame.present_tick); }
 
 	WindowContext&                                 m_window;
+	CommandScheduler&                              m_scheduler;
 	Common::Mutex                                  m_mutex;
 	Common::CondVar                                m_available;
 	std::vector<std::unique_ptr<Presenter::Frame>> m_frames;
@@ -330,7 +324,7 @@ public:
 	[[nodiscard]] Status AcquireNextImage();
 	[[nodiscard]] bool   PrepareImeOverlay();
 	void RecordPresentCommands(CommandBuffer& command, VulkanImage& source, bool draw_ime_overlay);
-	void Submit(CommandBuffer& command);
+	uint64_t             Submit(CommandScheduler& scheduler);
 	[[nodiscard]] Status Present();
 
 	[[nodiscard]] uint32_t ImageCount() const noexcept {
@@ -358,7 +352,7 @@ private:
 struct Presenter::Impl {
 	explicit Impl(WindowContext& owner)
 	    : renderer(*owner.render_context), window(owner), swapchain(owner),
-	      present_scheduler(renderer, owner.graphic_ctx), frames(owner) {
+	      present_scheduler(renderer, owner.graphic_ctx), frames(owner, present_scheduler) {
 		EXIT_IF(owner.render_context == nullptr);
 		swapchain.Create();
 		frames.Initialize(swapchain.ImageCount(), swapchain.Format());
@@ -628,7 +622,6 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 	}
 	EXIT_IF(m_image_index >= m_images.size());
 	auto vk_command = command.Handle();
-	command.Begin();
 
 	vk::ImageMemoryBarrier to_transfer {};
 	to_transfer.sType                           = vk::StructureType::eImageMemoryBarrier;
@@ -699,15 +692,14 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 		                           vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1,
 		                           &to_present);
 	}
-	command.End();
 }
 
-void Swapchain::Submit(CommandBuffer& command) {
+uint64_t Swapchain::Submit(CommandScheduler& scheduler) {
 	EXIT_IF(m_frame_index >= m_image_acquired.size() || m_image_index >= m_render_complete.size());
 	SubmitInfo submit;
 	submit.AddWait(m_image_acquired[m_frame_index], 1, vk::PipelineStageFlagBits::eTransfer);
 	submit.AddSignal(m_render_complete[m_image_index]);
-	command.Execute(submit);
+	return scheduler.Submit(submit);
 }
 
 Swapchain::Status Swapchain::Present() {
@@ -782,15 +774,9 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 		EXIT_IF(producer->IsInvalid());
 		frame->Clear(*producer, clear);
 	} else {
-		if (frame->present_commands == nullptr) {
-			frame->present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
-		}
-		auto& command = *frame->present_commands;
-		command.WaitForFenceAndReset();
-		command.Begin();
+		auto& command = m_impl->present_scheduler.BeginCommand();
 		frame->Clear(command, clear);
-		command.End();
-		command.Execute();
+		frame->present_tick = m_impl->present_scheduler.Submit();
 	}
 	return *frame;
 }
@@ -848,16 +834,12 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			m_impl->RecoverSwapchain(status);
 			continue;
 		}
-		if (frame.present_commands == nullptr) {
-			frame.present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
-		}
 		{
 			Common::LockGuard render_lock(m_impl->renderer.GetMutex());
-			frame.present_commands->WaitForFenceAndReset();
-			auto&      command          = *frame.present_commands;
-			const bool draw_ime_overlay = ime_visual.active && swapchain.PrepareImeOverlay();
+			auto&             command          = m_impl->present_scheduler.BeginCommand();
+			const bool        draw_ime_overlay = ime_visual.active && swapchain.PrepareImeOverlay();
 			swapchain.RecordPresentCommands(command, frame.image, draw_ime_overlay);
-			swapchain.Submit(command);
+			frame.present_tick = swapchain.Submit(m_impl->present_scheduler);
 		}
 		status = swapchain.Present();
 		if (status != Swapchain::Status::Success) {

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -7,7 +7,6 @@
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/recompiler/frontend/translate/Translate.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
@@ -49,7 +48,7 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
 	dump += "\nIR:\n";
 	dump += fmt::format("mode={} scratch_dwords={}\n",
 	                    ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords);
-	dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir);
+	dump += IR::ProgramToString(ir);
 	return dump;
 }
 
@@ -117,17 +116,8 @@ void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg
 	lanes->erase(first, last);
 }
 
-struct EmbeddedFetchLoad {
-	uint32_t              pc         = 0;
-	int                   attrib_id  = -1;
-	uint32_t              components = 0;
-	std::vector<uint32_t> prolog_loads;
-};
-
-struct EmbeddedFetchData {
-	std::vector<EmbeddedFetchLoad> loads;
-	int32_t                        vertex_offset_sgpr = -1;
-};
+using EmbeddedFetchLoad = Frontend::EmbeddedFetchLoad;
+using EmbeddedFetchData = Frontend::EmbeddedFetchPlan;
 
 bool IsDecodedSgpr(const Decoder::Operand& op) {
 	return op.kind == Decoder::OperandKind::Sgpr || op.kind == Decoder::OperandKind::VccLo ||
@@ -246,15 +236,6 @@ int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) {
 	return static_cast<int>((raw_offset + static_cast<uint32_t>(dword) * 4u) / 16u);
 }
 
-void AppendUniquePcs(std::vector<uint32_t>& dst, const std::vector<uint32_t>& src) {
-	dst.reserve(dst.size() + src.size());
-	for (auto pc: src) {
-		if (std::find(dst.begin(), dst.end(), pc) == dst.end()) {
-			dst.push_back(pc);
-		}
-	}
-}
-
 EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded,
                                             const ShaderVertexInputInfo* input_info,
                                             uint32_t user_data_base, uint32_t user_data_count,
@@ -489,103 +470,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 	return data;
 }
 
-const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const std::vector<EmbeddedFetchLoad>& loads,
-                                               uint32_t                              pc) {
-	for (const auto& load: loads) {
-		if (load.pc == pc) {
-			return &load;
-		}
-	}
-	return nullptr;
-}
-
-bool EmbeddedFetchPcInList(const std::vector<uint32_t>& pcs, uint32_t pc) {
-	return std::find(pcs.begin(), pcs.end(), pc) != pcs.end();
-}
-
-bool IsIrFetchPrologLoad(const IR::Instruction& inst) {
-	return inst.op == IR::Opcode::SLoadDword || inst.op == IR::Opcode::SBufferLoadDword;
-}
-
-int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
-                                 const EmbeddedFetchLoad&     load) {
-	if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num &&
-	    input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) {
-		return load.attrib_id;
-	}
-	for (int i = 0; i < input_info->resources_num; i++) {
-		const auto& dst = input_info->resources_dst[i];
-		if (dst.attr_id == load.attrib_id &&
-		    load.components <= static_cast<uint32_t>(std::max(dst.registers_num, 1))) {
-			return i;
-		}
-	}
-	for (int i = 0; i < input_info->resources_num; i++) {
-		if (input_info->resources_dst[i].attr_id == load.attrib_id) {
-			return i;
-		}
-	}
-	return -1;
-}
-
-uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputInfo* input_info,
-                                      const std::vector<EmbeddedFetchLoad>& loads) {
-	if (loads.empty()) {
-		return 0;
-	}
-
-	std::vector<uint32_t> prolog_pcs;
-	prolog_pcs.reserve(loads.size());
-	for (const auto& load: loads) {
-		AppendUniquePcs(prolog_pcs, load.prolog_loads);
-	}
-
-	auto*    mutable_input_info = const_cast<ShaderVertexInputInfo*>(input_info);
-	uint32_t rewritten          = 0;
-	for (auto& block: ir.blocks) {
-		for (auto& inst: block.instructions) {
-			if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) {
-				auto pc = inst.pc;
-				inst    = {};
-				inst.pc = pc;
-				inst.op = IR::Opcode::ControlNop;
-				continue;
-			}
-
-			if (inst.op != IR::Opcode::BufferLoadDword) {
-				continue;
-			}
-			const auto* load = FindEmbeddedFetchLoad(loads, inst.pc);
-			if (load == nullptr || inst.memory.data_dwords != load->components ||
-			    inst.dst.kind != IR::OperandKind::Register ||
-			    inst.dst.reg.file != IR::RegisterFile::Vector) {
-				continue;
-			}
-
-			const auto resource_id = ResolveEmbeddedFetchResource(input_info, *load);
-			if (resource_id < 0 || resource_id >= input_info->resources_num) {
-				LOGF("ShaderRecompiler VS embedded fetch remap failed: pc=0x%08" PRIx32
-				     " attrib=%d resources=%d\n",
-				     inst.pc, load->attrib_id, input_info->resources_num);
-				continue;
-			}
-
-			inst.op                         = IR::Opcode::LoadInputF32;
-			inst.input_info.attr            = static_cast<uint32_t>(resource_id);
-			inst.input_info.chan            = 0;
-			inst.input_info.component_count = load->components;
-			inst.memory                     = {};
-			inst.src_count                  = 0;
-			mutable_input_info->resource_fetch_components[resource_id] =
-			    std::max(mutable_input_info->resource_fetch_components[resource_id],
-			             static_cast<int>(load->components));
-			rewritten += load->components;
-		}
-	}
-
-	return rewritten;
-}
-
 } // namespace
 
 bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
@@ -677,27 +561,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	}
 
 	IR::Program ir;
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
-	     StageName(options.stage), options.shader_hash);
-	const auto lower_program = [&](IR::Program& output) {
-		output = {};
-		if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
-			return false;
-		}
-		output.shader_hash     = options.shader_hash;
-		output.user_data_base  = options.user_data_base;
-		output.user_data_count = options.user_data_count;
-		output.scratch_dwords  = options.scratch_dwords;
-		return true;
-	};
-	if (!lower_program(ir)) {
-		return false;
-	}
-
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
-	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
 	const ShaderVertexInputInfo*  vertex  = nullptr;
 	const ShaderPixelInputInfo*   pixel   = nullptr;
 	const ShaderComputeInputInfo* compute = nullptr;
@@ -709,42 +572,55 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	}
 	EmbeddedFetchData embedded_fetch;
 	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
-		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, ir.user_data_base,
-		                                           ir.user_data_count, options.wave_size);
-		auto rewritten = RewriteEmbeddedVertexFetches(ir, vertex, embedded_fetch.loads);
-		if (rewritten > 0 || !embedded_fetch.loads.empty()) {
-			LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n",
-			     GetDumpLabel(options), static_cast<uint64_t>(embedded_fetch.loads.size()),
-			     rewritten);
+		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base,
+		                                           options.user_data_count, options.wave_size);
+		if (!embedded_fetch.loads.empty()) {
+			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
+			     static_cast<uint64_t>(embedded_fetch.loads.size()));
 		}
 	}
-	ir.dispatcher_fallback = dispatcher_fallback;
-	if (!dispatcher_reason.empty()) {
-		ir.fallback_reason = dispatcher_reason;
-	}
-
-	ir.values = std::make_shared<IR::ValueProgram>();
-	if (!Frontend::TranslateProgram(ir, *ir.values, vertex, pixel, compute, error)) {
+	Frontend::TranslateOptions translate_options {
+	    .stage               = options.stage,
+	    .wave_size           = options.wave_size,
+	    .shader_hash         = options.shader_hash,
+	    .user_data_base      = options.user_data_base,
+	    .user_data_count     = options.user_data_count,
+	    .scratch_dwords      = options.scratch_dwords,
+	    .dispatcher_fallback = dispatcher_fallback,
+	    .cfg_failure_kind    = cfg.failure_kind,
+	    .fallback_reason     = dispatcher_reason.empty() ? cfg.unsupported_reason
+	                                                    : dispatcher_reason,
+	    .vertex              = vertex,
+	    .pixel               = pixel,
+	    .compute             = compute,
+	    .embedded_fetch      = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
+	};
+	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	if (!Frontend::TranslateProgram(decoded, cfg, translate_options, ir, error)) {
 		return false;
 	}
-	ir.blocks.clear();
-	IR::RewriteToSsa(ir.values->blocks);
-	IR::ConstantPropagationPass(ir.values->blocks);
-	IR::ResolveControlFlowIdentities(*ir.values);
-	IR::RemoveIdentities(ir.values->blocks);
-	IR::EliminateDeadCode(ir.values->blocks);
-	const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size);
+	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
+	     " elapsed_ms=%" PRIu64 "\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
+	IR::RewriteToSsa(ir.blocks);
+	IR::ConstantPropagationPass(ir.blocks);
+	IR::ResolveControlFlowIdentities(ir);
+	IR::RemoveIdentities(ir.blocks);
+	IR::EliminateDeadCode(ir.blocks);
+	const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size);
 	if (read_lane_stats.rewritten_reads != 0) {
 		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
 		     read_lane_stats.rewritten_reads);
-		IR::ConstantPropagationPass(ir.values->blocks);
-		IR::ResolveControlFlowIdentities(*ir.values);
-		IR::RemoveIdentities(ir.values->blocks);
-		IR::EliminateDeadCode(ir.values->blocks);
+		IR::ConstantPropagationPass(ir.blocks);
+		IR::ResolveControlFlowIdentities(ir);
+		IR::RemoveIdentities(ir.blocks);
+		IR::EliminateDeadCode(ir.blocks);
 	}
 	if (options.stage == ShaderType::Compute) {
 		const auto lds_barriers =
-		    IR::InsertSharedMemoryBarriers(*ir.values, ir.wave_size, *compute);
+		    IR::InsertSharedMemoryBarriers(ir, ir.wave_size, *compute);
 		if (lds_barriers.inserted_barriers != 0) {
 			LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options),
 			     lds_barriers.inserted_barriers);
@@ -758,11 +634,11 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		}
 		return false;
 	}
-	IR::EliminateDeadCode(ir.values->blocks);
+	IR::EliminateDeadCode(ir.blocks);
 	if (!IR::TrackResources(ir, error)) {
 		return false;
 	}
-	IR::EliminateDeadCode(ir.values->blocks);
+	IR::EliminateDeadCode(ir.blocks);
 	if (options.stage == ShaderType::Vertex) {
 		ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
 	}

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -1,7 +1,7 @@
 #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
@@ -51,22 +51,20 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
 	}
 	bool uses_gds = false;
-	if (program.values != nullptr) {
-		for (const auto* block: program.values->blocks) {
-			for (const auto& inst: *block) {
-				if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
-					continue;
-				}
-				const auto index = inst.Flags<IR::MemoryFlags>().index;
-				if (index >= program.values->memory_info.size()) {
-					return Fail(error, "shared operation has invalid memory metadata");
-				}
-				const auto kind = program.values->memory_info[index].kind;
-				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
-					return Fail(error, "shared operation has invalid resource kind");
-				}
-				uses_gds |= kind == IR::ResourceKind::Gds;
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
+				continue;
 			}
+			const auto index = inst.Flags<IR::MemoryFlags>().index;
+			if (index >= program.memory_info.size()) {
+				return Fail(error, "shared operation has invalid memory metadata");
+			}
+			const auto kind = program.memory_info[index].kind;
+			if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+				return Fail(error, "shared operation has invalid resource kind");
+			}
+			uses_gds |= kind == IR::ResourceKind::Gds;
 		}
 	}
 	if (uses_gds) {
@@ -76,11 +74,10 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
 	}
 	const bool uses_flattened_runtime =
-	    program.values != nullptr &&
-	    (!program.values->srt_reads.empty() ||
+	    !program.srt_reads.empty() ||
 	     std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
 		     return image.indirect_mapping_capacity != 0u;
-	     }));
+	     });
 	if (uses_flattened_runtime) {
 		Expect(Kind::FlattenedSrt);
 	}
@@ -120,9 +117,6 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		return Fail(error, "native user-data layout is inconsistent");
 	}
 
-	if (program.values == nullptr) {
-		return Fail(error, "native shader plan has no typed SSA");
-	}
 	const auto planning_only_handle = [&](const IR::Inst& handle) {
 		return !handle.Uses().empty() &&
 		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
@@ -132,11 +126,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 				       return false;
 			       }
 			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
-			       return index < program.values->memory_info.size() &&
-			              program.values->memory_info[index].planning_only;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].planning_only;
 		       });
 	};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto dense = inst.Flags<uint32_t>();
 			switch (inst.GetOpcode()) {
@@ -174,7 +168,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 				case IR::ValueOpcode::ReadConst: {
 					const auto slot = inst.Arg(1).Resolve();
 					if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
-					    slot.U32() >= program.values->srt_reads.size()) {
+					    slot.U32() >= program.srt_reads.size()) {
 						return Fail(error, "flattened SRT read has an invalid dense slot");
 					}
 					break;
@@ -191,19 +185,15 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 	program.spirv_requirements.reset();
 	IR::SpirvRequirements requirements {};
-	if (program.values == nullptr) {
-		program.spirv_requirements.emplace(requirements);
-		return true;
-	}
 	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
 				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
-				if (memory_index >= program.values->memory_info.size()) {
+				if (memory_index >= program.memory_info.size()) {
 					return Fail(error, "address operation has invalid memory metadata");
 				}
-				if (program.values->memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+				if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
 					if (program.scratch_dwords == 0) {
 						return Fail(error, "scratch operation has no per-thread storage");
 					}
@@ -212,10 +202,10 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 			}
 			if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
 				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
-				if (memory_index >= program.values->memory_info.size()) {
+				if (memory_index >= program.memory_info.size()) {
 					return Fail(error, "buffer operation has invalid memory metadata");
 				}
-				const auto& memory = program.values->memory_info[memory_index];
+				const auto& memory = program.memory_info[memory_index];
 				if (memory.kind == IR::ResourceKind::Buffer) {
 					if (memory.resource >= program.info.buffers.size()) {
 						return Fail(error, "buffer operation has invalid resource metadata");
@@ -231,10 +221,10 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 			const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
 			if (shared_access != IR::SharedAccess::None) {
 				const auto index = inst.Flags<IR::MemoryFlags>().index;
-				if (index >= program.values->memory_info.size()) {
+				if (index >= program.memory_info.size()) {
 					return Fail(error, "shared operation has invalid memory metadata");
 				}
-				const auto kind = program.values->memory_info[index].kind;
+				const auto kind = program.memory_info[index].kind;
 				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
 					return Fail(error, "shared operation has invalid resource kind");
 				}
@@ -288,11 +278,11 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
 					break;
 				case IR::ValueOpcode::SetAttribute: {
 					const auto index = inst.Flags<IR::ExportFlags>().index;
-					if (index >= program.values->export_info.size()) {
+					if (index >= program.export_info.size()) {
 						return Fail(error, "attribute export has invalid metadata");
 					}
 					if (program.stage == ShaderType::Pixel &&
-					    program.values->export_info[index].vm) {
+					    program.export_info[index].vm) {
 						requirements.pixel_valid_mask = true;
 					}
 					break;
@@ -330,12 +320,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
 	if (!ValidateNativeProgram(program, error)) {
 		return false;
 	}
-	if (program.values == nullptr) {
-		SetError(error, "SPIR-V emitter requires planned typed SSA");
-		return false;
-	}
-	const auto& value_program = *program.values;
-	if (!IR::ValidateValueProgram(value_program, true, error)) {
+	if (!IR::ValidateProgram(program, true, error)) {
 		return false;
 	}
 	EmitterState state(program, resources, input_info);
@@ -348,7 +333,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
 	AllocateInputVariables(state);
 	AllocateOutputVariables(state);
 	DefineModule(state);
-	if (!EmitValueProgram(state, value_program, error)) {
+	if (!EmitProgram(state, program, error)) {
 		return false;
 	}
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -2,7 +2,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -5,13 +5,11 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
 
-bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
-	if (reg.file != IR::RegisterFile::Scalar) {
-		return false;
-	}
+bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) {
+	const auto register_index = IR::RegIndex(reg);
 	const auto& registers = state.program.bindings.user_data_registers;
-	const auto  found     = std::lower_bound(registers.begin(), registers.end(), reg.index);
-	if (found == registers.end() || *found != reg.index) {
+	const auto  found     = std::lower_bound(registers.begin(), registers.end(), register_index);
+	if (found == registers.end() || *found != register_index) {
 		return false;
 	}
 	dword_index = static_cast<uint32_t>(found - registers.begin());
@@ -504,7 +502,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetUserData: {
 			const auto reg   = inst.Arg(0).ScalarRegister();
 			uint32_t   dword = 0;
-			if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) {
+			if (!UserDataDwordIndex(state, reg, dword)) {
 				ctx.Define(inst, ConstantU32(state, 0));
 			} else {
 				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -6,7 +6,7 @@
 #include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
@@ -407,7 +407,7 @@ inline void EmitLabel(EmitterState& state, uint32_t label) {
 }
 
 struct ValueEmitContext {
-	ValueEmitContext(EmitterState& state_, const IR::ValueProgram& program_)
+	ValueEmitContext(EmitterState& state_, const IR::Program& program_)
 	    : state(state_), program(program_) {}
 
 	uint32_t              Def(IR::Value value);
@@ -425,7 +425,7 @@ struct ValueEmitContext {
 	void                  Fail(const IR::Inst& inst, const char* reason);
 
 	EmitterState&                                                      state;
-	const IR::ValueProgram&                                            program;
+	const IR::Program&                                            program;
 	std::unordered_map<const IR::Inst*, uint32_t>                      definitions;
 	std::unordered_map<const IR::Block*, uint32_t>                     labels;
 	const std::unordered_map<const IR::Inst*, uint32_t>*               dispatcher_spills = nullptr;
@@ -847,7 +847,7 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
 
 bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
 
-bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error);
+bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error);
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -436,12 +436,12 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 		                     Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)),
 		              value);
 	};
-	if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
-		AtomicUpdate(ctx.state, pointer, mem.kind, merge);
-	} else {
+	if (mem.kind == IR::ResourceKind::Scratch) {
 		const auto old = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer});
 		ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
+	} else {
+		AtomicUpdate(ctx.state, pointer, mem.kind, merge);
 	}
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -79,9 +79,9 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
 	}
 }
 
-const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) {
+const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
 	const auto found = std::ranges::find_if(
-	    program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; });
+	    program.block_info, [&](const IR::BlockInfo& info) { return info.id == id; });
 	if (found == program.block_info.end()) {
 		return nullptr;
 	}
@@ -94,7 +94,7 @@ void EmitReturn(ValueEmitContext& ctx) {
 }
 
 void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
-                              const IR::ValueBlockInfo& info) {
+                              const IR::BlockInfo& info) {
 	const auto& term       = info.terminator;
 	const auto  emit_merge = [&]() {
 		if (term.loop_header) {
@@ -150,7 +150,7 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState&
 }
 
 uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
-                              const IR::Block* block, const IR::ValueBlockInfo& info) {
+                              const IR::Block* block, const IR::BlockInfo& info) {
 	const auto& term = info.terminator;
 	switch (term.kind) {
 		case CFG::TerminatorKind::Branch:
@@ -503,7 +503,7 @@ void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
 	error  = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
 }
 
-bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) {
+bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error) {
 	ValueEmitContext                       ctx(state, program);
 	std::optional<DispatcherFunctionState> dispatcher;
 	if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -1342,7 +1342,7 @@ bool IsolateSemanticLoopHeaders(Graph& graph, std::string* error) {
 		// SPIR-V requires OpLoopMerge and its branch to remain in the loop header's
 		// physical block. Keep that header as a dedicated control node, exactly like
 		// the separate Loop node in shadPS4's structured AST. Guest instructions live
-		// in the body so later lowering may introduce bounds/EXEC control flow without
+		// in the body so later translation may introduce bounds/EXEC control flow without
 		// displacing OpLoopMerge into a helper-created block.
 		if (!IsolateSemanticLoopHeader(graph, loop->header)) {
 			SetFailure(graph, FailureKind::StructuredControlFlow, loop->header,

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -423,7 +423,7 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	DecodeVectorGpr(vdst, inst.dst);
 	if (inst.opcode == Opcode::DS_READ_U16_D16) {
 		// Native D16 reads update only the selected destination half. Reuse the partial
-		// destination representation so the typed lowering preserves the upper word.
+		// destination representation so typed IR translation preserves the upper word.
 		inst.dst.sdwa_sel = 4u;
 	}
 	DecodeVectorGpr(addr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -299,6 +299,7 @@ enum class Opcode {
 	V_LSHRREV_B32,
 	V_ASHR_I32,
 	V_ASHRREV_I32,
+	V_LSHLREV_B64,
 	V_LSHRREV_B64,
 	V_LSHLREV_B16,
 	V_LSHRREV_B16,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -7,7 +7,7 @@ namespace {
 
 using Detail::OpcodeMap;
 
-// These profiles describe the selectors that the current lowering implements. They are not
+// These profiles describe the selectors that the current translator implements. They are not
 // architectural SDWA legality classes.
 enum class Vop2SdwaProfile {
 	None,
@@ -281,6 +281,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x16au, Opcode::V_MUL_HI_U32},
     {0x16bu, Opcode::V_MUL_LO_I32},
     {0x16cu, Opcode::V_MUL_HI_I32},
+    {0x2ffu, Opcode::V_LSHLREV_B64},
     {0x300u, Opcode::V_LSHRREV_B64},
     {0x303u, Opcode::V_ADD_NC_U16},
     {0x304u, Opcode::V_SUB_NC_U16},
@@ -1248,6 +1249,7 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 		case Opcode::V_MIN_I16:
 		case Opcode::V_ADD_NC_I16:
 		case Opcode::V_SUB_NC_I16:
+		case Opcode::V_LSHLREV_B64:
 		case Opcode::V_LSHRREV_B64:
 		case Opcode::V_LSHLREV_B16:
 		case Opcode::V_LSHRREV_B16:

src/graphics/shader/recompiler/frontend/translate/Attribute.cpp

@@ -1,42 +1,108 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
+#include "graphics/shader/shader.h"
 
 #include <algorithm>
 #include <array>
+#include <fmt/format.h>
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) {
-	const auto index = static_cast<uint32_t>(value_program.export_info.size());
-	value_program.export_info.push_back(inst.export_info);
+namespace {
+
+IR::ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t& index) {
+	index = 0;
+	switch (target) {
+		case 0x08u: return IR::ExportTargetKind::MrtZ;
+		case 0x09u: return IR::ExportTargetKind::Null;
+		case 0x14u: return IR::ExportTargetKind::Primitive;
+		default: break;
+	}
+	if (target <= 0x07u) {
+		index = target;
+		return IR::ExportTargetKind::Mrt;
+	}
+	if (target >= 0x0cu && target <= 0x0fu) {
+		index = target - 0x0cu;
+		return IR::ExportTargetKind::Position;
+	}
+	if (target >= 0x20u && target <= 0x3fu) {
+		index = target - 0x20u;
+		return IR::ExportTargetKind::Parameter;
+	}
+	return IR::ExportTargetKind::Unknown;
+}
+
+} // namespace
+
+IR::ExportFlags Translator::AddExportInfo(const Decoder::Instruction& inst) {
+	IR::ExportInfo info;
+	info.kind        = ExportTargetKindFromTarget(inst.exp.target, info.index);
+	info.target      = inst.exp.target;
+	info.en          = inst.exp.en;
+	info.done        = inst.exp.done;
+	info.compr       = inst.exp.compr;
+	info.vm          = inst.exp.vm;
+	const auto index = static_cast<uint32_t>(program.export_info.size());
+	program.export_info.push_back(info);
 	return {.index = index, .pc = inst.pc};
 }
 
-bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) {
-	if (inst.op == IR::Opcode::LoadInputF32) {
-		for (uint32_t component = 0; component < inst.input_info.component_count; component++) {
-			const auto chan  = inst.input_info.chan + component;
-			const auto value = inst.input_info.interpolation_mode < 3u
-			                       ? ir.Emit(IR::ValueOpcode::GetInterpolationParameter,
-			                                 {IR::Value(inst.input_info.attr), IR::Value(chan),
-			                                  IR::Value(inst.input_info.interpolation_mode)})
-			                       : ir.Emit(IR::ValueOpcode::GetAttribute,
-			                                 {IR::Value(inst.input_info.attr), IR::Value(chan)});
-			WriteOperand(OffsetOperand(inst.dst, component), value);
-		}
-		return true;
+bool Translator::TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
+                                        uint32_t component_count) {
+	for (uint32_t component = 0; component < component_count; component++) {
+		const auto value =
+		    ir.Emit(IR::ValueOpcode::GetAttribute, {IR::Value(attribute), IR::Value(component)});
+		WriteOperand(OffsetOperand(inst.dst, component), value);
 	}
-	if (inst.op == IR::Opcode::Export) {
-		std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u),
-		                                     IR::Value(0u)};
-		for (uint32_t index = 0; index < std::min(inst.src_count, 4u); index++) {
-			components[index] = ReadRawU32(PlainOperand(inst.src[index]));
-		}
-		const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
-		                          {components[0], components[1], components[2], components[3]});
-		ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
-		return true;
-	}
-	return false;
+	return true;
 }
 
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+void Translator::V_INTERP_P1_F32() {}
+
+void Translator::V_INTERP_P2_F32(const Decoder::Instruction& inst) {
+	const auto value = ir.Emit(IR::ValueOpcode::GetAttribute,
+	                           {IR::Value(inst.src1.value), IR::Value(inst.src2.value)});
+	WriteOperand(inst.dst, value);
+}
+
+bool Translator::V_INTERP_MOV_F32(const Decoder::Instruction& inst, std::string* error) {
+	if (inst.src0.value >= 3u) {
+		return ShaderError::Fail(error,
+		                         fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}",
+		                                     inst.src0.value, inst.pc));
+	}
+	const auto value = ir.Emit(
+	    IR::ValueOpcode::GetInterpolationParameter,
+	    {IR::Value(inst.src1.value), IR::Value(inst.src2.value), IR::Value(inst.src0.value)});
+	WriteOperand(inst.dst, value);
+	return true;
+}
+
+bool Translator::EXP(const Decoder::Instruction& inst, std::string* error) {
+	uint32_t index = 0;
+	if (ExportTargetKindFromTarget(inst.exp.target, index) == IR::ExportTargetKind::Unknown) {
+		return ShaderError::Fail(error,
+		                         fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}",
+		                                     inst.exp.target, inst.pc));
+	}
+	std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u),
+	                                     IR::Value(0u)};
+	for (uint32_t source = 0; source < std::min(inst.src_count, 4u); source++) {
+		components[source] = ReadRawU32(PlainOperand(SourceAt(inst, source)));
+	}
+	const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
+	                          {components[0], components[1], components[2], components[3]});
+	ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
+	return true;
+}
+
+bool Translator::EmitInterpolation(const Decoder::Instruction& inst, std::string* error) {
+	switch (inst.opcode) {
+		case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return true;
+		case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return true;
+		case Decoder::Opcode::V_INTERP_MOV_F32: return V_INTERP_MOV_F32(inst, error);
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -1,199 +1,65 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::TranslateIntegerCompare(const IR::Instruction& inst) {
-	IR::ValueOpcode opcode {};
-	IR::Type        type = IR::Type::U32;
-	switch (inst.op) {
-		case IR::Opcode::CompareFalse:
-			WriteCompareResult(inst.dst, IR::U1(IR::Value(false)));
-			return true;
-		case IR::Opcode::CompareTrue:
-			WriteCompareResult(inst.dst, IR::U1(IR::Value(true)));
-			return true;
-		case IR::Opcode::CompareEqU32:
-		case IR::Opcode::CompareMaskEqU32:
-		case IR::Opcode::CompareEqI32:
-		case IR::Opcode::CompareMaskEqI32: opcode = IR::ValueOpcode::IEqual32; break;
-		case IR::Opcode::CompareNeU32:
-		case IR::Opcode::CompareMaskNeU32:
-		case IR::Opcode::CompareNeI32:
-		case IR::Opcode::CompareMaskNeI32: opcode = IR::ValueOpcode::INotEqual32; break;
-		case IR::Opcode::CompareGtU32:
-		case IR::Opcode::CompareMaskGtU32: opcode = IR::ValueOpcode::UGreaterThan32; break;
-		case IR::Opcode::CompareGeU32:
-		case IR::Opcode::CompareMaskGeU32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break;
-		case IR::Opcode::CompareLtU32:
-		case IR::Opcode::CompareMaskLtU32: opcode = IR::ValueOpcode::ULessThan32; break;
-		case IR::Opcode::CompareLeU32:
-		case IR::Opcode::CompareMaskLeU32: opcode = IR::ValueOpcode::ULessThanEqual32; break;
-		case IR::Opcode::CompareGtI32:
-		case IR::Opcode::CompareMaskGtI32: opcode = IR::ValueOpcode::SGreaterThan32; break;
-		case IR::Opcode::CompareGeI32:
-		case IR::Opcode::CompareMaskGeI32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break;
-		case IR::Opcode::CompareLtI32:
-		case IR::Opcode::CompareMaskLtI32: opcode = IR::ValueOpcode::SLessThan32; break;
-		case IR::Opcode::CompareLeI32:
-		case IR::Opcode::CompareMaskLeI32: opcode = IR::ValueOpcode::SLessThanEqual32; break;
-		case IR::Opcode::CompareEqU64:
-			opcode = IR::ValueOpcode::IEqual64;
-			type   = IR::Type::U64;
-			break;
-		case IR::Opcode::CompareGtU64:
-			opcode = IR::ValueOpcode::UGreaterThan64;
-			type   = IR::Type::U64;
-			break;
-		case IR::Opcode::CompareNeU64:
-			opcode = IR::ValueOpcode::INotEqual64;
-			type   = IR::Type::U64;
-			break;
-		default: return false;
+void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar,
+                                   bool cmpx) {
+	if (scalar) {
+		ir.SetScc(value);
+		return;
 	}
-	const auto lhs = ReadOperand(inst.src[0], type);
-	const auto rhs = ReadOperand(inst.src[1], type);
-	WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs})));
-	return true;
+	const auto masked = ir.LogicalAnd(ir.GetExec(), value);
+	if (cmpx) {
+		const auto mask = BallotMask(masked);
+		ir.SetExec(masked);
+		ir.SetExecLo(mask[0]);
+		ir.SetExecHi(mask[1]);
+		return;
+	}
+	WriteMask(inst.dst, masked);
 }
 
-bool Translator::TranslateInteger16Compare(const IR::Instruction& inst) {
-	IR::ValueOpcode opcode {};
-	bool            signed_value = false;
-	switch (inst.op) {
-		case IR::Opcode::CompareEqU16:
-		case IR::Opcode::CompareEqI16: opcode = IR::ValueOpcode::IEqual32; break;
-		case IR::Opcode::CompareNeU16:
-		case IR::Opcode::CompareNeI16: opcode = IR::ValueOpcode::INotEqual32; break;
-		case IR::Opcode::CompareGtU16: opcode = IR::ValueOpcode::UGreaterThan32; break;
-		case IR::Opcode::CompareGeU16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break;
-		case IR::Opcode::CompareLtU16: opcode = IR::ValueOpcode::ULessThan32; break;
-		case IR::Opcode::CompareLeU16: opcode = IR::ValueOpcode::ULessThanEqual32; break;
-		case IR::Opcode::CompareGtI16:
-			opcode       = IR::ValueOpcode::SGreaterThan32;
-			signed_value = true;
-			break;
-		case IR::Opcode::CompareGeI16:
-			opcode       = IR::ValueOpcode::SGreaterThanEqual32;
-			signed_value = true;
-			break;
-		case IR::Opcode::CompareLtI16:
-			opcode       = IR::ValueOpcode::SLessThan32;
-			signed_value = true;
-			break;
-		case IR::Opcode::CompareLeI16:
-			opcode       = IR::ValueOpcode::SLessThanEqual32;
-			signed_value = true;
-			break;
-		default: return false;
-	}
-	if (inst.op == IR::Opcode::CompareEqI16 || inst.op == IR::Opcode::CompareNeI16) {
-		signed_value = true;
-	}
-	WriteCompareResult(inst.dst,
-	                   IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src[0], signed_value),
-	                                           ReadU16AsU32(inst.src[1], signed_value)})));
-	return true;
+void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar,
+                                     bool cmpx) {
+	EmitCompareResult(inst, IR::U1(IR::Value(value)), scalar, cmpx);
 }
 
-bool Translator::TranslateFloatCompare(const IR::Instruction& inst) {
-	IR::ValueOpcode opcode {};
-	bool            half = false;
-	switch (inst.op) {
-		case IR::Opcode::CompareEqF32:
-		case IR::Opcode::CompareMaskEqF32: opcode = IR::ValueOpcode::FPOrdEqual32; break;
-		case IR::Opcode::CompareNeF32:
-		case IR::Opcode::CompareMaskNeF32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break;
-		case IR::Opcode::CompareGtF32:
-		case IR::Opcode::CompareMaskGtF32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break;
-		case IR::Opcode::CompareGeF32:
-		case IR::Opcode::CompareMaskGeF32: opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; break;
-		case IR::Opcode::CompareLtF32:
-		case IR::Opcode::CompareMaskLtF32: opcode = IR::ValueOpcode::FPOrdLessThan32; break;
-		case IR::Opcode::CompareLeF32:
-		case IR::Opcode::CompareMaskLeF32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break;
-		case IR::Opcode::CompareUnordEqF32:
-		case IR::Opcode::CompareMaskUnordEqF32: opcode = IR::ValueOpcode::FPUnordEqual32; break;
-		case IR::Opcode::CompareUnordNeF32:
-		case IR::Opcode::CompareMaskUnordNeF32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break;
-		case IR::Opcode::CompareUnordGtF32:
-		case IR::Opcode::CompareMaskUnordGtF32:
-			opcode = IR::ValueOpcode::FPUnordGreaterThan32;
-			break;
-		case IR::Opcode::CompareUnordGeF32:
-		case IR::Opcode::CompareMaskUnordGeF32:
-			opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32;
-			break;
-		case IR::Opcode::CompareUnordLtF32:
-		case IR::Opcode::CompareMaskUnordLtF32: opcode = IR::ValueOpcode::FPUnordLessThan32; break;
-		case IR::Opcode::CompareUnordLeF32:
-		case IR::Opcode::CompareMaskUnordLeF32:
-			opcode = IR::ValueOpcode::FPUnordLessThanEqual32;
-			break;
-		case IR::Opcode::CompareEqF16:
-		case IR::Opcode::CompareMaskEqF16:
-			opcode = IR::ValueOpcode::FPOrdEqual32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareNeF16:
-		case IR::Opcode::CompareMaskNeF16:
-			opcode = IR::ValueOpcode::FPOrdNotEqual32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareGtF16:
-		case IR::Opcode::CompareMaskGtF16:
-			opcode = IR::ValueOpcode::FPOrdGreaterThan32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareGeF16:
-		case IR::Opcode::CompareMaskGeF16:
-			opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareLtF16:
-		case IR::Opcode::CompareMaskLtF16:
-			opcode = IR::ValueOpcode::FPOrdLessThan32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareLeF16:
-		case IR::Opcode::CompareMaskLeF16:
-			opcode = IR::ValueOpcode::FPOrdLessThanEqual32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareUnordNeF16:
-		case IR::Opcode::CompareMaskUnordNeF16:
-			opcode = IR::ValueOpcode::FPUnordNotEqual32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareMaskUnordGeF16:
-			opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32;
-			half   = true;
-			break;
-		case IR::Opcode::CompareOrderedF32:
-		case IR::Opcode::CompareUnorderedF32: {
-			const auto lhs       = IR::F32(ReadOperand(inst.src[0], IR::Type::F32));
-			const auto rhs       = IR::F32(ReadOperand(inst.src[1], IR::Type::F32));
-			const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})),
-			                                    IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs})));
-			const auto result =
-			    inst.op == IR::Opcode::CompareUnorderedF32 ? unordered : ir.LogicalNot(unordered);
-			WriteCompareResult(inst.dst, result);
-			return true;
-		}
-		case IR::Opcode::CompareClassF32: {
-			const auto value = ReadOperand(inst.src[0], IR::Type::F32);
-			const auto mask  = ReadOperand(inst.src[1], IR::Type::U32);
-			WriteCompareResult(inst.dst,
-			                   IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})));
-			return true;
-		}
-		default: return false;
-	}
+void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                                    IR::Type type, bool scalar, bool cmpx) {
+	const auto lhs = ReadOperand(inst.src0, type);
+	const auto rhs = ReadOperand(inst.src1, type);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx);
+}
+
+void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                                      bool signed_value, bool cmpx) {
+	const auto lhs = ReadU16AsU32(inst.src0, signed_value);
+	const auto rhs = ReadU16AsU32(inst.src1, signed_value);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
+}
+
+void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                                  bool half, bool cmpx) {
 	const auto lhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src[0])) : ReadOperand(inst.src[0], IR::Type::F32);
+	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
 	const auto rhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src[1])) : ReadOperand(inst.src[1], IR::Type::F32);
-	WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs})));
-	return true;
+	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
 }
 
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered) {
+	const auto lhs       = IR::F32(ReadOperand(inst.src0, IR::Type::F32));
+	const auto rhs       = IR::F32(ReadOperand(inst.src1, IR::Type::F32));
+	const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})),
+	                                    IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs})));
+	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, false);
+}
+
+void Translator::V_CMP_CLASS_F32(const Decoder::Instruction& inst) {
+	const auto value = ReadOperand(inst.src0, IR::Type::F32);
+	const auto mask  = ReadOperand(inst.src1, IR::Type::U32);
+	EmitCompareResult(inst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})), false,
+	                  false);
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -1,290 +1,370 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
+namespace {
 
-bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
-	const auto zero = IR::U32(IR::Value(0u));
-	switch (inst.op) {
-		case IR::Opcode::SaveexecB32:
-		case IR::Opcode::SaveexecB64: {
-			const auto old = ir.GetExec();
-			const auto src = ReadMask(inst.src[0]);
-			IR::U1     result;
-			switch (inst.saveexec_mode) {
-				case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break;
-				case IR::SaveexecMode::Andn1:
-					result = ir.LogicalAnd(old, ir.LogicalNot(src));
-					break;
-				case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break;
-			}
-			if (inst.op == IR::Opcode::SaveexecB64) {
-				WriteMask64(inst.dst, old);
-			} else {
-				WriteMask(inst.dst, old);
-			}
-			const auto mask = BallotMask(result);
-			ir.SetExec(result);
-			ir.SetExecLo(mask[0]);
-			ir.SetExecHi(mask[1]);
-			ir.SetScc(result);
-			return true;
-		}
-		case IR::Opcode::ScalarAddCarryU32:
-		case IR::Opcode::IAddCarryU32: {
-			const auto lhs      = ReadU32(inst.src[0]);
-			const auto rhs      = ReadU32(inst.src[1]);
-			const auto carry_in = ConditionBit(inst.src[2]);
-			const auto add0     = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs});
-			const auto partial  = ir.CompositeExtract(add0, 0);
-			const auto carry0   = ir.CompositeExtract(add0, 1);
-			const auto add1     = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in});
-			const auto result   = ir.CompositeExtract(add1, 0);
-			const auto carry1   = ir.CompositeExtract(add1, 1);
-			const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), IR::U32(IR::Value(0u)));
-			WriteOperand(inst.dst, result);
-			if (inst.op == IR::Opcode::ScalarAddCarryU32) {
-				ir.SetScc(carry);
-			} else {
-				WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry));
-			}
-			return true;
-		}
-		case IR::Opcode::ScalarSubBorrowU32:
-		case IR::Opcode::ISubBorrowU32: {
-			const auto lhs    = ReadU32(inst.src[0]);
-			const auto rhs    = ReadU32(inst.src[1]);
-			const auto result = ir.ISub(lhs, rhs);
-			const auto borrow = ir.UGreaterThan(rhs, lhs);
-			WriteOperand(inst.dst, result);
-			if (inst.op == IR::Opcode::ScalarSubBorrowU32) {
-				ir.SetScc(borrow);
-			} else {
-				WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
-			}
-			return true;
-		}
-		case IR::Opcode::ScalarSubBorrowCarryU32:
-		case IR::Opcode::ISubBorrowCarryU32: {
-			const auto lhs       = ReadU32(inst.src[0]);
-			const auto rhs       = ReadU32(inst.src[1]);
-			const auto borrow_in = ConditionBit(inst.src[2]);
-			const auto partial   = ir.ISub(lhs, rhs);
-			const auto result    = ir.ISub(partial, borrow_in);
-			const auto borrow0   = ir.UGreaterThan(rhs, lhs);
-			const auto borrow1   = ir.UGreaterThan(borrow_in, partial);
-			WriteOperand(inst.dst, result);
-			const auto borrow = ir.LogicalOr(borrow0, borrow1);
-			if (inst.op == IR::Opcode::ScalarSubBorrowCarryU32) {
-				ir.SetScc(borrow);
-			} else {
-				WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
-			}
-			return true;
-		}
-		case IR::Opcode::ScalarSignedAddOverflowI32:
-		case IR::Opcode::ScalarSignedSubOverflowI32: {
-			const auto lhs      = ReadU32(inst.src[0]);
-			const auto rhs      = ReadU32(inst.src[1]);
-			const bool subtract = inst.op == IR::Opcode::ScalarSignedSubOverflowI32;
-			const auto result   = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs);
-			const auto shift    = IR::U32(IR::Value(31u));
-			const auto lhs_sign = ir.ShiftRightLogical(lhs, shift);
-			const auto rhs_sign = ir.ShiftRightLogical(rhs, shift);
-			const auto out_sign = ir.ShiftRightLogical(result, shift);
-			const auto inputs =
-			    subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign);
-			const auto changed = ir.INotEqual(lhs_sign, out_sign);
-			WriteOperand(inst.dst, result);
-			ir.SetScc(ir.LogicalAnd(inputs, changed));
-			return true;
-		}
-		case IR::Opcode::ScalarShiftLeftAddCarryU32: {
-			const auto lhs           = ReadU32(inst.src[0]);
-			const auto shift         = ReadU32(inst.src[1]);
-			const auto rhs           = ReadU32(inst.src[2]);
-			const auto shifted       = ir.ShiftLeftLogical(lhs, shift);
-			const auto result        = ir.IAdd(shifted, rhs);
-			const auto add_carry     = ir.ULessThan(result, shifted);
-			const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift);
-			const auto shifted_out   = ir.ShiftRightLogical(lhs, inverse_shift);
-			const auto shift_carry   = ir.INotEqual(shifted_out, zero);
-			WriteOperand(inst.dst, result);
-			ir.SetScc(ir.LogicalOr(add_carry, shift_carry));
-			return true;
-		}
+bool IsExecOrVcc(const Decoder::Operand& operand) {
+	switch (operand.kind) {
+		case Decoder::OperandKind::ExecLo:
+		case Decoder::OperandKind::ExecHi:
+		case Decoder::OperandKind::VccLo:
+		case Decoder::OperandKind::VccHi: return true;
 		default: return false;
 	}
 }
 
-bool Translator::TranslateControlOperation(const IR::Instruction& inst) {
-	switch (inst.op) {
-		case IR::Opcode::ControlNop: ir.Emit(IR::ValueOpcode::ControlNop); return true;
-		case IR::Opcode::Waitcnt: ir.Emit(IR::ValueOpcode::Waitcnt); return true;
-		case IR::Opcode::Barrier: ir.Emit(IR::ValueOpcode::Barrier); return true;
-		case IR::Opcode::Sendmsg: ir.Emit(IR::ValueOpcode::Sendmsg); return true;
-		case IR::Opcode::TtraceData: ir.Emit(IR::ValueOpcode::TtraceData); return true;
-		case IR::Opcode::InstPrefetch: ir.Emit(IR::ValueOpcode::InstPrefetch); return true;
-		default: return false;
+Decoder::Operand ConditionOperand(Decoder::OperandKind kind) {
+	Decoder::Operand operand;
+	operand.kind = kind;
+	return operand;
+}
+
+} // namespace
+
+void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation,
+                            bool negate_exec, bool negate_source, bool write_64) {
+	const auto old    = ir.GetExec();
+	const auto src    = ReadMask(inst.src0);
+	const auto lhs    = negate_exec ? ir.LogicalNot(old) : old;
+	const auto rhs    = negate_source ? ir.LogicalNot(src) : src;
+	const auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
+	if (write_64) {
+		WriteMask64(inst.dst, old);
+	} else {
+		WriteMask(inst.dst, old);
+	}
+	const auto mask = BallotMask(result);
+	ir.SetExec(result);
+	ir.SetExecLo(mask[0]);
+	ir.SetExecHi(mask[1]);
+	ir.SetScc(result);
+}
+
+void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in) {
+	const auto zero     = IR::U32(IR::Value(0u));
+	const auto lhs      = ReadU32(inst.src0);
+	const auto rhs      = ReadU32(inst.src1);
+	const auto carry_in = !use_carry_in ? zero
+	                      : vector      ? ConditionBit(inst.src2)
+	                                    : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc));
+	const auto add0     = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs});
+	const auto partial  = ir.CompositeExtract(add0, 0);
+	const auto carry0   = ir.CompositeExtract(add0, 1);
+	const auto add1     = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in});
+	const auto result   = ir.CompositeExtract(add1, 0);
+	const auto carry1   = ir.CompositeExtract(add1, 1);
+	const auto carry    = ir.INotEqual(ir.BitwiseOr(carry0, carry1), zero);
+	WriteOperand(DestinationOperand(inst), result);
+	if (vector) {
+		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry));
+	} else {
+		ir.SetScc(carry);
 	}
 }
 
-bool Translator::TranslateMove(const IR::Instruction& inst) {
-	switch (inst.op) {
-		case IR::Opcode::MoveU32:
-			if (inst.src[0].kind == IR::OperandKind::Register &&
-			    (inst.src[0].reg.file == IR::RegisterFile::Exec ||
-			     inst.src[0].reg.file == IR::RegisterFile::Vcc) &&
-			    inst.dst.kind == IR::OperandKind::Register &&
-			    (inst.dst.reg.file == IR::RegisterFile::Exec ||
-			     inst.dst.reg.file == IR::RegisterFile::Vcc)) {
-				WriteMask(inst.dst, ReadMask(inst.src[0]));
+void Translator::SUB_U32(const Decoder::Instruction& inst, bool vector, bool reverse) {
+	const auto lhs    = ReadU32(reverse ? inst.src1 : inst.src0);
+	const auto rhs    = ReadU32(reverse ? inst.src0 : inst.src1);
+	const auto result = ir.ISub(lhs, rhs);
+	const auto borrow = ir.UGreaterThan(rhs, lhs);
+	WriteOperand(DestinationOperand(inst), result);
+	if (vector) {
+		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
+	} else {
+		ir.SetScc(borrow);
+	}
+}
+
+void Translator::SUBB_U32(const Decoder::Instruction& inst, bool vector) {
+	const auto lhs       = ReadU32(vector ? inst.src1 : inst.src0);
+	const auto rhs       = ReadU32(vector ? inst.src0 : inst.src1);
+	const auto borrow_in = vector ? ConditionBit(inst.src2)
+	                              : ConditionBit(ConditionOperand(Decoder::OperandKind::Scc));
+	const auto partial   = ir.ISub(lhs, rhs);
+	const auto result    = ir.ISub(partial, borrow_in);
+	const auto borrow0   = ir.UGreaterThan(rhs, lhs);
+	const auto borrow1   = ir.UGreaterThan(borrow_in, partial);
+	const auto borrow    = ir.LogicalOr(borrow0, borrow1);
+	WriteOperand(DestinationOperand(inst), result);
+	if (vector) {
+		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
+	} else {
+		ir.SetScc(borrow);
+	}
+}
+
+void Translator::S_ADD_SUB_I32(const Decoder::Instruction& inst, bool subtract) {
+	const auto lhs      = ReadU32(inst.src0);
+	const auto rhs      = ReadU32(inst.src1);
+	const auto result   = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs);
+	const auto shift    = IR::U32(IR::Value(31u));
+	const auto lhs_sign = ir.ShiftRightLogical(lhs, shift);
+	const auto rhs_sign = ir.ShiftRightLogical(rhs, shift);
+	const auto out_sign = ir.ShiftRightLogical(result, shift);
+	const auto inputs = subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign);
+	const auto changed = ir.INotEqual(lhs_sign, out_sign);
+	WriteOperand(DestinationOperand(inst), result);
+	ir.SetScc(ir.LogicalAnd(inputs, changed));
+}
+
+void Translator::S_LSHL_ADD_U32(const Decoder::Instruction& inst, uint32_t shift_amount) {
+	const auto lhs           = ReadU32(inst.src0);
+	const auto shift         = IR::U32(IR::Value(shift_amount));
+	const auto rhs           = ReadU32(inst.src1);
+	const auto shifted       = ir.ShiftLeftLogical(lhs, shift);
+	const auto result        = ir.IAdd(shifted, rhs);
+	const auto add_carry     = ir.ULessThan(result, shifted);
+	const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift);
+	const auto shifted_out   = ir.ShiftRightLogical(lhs, inverse_shift);
+	const auto shift_carry   = ir.INotEqual(shifted_out, IR::U32(IR::Value(0u)));
+	WriteOperand(DestinationOperand(inst), result);
+	ir.SetScc(ir.LogicalOr(add_carry, shift_carry));
+}
+
+void Translator::ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode,
+                                IR::ValueOpcode compare_opcode) {
+	const auto lhs = ReadU32(inst.src0);
+	const auto rhs = ReadU32(inst.src1);
+	WriteOperand(DestinationOperand(inst), ir.Emit(value_opcode, {lhs, rhs}));
+	ir.SetScc(IR::U1(ir.Emit(compare_opcode, {lhs, rhs})));
+}
+
+void Translator::EmitControlNop() {
+	ir.Emit(IR::ValueOpcode::ControlNop);
+}
+
+void Translator::EmitWaitcnt() {
+	ir.Emit(IR::ValueOpcode::Waitcnt);
+}
+
+void Translator::S_BARRIER() {
+	ir.Emit(IR::ValueOpcode::Barrier);
+}
+
+void Translator::S_SENDMSG() {
+	ir.Emit(IR::ValueOpcode::Sendmsg);
+}
+
+void Translator::S_TTRACEDATA() {
+	ir.Emit(IR::ValueOpcode::TtraceData);
+}
+
+void Translator::S_INST_PREFETCH() {
+	ir.Emit(IR::ValueOpcode::InstPrefetch);
+}
+
+bool Translator::S_GETPC_B64(const Decoder::Instruction& inst, std::string* error) {
+	const auto base    = IR::U64(ir.Emit(IR::ValueOpcode::GetShaderBase));
+	const auto address = IR::U64(
+	    ir.Emit(IR::ValueOpcode::IAdd64, {base, IR::Value(static_cast<uint64_t>(inst.pc) + 4u)}));
+	if (inst.dst.kind == Decoder::OperandKind::Null) {
+		return true;
+	}
+	auto high = PlainOperand(inst.dst);
+	switch (inst.dst.kind) {
+		case Decoder::OperandKind::Sgpr:
+			if (inst.dst.reg < 105u) {
+				high.reg++;
+			} else if (inst.dst.reg == 105u) {
+				high.kind = Decoder::OperandKind::VccLo;
+				high.reg  = 0u;
 			} else {
-				WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U32));
-			}
-			return true;
-		case IR::Opcode::MoveF32Bits:
-			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
-			return true;
-		case IR::Opcode::MoveU64: {
-			const auto is_exec_or_vcc = [](const IR::Operand& operand) {
-				return operand.kind == IR::OperandKind::Register &&
-				       (operand.reg.file == IR::RegisterFile::Exec ||
-				        operand.reg.file == IR::RegisterFile::Vcc);
-			};
-			if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) {
-				WriteMask64(inst.dst, ReadMask(inst.src[0]));
-				return true;
-			}
-			const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register &&
-			                         inst.src[0].kind == IR::OperandKind::Register &&
-			                         inst.dst.reg.file == IR::RegisterFile::Scalar &&
-			                         inst.src[0].reg.file == IR::RegisterFile::Scalar;
-			IR::U1     source_mask;
-			IR::U1     source_mask_valid;
-			if (scalar_copy) {
-				const auto src    = static_cast<IR::ScalarReg>(inst.src[0].reg.index);
-				source_mask       = ir.GetThreadBitScalarReg(src);
-				source_mask_valid = ir.GetScalarMaskTag(src);
-			}
-			WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0]));
-			if (scalar_copy) {
-				ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index),
-				                         source_mask);
-				ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg.index),
-				                    source_mask_valid);
-			}
-			return true;
-		}
-		case IR::Opcode::WqmB64: {
-			const auto is_exec_or_vcc = [](const IR::Operand& operand) {
-				return operand.kind == IR::OperandKind::Register &&
-				       (operand.reg.file == IR::RegisterFile::Exec ||
-				        operand.reg.file == IR::RegisterFile::Vcc);
-			};
-			if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) {
-				const auto mask_valid = ReadMaskValid(inst.src[0]);
-				const auto invocation_result =
-				    IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
-				const auto result = IR::U64(
-				    ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
-				WriteOperand(inst.dst, result);
-				if (inst.dst.kind == IR::OperandKind::Register &&
-				    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-					const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
-					ir.SetThreadBitScalarReg(dst, invocation_result);
-					ir.SetScalarMaskTag(dst, mask_valid);
-					const auto raw_nonzero = IR::U1(
-					    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
-					ir.SetScc(IR::U1(ir.Emit(
-					    IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
-				} else {
-					ir.SetScc(IR::U1(
-					    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+				if (error != nullptr) {
+					*error = "S_GETPC_B64 destination does not name a valid scalar pair";
 				}
-				return true;
+				return false;
 			}
-			const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
-			WriteMask64(inst.dst, result);
-			ir.SetScc(result);
-			return true;
-		}
-		default: return false;
+			break;
+		case Decoder::OperandKind::VccLo: high.kind = Decoder::OperandKind::VccHi; break;
+		case Decoder::OperandKind::M0: high.kind = Decoder::OperandKind::Null; break;
+		case Decoder::OperandKind::ExecLo: high.kind = Decoder::OperandKind::ExecHi; break;
+		default:
+			if (error != nullptr) {
+				*error = "S_GETPC_B64 destination does not name a valid scalar pair";
+			}
+			return false;
+	}
+	const auto words = ExtractU64(address);
+	WriteOperand(inst.dst, words[0]);
+	WriteOperand(high, words[1]);
+	return true;
+}
+
+void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
+	const auto result = ir.Select(ir.GetScc(), ReadU32(inst.src0), ReadU32(inst.src1));
+	WriteOperand(DestinationOperand(inst), result);
+}
+
+void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) {
+	const auto condition     = ir.GetScc();
+	const auto lhs           = ReadU32Pair(inst.src0);
+	const auto rhs           = ReadU32Pair(inst.src1);
+	const auto selected_mask = IR::U1(
+	    ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMask(inst.src0), ReadMask(inst.src1)}));
+	const auto selected_mask_valid =
+	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
+	                   {condition, ReadMaskValid(inst.src0), ReadMaskValid(inst.src1)}));
+	if (IsExecOrVcc(inst.dst)) {
+		WriteMask64(inst.dst, selected_mask);
+		return;
+	}
+	WriteU32Pair(inst.dst,
+	             {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])});
+	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
+		ir.SetThreadBitScalarReg(dst, selected_mask);
+		ir.SetScalarMaskTag(dst, selected_mask_valid);
 	}
 }
 
-bool Translator::TranslateLaneOperation(const IR::Instruction& inst) {
+void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) {
+	if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) {
+		WriteMask(inst.dst, ReadMask(inst.src0));
+	} else if (apply_float_modifiers && (inst.src0.negate || inst.src0.absolute)) {
+		WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::F32));
+	} else {
+		WriteOperand(DestinationOperand(inst), ReadOperand(inst.src0, IR::Type::U32));
+	}
+}
+
+void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
+	if (IsExecOrVcc(inst.dst) || IsExecOrVcc(inst.src0)) {
+		WriteMask64(inst.dst, ReadMask(inst.src0));
+		return;
+	}
+	const bool scalar_copy =
+	    inst.dst.kind == Decoder::OperandKind::Sgpr && inst.src0.kind == Decoder::OperandKind::Sgpr;
+	IR::U1 source_mask;
+	IR::U1 source_mask_valid;
+	if (scalar_copy) {
+		source_mask       = ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src0.reg));
+		source_mask_valid = ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(inst.src0.reg));
+	}
+	WriteU32Pair(inst.dst, ReadU32Pair(inst.src0));
+	if (scalar_copy) {
+		ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg), source_mask);
+		ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg), source_mask_valid);
+	}
+}
+
+void Translator::S_WQM_B64(const Decoder::Instruction& inst) {
+	if (!IsExecOrVcc(inst.dst) && !IsExecOrVcc(inst.src0)) {
+		const auto mask_valid = ReadMaskValid(inst.src0);
+		const auto invocation_result =
+		    IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
+		const auto result =
+		    IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)}));
+		WriteOperand(DestinationOperand(inst), result);
+		if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+			const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
+			ir.SetThreadBitScalarReg(dst, invocation_result);
+			ir.SetScalarMaskTag(dst, mask_valid);
+			const auto raw_nonzero =
+			    IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
+			ir.SetScc(IR::U1(
+			    ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
+		} else {
+			ir.SetScc(
+			    IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+		}
+		return;
+	}
+	const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src0)}));
+	WriteMask64(inst.dst, result);
+	ir.SetScc(result);
+}
+
+bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error) {
+	if (inst.dst.kind != Decoder::OperandKind::Vgpr ||
+	    inst.src0.kind != Decoder::OperandKind::Vgpr) {
+		if (error != nullptr) {
+			*error = "V_MOVRELS_B32 requires VGPR source and destination";
+		}
+		return false;
+	}
+	if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp ||
+	    inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute ||
+	    inst.src0.dpp) {
+		if (error != nullptr) {
+			*error = "V_MOVRELS_B32 modifiers are not implemented";
+		}
+		return false;
+	}
+	const auto base     = inst.src0.reg;
+	const auto m0       = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)),
+	                                    IR::U32(IR::Value(0xffu)));
+	auto       selected = ir.GetVectorReg(static_cast<IR::VectorReg>(base));
+	for (uint32_t index = base + 1u; index < current_vector_limit; index++) {
+		const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
+		selected = ir.Select(match, ir.GetVectorReg(static_cast<IR::VectorReg>(index)), selected);
+	}
+	WriteOperand(DestinationOperand(inst), selected);
+	return true;
+}
+
+bool Translator::V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* error) {
+	if (inst.dst.kind != Decoder::OperandKind::Vgpr) {
+		if (error != nullptr) {
+			*error = "V_MOVRELD_B32 requires VGPR destination";
+		}
+		return false;
+	}
+	if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp ||
+	    inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute ||
+	    inst.src0.dpp) {
+		if (error != nullptr) {
+			*error = "V_MOVRELD_B32 modifiers are not implemented";
+		}
+		return false;
+	}
+	const auto base  = inst.dst.reg;
+	const auto value = ReadU32(inst.src0);
+	const auto m0    = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)),
+	                                 IR::U32(IR::Value(0xffu)));
+	for (uint32_t index = base; index < current_vector_limit; index++) {
+		const auto reg   = static_cast<IR::VectorReg>(index);
+		const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
+		const auto write = ir.LogicalAnd(ir.GetExec(), match);
+		ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg)));
+	}
+	return true;
+}
+
+void Translator::V_READFIRSTLANE_B32(const Decoder::Instruction& inst) {
+	const auto result = ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src0), ir.GetExec()});
+	WriteOperand(DestinationOperand(inst), result);
+}
+
+void Translator::V_READLANE_B32(const Decoder::Instruction& inst) {
 	const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u));
-	switch (inst.op) {
-		case IR::Opcode::MoveRelSourceU32: {
-			EXIT_IF(inst.src[0].kind != IR::OperandKind::Register ||
-			        inst.src[0].reg.file != IR::RegisterFile::Vector);
-			const auto base     = inst.src[0].reg.index;
-			const auto m0       = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu)));
-			auto       selected = ir.GetVectorReg(static_cast<IR::VectorReg>(base));
-			for (uint32_t index = base + 1u; index < current_vector_limit; index++) {
-				const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
-				selected =
-				    ir.Select(match, ir.GetVectorReg(static_cast<IR::VectorReg>(index)), selected);
-			}
-			WriteOperand(inst.dst, selected);
-			return true;
-		}
-		case IR::Opcode::MoveRelDestU32: {
-			EXIT_IF(inst.dst.kind != IR::OperandKind::Register ||
-			        inst.dst.reg.file != IR::RegisterFile::Vector);
-			const auto base  = inst.dst.reg.index;
-			const auto value = ReadU32(inst.src[0]);
-			const auto m0    = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu)));
-			for (uint32_t index = base; index < current_vector_limit; index++) {
-				const auto reg   = static_cast<IR::VectorReg>(index);
-				const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
-				const auto write = ir.LogicalAnd(ir.GetExec(), match);
-				ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg)));
-			}
-			return true;
-		}
-		case IR::Opcode::ReadFirstLaneU32: {
-			const auto result =
-			    ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src[0]), ir.GetExec()});
-			WriteOperand(inst.dst, result);
-			return true;
-		}
-		case IR::Opcode::ReadLaneU32: {
-			const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask);
-			WriteOperand(inst.dst,
-			             ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src[0]), lane}));
-			return true;
-		}
-		case IR::Opcode::WriteLaneU32: {
-			EXIT_IF(inst.dst.kind != IR::OperandKind::Register ||
-			        inst.dst.reg.file != IR::RegisterFile::Vector);
-			const auto reg    = static_cast<IR::VectorReg>(inst.dst.reg.index);
-			const auto lane   = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask);
-			const auto result = IR::U32(ir.Emit(
-			    IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src[0]), lane}));
-			// V_WRITELANE_B32 ignores EXEC; bypass the ordinary VGPR destination path.
-			ir.SetVectorReg(reg, result);
-			return true;
-		}
-		case IR::Opcode::Permlane16B32:
-		case IR::Opcode::Permlanex16B32: {
-			const IR::PermlaneFlags flags {
-			    .x16            = inst.op == IR::Opcode::Permlanex16B32,
-			    .fetch_inactive = inst.dst.op_sel,
-			    .bound_control  = inst.dst.op_sel_hi,
-			};
-			const auto result = ir.Emit(
-			    IR::ValueOpcode::Permlane16U32,
-			    {ReadU32(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]), ir.GetExec()},
-			    flags);
-			auto dst      = inst.dst;
-			dst.op_sel    = false;
-			dst.op_sel_hi = false;
-			WriteOperand(dst, result);
-			return true;
-		}
-		default: return false;
-	}
+	const auto lane      = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask);
+	WriteOperand(DestinationOperand(inst),
+	             ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src0), lane}));
 }
 
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+void Translator::V_WRITELANE_B32(const Decoder::Instruction& inst) {
+	EXIT_IF(inst.dst.kind != Decoder::OperandKind::Vgpr);
+	const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u));
+	const auto reg       = static_cast<IR::VectorReg>(inst.dst.reg);
+	const auto lane      = ir.BitwiseAnd(ReadU32(inst.src1), lane_mask);
+	const auto result    = IR::U32(
+	    ir.Emit(IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src0), lane}));
+	ir.SetVectorReg(reg, result);
+}
+
+void Translator::V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16) {
+	const IR::PermlaneFlags flags {
+	    .x16            = x16,
+	    .fetch_inactive = inst.dst.op_sel,
+	    .bound_control  = inst.dst.op_sel_hi,
+	};
+	const auto result =
+	    ir.Emit(IR::ValueOpcode::Permlane16U32,
+	            {ReadU32(inst.src0), ReadU32(inst.src1), ReadU32(inst.src2), ir.GetExec()}, flags);
+	auto dst      = DestinationOperand(inst);
+	dst.op_sel    = false;
+	dst.op_sel_hi = false;
+	WriteOperand(dst, result);
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Convert.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 IR::F32 Translator::SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value) {
 	return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
@@ -38,132 +38,136 @@ IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, f
 	return ir.Select(nan, IR::U32(IR::Value(0u)), clamped);
 }
 
-bool Translator::TranslateConversion(const IR::Instruction& inst) {
-	const auto f32 = [&](uint32_t index) {
-		return IR::F32(ReadOperand(inst.src[index], IR::Type::F32));
-	};
-	const auto u32         = [&](uint32_t index) { return ReadU32(inst.src[index]); };
-	const auto convert_u32 = [&](IR::F32 value) {
-		return ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu);
-	};
-	const auto convert_i32 = [&](IR::F32 value) {
-		return ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f,
-		                                0x80000000u, 0x7fffffffu);
-	};
+void Translator::V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index) {
+	const auto source = ReadU32(SourceAt(inst, 0));
+	const auto byte =
+	    ir.BitwiseAnd(ir.ShiftRightLogical(source, IR::U32(IR::Value(byte_index * 8u))),
+	                  IR::U32(IR::Value(0xffu)));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte}));
+}
 
-	switch (inst.op) {
-		case IR::Opcode::ConvertByteU32ToF32: {
-			const auto index =
-			    inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm & 3u : 0u;
-			const auto byte =
-			    ir.BitwiseAnd(ir.ShiftRightLogical(u32(0), IR::U32(IR::Value(index * 8u))),
-			                  IR::U32(IR::Value(0xffu)));
-			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte}));
-			return true;
-		}
-		case IR::Opcode::ConvertU32ToF32:
-			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)}));
-			return true;
-		case IR::Opcode::ConvertI32ToF32:
-			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)}));
-			return true;
-		case IR::Opcode::ConvertF32ToU32: WriteOperand(inst.dst, convert_u32(f32(0))); return true;
-		case IR::Opcode::ConvertF32ToI32: WriteOperand(inst.dst, convert_i32(f32(0))); return true;
-		case IR::Opcode::ConvertF32ToF16: WriteF16(inst.dst, f32(0)); return true;
-		case IR::Opcode::ConvertF16ToF32:
-			WriteOperand(inst.dst, ReadF16AsF32(inst.src[0]));
-			return true;
-		case IR::Opcode::ConvertU16ToF16: {
-			const auto value = IR::F32(
-			    ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src[0], false)}));
-			WriteF16(inst.dst, value);
-			return true;
-		}
-		case IR::Opcode::ConvertI16ToF16: {
-			const auto value =
-			    IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src[0], true)}));
-			WriteF16(inst.dst, value);
-			return true;
-		}
-		case IR::Opcode::ConvertF16ToU16: {
-			const auto converted =
-			    ConvertF32ToU32Saturated(ReadF16AsF32(inst.src[0]), 65536.0f, 65535.0f, 0xffffu);
-			WriteU16(inst.dst, converted);
-			return true;
-		}
-		case IR::Opcode::ConvertF16ToI16: {
-			const auto converted = ConvertF32ToI32Saturated(
-			    ReadF16AsF32(inst.src[0]), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu);
-			WriteU16(inst.dst, ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu))));
-			return true;
-		}
-		case IR::Opcode::ConvertRoundPlusInfF32ToI32: {
-			const auto biased =
-			    IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {f32(0), IR::Value::F32(0.5f)}));
-			const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased}));
-			WriteOperand(inst.dst, convert_i32(rounded));
-			return true;
-		}
-		case IR::Opcode::ConvertFloorF32ToI32: {
-			const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {f32(0)}));
-			WriteOperand(inst.dst, convert_i32(rounded));
-			return true;
-		}
-		case IR::Opcode::FrexpExpI32F32: {
-			const auto bits     = ir.BitCastU32(f32(0));
-			const auto exponent = IR::U32(
-			    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
-			const auto mantissa  = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu)));
-			const auto normal    = ir.ISub(exponent, IR::U32(IR::Value(126u)));
-			const auto msb       = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa}));
-			const auto subnormal = ir.ISub(msb, IR::U32(IR::Value(148u)));
-			const auto denormal  = ir.Select(ir.INotEqual(mantissa, IR::U32(IR::Value(0u))),
-			                                 subnormal, IR::U32(IR::Value(0u)));
-			const auto finite    = ir.Select(
-			    ir.INotEqual(exponent, IR::U32(IR::Value(0xffu))),
-			    ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, denormal),
-			    IR::U32(IR::Value(0u)));
-			WriteOperand(inst.dst, finite);
-			return true;
-		}
-		case IR::Opcode::ConvertI4ToOffsetF32: {
-			const auto nibble = IR::U32(
-			    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {u32(0), IR::Value(0u), IR::Value(4u)}));
-			const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble}));
-			WriteOperand(inst.dst,
-			             ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)}));
-			return true;
-		}
-		case IR::Opcode::PackF32ToF16Rtz:
-			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)}));
-			return true;
-		case IR::Opcode::PackSnorm2x16F32:
-		case IR::Opcode::PackUnorm2x16F32: {
-			const auto pair   = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {f32(0), f32(1)});
-			const auto opcode = inst.op == IR::Opcode::PackSnorm2x16F32
-			                        ? IR::ValueOpcode::PackSnorm2x16
-			                        : IR::ValueOpcode::PackUnorm2x16;
-			WriteOperand(inst.dst, ir.Emit(opcode, {pair}));
-			return true;
-		}
-		case IR::Opcode::PackU8F32: {
-			const auto byte  = ConvertF32ToU32Saturated(f32(0), 255.0f, 255.0f, 255u);
-			const auto index = ir.BitwiseAnd(u32(1), IR::U32(IR::Value(3u)));
-			const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u)));
-			const auto mask  = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift);
-			const auto base  = ir.BitwiseAnd(u32(2), ir.BitwiseNot(mask));
-			WriteOperand(inst.dst, ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift)));
-			return true;
-		}
-		case IR::Opcode::PackB32F16: {
-			const auto low = ReadF16LaneBits(inst.src[0], false);
-			const auto high =
-			    ir.ShiftLeftLogical(ReadF16LaneBits(inst.src[1], false), IR::U32(IR::Value(16u)));
-			WriteOperand(inst.dst, ir.BitwiseOr(low, high));
-			return true;
-		}
-		default: return false;
+void Translator::V_CVT_F32_U32(const Decoder::Instruction& inst) {
+	WriteOperand(DestinationOperand(inst),
+	             ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU32(SourceAt(inst, 0))}));
+}
+
+void Translator::V_CVT_F32_I32(const Decoder::Instruction& inst) {
+	WriteOperand(DestinationOperand(inst),
+	             ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU32(SourceAt(inst, 0))}));
+}
+
+void Translator::V_CVT_U32_F32(const Decoder::Instruction& inst) {
+	const auto value = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
+	WriteOperand(DestinationOperand(inst),
+	             ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu));
+}
+
+void Translator::V_CVT_I32_F32(const Decoder::Instruction& inst) {
+	const auto value = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
+	WriteOperand(DestinationOperand(inst),
+	             ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f,
+	                                      0x80000000u, 0x7fffffffu));
+}
+
+void Translator::V_CVT_F16_F32(const Decoder::Instruction& inst) {
+	WriteF16(DestinationOperand(inst), IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32)));
+}
+
+void Translator::V_CVT_F32_F16(const Decoder::Instruction& inst) {
+	WriteOperand(DestinationOperand(inst), ReadF16AsF32(inst.src0));
+}
+
+void Translator::V_CVT_F16_16(const Decoder::Instruction& inst, bool signed_value) {
+	const auto value = IR::F32(
+	    ir.Emit(signed_value ? IR::ValueOpcode::ConvertF32S32 : IR::ValueOpcode::ConvertF32U32,
+	            {ReadU16AsU32(inst.src0, signed_value)}));
+	WriteF16(DestinationOperand(inst), value);
+}
+
+void Translator::V_CVT_16_F16(const Decoder::Instruction& inst, bool signed_value) {
+	const auto value = ReadF16AsF32(inst.src0);
+	if (signed_value) {
+		const auto converted =
+		    ConvertF32ToI32Saturated(value, -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu);
+		WriteU16(DestinationOperand(inst), ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu))));
+		return;
 	}
+	WriteU16(DestinationOperand(inst),
+	         ConvertF32ToU32Saturated(value, 65536.0f, 65535.0f, 0xffffu));
+}
+
+void Translator::V_CVT_RPI_I32_F32(const Decoder::Instruction& inst) {
+	const auto source  = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
+	const auto biased  = IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {source, IR::Value::F32(0.5f)}));
+	const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased}));
+	WriteOperand(DestinationOperand(inst),
+	             ConvertF32ToI32Saturated(rounded, -2147483648.0f, 2147483648.0f, 2147483520.0f,
+	                                      0x80000000u, 0x7fffffffu));
+}
+
+void Translator::V_CVT_FLR_I32_F32(const Decoder::Instruction& inst) {
+	const auto source  = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
+	const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {source}));
+	WriteOperand(DestinationOperand(inst),
+	             ConvertF32ToI32Saturated(rounded, -2147483648.0f, 2147483648.0f, 2147483520.0f,
+	                                      0x80000000u, 0x7fffffffu));
+}
+
+void Translator::V_FREXP_EXP_I32_F32(const Decoder::Instruction& inst) {
+	const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
+	const auto bits   = ir.BitCastU32(source);
+	const auto exponent =
+	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
+	const auto mantissa  = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu)));
+	const auto normal    = ir.ISub(exponent, IR::U32(IR::Value(126u)));
+	const auto msb       = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa}));
+	const auto subnormal = ir.ISub(msb, IR::U32(IR::Value(148u)));
+	const auto denormal  = ir.Select(ir.INotEqual(mantissa, IR::U32(IR::Value(0u))), subnormal,
+	                                 IR::U32(IR::Value(0u)));
+	const auto finite =
+	    ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0xffu))),
+	              ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, denormal),
+	              IR::U32(IR::Value(0u)));
+	WriteOperand(DestinationOperand(inst), finite);
+}
+
+void Translator::V_CVT_OFF_F32_I4(const Decoder::Instruction& inst) {
+	const auto nibble =
+	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldSExtract,
+	                    {ReadU32(SourceAt(inst, 0)), IR::Value(0u), IR::Value(4u)}));
+	const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble}));
+	WriteOperand(DestinationOperand(inst),
+	             ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)}));
+}
+
+void Translator::V_CVT_PKRTZ_F16_F32(const Decoder::Instruction& inst) {
+	const auto lhs = ReadOperand(SourceAt(inst, 0), IR::Type::F32);
+	const auto rhs = ReadOperand(SourceAt(inst, 1), IR::Type::F32);
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {lhs, rhs}));
+}
+
+void Translator::V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+	const auto lhs  = ReadOperand(SourceAt(inst, 0), IR::Type::F32);
+	const auto rhs  = ReadOperand(SourceAt(inst, 1), IR::Type::F32);
+	const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {lhs, rhs});
+	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {pair}));
+}
+
+void Translator::V_CVT_PK_U8_F32(const Decoder::Instruction& inst) {
+	const auto source = IR::F32(ReadOperand(SourceAt(inst, 0), IR::Type::F32));
+	const auto byte   = ConvertF32ToU32Saturated(source, 255.0f, 255.0f, 255u);
+	const auto index  = ir.BitwiseAnd(ReadU32(SourceAt(inst, 1)), IR::U32(IR::Value(3u)));
+	const auto shift  = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u)));
+	const auto mask   = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift);
+	const auto base   = ir.BitwiseAnd(ReadU32(SourceAt(inst, 2)), ir.BitwiseNot(mask));
+	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift)));
+}
+
+void Translator::V_PACK_B32_F16(const Decoder::Instruction& inst) {
+	const auto low = ReadF16LaneBits(inst.src0, false);
+	const auto high =
+	    ir.ShiftLeftLogical(ReadF16LaneBits(inst.src1, false), IR::U32(IR::Value(16u)));
+	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(low, high));
 }
 
 IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
@@ -172,4 +176,4 @@ IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
 	                    ir.ShiftLeftLogical(ir.BitwiseAnd(high, mask), IR::U32(IR::Value(16u))));
 }
 
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -1,103 +1,53 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
-	const auto invocation_result = EvaluateU64Mask(inst);
-	const auto is_exec_or_vcc    = [](const IR::Operand& operand) {
-		return operand.kind == IR::OperandKind::Register &&
-		       (operand.reg.file == IR::RegisterFile::Exec ||
-		        operand.reg.file == IR::RegisterFile::Vcc);
-	};
-	if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) ||
-	    (inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) {
-		WriteMask64(inst.dst, invocation_result);
-		ir.SetScc(invocation_result);
-		return true;
-	}
+bool Translator::TranslateInstruction(const Decoder::Instruction& inst, std::string* error) {
+	current_opcode = inst.opcode;
+	current_pc     = inst.pc;
 
-	const auto lhs        = ReadU32Pair(inst.src[0]);
-	auto       mask_valid = ReadMaskValid(inst.src[0]);
-	if (inst.src_count > 1u) {
-		mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1]));
-	}
-	std::array<IR::U32, 2> result;
-	if (inst.op == IR::Opcode::BitwiseNotU64) {
-		result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
-	} else {
-		const auto rhs = ReadU32Pair(inst.src[1]);
-		for (uint32_t component = 0; component < 2u; component++) {
-			switch (inst.op) {
-				case IR::Opcode::BitwiseAndU64:
-					result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
-					break;
-				case IR::Opcode::BitwiseAndNotU64:
-					result[component] =
-					    ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
-					break;
-				case IR::Opcode::BitwiseOrU64:
-					result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
-					break;
-				case IR::Opcode::BitwiseOrNotU64:
-					result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
-					break;
-				case IR::Opcode::BitwiseXorU64:
-					result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
-					break;
-				case IR::Opcode::BitwiseNandU64:
-					result[component] =
-					    ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
-					break;
-				case IR::Opcode::BitwiseNorU64:
-					result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
-					break;
-				case IR::Opcode::BitwiseXnorU64:
-					result[component] =
-					    ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
-					break;
-				default: return false;
+	switch (inst.opcode) {
+		case Decoder::Opcode::UNKNOWN:
+		case Decoder::Opcode::COUNT:
+			if (error != nullptr) {
+				*error = "decoded opcode has no IR translation";
 			}
-		}
+			return false;
+		case Decoder::Opcode::UNSUPPORTED:
+			if (error != nullptr) {
+				*error = "unsupported decoded instruction: " + Decoder::InstructionToString(inst);
+			}
+			return false;
+		default: break;
 	}
-	WriteU32Pair(inst.dst, result);
-	if (inst.dst.kind == IR::OperandKind::Register &&
-	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
-		ir.SetThreadBitScalarReg(dst, invocation_result);
-		ir.SetScalarMaskTag(dst, mask_valid);
-		const auto raw_nonzero =
-		    ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
-		ir.SetScc(IR::U1(
-		    ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
-	} else {
-		ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
+
+	bool translated = false;
+	switch (inst.family) {
+		case Decoder::Family::SOP1:
+		case Decoder::Family::SOP2:
+		case Decoder::Family::SOPK:
+		case Decoder::Family::SOPC:
+		case Decoder::Family::SOPP: translated = EmitScalar(inst, error); break;
+		case Decoder::Family::VOP1:
+		case Decoder::Family::VOP2:
+		case Decoder::Family::VOP3:
+		case Decoder::Family::VOP3P:
+		case Decoder::Family::VOPC: translated = EmitVector(inst, error); break;
+		case Decoder::Family::SMEM:
+		case Decoder::Family::MUBUF:
+		case Decoder::Family::MTBUF:
+		case Decoder::Family::FLAT:
+		case Decoder::Family::DS:
+		case Decoder::Family::MIMG: translated = EmitMemory(inst, error); break;
+		case Decoder::Family::VINTRP: translated = EmitInterpolation(inst, error); break;
+		case Decoder::Family::EXP: translated = EXP(inst, error); break;
+		default: break;
 	}
-	return true;
+
+	if (!translated && error != nullptr && error->empty()) {
+		*error = "decoded opcode has no IR translation";
+	}
+	return translated;
 }
 
-bool Translator::TranslateInstruction(const IR::Instruction& inst) {
-	switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
-		case IR::LoweringClass::Control: return TranslateControlOperation(inst);
-		case IR::LoweringClass::Move: return TranslateMove(inst);
-		case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
-		case IR::LoweringClass::State: return TranslateStateOperation(inst);
-		case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst);
-		case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
-		case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
-		case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);
-		case IR::LoweringClass::FloatCompare: return TranslateFloatCompare(inst);
-		case IR::LoweringClass::Conversion: return TranslateConversion(inst);
-		case IR::LoweringClass::Integer16: return TranslateInteger16Operation(inst);
-		case IR::LoweringClass::PackedInteger16: return TranslatePackedInteger16(inst);
-		case IR::LoweringClass::PackedFloat16: return TranslatePackedFloat16(inst);
-		case IR::LoweringClass::Float16: return TranslateFloat16Operation(inst);
-		case IR::LoweringClass::Float: return TranslateFloatOperation(inst);
-		case IR::LoweringClass::U64Mask: return TranslateU64MaskOperation(inst);
-		case IR::LoweringClass::SimpleInteger: return TranslateSimpleInteger(inst);
-		case IR::LoweringClass::ComposedInteger: return TranslateComposedInteger(inst);
-		case IR::LoweringClass::ExtendedInteger: return TranslateExtendedInteger(inst);
-	}
-	return false;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -3,36 +3,29 @@
 #include <array>
 #include <utility>
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) {
-	IR::ValueOpcode opcode;
-	switch (inst.op) {
-		case IR::Opcode::PackedAddF16: opcode = IR::ValueOpcode::FPAdd32; break;
-		case IR::Opcode::PackedMulF16: opcode = IR::ValueOpcode::FPMul32; break;
-		case IR::Opcode::PackedMinF16: opcode = IR::ValueOpcode::FPMin32; break;
-		case IR::Opcode::PackedMaxF16: opcode = IR::ValueOpcode::FPMax32; break;
-		case IR::Opcode::PackedFmaF16: opcode = IR::ValueOpcode::FPFma32; break;
-		default: return false;
-	}
+bool Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                               bool accumulator, bool quiet_snan) {
 	const auto translate_lane = [&](bool high) {
-		const auto lhs = ReadF16LaneAsF32(inst.src[0], high, true);
-		const auto rhs = ReadF16LaneAsF32(inst.src[1], high, true);
+		const auto lhs = ReadF16LaneAsF32(inst.src0, high, true);
+		const auto rhs = ReadF16LaneAsF32(inst.src1, high, true);
 		IR::F32    result;
-		if (inst.src_count == 3u) {
-			result = IR::F32(
-			    ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high, true)}));
+		if (accumulator) {
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high, true)}));
+		} else if (inst.src_count == 3u) {
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high, true)}));
 		} else {
 			result = IR::F32(ir.Emit(opcode, {lhs, rhs}));
 		}
 		return ApplyF32ResultModifiers(inst.dst, result);
 	};
-	auto raw    = inst.dst;
+	auto raw    = DestinationOperand(inst);
 	raw.omod    = 0u;
 	raw.clamp   = false;
 	auto result = PackHalf2x16(translate_lane(false), translate_lane(true));
-	if (inst.op == IR::Opcode::PackedMinF16 || inst.op == IR::Opcode::PackedMaxF16) {
-		const auto quiet_snan = [&](const IR::Operand& operand, bool high) {
+	if (quiet_snan) {
+		const auto quiet_snan_lane = [&](const Decoder::Operand& operand, bool high) {
 			const auto bits     = ReadU16LaneAsU32(operand, high, false);
 			const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u)));
 			const auto payload  = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu)));
@@ -41,8 +34,8 @@ bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) {
 			return std::pair {snan, ir.BitwiseOr(bits, IR::U32(IR::Value(0x0200u)))};
 		};
 		const auto override_lane = [&](bool high) {
-			const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src[0], high);
-			const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src[1], high);
+			const auto [lhs_snan, lhs_quiet] = quiet_snan_lane(inst.src0, high);
+			const auto [rhs_snan, rhs_quiet] = quiet_snan_lane(inst.src1, high);
 			const auto normal = high ? ir.ShiftRightLogical(result, IR::U32(IR::Value(16u)))
 			                         : ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu)));
 			return ir.Select(lhs_snan, lhs_quiet, ir.Select(rhs_snan, rhs_quiet, normal));
@@ -53,152 +46,133 @@ bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) {
 	return true;
 }
 
-bool Translator::TranslateFloat16Operation(const IR::Instruction& inst) {
-	IR::ValueOpcode opcode;
-	switch (inst.op) {
-		case IR::Opcode::AddF16: opcode = IR::ValueOpcode::FPAdd32; break;
-		case IR::Opcode::SubF16: opcode = IR::ValueOpcode::FPSub32; break;
-		case IR::Opcode::MulF16: opcode = IR::ValueOpcode::FPMul32; break;
-		case IR::Opcode::MinF16: opcode = IR::ValueOpcode::FPMin32; break;
-		case IR::Opcode::MaxF16: opcode = IR::ValueOpcode::FPMax32; break;
-		case IR::Opcode::FmaF16:
-		case IR::Opcode::MadMixF16: opcode = IR::ValueOpcode::FPFma32; break;
-		case IR::Opcode::RcpF16: opcode = IR::ValueOpcode::FPRecip32; break;
-		case IR::Opcode::SqrtF16: opcode = IR::ValueOpcode::FPSqrt; break;
-		case IR::Opcode::InverseSqrtF16: opcode = IR::ValueOpcode::FPRecipSqrt32; break;
-		case IR::Opcode::Log2F16: opcode = IR::ValueOpcode::FPLog2; break;
-		case IR::Opcode::Exp2F16: opcode = IR::ValueOpcode::FPExp2; break;
-		case IR::Opcode::FloorF16: opcode = IR::ValueOpcode::FPFloor32; break;
-		case IR::Opcode::CeilF16: opcode = IR::ValueOpcode::FPCeil32; break;
-		case IR::Opcode::TruncF16: opcode = IR::ValueOpcode::FPTrunc32; break;
-		case IR::Opcode::RoundEvenF16: opcode = IR::ValueOpcode::FPRoundEven32; break;
-		case IR::Opcode::Min3F16: opcode = IR::ValueOpcode::FPMinTri32; break;
-		case IR::Opcode::Max3F16: opcode = IR::ValueOpcode::FPMaxTri32; break;
-		case IR::Opcode::Med3F16: opcode = IR::ValueOpcode::FPMedTri32; break;
-		default: return false;
-	}
-	std::array<IR::Value, 3> args;
-	for (uint32_t index = 0; index < inst.src_count; index++) {
-		args[index] = inst.op == IR::Opcode::MadMixF16 ? IR::Value(ReadMixF32(inst.src[index]))
-		                                               : IR::Value(ReadF16AsF32(inst.src[index]));
-	}
-	IR::F32 result;
-	switch (inst.src_count) {
-		case 1: result = IR::F32(ir.Emit(opcode, {args[0]})); break;
-		case 2: result = IR::F32(ir.Emit(opcode, {args[0], args[1]})); break;
-		case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break;
-		default: EXIT("invalid half-float source count: %u", inst.src_count);
-	}
-	if (inst.op == IR::Opcode::SqrtF16 || inst.op == IR::Opcode::InverseSqrtF16 ||
-	    inst.op == IR::Opcode::Log2F16) {
-		const auto negative = IR::U1(
-		    ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {args[0], IR::Value::F32(0.0f)}));
-		result = ApplyF32ResultModifiers(inst.dst, result);
-		const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
-		const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
-		WriteU16(inst.dst, ir.Select(negative, invalid, bits));
+bool Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                              bool invalid_negative) {
+	const auto argument = ReadF16AsF32(inst.src0);
+	auto       result   = IR::F32(ir.Emit(opcode, {argument}));
+	if (!invalid_negative) {
+		WriteF16(DestinationOperand(inst), result);
 		return true;
 	}
-	WriteF16(inst.dst, result);
+	const auto negative =
+	    IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {argument, IR::Value::F32(0.0f)}));
+	result             = ApplyF32ResultModifiers(inst.dst, result);
+	const auto bits    = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
+	const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
+	WriteU16(DestinationOperand(inst), ir.Select(negative, invalid, bits));
 	return true;
 }
 
-bool Translator::TranslateFloatOperation(const IR::Instruction& inst) {
-	if (inst.op == IR::Opcode::FrexpMantF32) {
-		const auto bits     = ReadU32(inst.src[0]);
-		const auto exponent = IR::U32(
-		    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
-		const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu)));
-		const auto sign     = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x80000000u)));
-		const auto base     = ir.BitwiseOr(sign, IR::U32(IR::Value(0x3f000000u)));
-		const auto normal   = ir.BitwiseOr(base, mantissa);
-		const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa}));
-		const auto shift    = ir.ISub(IR::U32(IR::Value(23u)), msb);
-		const auto fraction =
-		    ir.BitwiseAnd(ir.ShiftLeftLogical(mantissa, shift), IR::U32(IR::Value(0x007fffffu)));
-		const auto subnormal = ir.BitwiseOr(base, fraction);
-		const auto zero      = ir.IEqual(mantissa, IR::U32(IR::Value(0u)));
-		const auto finite    = ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal,
-		                                 ir.Select(zero, bits, subnormal));
-		const auto result = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite);
-		WriteOperand(inst.dst, ir.BitCastF32(result));
-		return true;
-	}
-	IR::ValueOpcode opcode {};
-	switch (inst.op) {
-		case IR::Opcode::RcpF32: opcode = IR::ValueOpcode::FPRecip32; break;
-		case IR::Opcode::RcpIflagF32: opcode = IR::ValueOpcode::FPRecipIFlag32; break;
-		case IR::Opcode::FractF32: opcode = IR::ValueOpcode::FPFract32; break;
-		case IR::Opcode::TruncF32: opcode = IR::ValueOpcode::FPTrunc32; break;
-		case IR::Opcode::CeilF32: opcode = IR::ValueOpcode::FPCeil32; break;
-		case IR::Opcode::RoundEvenF32: opcode = IR::ValueOpcode::FPRoundEven32; break;
-		case IR::Opcode::FloorF32: opcode = IR::ValueOpcode::FPFloor32; break;
-		case IR::Opcode::Exp2F32: opcode = IR::ValueOpcode::FPExp2; break;
-		case IR::Opcode::Log2F32: opcode = IR::ValueOpcode::FPLog2; break;
-		case IR::Opcode::InverseSqrtF32: opcode = IR::ValueOpcode::FPRecipSqrt32; break;
-		case IR::Opcode::SqrtF32: opcode = IR::ValueOpcode::FPSqrt; break;
-		case IR::Opcode::SinF32: opcode = IR::ValueOpcode::FPSin; break;
-		case IR::Opcode::CosF32: opcode = IR::ValueOpcode::FPCos; break;
-		case IR::Opcode::FAddF32: opcode = IR::ValueOpcode::FPAdd32; break;
-		case IR::Opcode::FSubF32: opcode = IR::ValueOpcode::FPSub32; break;
-		case IR::Opcode::FMulF32: opcode = IR::ValueOpcode::FPMul32; break;
-		case IR::Opcode::FMinF32: opcode = IR::ValueOpcode::FPMin32; break;
-		case IR::Opcode::FMaxF32: opcode = IR::ValueOpcode::FPMax32; break;
-		case IR::Opcode::FMadF32: opcode = IR::ValueOpcode::FPFma32; break;
-		case IR::Opcode::FMin3F32: opcode = IR::ValueOpcode::FPMinTri32; break;
-		case IR::Opcode::FMax3F32: opcode = IR::ValueOpcode::FPMaxTri32; break;
-		case IR::Opcode::FMed3F32: opcode = IR::ValueOpcode::FPMedTri32; break;
-		case IR::Opcode::LdexpF32: opcode = IR::ValueOpcode::FPLdexp; break;
-		default: break;
-	}
-	if (opcode != IR::ValueOpcode {}) {
-		std::array<IR::Value, 3> args;
-		for (uint32_t index = 0; index < inst.src_count; index++) {
-			const auto arg_type = IR::ArgTypeOf(opcode, index);
-			args[index] =
-			    arg_type == IR::Type::F32
-			        ? (inst.op == IR::Opcode::FMadF32 ? IR::Value(ReadMixF32(inst.src[index]))
-			                                          : ReadOperand(inst.src[index], IR::Type::F32))
-			        : ReadOperand(inst.src[index], arg_type);
-		}
-		IR::Value result;
-		switch (inst.src_count) {
-			case 1: result = ir.Emit(opcode, {args[0]}); break;
-			case 2: result = ir.Emit(opcode, {args[0], args[1]}); break;
-			case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break;
-			default: EXIT("invalid floating-point source count: %u", inst.src_count);
-		}
-		WriteOperand(inst.dst, result);
-		return true;
-	}
+bool Translator::Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                               bool reverse) {
+	const auto lhs = ReadF16AsF32(reverse ? inst.src1 : inst.src0);
+	const auto rhs = ReadF16AsF32(reverse ? inst.src0 : inst.src1);
+	WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {lhs, rhs})));
+	return true;
+}
 
+bool Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                                bool accumulator, bool mix) {
+	std::array<IR::Value, 3> args;
+	for (uint32_t index = 0; index < args.size(); index++) {
+		const auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index);
+		args[index] = mix ? IR::Value(ReadMixF32(operand)) : IR::Value(ReadF16AsF32(operand));
+	}
+	WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})));
+	return true;
+}
+
+bool Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+	const auto type = IR::ArgTypeOf(opcode, 0);
+	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {ReadOperand(inst.src0, type)}));
+	return true;
+}
+
+bool Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                             bool reverse) {
+	std::array<IR::Value, 2> args;
+	for (uint32_t index = 0; index < args.size(); index++) {
+		const auto operand = SourceAt(inst, reverse ? 1u - index : index);
+		args[index]        = ReadOperand(operand, IR::ArgTypeOf(opcode, index));
+	}
+	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1]}));
+	return true;
+}
+
+bool Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                              bool accumulator, bool mix) {
+	std::array<IR::Value, 3> args;
+	for (uint32_t index = 0; index < args.size(); index++) {
+		const auto operand = accumulator && index == 2u ? inst.dst : SourceAt(inst, index);
+		const auto type    = IR::ArgTypeOf(opcode, index);
+		args[index]        = type == IR::Type::F32 && mix ? IR::Value(ReadMixF32(operand))
+		                                                  : ReadOperand(operand, type);
+	}
+	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1], args[2]}));
+	return true;
+}
+
+bool Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
+	const auto bits = ReadU32(inst.src0);
+	const auto exponent =
+	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
+	const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu)));
+	const auto sign     = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x80000000u)));
+	const auto base     = ir.BitwiseOr(sign, IR::U32(IR::Value(0x3f000000u)));
+	const auto normal   = ir.BitwiseOr(base, mantissa);
+	const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa}));
+	const auto shift    = ir.ISub(IR::U32(IR::Value(23u)), msb);
+	const auto fraction =
+	    ir.BitwiseAnd(ir.ShiftLeftLogical(mantissa, shift), IR::U32(IR::Value(0x007fffffu)));
+	const auto subnormal = ir.BitwiseOr(base, fraction);
+	const auto zero      = ir.IEqual(mantissa, IR::U32(IR::Value(0u)));
+	const auto finite    = ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal,
+	                                 ir.Select(zero, bits, subnormal));
+	const auto result    = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite);
+	WriteOperand(DestinationOperand(inst), ir.BitCastF32(result));
+	return true;
+}
+
+bool Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
+	auto a          = inst.src0;
+	a.op_sel        = false;
+	a.op_sel_hi     = true;
+	auto b          = inst.src1;
+	b.op_sel        = false;
+	b.op_sel_hi     = true;
+	const auto a_lo = ReadF16LaneAsF32(a, false);
+	const auto a_hi = ReadF16LaneAsF32(a, true);
+	const auto b_lo = ReadF16LaneAsF32(b, false);
+	const auto b_hi = ReadF16LaneAsF32(b, true);
+	const auto acc  = ReadMixF32(inst.dst);
+	const auto lo   = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc}));
+	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo}));
+	return true;
+}
+
+bool Translator::V_CUBEID_F32(const Decoder::Instruction& inst) {
+	return FloatCube(inst, 0u);
+}
+
+bool Translator::V_CUBESC_F32(const Decoder::Instruction& inst) {
+	return FloatCube(inst, 1u);
+}
+
+bool Translator::V_CUBETC_F32(const Decoder::Instruction& inst) {
+	return FloatCube(inst, 2u);
+}
+
+bool Translator::V_CUBEMA_F32(const Decoder::Instruction& inst) {
+	return FloatCube(inst, 3u);
+}
+
+bool Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kind) {
 	const auto select_f32 = [&](IR::U1 condition, IR::F32 true_value, IR::F32 false_value) {
 		return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
 	};
-	if (inst.op == IR::Opcode::Dot2AccF32F16) {
-		auto a          = inst.src[0];
-		a.op_sel        = false;
-		a.op_sel_hi     = true;
-		auto b          = inst.src[1];
-		b.op_sel        = false;
-		b.op_sel_hi     = true;
-		const auto a_lo = ReadF16LaneAsF32(a, false);
-		const auto a_hi = ReadF16LaneAsF32(a, true);
-		const auto b_lo = ReadF16LaneAsF32(b, false);
-		const auto b_hi = ReadF16LaneAsF32(b, true);
-		const auto acc  = ReadMixF32(inst.src[2]);
-		const auto lo   = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc}));
-		WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo}));
-		return true;
-	}
-	if (inst.op != IR::Opcode::CubeIdF32 && inst.op != IR::Opcode::CubeScF32 &&
-	    inst.op != IR::Opcode::CubeTcF32 && inst.op != IR::Opcode::CubeMaF32) {
-		return false;
-	}
-
-	const auto x  = ReadMixF32(inst.src[0]);
-	const auto y  = ReadMixF32(inst.src[1]);
-	const auto z  = ReadMixF32(inst.src[2]);
+	const auto x  = ReadMixF32(inst.src0);
+	const auto y  = ReadMixF32(inst.src1);
+	const auto z  = ReadMixF32(inst.src2);
 	const auto nx = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {x}));
 	const auto ny = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {y}));
 	const auto nz = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {z}));
@@ -216,28 +190,26 @@ bool Translator::TranslateFloatOperation(const IR::Instruction& inst) {
 		return select_f32(z_face, z_value, select_f32(y_face, y_value, x_value));
 	};
 	IR::F32 result;
-	switch (inst.op) {
-		case IR::Opcode::CubeIdF32:
+	switch (result_kind) {
+		case 0u:
 			result = select_face(
 			    select_f32(x_neg, IR::F32(IR::Value::F32(1.0f)), IR::F32(IR::Value::F32(0.0f))),
 			    select_f32(y_neg, IR::F32(IR::Value::F32(3.0f)), IR::F32(IR::Value::F32(2.0f))),
 			    select_f32(z_neg, IR::F32(IR::Value::F32(5.0f)), IR::F32(IR::Value::F32(4.0f))));
 			break;
-		case IR::Opcode::CubeScF32:
-			result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x));
-			break;
-		case IR::Opcode::CubeTcF32: result = select_face(ny, select_f32(y_neg, nz, z), ny); break;
-		case IR::Opcode::CubeMaF32: {
+		case 1u: result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x)); break;
+		case 2u: result = select_face(ny, select_f32(y_neg, nz, z), ny); break;
+		case 3u: {
 			const auto two = IR::F32(IR::Value::F32(2.0f));
 			result         = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})),
 			                             IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})),
 			                             IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {z, two})));
 			break;
 		}
-		default: return false;
+		default: EXIT("invalid cube result kind");
 	}
-	WriteOperand(inst.dst, result);
+	WriteOperand(DestinationOperand(inst), result);
 	return true;
 }
 
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -0,0 +1,221 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
+
+bool Translator::EmitScalar(const Decoder::Instruction& inst, std::string* error) {
+	using O = Decoder::Opcode;
+	switch (inst.opcode) {
+		case O::S_MOV_B32:
+		case O::S_MOVK_I32: MOV_B32(inst, false); return true;
+		case O::S_MOV_B64: S_MOV_B64(inst); return true;
+		case O::S_WQM_B64: S_WQM_B64(inst); return true;
+		case O::S_GETPC_B64: return S_GETPC_B64(inst, error);
+		case O::S_SETPC_B64: return true;
+		case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true;
+		case O::S_CSELECT_B64: S_CSELECT_B64(inst); return true;
+		case O::S_SETREG_B32: EmitControlNop(); return true;
+		case O::S_WAITCNT: EmitWaitcnt(); return true;
+
+		case O::S_AND_SAVEEXEC_B32:
+			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
+			return true;
+		case O::S_ANDN1_SAVEEXEC_B32:
+			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, false);
+			return true;
+		case O::S_AND_SAVEEXEC_B64:
+			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, true);
+			return true;
+		case O::S_ANDN1_SAVEEXEC_B64:
+			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, true);
+			return true;
+		case O::S_ORN2_SAVEEXEC_B64:
+			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, true);
+			return true;
+		case O::S_ADD_U32: ADD_U32(inst, false, false); return true;
+		case O::S_ADDC_U32: ADD_U32(inst, false, true); return true;
+		case O::S_SUB_U32: SUB_U32(inst, false, false); return true;
+		case O::S_SUBB_U32: SUBB_U32(inst, false); return true;
+		case O::S_ADD_I32: S_ADD_SUB_I32(inst, false); return true;
+		case O::S_SUB_I32: S_ADD_SUB_I32(inst, true); return true;
+		case O::S_LSHL1_ADD_U32: S_LSHL_ADD_U32(inst, 1u); return true;
+		case O::S_LSHL2_ADD_U32: S_LSHL_ADD_U32(inst, 2u); return true;
+		case O::S_LSHL3_ADD_U32: S_LSHL_ADD_U32(inst, 3u); return true;
+		case O::S_LSHL4_ADD_U32: S_LSHL_ADD_U32(inst, 4u); return true;
+		case O::S_MIN_I32:
+			ScalarMinMax32(inst, IR::ValueOpcode::SMin32, IR::ValueOpcode::SLessThan32);
+			return true;
+		case O::S_MAX_I32:
+			ScalarMinMax32(inst, IR::ValueOpcode::SMax32, IR::ValueOpcode::SGreaterThan32);
+			return true;
+		case O::S_MIN_U32:
+			ScalarMinMax32(inst, IR::ValueOpcode::UMin32, IR::ValueOpcode::ULessThan32);
+			return true;
+		case O::S_MAX_U32:
+			ScalarMinMax32(inst, IR::ValueOpcode::UMax32, IR::ValueOpcode::UGreaterThan32);
+			return true;
+
+		case O::S_CMP_EQ_U32:
+		case O::S_CMP_EQ_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_LG_U32:
+		case O::S_CMP_LG_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_GT_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_GE_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true,
+			                   false);
+			return true;
+		case O::S_CMP_LT_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_LE_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_GT_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_GE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true,
+			                   false);
+			return true;
+		case O::S_CMP_LT_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_LE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true, false);
+			return true;
+		case O::S_CMP_EQ_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true, false);
+			return true;
+		case O::S_CMP_LG_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true, false);
+			return true;
+
+		case O::S_AND_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32,
+			                  false, false, false);
+		case O::S_ANDN2_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32,
+			                  true, false, false);
+		case O::S_OR_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalOr, IR::ValueOpcode::BitwiseOr32, false,
+			                  false, false);
+		case O::S_ORN2_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalOr, IR::ValueOpcode::BitwiseOr32, true,
+			                  false, false);
+		case O::S_XOR_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalXor, IR::ValueOpcode::BitwiseXor32,
+			                  false, false, false);
+		case O::S_NAND_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32,
+			                  false, true, false);
+		case O::S_NOR_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalOr, IR::ValueOpcode::BitwiseOr32, false,
+			                  true, false);
+		case O::S_XNOR_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalXor, IR::ValueOpcode::BitwiseXor32,
+			                  false, true, false);
+		case O::S_NOT_B64:
+			return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32,
+			                  false, false, true);
+
+		case O::S_ABS_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::IAbs32, IR::Type::U32, false, false, true);
+		case O::S_MUL_I32:
+		case O::S_MULK_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false);
+		case O::S_MUL_HI_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMulHi, IR::Type::U32, false, false, false);
+		case O::S_AND_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseAnd32, IR::Type::U32, false, false,
+			                     true);
+		case O::S_OR_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseOr32, IR::Type::U32, false, false,
+			                     true);
+		case O::S_XOR_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseXor32, IR::Type::U32, false, false,
+			                     true);
+		case O::S_NOT_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseNot32, IR::Type::U32, false, false,
+			                     true);
+		case O::S_BREV_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false,
+			                     false);
+		case O::S_BCNT1_I32_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
+			                     true);
+		case O::S_BCNT1_I32_B64:
+			return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false,
+			                     true);
+		case O::S_FF1_I32_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false,
+			                     false);
+		case O::S_LSHL_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical32, IR::Type::U32, false,
+			                     true, true);
+		case O::S_LSHR_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical32, IR::Type::U32, false,
+			                     true, true);
+		case O::S_ASHR_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic32, IR::Type::U32,
+			                     false, true, true);
+		case O::S_LSHL_B64:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical64, IR::Type::U64, false,
+			                     false, true);
+		case O::S_LSHR_B64:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, false,
+			                     false, true);
+
+		case O::S_ANDN2_B32:
+			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseAnd32, true, false, true);
+		case O::S_ORN2_B32:
+			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, true, false, true);
+		case O::S_NAND_B32:
+			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseAnd32, false, true, true);
+		case O::S_NOR_B32:
+			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseOr32, false, true, true);
+		case O::S_XNOR_B32:
+			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, true);
+		case O::S_FF1_I32_B64: return S_FF1_I32_B64(inst);
+		case O::S_FLBIT_I32_B64: return S_FLBIT_I32_B64(inst);
+
+		case O::S_BITSET0_B32: return S_BITSET_B32(inst, false);
+		case O::S_BITSET1_B32: return S_BITSET_B32(inst, true);
+		case O::S_BITREPLICATE_B64_B32: return S_BITREPLICATE_B64_B32(inst);
+		case O::S_QUADMASK_B64: return S_QUADMASK_B64(inst);
+		case O::S_BFM_B32: return BFM_B32(inst);
+		case O::S_BFM_B64: return S_BFM_B64(inst);
+		case O::S_BFE_U32: return S_BFE_U32(inst, false);
+		case O::S_BFE_I32: return S_BFE_U32(inst, true);
+		case O::S_BFE_U64: return S_BFE_U64(inst);
+		case O::S_BITCMP0_B32: return S_BITCMP_B32(inst, false);
+		case O::S_BITCMP1_B32: return S_BITCMP_B32(inst, true);
+		case O::S_PACK_LL_B32_B16: return PackB16(inst, false, false);
+		case O::S_PACK_LH_B32_B16: return PackB16(inst, false, true);
+		case O::S_PACK_HH_B32_B16: return PackB16(inst, true, true);
+
+		case O::S_NOP:
+		case O::S_SLEEP:
+		case O::S_TRAP: EmitControlNop(); return true;
+		case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return true;
+		case O::S_BARRIER: S_BARRIER(); return true;
+		case O::S_SENDMSG: S_SENDMSG(); return true;
+		case O::S_TTRACEDATA: S_TTRACEDATA(); return true;
+		case O::S_INST_PREFETCH: S_INST_PREFETCH(); return true;
+		case O::S_BRANCH:
+		case O::S_CBRANCH_SCC0:
+		case O::S_CBRANCH_SCC1:
+		case O::S_CBRANCH_VCCZ:
+		case O::S_CBRANCH_VCCNZ:
+		case O::S_CBRANCH_EXECZ:
+		case O::S_CBRANCH_EXECNZ:
+		case O::S_ENDPGM: return true;
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Translate.h

@@ -1,12 +1,44 @@
 #pragma once
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+
+#include <cstdint>
+#include <string_view>
+#include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
-                      const ShaderVertexInputInfo*  vertex_input_info,
-                      const ShaderPixelInputInfo*   pixel_input_info,
-                      const ShaderComputeInputInfo* compute_input_info, std::string* error);
+struct EmbeddedFetchLoad {
+	uint32_t              pc         = 0;
+	int                   attrib_id  = -1;
+	uint32_t              components = 0;
+	std::vector<uint32_t> prolog_loads;
+};
+
+struct EmbeddedFetchPlan {
+	std::vector<EmbeddedFetchLoad> loads;
+	int32_t                        vertex_offset_sgpr = -1;
+};
+
+struct TranslateOptions {
+	ShaderType                    stage               = ShaderType::Unknown;
+	uint32_t                      wave_size           = 64;
+	uint64_t                      shader_hash         = 0;
+	uint32_t                      user_data_base      = 0;
+	uint32_t                      user_data_count     = 64;
+	uint32_t                      scratch_dwords      = 0;
+	bool                          dispatcher_fallback = false;
+	CFG::FailureKind              cfg_failure_kind    = CFG::FailureKind::None;
+	std::string_view              fallback_reason;
+	const ShaderVertexInputInfo*  vertex         = nullptr;
+	const ShaderPixelInputInfo*   pixel          = nullptr;
+	const ShaderComputeInputInfo* compute        = nullptr;
+	const EmbeddedFetchPlan*      embedded_fetch = nullptr;
+};
+
+bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
+                      const TranslateOptions& options, IR::Program& result, std::string* error);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -5,58 +5,59 @@
 
 #include <array>
 
-namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 class Translator {
 public:
-	Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
-	           uint32_t wave_size)
-	    : value_program(program), ir(block), current_vector_limit(vector_limit),
+	Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit, uint32_t wave_size)
+	    : program(program), ir(block), current_vector_limit(vector_limit),
 	      current_wave_size(wave_size) {}
 
-	bool TranslateBlock(const IR::BasicBlock& source, std::string* error);
-	bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
-	                        std::string* error);
+	bool TranslateInstruction(const Decoder::Instruction& inst, std::string* error);
+	bool TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
+	                            uint32_t component_count);
+	bool AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error);
 
 private:
-	IR::Operand            OffsetOperand(const IR::Operand& operand, uint32_t offset);
-	IR::Operand            ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset);
-	IR::Operand            PlainOperand(const IR::Operand& operand);
-	IR::U32                ReadPcRelativeU32(const IR::Operand& operand);
+	Decoder::Operand SourceAt(const Decoder::Instruction& inst, uint32_t index);
+	Decoder::Operand DestinationOperand(const Decoder::Instruction& inst);
+	Decoder::Operand OffsetOperand(const Decoder::Operand& operand, uint32_t offset);
+	Decoder::Operand ScalarDestinationOperand(const Decoder::Operand& operand, uint32_t offset);
+	Decoder::Operand PlainOperand(const Decoder::Operand& operand);
 	std::array<IR::U32, 2> BallotMask(IR::U1 value);
-	IR::U32                ReadRawU32(const IR::Operand& operand);
+	IR::U32                ReadRawU32(const Decoder::Operand& operand);
 	IR::U32                ReadScalarCode(uint32_t code);
-	IR::U32                ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value);
-	IR::Value              ReadOperand(const IR::Operand& operand, IR::Type type);
+	IR::U32                ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value);
+	IR::Value              ReadOperand(const Decoder::Operand& operand, IR::Type type);
 	IR::U1                 ThreadBit(IR::U32 low);
-	void                   WriteRawU32(const IR::Operand& operand, IR::U32 value);
-	IR::F32                ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value);
-	void                   WriteOperand(const IR::Operand& operand, IR::Value value);
+	void                   WriteRawU32(const Decoder::Operand& operand, IR::U32 value);
+	IR::F32                ApplyF32ResultModifiers(const Decoder::Operand& operand, IR::F32 value);
+	void                   WriteOperand(const Decoder::Operand& operand, IR::Value value);
 	IR::U32                PackHalf2x16(IR::F32 low, IR::F32 high);
-	void                   WriteF16(const IR::Operand& operand, IR::F32 value);
-	void                   WriteU16(const IR::Operand& operand, IR::U32 value);
-	IR::U32                ReadU32(const IR::Operand& operand);
-	std::array<IR::U32, 2> ReadU32Pair(const IR::Operand& operand);
-	IR::U64                ReadU64(const IR::Operand& operand);
-	IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false);
-	IR::F32 ReadF16AsF32(const IR::Operand& operand);
-	IR::F32 ReadMixF32(const IR::Operand& operand);
-	IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
-	IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
-	IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
-	IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
+	void                   WriteF16(const Decoder::Operand& operand, IR::F32 value);
+	void                   WriteU16(const Decoder::Operand& operand, IR::U32 value);
+	IR::U32                ReadU32(const Decoder::Operand& operand);
+	std::array<IR::U32, 2> ReadU32Pair(const Decoder::Operand& operand);
+	IR::U64                ReadU64(const Decoder::Operand& operand);
+	IR::F32 ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed = false);
+	IR::F32 ReadF16AsF32(const Decoder::Operand& operand);
+	IR::F32 ReadMixF32(const Decoder::Operand& operand);
+	IR::U32 ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane);
+	IR::U32 ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane, bool sign_extend);
+	IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend);
+	IR::U32 ReadF16LaneBits(const Decoder::Operand& operand, bool high_lane);
 	std::array<IR::U32, 2> ExtractU64(IR::U64 value);
-	void    WriteU32Pair(const IR::Operand& operand, const std::array<IR::U32, 2>& value);
-	IR::U1  ReadCondition(const IR::Operand& operand);
-	IR::U32 ConditionBit(const IR::Operand& operand);
-	IR::U1  ReadMask(const IR::Operand& operand);
-	IR::U1  ReadMaskValid(const IR::Operand& operand);
-	void    WriteMask(const IR::Operand& operand, IR::U1 value);
-	void    WriteMask64(const IR::Operand& operand, IR::U1 value);
-	void    WriteCompareResult(const IR::Operand& operand, IR::U1 value);
+	void    WriteU32Pair(const Decoder::Operand& operand, const std::array<IR::U32, 2>& value);
+	IR::U1  ReadCondition(const Decoder::Operand& operand);
+	IR::U32 ConditionBit(const Decoder::Operand& operand);
+	IR::U1  ReadMask(const Decoder::Operand& operand);
+	IR::U1  ReadMaskValid(const Decoder::Operand& operand);
+	void    WriteMask(const Decoder::Operand& operand, IR::U1 value);
+	void    WriteMask64(const Decoder::Operand& operand, IR::U1 value);
+	void    WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);
 
-	IR::MemoryFlags AddMemoryInfo(const IR::Instruction& inst);
-	IR::ExportFlags AddExportInfo(const IR::Instruction& inst);
+	IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc);
+	IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst);
 	struct AddressOperands {
 		IR::Value resource;
 		IR::Value low;
@@ -67,31 +68,47 @@ private:
 		IR::U32 offset;
 		IR::U32 soffset;
 	};
-	AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source);
+	AddressOperands ReadAddressOperands(const Decoder::Instruction& inst, uint32_t first_source);
 	IR::U32         GetResourceDword(uint32_t index, uint32_t dword);
 	IR::Value       GetBufferResource(const IR::MemoryInfo& memory);
 	IR::Value       GetAddressResource(IR::Value low, IR::Value high);
 	IR::Value       GetScalarAddressResource(uint32_t base);
 	IR::Value       GetImageResource(const IR::MemoryInfo& memory);
 	IR::Value       GetSamplerResource(const IR::MemoryInfo& memory);
-	IR::Value       MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
-	IR::Value       ConstructU32x4(const IR::Operand& base, uint32_t count);
-	void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory,
-	                          uint32_t component_limit);
-	IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
-	BufferAddress   ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
-	IR::U32         WidenSubdword(IR::Value value, uint32_t bits, bool sign);
-	IR::Value       NarrowSubdword(IR::U32 value, uint32_t bits);
-	IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
-	IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode);
-	bool            TranslateScalarMemory(const IR::Instruction& inst);
-	bool            TranslateBufferLoad(const IR::Instruction& inst);
-	bool            TranslateBufferStore(const IR::Instruction& inst);
-	bool            TranslateAtomicMemory(const IR::Instruction& inst);
-	bool            TranslateFlatLoad(const IR::Instruction& inst);
-	bool            TranslateFlatStore(const IR::Instruction& inst);
-	bool            TranslateImageMemory(const IR::Instruction& inst);
-	bool            TranslateSharedMemory(const IR::Instruction& inst);
+	IR::Value     MakeImageAddress(const Decoder::Instruction& inst, const Decoder::Operand& base);
+	IR::Value     ConstructU32x4(const Decoder::Operand& base, uint32_t count);
+	void          WriteImageComponents(const Decoder::Operand& dst, IR::Value value,
+	                                   const IR::MemoryInfo& memory, uint32_t component_limit);
+	BufferAddress ReadBufferAddress(const Decoder::Instruction& inst, uint32_t source_offset);
+	IR::U32       WidenSubdword(IR::Value value, uint32_t bits, bool sign);
+	IR::Value     NarrowSubdword(IR::U32 value, uint32_t bits);
+	bool          S_LOAD(const Decoder::Instruction& inst, bool raw);
+	bool          BUFFER_LOAD(const Decoder::Instruction& inst);
+	bool          BUFFER_STORE(const Decoder::Instruction& inst);
+	bool          BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	bool          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	bool DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
+	bool FLAT_LOAD(const Decoder::Instruction& inst);
+	bool FLAT_STORE(const Decoder::Instruction& inst);
+	bool IMAGE_GET_RESINFO(const Decoder::Instruction& inst);
+	bool IMAGE_GET_LOD(const Decoder::Instruction& inst);
+	bool IMAGE_LOAD(const Decoder::Instruction& inst);
+	bool IMAGE_STORE(const Decoder::Instruction& inst);
+	bool IMAGE_SAMPLE(const Decoder::Instruction& inst);
+	bool IMAGE_GATHER(const Decoder::Instruction& inst);
+	IR::Value LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory,
+	                        uint32_t pc);
+	IR::Value ExtractSharedU32(IR::Value value, uint32_t width, uint32_t index);
+	void WriteSharedU32(uint32_t width, IR::U32 address, const std::array<IR::Value, 4>& values,
+	                    const IR::MemoryInfo& memory, uint32_t pc);
+	bool DS_READ(const Decoder::Instruction& inst);
+	bool DS_READ2(const Decoder::Instruction& inst);
+	bool DS_WRITE(const Decoder::Instruction& inst);
+	bool DS_WRITE2(const Decoder::Instruction& inst);
+	bool DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	bool DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	bool DS_ADDTID(const Decoder::Instruction& inst, bool write);
+	bool DS_SWIZZLE_B32(const Decoder::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -100,35 +117,143 @@ private:
 	                                 float safe_upper, uint32_t lower_result,
 	                                 uint32_t upper_result);
 	IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
+	void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, bool cmpx);
+	void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, bool cmpx);
+	void EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
+	                        bool scalar, bool cmpx);
+	void EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	                          bool signed_value, bool cmpx);
+	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
+	                      bool cmpx);
+	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered);
+	void V_CMP_CLASS_F32(const Decoder::Instruction& inst);
+	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
+	void V_CVT_F32_U32(const Decoder::Instruction& inst);
+	void V_CVT_F32_I32(const Decoder::Instruction& inst);
+	void V_CVT_U32_F32(const Decoder::Instruction& inst);
+	void V_CVT_I32_F32(const Decoder::Instruction& inst);
+	void V_CVT_F16_F32(const Decoder::Instruction& inst);
+	void V_CVT_F32_F16(const Decoder::Instruction& inst);
+	void V_CVT_F16_16(const Decoder::Instruction& inst, bool signed_value);
+	void V_CVT_16_F16(const Decoder::Instruction& inst, bool signed_value);
+	void V_CVT_RPI_I32_F32(const Decoder::Instruction& inst);
+	void V_CVT_FLR_I32_F32(const Decoder::Instruction& inst);
+	void V_FREXP_EXP_I32_F32(const Decoder::Instruction& inst);
+	void V_CVT_OFF_F32_I4(const Decoder::Instruction& inst);
+	void V_CVT_PKRTZ_F16_F32(const Decoder::Instruction& inst);
+	void V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void V_CVT_PK_U8_F32(const Decoder::Instruction& inst);
+	void V_PACK_B32_F16(const Decoder::Instruction& inst);
+	bool PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
+	                   bool quiet_snan);
+	bool Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	                  bool invalid_negative);
+	bool Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
+	bool Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
+	                    bool mix);
+	bool FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	bool FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
+	bool FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
+	                  bool mix);
+	bool V_FREXP_MANT_F32(const Decoder::Instruction& inst);
+	bool V_DOT2C_F32_F16(const Decoder::Instruction& inst);
+	bool V_CUBEID_F32(const Decoder::Instruction& inst);
+	bool V_CUBESC_F32(const Decoder::Instruction& inst);
+	bool V_CUBETC_F32(const Decoder::Instruction& inst);
+	bool V_CUBEMA_F32(const Decoder::Instruction& inst);
+	bool FloatCube(const Decoder::Instruction& inst, uint32_t result_kind);
+	bool Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
+	bool Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
+	bool V_MED3_I16(const Decoder::Instruction& inst);
+	bool PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	                          bool arithmetic);
+	bool PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	bool PackedInteger16Mad(const Decoder::Instruction& inst, bool sign);
+	bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
+	bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
+	                IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary);
+	IR::U1  U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs,
+	                      bool negate_result);
+	bool    SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
+	                      bool reverse, bool mask_shift_count, bool update_scc);
+	bool    ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	                              bool negate_rhs, bool negate_result, bool update_scc);
+	bool    V_AND_OR_B32(const Decoder::Instruction& inst);
+	bool    V_OR3_B32(const Decoder::Instruction& inst);
+	bool    V_XOR3_B32(const Decoder::Instruction& inst);
+	bool    S_FF1_I32_B64(const Decoder::Instruction& inst);
+	bool    V_FFBH_U32(const Decoder::Instruction& inst);
+	bool    S_FLBIT_I32_B64(const Decoder::Instruction& inst);
+	bool    Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
+	bool    V_MAD_U64_U32(const Decoder::Instruction& inst);
+	bool    V_SAD_U32(const Decoder::Instruction& inst);
+	bool    V_ADD3_U32(const Decoder::Instruction& inst);
+	bool    S_BITSET_B32(const Decoder::Instruction& inst, bool set);
+	bool    V_BCNT_U32_B32(const Decoder::Instruction& inst);
+	bool    V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
+	bool    S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
+	bool    S_QUADMASK_B64(const Decoder::Instruction& inst);
+	bool    BFM_B32(const Decoder::Instruction& inst);
+	IR::U32 RightMask32(IR::U32 count);
+	IR::U64 RightMask64(IR::U32 count);
+	bool    S_BFM_B64(const Decoder::Instruction& inst);
+	bool    S_BFE_U32(const Decoder::Instruction& inst, bool sign);
+	bool    S_BFE_U64(const Decoder::Instruction& inst);
+	bool    V_BFE_U32(const Decoder::Instruction& inst, bool sign);
+	bool    V_BFI_B32(const Decoder::Instruction& inst);
+	bool    S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
+	bool    V_ALIGNBIT_B32(const Decoder::Instruction& inst);
+	bool    V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
+	bool    V_LSHL_ADD_U32(const Decoder::Instruction& inst);
+	bool    V_ADD_LSHL_U32(const Decoder::Instruction& inst);
+	bool    V_XAD_U32(const Decoder::Instruction& inst);
+	bool    V_LSHL_OR_B32(const Decoder::Instruction& inst);
+	bool    V_CNDMASK_B32(const Decoder::Instruction& inst);
+	bool    PackB16(const Decoder::Instruction& inst, bool high0, bool high1);
 
-	bool   TranslateInstruction(const IR::Instruction& inst);
-	bool   TranslateStateOperation(const IR::Instruction& inst);
-	bool   TranslateControlOperation(const IR::Instruction& inst);
-	bool   TranslateMove(const IR::Instruction& inst);
-	bool   TranslateLaneOperation(const IR::Instruction& inst);
-	bool   TranslateAttributeOperation(const IR::Instruction& inst);
-	bool   TranslateMemoryOperation(const IR::Instruction& inst);
-	bool   TranslateIntegerCompare(const IR::Instruction& inst);
-	bool   TranslateInteger16Compare(const IR::Instruction& inst);
-	bool   TranslateFloatCompare(const IR::Instruction& inst);
-	bool   TranslateConversion(const IR::Instruction& inst);
-	bool   TranslateInteger16Operation(const IR::Instruction& inst);
-	bool   TranslatePackedInteger16(const IR::Instruction& inst);
-	bool   TranslatePackedFloat16(const IR::Instruction& inst);
-	bool   TranslateFloat16Operation(const IR::Instruction& inst);
-	bool   TranslateFloatOperation(const IR::Instruction& inst);
-	IR::U1 EvaluateU64Mask(const IR::Instruction& inst);
-	bool   TranslateU64MaskOperation(const IR::Instruction& inst);
-	bool   TranslateSimpleInteger(const IR::Instruction& inst);
-	bool   TranslateComposedInteger(const IR::Instruction& inst);
-	bool   TranslateExtendedInteger(const IR::Instruction& inst);
+	void S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec,
+	                bool negate_source, bool write_64);
+	void ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in);
+	void SUB_U32(const Decoder::Instruction& inst, bool vector, bool reverse);
+	void SUBB_U32(const Decoder::Instruction& inst, bool vector);
+	void S_ADD_SUB_I32(const Decoder::Instruction& inst, bool subtract);
+	void S_LSHL_ADD_U32(const Decoder::Instruction& inst, uint32_t shift_amount);
+	void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode,
+	                    IR::ValueOpcode compare_opcode);
+	void EmitControlNop();
+	void EmitWaitcnt();
+	void S_BARRIER();
+	void S_SENDMSG();
+	void S_TTRACEDATA();
+	void S_INST_PREFETCH();
+	bool S_GETPC_B64(const Decoder::Instruction& inst, std::string* error);
+	void S_CSELECT_B32(const Decoder::Instruction& inst);
+	void S_CSELECT_B64(const Decoder::Instruction& inst);
+	void MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers);
+	void S_MOV_B64(const Decoder::Instruction& inst);
+	void S_WQM_B64(const Decoder::Instruction& inst);
+	bool V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error);
+	bool V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* error);
+	void V_READFIRSTLANE_B32(const Decoder::Instruction& inst);
+	void V_READLANE_B32(const Decoder::Instruction& inst);
+	void V_WRITELANE_B32(const Decoder::Instruction& inst);
+	void V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16);
+	void V_INTERP_P1_F32();
+	void V_INTERP_P2_F32(const Decoder::Instruction& inst);
+	bool V_INTERP_MOV_F32(const Decoder::Instruction& inst, std::string* error);
+	bool EXP(const Decoder::Instruction& inst, std::string* error);
 
-	IR::ValueProgram& value_program;
-	IR::IREmitter     ir;
-	IR::Opcode        current_opcode       = IR::Opcode::ControlNop;
-	uint32_t          current_pc           = 0;
-	uint32_t          current_vector_limit = 1;
-	uint32_t          current_wave_size    = 64;
+	bool EmitScalar(const Decoder::Instruction& inst, std::string* error);
+	bool EmitVector(const Decoder::Instruction& inst, std::string* error);
+	bool EmitInterpolation(const Decoder::Instruction& inst, std::string* error);
+	bool EmitMemory(const Decoder::Instruction& inst, std::string* error);
+
+	IR::Program&    program;
+	IR::IREmitter   ir;
+	Decoder::Opcode current_opcode       = Decoder::Opcode::UNKNOWN;
+	uint32_t        current_pc           = 0;
+	uint32_t        current_vector_limit = 1;
+	uint32_t        current_wave_size    = 64;
 };
 
-} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -0,0 +1,514 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend {
+
+bool Translator::EmitVector(const Decoder::Instruction& inst, std::string* error) {
+	using O = Decoder::Opcode;
+	switch (inst.opcode) {
+		case O::V_NOP: return true;
+		case O::V_ADD_I32: ADD_U32(inst, true, false); return true;
+		case O::V_ADDC_U32: ADD_U32(inst, true, true); return true;
+		case O::V_SUB_I32: SUB_U32(inst, true, false); return true;
+		case O::V_SUBREV_I32: SUB_U32(inst, true, true); return true;
+		case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true); return true;
+
+		case O::V_MOV_B32: MOV_B32(inst, true); return true;
+		case O::V_MOVRELS_B32: return V_MOVRELS_B32(inst, error);
+		case O::V_MOVRELD_B32: return V_MOVRELD_B32(inst, error);
+		case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return true;
+		case O::V_READLANE_B32: V_READLANE_B32(inst); return true;
+		case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return true;
+		case O::V_PERMLANE16_B32: V_PERMLANE16_B32(inst, false); return true;
+		case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return true;
+
+		case O::V_CMP_F_I32:
+		case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return true;
+		case O::V_CMP_T_I32:
+		case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return true;
+		case O::V_CMP_EQ_U32:
+		case O::V_CMP_EQ_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, false);
+			return true;
+		case O::V_CMPX_EQ_U32:
+		case O::V_CMPX_EQ_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_NE_U32:
+		case O::V_CMP_NE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, false);
+			return true;
+		case O::V_CMPX_NE_U32:
+		case O::V_CMPX_NE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_GT_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, false);
+			return true;
+		case O::V_CMPX_GT_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_GE_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
+			                   false);
+			return true;
+		case O::V_CMPX_GE_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
+			                   true);
+			return true;
+		case O::V_CMP_LT_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, false);
+			return true;
+		case O::V_CMPX_LT_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_LE_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false,
+			                   false);
+			return true;
+		case O::V_CMPX_LE_U32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_GT_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, false);
+			return true;
+		case O::V_CMPX_GT_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_GE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
+			                   false);
+			return true;
+		case O::V_CMPX_GE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
+			                   true);
+			return true;
+		case O::V_CMP_LT_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, false);
+			return true;
+		case O::V_CMPX_LT_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_LE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false,
+			                   false);
+			return true;
+		case O::V_CMPX_LE_I32:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true);
+			return true;
+		case O::V_CMP_EQ_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false);
+			return true;
+		case O::V_CMP_GT_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
+			return true;
+		case O::V_CMP_NE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
+			return true;
+		case O::V_CMPX_NE_I64:
+		case O::V_CMPX_NE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
+			return true;
+
+		case O::V_CMP_EQ_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
+			return true;
+		case O::V_CMP_EQ_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
+			return true;
+		case O::V_CMP_NE_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
+			return true;
+		case O::V_CMP_NE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
+			return true;
+		case O::V_CMP_GT_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
+			return true;
+		case O::V_CMP_GE_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false, false);
+			return true;
+		case O::V_CMP_LT_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, false);
+			return true;
+		case O::V_CMP_LE_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false);
+			return true;
+		case O::V_CMP_GT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
+			return true;
+		case O::V_CMP_GE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
+			return true;
+		case O::V_CMP_LT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
+			return true;
+		case O::V_CMP_LE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
+			return true;
+
+		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return true;
+		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return true;
+		case O::V_CMP_EQ_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
+			return true;
+		case O::V_CMPX_EQ_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
+			return true;
+		case O::V_CMP_LG_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, false);
+			return true;
+		case O::V_CMPX_LG_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, true);
+			return true;
+		case O::V_CMP_GT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, false);
+			return true;
+		case O::V_CMPX_GT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, true);
+			return true;
+		case O::V_CMP_GE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, false);
+			return true;
+		case O::V_CMPX_GE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, true);
+			return true;
+		case O::V_CMP_LT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, false);
+			return true;
+		case O::V_CMPX_LT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, true);
+			return true;
+		case O::V_CMP_LE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, false);
+			return true;
+		case O::V_CMPX_LE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, true);
+			return true;
+		case O::V_CMP_NLG_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, false);
+			return true;
+		case O::V_CMPX_NLG_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, true);
+			return true;
+		case O::V_CMP_NEQ_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, false);
+			return true;
+		case O::V_CMPX_NEQ_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, true);
+			return true;
+		case O::V_CMP_NLE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, false);
+			return true;
+		case O::V_CMPX_NLE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, true);
+			return true;
+		case O::V_CMP_NLT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, false);
+			return true;
+		case O::V_CMPX_NLT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, true);
+			return true;
+		case O::V_CMP_NGE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, false);
+			return true;
+		case O::V_CMPX_NGE_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, true);
+			return true;
+		case O::V_CMP_NGT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, false);
+			return true;
+		case O::V_CMPX_NGT_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, true);
+			return true;
+		case O::V_CMP_EQ_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, false);
+			return true;
+		case O::V_CMPX_EQ_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, true);
+			return true;
+		case O::V_CMP_LG_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true, false);
+			return true;
+		case O::V_CMP_GT_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, false);
+			return true;
+		case O::V_CMPX_GT_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, true);
+			return true;
+		case O::V_CMP_GE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, false);
+			return true;
+		case O::V_CMPX_GE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, true);
+			return true;
+		case O::V_CMP_LT_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, false);
+			return true;
+		case O::V_CMPX_LT_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, true);
+			return true;
+		case O::V_CMP_LE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, false);
+			return true;
+		case O::V_CMPX_LE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
+			return true;
+		case O::V_CMP_NEQ_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, false);
+			return true;
+		case O::V_CMPX_NEQ_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, true);
+			return true;
+		case O::V_CMPX_NLT_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, true);
+			return true;
+		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true); return true;
+		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false); return true;
+		case O::V_CMP_CLASS_F32: V_CMP_CLASS_F32(inst); return true;
+
+		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return true;
+		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return true;
+		case O::V_CVT_F32_UBYTE2: V_CVT_F32_UBYTE(inst, 2); return true;
+		case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return true;
+		case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return true;
+		case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return true;
+		case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return true;
+		case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return true;
+		case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return true;
+		case O::V_CVT_F32_F16: V_CVT_F32_F16(inst); return true;
+		case O::V_CVT_F16_U16: V_CVT_F16_16(inst, false); return true;
+		case O::V_CVT_F16_I16: V_CVT_F16_16(inst, true); return true;
+		case O::V_CVT_U16_F16: V_CVT_16_F16(inst, false); return true;
+		case O::V_CVT_I16_F16: V_CVT_16_F16(inst, true); return true;
+		case O::V_CVT_RPI_I32_F32: V_CVT_RPI_I32_F32(inst); return true;
+		case O::V_CVT_FLR_I32_F32: V_CVT_FLR_I32_F32(inst); return true;
+		case O::V_FREXP_EXP_I32_F32: V_FREXP_EXP_I32_F32(inst); return true;
+		case O::V_CVT_OFF_F32_I4: V_CVT_OFF_F32_I4(inst); return true;
+		case O::V_CVT_PKRTZ_F16_F32: V_CVT_PKRTZ_F16_F32(inst); return true;
+		case O::V_CVT_PKNORM_I16_F32:
+			V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackSnorm2x16);
+			return true;
+		case O::V_CVT_PKNORM_U16_F32:
+			V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackUnorm2x16);
+			return true;
+		case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return true;
+		case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return true;
+		case O::V_CVT_PK_U16_U32:
+		case O::V_CVT_PK_I16_I32: return PackB16(inst, false, false);
+
+		case O::V_LSHLREV_B16:
+			return Integer16Shift(inst, IR::ValueOpcode::ShiftLeftLogical32, false);
+		case O::V_LSHRREV_B16:
+			return Integer16Shift(inst, IR::ValueOpcode::ShiftRightLogical32, false);
+		case O::V_ASHRREV_I16:
+			return Integer16Shift(inst, IR::ValueOpcode::ShiftRightArithmetic32, true);
+		case O::V_ADD_NC_U16:
+		case O::V_ADD_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::IAdd32, false);
+		case O::V_SUB_NC_U16:
+		case O::V_SUB_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::ISub32, false);
+		case O::V_MED3_I16: return V_MED3_I16(inst);
+		case O::V_MIN_I16: return Integer16Binary(inst, IR::ValueOpcode::SMin32, true);
+		case O::V_MAX_I16: return Integer16Binary(inst, IR::ValueOpcode::SMax32, true);
+		case O::V_MIN_U16: return Integer16Binary(inst, IR::ValueOpcode::UMin32, false);
+		case O::V_MAX_U16: return Integer16Binary(inst, IR::ValueOpcode::UMax32, false);
+
+		case O::V_PK_LSHLREV_B16:
+			return PackedInteger16Shift(inst, IR::ValueOpcode::ShiftLeftLogical32, false);
+		case O::V_PK_LSHRREV_B16:
+			return PackedInteger16Shift(inst, IR::ValueOpcode::ShiftRightLogical32, false);
+		case O::V_PK_ASHRREV_I16:
+			return PackedInteger16Shift(inst, IR::ValueOpcode::ShiftRightArithmetic32, true);
+		case O::V_PK_MAD_I16: return PackedInteger16Mad(inst, true);
+		case O::V_PK_MAD_U16: return PackedInteger16Mad(inst, false);
+		case O::V_PK_MUL_LO_U16: return PackedInteger16Binary(inst, IR::ValueOpcode::IMul32);
+		case O::V_PK_ADD_I16:
+		case O::V_PK_ADD_U16: return PackedInteger16Binary(inst, IR::ValueOpcode::IAdd32);
+		case O::V_PK_SUB_I16:
+		case O::V_PK_SUB_U16: return PackedInteger16Binary(inst, IR::ValueOpcode::ISub32);
+		case O::V_PK_MAX_I16: return PackedInteger16MinMax(inst, IR::ValueOpcode::SMax32, true);
+		case O::V_PK_MIN_I16: return PackedInteger16MinMax(inst, IR::ValueOpcode::SMin32, true);
+		case O::V_PK_MAX_U16: return PackedInteger16MinMax(inst, IR::ValueOpcode::UMax32, false);
+		case O::V_PK_MIN_U16: return PackedInteger16MinMax(inst, IR::ValueOpcode::UMin32, false);
+
+		case O::V_PK_ADD_F16: return PackedFloat16(inst, IR::ValueOpcode::FPAdd32, false, false);
+		case O::V_PK_MUL_F16: return PackedFloat16(inst, IR::ValueOpcode::FPMul32, false, false);
+		case O::V_PK_MIN_F16: return PackedFloat16(inst, IR::ValueOpcode::FPMin32, false, true);
+		case O::V_PK_MAX_F16: return PackedFloat16(inst, IR::ValueOpcode::FPMax32, false, true);
+		case O::V_PK_FMA_F16: return PackedFloat16(inst, IR::ValueOpcode::FPFma32, false, false);
+		case O::V_PK_FMAC_F16: return PackedFloat16(inst, IR::ValueOpcode::FPFma32, true, false);
+
+		case O::V_ADD_F16: return Float16Binary(inst, IR::ValueOpcode::FPAdd32, false);
+		case O::V_SUB_F16: return Float16Binary(inst, IR::ValueOpcode::FPSub32, false);
+		case O::V_SUBREV_F16: return Float16Binary(inst, IR::ValueOpcode::FPSub32, true);
+		case O::V_MUL_F16: return Float16Binary(inst, IR::ValueOpcode::FPMul32, false);
+		case O::V_MIN_F16: return Float16Binary(inst, IR::ValueOpcode::FPMin32, false);
+		case O::V_MAX_F16: return Float16Binary(inst, IR::ValueOpcode::FPMax32, false);
+		case O::V_FMAC_F16: return Float16Ternary(inst, IR::ValueOpcode::FPFma32, true, false);
+		case O::V_FMAMK_F16:
+		case O::V_FMAAK_F16:
+		case O::V_FMA_F16: return Float16Ternary(inst, IR::ValueOpcode::FPFma32, false, false);
+		case O::V_MAD_MIXLO_F16:
+		case O::V_MAD_MIXHI_F16: return Float16Ternary(inst, IR::ValueOpcode::FPFma32, false, true);
+		case O::V_RCP_F16: return Float16Unary(inst, IR::ValueOpcode::FPRecip32, false);
+		case O::V_SQRT_F16: return Float16Unary(inst, IR::ValueOpcode::FPSqrt, true);
+		case O::V_RSQ_F16: return Float16Unary(inst, IR::ValueOpcode::FPRecipSqrt32, true);
+		case O::V_LOG_F16: return Float16Unary(inst, IR::ValueOpcode::FPLog2, true);
+		case O::V_EXP_F16: return Float16Unary(inst, IR::ValueOpcode::FPExp2, false);
+		case O::V_FLOOR_F16: return Float16Unary(inst, IR::ValueOpcode::FPFloor32, false);
+		case O::V_CEIL_F16: return Float16Unary(inst, IR::ValueOpcode::FPCeil32, false);
+		case O::V_TRUNC_F16: return Float16Unary(inst, IR::ValueOpcode::FPTrunc32, false);
+		case O::V_RNDNE_F16: return Float16Unary(inst, IR::ValueOpcode::FPRoundEven32, false);
+		case O::V_MIN3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMinTri32, false, false);
+		case O::V_MAX3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMaxTri32, false, false);
+		case O::V_MED3_F16: return Float16Ternary(inst, IR::ValueOpcode::FPMedTri32, false, false);
+
+		case O::V_FREXP_MANT_F32: return V_FREXP_MANT_F32(inst);
+		case O::V_RCP_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecip32);
+		case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32);
+		case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32);
+		case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32);
+		case O::V_CEIL_F32: return FloatUnary(inst, IR::ValueOpcode::FPCeil32);
+		case O::V_RNDNE_F32: return FloatUnary(inst, IR::ValueOpcode::FPRoundEven32);
+		case O::V_FLOOR_F32: return FloatUnary(inst, IR::ValueOpcode::FPFloor32);
+		case O::V_EXP_F32: return FloatUnary(inst, IR::ValueOpcode::FPExp2);
+		case O::V_LOG_F32: return FloatUnary(inst, IR::ValueOpcode::FPLog2);
+		case O::V_RSQ_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipSqrt32);
+		case O::V_SQRT_F32: return FloatUnary(inst, IR::ValueOpcode::FPSqrt);
+		case O::V_SIN_F32: return FloatUnary(inst, IR::ValueOpcode::FPSin);
+		case O::V_COS_F32: return FloatUnary(inst, IR::ValueOpcode::FPCos);
+		case O::V_ADD_F32: return FloatBinary(inst, IR::ValueOpcode::FPAdd32, false);
+		case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false);
+		case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true);
+		case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
+		case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
+		case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);
+		case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false);
+		case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true);
+		case O::V_MADMK_F32:
+		case O::V_MADAK_F32:
+		case O::V_MAD_F32:
+		case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true);
+		case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false);
+		case O::V_MAX3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMaxTri32, false, false);
+		case O::V_MED3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMedTri32, false, false);
+		case O::V_DOT2C_F32_F16: return V_DOT2C_F32_F16(inst);
+		case O::V_CUBEID_F32: return V_CUBEID_F32(inst);
+		case O::V_CUBESC_F32: return V_CUBESC_F32(inst);
+		case O::V_CUBETC_F32: return V_CUBETC_F32(inst);
+		case O::V_CUBEMA_F32: return V_CUBEMA_F32(inst);
+
+		case O::V_MUL_LO_U32:
+		case O::V_MUL_LO_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false);
+		case O::V_MUL_HI_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMulHi, IR::Type::U32, false, false, false);
+		case O::V_MUL_HI_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMulHi, IR::Type::U32, false, false, false);
+		case O::V_ADD_NC_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::IAdd32, IR::Type::U32, false, false, false);
+		case O::V_SUB_NC_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::ISub32, IR::Type::U32, false, false, false);
+		case O::V_SUBREV_NC_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::ISub32, IR::Type::U32, true, false, false);
+		case O::V_MIN_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMin32, IR::Type::U32, false, false, false);
+		case O::V_MAX_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMax32, IR::Type::U32, false, false, false);
+		case O::V_MIN_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMin32, IR::Type::U32, false, false, false);
+		case O::V_MAX_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMax32, IR::Type::U32, false, false, false);
+		case O::V_MIN3_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMinTri32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_MAX3_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMaxTri32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_MED3_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMedTri32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_MIN3_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMinTri32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_MAX3_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMaxTri32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_MED3_U32:
+			return SimpleInteger(inst, IR::ValueOpcode::UMedTri32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_AND_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseAnd32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_OR_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseOr32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_XOR_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseXor32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_NOT_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitwiseNot32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_BFREV_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_FFBL_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false,
+			                     false);
+		case O::V_LSHL_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical32, IR::Type::U32, false,
+			                     true, false);
+		case O::V_LSHLREV_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical32, IR::Type::U32, true,
+			                     true, false);
+		case O::V_LSHR_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical32, IR::Type::U32, false,
+			                     true, false);
+		case O::V_LSHRREV_B32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical32, IR::Type::U32, true,
+			                     true, false);
+		case O::V_ASHR_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic32, IR::Type::U32,
+			                     false, true, false);
+		case O::V_ASHRREV_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic32, IR::Type::U32, true,
+			                     true, false);
+		case O::V_LSHLREV_B64:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftLeftLogical64, IR::Type::U64, true,
+			                     false, false);
+		case O::V_LSHRREV_B64:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, true,
+			                     false, false);
+
+		case O::V_XNOR_B32:
+			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, false);
+		case O::V_AND_OR_B32: return V_AND_OR_B32(inst);
+		case O::V_OR3_B32: return V_OR3_B32(inst);
+		case O::V_XOR3_B32: return V_XOR3_B32(inst);
+		case O::V_FFBH_U32: return V_FFBH_U32(inst);
+
+		case O::V_MAD_I32_I24: return Integer24(inst, true, true);
+		case O::V_MAD_U32_U24: return Integer24(inst, false, true);
+		case O::V_MUL_I32_I24: return Integer24(inst, true, false);
+		case O::V_MUL_U32_U24: return Integer24(inst, false, false);
+		case O::V_MAD_U64_U32: return V_MAD_U64_U32(inst);
+		case O::V_SAD_U32: return V_SAD_U32(inst);
+		case O::V_ADD3_U32: return V_ADD3_U32(inst);
+		case O::V_BCNT_U32_B32: return V_BCNT_U32_B32(inst);
+		case O::V_MBCNT_LO_U32_B32: return V_MBCNT_U32_B32(inst, true);
+		case O::V_MBCNT_HI_U32_B32: return V_MBCNT_U32_B32(inst, false);
+		case O::V_BFM_B32: return BFM_B32(inst);
+		case O::V_BFE_U32: return V_BFE_U32(inst, false);
+		case O::V_BFE_I32: return V_BFE_U32(inst, true);
+		case O::V_BFI_B32: return V_BFI_B32(inst);
+		case O::V_ALIGNBIT_B32: return V_ALIGNBIT_B32(inst);
+		case O::V_ALIGNBYTE_B32: return V_ALIGNBYTE_B32(inst);
+		case O::V_LSHL_ADD_U32: return V_LSHL_ADD_U32(inst);
+		case O::V_ADD_LSHL_U32: return V_ADD_LSHL_U32(inst);
+		case O::V_XAD_U32: return V_XAD_U32(inst);
+		case O::V_LSHL_OR_B32: return V_LSHL_OR_B32(inst);
+		case O::V_CNDMASK_B32: return V_CNDMASK_B32(inst);
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/ir/Program.cpp

@@ -1,11 +1,13 @@
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 #include "graphics/shader/shader.h"
 
 #include <fmt/format.h>
 #include <map>
+#include <new>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
@@ -50,7 +52,7 @@ bool IsRuntimeRead(ValueOpcode opcode) {
 	       AddressOpcodeInfoOf(opcode).access == AddressAccess::Read;
 }
 
-bool EquivalentValue(const ValueProgram& program, Value left, Value right,
+bool EquivalentValue(const Program& program, Value left, Value right,
                      std::vector<std::pair<const Inst*, const Inst*>>& visited) {
 	left  = left.Resolve();
 	right = right.Resolve();
@@ -93,7 +95,7 @@ bool EquivalentValue(const ValueProgram& program, Value left, Value right,
 
 } // namespace
 
-ValueProgram::~ValueProgram() {
+Program::~Program() {
 	// Values may cross block boundaries. Detach all arguments before any block starts destroying
 	// its instruction storage so reverse-use links always point to live definitions.
 	for (auto* block: blocks) {
@@ -103,12 +105,20 @@ ValueProgram::~ValueProgram() {
 	}
 }
 
-bool EquivalentValue(const ValueProgram& program, Value left, Value right) {
+Program& Program::operator=(Program&& other) noexcept {
+	if (this != &other) {
+		this->~Program();
+		new (this) Program(std::move(other));
+	}
+	return *this;
+}
+
+bool EquivalentValue(const Program& program, Value left, Value right) {
 	std::vector<std::pair<const Inst*, const Inst*>> visited;
 	return EquivalentValue(program, left, right, visited);
 }
 
-Value ResolveInvariantPhi(const ValueProgram& program, Value value) {
+Value ResolveInvariantPhi(const Program& program, Value value) {
 	value            = value.Resolve();
 	const auto* root = value.TryInstruction();
 	if (root == nullptr || root->GetOpcode() != ValueOpcode::Phi) {
@@ -139,7 +149,7 @@ Value ResolveInvariantPhi(const ValueProgram& program, Value value) {
 	return invariant;
 }
 
-bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error) {
+bool ValidateProgram(const Program& program, bool require_ssa, std::string* error) {
 	if (program.blocks.size() != program.block_info.size() ||
 	    program.blocks.size() != program.block_storage.size()) {
 		return Fail(error, "value IR block storage is inconsistent");
@@ -591,14 +601,14 @@ bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::st
 	return true;
 }
 
-void ResolveControlFlowIdentities(ValueProgram& program) {
+void ResolveControlFlowIdentities(Program& program) {
 	for (auto& info: program.block_info) {
 		info.condition       = info.condition.Resolve();
 		info.indirect_target = info.indirect_target.Resolve();
 	}
 }
 
-std::string ValueProgramToString(const ValueProgram& program) {
+std::string ProgramToString(const Program& program) {
 	std::map<const Inst*, size_t> ids;
 	size_t                        next_id = 1;
 	for (const auto* block: program.blocks) {

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -6,7 +6,7 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
-#include "graphics/shader/recompiler/ir/opcodes/Opcodes.h"
+#include "graphics/shader/recompiler/ir/Block.h"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h"
 #include "graphics/shader/shader.h"
 
@@ -19,47 +19,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-struct ValueProgram;
-
-enum class RegisterFile { Scalar, Vector, Vcc, Exec, Scc, M0 };
-
-struct Register {
-	RegisterFile file  = RegisterFile::Scalar;
-	uint32_t     index = 0;
-
-	bool operator==(const Register& other) const {
-		return file == other.file && index == other.index;
-	}
-};
-
-enum class OperandKind { Register, ImmediateU32, PcRelativeU32, PcRelativeHighU32, Null };
-
-struct Operand {
-	OperandKind kind = OperandKind::Null;
-	Register    reg;
-	uint32_t    imm                = 0;
-	bool        float_inline       = false;
-	bool        sext_64            = false;
-	uint32_t    sdwa_sel           = 6;
-	uint32_t    sdwa_dst_unused    = 2;
-	uint32_t    omod               = 0;
-	bool        sdwa_sext          = false;
-	bool        op_sel             = false;
-	bool        op_sel_hi          = false;
-	bool        negate             = false;
-	bool        negate_hi          = false;
-	bool        absolute           = false;
-	bool        clamp              = false;
-	uint32_t    dpp_ctrl           = 0;
-	uint32_t    dpp_row_mask       = 0xf;
-	uint32_t    dpp_bank_mask      = 0xf;
-	bool        dpp_fetch_inactive = false;
-	bool        dpp_bound_ctrl     = false;
-	bool        dpp                = false;
-
-	bool operator==(const Operand& other) const = default;
-};
-
 enum class ResourceKind {
 	None,
 	ScalarBuffer,
@@ -144,44 +103,6 @@ struct ExportInfo {
 	bool operator==(const ExportInfo& other) const = default;
 };
 
-struct InputInfo {
-	uint32_t attr               = 0;
-	uint32_t chan               = 0;
-	uint32_t component_count    = 1;
-	uint32_t interpolation_mode = 3;
-
-	bool operator==(const InputInfo& other) const = default;
-};
-
-enum class SaveexecMode { And, Orn2, Andn1 };
-
-struct Instruction {
-	uint32_t     pc = 0;
-	Opcode       op = Opcode::MoveU32;
-	Operand      dst;
-	Operand      dst2;
-	Operand      src[4];
-	uint32_t     src_count = 0;
-	MemoryInfo   memory;
-	ExportInfo   export_info;
-	InputInfo    input_info;
-	SaveexecMode saveexec_mode = SaveexecMode::And;
-
-	bool operator==(const Instruction& other) const = default;
-};
-
-struct BasicBlock {
-	uint32_t                 id         = 0;
-	uint32_t                 start_pc   = 0;
-	uint32_t                 end_pc     = 0;
-	uint32_t                 inst_begin = 0;
-	uint32_t                 inst_end   = 0;
-	std::vector<uint32_t>    predecessors;
-	std::vector<uint32_t>    successors;
-	CFG::Terminator          terminator;
-	std::vector<Instruction> instructions;
-};
-
 struct DescriptorValue {
 	std::array<uint32_t, 8> dwords      = {};
 	uint32_t                dword_count = 0;
@@ -522,6 +443,7 @@ struct ShaderInfo {
 	std::vector<SampledResourcePair> sampled_pairs;
 	std::vector<StageInput>          inputs;
 	std::vector<StageOutput>         outputs;
+	std::array<uint8_t, 32>          vertex_fetch_components {};
 	int32_t                          vertex_offset_sgpr = -1;
 	bool                             has_bitwise_xor    = false;
 
@@ -539,7 +461,50 @@ struct SpirvRequirements {
 	bool pixel_valid_mask             = false;
 };
 
+struct BlockInfo {
+	uint32_t        id       = 0;
+	uint32_t        start_pc = 0;
+	uint32_t        end_pc   = 0;
+	CFG::Terminator terminator;
+	Value           condition;
+	Value           indirect_target;
+};
+
+struct DescriptorSource {
+	struct IndirectImage {
+		uint32_t material_source = 0;
+		uint32_t heap_source     = 0;
+		uint32_t selector_stride = 0;
+		uint32_t selector_offset = 0;
+		uint32_t key_arg         = 0;
+
+		bool operator==(const IndirectImage& other) const = default;
+	};
+
+	std::array<Value, 8>         dwords {};
+	uint32_t                     dword_count = 0;
+	std::optional<IndirectImage> indirect_image;
+
+	bool operator==(const DescriptorSource& other) const = default;
+};
+
+struct SrtRead {
+	Value    value;
+	uint32_t flat_offset = 0;
+	uint32_t use_pc      = 0;
+
+	bool operator==(const SrtRead& other) const = default;
+};
+
 struct Program {
+	Program() = default;
+	~Program();
+
+	Program(const Program&)            = delete;
+	Program& operator=(const Program&) = delete;
+	Program(Program&&) noexcept         = default;
+	Program& operator=(Program&& other) noexcept;
+
 	ShaderType                    stage               = ShaderType::Unknown;
 	uint64_t                      shader_hash         = 0;
 	uint32_t                      wave_size           = 64;
@@ -549,8 +514,17 @@ struct Program {
 	bool                          dispatcher_fallback = false;
 	CFG::FailureKind              cfg_failure_kind    = CFG::FailureKind::None;
 	std::string                   fallback_reason;
-	std::vector<BasicBlock>       blocks;
-	std::shared_ptr<ValueProgram> values;
+	std::vector<std::unique_ptr<Block>> block_storage;
+	BlockList                           blocks;
+	std::vector<BlockInfo>              block_info;
+	// Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and
+	// storage-image swizzles. Typed memory instructions carry a dense index into these shader-local
+	// tables until those fields are consumed by emission.
+	std::vector<MemoryInfo>       memory_info;
+	std::vector<ExportInfo>       export_info;
+	std::vector<DescriptorSource> descriptor_sources;
+	std::vector<SrtRead>          srt_reads;
+	std::vector<Value>            dynamic_reads;
 	bool                          srt_plan_complete = false;
 	ShaderInfo                    info;
 	bool                          resource_tracking_complete = false;
@@ -561,15 +535,13 @@ struct Program {
 	std::optional<SpirvRequirements> spirv_requirements;
 };
 
-bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage,
-                  uint32_t wave_size, Program& program, std::string* error);
-
-std::string RegisterToString(Register reg);
-std::string OperandToString(const Operand& operand);
-std::string ExportTargetKindToString(ExportTargetKind kind);
-std::string InstructionToString(const Instruction& inst);
 std::string ProgramToString(const Program& program);
 
+bool  ValidateProgram(const Program& program, bool require_ssa, std::string* error);
+void  ResolveControlFlowIdentities(Program& program);
+bool  EquivalentValue(const Program& program, Value left, Value right);
+Value ResolveInvariantPhi(const Program& program, Value value);
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_ */

src/graphics/shader/recompiler/ir/ShaderIRInternal.h

@@ -1,51 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_
-#define EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_
-
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-#include <optional>
-
-namespace Libs::Graphics::ShaderRecompiler::IR {
-
-void SetError(std::string* error, const char* message);
-
-std::optional<Opcode> LookupIrOpcode(Decoder::Opcode opcode);
-bool                  IsReversedBinary(Decoder::Opcode opcode);
-bool                  IsVectorCarryOutOpcode(Decoder::Opcode opcode);
-bool                  ScalarResultWritesSccNonZero(Decoder::Opcode opcode);
-bool                  ScalarResultIs64Bit(Decoder::Opcode opcode);
-
-Operand  MakeSccOperand();
-Operand  MakeImmediateU32(uint32_t value);
-Operand  MakePcRelativeU32(uint32_t value);
-Operand  MakePcRelativeHighU32(uint32_t value);
-bool     AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block,
-                                      std::string* error);
-uint32_t VectorByteConvertIndex(Decoder::Opcode opcode);
-void     CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand);
-bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error);
-bool LowerSourceOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error);
-void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst);
-uint32_t         ResourceIndexFromOperand(const Decoder::Operand& operand);
-MemoryInfo       MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind);
-uint32_t         RawScalarLoadBase(const Decoder::Operand& operand);
-void             ClearRegisterOffsetModifiers(Decoder::Operand& operand);
-bool             TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code);
-bool             TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
-                                            Decoder::Operand& result);
-Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
-MemoryInfo       OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
-                                  uint32_t dword_index);
-MemoryInfo       ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
-                                      uint32_t byte_offset);
-ResourceKind     FlatSegmentResourceKind(uint32_t segment);
-const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index);
-Decoder::Operand        M0Operand();
-
-bool IsMemoryOpcode(Decoder::Opcode opcode);
-bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& block,
-                            std::string* error);
-
-} // namespace Libs::Graphics::ShaderRecompiler::IR
-
-#endif // EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_

src/graphics/shader/recompiler/ir/ShaderIRLog.cpp

@@ -1,214 +0,0 @@
-#include "common/assert.h"
-#include "graphics/shader/recompiler/BufferFormat.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ShaderIRInternal.h"
-
-#include <algorithm>
-#include <fmt/format.h>
-#include <utility>
-
-namespace Libs::Graphics::ShaderRecompiler::IR {
-
-std::string RegisterToString(Register reg) {
-	switch (reg.file) {
-		case RegisterFile::Scalar: return fmt::format("s{}", reg.index);
-		case RegisterFile::Vector: return fmt::format("v{}", reg.index);
-		case RegisterFile::Vcc:
-			return reg.index == 0 ? std::string("vcc_lo") : std::string("vcc_hi");
-		case RegisterFile::Exec:
-			return reg.index == 0 ? std::string("exec_lo") : std::string("exec_hi");
-		case RegisterFile::Scc:
-			if (reg.index == static_cast<uint32_t>(Decoder::OperandKind::Scc)) {
-				return "scc";
-			}
-			if (reg.index == static_cast<uint32_t>(Decoder::OperandKind::VccZ)) {
-				return "vccz";
-			}
-			if (reg.index == static_cast<uint32_t>(Decoder::OperandKind::ExecZ)) {
-				return "execz";
-			}
-			return fmt::format("scc_like_{}", reg.index);
-		case RegisterFile::M0: return "m0";
-		default: return "unknown_reg";
-	}
-}
-
-std::string OperandToString(const Operand& operand) {
-	std::string text;
-	switch (operand.kind) {
-		case OperandKind::Register: text = RegisterToString(operand.reg); break;
-		case OperandKind::ImmediateU32: text = fmt::format("0x{:08x}", operand.imm); break;
-		case OperandKind::PcRelativeU32: text = fmt::format("pc_rel(0x{:08x})", operand.imm); break;
-		case OperandKind::PcRelativeHighU32:
-			text = fmt::format("pc_rel_hi(0x{:08x})", operand.imm);
-			break;
-		case OperandKind::Null: text = "null"; break;
-		default: text = "unknown"; break;
-	}
-	if (operand.sdwa_sel != 6 || operand.sdwa_sext) {
-		text += fmt::format(".sdwa(sel={},sext={})", operand.sdwa_sel, operand.sdwa_sext ? 1u : 0u);
-	}
-	if (operand.absolute) {
-		text += ".abs";
-	}
-	if (operand.negate) {
-		text += ".neg";
-	}
-	if (operand.op_sel || operand.op_sel_hi || operand.negate_hi) {
-		text += fmt::format(".opsel(lo={},hi={},neghi={})", operand.op_sel ? 1u : 0u,
-		                    operand.op_sel_hi ? 1u : 0u, operand.negate_hi ? 1u : 0u);
-	}
-	if (operand.omod != 0) {
-		text += fmt::format(".omod({})", operand.omod);
-	}
-	if (operand.clamp) {
-		text += ".clamp";
-	}
-	if (operand.dpp) {
-		text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl,
-		                    operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u);
-	}
-	return text;
-}
-
-std::string ResourceKindToString(ResourceKind kind) {
-	switch (kind) {
-		case ResourceKind::ScalarBuffer: return "scalar_buffer";
-		case ResourceKind::ScalarAddress: return "scalar_address";
-		case ResourceKind::Buffer: return "buffer";
-		case ResourceKind::Flat: return "flat";
-		case ResourceKind::Global: return "global";
-		case ResourceKind::Scratch: return "scratch";
-		case ResourceKind::Lds: return "lds";
-		case ResourceKind::Gds: return "gds";
-		case ResourceKind::Image: return "image";
-		case ResourceKind::ImageUint: return "image_uint";
-		case ResourceKind::StorageImage: return "storage_image";
-		case ResourceKind::StorageImageUint: return "storage_image_uint";
-		case ResourceKind::Sampler: return "sampler";
-		default: return "none";
-	}
-}
-
-std::string MemoryInfoToString(const MemoryInfo& mem) {
-	if (mem.kind == ResourceKind::None) {
-		return "";
-	}
-	std::string text = fmt::format(
-	    " ; {} resource={} sampler={} offset={} offset2={} dmask=0x{:x} data_dwords={} "
-	    "data_bits={} component={}/{} dfmt={} nfmt={} signed={} typed={} formatted={} segment={} "
-	    "glc={} slc={} idxen={} offen={} image_flags=0x{:x} image_dim={} image_addr={} "
-	    "image_mip={} image_r128={}",
-	    ResourceKindToString(mem.kind).c_str(), mem.resource, mem.sampler, mem.offset,
-	    mem.secondary_offset, mem.dmask, mem.data_dwords, mem.data_bits, mem.component_index,
-	    mem.component_count, mem.data_format, mem.number_format, mem.data_signed ? 1u : 0u,
-	    mem.typed ? 1u : 0u, mem.formatted ? 1u : 0u, mem.memory_segment, mem.glc ? 1u : 0u,
-	    mem.slc ? 1u : 0u, mem.idxen ? 1u : 0u, mem.offen ? 1u : 0u, mem.image_sample_flags,
-	    Decoder::ImageDimensionToString(mem.image_dimension), mem.image_address_components,
-	    mem.image_has_mip ? 1u : 0u, mem.image_r128 ? 1u : 0u);
-	if (mem.image_nsa_dwords != 0) {
-		text += fmt::format(" image_nsa_dwords={} image_nsa_addr=", mem.image_nsa_dwords);
-		const auto count =
-		    std::min<uint32_t>(mem.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
-		for (uint32_t i = 0; i < count; i++) {
-			if (i != 0) {
-				text += ",";
-			}
-			text += fmt::format("{}", mem.image_nsa_addr[i]);
-		}
-	}
-	return text;
-}
-
-std::string ExportInfoToString(const ExportInfo& exp) {
-	return fmt::format(" ; target={} index={} raw=0x{:02x} en=0x{:x} done={} compr={} vm={}",
-	                   ExportTargetKindToString(exp.kind).c_str(), exp.index, exp.target, exp.en,
-	                   exp.done ? 1u : 0u, exp.compr ? 1u : 0u, exp.vm ? 1u : 0u);
-}
-
-std::string InputInfoToString(Opcode op, const InputInfo& input) {
-	if (op != Opcode::LoadInputF32) {
-		return "";
-	}
-	return fmt::format(" ; input_attr={} input_chan={} input_components={}", input.attr, input.chan,
-	                   input.component_count);
-}
-
-std::string TerminatorToString(const CFG::Terminator& term) {
-	return fmt::format(
-	    "term={} cond={} true={} false={} merge={} continue={} loop={} indirect_sgpr={} "
-	    "indirect_selector={} indirect_targets={} selector_values={}",
-	    static_cast<uint32_t>(term.kind), CFG::BranchConditionToString(term.condition).c_str(),
-	    term.true_block, term.false_block, term.merge_block, term.continue_block,
-	    term.loop_header ? 1u : 0u, term.indirect_pc_sgpr, term.indirect_selector_code,
-	    static_cast<uint32_t>(term.indirect_targets.size()),
-	    static_cast<uint32_t>(term.indirect_selector_values.size()));
-}
-
-std::string ExportSourcesToString(const Instruction& inst) {
-	std::string text;
-	for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
-		text += i == 0 ? " " : ", ";
-		text += OperandToString(inst.src[i]);
-	}
-	return text;
-}
-
-std::string InstructionToString(const Instruction& inst) {
-	std::string text = fmt::format("0x{:08x}: ", inst.pc);
-	if (inst.op == Opcode::Export) {
-		text += "Export";
-		text += ExportSourcesToString(inst);
-		text += ExportInfoToString(inst.export_info);
-		return text;
-	}
-	text += OpcodeName(inst.op);
-	text += " ";
-	text += OperandToString(inst.dst);
-	if (inst.dst2.kind != OperandKind::Null) {
-		text += ", ";
-		text += OperandToString(inst.dst2);
-	}
-	for (uint32_t i = 0; i < inst.src_count; i++) {
-		text += i == 0 ? ", " : ", ";
-		text += OperandToString(inst.src[i]);
-	}
-	text += MemoryInfoToString(inst.memory);
-	text += InputInfoToString(inst.op, inst.input_info);
-	return text;
-}
-
-std::string ExportTargetKindToString(ExportTargetKind kind) {
-	switch (kind) {
-		case ExportTargetKind::Null: return "null";
-		case ExportTargetKind::Position: return "position";
-		case ExportTargetKind::Primitive: return "primitive";
-		case ExportTargetKind::Parameter: return "parameter";
-		case ExportTargetKind::Mrt: return "mrt";
-		case ExportTargetKind::MrtZ: return "mrtz";
-		default: return "unknown";
-	}
-}
-
-std::string ProgramToString(const Program& program) {
-	std::string text;
-	text += fmt::format("mode={} fallback={} reason={}\n",
-	                    program.dispatcher_fallback ? "dispatcher" : "structured",
-	                    CFG::FailureKindToString(program.cfg_failure_kind).c_str(),
-	                    program.fallback_reason.c_str());
-	for (const auto& block: program.blocks) {
-		text += fmt::format("block_{} pc=0x{:08x} end=0x{:08x} preds={} succs={} {}\n", block.id,
-		                    block.start_pc, block.end_pc,
-		                    static_cast<uint32_t>(block.predecessors.size()),
-		                    static_cast<uint32_t>(block.successors.size()),
-		                    TerminatorToString(block.terminator).c_str());
-		for (const auto& inst: block.instructions) {
-			text += "  ";
-			text += InstructionToString(inst);
-			text += "\n";
-		}
-	}
-	return text;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp

@@ -1,812 +0,0 @@
-#include "common/assert.h"
-#include "graphics/shader/recompiler/BufferFormat.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ShaderIRInternal.h"
-
-#include <algorithm>
-#include <fmt/format.h>
-#include <utility>
-
-namespace Libs::Graphics::ShaderRecompiler::IR {
-namespace {
-
-Opcode BufferLoadOpcode(const Decoder::Instruction& decoded) {
-	switch (decoded.data_bits) {
-		case 8u: return decoded.data_signed ? Opcode::BufferLoadSbyte : Opcode::BufferLoadUbyte;
-		case 16u: return decoded.data_signed ? Opcode::BufferLoadSshort : Opcode::BufferLoadUshort;
-		default: return Opcode::BufferLoadDword;
-	}
-}
-
-Opcode BufferStoreOpcode(uint32_t data_bits) {
-	switch (data_bits) {
-		case 8u: return Opcode::BufferStoreByte;
-		case 16u: return Opcode::BufferStoreShort;
-		default: return Opcode::BufferStoreDword;
-	}
-}
-
-Opcode DsReadOpcode(const Decoder::Instruction& decoded) {
-	switch (decoded.data_bits) {
-		case 8u: return decoded.data_signed ? Opcode::DsReadSbyte : Opcode::DsReadUbyte;
-		case 16u: return decoded.data_signed ? Opcode::DsReadSshort : Opcode::DsReadUshort;
-		default: return Opcode::DsReadB32;
-	}
-}
-
-Opcode FlatLoadOpcode(const Decoder::Instruction& decoded) {
-	switch (decoded.data_bits) {
-		case 8u: return decoded.data_signed ? Opcode::FlatLoadSbyte : Opcode::FlatLoadUbyte;
-		case 16u: return decoded.data_signed ? Opcode::FlatLoadSshort : Opcode::FlatLoadUshort;
-		default: return Opcode::FlatLoadDword;
-	}
-}
-
-Opcode FlatStoreOpcode(uint32_t data_bits) {
-	switch (data_bits) {
-		case 8u: return Opcode::FlatStoreByte;
-		case 16u: return Opcode::FlatStoreShort;
-		default: return Opcode::FlatStoreDword;
-	}
-}
-
-Opcode DsWriteOpcode(uint32_t data_bits) {
-	switch (data_bits) {
-		case 8u: return Opcode::DsWriteByte;
-		case 16u: return Opcode::DsWriteShort;
-		default: return Opcode::DsWriteB32;
-	}
-}
-
-Opcode LoweredImageOpcode(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::IMAGE_GET_RESINFO: return Opcode::ImageGetResinfo;
-		case Decoder::Opcode::IMAGE_GET_LOD: return Opcode::ImageGetLod;
-		case Decoder::Opcode::IMAGE_LOAD:
-		case Decoder::Opcode::IMAGE_LOAD_MIP: return Opcode::ImageLoad;
-		case Decoder::Opcode::IMAGE_GATHER4_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_C:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4H: return Opcode::ImageGather4;
-		default: return Opcode::ImageSample;
-	}
-}
-
-bool LowerScalarMemoryLoadDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
-                                std::string* error) {
-	uint32_t destination_code = 0;
-	const auto alignment = decoded.data_dwords == 1u ? 1u
-	                       : decoded.data_dwords == 2u ? 2u
-	                                                    : 4u;
-	if (!TryGetEncodedScalarCode(decoded.dst, destination_code) ||
-	    destination_code % alignment != 0u ||
-	    destination_code + decoded.data_dwords > 108u) {
-		SetError(error, "scalar-memory destination has an invalid aligned SGPR span");
-		return false;
-	}
-	uint32_t base_code = 0;
-	const auto base_alignment = op == Opcode::SLoadDword ? 2u : 4u;
-	const auto base_width     = op == Opcode::SLoadDword ? 2u : 4u;
-	if (!TryGetEncodedScalarCode(decoded.src0, base_code) ||
-	    base_code % base_alignment != 0u || base_code + base_width > 108u) {
-		SetError(error, "scalar-memory base has an invalid aligned SGPR span");
-		return false;
-	}
-	Instruction inst;
-	inst.pc        = decoded.pc;
-	inst.op        = op;
-	inst.src_count = 1;
-	const auto kind =
-	    op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer;
-	inst.memory = MemoryInfoFromDecoded(decoded, kind);
-	// Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR
-	// descriptor resource index.
-	inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
-	                                                : ResourceIndexFromOperand(decoded.src0);
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(decoded.src1, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction& inst,
-                               uint32_t first_src, std::string* error) {
-	uint32_t src = first_src;
-	if (decoded.idxen) {
-		if (!LowerSourceOperand(decoded.src0, inst.src[src++], error)) {
-			return false;
-		}
-	}
-	if (decoded.offen) {
-		const auto offset_source =
-		    decoded.idxen ? OffsetDecodedRegister(decoded.src0, 1) : decoded.src0;
-		if (!LowerSourceOperand(offset_source, inst.src[src++], error)) {
-			return false;
-		}
-	}
-	if (!LowerSourceOperand(decoded.src2, inst.src[src++], error)) {
-		return false;
-	}
-	inst.src_count = src;
-	return true;
-}
-
-bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	Instruction inst;
-	inst.pc     = decoded.pc;
-	inst.op     = BufferLoadOpcode(decoded);
-	inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerBufferAddressSources(decoded, inst, 0, error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	Instruction inst;
-	inst.pc       = decoded.pc;
-	inst.op       = BufferStoreOpcode(decoded.data_bits);
-	inst.memory   = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
-	inst.dst.kind = OperandKind::Null;
-	if (!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
-	    !LowerBufferAddressSources(decoded, inst, 1, error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerBufferAtomicDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
-                            std::string* error) {
-	Instruction inst;
-	inst.pc       = decoded.pc;
-	inst.op       = op;
-	inst.memory   = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
-	inst.dst.kind = OperandKind::Null;
-	if ((decoded.glc && !LowerRegisterOperand(decoded.dst, inst.dst, error)) ||
-	    !LowerSourceOperand(decoded.dst, inst.src[0], error) ||
-	    !LowerBufferAddressSources(decoded, inst, 1, error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) {
-	return decoded.gds ? ResourceKind::Gds : ResourceKind::Lds;
-}
-
-bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto  ir_op = DsReadOpcode(decoded);
-	Instruction inst;
-	inst.pc        = decoded.pc;
-	inst.op        = ir_op;
-	inst.src_count = 1;
-	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read,
-                  std::string* error) {
-	Instruction inst;
-	inst.pc                 = decoded.pc;
-	inst.op                 = Opcode::DsRead2B32;
-	inst.src_count          = 1;
-	inst.memory             = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.memory.data_dwords = dwords_per_read * 2u;
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
-                      bool return_old, std::string* error) {
-	Instruction inst;
-	inst.pc        = decoded.pc;
-	inst.op        = op;
-	inst.src_count = 2;
-	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.dst.kind  = return_old ? OperandKind::Register : OperandKind::Null;
-	if ((return_old && !LowerRegisterOperand(decoded.dst, inst.dst, error)) ||
-	    !LowerSourceOperand(decoded.src1, inst.src[0], error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsSwizzleB32(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	Instruction inst;
-	inst.pc          = decoded.pc;
-	inst.op          = Opcode::DsSwizzleB32;
-	inst.src_count   = 2;
-	inst.src[1].kind = OperandKind::ImmediateU32;
-	inst.src[1].imm  = decoded.offset & 0xffffu;
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
-                           std::string* error) {
-	Instruction inst;
-	inst.pc              = decoded.pc;
-	inst.op              = op;
-	inst.src_count       = 2;
-	inst.memory          = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.memory.resource = 0;
-	inst.dst.kind        = OperandKind::Null;
-	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[1], error) ||
-	    !LowerSourceOperand(decoded.src2, inst.src[2], error)) {
-		return false;
-	}
-	inst.src_count = 3;
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock& block,
-                           std::string* error) {
-	Instruction inst;
-	inst.pc              = decoded.pc;
-	inst.op              = Opcode::DsWriteAddtidB32;
-	inst.src_count       = 2;
-	inst.memory          = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.memory.resource = 0;
-	inst.dst.kind        = OperandKind::Null;
-	const auto m0        = M0Operand();
-	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
-	    !LowerSourceOperand(m0, inst.src[1], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock& block,
-                          std::string* error) {
-	Instruction inst;
-	inst.pc              = decoded.pc;
-	inst.op              = Opcode::DsReadAddtidB32;
-	inst.src_count       = 1;
-	inst.memory          = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.memory.resource = 0;
-	const auto m0        = M0Operand();
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(m0, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
-                          std::string* error) {
-	Instruction inst;
-	inst.pc        = decoded.pc;
-	inst.op        = op;
-	inst.src_count = 1;
-	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	const auto m0  = M0Operand();
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(m0, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-Opcode ImageAtomicIrOpcode(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::IMAGE_ATOMIC_ADD: return Opcode::AtomicAddU32;
-		case Decoder::Opcode::IMAGE_ATOMIC_UMIN: return Opcode::AtomicUMinU32;
-		case Decoder::Opcode::IMAGE_ATOMIC_UMAX: return Opcode::AtomicUMaxU32;
-		case Decoder::Opcode::IMAGE_ATOMIC_AND: return Opcode::AtomicAndU32;
-		case Decoder::Opcode::IMAGE_ATOMIC_OR: return Opcode::AtomicOrU32;
-		case Decoder::Opcode::IMAGE_ATOMIC_XOR: return Opcode::AtomicXorU32;
-		default: return Opcode::AtomicAddU32;
-	}
-}
-
-bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock& block,
-                         std::string* error) {
-	Instruction inst;
-	inst.pc             = decoded.pc;
-	inst.op             = ImageAtomicIrOpcode(decoded.opcode);
-	inst.src_count      = 2;
-	inst.memory         = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImageUint);
-	inst.memory.sampler = 0;
-	inst.dst.kind       = OperandKind::Null;
-	if ((decoded.glc && !LowerRegisterOperand(decoded.dst, inst.dst, error)) ||
-	    !LowerSourceOperand(decoded.dst, inst.src[0], error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto kind  = FlatSegmentResourceKind(decoded.memory_segment);
-	const auto ir_op = FlatLoadOpcode(decoded);
-	const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	if (decoded.data_bits == 32u && count > 1u) {
-		Instruction inst;
-		inst.pc              = decoded.pc;
-		inst.op              = ir_op;
-		inst.src_count       = decoded.src_count;
-		inst.memory          = MemoryInfoFromDecoded(decoded, kind);
-		inst.memory.resource = 0;
-		if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) {
-			return false;
-		}
-		for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
-			if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i], error)) {
-				return false;
-			}
-		}
-		block.instructions.push_back(inst);
-		return true;
-	}
-
-	for (uint32_t dword = 0; dword < count; dword++) {
-		Instruction inst;
-		inst.pc              = decoded.pc;
-		inst.op              = ir_op;
-		inst.src_count       = decoded.src_count;
-		inst.memory          = OffsetMemoryInfo(decoded, kind, dword);
-		inst.memory.resource = 0;
-		if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, dword), inst.dst, error)) {
-			return false;
-		}
-		for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
-			if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i], error)) {
-				return false;
-			}
-		}
-		block.instructions.push_back(inst);
-	}
-	return true;
-}
-
-bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto kind  = FlatSegmentResourceKind(decoded.memory_segment);
-	const auto ir_op = FlatStoreOpcode(decoded.data_bits);
-	const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	for (uint32_t dword = 0; dword < count; dword++) {
-		Instruction inst;
-		inst.pc              = decoded.pc;
-		inst.op              = ir_op;
-		inst.src_count       = decoded.src_count + 1u;
-		inst.memory          = OffsetMemoryInfo(decoded, kind, dword);
-		inst.memory.resource = 0;
-		inst.dst.kind        = OperandKind::Null;
-		if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, dword), inst.src[0], error)) {
-			return false;
-		}
-		for (uint32_t i = 0; i < decoded.src_count && i + 1u < 3u; i++) {
-			if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i + 1u], error)) {
-				return false;
-			}
-		}
-		block.instructions.push_back(inst);
-	}
-	return true;
-}
-
-bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto  ir_op = DsWriteOpcode(decoded.data_bits);
-	Instruction inst;
-	inst.pc        = decoded.pc;
-	inst.op        = ir_op;
-	inst.src_count = 2;
-	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.dst.kind  = OperandKind::Null;
-	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block,
-                   uint32_t dwords_per_write, std::string* error) {
-	Instruction inst;
-	inst.pc                 = decoded.pc;
-	inst.op                 = Opcode::DsWrite2B32;
-	inst.src_count          = 3;
-	inst.memory             = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
-	inst.memory.data_dwords = dwords_per_write * 2u;
-	inst.dst.kind           = OperandKind::Null;
-	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[1], error) ||
-	    !LowerSourceOperand(decoded.src2, inst.src[2], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock& block,
-                         std::string* error) {
-	Instruction inst;
-	inst.pc = decoded.pc;
-	if (decoded.opcode == Decoder::Opcode::IMAGE_STORE ||
-	    decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP) {
-		inst.op             = Opcode::ImageStore;
-		inst.src_count      = 2;
-		inst.memory         = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImage);
-		inst.memory.sampler = 0;
-		inst.dst.kind       = OperandKind::Null;
-		if (!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
-		    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
-			return false;
-		}
-		block.instructions.push_back(inst);
-		return true;
-	}
-
-	inst.op        = LoweredImageOpcode(decoded.opcode);
-	inst.src_count = 1;
-	inst.memory    = MemoryInfoFromDecoded(decoded, ResourceKind::Image);
-	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
-	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-} // namespace
-
-bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& block,
-                            std::string* error) {
-	switch (decoded.opcode) {
-		case Decoder::Opcode::S_LOAD_DWORD:
-		case Decoder::Opcode::S_LOAD_DWORDX2:
-		case Decoder::Opcode::S_LOAD_DWORDX4:
-		case Decoder::Opcode::S_LOAD_DWORDX8:
-		case Decoder::Opcode::S_LOAD_DWORDX16:
-			return LowerScalarMemoryLoadDword(decoded, block, Opcode::SLoadDword, error);
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORD:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16:
-			return LowerScalarMemoryLoadDword(decoded, block, Opcode::SBufferLoadDword, error);
-		case Decoder::Opcode::BUFFER_LOAD_UBYTE:
-		case Decoder::Opcode::BUFFER_LOAD_SBYTE:
-		case Decoder::Opcode::BUFFER_LOAD_USHORT:
-		case Decoder::Opcode::BUFFER_LOAD_SSHORT:
-		case Decoder::Opcode::BUFFER_LOAD_DWORD:
-		case Decoder::Opcode::BUFFER_LOAD_DWORDX2:
-		case Decoder::Opcode::BUFFER_LOAD_DWORDX3:
-		case Decoder::Opcode::BUFFER_LOAD_DWORDX4:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_X:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW:
-			return LowerBufferLoad(decoded, block, error);
-		case Decoder::Opcode::BUFFER_STORE_DWORD:
-		case Decoder::Opcode::BUFFER_STORE_DWORDX2:
-		case Decoder::Opcode::BUFFER_STORE_DWORDX3:
-		case Decoder::Opcode::BUFFER_STORE_DWORDX4:
-		case Decoder::Opcode::BUFFER_STORE_BYTE:
-		case Decoder::Opcode::BUFFER_STORE_SHORT:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_X:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_XY:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZ:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_X:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW:
-			return LowerBufferStore(decoded, block, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_SWAP:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSwapU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_ADD:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAddU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_SUB:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSubU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_SMIN:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMinI32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_UMIN:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMinU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_SMAX:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMaxI32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_UMAX:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMaxU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_AND:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAndU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_OR:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicOrU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_XOR:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicXorU32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_FMIN:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicFMinF32, error);
-		case Decoder::Opcode::BUFFER_ATOMIC_FMAX:
-			return LowerBufferAtomicDword(decoded, block, Opcode::AtomicFMaxF32, error);
-		case Decoder::Opcode::FLAT_LOAD_UBYTE:
-		case Decoder::Opcode::FLAT_LOAD_SBYTE:
-		case Decoder::Opcode::FLAT_LOAD_USHORT:
-		case Decoder::Opcode::FLAT_LOAD_SSHORT:
-		case Decoder::Opcode::FLAT_LOAD_DWORD:
-		case Decoder::Opcode::FLAT_LOAD_DWORDX2:
-		case Decoder::Opcode::FLAT_LOAD_DWORDX3:
-		case Decoder::Opcode::FLAT_LOAD_DWORDX4: return LowerFlatLoad(decoded, block, error);
-		case Decoder::Opcode::FLAT_STORE_BYTE:
-		case Decoder::Opcode::FLAT_STORE_SHORT:
-		case Decoder::Opcode::FLAT_STORE_DWORD:
-		case Decoder::Opcode::FLAT_STORE_DWORDX2:
-		case Decoder::Opcode::FLAT_STORE_DWORDX3:
-		case Decoder::Opcode::FLAT_STORE_DWORDX4: return LowerFlatStore(decoded, block, error);
-		case Decoder::Opcode::DS_ADD_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, false, error);
-		case Decoder::Opcode::DS_ADD_RTN_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, true, error);
-		case Decoder::Opcode::DS_SUB_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, false, error);
-		case Decoder::Opcode::DS_SUB_RTN_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, true, error);
-		case Decoder::Opcode::DS_MIN_I32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, false, error);
-		case Decoder::Opcode::DS_MIN_RTN_I32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, true, error);
-		case Decoder::Opcode::DS_MAX_I32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, false, error);
-		case Decoder::Opcode::DS_MAX_RTN_I32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, true, error);
-		case Decoder::Opcode::DS_MIN_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, false, error);
-		case Decoder::Opcode::DS_MIN_RTN_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, true, error);
-		case Decoder::Opcode::DS_MAX_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, false, error);
-		case Decoder::Opcode::DS_MAX_RTN_U32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, true, error);
-		case Decoder::Opcode::DS_AND_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, false, error);
-		case Decoder::Opcode::DS_AND_RTN_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, true, error);
-		case Decoder::Opcode::DS_OR_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, false, error);
-		case Decoder::Opcode::DS_OR_RTN_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, true, error);
-		case Decoder::Opcode::DS_XOR_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, false, error);
-		case Decoder::Opcode::DS_XOR_RTN_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, true, error);
-		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
-			return LowerDsAtomicU32(decoded, block, Opcode::AtomicSwapU32, true, error);
-		case Decoder::Opcode::DS_MIN_F32:
-			return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMinF32, error);
-		case Decoder::Opcode::DS_MAX_F32:
-			return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMaxF32, error);
-		case Decoder::Opcode::DS_SWIZZLE_B32: return LowerDsSwizzleB32(decoded, block, error);
-		case Decoder::Opcode::DS_CONSUME:
-			return LowerDsAppendConsume(decoded, block, Opcode::DsConsume, error);
-		case Decoder::Opcode::DS_APPEND:
-			return LowerDsAppendConsume(decoded, block, Opcode::DsAppend, error);
-		case Decoder::Opcode::DS_WRITE_ADDTID_B32:
-			return LowerDsWriteAddtidB32(decoded, block, error);
-		case Decoder::Opcode::DS_READ_ADDTID_B32:
-			return LowerDsReadAddtidB32(decoded, block, error);
-		case Decoder::Opcode::DS_READ2_B32:
-		case Decoder::Opcode::DS_READ2ST64_B32: return LowerDsRead2(decoded, block, 1, error);
-		case Decoder::Opcode::DS_READ2_B64:
-		case Decoder::Opcode::DS_READ2ST64_B64: return LowerDsRead2(decoded, block, 2, error);
-		case Decoder::Opcode::DS_READ_I8:
-		case Decoder::Opcode::DS_READ_U8:
-		case Decoder::Opcode::DS_READ_I16:
-		case Decoder::Opcode::DS_READ_U16:
-		case Decoder::Opcode::DS_READ_U16_D16:
-		case Decoder::Opcode::DS_READ_B32:
-		case Decoder::Opcode::DS_READ_B64:
-		case Decoder::Opcode::DS_READ_B96:
-		case Decoder::Opcode::DS_READ_B128: return LowerDsRead(decoded, block, error);
-		case Decoder::Opcode::DS_WRITE2_B32:
-		case Decoder::Opcode::DS_WRITE2ST64_B32: return LowerDsWrite2(decoded, block, 1, error);
-		case Decoder::Opcode::DS_WRITE2_B64:
-		case Decoder::Opcode::DS_WRITE2ST64_B64: return LowerDsWrite2(decoded, block, 2, error);
-		case Decoder::Opcode::DS_WRITE_B8:
-		case Decoder::Opcode::DS_WRITE_B16:
-		case Decoder::Opcode::DS_WRITE_B32:
-		case Decoder::Opcode::DS_WRITE_B64:
-		case Decoder::Opcode::DS_WRITE_B96:
-		case Decoder::Opcode::DS_WRITE_B128: return LowerDsWrite(decoded, block, error);
-		case Decoder::Opcode::IMAGE_GET_RESINFO:
-		case Decoder::Opcode::IMAGE_GET_LOD:
-		case Decoder::Opcode::IMAGE_LOAD:
-		case Decoder::Opcode::IMAGE_LOAD_MIP:
-		case Decoder::Opcode::IMAGE_STORE:
-		case Decoder::Opcode::IMAGE_STORE_MIP:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_C:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4H:
-		case Decoder::Opcode::IMAGE_SAMPLE: return LowerImageOperation(decoded, block, error);
-		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
-		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
-		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
-		case Decoder::Opcode::IMAGE_ATOMIC_AND:
-		case Decoder::Opcode::IMAGE_ATOMIC_OR:
-		case Decoder::Opcode::IMAGE_ATOMIC_XOR: return LowerImageAtomicU32(decoded, block, error);
-		default:
-			if (error != nullptr) {
-				*error = fmt::format("memory opcode has no specialized IR lowering: {}",
-				                     magic_enum::enum_name(decoded.opcode));
-			}
-			return false;
-	}
-}
-
-bool IsMemoryOpcode(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::S_LOAD_DWORD:
-		case Decoder::Opcode::S_LOAD_DWORDX2:
-		case Decoder::Opcode::S_LOAD_DWORDX4:
-		case Decoder::Opcode::S_LOAD_DWORDX8:
-		case Decoder::Opcode::S_LOAD_DWORDX16:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORD:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX2:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8:
-		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16:
-		case Decoder::Opcode::BUFFER_LOAD_UBYTE:
-		case Decoder::Opcode::BUFFER_LOAD_SBYTE:
-		case Decoder::Opcode::BUFFER_LOAD_USHORT:
-		case Decoder::Opcode::BUFFER_LOAD_SSHORT:
-		case Decoder::Opcode::BUFFER_LOAD_DWORD:
-		case Decoder::Opcode::BUFFER_LOAD_DWORDX2:
-		case Decoder::Opcode::BUFFER_LOAD_DWORDX3:
-		case Decoder::Opcode::BUFFER_LOAD_DWORDX4:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_X:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ:
-		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ:
-		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZW:
-		case Decoder::Opcode::BUFFER_STORE_DWORD:
-		case Decoder::Opcode::BUFFER_STORE_DWORDX2:
-		case Decoder::Opcode::BUFFER_STORE_DWORDX3:
-		case Decoder::Opcode::BUFFER_STORE_DWORDX4:
-		case Decoder::Opcode::BUFFER_STORE_BYTE:
-		case Decoder::Opcode::BUFFER_STORE_SHORT:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_X:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_XY:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZ:
-		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_X:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ:
-		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZW:
-		case Decoder::Opcode::BUFFER_ATOMIC_SWAP:
-		case Decoder::Opcode::BUFFER_ATOMIC_ADD:
-		case Decoder::Opcode::BUFFER_ATOMIC_SUB:
-		case Decoder::Opcode::BUFFER_ATOMIC_SMIN:
-		case Decoder::Opcode::BUFFER_ATOMIC_UMIN:
-		case Decoder::Opcode::BUFFER_ATOMIC_SMAX:
-		case Decoder::Opcode::BUFFER_ATOMIC_UMAX:
-		case Decoder::Opcode::BUFFER_ATOMIC_AND:
-		case Decoder::Opcode::BUFFER_ATOMIC_OR:
-		case Decoder::Opcode::BUFFER_ATOMIC_XOR:
-		case Decoder::Opcode::BUFFER_ATOMIC_FMIN:
-		case Decoder::Opcode::BUFFER_ATOMIC_FMAX:
-		case Decoder::Opcode::FLAT_LOAD_UBYTE:
-		case Decoder::Opcode::FLAT_LOAD_SBYTE:
-		case Decoder::Opcode::FLAT_LOAD_USHORT:
-		case Decoder::Opcode::FLAT_LOAD_SSHORT:
-		case Decoder::Opcode::FLAT_LOAD_DWORD:
-		case Decoder::Opcode::FLAT_LOAD_DWORDX2:
-		case Decoder::Opcode::FLAT_LOAD_DWORDX3:
-		case Decoder::Opcode::FLAT_LOAD_DWORDX4:
-		case Decoder::Opcode::FLAT_STORE_BYTE:
-		case Decoder::Opcode::FLAT_STORE_SHORT:
-		case Decoder::Opcode::FLAT_STORE_DWORD:
-		case Decoder::Opcode::FLAT_STORE_DWORDX2:
-		case Decoder::Opcode::FLAT_STORE_DWORDX3:
-		case Decoder::Opcode::FLAT_STORE_DWORDX4:
-		case Decoder::Opcode::DS_ADD_U32:
-		case Decoder::Opcode::DS_ADD_RTN_U32:
-		case Decoder::Opcode::DS_SUB_U32:
-		case Decoder::Opcode::DS_SUB_RTN_U32:
-		case Decoder::Opcode::DS_MIN_I32:
-		case Decoder::Opcode::DS_MIN_RTN_I32:
-		case Decoder::Opcode::DS_MAX_I32:
-		case Decoder::Opcode::DS_MAX_RTN_I32:
-		case Decoder::Opcode::DS_MIN_U32:
-		case Decoder::Opcode::DS_MIN_RTN_U32:
-		case Decoder::Opcode::DS_MAX_U32:
-		case Decoder::Opcode::DS_MAX_RTN_U32:
-		case Decoder::Opcode::DS_AND_B32:
-		case Decoder::Opcode::DS_AND_RTN_B32:
-		case Decoder::Opcode::DS_OR_B32:
-		case Decoder::Opcode::DS_OR_RTN_B32:
-		case Decoder::Opcode::DS_XOR_B32:
-		case Decoder::Opcode::DS_XOR_RTN_B32:
-		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
-		case Decoder::Opcode::DS_MIN_F32:
-		case Decoder::Opcode::DS_MAX_F32:
-		case Decoder::Opcode::DS_SWIZZLE_B32:
-		case Decoder::Opcode::DS_CONSUME:
-		case Decoder::Opcode::DS_APPEND:
-		case Decoder::Opcode::DS_READ_I8:
-		case Decoder::Opcode::DS_READ_U8:
-		case Decoder::Opcode::DS_READ_I16:
-		case Decoder::Opcode::DS_READ_U16:
-		case Decoder::Opcode::DS_READ_U16_D16:
-		case Decoder::Opcode::DS_READ2_B32:
-		case Decoder::Opcode::DS_READ2ST64_B32:
-		case Decoder::Opcode::DS_READ_B32:
-		case Decoder::Opcode::DS_READ_B64:
-		case Decoder::Opcode::DS_READ2_B64:
-		case Decoder::Opcode::DS_READ2ST64_B64:
-		case Decoder::Opcode::DS_READ_B96:
-		case Decoder::Opcode::DS_READ_B128:
-		case Decoder::Opcode::DS_WRITE_B8:
-		case Decoder::Opcode::DS_WRITE_B16:
-		case Decoder::Opcode::DS_WRITE2_B32:
-		case Decoder::Opcode::DS_WRITE2ST64_B32:
-		case Decoder::Opcode::DS_WRITE2_B64:
-		case Decoder::Opcode::DS_WRITE2ST64_B64:
-		case Decoder::Opcode::DS_WRITE_B32:
-		case Decoder::Opcode::DS_WRITE_B64:
-		case Decoder::Opcode::DS_WRITE_B96:
-		case Decoder::Opcode::DS_WRITE_B128:
-		case Decoder::Opcode::DS_WRITE_ADDTID_B32:
-		case Decoder::Opcode::DS_READ_ADDTID_B32:
-		case Decoder::Opcode::IMAGE_GET_RESINFO:
-		case Decoder::Opcode::IMAGE_GET_LOD:
-		case Decoder::Opcode::IMAGE_LOAD:
-		case Decoder::Opcode::IMAGE_LOAD_MIP:
-		case Decoder::Opcode::IMAGE_STORE:
-		case Decoder::Opcode::IMAGE_STORE_MIP:
-		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
-		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
-		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
-		case Decoder::Opcode::IMAGE_ATOMIC_AND:
-		case Decoder::Opcode::IMAGE_ATOMIC_OR:
-		case Decoder::Opcode::IMAGE_ATOMIC_XOR:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_C:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4H:
-		case Decoder::Opcode::IMAGE_SAMPLE: return true;
-		default: return false;
-	}
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/ValueProgram.h

@@ -1,76 +0,0 @@
-#pragma once
-
-#include "graphics/shader/recompiler/ir/Block.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-#include <memory>
-#include <optional>
-#include <string>
-#include <vector>
-
-namespace Libs::Graphics::ShaderRecompiler::IR {
-
-struct ValueBlockInfo {
-	uint32_t        id       = 0;
-	uint32_t        start_pc = 0;
-	uint32_t        end_pc   = 0;
-	CFG::Terminator terminator;
-	Value           condition;
-	Value           indirect_target;
-};
-
-struct DescriptorSource {
-	struct IndirectImage {
-		uint32_t material_source = 0;
-		uint32_t heap_source     = 0;
-		uint32_t selector_stride = 0;
-		uint32_t selector_offset = 0;
-		uint32_t key_arg         = 0;
-
-		bool operator==(const IndirectImage& other) const = default;
-	};
-
-	std::array<Value, 8>         dwords {};
-	uint32_t                     dword_count = 0;
-	std::optional<IndirectImage> indirect_image;
-
-	bool operator==(const DescriptorSource& other) const = default;
-};
-
-struct SrtRead {
-	Value    value;
-	uint32_t flat_offset = 0;
-	uint32_t use_pc      = 0;
-
-	bool operator==(const SrtRead& other) const = default;
-};
-
-struct ValueProgram {
-	ValueProgram() = default;
-	~ValueProgram();
-
-	ValueProgram(const ValueProgram&)            = delete;
-	ValueProgram& operator=(const ValueProgram&) = delete;
-	ValueProgram(ValueProgram&&)                 = default;
-	ValueProgram& operator=(ValueProgram&&)      = default;
-
-	std::vector<std::unique_ptr<Block>> block_storage;
-	BlockList                           blocks;
-	std::vector<ValueBlockInfo>         block_info;
-	// Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and
-	// storage-image swizzles. Typed memory instructions carry a dense index into these shader-local
-	// tables until those fields are consumed by emission.
-	std::vector<MemoryInfo>       memory_info;
-	std::vector<ExportInfo>       export_info;
-	std::vector<DescriptorSource> descriptor_sources;
-	std::vector<SrtRead>          srt_reads;
-	std::vector<Value>            dynamic_reads;
-};
-
-bool        ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error);
-void        ResolveControlFlowIdentities(ValueProgram& program);
-bool        EquivalentValue(const ValueProgram& program, Value left, Value right);
-Value       ResolveInvariantPhi(const ValueProgram& program, Value value);
-std::string ValueProgramToString(const ValueProgram& program);
-
-} // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/Opcodes.h

@@ -1,71 +0,0 @@
-#pragma once
-
-#include "graphics/shader/recompiler/ir/Type.h"
-
-#include <array>
-#include <cstddef>
-#include <string_view>
-
-namespace Libs::Graphics::ShaderRecompiler::IR {
-
-enum class Opcode {
-#define IR_OPCODE(name, category, lowering, ...) name,
-#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc"
-#undef IR_OPCODE
-	Count,
-};
-
-enum class OpcodeClass { General, Compare, Compare64 };
-enum class LoweringClass {
-	Control,
-	Move,
-	Lane,
-	State,
-	Memory,
-	Attribute,
-	IntegerCompare,
-	Integer16Compare,
-	FloatCompare,
-	Conversion,
-	Integer16,
-	PackedInteger16,
-	PackedFloat16,
-	Float16,
-	Float,
-	U64Mask,
-	SimpleInteger,
-	ComposedInteger,
-	ExtendedInteger,
-};
-
-struct OpcodeInfo {
-	std::string_view name;
-	OpcodeClass      opcode_class   = OpcodeClass::General;
-	LoweringClass    lowering_class = LoweringClass::Control;
-};
-
-inline constexpr OpcodeInfo OPCODE_INFO[] = {
-#define IR_OPCODE(name, category, lowering, ...)                                                   \
-	{#name, OpcodeClass::category, LoweringClass::lowering},
-#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc"
-#undef IR_OPCODE
-};
-static_assert(std::size(OPCODE_INFO) == static_cast<size_t>(Opcode::Count));
-
-constexpr const OpcodeInfo& GetOpcodeInfo(Opcode opcode) {
-	return OPCODE_INFO[static_cast<size_t>(opcode)];
-}
-
-constexpr std::string_view OpcodeName(Opcode opcode) {
-	return GetOpcodeInfo(opcode).name;
-}
-
-constexpr bool IsCompareOpcode(Opcode opcode) {
-	return GetOpcodeInfo(opcode).opcode_class != OpcodeClass::General;
-}
-
-constexpr bool IsCompare64Opcode(Opcode opcode) {
-	return GetOpcodeInfo(opcode).opcode_class == OpcodeClass::Compare64;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp

@@ -1,546 +0,0 @@
-#include "common/assert.h"
-#include "graphics/shader/recompiler/BufferFormat.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ShaderIRInternal.h"
-
-#include <algorithm>
-#include <fmt/format.h>
-#include <utility>
-
-namespace Libs::Graphics::ShaderRecompiler::IR {
-namespace {
-
-struct LowerMap {
-	Decoder::Opcode decoded = Decoder::Opcode::UNKNOWN;
-	Opcode          ir      = Opcode::MoveU32;
-};
-
-constexpr LowerMap LOWER_OPS[] = {
-    {Decoder::Opcode::S_MOV_B32, Opcode::MoveU32},
-    {Decoder::Opcode::S_MOV_B64, Opcode::MoveU64},
-    {Decoder::Opcode::S_WQM_B64, Opcode::WqmB64},
-    {Decoder::Opcode::S_MOVK_I32, Opcode::MoveU32},
-    {Decoder::Opcode::S_ABS_I32, Opcode::AbsI32},
-    {Decoder::Opcode::S_BREV_B32, Opcode::BitReverseU32},
-    {Decoder::Opcode::S_BCNT1_I32_B32, Opcode::BitCountU32},
-    {Decoder::Opcode::S_BCNT1_I32_B64, Opcode::BitCountU64},
-    {Decoder::Opcode::S_FF1_I32_B32, Opcode::FindLsbU32},
-    {Decoder::Opcode::S_FF1_I32_B64, Opcode::FindLsbU64},
-    {Decoder::Opcode::S_FLBIT_I32_B64, Opcode::FindMsbFromHighU64},
-    {Decoder::Opcode::S_BITREPLICATE_B64_B32, Opcode::BitReplicateB64B32},
-    {Decoder::Opcode::S_QUADMASK_B64, Opcode::QuadmaskB64},
-    {Decoder::Opcode::S_NOT_B32, Opcode::BitwiseNotU32},
-    {Decoder::Opcode::S_NOT_B64, Opcode::BitwiseNotU64},
-    {Decoder::Opcode::S_ADD_U32, Opcode::IAddU32},
-    {Decoder::Opcode::S_SUB_U32, Opcode::ISubU32},
-    {Decoder::Opcode::S_ADD_I32, Opcode::IAddU32},
-    {Decoder::Opcode::S_SUB_I32, Opcode::ISubU32},
-    {Decoder::Opcode::S_MIN_I32, Opcode::IMinI32},
-    {Decoder::Opcode::S_MAX_I32, Opcode::IMaxI32},
-    {Decoder::Opcode::S_MIN_U32, Opcode::UMinU32},
-    {Decoder::Opcode::S_MAX_U32, Opcode::UMaxU32},
-    {Decoder::Opcode::S_AND_B32, Opcode::BitwiseAndU32},
-    {Decoder::Opcode::S_AND_B64, Opcode::BitwiseAndU64},
-    {Decoder::Opcode::S_ANDN2_B32, Opcode::BitwiseAndNotU32},
-    {Decoder::Opcode::S_ANDN2_B64, Opcode::BitwiseAndNotU64},
-    {Decoder::Opcode::S_OR_B32, Opcode::BitwiseOrU32},
-    {Decoder::Opcode::S_OR_B64, Opcode::BitwiseOrU64},
-    {Decoder::Opcode::S_ORN2_B32, Opcode::BitwiseOrNotU32},
-    {Decoder::Opcode::S_ORN2_B64, Opcode::BitwiseOrNotU64},
-    {Decoder::Opcode::S_XOR_B32, Opcode::BitwiseXorU32},
-    {Decoder::Opcode::S_XOR_B64, Opcode::BitwiseXorU64},
-    {Decoder::Opcode::S_NAND_B32, Opcode::BitwiseNandU32},
-    {Decoder::Opcode::S_NAND_B64, Opcode::BitwiseNandU64},
-    {Decoder::Opcode::S_NOR_B32, Opcode::BitwiseNorU32},
-    {Decoder::Opcode::S_NOR_B64, Opcode::BitwiseNorU64},
-    {Decoder::Opcode::S_XNOR_B32, Opcode::BitwiseXnorU32},
-    {Decoder::Opcode::S_XNOR_B64, Opcode::BitwiseXnorU64},
-    {Decoder::Opcode::S_LSHL_B32, Opcode::ShiftLeftLogicalU32},
-    {Decoder::Opcode::S_LSHL_B64, Opcode::ShiftLeftLogicalU64},
-    {Decoder::Opcode::S_LSHR_B32, Opcode::ShiftRightLogicalU32},
-    {Decoder::Opcode::S_LSHR_B64, Opcode::ShiftRightLogicalU64},
-    {Decoder::Opcode::S_ASHR_I32, Opcode::ShiftRightArithmeticI32},
-    {Decoder::Opcode::S_MUL_I32, Opcode::IMulU32},
-    {Decoder::Opcode::S_MUL_HI_U32, Opcode::UMulHighU32},
-    {Decoder::Opcode::S_MULK_I32, Opcode::IMulU32},
-    {Decoder::Opcode::S_BFE_U32, Opcode::BitFieldExtractU32},
-    {Decoder::Opcode::S_BFE_I32, Opcode::BitFieldExtractI32},
-    {Decoder::Opcode::S_BFE_U64, Opcode::BitFieldExtractU64},
-    {Decoder::Opcode::S_BFM_B32, Opcode::BitFieldMaskU32},
-    {Decoder::Opcode::S_BFM_B64, Opcode::BitFieldMaskU64},
-    {Decoder::Opcode::S_BITCMP0_B32, Opcode::BitCompare0B32},
-    {Decoder::Opcode::S_BITCMP1_B32, Opcode::BitCompare1B32},
-    {Decoder::Opcode::S_PACK_LL_B32_B16, Opcode::PackLowLowU16},
-    {Decoder::Opcode::S_PACK_LH_B32_B16, Opcode::PackLowHighU16},
-    {Decoder::Opcode::S_PACK_HH_B32_B16, Opcode::PackHighHighU16},
-    {Decoder::Opcode::S_CMP_EQ_I32, Opcode::CompareEqI32},
-    {Decoder::Opcode::S_CMP_LG_I32, Opcode::CompareNeI32},
-    {Decoder::Opcode::S_CMP_GT_I32, Opcode::CompareGtI32},
-    {Decoder::Opcode::S_CMP_GE_I32, Opcode::CompareGeI32},
-    {Decoder::Opcode::S_CMP_LT_I32, Opcode::CompareLtI32},
-    {Decoder::Opcode::S_CMP_LE_I32, Opcode::CompareLeI32},
-    {Decoder::Opcode::S_CMP_EQ_U32, Opcode::CompareEqU32},
-    {Decoder::Opcode::S_CMP_LG_U32, Opcode::CompareNeU32},
-    {Decoder::Opcode::S_CMP_GT_U32, Opcode::CompareGtU32},
-    {Decoder::Opcode::S_CMP_GE_U32, Opcode::CompareGeU32},
-    {Decoder::Opcode::S_CMP_LT_U32, Opcode::CompareLtU32},
-    {Decoder::Opcode::S_CMP_LE_U32, Opcode::CompareLeU32},
-    {Decoder::Opcode::S_CMP_EQ_U64, Opcode::CompareEqU64},
-    {Decoder::Opcode::S_CMP_LG_U64, Opcode::CompareNeU64},
-    {Decoder::Opcode::V_READFIRSTLANE_B32, Opcode::ReadFirstLaneU32},
-    {Decoder::Opcode::V_READLANE_B32, Opcode::ReadLaneU32},
-    {Decoder::Opcode::V_WRITELANE_B32, Opcode::WriteLaneU32},
-    {Decoder::Opcode::V_PERMLANE16_B32, Opcode::Permlane16B32},
-    {Decoder::Opcode::V_PERMLANEX16_B32, Opcode::Permlanex16B32},
-    {Decoder::Opcode::V_CVT_F32_I32, Opcode::ConvertI32ToF32},
-    {Decoder::Opcode::V_CVT_F32_U32, Opcode::ConvertU32ToF32},
-    {Decoder::Opcode::V_CVT_U32_F32, Opcode::ConvertF32ToU32},
-    {Decoder::Opcode::V_CVT_I32_F32, Opcode::ConvertF32ToI32},
-    {Decoder::Opcode::V_CVT_F16_F32, Opcode::ConvertF32ToF16},
-    {Decoder::Opcode::V_CVT_F32_F16, Opcode::ConvertF16ToF32},
-    {Decoder::Opcode::V_CVT_F16_U16, Opcode::ConvertU16ToF16},
-    {Decoder::Opcode::V_CVT_U16_F16, Opcode::ConvertF16ToU16},
-    {Decoder::Opcode::V_CVT_F16_I16, Opcode::ConvertI16ToF16},
-    {Decoder::Opcode::V_CVT_I16_F16, Opcode::ConvertF16ToI16},
-    {Decoder::Opcode::V_CVT_RPI_I32_F32, Opcode::ConvertRoundPlusInfF32ToI32},
-    {Decoder::Opcode::V_CVT_FLR_I32_F32, Opcode::ConvertFloorF32ToI32},
-    {Decoder::Opcode::V_FREXP_EXP_I32_F32, Opcode::FrexpExpI32F32},
-    {Decoder::Opcode::V_FREXP_MANT_F32, Opcode::FrexpMantF32},
-    {Decoder::Opcode::V_CVT_OFF_F32_I4, Opcode::ConvertI4ToOffsetF32},
-    {Decoder::Opcode::V_RCP_F32, Opcode::RcpF32},
-    {Decoder::Opcode::V_RCP_IFLAG_F32, Opcode::RcpIflagF32},
-    {Decoder::Opcode::V_FRACT_F32, Opcode::FractF32},
-    {Decoder::Opcode::V_TRUNC_F32, Opcode::TruncF32},
-    {Decoder::Opcode::V_CEIL_F32, Opcode::CeilF32},
-    {Decoder::Opcode::V_RNDNE_F32, Opcode::RoundEvenF32},
-    {Decoder::Opcode::V_FLOOR_F32, Opcode::FloorF32},
-    {Decoder::Opcode::V_EXP_F32, Opcode::Exp2F32},
-    {Decoder::Opcode::V_LOG_F32, Opcode::Log2F32},
-    {Decoder::Opcode::V_RSQ_F32, Opcode::InverseSqrtF32},
-    {Decoder::Opcode::V_SQRT_F32, Opcode::SqrtF32},
-    {Decoder::Opcode::V_RCP_F16, Opcode::RcpF16},
-    {Decoder::Opcode::V_SQRT_F16, Opcode::SqrtF16},
-    {Decoder::Opcode::V_RSQ_F16, Opcode::InverseSqrtF16},
-    {Decoder::Opcode::V_LOG_F16, Opcode::Log2F16},
-    {Decoder::Opcode::V_EXP_F16, Opcode::Exp2F16},
-    {Decoder::Opcode::V_FLOOR_F16, Opcode::FloorF16},
-    {Decoder::Opcode::V_CEIL_F16, Opcode::CeilF16},
-    {Decoder::Opcode::V_TRUNC_F16, Opcode::TruncF16},
-    {Decoder::Opcode::V_RNDNE_F16, Opcode::RoundEvenF16},
-    {Decoder::Opcode::V_SIN_F32, Opcode::SinF32},
-    {Decoder::Opcode::V_COS_F32, Opcode::CosF32},
-    {Decoder::Opcode::V_NOT_B32, Opcode::BitwiseNotU32},
-    {Decoder::Opcode::V_BFREV_B32, Opcode::BitReverseU32},
-    {Decoder::Opcode::V_FFBL_B32, Opcode::FindLsbU32},
-    {Decoder::Opcode::V_FFBH_U32, Opcode::FindMsbFromHighU32},
-    {Decoder::Opcode::V_ADD_F32, Opcode::FAddF32},
-    {Decoder::Opcode::V_SUB_F32, Opcode::FSubF32},
-    {Decoder::Opcode::V_SUBREV_F32, Opcode::FSubF32},
-    {Decoder::Opcode::V_MUL_F32, Opcode::FMulF32},
-    {Decoder::Opcode::V_MAC_F32, Opcode::FMadF32},
-    {Decoder::Opcode::V_MADMK_F32, Opcode::FMadF32},
-    {Decoder::Opcode::V_MADAK_F32, Opcode::FMadF32},
-    {Decoder::Opcode::V_MIN_F32, Opcode::FMinF32},
-    {Decoder::Opcode::V_MAX_F32, Opcode::FMaxF32},
-    {Decoder::Opcode::V_MIN_I32, Opcode::IMinI32},
-    {Decoder::Opcode::V_MAX_I32, Opcode::IMaxI32},
-    {Decoder::Opcode::V_MAD_F32, Opcode::FMadF32},
-    {Decoder::Opcode::V_MAD_I32_I24, Opcode::IMadI24U32},
-    {Decoder::Opcode::V_MAD_U32_U24, Opcode::UMadU24U32},
-    {Decoder::Opcode::V_CUBEID_F32, Opcode::CubeIdF32},
-    {Decoder::Opcode::V_CUBESC_F32, Opcode::CubeScF32},
-    {Decoder::Opcode::V_CUBETC_F32, Opcode::CubeTcF32},
-    {Decoder::Opcode::V_CUBEMA_F32, Opcode::CubeMaF32},
-    {Decoder::Opcode::V_FMA_F32, Opcode::FMadF32},
-    {Decoder::Opcode::V_BFE_U32, Opcode::BitFieldExtract3U32},
-    {Decoder::Opcode::V_BFE_I32, Opcode::BitFieldExtract3I32},
-    {Decoder::Opcode::V_BFI_B32, Opcode::BitFieldInsertSelectU32},
-    {Decoder::Opcode::V_ALIGNBIT_B32, Opcode::AlignBitU32},
-    {Decoder::Opcode::V_ALIGNBYTE_B32, Opcode::AlignByteU32},
-    {Decoder::Opcode::V_MIN3_F32, Opcode::FMin3F32},
-    {Decoder::Opcode::V_MIN3_I32, Opcode::IMin3I32},
-    {Decoder::Opcode::V_MIN3_U32, Opcode::UMin3U32},
-    {Decoder::Opcode::V_MIN3_F16, Opcode::Min3F16},
-    {Decoder::Opcode::V_MAX3_F32, Opcode::FMax3F32},
-    {Decoder::Opcode::V_MAX3_I32, Opcode::IMax3I32},
-    {Decoder::Opcode::V_MAX3_U32, Opcode::UMax3U32},
-    {Decoder::Opcode::V_MAX3_F16, Opcode::Max3F16},
-    {Decoder::Opcode::V_MED3_F32, Opcode::FMed3F32},
-    {Decoder::Opcode::V_MED3_I32, Opcode::IMed3I32},
-    {Decoder::Opcode::V_MED3_U32, Opcode::UMed3U32},
-    {Decoder::Opcode::V_MED3_F16, Opcode::Med3F16},
-    {Decoder::Opcode::V_MED3_I16, Opcode::IMed3I16},
-    {Decoder::Opcode::V_SAD_U32, Opcode::SadU32},
-    {Decoder::Opcode::V_ADD3_U32, Opcode::IAdd3U32},
-    {Decoder::Opcode::V_LSHL_ADD_U32, Opcode::ShiftLeftAddU32},
-    {Decoder::Opcode::V_ADD_LSHL_U32, Opcode::AddShiftLeftU32},
-    {Decoder::Opcode::V_XAD_U32, Opcode::XorAddU32},
-    {Decoder::Opcode::V_LSHL_OR_B32, Opcode::ShiftLeftOrU32},
-    {Decoder::Opcode::V_AND_OR_B32, Opcode::BitwiseAndOrU32},
-    {Decoder::Opcode::V_OR3_B32, Opcode::BitwiseOr3U32},
-    {Decoder::Opcode::V_XOR3_B32, Opcode::BitwiseXor3U32},
-    {Decoder::Opcode::V_ADD_NC_U32, Opcode::IAddU32},
-    {Decoder::Opcode::V_SUB_NC_U32, Opcode::ISubU32},
-    {Decoder::Opcode::V_SUBREV_NC_U32, Opcode::ISubU32},
-    {Decoder::Opcode::V_ADD_NC_U16, Opcode::IAddU16},
-    {Decoder::Opcode::V_SUB_NC_U16, Opcode::ISubI16},
-    {Decoder::Opcode::V_MAX_U16, Opcode::UMaxU16},
-    {Decoder::Opcode::V_MAX_I16, Opcode::IMaxI16},
-    {Decoder::Opcode::V_MIN_U16, Opcode::UMinU16},
-    {Decoder::Opcode::V_MIN_I16, Opcode::IMinI16},
-    {Decoder::Opcode::V_ADD_NC_I16, Opcode::IAddU16},
-    {Decoder::Opcode::V_SUB_NC_I16, Opcode::ISubI16},
-    {Decoder::Opcode::V_MUL_I32_I24, Opcode::IMulI24U32},
-    {Decoder::Opcode::V_MUL_U32_U24, Opcode::UMulU24U32},
-    {Decoder::Opcode::V_MUL_LO_U32, Opcode::IMulU32},
-    {Decoder::Opcode::V_MUL_HI_U32, Opcode::UMulHighU32},
-    {Decoder::Opcode::V_MUL_LO_I32, Opcode::IMulU32},
-    {Decoder::Opcode::V_MUL_HI_I32, Opcode::SMulHighI32},
-    {Decoder::Opcode::V_ADD_I32, Opcode::IAddU32},
-    {Decoder::Opcode::V_SUB_I32, Opcode::ISubU32},
-    {Decoder::Opcode::V_SUBREV_I32, Opcode::ISubU32},
-    {Decoder::Opcode::V_BFM_B32, Opcode::BitFieldMaskU32},
-    {Decoder::Opcode::V_CVT_PKRTZ_F16_F32, Opcode::PackF32ToF16Rtz},
-    {Decoder::Opcode::V_LDEXP_F32, Opcode::LdexpF32},
-    {Decoder::Opcode::V_CVT_PKNORM_I16_F32, Opcode::PackSnorm2x16F32},
-    {Decoder::Opcode::V_CVT_PKNORM_U16_F32, Opcode::PackUnorm2x16F32},
-    {Decoder::Opcode::V_CVT_PK_U16_U32, Opcode::PackU16U32},
-    {Decoder::Opcode::V_CVT_PK_I16_I32, Opcode::PackU16U32},
-    {Decoder::Opcode::V_CVT_PK_U8_F32, Opcode::PackU8F32},
-    {Decoder::Opcode::V_PACK_B32_F16, Opcode::PackB32F16},
-    {Decoder::Opcode::V_PK_MAD_I16, Opcode::PackedMadI16},
-    {Decoder::Opcode::V_PK_MUL_LO_U16, Opcode::PackedMulLoU16},
-    {Decoder::Opcode::V_PK_ADD_I16, Opcode::PackedAddI16},
-    {Decoder::Opcode::V_PK_SUB_I16, Opcode::PackedSubI16},
-    {Decoder::Opcode::V_PK_LSHLREV_B16, Opcode::PackedLshlrevB16},
-    {Decoder::Opcode::V_PK_LSHRREV_B16, Opcode::PackedLshrrevB16},
-    {Decoder::Opcode::V_PK_ASHRREV_I16, Opcode::PackedAshrrevI16},
-    {Decoder::Opcode::V_PK_MAX_I16, Opcode::PackedMaxI16},
-    {Decoder::Opcode::V_PK_MIN_I16, Opcode::PackedMinI16},
-    {Decoder::Opcode::V_PK_MAD_U16, Opcode::PackedMadU16},
-    {Decoder::Opcode::V_PK_ADD_U16, Opcode::PackedAddU16},
-    {Decoder::Opcode::V_PK_SUB_U16, Opcode::PackedSubU16},
-    {Decoder::Opcode::V_PK_MAX_U16, Opcode::PackedMaxU16},
-    {Decoder::Opcode::V_PK_MIN_U16, Opcode::PackedMinU16},
-    {Decoder::Opcode::V_PK_ADD_F16, Opcode::PackedAddF16},
-    {Decoder::Opcode::V_PK_MUL_F16, Opcode::PackedMulF16},
-    {Decoder::Opcode::V_PK_MIN_F16, Opcode::PackedMinF16},
-    {Decoder::Opcode::V_PK_MAX_F16, Opcode::PackedMaxF16},
-    {Decoder::Opcode::V_PK_FMA_F16, Opcode::PackedFmaF16},
-    {Decoder::Opcode::V_PK_FMAC_F16, Opcode::PackedFmaF16},
-    {Decoder::Opcode::V_ADD_F16, Opcode::AddF16},
-    {Decoder::Opcode::V_SUB_F16, Opcode::SubF16},
-    {Decoder::Opcode::V_SUBREV_F16, Opcode::SubF16},
-    {Decoder::Opcode::V_MUL_F16, Opcode::MulF16},
-    {Decoder::Opcode::V_FMAC_F16, Opcode::FmaF16},
-    {Decoder::Opcode::V_FMAMK_F16, Opcode::FmaF16},
-    {Decoder::Opcode::V_FMAAK_F16, Opcode::FmaF16},
-    {Decoder::Opcode::V_FMA_F16, Opcode::FmaF16},
-    {Decoder::Opcode::V_MAD_MIXLO_F16, Opcode::MadMixF16},
-    {Decoder::Opcode::V_MAD_MIXHI_F16, Opcode::MadMixF16},
-    {Decoder::Opcode::V_AND_B32, Opcode::BitwiseAndU32},
-    {Decoder::Opcode::V_OR_B32, Opcode::BitwiseOrU32},
-    {Decoder::Opcode::V_XOR_B32, Opcode::BitwiseXorU32},
-    {Decoder::Opcode::V_XNOR_B32, Opcode::BitwiseXnorU32},
-    {Decoder::Opcode::V_BCNT_U32_B32, Opcode::BitCountAddU32},
-    {Decoder::Opcode::V_MBCNT_LO_U32_B32, Opcode::MaskedBitCountLowU32},
-    {Decoder::Opcode::V_MBCNT_HI_U32_B32, Opcode::MaskedBitCountHighU32},
-    {Decoder::Opcode::V_LSHL_B32, Opcode::ShiftLeftLogicalU32},
-    {Decoder::Opcode::V_LSHLREV_B32, Opcode::ShiftLeftLogicalU32},
-    {Decoder::Opcode::V_LSHLREV_B16, Opcode::ShiftLeftLogicalU16},
-    {Decoder::Opcode::V_LSHR_B32, Opcode::ShiftRightLogicalU32},
-    {Decoder::Opcode::V_LSHRREV_B32, Opcode::ShiftRightLogicalU32},
-    {Decoder::Opcode::V_LSHRREV_B64, Opcode::ShiftRightLogicalU64},
-    {Decoder::Opcode::V_LSHRREV_B16, Opcode::ShiftRightLogicalU16},
-    {Decoder::Opcode::V_ASHR_I32, Opcode::ShiftRightArithmeticI32},
-    {Decoder::Opcode::V_ASHRREV_I32, Opcode::ShiftRightArithmeticI32},
-    {Decoder::Opcode::V_ASHRREV_I16, Opcode::ShiftRightArithmeticI16},
-    {Decoder::Opcode::V_MIN_U32, Opcode::UMinU32},
-    {Decoder::Opcode::V_MAX_U32, Opcode::UMaxU32},
-    {Decoder::Opcode::V_MAX_F16, Opcode::MaxF16},
-    {Decoder::Opcode::V_MIN_F16, Opcode::MinF16},
-    {Decoder::Opcode::V_CMP_F_F32, Opcode::CompareFalse},
-    {Decoder::Opcode::V_CMP_LT_F32, Opcode::CompareLtF32},
-    {Decoder::Opcode::V_CMP_EQ_F32, Opcode::CompareEqF32},
-    {Decoder::Opcode::V_CMP_LE_F32, Opcode::CompareLeF32},
-    {Decoder::Opcode::V_CMP_GT_F32, Opcode::CompareGtF32},
-    {Decoder::Opcode::V_CMP_LG_F32, Opcode::CompareNeF32},
-    {Decoder::Opcode::V_CMP_GE_F32, Opcode::CompareGeF32},
-    {Decoder::Opcode::V_CMP_O_F32, Opcode::CompareOrderedF32},
-    {Decoder::Opcode::V_CMP_U_F32, Opcode::CompareUnorderedF32},
-    {Decoder::Opcode::V_CMP_NGE_F32, Opcode::CompareUnordLtF32},
-    {Decoder::Opcode::V_CMP_NLG_F32, Opcode::CompareUnordEqF32},
-    {Decoder::Opcode::V_CMP_NGT_F32, Opcode::CompareUnordLeF32},
-    {Decoder::Opcode::V_CMP_NLE_F32, Opcode::CompareUnordGtF32},
-    {Decoder::Opcode::V_CMP_NEQ_F32, Opcode::CompareUnordNeF32},
-    {Decoder::Opcode::V_CMP_NLT_F32, Opcode::CompareUnordGeF32},
-    {Decoder::Opcode::V_CMP_TRU_F32, Opcode::CompareTrue},
-    {Decoder::Opcode::V_CMP_CLASS_F32, Opcode::CompareClassF32},
-    {Decoder::Opcode::V_CMP_LT_F16, Opcode::CompareLtF16},
-    {Decoder::Opcode::V_CMP_EQ_F16, Opcode::CompareEqF16},
-    {Decoder::Opcode::V_CMP_LE_F16, Opcode::CompareLeF16},
-    {Decoder::Opcode::V_CMP_GT_F16, Opcode::CompareGtF16},
-    {Decoder::Opcode::V_CMP_LG_F16, Opcode::CompareNeF16},
-    {Decoder::Opcode::V_CMP_GE_F16, Opcode::CompareGeF16},
-    {Decoder::Opcode::V_CMP_NEQ_F16, Opcode::CompareUnordNeF16},
-    {Decoder::Opcode::V_CMPX_LT_F16, Opcode::CompareMaskLtF16},
-    {Decoder::Opcode::V_CMPX_EQ_F16, Opcode::CompareMaskEqF16},
-    {Decoder::Opcode::V_CMPX_LE_F16, Opcode::CompareMaskLeF16},
-    {Decoder::Opcode::V_CMPX_GT_F16, Opcode::CompareMaskGtF16},
-    {Decoder::Opcode::V_CMPX_GE_F16, Opcode::CompareMaskGeF16},
-    {Decoder::Opcode::V_CMPX_NEQ_F16, Opcode::CompareMaskUnordNeF16},
-    {Decoder::Opcode::V_CMPX_NLT_F16, Opcode::CompareMaskUnordGeF16},
-    {Decoder::Opcode::V_CMPX_LT_F32, Opcode::CompareMaskLtF32},
-    {Decoder::Opcode::V_CMPX_EQ_F32, Opcode::CompareMaskEqF32},
-    {Decoder::Opcode::V_CMPX_LE_F32, Opcode::CompareMaskLeF32},
-    {Decoder::Opcode::V_CMPX_GT_F32, Opcode::CompareMaskGtF32},
-    {Decoder::Opcode::V_CMPX_LG_F32, Opcode::CompareMaskNeF32},
-    {Decoder::Opcode::V_CMPX_GE_F32, Opcode::CompareMaskGeF32},
-    {Decoder::Opcode::V_CMPX_NGE_F32, Opcode::CompareMaskUnordLtF32},
-    {Decoder::Opcode::V_CMPX_NLG_F32, Opcode::CompareMaskUnordEqF32},
-    {Decoder::Opcode::V_CMPX_NGT_F32, Opcode::CompareMaskUnordLeF32},
-    {Decoder::Opcode::V_CMPX_NLE_F32, Opcode::CompareMaskUnordGtF32},
-    {Decoder::Opcode::V_CMPX_NEQ_F32, Opcode::CompareMaskUnordNeF32},
-    {Decoder::Opcode::V_CMPX_NLT_F32, Opcode::CompareMaskUnordGeF32},
-    {Decoder::Opcode::V_CMP_F_I32, Opcode::CompareFalse},
-    {Decoder::Opcode::V_CMP_LT_I32, Opcode::CompareLtI32},
-    {Decoder::Opcode::V_CMP_EQ_I32, Opcode::CompareEqI32},
-    {Decoder::Opcode::V_CMP_LE_I32, Opcode::CompareLeI32},
-    {Decoder::Opcode::V_CMP_GT_I32, Opcode::CompareGtI32},
-    {Decoder::Opcode::V_CMP_NE_I32, Opcode::CompareNeI32},
-    {Decoder::Opcode::V_CMP_GE_I32, Opcode::CompareGeI32},
-    {Decoder::Opcode::V_CMP_T_I32, Opcode::CompareTrue},
-    {Decoder::Opcode::V_CMP_LT_I16, Opcode::CompareLtI16},
-    {Decoder::Opcode::V_CMP_EQ_I16, Opcode::CompareEqI16},
-    {Decoder::Opcode::V_CMP_LE_I16, Opcode::CompareLeI16},
-    {Decoder::Opcode::V_CMP_GT_I16, Opcode::CompareGtI16},
-    {Decoder::Opcode::V_CMP_NE_I16, Opcode::CompareNeI16},
-    {Decoder::Opcode::V_CMP_GE_I16, Opcode::CompareGeI16},
-    {Decoder::Opcode::V_CMPX_LT_I32, Opcode::CompareMaskLtI32},
-    {Decoder::Opcode::V_CMPX_EQ_I32, Opcode::CompareMaskEqI32},
-    {Decoder::Opcode::V_CMPX_LE_I32, Opcode::CompareMaskLeI32},
-    {Decoder::Opcode::V_CMPX_GT_I32, Opcode::CompareMaskGtI32},
-    {Decoder::Opcode::V_CMPX_NE_I32, Opcode::CompareMaskNeI32},
-    {Decoder::Opcode::V_CMPX_GE_I32, Opcode::CompareMaskGeI32},
-    {Decoder::Opcode::V_CMP_LT_U16, Opcode::CompareLtU16},
-    {Decoder::Opcode::V_CMP_EQ_U16, Opcode::CompareEqU16},
-    {Decoder::Opcode::V_CMP_LE_U16, Opcode::CompareLeU16},
-    {Decoder::Opcode::V_CMP_GT_U16, Opcode::CompareGtU16},
-    {Decoder::Opcode::V_CMP_NE_U16, Opcode::CompareNeU16},
-    {Decoder::Opcode::V_CMP_GE_U16, Opcode::CompareGeU16},
-    {Decoder::Opcode::V_CMP_F_U32, Opcode::CompareFalse},
-    {Decoder::Opcode::V_CMP_LT_U32, Opcode::CompareLtU32},
-    {Decoder::Opcode::V_CMP_EQ_U32, Opcode::CompareEqU32},
-    {Decoder::Opcode::V_CMP_LE_U32, Opcode::CompareLeU32},
-    {Decoder::Opcode::V_CMP_GT_U32, Opcode::CompareGtU32},
-    {Decoder::Opcode::V_CMP_NE_U32, Opcode::CompareNeU32},
-    {Decoder::Opcode::V_CMP_GE_U32, Opcode::CompareGeU32},
-    {Decoder::Opcode::V_CMP_T_U32, Opcode::CompareTrue},
-    {Decoder::Opcode::V_CMP_EQ_I64, Opcode::CompareEqU64},
-    {Decoder::Opcode::V_CMP_GT_U64, Opcode::CompareGtU64},
-    {Decoder::Opcode::V_CMP_NE_U64, Opcode::CompareNeU64},
-    {Decoder::Opcode::V_CMPX_NE_I64, Opcode::CompareNeU64},
-    {Decoder::Opcode::V_CMPX_NE_U64, Opcode::CompareNeU64},
-    {Decoder::Opcode::V_CMPX_LT_U32, Opcode::CompareMaskLtU32},
-    {Decoder::Opcode::V_CMPX_EQ_U32, Opcode::CompareMaskEqU32},
-    {Decoder::Opcode::V_CMPX_LE_U32, Opcode::CompareMaskLeU32},
-    {Decoder::Opcode::V_CMPX_GT_U32, Opcode::CompareMaskGtU32},
-    {Decoder::Opcode::V_CMPX_NE_U32, Opcode::CompareMaskNeU32},
-    {Decoder::Opcode::V_CMPX_GE_U32, Opcode::CompareMaskGeU32},
-    {Decoder::Opcode::S_LOAD_DWORD, Opcode::SLoadDword},
-    {Decoder::Opcode::S_BUFFER_LOAD_DWORD, Opcode::SBufferLoadDword},
-    {Decoder::Opcode::BUFFER_LOAD_DWORD, Opcode::BufferLoadDword},
-    {Decoder::Opcode::BUFFER_LOAD_SBYTE, Opcode::BufferLoadSbyte},
-    {Decoder::Opcode::BUFFER_LOAD_SSHORT, Opcode::BufferLoadSshort},
-    {Decoder::Opcode::BUFFER_STORE_BYTE, Opcode::BufferStoreByte},
-    {Decoder::Opcode::BUFFER_STORE_SHORT, Opcode::BufferStoreShort},
-    {Decoder::Opcode::BUFFER_STORE_DWORD, Opcode::BufferStoreDword},
-    {Decoder::Opcode::BUFFER_ATOMIC_SWAP, Opcode::AtomicSwapU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_ADD, Opcode::AtomicAddU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_SUB, Opcode::AtomicSubU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_SMIN, Opcode::AtomicSMinI32},
-    {Decoder::Opcode::BUFFER_ATOMIC_UMIN, Opcode::AtomicUMinU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_SMAX, Opcode::AtomicSMaxI32},
-    {Decoder::Opcode::BUFFER_ATOMIC_UMAX, Opcode::AtomicUMaxU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_AND, Opcode::AtomicAndU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_OR, Opcode::AtomicOrU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_XOR, Opcode::AtomicXorU32},
-    {Decoder::Opcode::BUFFER_ATOMIC_FMIN, Opcode::AtomicFMinF32},
-    {Decoder::Opcode::BUFFER_ATOMIC_FMAX, Opcode::AtomicFMaxF32},
-    {Decoder::Opcode::FLAT_LOAD_UBYTE, Opcode::FlatLoadUbyte},
-    {Decoder::Opcode::FLAT_LOAD_SBYTE, Opcode::FlatLoadSbyte},
-    {Decoder::Opcode::FLAT_LOAD_SSHORT, Opcode::FlatLoadSshort},
-    {Decoder::Opcode::FLAT_STORE_BYTE, Opcode::FlatStoreByte},
-    {Decoder::Opcode::FLAT_STORE_SHORT, Opcode::FlatStoreShort},
-    {Decoder::Opcode::FLAT_STORE_DWORD, Opcode::FlatStoreDword},
-    {Decoder::Opcode::DS_ADD_U32, Opcode::AtomicAddU32},
-    {Decoder::Opcode::DS_ADD_RTN_U32, Opcode::AtomicAddU32},
-    {Decoder::Opcode::DS_SUB_U32, Opcode::AtomicSubU32},
-    {Decoder::Opcode::DS_SUB_RTN_U32, Opcode::AtomicSubU32},
-    {Decoder::Opcode::DS_MIN_I32, Opcode::AtomicSMinI32},
-    {Decoder::Opcode::DS_MIN_RTN_I32, Opcode::AtomicSMinI32},
-    {Decoder::Opcode::DS_MAX_I32, Opcode::AtomicSMaxI32},
-    {Decoder::Opcode::DS_MAX_RTN_I32, Opcode::AtomicSMaxI32},
-    {Decoder::Opcode::DS_MIN_U32, Opcode::AtomicUMinU32},
-    {Decoder::Opcode::DS_MIN_RTN_U32, Opcode::AtomicUMinU32},
-    {Decoder::Opcode::DS_MAX_U32, Opcode::AtomicUMaxU32},
-    {Decoder::Opcode::DS_MAX_RTN_U32, Opcode::AtomicUMaxU32},
-    {Decoder::Opcode::DS_AND_B32, Opcode::AtomicAndU32},
-    {Decoder::Opcode::DS_AND_RTN_B32, Opcode::AtomicAndU32},
-    {Decoder::Opcode::DS_OR_B32, Opcode::AtomicOrU32},
-    {Decoder::Opcode::DS_OR_RTN_B32, Opcode::AtomicOrU32},
-    {Decoder::Opcode::DS_XOR_B32, Opcode::AtomicXorU32},
-    {Decoder::Opcode::DS_XOR_RTN_B32, Opcode::AtomicXorU32},
-    {Decoder::Opcode::DS_WRXCHG_RTN_B32, Opcode::AtomicSwapU32},
-    {Decoder::Opcode::DS_READ_I8, Opcode::DsReadSbyte},
-    {Decoder::Opcode::DS_READ_U8, Opcode::DsReadUbyte},
-    {Decoder::Opcode::DS_READ_I16, Opcode::DsReadSshort},
-    {Decoder::Opcode::DS_READ_U16, Opcode::DsReadUshort},
-    {Decoder::Opcode::DS_READ_U16_D16, Opcode::DsReadUshort},
-    {Decoder::Opcode::DS_READ2_B32, Opcode::DsRead2B32},
-    {Decoder::Opcode::DS_READ2ST64_B32, Opcode::DsRead2B32},
-    {Decoder::Opcode::DS_READ_B32, Opcode::DsReadB32},
-    {Decoder::Opcode::DS_READ_B64, Opcode::DsReadB32},
-    {Decoder::Opcode::DS_READ2_B64, Opcode::DsRead2B32},
-    {Decoder::Opcode::DS_READ2ST64_B64, Opcode::DsRead2B32},
-    {Decoder::Opcode::DS_READ_B96, Opcode::DsReadB32},
-    {Decoder::Opcode::DS_READ_B128, Opcode::DsReadB32},
-    {Decoder::Opcode::DS_WRITE_B8, Opcode::DsWriteByte},
-    {Decoder::Opcode::DS_WRITE_B16, Opcode::DsWriteShort},
-    {Decoder::Opcode::DS_WRITE2_B32, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE2ST64_B32, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE2_B64, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE2ST64_B64, Opcode::DsWrite2B32},
-    {Decoder::Opcode::DS_WRITE_B32, Opcode::DsWriteB32},
-    {Decoder::Opcode::DS_WRITE_B64, Opcode::DsWriteB32},
-    {Decoder::Opcode::DS_WRITE_B96, Opcode::DsWriteB32},
-    {Decoder::Opcode::DS_WRITE_B128, Opcode::DsWriteB32},
-    {Decoder::Opcode::DS_MIN_F32, Opcode::DsMinF32},
-    {Decoder::Opcode::DS_MAX_F32, Opcode::DsMaxF32},
-    {Decoder::Opcode::DS_CONSUME, Opcode::DsConsume},
-    {Decoder::Opcode::DS_APPEND, Opcode::DsAppend},
-    {Decoder::Opcode::DS_WRITE_ADDTID_B32, Opcode::DsWriteAddtidB32},
-    {Decoder::Opcode::DS_READ_ADDTID_B32, Opcode::DsReadAddtidB32},
-    {Decoder::Opcode::IMAGE_GET_RESINFO, Opcode::ImageGetResinfo},
-    {Decoder::Opcode::IMAGE_GET_LOD, Opcode::ImageGetLod},
-    {Decoder::Opcode::IMAGE_LOAD, Opcode::ImageLoad},
-    {Decoder::Opcode::IMAGE_LOAD_MIP, Opcode::ImageLoad},
-    {Decoder::Opcode::IMAGE_STORE, Opcode::ImageStore},
-    {Decoder::Opcode::IMAGE_STORE_MIP, Opcode::ImageStore},
-    {Decoder::Opcode::IMAGE_ATOMIC_ADD, Opcode::AtomicAddU32},
-    {Decoder::Opcode::IMAGE_ATOMIC_UMIN, Opcode::AtomicUMinU32},
-    {Decoder::Opcode::IMAGE_ATOMIC_UMAX, Opcode::AtomicUMaxU32},
-    {Decoder::Opcode::IMAGE_ATOMIC_AND, Opcode::AtomicAndU32},
-    {Decoder::Opcode::IMAGE_ATOMIC_OR, Opcode::AtomicOrU32},
-    {Decoder::Opcode::IMAGE_ATOMIC_XOR, Opcode::AtomicXorU32},
-    {Decoder::Opcode::IMAGE_SAMPLE, Opcode::ImageSample},
-    {Decoder::Opcode::IMAGE_GATHER4_LZ, Opcode::ImageGather4},
-    {Decoder::Opcode::IMAGE_GATHER4_C, Opcode::ImageGather4},
-    {Decoder::Opcode::IMAGE_GATHER4_C_LZ, Opcode::ImageGather4},
-    {Decoder::Opcode::IMAGE_GATHER4_LZ_O, Opcode::ImageGather4},
-    {Decoder::Opcode::IMAGE_GATHER4_C_O, Opcode::ImageGather4},
-    {Decoder::Opcode::IMAGE_GATHER4_C_LZ_O, Opcode::ImageGather4},
-    {Decoder::Opcode::IMAGE_GATHER4H, Opcode::ImageGather4},
-};
-
-} // namespace
-
-void SetError(std::string* error, const char* message) {
-	if (error != nullptr) {
-		*error = message;
-	}
-}
-
-std::optional<Opcode> LookupIrOpcode(Decoder::Opcode opcode) {
-	for (const auto& op: LOWER_OPS) {
-		if (op.decoded == opcode) {
-			return op.ir;
-		}
-	}
-	return std::nullopt;
-}
-
-bool IsReversedBinary(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::V_SUBREV_F32:
-		case Decoder::Opcode::V_SUBREV_F16:
-		case Decoder::Opcode::V_SUBREV_NC_U32:
-		case Decoder::Opcode::V_LSHLREV_B32:
-		case Decoder::Opcode::V_LSHRREV_B32:
-		case Decoder::Opcode::V_ASHRREV_I32:
-		case Decoder::Opcode::V_LSHRREV_B64:
-		case Decoder::Opcode::V_LSHLREV_B16:
-		case Decoder::Opcode::V_LSHRREV_B16:
-		case Decoder::Opcode::V_ASHRREV_I16:
-		case Decoder::Opcode::V_SUBREV_I32: return true;
-		default: return false;
-	}
-}
-
-bool IsVectorCarryOutOpcode(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::V_ADD_I32:
-		case Decoder::Opcode::V_SUB_I32:
-		case Decoder::Opcode::V_SUBREV_I32: return true;
-		default: return false;
-	}
-}
-
-bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::S_ABS_I32:
-		case Decoder::Opcode::S_BCNT1_I32_B32:
-		case Decoder::Opcode::S_BCNT1_I32_B64:
-		case Decoder::Opcode::S_AND_B32:
-		case Decoder::Opcode::S_ANDN2_B32:
-		case Decoder::Opcode::S_OR_B32:
-		case Decoder::Opcode::S_ORN2_B32:
-		case Decoder::Opcode::S_XOR_B32:
-		case Decoder::Opcode::S_NAND_B32:
-		case Decoder::Opcode::S_NOR_B32:
-		case Decoder::Opcode::S_XNOR_B32:
-		case Decoder::Opcode::S_NOT_B32:
-		case Decoder::Opcode::S_LSHL_B32:
-		case Decoder::Opcode::S_LSHR_B32:
-		case Decoder::Opcode::S_ASHR_I32:
-		case Decoder::Opcode::S_BFE_U32:
-		case Decoder::Opcode::S_BFE_I32:
-		case Decoder::Opcode::S_AND_B64:
-		case Decoder::Opcode::S_ANDN2_B64:
-		case Decoder::Opcode::S_NOT_B64:
-		case Decoder::Opcode::S_OR_B64:
-		case Decoder::Opcode::S_ORN2_B64:
-		case Decoder::Opcode::S_XOR_B64:
-		case Decoder::Opcode::S_NAND_B64:
-		case Decoder::Opcode::S_NOR_B64:
-		case Decoder::Opcode::S_XNOR_B64:
-		case Decoder::Opcode::S_LSHL_B64:
-		case Decoder::Opcode::S_LSHR_B64:
-		case Decoder::Opcode::S_BFE_U64:
-		case Decoder::Opcode::S_WQM_B64:
-		case Decoder::Opcode::S_QUADMASK_B64: return true;
-		default: return false;
-	}
-}
-
-bool ScalarResultIs64Bit(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::S_AND_B64:
-		case Decoder::Opcode::S_ANDN2_B64:
-		case Decoder::Opcode::S_NOT_B64:
-		case Decoder::Opcode::S_OR_B64:
-		case Decoder::Opcode::S_ORN2_B64:
-		case Decoder::Opcode::S_XOR_B64:
-		case Decoder::Opcode::S_NAND_B64:
-		case Decoder::Opcode::S_NOR_B64:
-		case Decoder::Opcode::S_XNOR_B64:
-		case Decoder::Opcode::S_LSHL_B64:
-		case Decoder::Opcode::S_LSHR_B64:
-		case Decoder::Opcode::S_BFE_U64:
-		case Decoder::Opcode::S_WQM_B64:
-		case Decoder::Opcode::S_QUADMASK_B64: return true;
-		default: return false;
-	}
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc

@@ -1,354 +0,0 @@
-// Semantic operations produced by RDNA2 instruction lowering. The typed frontend consumes this
-// baseline set before optimization and resource planning; it must not grow without decoder support.
-IR_OPCODE(ControlNop, General, Control, Void)
-IR_OPCODE(Waitcnt, General, Control, Void)
-IR_OPCODE(Barrier, General, Control, Void)
-IR_OPCODE(Sendmsg, General, Control, Void)
-IR_OPCODE(TtraceData, General, Control, Void)
-IR_OPCODE(InstPrefetch, General, Control, Void)
-IR_OPCODE(MoveU32, General, Move)
-IR_OPCODE(MoveF32Bits, General, Move)
-IR_OPCODE(MoveRelDestU32, General, Lane)
-IR_OPCODE(MoveRelSourceU32, General, Lane)
-IR_OPCODE(MoveU64, General, Move)
-IR_OPCODE(WqmB64, General, Move)
-IR_OPCODE(SaveexecB32, General, State)
-IR_OPCODE(SaveexecB64, General, State)
-IR_OPCODE(ReadFirstLaneU32, General, Lane)
-IR_OPCODE(ReadLaneU32, General, Lane)
-IR_OPCODE(WriteLaneU32, General, Lane)
-IR_OPCODE(Permlane16B32, General, Lane)
-IR_OPCODE(Permlanex16B32, General, Lane)
-IR_OPCODE(AbsI32, General, SimpleInteger)
-IR_OPCODE(IAddU32, General, SimpleInteger)
-IR_OPCODE(IAddCarryU32, General, State)
-IR_OPCODE(ISubBorrowCarryU32, General, State)
-IR_OPCODE(ISubBorrowU32, General, State)
-IR_OPCODE(ScalarAddCarryU32, General, State)
-IR_OPCODE(ScalarSubBorrowU32, General, State)
-IR_OPCODE(ScalarSubBorrowCarryU32, General, State)
-IR_OPCODE(ScalarSignedAddOverflowI32, General, State)
-IR_OPCODE(ScalarSignedSubOverflowI32, General, State)
-IR_OPCODE(ScalarShiftLeftAddCarryU32, General, State)
-IR_OPCODE(ISubU32, General, SimpleInteger)
-IR_OPCODE(IMulU32, General, SimpleInteger)
-IR_OPCODE(UMulHighU32, General, SimpleInteger)
-IR_OPCODE(SMulHighI32, General, SimpleInteger)
-IR_OPCODE(IMadI24U32, General, ExtendedInteger)
-IR_OPCODE(UMadU24U32, General, ExtendedInteger)
-IR_OPCODE(UMadU64U32, General, ExtendedInteger)
-IR_OPCODE(SadU32, General, ExtendedInteger)
-IR_OPCODE(IAdd3U32, General, ExtendedInteger)
-IR_OPCODE(IMulI24U32, General, ExtendedInteger)
-IR_OPCODE(UMulU24U32, General, ExtendedInteger)
-IR_OPCODE(IMinI32, General, SimpleInteger)
-IR_OPCODE(IMaxI32, General, SimpleInteger)
-IR_OPCODE(IMin3I32, General, SimpleInteger)
-IR_OPCODE(IMax3I32, General, SimpleInteger)
-IR_OPCODE(IMed3I32, General, SimpleInteger)
-IR_OPCODE(UMinU32, General, SimpleInteger)
-IR_OPCODE(UMaxU32, General, SimpleInteger)
-IR_OPCODE(UMin3U32, General, SimpleInteger)
-IR_OPCODE(UMax3U32, General, SimpleInteger)
-IR_OPCODE(UMed3U32, General, SimpleInteger)
-IR_OPCODE(BitwiseAndU32, General, SimpleInteger)
-IR_OPCODE(BitwiseAndU64, General, U64Mask)
-IR_OPCODE(BitwiseAndNotU32, General, ComposedInteger)
-IR_OPCODE(BitwiseAndNotU64, General, U64Mask)
-IR_OPCODE(BitwiseOrU32, General, SimpleInteger)
-IR_OPCODE(BitwiseOrU64, General, U64Mask)
-IR_OPCODE(BitwiseOrNotU32, General, ComposedInteger)
-IR_OPCODE(BitwiseOrNotU64, General, U64Mask)
-IR_OPCODE(BitwiseAndOrU32, General, ComposedInteger)
-IR_OPCODE(BitwiseOr3U32, General, ComposedInteger)
-IR_OPCODE(BitwiseXorU32, General, SimpleInteger)
-IR_OPCODE(BitwiseXorU64, General, U64Mask)
-IR_OPCODE(BitwiseXor3U32, General, ComposedInteger)
-IR_OPCODE(BitwiseNandU32, General, ComposedInteger)
-IR_OPCODE(BitwiseNandU64, General, U64Mask)
-IR_OPCODE(BitwiseNorU32, General, ComposedInteger)
-IR_OPCODE(BitwiseNorU64, General, U64Mask)
-IR_OPCODE(BitwiseXnorU32, General, ComposedInteger)
-IR_OPCODE(BitwiseXnorU64, General, U64Mask)
-IR_OPCODE(BitwiseNotU32, General, SimpleInteger)
-IR_OPCODE(BitwiseNotU64, General, U64Mask)
-IR_OPCODE(BitClearU32, General, ExtendedInteger)
-IR_OPCODE(BitSetU32, General, ExtendedInteger)
-IR_OPCODE(BitReverseU32, General, SimpleInteger)
-IR_OPCODE(BitCountU32, General, SimpleInteger)
-IR_OPCODE(BitCountU64, General, SimpleInteger)
-IR_OPCODE(BitReplicateB64B32, General, ExtendedInteger)
-IR_OPCODE(QuadmaskB64, General, ExtendedInteger)
-IR_OPCODE(BitCountAddU32, General, ExtendedInteger)
-IR_OPCODE(MaskedBitCountLowU32, General, ExtendedInteger)
-IR_OPCODE(MaskedBitCountHighU32, General, ExtendedInteger)
-IR_OPCODE(FindLsbU32, General, SimpleInteger)
-IR_OPCODE(FindLsbU64, General, ComposedInteger)
-IR_OPCODE(FindMsbFromHighU32, General, ComposedInteger)
-IR_OPCODE(FindMsbFromHighU64, General, ComposedInteger)
-IR_OPCODE(BitFieldMaskU32, General, ExtendedInteger)
-IR_OPCODE(BitFieldMaskU64, General, ExtendedInteger)
-IR_OPCODE(BitFieldExtractU32, General, ExtendedInteger)
-IR_OPCODE(BitFieldExtractI32, General, ExtendedInteger)
-IR_OPCODE(BitFieldExtractU64, General, ExtendedInteger)
-IR_OPCODE(BitFieldExtract3U32, General, ExtendedInteger)
-IR_OPCODE(BitFieldExtract3I32, General, ExtendedInteger)
-IR_OPCODE(BitFieldInsertSelectU32, General, ExtendedInteger)
-IR_OPCODE(BitCompare0B32, General, ExtendedInteger)
-IR_OPCODE(BitCompare1B32, General, ExtendedInteger)
-IR_OPCODE(AlignBitU32, General, ExtendedInteger)
-IR_OPCODE(AlignByteU32, General, ExtendedInteger)
-IR_OPCODE(ShiftLeftAddU32, General, ExtendedInteger)
-IR_OPCODE(AddShiftLeftU32, General, ExtendedInteger)
-IR_OPCODE(XorAddU32, General, ExtendedInteger)
-IR_OPCODE(ShiftLeftOrU32, General, ExtendedInteger)
-IR_OPCODE(ShiftLeftLogicalU32, General, SimpleInteger)
-IR_OPCODE(ShiftLeftLogicalU64, General, SimpleInteger)
-IR_OPCODE(ShiftLeftLogicalU16, General, Integer16)
-IR_OPCODE(ShiftRightLogicalU32, General, SimpleInteger)
-IR_OPCODE(ShiftRightLogicalU64, General, SimpleInteger)
-IR_OPCODE(ShiftRightLogicalU16, General, Integer16)
-IR_OPCODE(ShiftRightArithmeticI32, General, SimpleInteger)
-IR_OPCODE(ShiftRightArithmeticI16, General, Integer16)
-IR_OPCODE(SelectU32, General, ExtendedInteger, U32, U1, U32, U32)
-IR_OPCODE(SelectMaskU32, General, ExtendedInteger)
-IR_OPCODE(SelectF32Bits, General, ExtendedInteger)
-IR_OPCODE(SelectMaskF32Bits, General, ExtendedInteger)
-IR_OPCODE(SelectU64, General, ExtendedInteger)
-IR_OPCODE(PackLowLowU16, General, ExtendedInteger)
-IR_OPCODE(PackLowHighU16, General, ExtendedInteger)
-IR_OPCODE(PackHighHighU16, General, ExtendedInteger)
-IR_OPCODE(CompareFalse, Compare, IntegerCompare)
-IR_OPCODE(CompareTrue, Compare, IntegerCompare)
-IR_OPCODE(CompareEqU32, Compare, IntegerCompare)
-IR_OPCODE(CompareNeU32, Compare, IntegerCompare)
-IR_OPCODE(CompareGtU32, Compare, IntegerCompare)
-IR_OPCODE(CompareGeU32, Compare, IntegerCompare)
-IR_OPCODE(CompareLtU32, Compare, IntegerCompare)
-IR_OPCODE(CompareLeU32, Compare, IntegerCompare)
-IR_OPCODE(CompareEqU64, Compare64, IntegerCompare)
-IR_OPCODE(CompareGtU64, Compare64, IntegerCompare)
-IR_OPCODE(CompareNeU64, Compare64, IntegerCompare)
-IR_OPCODE(CompareMaskEqU32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskNeU32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskGtU32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskGeU32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskLtU32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskLeU32, Compare, IntegerCompare)
-IR_OPCODE(CompareEqI32, Compare, IntegerCompare)
-IR_OPCODE(CompareNeI32, Compare, IntegerCompare)
-IR_OPCODE(CompareGtI32, Compare, IntegerCompare)
-IR_OPCODE(CompareGeI32, Compare, IntegerCompare)
-IR_OPCODE(CompareLtI32, Compare, IntegerCompare)
-IR_OPCODE(CompareLeI32, Compare, IntegerCompare)
-IR_OPCODE(CompareEqI16, Compare, Integer16Compare)
-IR_OPCODE(CompareNeI16, Compare, Integer16Compare)
-IR_OPCODE(CompareGtI16, Compare, Integer16Compare)
-IR_OPCODE(CompareGeI16, Compare, Integer16Compare)
-IR_OPCODE(CompareLtI16, Compare, Integer16Compare)
-IR_OPCODE(CompareLeI16, Compare, Integer16Compare)
-IR_OPCODE(CompareMaskEqI32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskNeI32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskGtI32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskGeI32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskLtI32, Compare, IntegerCompare)
-IR_OPCODE(CompareMaskLeI32, Compare, IntegerCompare)
-IR_OPCODE(CompareEqU16, Compare, Integer16Compare)
-IR_OPCODE(CompareNeU16, Compare, Integer16Compare)
-IR_OPCODE(CompareGtU16, Compare, Integer16Compare)
-IR_OPCODE(CompareGeU16, Compare, Integer16Compare)
-IR_OPCODE(CompareLtU16, Compare, Integer16Compare)
-IR_OPCODE(CompareLeU16, Compare, Integer16Compare)
-IR_OPCODE(CompareEqF32, Compare, FloatCompare)
-IR_OPCODE(CompareNeF32, Compare, FloatCompare)
-IR_OPCODE(CompareGtF32, Compare, FloatCompare)
-IR_OPCODE(CompareGeF32, Compare, FloatCompare)
-IR_OPCODE(CompareLtF32, Compare, FloatCompare)
-IR_OPCODE(CompareLeF32, Compare, FloatCompare)
-IR_OPCODE(CompareOrderedF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnorderedF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnordEqF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnordNeF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnordGtF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnordGeF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnordLtF32, Compare, FloatCompare)
-IR_OPCODE(CompareUnordLeF32, Compare, FloatCompare)
-IR_OPCODE(CompareClassF32, Compare, FloatCompare)
-IR_OPCODE(CompareEqF16, Compare, FloatCompare)
-IR_OPCODE(CompareNeF16, Compare, FloatCompare)
-IR_OPCODE(CompareGtF16, Compare, FloatCompare)
-IR_OPCODE(CompareGeF16, Compare, FloatCompare)
-IR_OPCODE(CompareLtF16, Compare, FloatCompare)
-IR_OPCODE(CompareLeF16, Compare, FloatCompare)
-IR_OPCODE(CompareUnordNeF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskEqF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskNeF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskGtF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskGeF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskLtF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskLeF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordNeF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordGeF16, Compare, FloatCompare)
-IR_OPCODE(CompareMaskEqF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskNeF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskGtF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskGeF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskLtF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskLeF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordEqF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordNeF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordGtF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordGeF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordLtF32, Compare, FloatCompare)
-IR_OPCODE(CompareMaskUnordLeF32, Compare, FloatCompare)
-IR_OPCODE(ConvertByteU32ToF32, General, Conversion)
-IR_OPCODE(ConvertU32ToF32, General, Conversion)
-IR_OPCODE(ConvertI32ToF32, General, Conversion)
-IR_OPCODE(ConvertF32ToU32, General, Conversion)
-IR_OPCODE(ConvertF32ToI32, General, Conversion)
-IR_OPCODE(ConvertF32ToF16, General, Conversion)
-IR_OPCODE(ConvertF16ToF32, General, Conversion)
-IR_OPCODE(ConvertU16ToF16, General, Conversion)
-IR_OPCODE(ConvertF16ToU16, General, Conversion)
-IR_OPCODE(ConvertI16ToF16, General, Conversion)
-IR_OPCODE(ConvertF16ToI16, General, Conversion)
-IR_OPCODE(ConvertRoundPlusInfF32ToI32, General, Conversion)
-IR_OPCODE(ConvertFloorF32ToI32, General, Conversion)
-IR_OPCODE(FrexpExpI32F32, General, Conversion)
-IR_OPCODE(FrexpMantF32, General, Float)
-IR_OPCODE(ConvertI4ToOffsetF32, General, Conversion)
-IR_OPCODE(LdexpF32, General, Float)
-IR_OPCODE(PackF32ToF16Rtz, General, Conversion)
-IR_OPCODE(PackSnorm2x16F32, General, Conversion)
-IR_OPCODE(PackUnorm2x16F32, General, Conversion)
-IR_OPCODE(PackU16U32, General, ExtendedInteger)
-IR_OPCODE(PackU8F32, General, Conversion)
-IR_OPCODE(PackB32F16, General, Conversion)
-IR_OPCODE(PackedMadI16, General, PackedInteger16)
-IR_OPCODE(PackedMulLoU16, General, PackedInteger16)
-IR_OPCODE(PackedAddI16, General, PackedInteger16)
-IR_OPCODE(PackedSubI16, General, PackedInteger16)
-IR_OPCODE(PackedLshlrevB16, General, PackedInteger16)
-IR_OPCODE(PackedLshrrevB16, General, PackedInteger16)
-IR_OPCODE(PackedAshrrevI16, General, PackedInteger16)
-IR_OPCODE(PackedMaxI16, General, PackedInteger16)
-IR_OPCODE(PackedMinI16, General, PackedInteger16)
-IR_OPCODE(PackedMadU16, General, PackedInteger16)
-IR_OPCODE(PackedAddU16, General, PackedInteger16)
-IR_OPCODE(PackedSubU16, General, PackedInteger16)
-IR_OPCODE(PackedMaxU16, General, PackedInteger16)
-IR_OPCODE(PackedMinU16, General, PackedInteger16)
-IR_OPCODE(PackedAddF16, General, PackedFloat16)
-IR_OPCODE(PackedMulF16, General, PackedFloat16)
-IR_OPCODE(PackedMinF16, General, PackedFloat16)
-IR_OPCODE(PackedMaxF16, General, PackedFloat16)
-IR_OPCODE(PackedFmaF16, General, PackedFloat16)
-IR_OPCODE(AddF16, General, Float16)
-IR_OPCODE(SubF16, General, Float16)
-IR_OPCODE(MulF16, General, Float16)
-IR_OPCODE(MinF16, General, Float16)
-IR_OPCODE(MaxF16, General, Float16)
-IR_OPCODE(FmaF16, General, Float16)
-IR_OPCODE(MadMixF16, General, Float16)
-IR_OPCODE(IAddU16, General, Integer16)
-IR_OPCODE(ISubI16, General, Integer16)
-IR_OPCODE(IMinI16, General, Integer16)
-IR_OPCODE(IMaxI16, General, Integer16)
-IR_OPCODE(IMed3I16, General, Integer16)
-IR_OPCODE(UMinU16, General, Integer16)
-IR_OPCODE(UMaxU16, General, Integer16)
-IR_OPCODE(RcpF32, General, Float)
-IR_OPCODE(RcpIflagF32, General, Float)
-IR_OPCODE(FractF32, General, Float)
-IR_OPCODE(TruncF32, General, Float)
-IR_OPCODE(CeilF32, General, Float)
-IR_OPCODE(RoundEvenF32, General, Float)
-IR_OPCODE(FloorF32, General, Float)
-IR_OPCODE(Exp2F32, General, Float)
-IR_OPCODE(Log2F32, General, Float)
-IR_OPCODE(InverseSqrtF32, General, Float)
-IR_OPCODE(SqrtF32, General, Float)
-IR_OPCODE(RcpF16, General, Float16)
-IR_OPCODE(SqrtF16, General, Float16)
-IR_OPCODE(InverseSqrtF16, General, Float16)
-IR_OPCODE(Log2F16, General, Float16)
-IR_OPCODE(Exp2F16, General, Float16)
-IR_OPCODE(FloorF16, General, Float16)
-IR_OPCODE(CeilF16, General, Float16)
-IR_OPCODE(TruncF16, General, Float16)
-IR_OPCODE(RoundEvenF16, General, Float16)
-IR_OPCODE(SinF32, General, Float)
-IR_OPCODE(CosF32, General, Float)
-IR_OPCODE(CubeIdF32, General, Float)
-IR_OPCODE(CubeScF32, General, Float)
-IR_OPCODE(CubeTcF32, General, Float)
-IR_OPCODE(CubeMaF32, General, Float)
-IR_OPCODE(FAddF32, General, Float)
-IR_OPCODE(FSubF32, General, Float)
-IR_OPCODE(FMulF32, General, Float)
-IR_OPCODE(FMinF32, General, Float)
-IR_OPCODE(FMaxF32, General, Float)
-IR_OPCODE(FMadF32, General, Float)
-IR_OPCODE(Dot2AccF32F16, General, Float)
-IR_OPCODE(FMin3F32, General, Float)
-IR_OPCODE(FMax3F32, General, Float)
-IR_OPCODE(FMed3F32, General, Float)
-IR_OPCODE(Min3F16, General, Float16)
-IR_OPCODE(Max3F16, General, Float16)
-IR_OPCODE(Med3F16, General, Float16)
-IR_OPCODE(LoadSrtDword, General, Memory)
-IR_OPCODE(SLoadDword, General, Memory)
-IR_OPCODE(SBufferLoadDword, General, Memory)
-IR_OPCODE(BufferLoadUbyte, General, Memory)
-IR_OPCODE(BufferLoadSbyte, General, Memory)
-IR_OPCODE(BufferLoadUshort, General, Memory)
-IR_OPCODE(BufferLoadSshort, General, Memory)
-IR_OPCODE(BufferLoadDword, General, Memory)
-IR_OPCODE(BufferStoreByte, General, Memory)
-IR_OPCODE(BufferStoreShort, General, Memory)
-IR_OPCODE(BufferStoreDword, General, Memory)
-IR_OPCODE(AtomicSwapU32, General, Memory)
-IR_OPCODE(AtomicAddU32, General, Memory)
-IR_OPCODE(AtomicSubU32, General, Memory)
-IR_OPCODE(AtomicSMinI32, General, Memory)
-IR_OPCODE(AtomicUMinU32, General, Memory)
-IR_OPCODE(AtomicSMaxI32, General, Memory)
-IR_OPCODE(AtomicUMaxU32, General, Memory)
-IR_OPCODE(AtomicAndU32, General, Memory)
-IR_OPCODE(AtomicOrU32, General, Memory)
-IR_OPCODE(AtomicXorU32, General, Memory)
-IR_OPCODE(AtomicFMinF32, General, Memory)
-IR_OPCODE(AtomicFMaxF32, General, Memory)
-IR_OPCODE(FlatLoadUbyte, General, Memory)
-IR_OPCODE(FlatLoadSbyte, General, Memory)
-IR_OPCODE(FlatLoadUshort, General, Memory)
-IR_OPCODE(FlatLoadSshort, General, Memory)
-IR_OPCODE(FlatLoadDword, General, Memory)
-IR_OPCODE(FlatStoreByte, General, Memory)
-IR_OPCODE(FlatStoreShort, General, Memory)
-IR_OPCODE(FlatStoreDword, General, Memory)
-IR_OPCODE(DsReadUbyte, General, Memory)
-IR_OPCODE(DsReadSbyte, General, Memory)
-IR_OPCODE(DsReadUshort, General, Memory)
-IR_OPCODE(DsReadSshort, General, Memory)
-IR_OPCODE(DsReadB32, General, Memory)
-IR_OPCODE(DsRead2B32, General, Memory)
-IR_OPCODE(DsWriteByte, General, Memory)
-IR_OPCODE(DsWriteShort, General, Memory)
-IR_OPCODE(DsWriteB32, General, Memory)
-IR_OPCODE(DsWrite2B32, General, Memory)
-IR_OPCODE(DsMinF32, General, Memory)
-IR_OPCODE(DsMaxF32, General, Memory)
-IR_OPCODE(DsSwizzleB32, General, Memory)
-IR_OPCODE(DsConsume, General, Memory)
-IR_OPCODE(DsAppend, General, Memory)
-IR_OPCODE(DsWriteAddtidB32, General, Memory)
-IR_OPCODE(DsReadAddtidB32, General, Memory)
-IR_OPCODE(ImageGetResinfo, General, Memory)
-IR_OPCODE(ImageGetLod, General, Memory)
-IR_OPCODE(ImageLoad, General, Memory)
-IR_OPCODE(ImageStore, General, Memory)
-IR_OPCODE(ImageSample, General, Memory)
-IR_OPCODE(ImageGather4, General, Memory)
-IR_OPCODE(LoadInputF32, General, Attribute)
-IR_OPCODE(Export, General, Attribute)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 #include <algorithm>
 #include <array>
@@ -16,7 +16,7 @@ constexpr uint32_t ImageBindingCount =
 
 bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
 	std::array<bool, NumScalarRegs> registers {};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) {
 				continue;
@@ -47,16 +47,16 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
 
 bool UsesGds(const Program& program, bool& uses_gds) {
 	uses_gds = false;
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) {
 				continue;
 			}
 			const auto index = inst.Flags<MemoryFlags>().index;
-			if (index >= program.values->memory_info.size()) {
+			if (index >= program.memory_info.size()) {
 				return false;
 			}
-			const auto kind = program.values->memory_info[index].kind;
+			const auto kind = program.memory_info[index].kind;
 			if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) {
 				return false;
 			}
@@ -76,12 +76,6 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
 		}
 		return false;
 	}
-	if (program.values == nullptr) {
-		if (error != nullptr) {
-			*error = "typed value program is not available";
-		}
-		return false;
-	}
 	if (push_constant_offset > NativePushConstantSize ||
 	    push_constant_offset % sizeof(uint32_t) != 0) {
 		if (error != nullptr) {
@@ -171,7 +165,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
 		AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
 	}
 	const bool uses_flattened_runtime =
-	    !program.values->srt_reads.empty() ||
+	    !program.srt_reads.empty() ||
 	    std::ranges::any_of(program.info.images, [](const ImageResource& image) {
 		    return image.indirect_mapping_capacity != 0u;
 	    });

src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp

@@ -101,7 +101,7 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s
 
 } // namespace
 
-ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size) {
+ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) {
 	ReadLaneStats stats;
 	if (wave_size != 32u && wave_size != 64u) {
 		return stats;

src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
@@ -8,6 +8,6 @@ struct ReadLaneStats {
 	uint32_t rewritten_reads = 0;
 };
 
-[[nodiscard]] ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size);
+[[nodiscard]] ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -1,7 +1,7 @@
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
 #include "graphics/guest_gpu/gpu_format.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shaderBindings.h"
 
 #include <algorithm>
@@ -105,10 +105,10 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou
 }
 
 const DescriptorSource* Source(const Program& program, uint32_t source) {
-	if (program.values == nullptr || source >= program.values->descriptor_sources.size()) {
+	if (source >= program.descriptor_sources.size()) {
 		return nullptr;
 	}
-	return &program.values->descriptor_sources[source];
+	return &program.descriptor_sources[source];
 }
 
 void MarkCleanFlatSlots(const Program& program, const DescriptorSource* source,
@@ -131,7 +131,7 @@ void MarkCleanFlatSlots(const Program& program, const DescriptorSource* source,
 			const auto slot = inst->Arg(1).Resolve();
 			if (slot.IsImmediate() && slot.GetType() == Type::U32 && slot.U32() < slots.size()) {
 				slots[slot.U32()] = 1u;
-				pending.push_back(program.values->srt_reads[slot.U32()].value);
+				pending.push_back(program.srt_reads[slot.U32()].value);
 			}
 			continue;
 		}
@@ -275,7 +275,7 @@ uint64_t AddressSpecialization(const AddressResource&           resource,
 }
 
 size_t FlattenedRuntimeDwords(const Program& program) {
-	size_t size = program.values != nullptr ? program.values->srt_reads.size() : 0u;
+	size_t size = program.srt_reads.size();
 	for (uint32_t resource = 0; resource < program.info.images.size(); resource++) {
 		const auto& image = program.info.images[resource];
 		if (image.indirect_root == resource) {
@@ -305,8 +305,7 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
 		}
 		return false;
 	}
-	if (program.values == nullptr ||
-	    snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) {
+	if (snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) {
 		if (error != nullptr) {
 			*error = "flattened SRT snapshot does not match the shader plan";
 		}
@@ -549,7 +548,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
 	}
 
 	std::vector<DescriptorSourceRequest> requests;
-	std::vector<uint8_t>                 clean_flat_slots(program.values->srt_reads.size());
+	std::vector<uint8_t>                 clean_flat_slots(program.srt_reads.size());
 	requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u +
 	                 program.info.samplers.size() + program.info.addresses.size());
 	for (const auto& buffer: program.info.buffers) {
@@ -925,13 +924,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
 			}
 		}
 	}
-	if (program.values == nullptr) {
-		if (error != nullptr) {
-			*error = "typed SSA is not ready";
-		}
-		return false;
-	}
-	auto memory_info = program.values->memory_info;
+	auto memory_info = program.memory_info;
 
 	struct SamplerUsage {
 		bool native     = false;
@@ -980,7 +973,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
 			pair.sampler = point_sampler[pair.sampler];
 		}
 	}
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto image_opcode = ImageOpcodeInfoOf(inst.GetOpcode());
 			if (image_opcode.access == ImageAccess::None) {
@@ -1023,7 +1016,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
 		}
 	}
 	program.info                = std::move(next);
-	program.values->memory_info = std::move(memory_info);
+	program.memory_info = std::move(memory_info);
 	snapshot                    = std::move(next_snapshot);
 	return true;
 }

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 #include "graphics/shader/shader.h"
 
@@ -94,8 +94,7 @@ ImageMipMode MipMode(const MemoryInfo& memory) {
 
 class Tracker {
 public:
-	Tracker(Program& program, ValueProgram& values)
-	    : m_program(program), m_values(values), m_info(program.info) {
+	explicit Tracker(Program& program): m_program(program), m_info(program.info) {
 		m_info.buffers.clear();
 		m_info.addresses.clear();
 		m_info.images.clear();
@@ -111,7 +110,7 @@ public:
 			return Fail(0, error, "SRT plan is not ready");
 		}
 		PlanIndirectImages();
-		for (auto* block: m_values.blocks) {
+		for (auto* block: m_program.blocks) {
 			for (auto& inst: *block) {
 				if (!Collect(inst, error)) {
 					return false;
@@ -123,7 +122,7 @@ public:
 			patch.handle->SetFlags<uint32_t>(patch.resource);
 		}
 		for (const auto& patch: m_memory_patches) {
-			auto& memory    = m_values.memory_info[patch.index];
+			auto& memory    = m_program.memory_info[patch.index];
 			memory.resource = patch.resource;
 			if (patch.has_sampler) {
 				memory.sampler = patch.sampler;
@@ -138,16 +137,16 @@ public:
 				plan.handle->SetArg(dword, plan.key);
 			}
 			for (const auto index: plan.memory) {
-				m_values.memory_info[index].planning_only = true;
+				m_program.memory_info[index].planning_only = true;
 			}
 		}
-		std::erase_if(m_values.dynamic_reads, [&](Value value) {
+		std::erase_if(m_program.dynamic_reads, [&](Value value) {
 			const auto* inst = value.Resolve().TryInstruction();
 			return std::ranges::any_of(m_indirect_images, [&](const IndirectImagePlan& plan) {
 				return std::ranges::find(plan.reads, inst) != plan.reads.end();
 			});
 		});
-		m_values.descriptor_sources          = std::move(m_sources);
+		m_program.descriptor_sources          = std::move(m_sources);
 		m_program.info                       = std::move(m_info);
 		m_program.resource_tracking_complete = true;
 		return true;
@@ -208,19 +207,19 @@ private:
 			if (left.value.IsEmpty() || right.value.IsEmpty()) {
 				return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left);
 			}
-			return EquivalentValue(m_values, left.value, right.value)
+			return EquivalentValue(m_program, left.value, right.value)
 			           ? AddressPart {left.value, left.rooted || right.rooted}
 			           : AddressPart {};
 		};
 		switch (inst->GetOpcode()) {
 			case ValueOpcode::SelectU32:
-				if (EquivalentValue(m_values, inst->Arg(0), active)) {
+				if (EquivalentValue(m_program, inst->Arg(0), active)) {
 					return finish(FindAddressPart(inst->Arg(1), active, visiting));
 				}
 				return finish(merge(FindAddressPart(inst->Arg(1), active, visiting),
 				                    FindAddressPart(inst->Arg(2), active, visiting), true));
 			case ValueOpcode::Phi: {
-				const auto invariant = ResolveInvariantPhi(m_values, value);
+				const auto invariant = ResolveInvariantPhi(m_program, value);
 				return finish(invariant.IsEmpty() ? AddressPart {}
 				                                  : FindAddressPart(invariant, active, visiting));
 			}
@@ -312,7 +311,7 @@ private:
 			}
 			bool same = true;
 			for (uint32_t i = 0; i < descriptor.dword_count; i++) {
-				same = same && EquivalentValue(m_values, current.dwords[i], descriptor.dwords[i]);
+				same = same && EquivalentValue(m_program, current.dwords[i], descriptor.dwords[i]);
 			}
 			if (same) {
 				return candidate;
@@ -342,10 +341,10 @@ private:
 			return nullptr;
 		}
 		index = read.Flags<MemoryFlags>().index;
-		if (index >= m_values.memory_info.size()) {
+		if (index >= m_program.memory_info.size()) {
 			return nullptr;
 		}
-		const auto& memory = m_values.memory_info[index];
+		const auto& memory = m_program.memory_info[index];
 		return memory.kind == ResourceKind::ScalarBuffer && memory.data_bits == 32u &&
 		               memory.data_dwords == 1u
 		           ? &memory
@@ -353,7 +352,7 @@ private:
 	}
 
 	bool MemoryIndexBelongsTo(uint32_t index, const Inst& owner) const {
-		for (const auto* block: m_values.blocks) {
+		for (const auto* block: m_program.blocks) {
 			for (const auto& inst: *block) {
 				const auto op = inst.GetOpcode();
 				if ((BufferAccessOf(op) == BufferAccess::None &&
@@ -446,7 +445,7 @@ private:
 			heap_handle = current_handle;
 			if (dword == 0u) {
 				heap_offset = heap_reads[dword]->Arg(1).Resolve();
-			} else if (!EquivalentValue(m_values, heap_offset, heap_reads[dword]->Arg(1))) {
+			} else if (!EquivalentValue(m_program, heap_offset, heap_reads[dword]->Arg(1))) {
 				return false;
 			}
 			plan.memory[dword] = memory_index;
@@ -537,7 +536,7 @@ private:
 	}
 
 	void PlanIndirectImages() {
-		for (auto* block: m_values.blocks) {
+		for (auto* block: m_program.blocks) {
 			for (auto& inst: *block) {
 				if (ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None ||
 				    inst.NumArgs() == 0u) {
@@ -588,7 +587,7 @@ private:
 			return Fail(pc, error, "GetAddressResource must have two address dwords");
 		}
 		DescriptorSource descriptor;
-		const auto&      memory = m_values.memory_info[memory_inst.Flags<MemoryFlags>().index];
+		const auto&      memory = m_program.memory_info[memory_inst.Flags<MemoryFlags>().index];
 		if (memory.address_is_full) {
 			const auto active = memory_inst.Arg(memory_inst.NumArgs() - 1u);
 			if (memory.kind != ResourceKind::Flat ||
@@ -797,14 +796,14 @@ private:
 			return true;
 		}
 		const auto flags = inst.Flags<MemoryFlags>();
-		if (flags.index >= m_values.memory_info.size()) {
+		if (flags.index >= m_program.memory_info.size()) {
 			return Fail(flags.pc, error,
 			            fmt::format("memory metadata index {} is out of range", flags.index));
 		}
 		if (inst.NumArgs() == 0) {
 			return Fail(flags.pc, error, "memory operation has no resource handle");
 		}
-		const auto& memory = m_values.memory_info[flags.index];
+		const auto& memory = m_program.memory_info[flags.index];
 		if (memory.planning_only || IsIndirectPlanningMemory(flags.index)) {
 			return true;
 		}
@@ -914,7 +913,7 @@ private:
 				}
 				bool alias = true;
 				for (uint32_t dword = 0; dword < 4; dword++) {
-					alias = alias && EquivalentValue(m_values, buffer_source->dwords[dword],
+					alias = alias && EquivalentValue(m_program, buffer_source->dwords[dword],
 					                                 image_source->dwords[dword]);
 				}
 				if (alias) {
@@ -926,7 +925,6 @@ private:
 	}
 
 	Program&                       m_program;
-	ValueProgram&                  m_values;
 	ShaderInfo                     m_info;
 	std::vector<DescriptorSource>  m_sources;
 	std::vector<HandlePatch>       m_handle_patches;
@@ -937,13 +935,7 @@ private:
 } // namespace
 
 bool TrackResources(Program& program, std::string* error) {
-	if (program.values == nullptr) {
-		if (error != nullptr) {
-			*error = "shader resource tracking requires typed IR";
-		}
-		return false;
-	}
-	return Tracker(program, *program.values).Run(error);
+	return Tracker(program).Run(error);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 #include <algorithm>
 #include <fmt/format.h>
@@ -66,7 +66,7 @@ bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& op
 		}
 		return false;
 	};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			switch (inst.GetOpcode()) {
 				case ValueOpcode::GetAttribute: {
@@ -140,7 +140,7 @@ bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& op
 					break;
 				}
 				case ValueOpcode::SetAttribute:
-					if (inst.Flags<ExportFlags>().index >= program.values->export_info.size()) {
+					if (inst.Flags<ExportFlags>().index >= program.export_info.size()) {
 						return Fail("typed export metadata index is out of range");
 					}
 					break;
@@ -156,7 +156,7 @@ void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* ve
 	AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex");
 	AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex");
 	uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() == ValueOpcode::GetAttribute) {
 				const auto attr       = inst.Arg(0).U32();
@@ -185,7 +185,7 @@ void CollectPixelInputs(const Program& program, const ShaderPixelInputInfo* pixe
 	}
 	std::array<bool, 32> per_vertex {};
 	std::array<bool, 32> interpolated {};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() == ValueOpcode::GetAttribute) {
 				interpolated[inst.Arg(0).U32()] = true;
@@ -228,7 +228,7 @@ void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& inf
 }
 
 void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() != ValueOpcode::GetBuiltin) {
 				continue;
@@ -277,12 +277,12 @@ bool CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex,
 		}
 		return false;
 	};
-	for (const auto* block: program.values->blocks) {
+	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() != ValueOpcode::SetAttribute) {
 				continue;
 			}
-			const auto& export_info = program.values->export_info[inst.Flags<ExportFlags>().index];
+			const auto& export_info = program.export_info[inst.Flags<ExportFlags>().index];
 			if (export_info.kind == ExportTargetKind::MrtZ) {
 				if (program.stage == ShaderType::Pixel && (export_info.en & 0x1u) != 0 &&
 				    pixel->ps_depth_export_enable) {
@@ -360,12 +360,6 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::
 		}
 		return false;
 	}
-	if (program.values == nullptr) {
-		if (error != nullptr) {
-			*error = "typed value program is not available";
-		}
-		return false;
-	}
 	if (!ValidateOptions(program, options, error)) {
 		return false;
 	}
@@ -377,7 +371,7 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::
 	next.inputs.clear();
 	next.outputs.clear();
 	next.has_bitwise_xor = std::any_of(
-	    program.values->blocks.begin(), program.values->blocks.end(), [](const auto* block) {
+	    program.blocks.begin(), program.blocks.end(), [](const auto* block) {
 		    return std::any_of(block->begin(), block->end(), [](const auto& inst) {
 			    return inst.GetOpcode() == ValueOpcode::BitwiseXor32;
 		    });

src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h"
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 
 #include <algorithm>
 #include <queue>
@@ -12,7 +12,7 @@ namespace {
 
 enum class LdsPhase { None, Read, Write };
 
-LdsPhase GetLdsPhase(const ValueProgram& program, const Inst& inst) {
+LdsPhase GetLdsPhase(const Program& program, const Inst& inst) {
 	const auto access = SharedAccessOf(inst.GetOpcode());
 	if (access != SharedAccess::Read && access != SharedAccess::Write) {
 		return LdsPhase::None;
@@ -59,7 +59,7 @@ uint32_t InsertBarrier(Block& block, Block::iterator position) {
 	return 1;
 }
 
-uint32_t InsertBlockBarriers(ValueProgram& program, Block& block) {
+uint32_t InsertBlockBarriers(Program& program, Block& block) {
 	uint32_t inserted = 0;
 	LdsPhase phase    = LdsPhase::None;
 	for (auto inst = block.begin(); inst != block.end(); ++inst) {
@@ -94,7 +94,7 @@ uint32_t InsertMergeBarrier(Block& block) {
 
 } // namespace
 
-SharedMemoryBarrierStats InsertSharedMemoryBarriers(ValueProgram& program, uint32_t wave_size,
+SharedMemoryBarrierStats InsertSharedMemoryBarriers(Program& program, uint32_t wave_size,
 	                                                 const ShaderComputeInputInfo& compute_info) {
 	SharedMemoryBarrierStats stats;
 	const auto threadgroup_size = compute_info.threads_num[0] * compute_info.threads_num[1] *

src/graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h

@@ -9,7 +9,7 @@ struct SharedMemoryBarrierStats {
 };
 
 [[nodiscard]] SharedMemoryBarrierStats
-InsertSharedMemoryBarriers(ValueProgram& program, uint32_t wave_size,
+InsertSharedMemoryBarriers(Program& program, uint32_t wave_size,
 	                       const ShaderComputeInputInfo& compute_info);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
 
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
@@ -50,7 +50,7 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
 	return true;
 }
 
-bool IsRawRead(const ValueProgram& values, const Inst& inst) {
+bool IsRawRead(const Program& values, const Inst& inst) {
 	const auto op = inst.GetOpcode();
 	if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) {
 		return false;
@@ -118,8 +118,7 @@ bool IsRuntimeIntegerOp(ValueOpcode op) {
 
 class RuntimeValidator {
 public:
-	RuntimeValidator(const Program& program, const ValueProgram& values)
-	    : m_program(program), m_values(values) {}
+	explicit RuntimeValidator(const Program& program): m_program(program) {}
 
 	bool Run(Value value, std::string& reason) { return Validate(value, reason); }
 
@@ -176,7 +175,7 @@ private:
 			return finish(true);
 		}
 		if (op == ValueOpcode::Phi) {
-			const auto invariant = ResolveInvariantPhi(m_values, value);
+			const auto invariant = ResolveInvariantPhi(m_program, value);
 			if (invariant.IsEmpty()) {
 				reason = "contains a control-dependent phi";
 				return finish(false);
@@ -196,7 +195,7 @@ private:
 			    inst->Arg(0).Resolve().TryInstruction()->GetOpcode() !=
 			        ValueOpcode::GetSrtResource ||
 			    !slot.IsImmediate() || slot.GetType() != Type::U32 ||
-			    slot.U32() >= m_values.srt_reads.size()) {
+			    slot.U32() >= m_program.srt_reads.size()) {
 				reason = "contains a malformed flattened SRT read";
 				return finish(false);
 			}
@@ -205,7 +204,7 @@ private:
 			                           ? ValueOpcode::GetAddressResource
 			                           : ValueOpcode::GetBufferResource;
 			const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr;
-			if (!IsRawRead(m_values, *inst) || handle == nullptr ||
+			if (!IsRawRead(m_program, *inst) || handle == nullptr ||
 			    handle->GetOpcode() != expected) {
 				reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op));
 				return finish(false);
@@ -252,24 +251,23 @@ private:
 	}
 
 	const Program&                  m_program;
-	const ValueProgram&             m_values;
 	std::unordered_set<const Inst*> m_visiting;
 };
 
 class PlanBuilder {
 public:
-	PlanBuilder(Program& program, ValueProgram& values): m_program(program), m_values(values) {}
+	explicit PlanBuilder(Program& program): m_program(program) {}
 
 	bool Run(std::string* error) {
-		m_values.srt_reads.clear();
-		m_values.dynamic_reads.clear();
-		for (auto* block: m_values.blocks) {
+		m_program.srt_reads.clear();
+		m_program.dynamic_reads.clear();
+		for (auto* block: m_program.blocks) {
 			for (auto& inst: *block) {
 				const auto op = inst.GetOpcode();
 				if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
 					const auto flags = inst.Flags<MemoryFlags>();
-					if (flags.index < m_values.memory_info.size()) {
-						const auto kind       = m_values.memory_info[flags.index].kind;
+					if (flags.index < m_program.memory_info.size()) {
+						const auto kind       = m_program.memory_info[flags.index].kind;
 						const bool crosswired = (op == ValueOpcode::LoadAddressU32 &&
 						                         kind == ResourceKind::ScalarBuffer) ||
 						                        (op == ValueOpcode::ReadConstBuffer &&
@@ -290,10 +288,10 @@ public:
 				}
 			}
 		}
-		for (auto* block: m_values.blocks) {
+		for (auto* block: m_program.blocks) {
 			for (auto& inst: *block) {
 				std::string reason;
-				if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_values, inst) &&
+				if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) &&
 				    inst.Arg(1).Resolve().IsImmediate() &&
 				    ValidateRuntimeValue(m_program, Value(&inst), reason) &&
 				    !Collect(Value(&inst), inst.Flags<MemoryFlags>().pc, error)) {
@@ -348,25 +346,25 @@ private:
 		}
 		m_visiting.pop_back();
 		m_visited.push_back(inst);
-		if (!IsRawRead(m_values, *inst)) {
+		if (!IsRawRead(m_program, *inst)) {
 			return true;
 		}
 		const auto offset = inst->Arg(1).Resolve();
 		if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
-			if (std::ranges::find(m_values.dynamic_reads, value) == m_values.dynamic_reads.end()) {
-				m_values.dynamic_reads.push_back(value);
+			if (std::ranges::find(m_program.dynamic_reads, value) == m_program.dynamic_reads.end()) {
+				m_program.dynamic_reads.push_back(value);
 			}
 			return true;
 		}
 		use_pc = inst->Flags<MemoryFlags>().pc;
-		for (uint32_t slot = 0; slot < m_values.srt_reads.size(); slot++) {
-			if (EquivalentValue(m_values, value, m_values.srt_reads[slot].value)) {
+		for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) {
+			if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) {
 				m_patches.push_back({inst, slot, false});
 				return true;
 			}
 		}
-		const auto slot = static_cast<uint32_t>(m_values.srt_reads.size());
-		m_values.srt_reads.push_back({value, slot, use_pc});
+		const auto slot = static_cast<uint32_t>(m_program.srt_reads.size());
+		m_program.srt_reads.push_back({value, slot, use_pc});
 		m_patches.push_back({inst, slot, true});
 		return true;
 	}
@@ -385,7 +383,7 @@ private:
 			for (const auto& use: uses) {
 				use.user->SetArg(use.operand, flat);
 			}
-			for (auto& info: m_values.block_info) {
+			for (auto& info: m_program.block_info) {
 				if (info.condition.Resolve() == Value(patch.inst)) {
 					info.condition = flat;
 				}
@@ -395,8 +393,8 @@ private:
 			}
 			if (patch.keep) {
 				const auto memory = patch.inst->Flags<MemoryFlags>().index;
-				if (memory < m_values.memory_info.size()) {
-					m_values.memory_info[memory].planning_only = true;
+				if (memory < m_program.memory_info.size()) {
+					m_program.memory_info[memory].planning_only = true;
 				}
 				block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
 			}
@@ -404,7 +402,6 @@ private:
 	}
 
 	Program&           m_program;
-	ValueProgram&      m_values;
 	std::vector<Inst*> m_visiting;
 	std::vector<Inst*> m_visited;
 	std::vector<Patch> m_patches;
@@ -414,7 +411,7 @@ class Evaluator {
 public:
 	Evaluator(const Program& program, const SrtRuntime& runtime,
 	          std::span<const uint8_t> clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr)
-	    : m_program(program), m_values(*program.values), m_runtime(runtime),
+	    : m_program(program), m_runtime(runtime),
 	      m_clean_flat_slots(clean_flat_slots), m_clean_evaluator(clean_evaluator) {}
 
 	void SetUsePc(uint32_t pc) { m_use_pc = pc; }
@@ -472,7 +469,7 @@ private:
 	}
 
 	bool EvaluatePhi(const Inst& inst, uint64_t& result, std::string* error) {
-		const auto value = ResolveInvariantPhi(m_values, Value(const_cast<Inst*>(&inst)));
+		const auto value = ResolveInvariantPhi(m_program, Value(const_cast<Inst*>(&inst)));
 		return !value.IsEmpty() ? EvaluateWide(value, result, error)
 		                        : Fail(error, "typed phi has runtime-dependent values");
 	}
@@ -518,10 +515,10 @@ private:
 
 	bool EvaluateRawRead(const Inst& inst, uint64_t& result, std::string* error) {
 		const auto flags = inst.Flags<MemoryFlags>();
-		if (flags.index >= m_values.memory_info.size()) {
+		if (flags.index >= m_program.memory_info.size()) {
 			return Fail(error, "raw scalar read has invalid metadata");
 		}
-		const auto& mem    = m_values.memory_info[flags.index];
+		const auto& mem    = m_program.memory_info[flags.index];
 		const auto* handle = inst.Arg(0).ResolveInstruction();
 		if (handle == nullptr) {
 			return Fail(error, "raw scalar read has no descriptor handle");
@@ -609,20 +606,20 @@ private:
 			case ValueOpcode::ReadConst: {
 				const auto slot = inst.Arg(1).Resolve();
 				if (!slot.IsImmediate() || slot.GetType() != Type::U32 ||
-				    slot.U32() >= m_values.srt_reads.size()) {
+				    slot.U32() >= m_program.srt_reads.size()) {
 					return Fail(error, "invalid flattened SRT index");
 				}
 				if (slot.U32() < m_clean_flat_slots.size() &&
 				    m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) {
 					m_clean_evaluator->SetUsePc(m_use_pc);
-					return m_clean_evaluator->EvaluateWide(m_values.srt_reads[slot.U32()].value,
+					return m_clean_evaluator->EvaluateWide(m_program.srt_reads[slot.U32()].value,
 					                                       result, error);
 				}
-				return EvaluateWide(m_values.srt_reads[slot.U32()].value, result, error);
+				return EvaluateWide(m_program.srt_reads[slot.U32()].value, result, error);
 			}
 			case ValueOpcode::LoadAddressU32:
 			case ValueOpcode::ReadConstBuffer:
-				if (IsRawRead(m_values, inst)) {
+				if (IsRawRead(m_program, inst)) {
 					return EvaluateRawRead(inst, result, error);
 				}
 				break;
@@ -856,7 +853,6 @@ private:
 	}
 
 	const Program&                            m_program;
-	const ValueProgram&                       m_values;
 	const SrtRuntime&                         m_runtime;
 	std::span<const uint8_t>                  m_clean_flat_slots;
 	Evaluator*                                m_clean_evaluator = nullptr;
@@ -866,10 +862,10 @@ private:
 };
 
 const DescriptorSource* Source(const Program& program, uint32_t source) {
-	if (program.values == nullptr || source >= program.values->descriptor_sources.size()) {
+	if (source >= program.descriptor_sources.size()) {
 		return nullptr;
 	}
-	return &program.values->descriptor_sources[source];
+	return &program.descriptor_sources[source];
 }
 
 bool EvaluateRuntimeSourcesImpl(const Program&                           program,
@@ -877,7 +873,7 @@ bool EvaluateRuntimeSourcesImpl(const Program&                           program
                                 const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
                                 std::vector<uint32_t>& flat, bool evaluate_flat,
                                 std::span<const uint8_t> clean_flat_slots, std::string* error) {
-	if (program.values == nullptr || !program.srt_plan_complete) {
+	if (!program.srt_plan_complete) {
 		if (error != nullptr) {
 			*error = Diagnostic(program, 0, "typed SRT plan is not ready");
 		}
@@ -917,8 +913,8 @@ bool EvaluateRuntimeSourcesImpl(const Program&                           program
 	}
 	std::vector<uint32_t> flattened;
 	if (evaluate_flat) {
-		flattened.resize(program.values->srt_reads.size());
-		for (const auto& read: program.values->srt_reads) {
+		flattened.resize(program.srt_reads.size());
+		for (const auto& read: program.srt_reads) {
 			const bool clean    = read.flat_offset < clean_flat_slots.size() &&
 			                      clean_flat_slots[read.flat_offset] != 0u;
 			auto&      selected = clean ? clean_evaluator : evaluator;
@@ -939,23 +935,18 @@ bool EvaluateRuntimeSourcesImpl(const Program&                           program
 } // namespace
 
 bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason) {
-	if (program.values == nullptr) {
-		reason = "typed SSA is not ready";
-		return false;
-	}
-	return RuntimeValidator(program, *program.values).Run(value, reason);
+	return RuntimeValidator(program).Run(value, reason);
 }
 
 bool BuildSrtPlan(Program& program, std::string* error) {
-	if (program.values == nullptr || program.resource_tracking_complete) {
+	if (program.resource_tracking_complete) {
 		if (error != nullptr) {
-			*error = program.values == nullptr ? "typed SSA is not ready"
-			                                   : "cannot rebuild SRT after resource tracking";
+			*error = "cannot rebuild SRT after resource tracking";
 		}
 		return false;
 	}
 	program.srt_plan_complete = false;
-	if (!PlanBuilder(program, *program.values).Run(error)) {
+	if (!PlanBuilder(program).Run(error)) {
 		return false;
 	}
 	program.srt_plan_complete = true;

src/graphics/shader/shader.cpp

@@ -15,7 +15,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shaderVertexMetadata.h"
 #include "kernel/memory.h"
 #include "libs/errno.h"
@@ -49,9 +49,6 @@ namespace Libs::Graphics {
 
 namespace {
 
-constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
-constexpr uint32_t PsInputFlatShade  = 0x00000400u;
-
 bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) {
 	return value != nullptr &&
 	       Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, value, sizeof(*value));
@@ -59,40 +56,6 @@ bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) {
 
 } // namespace
 
-uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input) {
-	return input < info.input_num ? info.interpolator_settings[input] & PsInputOffsetMask : input;
-}
-
-uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info,
-                                      std::span<const uint32_t> active_inputs, uint32_t input) {
-	std::array<bool, 32> used_locations {};
-	for (const auto active_input: active_inputs) {
-		auto location = ShaderPixelParameterMappedLocation(info, active_input);
-		if (location < used_locations.size() && used_locations[location]) {
-			location = active_input;
-			while (location < used_locations.size() && used_locations[location]) {
-				location++;
-			}
-			EXIT_NOT_IMPLEMENTED(location >= used_locations.size());
-		}
-
-		if (active_input == input) {
-			return location;
-		}
-		used_locations[location] = true;
-	}
-	return ShaderPixelParameterMappedLocation(info, input);
-}
-
-bool ShaderPixelParameterIsFlat(const ShaderPixelInputInfo& info, uint32_t input) {
-	return input < info.input_num && (info.interpolator_settings[input] & PsInputFlatShade) != 0 &&
-	       !ShaderPixelParameterIsCustom(info, input);
-}
-
-bool ShaderPixelParameterIsCustom(const ShaderPixelInputInfo& info, uint32_t input) {
-	return input < 32u && (info.custom_interpolation_mask & (1u << input)) != 0;
-}
-
 struct ShaderBinaryInfo {
 	uint8_t  signature[7];
 	uint8_t  version;
@@ -1035,7 +998,7 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog
 	    program.bindings.push_constant_size, program.bindings.descriptors.size(),
 	    program.bindings.user_data_registers.size(), program.info.buffers.size(),
 	    program.info.images.size(), program.info.samplers.size(), program.info.addresses.size(),
-	    program.values != nullptr ? program.values->srt_reads.size() : 0u);
+	    program.srt_reads.size());
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		const auto& buffer = program.info.buffers[i];
 		ret +=

src/graphics/shader/shader.h

@@ -84,7 +84,6 @@ struct ShaderVertexInputInfo {
 
 	ShaderBufferResource    resources[RES_MAX];
 	ShaderVertexDestination resources_dst[RES_MAX];
-	int                     resource_fetch_components[RES_MAX] = {};
 	ShaderVertexInputBuffer buffers[RES_MAX];
 	ShaderStageRuntime      stage;
 	int                     resources_num       = 0;

src/graphics/shader/shaderPixelParameter.cpp

@@ -0,0 +1,48 @@
+#include "graphics/shader/shader.h"
+
+#include "common/assert.h"
+
+namespace Libs::Graphics {
+
+namespace {
+
+constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
+constexpr uint32_t PsInputFlatShade  = 0x00000400u;
+
+} // namespace
+
+uint32_t ShaderPixelParameterMappedLocation(const ShaderPixelInputInfo& info, uint32_t input) {
+	return input < info.input_num ? info.interpolator_settings[input] & PsInputOffsetMask : input;
+}
+
+uint32_t ShaderPixelParameterLocation(const ShaderPixelInputInfo& info,
+                                      std::span<const uint32_t> active_inputs, uint32_t input) {
+	std::array<bool, 32> used_locations {};
+	for (const auto active_input: active_inputs) {
+		auto location = ShaderPixelParameterMappedLocation(info, active_input);
+		if (location < used_locations.size() && used_locations[location]) {
+			location = active_input;
+			while (location < used_locations.size() && used_locations[location]) {
+				location++;
+			}
+			EXIT_NOT_IMPLEMENTED(location >= used_locations.size());
+		}
+
+		if (active_input == input) {
+			return location;
+		}
+		used_locations[location] = true;
+	}
+	return ShaderPixelParameterMappedLocation(info, input);
+}
+
+bool ShaderPixelParameterIsFlat(const ShaderPixelInputInfo& info, uint32_t input) {
+	return input < info.input_num && (info.interpolator_settings[input] & PsInputFlatShade) != 0 &&
+	       !ShaderPixelParameterIsCustom(info, input);
+}
+
+bool ShaderPixelParameterIsCustom(const ShaderPixelInputInfo& info, uint32_t input) {
+	return input < 32u && (info.custom_interpolation_mask & (1u << input)) != 0;
+}
+
+} // namespace Libs::Graphics

src/libs/agc.cpp

@@ -67,7 +67,7 @@ void Shutdown() {
 	g_renderer = nullptr;
 }
 
-void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, uint32_t* cmd_buffer) {
+void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, const uint32_t* cmd_buffer) {
 	EXIT_IF(type == nullptr);
 
 	static std::atomic_int id = 0;
@@ -3963,39 +3963,27 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchAddress(uint32_t*             cmd,
 	return OK;
 }
 
-int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchData(uint32_t* cmd, uint32_t context_id,
-                                                   uint32_t data_sel, uint64_t data) {
+int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchData(uint32_t* cmd, uint64_t data) {
 	PRINT_NAME();
 
-	LOGF("\t cmd        = 0x%016" PRIx64 "\n"
-	     "\t context_id = 0x%08" PRIx32 "\n"
-	     "\t data_sel   = 0x%08" PRIx32 "\n"
-	     "\t data       = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), context_id, data_sel, data);
+	LOGF("\t cmd  = 0x%016" PRIx64 "\n"
+	     "\t data = 0x%016" PRIx64 "\n",
+	     reinterpret_cast<uint64_t>(cmd), data);
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
-	auto op = (cmd[0] >> 8u) & 0xffu;
+	const auto op             = (cmd[0] >> 8u) & 0xffu;
+	const bool is_release_mem = op == Pm4::IT_RELEASE_MEM ||
+	                            (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM);
+	if (!is_release_mem) {
+		return GRAPHICS5_ERROR_INVALID_PACKET;
+	}
 
-	if ((op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM) ||
-	    op == Pm4::IT_RELEASE_MEM) {
-		uint64_t packet_data = data;
-		if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM && context_id > 1 &&
-		    data_sel == 1) {
-			// Agc Core ring-buffer release packets pack the segment generation
-			// into bits 24..31 and wrap that byte every 256 submissions. The
-			// patch context carries the monotonic generation, so expand the
-			// packed value before it reaches the command processor.
-			packet_data = (static_cast<uint64_t>(context_id - 2u) << 24u) | (data & 0x00ffffffull);
-		}
-
-		cmd[5] = static_cast<uint32_t>(packet_data & 0xffffffffu);
-		cmd[6] = static_cast<uint32_t>((packet_data >> 32u) & 0xffffffffu);
-	} else if (op == Pm4::IT_EVENT_WRITE_EOP) {
-		cmd[4] = static_cast<uint32_t>(data & 0xffffffffu);
-		cmd[5] = static_cast<uint32_t>((data >> 32u) & 0xffffffffu);
-	} else {
-		EXIT("unsupported queueEndOfPipeAction packet for data patch: 0x%08" PRIx32 "\n", cmd[0]);
+	const auto interrupt = (cmd[2] >> 24u) & 0x7u;
+	const auto data_sel  = (cmd[2] >> 29u) & 0x7u;
+	if (interrupt != 4u && data_sel != 5u) {
+		cmd[5] = static_cast<uint32_t>(data & 0xffffffffu);
+		cmd[6] = static_cast<uint32_t>((data >> 32u) & 0xffffffffu);
 	}
 
 	return OK;
@@ -4228,37 +4216,10 @@ struct TessellationDriverState {
 
 static TessellationDriverState g_tessellation_driver_state {};
 
-static bool dcb_has_queued_interrupt(const uint32_t* dcb, uint32_t size_in_dwords) {
-	if (dcb == nullptr) {
-		return false;
-	}
-
-	for (uint32_t offset = 0; offset < size_in_dwords;) {
-		auto cmd_id = dcb[offset];
-		auto len    = KYTY_PM4_LEN(cmd_id);
-		if (len == 0 || len > size_in_dwords - offset) {
-			return false;
-		}
-
-		auto op = (cmd_id >> 8u) & 0xffu;
-		if (op == Pm4::IT_NOP && KYTY_PM4_R(cmd_id) == Pm4::R_RELEASE_MEM && len >= 6) {
-			auto interrupt = (dcb[offset + 2] >> 24u) & 0x7u;
-			if (interrupt == 1 || interrupt == 2 || interrupt >= 4) {
-				return true;
-			}
-		}
-
-		offset += len;
-	}
-
-	return false;
-}
-
 static void submit_dcb(uint32_t* dcb, uint32_t size_in_dwords) {
 	GraphicsDbgDumpDcb("d", size_in_dwords, dcb);
 	EXIT_IF(g_renderer == nullptr);
-	g_renderer->GetGpu().Submit(dcb, size_in_dwords, nullptr, 0,
-	                            !dcb_has_queued_interrupt(dcb, size_in_dwords));
+	g_renderer->GetGpu().Submit(std::span {dcb, size_in_dwords}, {});
 }
 
 int KYTY_SYSV_ABI AgcDriverSubmitDcb(const Packet* packet) {
@@ -4306,23 +4267,6 @@ int KYTY_SYSV_ABI AgcDriverSubmitMultiDcbs(uint32_t* const* dcb_gpu_addrs,
 }
 
 static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
-	if (acb != nullptr && size_in_dwords >= 5) {
-		auto descriptor_addr =
-		    static_cast<uint64_t>(acb[0]) | (static_cast<uint64_t>(acb[1]) << 32u);
-		auto descriptor_size  = acb[2];
-		auto descriptor_flags = acb[3];
-		auto descriptor_magic = acb[4];
-		if (descriptor_addr != 0 && descriptor_size != 0 && descriptor_flags == 0 &&
-		    descriptor_magic == 0x5533ccaau) {
-			LOGF("\t descriptor addr = 0x%016" PRIx64 "\n"
-			     "\t descriptor size = 0x%08" PRIx32 "\n"
-			     "\t descriptor magic = 0x%08" PRIx32 "\n",
-			     descriptor_addr, descriptor_size, descriptor_magic);
-			acb            = reinterpret_cast<uint32_t*>(descriptor_addr);
-			size_in_dwords = descriptor_size;
-		}
-	}
-
 	if (acb == nullptr || size_in_dwords == 0) {
 		return;
 	}
@@ -4333,9 +4277,8 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
 
 	GraphicsDbgDumpDcb("a", size_in_dwords, acb);
 
-	const bool trigger_interrupt_on_done = !dcb_has_queued_interrupt(acb, size_in_dwords);
 	EXIT_IF(g_renderer == nullptr);
-	g_renderer->GetGpu().SubmitCompute(queue, acb, size_in_dwords, trigger_interrupt_on_done);
+	g_renderer->GetGpu().SubmitCompute(queue, std::span {acb, size_in_dwords});
 }
 
 static uint32_t get_driver_queue(const void* queue_context) {
@@ -4461,7 +4404,8 @@ int KYTY_SYSV_ABI AgcDriverDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq,
 		return LibKernel::KERNEL_ERROR_EBADF;
 	}
 
-	return Sync::DeleteEqEvent(eq, id);
+	EXIT_IF(g_renderer == nullptr);
+	return Sync::DeleteEqEvent(*g_renderer, eq, id);
 }
 
 int KYTY_SYSV_ABI AgcDriverGetEqEventType(const LibKernel::EventQueue::KernelEvent* ev) {

src/libs/agc.h

@@ -29,7 +29,7 @@ struct Lifecycle {
 	static constexpr auto        shutdown   = Libs::Graphics::Shutdown;
 };
 
-void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, uint32_t* cmd_buffer);
+void GraphicsDbgDumpDcb(const char* type, uint32_t num_dw, const uint32_t* cmd_buffer);
 
 namespace Gen5 {
 
@@ -287,8 +287,7 @@ uint32_t* KYTY_SYSV_ABI AgcDcbPopMarker(CommandBuffer* buf);
 int KYTY_SYSV_ABI AgcWaitRegMemPatchAddress(uint32_t* cmd, const volatile void* address);
 int KYTY_SYSV_ABI AgcWaitRegMemPatchReference(uint32_t* cmd, uint64_t reference);
 int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchAddress(uint32_t* cmd, const volatile Label* address);
-int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchData(uint32_t* cmd, uint32_t context_id,
-                                                   uint32_t data_sel, uint64_t data);
+int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchData(uint32_t* cmd, uint64_t data);
 
 uint32_t* KYTY_SYSV_ABI AgcDcbSetFlip(CommandBuffer* buf, uint32_t video_out_handle,
                                       int32_t display_buffer_index, uint32_t flip_mode,

src/libs/audio.cpp

@@ -21,6 +21,8 @@
 #include <limits>
 #include <vector>
 
+#include "libatrac9.h"
+
 namespace Libs::Audio {
 
 namespace {
@@ -1360,6 +1362,17 @@ namespace Ngs2 {
 
 LIB_NAME("Ngs2", "Ngs2");
 
+constexpr int32_t NGS2_ERROR_INVALID_OUT_ADDRESS =
+    static_cast<int32_t>(0x804a8010u);
+constexpr int32_t NGS2_ERROR_INVALID_WAVEFORM_DATA =
+    static_cast<int32_t>(0x804a8430u);
+constexpr int32_t NGS2_ERROR_INVALID_WAVEFORM_FORMAT =
+    static_cast<int32_t>(0x804a8431u);
+constexpr int32_t NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT =
+    static_cast<int32_t>(0x804a8432u);
+
+constexpr uint32_t NGS2_WAVEFORM_TYPE_ATRAC9 = 0x40;
+
 struct Ngs2SystemOption {
 	size_t    size                     = 0;
 	char      name[64]                 = {};
@@ -1773,6 +1786,9 @@ static_assert(sizeof(Ngs2VoiceState) == 8);
 static_assert(sizeof(Ngs2SubmixerVoiceState) == 20);
 static_assert(sizeof(Ngs2CustomMasteringVoiceState) == 16);
 static_assert(sizeof(Ngs2SamplerVoiceState) == 56);
+static_assert(sizeof(Ngs2WaveformFormat) == 24);
+static_assert(sizeof(Ngs2WaveformBlock) == 40);
+static_assert(sizeof(Ngs2WaveformInfo) == 232);
 
 static uint32_t Ngs2GetStateFlags(const Ngs2VoiceInternal* voice) {
 	switch (voice->state) {
@@ -2443,24 +2459,143 @@ int KYTY_SYSV_ABI Ngs2SystemRender(uintptr_t system_handle, const Ngs2RenderBuff
 	return OK;
 }
 
+static uint16_t Ngs2ReadLe16(const uint8_t* data) {
+	return static_cast<uint16_t>(data[0]) | (static_cast<uint16_t>(data[1]) << 8u);
+}
+
+static uint32_t Ngs2ReadLe32(const uint8_t* data) {
+	return static_cast<uint32_t>(data[0]) | (static_cast<uint32_t>(data[1]) << 8u) |
+	       (static_cast<uint32_t>(data[2]) << 16u) | (static_cast<uint32_t>(data[3]) << 24u);
+}
+
+static bool Ngs2FourCcEquals(const uint8_t* data, const char* four_cc) {
+	return std::memcmp(data, four_cc, 4) == 0;
+}
+
+static int Ngs2ParseAtrac9Riff(const void* data, size_t data_size, Ngs2WaveformInfo* info) {
+	static constexpr uint8_t ATRAC9_GUID[16] = {0xd2, 0x42, 0xe1, 0x47, 0xba, 0x36,
+	                                            0x8d, 0x4d, 0x88, 0xfc, 0x61, 0x65,
+	                                            0x4f, 0x8c, 0x83, 0x6c};
+	const auto* bytes = static_cast<const uint8_t*>(data);
+	if (bytes == nullptr || data_size < 12) {
+		return NGS2_ERROR_INVALID_WAVEFORM_DATA;
+	}
+	if (!Ngs2FourCcEquals(bytes, "RIFF") || !Ngs2FourCcEquals(bytes + 8, "WAVE")) {
+		return NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT;
+	}
+
+	const uint64_t riff_end64 = 8ull + Ngs2ReadLe32(bytes + 4);
+	if (riff_end64 < 12 || riff_end64 > data_size) {
+		return NGS2_ERROR_INVALID_WAVEFORM_DATA;
+	}
+	const auto riff_end = static_cast<size_t>(riff_end64);
+
+	const uint8_t* format           = nullptr;
+	const uint8_t* fact             = nullptr;
+	size_t         waveform_offset  = 0;
+	uint32_t       waveform_size    = 0;
+
+	for (size_t offset = 12; offset + 8 <= riff_end;) {
+		const auto* chunk       = bytes + offset;
+		const auto  chunk_size  = static_cast<size_t>(Ngs2ReadLe32(chunk + 4));
+		const auto  payload     = offset + 8;
+		if (chunk_size > riff_end - payload) {
+			return NGS2_ERROR_INVALID_WAVEFORM_DATA;
+		}
+
+		if (Ngs2FourCcEquals(chunk, "fmt ")) {
+			if (chunk_size < 52) {
+				return NGS2_ERROR_INVALID_WAVEFORM_FORMAT;
+			}
+			format = bytes + payload;
+		} else if (Ngs2FourCcEquals(chunk, "fact")) {
+			if (chunk_size < 12) {
+				return NGS2_ERROR_INVALID_WAVEFORM_FORMAT;
+			}
+			fact = bytes + payload;
+		} else if (Ngs2FourCcEquals(chunk, "data")) {
+			if (payload > std::numeric_limits<uint32_t>::max()) {
+				return NGS2_ERROR_INVALID_WAVEFORM_DATA;
+			}
+			waveform_offset = payload;
+			waveform_size   = static_cast<uint32_t>(chunk_size);
+		}
+
+		const uint64_t next = static_cast<uint64_t>(payload) + chunk_size + (chunk_size & 1u);
+		if (next > riff_end) {
+			return NGS2_ERROR_INVALID_WAVEFORM_DATA;
+		}
+		offset = static_cast<size_t>(next);
+	}
+
+	if (format == nullptr || fact == nullptr || waveform_offset == 0) {
+		return NGS2_ERROR_INVALID_WAVEFORM_FORMAT;
+	}
+	if (Ngs2ReadLe16(format) != 0xfffe ||
+	    std::memcmp(format + 24, ATRAC9_GUID, sizeof(ATRAC9_GUID)) != 0) {
+		return NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT;
+	}
+
+	std::array<uint8_t, ATRAC9_CONFIG_DATA_SIZE> config {};
+	std::memcpy(config.data(), format + 44, config.size());
+	if (config[0] != 0xfe || (config[1] & 1u) != 0 || ((config[1] >> 1u) & 7u) >= 6u) {
+		return NGS2_ERROR_INVALID_WAVEFORM_FORMAT;
+	}
+
+	Atrac9CodecInfo codec {};
+	void*           decoder = Atrac9GetHandle();
+	const bool valid_codec =
+	    decoder != nullptr && Atrac9InitDecoder(decoder, config.data()) == 0 &&
+	    Atrac9GetCodecInfo(decoder, &codec) == 0 && codec.channels > 0 &&
+	    codec.samplingRate > 0 && codec.superframeSize > 0 && codec.framesInSuperframe > 0 &&
+	    codec.frameSamples > 0 && codec.superframeSize % codec.framesInSuperframe == 0;
+	if (decoder != nullptr) {
+		Atrac9ReleaseHandle(decoder);
+	}
+	const uint64_t frame_samples =
+	    static_cast<uint64_t>(codec.frameSamples) * static_cast<uint64_t>(codec.framesInSuperframe);
+	if (!valid_codec || codec.channels != Ngs2ReadLe16(format + 2) ||
+	    codec.samplingRate != static_cast<int>(Ngs2ReadLe32(format + 4)) ||
+	    codec.superframeSize != Ngs2ReadLe16(format + 12) ||
+	    frame_samples != Ngs2ReadLe16(format + 18) ||
+	    frame_samples > std::numeric_limits<uint32_t>::max()) {
+		return NGS2_ERROR_INVALID_WAVEFORM_FORMAT;
+	}
+
+	info->format.waveform_type = NGS2_WAVEFORM_TYPE_ATRAC9;
+	info->format.num_channels  = static_cast<uint32_t>(codec.channels);
+	info->format.sample_rate   = static_cast<uint32_t>(codec.samplingRate);
+	std::memcpy(&info->format.config_data, config.data(), config.size());
+	info->data_offset              = static_cast<uint32_t>(waveform_offset);
+	info->data_size                = waveform_size;
+	info->num_samples              = Ngs2ReadLe32(fact);
+	info->audio_unit_size          = static_cast<uint32_t>(codec.superframeSize /
+	                                                       codec.framesInSuperframe);
+	info->num_audio_unit_samples   = static_cast<uint32_t>(codec.frameSamples);
+	info->num_audio_unit_per_frame = static_cast<uint32_t>(codec.framesInSuperframe);
+	info->audio_frame_size         = static_cast<uint32_t>(codec.superframeSize);
+	info->num_audio_frame_samples  = static_cast<uint32_t>(frame_samples);
+	info->num_delay_samples        = Ngs2ReadLe32(fact + 4);
+	info->num_blocks               = 1;
+	info->blocks[0].data_offset    = waveform_offset;
+	info->blocks[0].data_size      = waveform_size;
+	info->blocks[0].num_skip_samples = Ngs2ReadLe32(fact + 8);
+	info->blocks[0].num_samples      = info->num_samples;
+	return OK;
+}
+
 int KYTY_SYSV_ABI Ngs2ParseWaveformData(const void* data, size_t data_size,
                                         Ngs2WaveformInfo* info) {
 	PRINT_NAME();
 	LOGF("\t data = 0x%016" PRIx64 ", data_size = 0x%016" PRIx64 "\n",
 	     reinterpret_cast<uint64_t>(data), static_cast<uint64_t>(data_size));
 
-	EXIT_NOT_IMPLEMENTED(info == nullptr);
+	if (info == nullptr) {
+		return NGS2_ERROR_INVALID_OUT_ADDRESS;
+	}
 
 	std::memset(info, 0, sizeof(Ngs2WaveformInfo));
-	info->format.waveform_type = 0x80;
-	info->format.num_channels  = 1;
-	info->format.sample_rate   = 48000;
-	info->data_size =
-	    static_cast<uint32_t>(std::min<size_t>(data_size, std::numeric_limits<uint32_t>::max()));
-	info->num_audio_unit_samples   = 1;
-	info->num_audio_unit_per_frame = 1;
-	info->num_audio_frame_samples  = 1;
-	return OK;
+	return Ngs2ParseAtrac9Riff(data, data_size, info);
 }
 
 int KYTY_SYSV_ABI Ngs2CalcWaveformBlock(const Ngs2WaveformFormat* format, uint32_t sample_pos,

tests/MemoryTrackerTests.cpp

@@ -191,6 +191,10 @@ void TestRangeSet() {
   ranges.Add(0x1200, 0x40);
   Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200),
         "range set containment did not require full coverage");
+  Check(ranges.Intersects(0x0fff, 2) && ranges.Intersects(0x11ff, 2) &&
+            !ranges.Intersects(0x1100, 0x100) &&
+            !ranges.Intersects(0x1240, 1),
+        "range set intersection did not preserve half-open boundaries");
   auto intersections = ranges.Intersections(0x1070, 0x1b0);
   Check(intersections.size() == 2 && intersections[0].address == 0x1070 &&
             intersections[0].size == 0x90 &&

tests/ResourceTrackingTests.cpp

@@ -1,5 +1,5 @@
 #include "graphics/guest_gpu/gpu_defs.h"
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
@@ -35,7 +35,6 @@ struct Fixture {
 
   explicit Fixture(ShaderType stage = ShaderType::Compute) {
     program.stage = stage;
-    program.values = std::make_shared<ValueProgram>();
     program.user_data_count = 64;
     block = AddBlock();
   }
@@ -43,10 +42,10 @@ struct Fixture {
   Block *AddBlock() {
     auto storage = std::make_unique<Block>();
     auto *result = storage.get();
-    program.values->block_storage.push_back(std::move(storage));
-    program.values->blocks.push_back(result);
-    program.values->block_info.push_back(
-        {.id = static_cast<uint32_t>(program.values->block_info.size())});
+    program.block_storage.push_back(std::move(storage));
+    program.blocks.push_back(result);
+    program.block_info.push_back(
+        {.id = static_cast<uint32_t>(program.block_info.size())});
     return result;
   }
 
@@ -77,8 +76,8 @@ struct Fixture {
 
   MemoryFlags AddMemory(MemoryInfo memory, uint32_t pc) {
     const auto index =
-        static_cast<uint32_t>(program.values->memory_info.size());
-    program.values->memory_info.push_back(memory);
+        static_cast<uint32_t>(program.memory_info.size());
+    program.memory_info.push_back(memory);
     return {index, pc};
   }
 
@@ -241,19 +240,19 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
 void TestInvariantIndirectImageMaterialization() {
   auto fixture = MakeIndirectImageFixture(false);
   fixture->PlanAndTrack();
-  EliminateDeadCode(fixture->program.values->blocks);
+  EliminateDeadCode(fixture->program.blocks);
   std::string validation_error;
   Check(
-      ValidateValueProgram(*fixture->program.values, true, &validation_error),
+      ValidateProgram(fixture->program, true, &validation_error),
       "post-tracking dead-code elimination invalidated descriptor provenance");
 
   Check(fixture->program.info.buffers.size() == 1 &&
             fixture->program.info.images.size() == 1 &&
-            fixture->program.values->dynamic_reads.size() == 1,
+            fixture->program.dynamic_reads.size() == 1,
         "indirect image key was not retained as a scalar-buffer read");
   const auto source = fixture->program.info.images[0].source;
-  Check(source < fixture->program.values->descriptor_sources.size() &&
-            fixture->program.values->descriptor_sources[source]
+  Check(source < fixture->program.descriptor_sources.size() &&
+            fixture->program.descriptor_sources[source]
                 .indirect_image.has_value(),
         "indirect image source was not retained for runtime proof");
   const auto image_handle =
@@ -407,7 +406,7 @@ void TestInvariantIndirectImageMaterialization() {
 
   auto memory_backed = MakeIndirectImageFixture(false, 0u, true);
   memory_backed->PlanAndTrack();
-  EliminateDeadCode(memory_backed->program.values->blocks);
+  EliminateDeadCode(memory_backed->program.blocks);
   std::array<uint32_t, 11> memory_backed_user_data{0x1000u, 224u << 16u, 2u, 0u,
                                                    0x2000u, 16u << 16u,  4u, 0u,
                                                    7u,      0x3100u,     0u};
@@ -431,7 +430,7 @@ void TestInvariantIndirectImageMaterialization() {
             error.find("ReadFirstLane") != std::string::npos &&
             !malformed->program.resource_tracking_complete &&
             malformed->program.info.images.empty() &&
-            malformed->program.values->descriptor_sources.empty(),
+            malformed->program.descriptor_sources.empty(),
         "malformed indirect image pattern was partially accepted");
 
   auto wrapped_immediate = MakeIndirectImageFixture(false, 4u);
@@ -484,7 +483,7 @@ void TestDenseBufferTracking() {
 
   Check(fixture.program.info.buffers.size() == 2,
         "typed buffer sources were not densely interned");
-  Check(fixture.program.values->descriptor_sources.size() == 2,
+  Check(fixture.program.descriptor_sources.size() == 2,
         "descriptor source table did not match dense topology");
   const auto &resource = fixture.program.info.buffers[0];
   Check(resource.read && resource.written && resource.atomic &&
@@ -494,10 +493,10 @@ void TestDenseBufferTracking() {
   Check(first.Instruction()->Flags<uint32_t>() == 0 &&
             second.Instruction()->Flags<uint32_t>() == 1,
         "typed handles were not assigned dense indices");
-  Check(fixture.program.values->memory_info[load_flags.index].resource == 0 &&
-            fixture.program.values->memory_info[store_flags.index].resource ==
+  Check(fixture.program.memory_info[load_flags.index].resource == 0 &&
+            fixture.program.memory_info[store_flags.index].resource ==
                 0 &&
-            fixture.program.values->memory_info[other_flags.index].resource ==
+            fixture.program.memory_info[other_flags.index].resource ==
                 1,
         "typed memory metadata was not patched to dense indices");
 
@@ -531,8 +530,8 @@ void TestScalarAndVectorBufferAlias() {
   Check(fixture.program.info.buffers.size() == 1 &&
             fixture.program.info.buffers[0].scalar,
         "typed scalar and vector uses of one descriptor were split");
-  Check(fixture.program.values->memory_info[scalar_flags.index].resource == 0 &&
-            fixture.program.values->memory_info[vector_flags.index].resource ==
+  Check(fixture.program.memory_info[scalar_flags.index].resource == 0 &&
+            fixture.program.memory_info[vector_flags.index].resource ==
                 0,
         "scalar/vector alias did not share a dense index");
 }
@@ -625,7 +624,7 @@ void TestImagesSamplersAndAliases() {
             repeated.second.Instruction()->Flags<uint32_t>() == 0,
         "unused sampler border colors prevented source interning");
   const auto sampler_source = fixture.program.info.samplers[0].source;
-  Check(fixture.program.values->descriptor_sources[sampler_source]
+  Check(fixture.program.descriptor_sources[sampler_source]
                 .dwords[3]
                 .U32() == 0,
         "unused sampler border color was not canonicalized");
@@ -665,7 +664,7 @@ void TestSampleAdjustSamplerScratch() {
   fixture.PlanAndTrack();
 
   const auto source = fixture.program.info.samplers[0].source;
-  const auto stored = fixture.program.values->descriptor_sources[source]
+  const auto stored = fixture.program.descriptor_sources[source]
                           .dwords[3]
                           .Resolve()
                           .TryInstruction();
@@ -758,10 +757,10 @@ void TestDynamicStorageMipTracking() {
             mip1.first.Instruction()->Flags<uint32_t>() == 1 &&
             mip2.first.Instruction()->Flags<uint32_t>() == 1 &&
             dynamic.first.Instruction()->Flags<uint32_t>() == 1 &&
-            fixture.program.values->memory_info[plain.second].resource == 0 &&
-            fixture.program.values->memory_info[mip1.second].resource == 1 &&
-            fixture.program.values->memory_info[mip2.second].resource == 1 &&
-            fixture.program.values->memory_info[dynamic.second].resource == 1,
+            fixture.program.memory_info[plain.second].resource == 0 &&
+            fixture.program.memory_info[mip1.second].resource == 1 &&
+            fixture.program.memory_info[mip2.second].resource == 1 &&
+            fixture.program.memory_info[dynamic.second].resource == 1,
         "dynamic storage mip handles and memory metadata were not patched");
 
   DescriptorValue descriptor{};
@@ -795,7 +794,6 @@ void TestDynamicStorageMipTracking() {
         "dynamic storage mip descriptors were not expanded consecutively");
 
   Program null_program;
-  null_program.values = std::make_shared<ValueProgram>();
   null_program.resource_tracking_complete = true;
   ImageResource null_image;
   null_image.kind = ResourceKind::StorageImage;
@@ -847,12 +845,12 @@ void TestSrtFlatteningAndRuntimeMemoization() {
                fixture.AddMemory(buffer, 16));
   fixture.PlanAndTrack();
 
-  Check(fixture.program.values->srt_reads.size() == 1,
+  Check(fixture.program.srt_reads.size() == 1,
         "shared typed scalar read did not receive one flat SRT slot");
   Check(fixture.program.info.buffers.size() == 1 &&
             fixture.program.info.addresses.empty(),
         "planning-only scalar reads leaked into resource topology");
-  Check(fixture.program.values->memory_info[0].planning_only,
+  Check(fixture.program.memory_info[0].planning_only,
         "canonical runtime scalar read was not marked planning-only");
 
   std::array<uint32_t, 2> user_data{0x1000u, 0u};
@@ -910,8 +908,8 @@ void TestDynamicSrtReadRemainsExplicit() {
                fixture.AddMemory(buffer, 8));
   fixture.PlanAndTrack();
 
-  Check(fixture.program.values->srt_reads.empty() &&
-            fixture.program.values->dynamic_reads.size() == 1 &&
+  Check(fixture.program.srt_reads.empty() &&
+            fixture.program.dynamic_reads.size() == 1 &&
             fixture.program.info.addresses.size() == 1,
         "dynamic scalar read was incorrectly flattened or lost");
   std::array<uint32_t, 3> user_data{0x1000u, 0u, 4u};
@@ -973,7 +971,7 @@ void TestPhiValidation() {
             error.find("control-dependent phi") != std::string::npos &&
             !fixture.program.resource_tracking_complete &&
             fixture.program.info.buffers.empty() &&
-            fixture.program.values->descriptor_sources.empty(),
+            fixture.program.descriptor_sources.empty(),
         "control-dependent descriptor phi was not rejected transactionally");
 }
 
@@ -1295,7 +1293,7 @@ void TestResourceLimitIsTransactional() {
             error.find("buffer resource limit exceeded") != std::string::npos &&
             !fixture.program.resource_tracking_complete &&
             fixture.program.info.buffers.empty() &&
-            fixture.program.values->descriptor_sources.empty(),
+            fixture.program.descriptor_sources.empty(),
         "resource-limit failure partially mutated typed resource state");
 }
 
@@ -1412,13 +1410,18 @@ int main() {
 
 // The full emulator supplies these assertion hooks through common. This focused
 // target links only fmt; keep assertion failures observable without widening
-// its legacy build manifest.
+// its focused build manifest.
 namespace Common {
 int DbgExitIfHandler(const char *expression, const char *file, int line) {
   throw std::runtime_error(std::string("typed IR assertion: ") + expression +
                            " at " + file + ':' + std::to_string(line));
 }
 
+int DbgNotImplementedHandler(const char *expression, const char *file, int line) {
+  throw std::runtime_error(std::string("typed IR not implemented: ") + expression +
+                           " at " + file + ':' + std::to_string(line));
+}
+
 void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
 } // namespace Common
 
@@ -1427,6 +1430,6 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
 #include "graphics/shader/recompiler/ir/Block.cpp"
 #include "graphics/shader/recompiler/ir/Type.cpp"
 #include "graphics/shader/recompiler/ir/Value.cpp"
-#include "graphics/shader/recompiler/ir/ValueProgram.cpp"
+#include "graphics/shader/recompiler/ir/Program.cpp"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"

tests/ScalarProvenanceTests.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
@@ -27,16 +27,15 @@ struct Fixture {
     program.stage = ShaderType::Compute;
     program.shader_hash = 0x12345678u;
     program.user_data_base = 2;
-    program.values = std::make_shared<ValueProgram>();
     for (uint32_t index = 0; index < block_count; index++) {
-      program.values->block_storage.push_back(std::make_unique<Block>());
-      auto *block = program.values->block_storage.back().get();
-      program.values->blocks.push_back(block);
-      program.values->block_info.push_back({.id = index});
+      program.block_storage.push_back(std::make_unique<Block>());
+      auto *block = program.block_storage.back().get();
+      program.blocks.push_back(block);
+      program.block_info.push_back({.id = index});
     }
   }
 
-  Block &BlockAt(uint32_t index = 0) { return *program.values->blocks[index]; }
+  Block &BlockAt(uint32_t index = 0) { return *program.blocks[index]; }
 
   Value Emit(ValueOpcode opcode, std::initializer_list<Value> args = {},
              uint64_t flags = 0, uint32_t block = 0) {
@@ -55,8 +54,8 @@ struct Fixture {
     MemoryInfo info;
     info.kind = kind;
     info.offset = static_cast<uint32_t>(offset);
-    program.values->memory_info.push_back(info);
-    return static_cast<uint32_t>(program.values->memory_info.size() - 1u);
+    program.memory_info.push_back(info);
+    return static_cast<uint32_t>(program.memory_info.size() - 1u);
   }
 
   void Plan() {
@@ -103,11 +102,11 @@ void TestImmediateFlatteningAndGvn() {
                {first, second, Value(16u), Value(0u)});
 
   fixture.Plan();
-  Check(fixture.program.values->srt_reads.size() == 1,
+  Check(fixture.program.srt_reads.size() == 1,
         "equivalent typed scalar reads were not coalesced");
-  Check(fixture.program.values->dynamic_reads.empty(),
+  Check(fixture.program.dynamic_reads.empty(),
         "immediate scalar read was classified as dynamic");
-  Check(fixture.program.values->memory_info[memory].planning_only,
+  Check(fixture.program.memory_info[memory].planning_only,
         "flattened raw read was not kept as a planning-only root");
 
   TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}};
@@ -173,8 +172,8 @@ void TestDynamicReadRemainsTyped() {
                {read, Value(0u), Value(16u), Value(0u)});
 
   fixture.Plan();
-  Check(fixture.program.values->srt_reads.empty() &&
-            fixture.program.values->dynamic_reads == std::vector<Value>{read},
+  Check(fixture.program.srt_reads.empty() &&
+            fixture.program.dynamic_reads == std::vector<Value>{read},
         "dynamic scalar read received a fake flattened slot");
 }
 
@@ -209,7 +208,7 @@ void TestShaderBaseAndUserData() {
                                  {Value(static_cast<ScalarReg>(2))});
   const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)});
   fixture.Plan();
-  fixture.program.values->descriptor_sources.push_back(
+  fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high, sum}, .dword_count = 3});
 
   const std::array user_data{0x20u};
@@ -239,7 +238,7 @@ void TestCarryAndBitFields() {
   const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract,
                                  {Value(0x000000f0u), Value(4u), Value(4u)});
   fixture.Plan();
-  fixture.program.values->descriptor_sources.push_back(
+  fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high, inserted, sign}, .dword_count = 4});
 
   DescriptorValue result;
@@ -262,9 +261,9 @@ void TestInvariantAndDivergentPhi() {
   divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u));
   divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u));
   fixture.Plan();
-  fixture.program.values->descriptor_sources.push_back(
+  fixture.program.descriptor_sources.push_back(
       {.dwords = {Value(&invariant)}, .dword_count = 1});
-  fixture.program.values->descriptor_sources.push_back(
+  fixture.program.descriptor_sources.push_back(
       {.dwords = {Value(&divergent)}, .dword_count = 1});
 
   DescriptorValue result;
@@ -293,10 +292,10 @@ void TestControlDependentStandaloneLoadStaysTyped() {
       RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u),
               memory, 2);
   fixture.Plan();
-  Check(fixture.program.values->srt_reads.empty() &&
+  Check(fixture.program.srt_reads.empty() &&
             read.ResolveInstruction()->GetOpcode() ==
                 ValueOpcode::LoadAddressU32 &&
-            !fixture.program.values->memory_info[memory].planning_only,
+            !fixture.program.memory_info[memory].planning_only,
         "control-dependent standalone scalar load was flattened into a host "
         "snapshot");
 }
@@ -315,7 +314,7 @@ void TestRuntime64BitDescriptorOps() {
   const auto high =
       fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)});
   fixture.Plan();
-  fixture.program.values->descriptor_sources.push_back(
+  fixture.program.descriptor_sources.push_back(
       {.dwords = {low, high}, .dword_count = 2});
 
   DescriptorValue result;
@@ -370,7 +369,7 @@ void TestReadLaneElimination() {
                                   {undef, Value(0xdeadbeefu), Value(5u)});
   const auto read = fixture.Emit(ValueOpcode::ReadLane, {write, Value(5u)});
   const auto use = fixture.Emit(ValueOpcode::IAdd32, {read, Value(1u)});
-  const auto stats = EliminateReadLane(*fixture.program.values, 64);
+  const auto stats = EliminateReadLane(fixture.program, 64);
   Check(stats.rewritten_reads == 1 &&
             use.ResolveInstruction()->Arg(0).Resolve() == Value(0xdeadbeefu),
         "fixed-lane typed read was not rewritten from its SSA write chain");
@@ -379,7 +378,7 @@ void TestReadLaneElimination() {
                                      {Value(static_cast<ScalarReg>(2))});
   const auto dynamic = fixture.Emit(ValueOpcode::ReadLane, {write, selector});
   fixture.Emit(ValueOpcode::IAdd32, {dynamic, Value(1u)});
-  Check(EliminateReadLane(*fixture.program.values, 64).rewritten_reads == 0,
+  Check(EliminateReadLane(fixture.program, 64).rewritten_reads == 0,
         "dynamic-lane read was rewritten unsafely");
 }
 
@@ -390,9 +389,9 @@ void TestOptimizationPipeline() {
   fixture.Emit(ValueOpcode::ReferenceU32, {kept});
   fixture.Emit(ValueOpcode::IMul32, {Value(6u), Value(7u)});
 
-  ConstantPropagationPass(fixture.program.values->blocks);
-  RemoveIdentities(fixture.program.values->blocks);
-  EliminateDeadCode(fixture.program.values->blocks);
+  ConstantPropagationPass(fixture.program.blocks);
+  RemoveIdentities(fixture.program.blocks);
+  EliminateDeadCode(fixture.program.blocks);
 
   const auto &instructions = fixture.BlockAt().Instructions();
   Check(instructions.size() == 1 &&
@@ -404,20 +403,20 @@ void TestOptimizationPipeline() {
 
 void TestControlFlowValueSurvivesReadLaneFolding() {
   Fixture fixture(3);
-  auto *entry = fixture.program.values->blocks[0];
-  auto *taken = fixture.program.values->blocks[1];
-  auto *other = fixture.program.values->blocks[2];
+  auto *entry = fixture.program.blocks[0];
+  auto *taken = fixture.program.blocks[1];
+  auto *other = fixture.program.blocks[2];
   entry->AddBranch(taken);
   entry->AddBranch(other);
 
-  auto &entry_info = fixture.program.values->block_info[0];
+  auto &entry_info = fixture.program.block_info[0];
   entry_info.terminator.kind =
       Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch;
   entry_info.terminator.true_block = 1;
   entry_info.terminator.false_block = 2;
-  fixture.program.values->block_info[1].terminator.kind =
+  fixture.program.block_info[1].terminator.kind =
       Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return;
-  fixture.program.values->block_info[2].terminator.kind =
+  fixture.program.block_info[2].terminator.kind =
       Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return;
 
   const auto undef = fixture.Emit(ValueOpcode::UndefU32);
@@ -429,16 +428,16 @@ void TestControlFlowValueSurvivesReadLaneFolding() {
       fixture.Emit(ValueOpcode::IEqual32, {read, Value(42u)});
   fixture.Emit(ValueOpcode::Reference, {entry_info.condition});
 
-  Check(EliminateReadLane(*fixture.program.values, 64).rewritten_reads == 1,
+  Check(EliminateReadLane(fixture.program, 64).rewritten_reads == 1,
         "control-flow fixture did not eliminate its fixed-lane read");
-  ConstantPropagationPass(fixture.program.values->blocks);
-  ResolveControlFlowIdentities(*fixture.program.values);
-  RemoveIdentities(fixture.program.values->blocks);
-  EliminateDeadCode(fixture.program.values->blocks);
+  ConstantPropagationPass(fixture.program.blocks);
+  ResolveControlFlowIdentities(fixture.program);
+  RemoveIdentities(fixture.program.blocks);
+  EliminateDeadCode(fixture.program.blocks);
 
   std::string error;
   Check(entry_info.condition == Value(true) &&
-            ValidateValueProgram(*fixture.program.values, true, &error),
+            ValidateProgram(fixture.program, true, &error),
         error.empty() ? "folded branch condition did not survive identity removal"
                       : error.c_str());
 }
@@ -447,7 +446,7 @@ void TestUndefinedRuntimeValueFails() {
   Fixture fixture;
   const auto undef = fixture.Emit(ValueOpcode::UndefU32);
   fixture.Plan();
-  fixture.program.values->descriptor_sources.push_back(
+  fixture.program.descriptor_sources.push_back(
       {.dwords = {undef}, .dword_count = 1});
   DescriptorValue result;
   result.dword_count = 3;
@@ -499,7 +498,7 @@ int main() {
 #include "../src/graphics/shader/recompiler/ir/Block.cpp"
 #include "../src/graphics/shader/recompiler/ir/Type.cpp"
 #include "../src/graphics/shader/recompiler/ir/Value.cpp"
-#include "../src/graphics/shader/recompiler/ir/ValueProgram.cpp"
+#include "../src/graphics/shader/recompiler/ir/Program.cpp"
 #include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
 #include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
 #include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"

tests/ShaderStageRuntimeTests.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 
@@ -23,12 +23,11 @@ bool RejectSpecializationRead(void *userdata, uint64_t, uint32_t *) {
 Libs::Graphics::ShaderRecompiler::IR::Block &
 AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
-  program.values = std::make_shared<ValueProgram>();
   auto block = std::make_unique<Block>();
   auto *result = block.get();
-  program.values->blocks.push_back(result);
-  program.values->block_info.push_back({.id = 0});
-  program.values->block_storage.push_back(std::move(block));
+  program.blocks.push_back(result);
+  program.block_info.push_back({.id = 0});
+  program.block_storage.push_back(std::move(block));
   return *result;
 }
 
@@ -44,7 +43,7 @@ SrtProgram(uint64_t address) {
   MemoryInfo memory;
   memory.kind = ResourceKind::ScalarAddress;
   memory.planning_only = true;
-  program.values->memory_info.push_back(memory);
+  program.memory_info.push_back(memory);
   const auto low = Value(static_cast<uint32_t>(address));
   const auto high = Value(static_cast<uint32_t>(address >> 32u));
   auto &handle =
@@ -53,7 +52,7 @@ SrtProgram(uint64_t address) {
       ValueOpcode::LoadAddressU32,
       {Value(&handle), Value(0u), Value(0u), Value(true)});
   raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40});
-  program.values->srt_reads.push_back({Value(&raw), 0, 0x40});
+  program.srt_reads.push_back({Value(&raw), 0, 0x40});
 
   auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource);
   auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst,
@@ -62,7 +61,7 @@ SrtProgram(uint64_t address) {
   source.dwords[0] = Value(&flat);
   source.dwords[1] = Value(0u);
   source.dword_count = 2;
-  program.values->descriptor_sources.push_back(source);
+  program.descriptor_sources.push_back(source);
   return std::make_shared<const Program>(std::move(program));
 }
 
@@ -141,5 +140,5 @@ int main() {
 #include "graphics/shader/recompiler/ir/Block.cpp"
 #include "graphics/shader/recompiler/ir/Type.cpp"
 #include "graphics/shader/recompiler/ir/Value.cpp"
-#include "graphics/shader/recompiler/ir/ValueProgram.cpp"
+#include "graphics/shader/recompiler/ir/Program.cpp"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"