json2: export Value::clear (#584)

Export the existing clear implementation while preserving tree ownership and root handler metadata.

CMakeLists.txt

@@ -342,6 +342,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/SDL2/include
 	${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
+	${KYTY_THIRD_PARTY_DIR}/SPIRV-Headers/include
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/xxhash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
@@ -550,6 +551,10 @@ if(BUILD_TESTING)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME shader_recompiler_alignbyte
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
+	add_test(NAME graphics_pipeline_rasterization
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --polygon-mode-only)
+	add_test(NAME graphics_draw_offsets
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --draw-offset-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME guest_red_zone_patcher

README.md

@@ -90,8 +90,14 @@ starting a large change.
 
 Set up the clang-format hook after cloning:
 
-```powershell
-python -m pip install pre-commit
+Install `pre-commit` using the method appropriate for your platform:
+
+- **Arch Linux / CachyOS:** `sudo pacman -S pre-commit`
+- **Other Linux / macOS / Windows:** `python -m pip install pre-commit`
+
+Then install the Git hook:
+
+```bash
 python -m pre_commit install --install-hooks
 ```
 

src/common/alignment.h

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2014 Jannik Vogel <email@jannikvogel.de>
+// SPDX-License-Identifier: CC0-1.0
+
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+
+namespace Common {
+
+template <typename T>
+[[nodiscard]] constexpr T AlignUp(T value, std::size_t size) {
+	static_assert(std::is_unsigned_v<T>, "T must be an unsigned value.");
+	auto mod{static_cast<T>(value % size)};
+	value -= mod;
+	return static_cast<T>(mod == T{0} ? value : value + size);
+}
+
+template <typename T>
+[[nodiscard]] constexpr T AlignDown(T value, std::size_t size) {
+	static_assert(std::is_unsigned_v<T>, "T must be an unsigned value.");
+	return static_cast<T>(value - value % size);
+}
+
+} // namespace Common

src/common/assert.cpp

@@ -1,6 +1,5 @@
 #include "common/assert.h"
 
-#include "common/debug.h"
 #include "common/logging/log.h"
 #include "common/subsystems.h"
 #include "kytyGitVersion.h"
@@ -12,25 +11,10 @@
 
 namespace Common {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_BUILD == KYTY_BUILD_DEBUG &&                    \
-    KYTY_COMPILER == KYTY_COMPILER_CLANG
-constexpr int PRINT_STACK_FROM = 4;
-#else
-constexpr int PRINT_STACK_FROM = 2;
-#endif
-
 static std::string BuildFatalReport(const char* title, std::string_view text, const char* file,
                                     int line) {
-	DebugStack stack;
-	DebugStack::Trace(&stack);
-
-	std::string report = "--- Build ---\n" KYTY_BUILD_LABEL "\n--- Stack Trace ---\n";
-	for (int i = PRINT_STACK_FROM; i < stack.depth; i++) {
-		report += fmt::format("[{}] {:016x}\n", i - PRINT_STACK_FROM,
-		                      static_cast<uint64_t>(stack.GetAddr(i)));
-	}
-	report += fmt::format("{}\n{} in {}:{}\n", title, text, file, line);
-	return report;
+	return fmt::format("--- Build ---\n{}\n{}\n{} in {}:{}\n", KYTY_BUILD_LABEL, title, text, file,
+	                   line);
 }
 
 static int DbgReport(const char* title, std::string_view text, const char* file, int line) {

src/common/debug.cpp

@@ -1,20 +1,7 @@
 #include "common/debug.h"
 
-#include "common/platform/sysDbg.h"
-
 namespace Common {
 
-void DebugStack::Print(int from, [[maybe_unused]] bool with_name) const {
-	for (int i = from; i < depth; i++) {
-		printf("[%d] %016" PRIx64 "\n", i - from, static_cast<uint64_t>(GetAddr(i)));
-	}
-}
-
-void DebugStack::Trace(DebugStack* stack) {
-	stack->depth = DEBUG_MAX_STACK_DEPTH;
-	SysStackWalk(stack->stack.data(), &stack->depth);
-}
-
 std::string Debug::GetCompiler() {
 #if KYTY_COMPILER == KYTY_COMPILER_CLANG
 	return "clang";

src/common/debug.h

@@ -2,42 +2,17 @@
 #define KYTY_COMMON_DEBUG_H_
 
 #include "common/common.h"
-#include "common/stringUtils.h"
 
-#include <array>
-#include <cstring>
+#include <string>
 
 namespace Common {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_BUILD == KYTY_BUILD_DEBUG &&                    \
-    KYTY_COMPILER == KYTY_COMPILER_CLANG
-constexpr int DEBUG_MAX_STACK_DEPTH = 20;
-#else
-constexpr int DEBUG_MAX_STACK_DEPTH = 15;
-#endif
-
 namespace Debug {
 std::string GetCompiler();
 std::string GetLinker();
 std::string GetBitness();
 } // namespace Debug
 
-struct DebugStack {
-	int                                      depth {0};
-	std::array<void*, DEBUG_MAX_STACK_DEPTH> stack {};
-
-	[[nodiscard]] uintptr_t GetAddr(int i) const { return reinterpret_cast<uintptr_t>(stack[i]); }
-
-	void Print(int from, bool with_name = true) const;
-
-	void CopyTo(DebugStack* s) const {
-		std::memcpy(s->stack.data(), stack.data(), sizeof(void*) * depth);
-		s->depth = depth;
-	}
-
-	static void Trace(DebugStack* stack);
-};
-
 } // namespace Common
 
 #endif /* KYTY_COMMON_DEBUG_H_ */

src/common/emulatorConfig.cpp

@@ -75,7 +75,7 @@ ShaderOptimizationType GetShaderOptimizationType() {
 	return g_config->shader_optimization_type;
 }
 
-ShaderLogDirection GetShaderLogDirection() {
+LogDirection GetShaderLogDirection() {
 	return g_config->shader_log_direction;
 }
 
@@ -95,7 +95,7 @@ bool GraphicsDebugDumpEnabled() {
 	return g_config->graphics_debug_dump_enabled;
 }
 
-OutputDirection GetPrintfDirection() {
+LogDirection GetPrintfDirection() {
 	return g_config->printf_direction;
 }
 
@@ -103,8 +103,8 @@ std::filesystem::path GetPrintfOutputFile() {
 	return g_config->printf_output_file;
 }
 
-ProfilerDirection GetProfilerDirection() {
-	return g_config->profiler_direction;
+bool ProfilerEnabled() {
+	return g_config->profiler_enabled;
 }
 
 bool SpirvDebugPrintfEnabled() {

src/common/emulatorConfig.h

@@ -21,11 +21,7 @@ struct Lifecycle {
 
 enum class ShaderOptimizationType { None, Size, Performance };
 
-enum class ShaderLogDirection { Silent, Console, File };
-
-enum class ProfilerDirection { None, Network };
-
-enum class OutputDirection { Silent, Console, File };
+enum class LogDirection { Silent, Console, File };
 
 enum class PresentMode { Fifo, Mailbox, Immediate };
 
@@ -47,7 +43,7 @@ struct ConfigOptions {
 	uint32_t               screen_height               = 720;
 	std::string            user_name                   = "Kyty";
 	int32_t                user_id                     = DEFAULT_USER_ID;
-	PresentMode            present_mode                = PresentMode::Fifo;
+	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
 	uint32_t               vblank_frequency            = 60;
@@ -55,14 +51,14 @@ struct ConfigOptions {
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
-	ShaderLogDirection     shader_log_direction        = ShaderLogDirection::Silent;
+	LogDirection           shader_log_direction        = LogDirection::Silent;
 	std::filesystem::path  shader_log_folder           = "_Shaders";
 	bool                   command_buffer_dump_enabled = false;
 	std::filesystem::path  command_buffer_dump_folder  = "_Buffers";
 	bool                   graphics_debug_dump_enabled = false;
-	OutputDirection        printf_direction            = OutputDirection::Silent;
+	LogDirection           printf_direction            = LogDirection::Silent;
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
-	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
+	bool                   profiler_enabled            = false;
 	bool                   spirv_debug_printf_enabled  = false;
 	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
@@ -89,7 +85,7 @@ bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
 ShaderOptimizationType GetShaderOptimizationType();
-ShaderLogDirection     GetShaderLogDirection();
+LogDirection           GetShaderLogDirection();
 std::filesystem::path  GetShaderLogFolder();
 
 bool                  CommandBufferDumpEnabled();
@@ -97,10 +93,10 @@ std::filesystem::path GetCommandBufferDumpFolder();
 
 bool GraphicsDebugDumpEnabled();
 
-OutputDirection       GetPrintfDirection();
+LogDirection          GetPrintfDirection();
 std::filesystem::path GetPrintfOutputFile();
 
-ProfilerDirection GetProfilerDirection();
+bool ProfilerEnabled();
 
 bool SpirvDebugPrintfEnabled();
 

src/common/logging/log.cpp

@@ -152,9 +152,9 @@ void WriteFatal(fmt::text_style style, std::string_view text) {
 void Initialize() {
 	g_initialized = true;
 	switch (Config::GetPrintfDirection()) {
-		case Config::OutputDirection::Silent: g_direction = Direction::Silent; break;
-		case Config::OutputDirection::Console: g_direction = Direction::Console; break;
-		case Config::OutputDirection::File: g_direction = Direction::File; break;
+		case Config::LogDirection::Silent: g_direction = Direction::Silent; break;
+		case Config::LogDirection::Console: g_direction = Direction::Console; break;
+		case Config::LogDirection::File: g_direction = Direction::File; break;
 	}
 	g_output_file =
 	    (g_direction == Direction::File ? Config::GetPrintfOutputFile() : std::filesystem::path {});

src/common/platform/sysDbg.h

@@ -30,8 +30,6 @@ struct sys_dbg_stack_info_t {
 #endif
 };
 
-void SysStackWalk(void** stack, int* depth);
-void SysStackUsage(sys_dbg_stack_info_t& s);          // NOLINT(google-runtime-references)
-void SysStackUsagePrint(sys_dbg_stack_info_t& stack); // NOLINT(google-runtime-references)
+void SysStackUsage(sys_dbg_stack_info_t& s); // NOLINT(google-runtime-references)
 
 #endif /* KYTY_COMMON_PLATFORM_SYSDBG_H_ */

src/common/platform/sysLinuxDbg.cpp

@@ -8,7 +8,6 @@
 
 #include <cstdlib>
 #include <cstring>
-#include <execinfo.h>
 #include <pthread.h>
 #include <sys/param.h>
 #include <sys/types.h>
@@ -17,56 +16,6 @@
 #include <libgen.h> // POSIX basename() lives here on macOS, not in <cstring>
 #endif
 
-// Avoid unwinding a guest-owned stack.
-static bool OnOwnStack() {
-	const char* probe = reinterpret_cast<const char*>(&probe);
-
-	pthread_attr_t attr {};
-#if defined(__APPLE__)
-	const auto* top  = static_cast<const char*>(pthread_get_stackaddr_np(pthread_self()));
-	const auto  size = pthread_get_stacksize_np(pthread_self());
-	(void)attr;
-	return top != nullptr && size != 0 && probe < top && probe >= top - size;
-#else
-	if (pthread_getattr_np(pthread_self(), &attr) != 0) {
-		return false;
-	}
-	void*      base = nullptr;
-	size_t     size = 0;
-	const bool ok = pthread_attr_getstack(&attr, &base, &size) == 0 && base != nullptr && size != 0;
-	pthread_attr_destroy(&attr);
-	if (!ok) {
-		return false;
-	}
-	const auto* low = static_cast<const char*>(base);
-	return probe >= low && probe < low + size;
-#endif
-}
-
-void SysStackWalk(void** stack, int* depth) {
-	if (stack == nullptr || depth == nullptr || *depth <= 0) {
-		if (depth != nullptr) {
-			*depth = 0;
-		}
-		return;
-	}
-
-	if (!OnOwnStack()) {
-		*depth = 0;
-		return;
-	}
-
-	const int n = ::backtrace(stack, *depth);
-	*depth      = (n < 0 ? 0 : n);
-}
-
-void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
-	printf("stack: (0x%" PRIx64 ", %" PRIu64 ")\n", static_cast<uint64_t>(stack.commited_addr),
-	       static_cast<uint64_t>(stack.commited_size));
-	printf("code: (0x%" PRIx64 ", %" PRIu64 ")\n", static_cast<uint64_t>(stack.code_addr),
-	       static_cast<uint64_t>(stack.code_size));
-}
-
 void SysStackUsage(sys_dbg_stack_info_t& s) {
 	pid_t pid = getpid();
 

src/common/platform/sysLinuxVirtual.cpp

@@ -31,8 +31,7 @@
 namespace Common {
 
 static pthread_mutex_t              g_virtual_mutex {};
-static std::map<uintptr_t, size_t>* g_allocs   = nullptr;
-static std::map<uintptr_t, int>*    g_protects = nullptr;
+static std::map<uintptr_t, size_t>* g_allocs = nullptr;
 
 void SysVirtualInit() {
 	pthread_mutexattr_t attr {};
@@ -46,8 +45,7 @@ void SysVirtualInit() {
 	pthread_mutex_init(&g_virtual_mutex, &attr);
 	pthread_mutexattr_destroy(&attr);
 
-	g_allocs   = new std::map<uintptr_t, size_t>;
-	g_protects = new std::map<uintptr_t, int>;
+	g_allocs = new std::map<uintptr_t, size_t>;
 }
 
 static int get_protection_flag(VirtualMemory::Mode mode) {
@@ -68,21 +66,6 @@ static int get_protection_flag(VirtualMemory::Mode mode) {
 	return protect;
 }
 
-static VirtualMemory::Mode get_protection_flag(int mode) {
-	switch (mode) {
-		case PROT_NONE: return VirtualMemory::Mode::NoAccess;
-		case PROT_READ: return VirtualMemory::Mode::Read;
-		case PROT_WRITE: return VirtualMemory::Mode::Write;
-		case PROT_READ | PROT_WRITE: return VirtualMemory::Mode::ReadWrite; // NOLINT
-		case PROT_EXEC: return VirtualMemory::Mode::Execute;
-		case PROT_EXEC | PROT_WRITE: return VirtualMemory::Mode::ExecuteWrite; // NOLINT
-		case PROT_EXEC | PROT_READ: return VirtualMemory::Mode::ExecuteRead;   // NOLINT
-		case PROT_EXEC | PROT_WRITE | PROT_READ:
-			return VirtualMemory::Mode::ExecuteReadWrite; // NOLINT
-		default: return VirtualMemory::Mode::NoAccess;
-	}
-}
-
 // Keep automatic mappings inside the guest and GPU-addressable low window.
 #ifdef KYTY_FIXED_NOREPLACE
 static constexpr uintptr_t LOW_ARENA_LIMIT = 0x000000FC00000000ULL; // libc mspace window ceiling
@@ -161,11 +144,6 @@ uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mo
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start = ret_addr >> 12u;
-		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = protect;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 	}
 
@@ -251,11 +229,6 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
 
 	pthread_mutex_lock(&g_virtual_mutex);
 	record_alloc(ret_addr, size);
-	uintptr_t page_start = ret_addr >> 12u;
-	uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-	for (uintptr_t page = page_start; page <= page_end; page++) {
-		(*g_protects)[page] = protect;
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 
 	return ret_addr;
@@ -337,11 +310,6 @@ bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode m
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start = ret_addr >> 12u;
-		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = protect;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 
 		return true;
@@ -516,18 +484,7 @@ bool SysVirtualFree(uint64_t address) {
 		return false;
 	}
 
-	if (munmap(reinterpret_cast<void*>(addr), size) == 0) {
-		uintptr_t page_start = addr >> 12u;
-		uintptr_t page_end   = (addr + size - 1) >> 12u;
-		pthread_mutex_lock(&g_virtual_mutex);
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			g_protects->erase(page);
-		}
-		pthread_mutex_unlock(&g_virtual_mutex);
-		return true;
-	}
-
-	return false;
+	return munmap(reinterpret_cast<void*>(addr), size) == 0;
 }
 
 bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
@@ -582,40 +539,16 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 	if (end < alloc_end) {
 		(*g_allocs)[end] = alloc_end - end;
 	}
-	for (uintptr_t page = addr >> 12u; page <= (end - 1u) >> 12u; page++) {
-		g_protects->erase(page);
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 	return true;
 }
 
-bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                       VirtualMemory::Mode* old_mode) {
-	auto addr = static_cast<uintptr_t>(address);
-
-	pthread_mutex_lock(&g_virtual_mutex);
-	if (old_mode != nullptr) {
-		if (auto s = g_protects->find(addr >> 12u); s != g_protects->end()) {
-			*old_mode = get_protection_flag(s->second);
-		} else {
-			*old_mode = VirtualMemory::Mode::NoAccess;
-		}
-	}
-	pthread_mutex_unlock(&g_virtual_mutex);
-
-	uintptr_t page_start = addr >> 12u;
-	uintptr_t page_end   = (addr + size - 1) >> 12u;
-	if (mprotect(reinterpret_cast<void*>(page_start << 12u), (page_end - page_start + 1) << 12u,
-	             get_protection_flag(mode)) == 0) {
-		pthread_mutex_lock(&g_virtual_mutex);
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = get_protection_flag(mode);
-		}
-		pthread_mutex_unlock(&g_virtual_mutex);
-		return true;
-	}
-
-	return false;
+bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
+	const auto addr       = static_cast<uintptr_t>(address);
+	const auto page_start = addr >> 12u;
+	const auto page_end   = (addr + size - 1) >> 12u;
+	return mprotect(reinterpret_cast<void*>(page_start << 12u), (page_end - page_start + 1) << 12u,
+	                get_protection_flag(mode)) == 0;
 }
 
 bool SysVirtualFlushInstructionCache(uint64_t /*address*/, uint64_t /*size*/) {

src/common/platform/sysVirtual.h

@@ -19,8 +19,7 @@ bool     SysVirtualReserveFixed(uint64_t address, uint64_t size);
 bool     SysVirtualDecommit(uint64_t address, uint64_t size);
 bool     SysVirtualFree(uint64_t address);
 bool     SysVirtualFreeRange(uint64_t address, uint64_t size);
-bool     SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                           VirtualMemory::Mode* old_mode = nullptr);
+bool     SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode);
 bool     SysVirtualFlushInstructionCache(uint64_t address, uint64_t size);
 
 } // namespace Common

src/common/platform/sysWindowsDbg.cpp

@@ -15,18 +15,6 @@
 #include "common/assert.h"
 #include "common/platform/sysDbg.h"
 
-void SysStackWalk(void** stack, int* depth) {
-	*depth = static_cast<int>(CaptureStackBackTrace(0, static_cast<DWORD>(*depth), stack, nullptr));
-}
-
-void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
-	printf("stack: (0x%" PRIx64 ", %" PRIu64 ") + (0x%" PRIx64 ", %" PRIu64 ") + (0x%" PRIx64
-	       ", %" PRIu64 ")\n",
-	       static_cast<uint64_t>(stack.reserved_addr), static_cast<uint64_t>(stack.reserved_size),
-	       static_cast<uint64_t>(stack.guard_addr), static_cast<uint64_t>(stack.guard_size),
-	       static_cast<uint64_t>(stack.commited_addr), static_cast<uint64_t>(stack.commited_size));
-}
-
 void SysStackUsage(sys_dbg_stack_info_t& s) {
 	MEMORY_BASIC_INFORMATION mbi {};
 	[[maybe_unused]] size_t  ss = VirtualQuery(&mbi, &mbi, sizeof(mbi));

src/common/platform/sysWindowsVirtual.cpp

@@ -45,18 +45,6 @@ static DWORD GetProtectionFlag(VirtualMemory::Mode mode) {
 	return protect;
 }
 
-static VirtualMemory::Mode GetProtectionFlag(DWORD mode) {
-	switch (mode) {
-		case PAGE_NOACCESS: return VirtualMemory::Mode::NoAccess;
-		case PAGE_READONLY: return VirtualMemory::Mode::Read;
-		case PAGE_READWRITE: return VirtualMemory::Mode::ReadWrite;
-		case PAGE_EXECUTE: return VirtualMemory::Mode::Execute;
-		case PAGE_EXECUTE_READ: return VirtualMemory::Mode::ExecuteRead;
-		case PAGE_EXECUTE_READWRITE: return VirtualMemory::Mode::ExecuteReadWrite;
-		default: return VirtualMemory::Mode::NoAccess;
-	}
-}
-
 void SysVirtualInit() {}
 
 uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
@@ -328,17 +316,13 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 	return current - address == size && SysVirtualFree(address);
 }
 
-bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                       VirtualMemory::Mode* old_mode) {
+bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
 	DWORD old_protect = 0;
 	if (VirtualProtect(reinterpret_cast<LPVOID>(static_cast<uintptr_t>(address)), size,
 	                   GetProtectionFlag(mode), &old_protect) == 0) {
 		printf("VirtualProtect() failed: 0x%08" PRIx32 "\n", static_cast<uint32_t>(GetLastError()));
 		return false;
 	}
-	if (old_mode != nullptr) {
-		*old_mode = GetProtectionFlag(old_protect);
-	}
 	return true;
 }
 

src/common/profiler.cpp

@@ -55,19 +55,13 @@ void SetThreadName(const char* name) {
 }
 
 void Initialize() {
-	switch (Config::GetProfilerDirection()) {
-		case Config::ProfilerDirection::Network:
-			if (!tracy::ProfilerAvailable()) {
-				tracy::StartupProfiler();
-				TracySetProgramName("KytyPS5");
-				::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, "
-				         "broadcast %u, connect to 127.0.0.1:8086\n",
-				         tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch,
-				         tracy::ProtocolVersion, tracy::BroadcastVersion);
-			}
-			break;
-		case Config::ProfilerDirection::None:
-		default: break;
+	if (Config::ProfilerEnabled() && !tracy::ProfilerAvailable()) {
+		tracy::StartupProfiler();
+		TracySetProgramName("KytyPS5");
+		::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, "
+		         "broadcast %u, connect to 127.0.0.1:8086\n",
+		         tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch,
+		         tracy::ProtocolVersion, tracy::BroadcastVersion);
 	}
 }
 

src/common/threads.cpp

@@ -1,7 +1,6 @@
 #include "common/threads.h"
 
 #include "common/assert.h"
-#include "common/debug.h"
 
 #include <algorithm>
 #include <atomic>

src/common/virtualMemory.cpp

@@ -50,8 +50,8 @@ bool FreeRange(uint64_t address, uint64_t size) {
 	return SysVirtualFreeRange(address, size);
 }
 
-bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode) {
-	return SysVirtualProtect(address, size, mode, old_mode);
+bool Protect(uint64_t address, uint64_t size, Mode mode) {
+	return SysVirtualProtect(address, size, mode);
 }
 
 bool FlushInstructionCache(uint64_t address, uint64_t size) {

src/common/virtualMemory.h

@@ -35,7 +35,7 @@ bool     ReserveFixed(uint64_t address, uint64_t size);
 bool     Decommit(uint64_t address, uint64_t size);
 bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
-bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
+bool     Protect(uint64_t address, uint64_t size, Mode mode);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
 
 } // namespace VirtualMemory

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -148,7 +148,6 @@ private:
 	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
 	void                CheckBuffer() const { GetScheduler().CheckActive(); }
-	GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); }
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -315,8 +315,17 @@ static void HwCtxSetDepthBoundsRegister(CommandProcessor& cp, uint32_t cmd_offse
 	}
 }
 
-static void HwCtxIgnoreDepthMetadataRegister([[maybe_unused]] uint32_t cmd_offset,
-                                             [[maybe_unused]] uint32_t value) {}
+static void HwCtxSetDepthMetadataRegister(CommandProcessor& cp, uint32_t cmd_offset,
+                                         uint32_t value) {
+	if (cmd_offset == Pm4::DB_RENDER_OVERRIDE) {
+		HW::DepthRenderOverride control;
+		control.force_z_valid       = (value & 0x20000000u) != 0;
+		control.force_z_dirty       = (value & 0x08000000u) != 0;
+		control.force_stencil_valid = (value & 0x40000000u) != 0;
+		control.force_stencil_dirty = (value & 0x10000000u) != 0;
+		cp.GetCtx().SetDepthRenderOverride(control);
+	}
+}
 
 static void HwCtxIgnoreFovWindow([[maybe_unused]] uint32_t value) {}
 
@@ -389,7 +398,7 @@ KYTY_HW_CTX_PARSER(HwCtxSetDepthMetadataRegisters) {
 	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
 
 	for (uint32_t i = 0; i < num_values; i++) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset + i, buffer[i]);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset + i, buffer[i]);
 	}
 
 	return num_values;
@@ -3003,28 +3012,28 @@ void GraphicsInitJmpTablesCxIndirect() {
 		HwCtxIgnoreCbDccControl(value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_COUNT_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	for (auto cmd_offset = Pm4::DB_SRESULTS_COMPARE_STATE0;
 	     cmd_offset <= Pm4::DB_SRESULTS_COMPARE_STATE1; cmd_offset++) {
 		g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-			HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+			HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 		};
 	}
 	g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE2] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_DFSM_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_RMI_L2_CACHE_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::CB_RMI_GL2_CACHE_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::TA_BC_BASE_ADDR] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
 		HwCtxIgnoreBorderColorTableAddr(cmd_offset, value);

src/graphics/guest_gpu/graphicsRun.cpp

@@ -77,7 +77,7 @@ private:
 
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
-	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
+	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
 }
 
 GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
@@ -427,7 +427,7 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 	if (!decode_gds(dst_sel, dst_gds)) {
 		EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel);
 	}
-	auto& buffer_cache = GetGpuResources().GetBufferCache();
+	auto& buffer_cache = m_renderer.GetBufferCache();
 	if (src_sel == 2) {
 		buffer_cache.FillBuffer(
 		    dst_address_or_offset, num_bytes,
@@ -603,11 +603,11 @@ bool GuestGpu::Process(Submission& submission) {
 			}
 			if (progressed) {
 				if (complete) {
-					m_renderer.GetGpuResources().RunGarbageCollector();
+					m_renderer.RunGarbageCollector();
 				}
 				cp.BufferFlush();
 			} else if (complete) {
-				m_renderer.GetGpuResources().RunGarbageCollector();
+				m_renderer.RunGarbageCollector();
 			}
 			break;
 		}
@@ -629,16 +629,16 @@ bool GuestGpu::Process(Submission& submission) {
 			           Pm4ProcessResult::Complete;
 			if (submission.command_execution.MadeProgress()) {
 				if (complete) {
-					m_renderer.GetGpuResources().RunGarbageCollector();
+					m_renderer.RunGarbageCollector();
 				}
 				cp.BufferFlush();
 			} else if (complete) {
-				m_renderer.GetGpuResources().RunGarbageCollector();
+				m_renderer.RunGarbageCollector();
 			}
 			break;
 		}
 		case SubmissionType::FlipPreparation:
-			m_renderer.GetGpuResources().RunGarbageCollector();
+			m_renderer.RunGarbageCollector();
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
 	}

src/graphics/guest_gpu/hardwareContext.h

@@ -202,6 +202,13 @@ struct RenderControl {
 	uint8_t copy_sample              = 0;
 };
 
+struct DepthRenderOverride {
+	bool force_z_valid       = false;
+	bool force_z_dirty       = false;
+	bool force_stencil_valid = false;
+	bool force_stencil_dirty = false;
+};
+
 struct GdsOaCounter {
 	uint32_t counter = 0;
 	uint32_t address = 0;
@@ -809,6 +816,12 @@ public:
 	void SetClipControl(const ClipControl& control) { m_clip_control = control; }
 	[[nodiscard]] const RenderControl& GetRenderControl() const { return m_render_control; }
 	void SetRenderControl(const RenderControl& control) { m_render_control = control; }
+	[[nodiscard]] const DepthRenderOverride& GetDepthRenderOverride() const {
+		return m_depth_render_override;
+	}
+	void SetDepthRenderOverride(const DepthRenderOverride& control) {
+		m_depth_render_override = control;
+	}
 	[[nodiscard]] const DepthControl& GetDepthControl() const { return m_depth_control; }
 	void SetDepthControl(const DepthControl& control) { m_depth_control = control; }
 	[[nodiscard]] const ModeControl& GetModeControl() const { return m_mode_control; }
@@ -904,15 +917,16 @@ private:
 
 	uint32_t m_shader_stages = 0;
 
-	DepthRenderTarget m_depth_render_target;
-	RenderControl     m_render_control;
-	DepthControl      m_depth_control;
-	StencilControl    m_stencil_control;
-	StencilMask       m_stencil_mask;
-	float             m_depth_clear_value   = 0.0f;
-	float             m_depth_bounds_min    = 0.0f;
-	float             m_depth_bounds_max    = 1.0f;
-	uint8_t           m_stencil_clear_value = 0;
+	DepthRenderTarget   m_depth_render_target;
+	RenderControl       m_render_control;
+	DepthRenderOverride m_depth_render_override;
+	DepthControl        m_depth_control;
+	StencilControl      m_stencil_control;
+	StencilMask         m_stencil_mask;
+	float               m_depth_clear_value   = 0.0f;
+	float               m_depth_bounds_min    = 0.0f;
+	float               m_depth_bounds_max    = 1.0f;
+	uint8_t             m_stencil_clear_value = 0;
 
 	ModeControl m_mode_control;
 	PolyOffset  m_poly_offset;

src/graphics/guest_gpu/tile.cpp

@@ -1,5 +1,6 @@
 #include "graphics/guest_gpu/tile.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -15,10 +16,6 @@
 
 namespace Libs::Graphics {
 
-static uint32_t AlignUp(uint32_t value, uint32_t alignment) {
-	return (value + alignment - 1u) & ~(alignment - 1u);
-}
-
 static uint32_t ShiftCeil(uint32_t value, uint32_t shift) {
 	return static_cast<uint32_t>((static_cast<uint64_t>(value) + (1ull << shift) - 1ull) >> shift);
 }
@@ -33,7 +30,7 @@ static uint32_t CalcLinearAlignedLevelPitch(uint32_t base_width, uint32_t base_h
 	const uint32_t level_width  = ShiftCeil(base_width, level);
 	const uint32_t level_height = ShiftCeil(base_height, level);
 	const uint32_t padded_width =
-	    AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element));
+	    Common::AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element));
 	const uint64_t size =
 	    static_cast<uint64_t>(padded_width) * std::max(level_height, 1u) * bytes_per_element;
 	EXIT_NOT_IMPLEMENTED(size > 0xffffffffull);
@@ -61,7 +58,7 @@ static uint32_t SetLinearMipChainLayout(uint32_t levels, const uint32_t* mip_pit
 		offset += mip_size[level];
 	}
 
-	return AlignUp(offset, 256u);
+	return Common::AlignUp(offset, 256u);
 }
 
 struct Log2BlockDimensions {
@@ -366,8 +363,8 @@ bool TileGetTiledTextureLayout(const TileSurfaceDescription& description, TileSu
 		    ((description.width >> level) + texture.texel_width - 1u) / texture.texel_width, 1u);
 		mip.height = std::max(
 		    ((description.height >> level) + texture.texel_height - 1u) / texture.texel_height, 1u);
-		mip.padded_width  = AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width);
-		mip.padded_height = AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height);
+		mip.padded_width  = Common::AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width);
+		mip.padded_height = Common::AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height);
 		mip.size = static_cast<uint64_t>(block.block_depth) * mip.padded_width * mip.padded_height *
 		           block.bytes_per_element;
 		result.block_slice_size += mip.size;
@@ -1121,8 +1118,8 @@ static bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign& hti
 	}
 	// Prospero HTile stores one DWORD per depth tile. Its 32 KiB allocation blocks cover
 	// 1024x512 pixels, independently of the attachment's fragment count.
-	const uint64_t size = static_cast<uint64_t>(AlignUp(width, 1024u) / 1024u) *
-	                      (AlignUp(height, 512u) / 512u) * 32768u;
+	const uint64_t size = static_cast<uint64_t>(Common::AlignUp(width, 1024u) / 1024u) *
+	                      (Common::AlignUp(height, 512u) / 512u) * 32768u;
 	if (size == 0 || size > UINT32_MAX) {
 		return false;
 	}
@@ -1154,13 +1151,13 @@ bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch,
 		                                           &stencil_block_height));
 		const uint32_t fragments = 1u << num_fragments_log2;
 		const uint64_t depth_bytes_total =
-		    valid_blocks ? static_cast<uint64_t>(AlignUp(width, depth_block_width)) *
-		                       AlignUp(height, depth_block_height) * depth_bytes * fragments
+		    valid_blocks ? static_cast<uint64_t>(Common::AlignUp(width, depth_block_width)) *
+		                       Common::AlignUp(height, depth_block_height) * depth_bytes * fragments
 		                 : 0;
 		const uint64_t stencil_bytes_total =
 		    stencil_format == Prospero::StencilFormat::k8UInt && valid_blocks
-		        ? static_cast<uint64_t>(AlignUp(width, stencil_block_width)) *
-		              AlignUp(height, stencil_block_height) * fragments
+		        ? static_cast<uint64_t>(Common::AlignUp(width, stencil_block_width)) *
+		              Common::AlignUp(height, stencil_block_height) * fragments
 		        : 0;
 		TileSizeAlign calculated_htile {};
 		const bool    htile_valid = !htile || TileGetHtileSize(width, height, calculated_htile);
@@ -1187,8 +1184,7 @@ uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element,
 	                                                         &block_width, &block_height)) {
 		return 0;
 	}
-	const uint64_t pitch = (static_cast<uint64_t>(width) + block_width - 1u) &
-	                       ~static_cast<uint64_t>(block_width - 1u);
+	const uint64_t pitch = Common::AlignUp(static_cast<uint64_t>(width), block_width);
 	return pitch <= UINT32_MAX ? static_cast<uint32_t>(pitch) : 0;
 }
 
@@ -1209,8 +1205,7 @@ bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
 	    pitch != TileGetRenderTargetPitch(width, bytes_per_element, num_fragments_log2)) {
 		return false;
 	}
-	const uint64_t padded_height = (static_cast<uint64_t>(height) + block_height - 1u) &
-	                               ~static_cast<uint64_t>(block_height - 1u);
+	const uint64_t padded_height = Common::AlignUp(static_cast<uint64_t>(height), block_height);
 	const uint64_t size = static_cast<uint64_t>(pitch) * padded_height * bytes_per_element *
 	                      (1u << num_fragments_log2);
 	if (size == 0 || size > UINT32_MAX) {
@@ -1372,7 +1367,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 			TileTextureElementLayout element {};
 			if (TileGetTextureElementLayout(format, element) && element.texel_width == 1 &&
 			    element.texel_height == 1) {
-				pitch = AlignUp(pitch, CalcLinearBlockWidth(element.bytes));
+				pitch = Common::AlignUp(pitch, CalcLinearBlockWidth(element.bytes));
 			}
 			break;
 		}
@@ -1383,7 +1378,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 		case Prospero::TileMode::kRenderTarget: {
 			TileTextureBlockLayout layout {};
 			if (TileGetTextureBlockLayout(format, tile, false, layout)) {
-				pitch = AlignUp(pitch, layout.block.block_width * layout.texel_width);
+				pitch = Common::AlignUp(pitch, layout.block.block_width * layout.texel_width);
 			}
 			break;
 		}

src/graphics/host_gpu/graphicContext.h

@@ -27,7 +27,6 @@ struct GraphicContext {
 	vk::Device                         device                                = nullptr;
 	VmaAllocator                       allocator                             = nullptr;
 	bool                               memory_budget_ext_enabled             = false;
-	bool                               rt_extensions_enabled                 = false;
 	bool                               compute_subgroup_size_control_enabled = false;
 	bool                               sample_rate_shading_enabled           = false;
 	bool                               attachment_feedback_loop_enabled      = false;
@@ -99,10 +98,6 @@ struct GraphicContext {
 	[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
 	[[nodiscard]] bool     CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
 	void                   DeleteImage(VulkanImage& image);
-	void                   AppendHardwareRayTracingDeviceExtensions(
-	    const std::vector<vk::ExtensionProperties>& available_extensions,
-	    std::vector<const char*>&                   device_extensions);
-	void LoadHardwareRayTracingFunctions() const;
 
 	uint32_t screen_width  = 0;
 	uint32_t screen_height = 0;

src/graphics/host_gpu/memoryTracker.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/memoryTracker.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 
 namespace Libs::Graphics {
@@ -8,9 +9,6 @@ static_assert(std::atomic<void*>::is_always_lock_free);
 
 MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) {
 	m_regions = std::make_unique<std::atomic<RegionManager*>[]>(REGION_COUNT);
-	for (size_t i = 0; i < REGION_COUNT; i++) {
-		m_regions[i].store(nullptr, std::memory_order_relaxed);
-	}
 }
 
 MemoryTracker::~MemoryTracker() = default;
@@ -34,8 +32,8 @@ void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr,
 void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
                                               const char* operation) {
 	ValidateRange(vaddr, size);
-	const auto begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
+	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
 	for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
 		const bool has_dirty_bytes = dirty.Intersects(page, TRACKER_PAGE_SIZE);
 		if (IsRegionGpuModified(page, TRACKER_PAGE_SIZE) != has_dirty_bytes) {

src/graphics/host_gpu/memoryTracker.h

@@ -62,43 +62,18 @@ public:
 	void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {}
 #endif
 
-	template <bool clear, typename Preflight, typename Func>
-	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) {
-		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
-		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
-		CheckNotInUploadCallback();
-		std::vector<RegionManager*> managers;
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
-			managers.push_back(manager);
-		});
-		std::vector<std::unique_lock<TrackingSpinLock>> locks;
-		locks.reserve(managers.size());
-		for (auto* manager: managers) {
-			locks.emplace_back(manager->lock);
-		}
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			const auto address = manager->GetCpuAddr() + offset;
-			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes,
-			                                                                preflight);
-		});
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(
-			    manager->GetCpuAddr() + offset, bytes, func);
-		});
-		if constexpr (clear) {
-			Iterate<false>(vaddr, size,
-			               [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				               const auto address = manager->GetCpuAddr() + offset;
-				               manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
-				                   address, bytes, [](uint64_t, uint64_t) noexcept {});
-			               });
-		}
-	}
-
 	template <bool clear, typename Func>
 	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Func&& func) {
-		ForEachDownloadRange<clear>(
-		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
+		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
+		CheckNotInUploadCallback();
+		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+			std::scoped_lock lock(manager->lock);
+			const auto       address = manager->GetCpuAddr() + offset;
+			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes, func);
+			if constexpr (clear) {
+				manager->template ChangeState<DirtySource::Gpu, false>(address, bytes);
+			}
+		});
 	}
 
 	template <typename RangeFunc, typename UploadFunc>

src/graphics/host_gpu/pageManager.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/pageManager.h"
 
-#include "common/platform/sysDbg.h"
+#include "common/alignment.h"
+#include "common/virtualMemory.h"
 #include "graphics/host_gpu/regionDefinitions.h"
 #include "kernel/memory.h"
 
@@ -21,10 +22,7 @@
 #include <windows.h>
 #undef min
 #undef max
-#elif defined(__APPLE__)
-#include <unistd.h>
 #else
-#include <execinfo.h>
 #include <unistd.h>
 #endif
 
@@ -36,38 +34,12 @@ constexpr uint64_t REGION_SIZE  = TRACKER_REGION_SIZE;
 constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
 constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
 
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-// The tracker reuses Win32 memory-protection tags as internal page-state values.
-// Mirror their canonical numeric values so the shared state-machine logic is identical.
-constexpr uint32_t PAGE_NOACCESS  = 0x01;
-constexpr uint32_t PAGE_READONLY  = 0x02;
-constexpr uint32_t PAGE_READWRITE = 0x04;
-#endif
 constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 
-constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
-constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
-constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-
 [[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
 	std::fputs("PageManager fail-fast: ", stderr);
 	std::fputs(reason != nullptr ? reason : "invalid page state", stderr);
 	std::fputc('\n', stderr);
-#if !defined(__APPLE__)
-	void* frames[16] {};
-	int   frame_count = static_cast<int>(std::size(frames));
-	SysStackWalk(frames, &frame_count);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	const auto image_base = reinterpret_cast<uintptr_t>(GetModuleHandleW(nullptr));
-	for (int i = 0; i < frame_count; i++) {
-		const auto address = reinterpret_cast<uintptr_t>(frames[i]);
-		std::fprintf(stderr, "  frame[%d]=0x%016" PRIxPTR " image_rva=0x%016" PRIxPTR "\n", i,
-		             address, address >= image_base ? address - image_base : 0);
-	}
-#else
-	::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
-#endif
-#endif
 	std::fflush(stderr);
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
@@ -86,15 +58,6 @@ constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
 	std::_Exit(322);
 }
 
-Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
-	switch (protection) {
-		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
-		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
-		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
-		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
-	}
-}
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -115,15 +78,6 @@ void ValidateRange(uint64_t vaddr, uint64_t size) {
 	}
 }
 
-uint64_t PageStart(uint64_t vaddr) {
-	return vaddr & ~(PAGE_SIZE - 1);
-}
-
-uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
-	ValidateRange(vaddr, size);
-	return PageStart(vaddr + size - 1) + PAGE_SIZE;
-}
-
 } // namespace
 
 struct PageManager::Impl {
@@ -131,14 +85,14 @@ struct PageManager::Impl {
 		uint8_t write_watchers  : 7 = 0;
 		uint8_t access_watchers : 1 = 0;
 
-		[[nodiscard]] uint32_t Perms() const noexcept {
+		[[nodiscard]] Common::VirtualMemory::Mode Perms() const noexcept {
 			if (access_watchers != 0) {
-				return NO_ACCESS_PROTECTION;
+				return Common::VirtualMemory::Mode::NoAccess;
 			}
 			if (write_watchers != 0) {
-				return READ_ONLY_PROTECTION;
+				return Common::VirtualMemory::Mode::Read;
 			}
-			return READ_WRITE_PROTECTION;
+			return Common::VirtualMemory::Mode::ReadWrite;
 		}
 
 		template <int delta, bool is_read>
@@ -202,9 +156,6 @@ struct PageManager::Impl {
 		}
 #endif
 		regions = std::make_unique<std::atomic<Region*>[]>(REGION_COUNT);
-		for (uint64_t i = 0; i < REGION_COUNT; i++) {
-			regions[i].store(nullptr, std::memory_order_relaxed);
-		}
 	}
 
 	~Impl() {
@@ -239,11 +190,10 @@ struct PageManager::Impl {
 		return ptr;
 	}
 
-	void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept {
-		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
-		                                                     ToMemoryMode(protection))) {
-			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
-			      protection);
+	void Protect(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) noexcept {
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, mode)) {
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", mode=0x%08" PRIx32, vaddr,
+			      static_cast<uint32_t>(mode));
 		}
 	}
 
@@ -300,11 +250,12 @@ struct PageManager::Impl {
 
 	template <bool track, bool is_read>
 	void UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
-		const auto begin = PageStart(vaddr);
-		const auto end   = PageEnd(vaddr, size);
+		ValidateRange(vaddr, size);
+		const auto begin = Common::AlignDown(vaddr, PAGE_SIZE);
+		const auto end   = Common::AlignUp(vaddr + size, PAGE_SIZE);
 		for (auto chunk_begin = begin; chunk_begin < end;) {
-			const auto chunk_end   = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
-			const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE;
+			const auto chunk_end = std::min(end, Common::AlignUp(chunk_begin + 1, REGION_SIZE));
+			const auto region_base = Common::AlignDown(chunk_begin, REGION_SIZE);
 			auto*      region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin);
 			if (region == nullptr) {
 				Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);

src/graphics/host_gpu/rangeSet.h

@@ -6,17 +6,11 @@
 #include <algorithm>
 #include <cstdint>
 #include <map>
-#include <vector>
 
 namespace Libs::Graphics {
 
 class RangeSet final {
 public:
-	struct Range {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-
 	void Add(uint64_t address, uint64_t size) {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.lower_bound(address);
@@ -62,12 +56,6 @@ public:
 		}
 	}
 
-	[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
-		std::vector<Range> result;
-		ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
-		return result;
-	}
-
 	[[nodiscard]] bool Intersects(uint64_t address, uint64_t size) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.lower_bound(address);
@@ -88,7 +76,7 @@ public:
 	}
 
 	template <typename Func>
-	void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
+	void ForEachInRange(uint64_t address, uint64_t size, Func&& func) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.upper_bound(address);
 		if (it != m_ranges.begin()) {
@@ -98,7 +86,7 @@ public:
 			const auto begin = std::max(address, it->first);
 			const auto last  = std::min(end, it->second);
 			if (begin < last) {
-				func(Range {begin, last - begin});
+				func(begin, last);
 			}
 		}
 	}

src/graphics/host_gpu/regionManager.h

@@ -111,56 +111,43 @@ public:
 			bits.UnsetRange(start, end);
 		}
 		if constexpr (source == DirtySource::Cpu) {
-			UpdateCpuProtection<!enable>();
+			UpdateProtection<!enable, false>();
 		} else {
-			UpdateGpuProtection<enable>();
+			UpdateProtection<enable, true>();
 		}
 	}
 
 	template <DirtySource source, bool clear, typename Func>
 	void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
 		const auto [start, end] = GetPageRange(vaddr, size);
-		RegionBits mask(GetBits<source>(), start, end);
+		auto&      bits         = GetBits<source>();
+		RegionBits mask(bits, start, end);
 		if constexpr (clear) {
-			GetBits<source>().UnsetRange(start, end);
+			bits.UnsetRange(start, end);
+			if constexpr (source == DirtySource::Cpu) {
+				UpdateProtection<true, false>();
+			} else {
+				UpdateProtection<false, true>();
+			}
 		}
-		if constexpr (source == DirtySource::Cpu && clear) {
-			UpdateCpuProtection<true>();
-			ForEachRange(mask, std::forward<Func>(func));
-			return;
+		for (const auto [first, last]: mask) {
+			func(m_cpu_addr + first * TRACKER_PAGE_SIZE, (last - first) * TRACKER_PAGE_SIZE);
 		}
-		if constexpr (source == DirtySource::Gpu && clear) {
-			UpdateGpuProtection<false>();
-		}
-		ForEachRange(mask, std::forward<Func>(func));
 	}
 
 	TrackingSpinLock lock;
 
 private:
-	template <bool track>
-	void UpdateCpuProtection() {
-		auto mask  = m_cpu_dirty ^ m_writable;
-		m_writable = m_cpu_dirty;
+	template <bool track, bool is_read>
+	void UpdateProtection() {
+		const auto protection = is_read ? ~m_gpu_dirty : m_cpu_dirty;
+		auto&      previous   = is_read ? m_readable : m_writable;
+		auto       mask       = protection ^ previous;
 		if (mask.None()) {
 			return;
 		}
-		m_page_manager.UpdatePageWatchersForRegion<track>(m_cpu_addr, mask);
-	}
-
-	template <bool track>
-	void UpdateGpuProtection() {
-		auto readable = ~m_gpu_dirty;
-		auto mask     = readable ^ m_readable;
-		m_readable    = readable;
-		if (mask.None()) {
-			return;
-		}
-		if constexpr (track) {
-			m_page_manager.UpdatePageWatchersForRegion<true, true>(m_cpu_addr, mask);
-		} else {
-			m_page_manager.UpdatePageWatchersForRegion<false, true>(m_cpu_addr, mask);
-		}
+		previous = protection;
+		m_page_manager.UpdatePageWatchersForRegion<track, is_read>(m_cpu_addr, mask);
 	}
 
 	template <DirtySource source>
@@ -191,13 +178,6 @@ private:
 		        static_cast<size_t>((offset + size + TRACKER_PAGE_SIZE - 1) / TRACKER_PAGE_SIZE)};
 	}
 
-	template <typename Func>
-	void ForEachRange(const RegionBits& bits, Func&& func) const {
-		for (const auto [start, end]: bits) {
-			func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE);
-		}
-	}
-
 	PageManager& m_page_manager;
 	uint64_t     m_cpu_addr = 0;
 	RegionBits   m_cpu_dirty;

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -42,13 +43,6 @@ void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void*
 	}
 }
 
-struct BufferCache::DownloadCopy {
-	Buffer*  buffer        = nullptr;
-	uint64_t source_offset = 0;
-	uint64_t address       = 0;
-	uint64_t size          = 0;
-};
-
 void BufferCache::Register(BufferId id) {
 	ChangeRegister<true>(id);
 }
@@ -103,8 +97,7 @@ void BufferCache::TouchBuffer(const Buffer& buffer) {
 }
 
 void BufferCache::DeleteBuffer(BufferId id) {
-	auto* buffer = m_slot_buffers.try_get(id);
-	if (buffer == nullptr || buffer->is_deleted) {
+	if (IsBufferInvalid(id)) {
 		return;
 	}
 	Unregister(id);
@@ -115,79 +108,70 @@ void BufferCache::DeleteBuffer(BufferId id) {
 	}
 }
 
-std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
-	if (copy.buffer == nullptr || copy.size == 0 || copy.source_offset > copy.buffer->Size() ||
-	    copy.size > copy.buffer->Size() - copy.source_offset) {
-		EXIT("BufferCache: invalid download copy\n");
+bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size) {
+	std::vector<vk::BufferCopy> copies;
+	uint64_t                    total_size     = 0;
+	const auto                  buffer_address = buffer.CpuAddress();
+	m_memory_tracker.ForEachDownloadRange<false>(
+	    vaddr, size, [&](uint64_t address, uint64_t bytes) noexcept {
+		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
+		                                           "buffer download");
+		    m_gpu_modified_ranges.ForEachInRange(address, bytes, [&](uint64_t start, uint64_t end) {
+			    copies.emplace_back(start - buffer_address, total_size, end - start);
+			    // Keep packed ranges on separate cache lines, as in shadPS4.
+			    total_size += Common::AlignUp(end - start, 64);
+		    });
+		    m_gpu_modified_ranges.Subtract(address, bytes);
+	    });
+	if (copies.empty()) {
+		return false;
 	}
-	const auto begin = copy.source_offset & ~uint64_t {3};
-	if (copy.source_offset > UINT64_MAX - copy.size ||
-	    copy.source_offset + copy.size > UINT64_MAX - 3) {
-		EXIT("BufferCache: download copy alignment overflow\n");
-	}
-	const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3};
-	if (end > copy.buffer->Size()) {
-		EXIT("BufferCache: aligned download copy exceeds its owner\n");
-	}
-	return {begin, end - begin};
-}
 
-void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
-	std::vector<DownloadCopy> batch;
-	batch.reserve(copies.size());
-	uint64_t                  packed_size = 0;
-	auto&                     download    = m_download_buffer;
-	const auto flush = [&] {
-		const auto [mapped, base_offset] = download.Map(packed_size, DOWNLOAD_ALIGNMENT);
-		EXIT_IF(mapped == nullptr);
-		uint64_t cursor = 0;
-		for (const auto& copy: batch) {
-			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-			download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
-			                  envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
-			                  vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
-			                  vk::AccessFlagBits::eHostRead);
-			cursor += AlignDownload(envelope_size);
-		}
-		download.Commit();
-		const auto completion_tick = m_scheduler.CurrentTick();
-		m_scheduler.Finish();
-		m_scheduler.WaitPriorityOperations(completion_tick);
-		cursor = 0;
-		for (const auto& copy: batch) {
-			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-			const auto offset = cursor + copy.source_offset - source_begin;
-			download.Invalidate(base_offset + offset, copy.size);
-			Libs::LibKernel::Memory::WriteBacking(copy.address, mapped + offset, copy.size);
-			cursor += AlignDownload(envelope_size);
-		}
-		batch.clear();
-		packed_size = 0;
-	};
-	for (auto copy: copies) {
-		while (copy.size != 0) {
-			const auto available = download.Size() - packed_size;
-			const auto prefix    = copy.source_offset & 3u;
-			const auto bytes     = std::min(copy.size, available - prefix);
-			DownloadCopy part {copy.buffer, copy.source_offset, copy.address, bytes};
-			const auto [source_begin, envelope_size] = DownloadEnvelope(part);
-			(void)source_begin;
-			packed_size += AlignDownload(envelope_size);
-			batch.push_back(part);
-			copy.source_offset += bytes;
-			copy.address += bytes;
-			copy.size -= bytes;
-			if (packed_size == download.Size()) {
-				flush();
-			}
-		}
+	const auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
+	if (mapped == nullptr) {
+		EXIT("BufferCache: download exceeds 32 MiB staging buffer capacity\n");
 	}
-	if (!batch.empty()) {
-		flush();
-	}
-	for (const auto& copy: copies) {
-		m_gpu_modified_ranges.Subtract(copy.address, copy.size);
+	m_download_buffer.Commit();
+	for (auto& copy: copies) {
+		copy.dstOffset += offset;
 	}
+
+	auto& command = m_scheduler.Current();
+	command.EndRendering();
+	const auto              native = command.Handle();
+	vk::BufferMemoryBarrier before {};
+	before.srcAccessMask       = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite;
+	before.dstAccessMask       = vk::AccessFlagBits::eTransferRead;
+	before.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+	before.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+	before.buffer              = buffer.Handle();
+	before.offset              = 0;
+	before.size                = buffer.Size();
+	native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
+	                       vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &before, 0,
+	                       nullptr);
+	native.copyBuffer(buffer.Handle(), m_download_buffer.Handle(),
+	                  static_cast<uint32_t>(copies.size()), copies.data());
+
+	auto after          = before;
+	after.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+	after.dstAccessMask = vk::AccessFlagBits::eHostRead;
+	after.buffer        = m_download_buffer.Handle();
+	after.offset        = offset;
+	after.size          = total_size;
+	native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+	                       vk::PipelineStageFlagBits::eAllCommands |
+	                           vk::PipelineStageFlagBits::eHost,
+	                       {}, 0, nullptr, 1, &after, 0, nullptr);
+	m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address,
+	                                    copies = std::move(copies)] {
+		m_download_buffer.Invalidate(offset, total_size);
+		for (const auto& copy: copies) {
+			Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset,
+			                                      mapped + (copy.dstOffset - offset), copy.size);
+		}
+	});
+	return true;
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -252,44 +236,29 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 		     vaddr, size);
 	}
-	m_scheduler.Context().GetGpu().SendCommandSync(
-	    [this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); });
-}
+	m_scheduler.Context().GetGpu().SendCommandSync([this, vaddr, size, is_write] {
+		if (is_write && !IsRegionRegistered(vaddr, size)) {
+			return;
+		}
+		auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
 
-void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) {
-	if (is_write && !IsRegionRegistered(vaddr, size)) {
-		return;
-	}
-	auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
+		// Widen nearby CPU reads so they share one GPU drain.
+		constexpr uint64_t WindowSize   = 512 * 1024;
+		const auto         buffer_begin = buffer.CpuAddress();
+		const auto         buffer_end   = buffer_begin + buffer.Size();
+		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
+		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
 
-	// Widen nearby CPU reads so they share one GPU drain.
-	constexpr uint64_t WindowSize   = 512 * 1024;
-	const auto         buffer_begin = buffer.CpuAddress();
-	const auto         buffer_end   = buffer_begin + buffer.Size();
-	const auto         window_begin = std::max(vaddr & ~(WindowSize - 1), buffer_begin);
-	const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
-
-	std::vector<DownloadCopy> copies;
-	m_memory_tracker.ForEachDownloadRange<false>(
-	    window_begin, window_end - window_begin,
-	    [&](uint64_t address, uint64_t bytes) noexcept {
-		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
-		                                           "memory invalidation");
-	    },
-	    [&](uint64_t address, uint64_t bytes) noexcept {
-		    m_gpu_modified_ranges.ForEachIntersection(address, bytes, [&](RangeSet::Range range) {
-			    copies.push_back(
-			        {&buffer, buffer.Offset(range.address), range.address, range.size});
-		    });
-	    });
-	if (!copies.empty()) {
-		DownloadBufferMemory(copies);
-		// The enumeration covered whole dirty pages and every exact interval on them.
-		m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
-	}
-	if (is_write) {
-		m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
-	}
+		if (DownloadBufferMemory(buffer, window_begin, window_end - window_begin)) {
+			const auto tick = m_scheduler.CurrentTick();
+			m_scheduler.Wait(tick);
+			m_scheduler.WaitPriorityOperations(tick);
+			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
+		}
+		if (is_write) {
+			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
+		}
+	});
 }
 
 BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
@@ -340,10 +309,12 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t
 		end                       = std::max(end, buffer_end);
 		if (!has_stream_leap && (stream_score += buffer.StreamScore()) > StreamLeapThreshold) {
 			has_stream_leap = true;
-			if (expands_right) {
+			// Reserve space in the incoming stream's direction of growth.
+			// The old buffer extending left of the request predicts growth to the right, and vice versa.
+			if (expands_left) {
 				end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
 			}
-			if (expands_left) {
+			if (expands_right) {
 				const auto minimum = CACHING_PAGESIZE * 2;
 				if (begin > minimum) {
 					begin -= std::min(StreamLeapSize, begin - minimum);
@@ -369,8 +340,8 @@ void BufferCache::JoinOverlap(BufferId new_id, BufferId overlap_id, bool accumul
 
 BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
 	EXIT_IF(m_scheduler.Current().IsInvalid());
-	const auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
-	vaddr &= ~(CACHING_PAGESIZE - 1);
+	const auto end = Common::AlignUp(vaddr + size, CACHING_PAGESIZE);
+	vaddr = Common::AlignDown(vaddr, CACHING_PAGESIZE);
 	size               = end - vaddr;
 	const auto overlap = ResolveOverlaps(vaddr, size);
 
@@ -483,17 +454,16 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
 		}
 	}
 
-	auto* buffer = m_slot_buffers.try_get(id);
-	if (buffer == nullptr || buffer->is_deleted || !buffer->IsInBounds(vaddr, size)) {
-		id     = FindBuffer(vaddr, size);
-		buffer = &m_slot_buffers[id];
+	if (IsBufferInvalid(id) || !m_slot_buffers[id].IsInBounds(vaddr, size)) {
+		id = FindBuffer(vaddr, size);
 	}
-	TouchBuffer(*buffer);
-	(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
+	auto& buffer = m_slot_buffers[id];
+	TouchBuffer(buffer);
+	(void)SynchronizeBuffer(buffer, vaddr, size, is_written, is_texel_buffer);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
-	return {buffer, buffer->Offset(vaddr)};
+	return {&buffer, buffer.Offset(vaddr)};
 }
 
 std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
@@ -545,9 +515,7 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	}
 
 	m_texture_cache.InvalidateMemoryFromGPU(vaddr, size);
-	const auto id          = FindBuffer(vaddr, size);
-	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true, id);
-	EXIT_IF(dst == nullptr);
+	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true);
 	dst->Fill(dst_offset, size, value);
 }
 
@@ -581,10 +549,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	                                    : std::pair {&m_gds_buffer, src_vaddr};
 	auto [dst, dst_offset] = dst_memory ? ObtainBuffer(dst_vaddr, size, true, true, dst_id)
 	                                    : std::pair {&m_gds_buffer, dst_vaddr};
-	EXIT_IF(src == nullptr || dst == nullptr);
-	if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) {
-		EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
-	}
 	dst->CopyFrom(command, *src, src_offset, dst_offset, size);
 }
 
@@ -628,8 +592,7 @@ void BufferCache::RunGarbageCollector() {
 	const size_t   limit      = aggressive ? 64 : 32;
 
 	std::vector<BufferId> dirty_buffers;
-	std::vector<DownloadCopy> copies;
-	size_t                    retire_count = 0;
+	size_t                retire_count = 0;
 	m_lru_cache.ForEachItemBelow(tick - age, [&](BufferId id) {
 		auto& buffer = m_slot_buffers[id];
 		EXIT_IF(buffer.is_deleted);
@@ -640,19 +603,7 @@ void BufferCache::RunGarbageCollector() {
 			return false;
 		}
 		if (dirty) {
-			m_memory_tracker.ForEachDownloadRange<false>(
-			    buffer.CpuAddress(), buffer.Size(),
-			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address,
-				                                           dirty_size, "garbage collection");
-			    },
-			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    m_gpu_modified_ranges.ForEachIntersection(
-				        dirty_address, dirty_size, [&](RangeSet::Range range) {
-					    copies.push_back({&buffer, range.address - buffer.CpuAddress(),
-					                      range.address, range.size});
-				        });
-				});
+			EXIT_IF(!DownloadBufferMemory(buffer, buffer.CpuAddress(), buffer.Size()));
 			dirty_buffers.push_back(id);
 		} else {
 			m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());
@@ -664,8 +615,10 @@ void BufferCache::RunGarbageCollector() {
 		return;
 	}
 
-	EXIT_IF(copies.empty());
-	DownloadBufferMemory(copies);
+	// Publish all queued downloads before releasing their tracked pages and owners.
+	const auto completion_tick = m_scheduler.CurrentTick();
+	m_scheduler.Wait(completion_tick);
+	m_scheduler.WaitPriorityOperations(completion_tick);
 	for (const auto id: dirty_buffers) {
 		auto& buffer = m_slot_buffers[id];
 		m_memory_tracker.UnmarkRegionAsGpuModified(buffer.CpuAddress(), buffer.Size());

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -74,6 +74,11 @@ public:
 private:
 	friend struct BufferCacheTestAccess;
 
+	bool IsBufferInvalid(BufferId id) const {
+		const auto* buffer = m_slot_buffers.try_get(id);
+		return buffer == nullptr || buffer->is_deleted;
+	}
+
 	using BufferMap = std::map<uint64_t, BufferId>;
 	struct OverlapResult {
 		BufferMap::iterator first;
@@ -83,14 +88,8 @@ private:
 		bool                has_stream_leap;
 	};
 
-	struct DownloadCopy;
 	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
 	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
-	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
-		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
-	}
-	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
 	void TouchBuffer(const Buffer& buffer);
 	[[nodiscard]] OverlapResult ResolveOverlaps(uint64_t vaddr, uint64_t size);
@@ -106,8 +105,8 @@ private:
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
-	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
+	// Queues backing publication; callers wait before clearing dirty pages or reusing their data.
+	[[nodiscard]] bool DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -1,96 +0,0 @@
-#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
-
-#include "common/assert.h"
-#include "graphics/guest_gpu/graphicsRun.h"
-#include "graphics/host_gpu/renderer/commandScheduler.h"
-namespace Libs::Graphics {
-
-GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
-    : m_scheduler(scheduler), m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache),
-      m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache) {}
-
-GpuResourceManager::~GpuResourceManager() = default;
-
-bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	// The host reports the faulting byte, not the instruction's access width. Both caches
-	// resolve its page; guessing a width can cross the end of a valid guest mapping.
-	constexpr uint64_t fault_size = 1;
-	if (!IsMapped(fault_vaddr, fault_size)) {
-		return false;
-	}
-	if (access == PageFaultAccess::Write) {
-		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
-		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
-	} else {
-		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
-	}
-	return true;
-}
-
-bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
-	if (!IsMapped(vaddr, size)) {
-		return false;
-	}
-	m_buffer_cache.InvalidateMemory(vaddr, size);
-	m_texture_cache.InvalidateMemory(vaddr, size);
-	return true;
-}
-
-bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (!GuestRange {vaddr, size}.Valid()) {
-		return false;
-	}
-	std::shared_lock lock(m_mapped_ranges_mutex);
-	return m_mapped_ranges.Contains(vaddr, size);
-}
-
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
-	{
-		std::lock_guard lock(m_mapped_ranges_mutex);
-		m_mapped_ranges.Add(vaddr, size);
-	}
-}
-
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	if (CommandScheduler::InDeferredOperation()) {
-		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
-		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	const auto unmap = [this, vaddr, size] {
-		if (m_scheduler.Active()) {
-			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.Finish();
-			m_scheduler.WaitPriorityOperations(tick);
-		}
-		m_buffer_cache.InvalidateMemory(vaddr, size);
-		m_texture_cache.UnmapMemory(vaddr, size);
-		std::lock_guard lock(m_mapped_ranges_mutex);
-		m_mapped_ranges.Subtract(vaddr, size);
-	};
-	if (m_gpu == nullptr) {
-		unmap();
-		return;
-	}
-	m_gpu->SendCommandSync(unmap);
-}
-
-void GpuResourceManager::PrepareBda() {
-	std::shared_lock lock(m_mapped_ranges_mutex);
-	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
-		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
-	});
-	m_fault_process_pending = true;
-}
-
-void GpuResourceManager::RunGarbageCollector() {
-	if (m_fault_process_pending) {
-		m_fault_process_pending = false;
-		m_buffer_cache.ProcessFaultBuffer();
-	}
-	m_texture_cache.ProcessDownloadImages();
-	m_texture_cache.RunGarbageCollector();
-	m_buffer_cache.RunGarbageCollector();
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -1,49 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
-
-#include "common/abi.h"
-#include "common/common.h"
-#include "graphics/host_gpu/pageManager.h"
-#include "graphics/host_gpu/renderer/cache/bufferCache.h"
-#include "graphics/host_gpu/renderer/cache/textureCache.h"
-
-#include <cstdint>
-#include <shared_mutex>
-
-namespace Libs::Graphics {
-
-class CommandScheduler;
-class GuestGpu;
-
-class GpuResourceManager {
-public:
-	GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler);
-	~GpuResourceManager();
-	KYTY_CLASS_NO_COPY(GpuResourceManager);
-
-	[[nodiscard]] BufferCache&  GetBufferCache() { return m_buffer_cache; }
-	[[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; }
-	void                        SetGpu(GuestGpu* gpu) noexcept { m_gpu = gpu; }
-
-	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
-	void               PrepareBda();
-	void               RunGarbageCollector();
-
-private:
-	PageManager               m_page_manager;
-	CommandScheduler&         m_scheduler;
-	BufferCache               m_buffer_cache;
-	TextureCache              m_texture_cache;
-	mutable std::shared_mutex m_mapped_ranges_mutex;
-	RangeSet                  m_mapped_ranges;
-	GuestGpu*                 m_gpu = nullptr;
-	bool                      m_fault_process_pending = false;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -1,12 +1,12 @@
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/profiler.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 
 #include <cstring>
-#include <limits>
 #include <numeric>
 #include <vk_mem_alloc.h>
 
@@ -45,16 +45,11 @@ constexpr size_t WATCHES_RESERVE_CHUNK   = 0x1000;
 		result = value;
 		return true;
 	}
-	const auto remainder = value % alignment;
-	if (remainder == 0) {
-		result = value;
-		return true;
-	}
-	const auto increment = alignment - remainder;
-	if (value > std::numeric_limits<uint64_t>::max() - increment) {
+	const auto aligned = Common::AlignUp(value, alignment);
+	if (aligned < value) {
 		return false;
 	}
-	result = value + increment;
+	result = aligned;
 	return true;
 }
 
@@ -124,12 +119,6 @@ bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
 	return address >= m_cpu_address && size <= Size() && address - m_cpu_address <= Size() - size;
 }
 
-void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
-	EXIT_IF(source == nullptr || m_mapped.empty() || offset > Size() || size > Size() - offset);
-	std::memcpy(m_mapped.data() + offset, source, static_cast<size_t>(size));
-	Flush(offset, size);
-}
-
 void Buffer::Flush(uint64_t offset, uint64_t size) {
 	EXIT_IF(m_mapped.empty() || offset > Size() || size > Size() - offset);
 	if (!IsCoherent() && size != 0) {

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -55,7 +55,6 @@ public:
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
 	void               IncreaseStreamScore(int score) noexcept { stream_score += score; }
 	[[nodiscard]] int  StreamScore() const noexcept { return stream_score; }
-	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
 	void               Invalidate(uint64_t offset, uint64_t size);
 	void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
@@ -266,9 +267,9 @@ void TextureCache::RegisterImage(ImageId id) {
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
 		EXIT("TextureCache: image registration is outside the guest address space\n");
 	}
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(image.info.data.address, image.info.data.size, [this, id](uint64_t page) {
 		m_image_page_table[page].push_back(id);
-	}
+	});
 	image.registered = true;
 	image.lru_id     = m_lru_cache.Insert(id, m_gc_tick);
 	m_total_used_memory += image.AccountedSize();
@@ -284,12 +285,12 @@ void TextureCache::UnregisterImage(ImageId id) {
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
 		EXIT("TextureCache: registered image is outside the guest address space\n");
 	}
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(image.info.data.address, image.info.data.size, [this, id](uint64_t page) {
 		auto* owners = m_image_page_table.Find(page);
 		if (owners == nullptr || !owners->Erase(id)) {
 			EXIT("TextureCache: image missing from page owner index\n");
 		}
-	}
+	});
 	m_lru_cache.Free(image.lru_id);
 	const auto accounted = image.AccountedSize();
 	if (accounted > m_total_used_memory) {
@@ -312,11 +313,7 @@ void TextureCache::DeleteImage(ImageId id) {
 			}
 		});
 		for (const auto association: associations) {
-			auto& associated = m_slot_images[association];
-			if (associated.IsGpuModified()) {
-				associated.ClearGpuModified();
-			}
-			DeleteImage(association);
+			FreeImage(association);
 		}
 	}
 	if (image->IsGpuModified()) {
@@ -443,7 +440,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
 	if (!image.IsTracked() || begin < image.track_addr) {
 		return;
 	}
-	const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE);
 	const auto size    = address - begin;
 	image.track_addr   = address;
 	if (image.track_addr == image.track_addr_end) {
@@ -460,7 +457,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
 	if (!image.IsTracked() || image.track_addr_end < end) {
 		return;
 	}
-	const auto address   = end & ~(TRACKER_PAGE_SIZE - 1);
+	const auto address   = Common::AlignDown(end, TRACKER_PAGE_SIZE);
 	const auto size      = end - address;
 	image.track_addr_end = address;
 	if (image.track_addr == image.track_addr_end) {
@@ -494,10 +491,10 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 	}
 
 	ImageIds result;
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(address, size, [&](uint64_t page) {
 		const auto* owners = m_image_page_table.Find(page);
 		if (owners == nullptr) {
-			continue;
+			return;
 		}
 		owners->ForEach([&](ImageId id) {
 			auto* image = m_slot_images.try_get(id);
@@ -512,7 +509,7 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 				result.push_back(id);
 			}
 		});
-	}
+	});
 	return result;
 }
 
@@ -927,7 +924,7 @@ ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) {
 	return expanded_id;
 }
 
-struct TextureCache::TextureTransferPlan {
+struct TextureCache::TextureTransfer {
 	TextureUploadLayout              layout;
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
@@ -943,13 +940,13 @@ struct TextureCache::TextureTransferPlan {
 	}
 };
 
-struct TextureCache::DownloadPlan {
-	TextureTransferPlan texture;
+struct TextureCache::ImageDownload {
+	TextureTransfer texture;
 	bool                depth_target = false;
 	bool                valid        = false;
 };
 
-TextureCache::TextureTransferPlan
+TextureCache::TextureTransfer
 TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
                                     TransferDirection direction) const {
 	const auto& info             = image.info;
@@ -962,8 +959,8 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 	bool        allow_depth_tile = upload;
 	const char* owner            = "TextureCache readback";
 
-	TextureTransferPlan plan;
-	plan.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
+	TextureTransfer transfer;
+	transfer.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
 	if (render_target) {
 		format = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
 	}
@@ -993,44 +990,44 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 		}
 	}
 
-	plan.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
+	transfer.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
 	                                       info.resources.levels, layers, info.tile_mode,
 	                                       info.data.size, allow_depth_tile, volume, owner);
-	plan.regions = TextureBuildImageCopies(plan.layout);
+	transfer.regions = TextureBuildImageCopies(transfer.layout);
 	if (info.IsDepth()) {
-		for (auto& region: plan.regions) {
+		for (auto& region: transfer.regions) {
 			region.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth;
 		}
 	}
-	if (plan.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
-		if (!TextureBuildGpuTileInfos(info.data.size, plan.regions, plan.layout,
-		                              info.resources.levels, plan.tiles)) {
-			return plan;
+	if (transfer.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
+		if (!TextureBuildGpuTileInfos(info.data.size, transfer.regions, transfer.layout,
+		                              info.resources.levels, transfer.tiles)) {
+			return transfer;
 		}
 	}
-	plan.valid = true;
-	return plan;
+	transfer.valid = true;
+	return transfer;
 }
 
-TextureCache::DownloadPlan TextureCache::BuildDownload(const Image& image) const {
+TextureCache::ImageDownload TextureCache::BuildDownload(const Image& image) const {
 	const auto&  info    = image.info;
 	const auto   binding = UploadBinding(image);
-	DownloadPlan plan {.depth_target = binding == BindingType::DepthTarget};
+	ImageDownload transfer {.depth_target = binding == BindingType::DepthTarget};
 	if (info.samples != 1 || image.backing.samples != 1) {
-		return plan;
+		return transfer;
 	}
-	if (plan.depth_target) {
-		plan.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 &&
+	if (transfer.depth_target) {
+		transfer.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 &&
 		             info.data.size % info.resources.layers == 0 &&
 		             Prospero::NumBytesPerElement(info.guest_format) == info.bytes_per_block;
-		return plan;
+		return transfer;
 	}
 	if (info.metadata.compression != VideoOutCompression::Uncompressed) {
-		return plan;
+		return transfer;
 	}
-	plan.texture = BuildTextureTransfer(image, binding, TransferDirection::Download);
-	plan.valid   = plan.texture.valid;
-	return plan;
+	transfer.texture = BuildTextureTransfer(image, binding, TransferDirection::Download);
+	transfer.valid   = transfer.texture.valid;
+	return transfer;
 }
 
 void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_offset) {
@@ -1044,26 +1041,26 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	};
 
 	if (binding != BindingType::DepthTarget) {
-		auto plan = BuildTextureTransfer(image, binding, TransferDirection::Upload);
-		if (!plan.valid) {
+		auto transfer = BuildTextureTransfer(image, binding, TransferDirection::Upload);
+		if (!transfer.valid) {
 			EXIT("TextureCache: invalid texture upload: binding=%u addr=0x%016" PRIx64
 			     " size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u "
 			     "pitch=%u levels=%u layers=%u samples=%u\n",
 			     static_cast<uint32_t>(binding), info.data.address, info.data.size,
 			     static_cast<uint32_t>(info.guest_format), static_cast<uint32_t>(info.tile_mode),
-			     static_cast<uint32_t>(plan.layout.surface.texture.block.family), info.extent.width,
+			     static_cast<uint32_t>(transfer.layout.surface.texture.block.family), info.extent.width,
 			     info.extent.height, info.extent.depth, info.pitch, info.resources.levels,
 			     info.resources.layers, info.samples);
 		}
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
-		if (!plan.tiles.empty()) {
+		if (!transfer.tiles.empty()) {
 			linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size,
-			                        plan.LinearSize(), plan.tiles);
+			                        transfer.LinearSize(), transfer.tiles);
 		}
-		if (plan.swap_bgra16) {
+		if (transfer.swap_bgra16) {
 			linear = m_tiler.SwapBgra16(linear);
 		}
-		upload(plan.regions, linear);
+		upload(transfer.regions, linear);
 		return;
 	}
 
@@ -1386,9 +1383,6 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
 		case BindingType::Texture: break;
 		case BindingType::Storage:
 			if (!image.info.data.Empty()) {
-				if (!image.registered || image.depth_id) {
-					EXIT("TextureCache: cannot acquire an unavailable storage image\n");
-				}
 				CommitGpuWrite(image);
 			}
 			TrackImageDownload(id, image);
@@ -1675,12 +1669,12 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	             destination_offset, info.data.size, tiles);
 }
 
-void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-                                     uint64_t destination_size, DownloadPlan plan) {
-	if (!plan.valid) {
-		EXIT("TextureCache: invalid image download plan\n");
+void TextureCache::DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset,
+                                     uint64_t destination_size, ImageDownload transfer) {
+	if (!transfer.valid) {
+		EXIT("TextureCache: invalid image download transfer\n");
 	}
-	if (plan.depth_target) {
+	if (transfer.depth_target) {
 		if (destination_size != image.info.data.size) {
 			EXIT("TextureCache: partial depth image download is unsupported\n");
 		}
@@ -1688,7 +1682,7 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t
 		return;
 	}
 
-	auto&      texture   = plan.texture;
+	auto&      texture   = transfer.texture;
 	const auto transform = texture.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16
 	                                           : TileManager::ColorTransform::None;
 	if (texture.tiles.empty()) {
@@ -1749,15 +1743,15 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 	if (copy_size == 0) {
 		return false;
 	}
-	auto plan = m_texture_cache.BuildDownload(image);
-	if (!plan.valid) {
+	auto transfer = m_texture_cache.BuildDownload(image);
+	if (!transfer.valid) {
 		return false;
 	}
-	if (plan.depth_target && copy_size != image.info.data.size) {
+	if (transfer.depth_target && copy_size != image.info.data.size) {
 		return false;
 	}
-	if (!plan.depth_target && levels < image.info.resources.levels) {
-		auto& texture = plan.texture;
+	if (!transfer.depth_target && levels < image.info.resources.levels) {
+		auto& texture = transfer.texture;
 		std::erase_if(texture.regions, [levels](const vk::BufferImageCopy& region) {
 			return region.imageSubresource.mipLevel >= levels;
 		});
@@ -1772,17 +1766,17 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 			}
 		}
 	}
-	m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan));
+	m_texture_cache.DownloadImage(image, buffer, buf_offset, copy_size, std::move(transfer));
 	return true;
 }
 
-bool TextureCache::TryDownloadImage(ImageId id) {
+bool TextureCache::DownloadImageMemory(ImageId id) {
 	auto& image = m_slot_images[id];
 	if (image.depth_id) {
 		return false;
 	}
-	auto plan = BuildDownload(image);
-	if (!plan.valid || !SafeToDownload(image)) {
+	auto transfer = BuildDownload(image);
+	if (!transfer.valid || !SafeToDownload(image)) {
 		return false;
 	}
 	const auto range    = image.info.data;
@@ -1798,7 +1792,7 @@ bool TextureCache::TryDownloadImage(ImageId id) {
 	}
 	download.Flush(offset, range.size);
 
-	DownloadImageData(image, download, offset, range.size, std::move(plan));
+	DownloadImage(image, download, offset, range.size, std::move(transfer));
 	vk::BufferMemoryBarrier barrier {};
 	barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eTransferWrite |
 	                        vk::AccessFlagBits::eShaderWrite;
@@ -1851,8 +1845,8 @@ bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
 }
 
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
-	const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
-	const auto page_end   = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
+	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {
@@ -1970,10 +1964,7 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 		if (owner == nullptr) {
 			continue;
 		}
-		if (owner->IsGpuModified()) {
-			owner->ClearGpuModified();
-		}
-		DeleteImage(id);
+		FreeImage(id);
 	}
 }
 
@@ -2016,12 +2007,11 @@ void TextureCache::RunGarbageCollector() {
 				if (safe && !pressured) {
 					continue;
 				}
-				if (safe && !TryDownloadImage(id)) {
+				if (safe && !DownloadImageMemory(id)) {
 					continue;
 				}
-				owner->ClearGpuModified();
 			}
-			DeleteImage(id);
+			FreeImage(id);
 			if (m_total_used_memory < m_critical_gc_memory && aggressive) {
 				deletions >>= 2;
 				aggressive = false;
@@ -2043,7 +2033,7 @@ void TextureCache::ProcessDownloadImages() {
 	for (const auto id: m_download_images) {
 		const auto owner = m_slot_images.try_get(id);
 		if (owner != nullptr && owner->registered && owner->IsGpuModified()) {
-			(void)TryDownloadImage(id);
+			(void)DownloadImageMemory(id);
 		}
 	}
 	m_download_images.clear();

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -13,6 +13,7 @@
 #include "graphics/host_gpu/renderer/image/tiler.h"
 
 #include <map>
+#include <type_traits>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -76,8 +77,8 @@ public:
 
 private:
 	enum class TransferDirection { Upload, Download };
-	struct TextureTransferPlan;
-	struct DownloadPlan;
+	struct TextureTransfer;
+	struct ImageDownload;
 
 	struct MetaDataInfo {
 		// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
@@ -102,6 +103,23 @@ private:
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
 	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
+	// Callers have validated the nonempty 40-bit range with TryGetPageRange.
+	template <typename Func>
+	static void ForEachPage(uint64_t address, size_t size, Func&& func) {
+		using FuncReturn = typename std::invoke_result<Func, uint64_t>::type;
+		static constexpr bool RETURNS_BOOL = std::is_same_v<FuncReturn, bool>;
+		const uint64_t page_end = (address + size - 1) >> ImagePageTable::kPageBits;
+		for (uint64_t page = address >> ImagePageTable::kPageBits; page <= page_end; ++page) {
+			if constexpr (RETURNS_BOOL) {
+				if (func(page)) {
+					break;
+				}
+			} else {
+				func(page);
+			}
+		}
+	}
+
 	[[nodiscard]] ImageId     InsertImage(const ImageInfo& info);
 	[[nodiscard]] ImageId     GetNullImage(const ImageDesc& desc);
 	void                      RegisterImage(ImageId id);
@@ -133,12 +151,12 @@ private:
 	void                        RefreshImage(ImageId id);
 	void                        PrepareDccClear(ImageId id, const ImageDesc& desc);
 	void                        InitializeImage(ImageId id);
-	[[nodiscard]] TextureTransferPlan
+	[[nodiscard]] TextureTransfer
 	BuildTextureTransfer(const Image& image, BindingType binding, TransferDirection direction) const;
-	[[nodiscard]] DownloadPlan BuildDownload(const Image& image) const;
+	[[nodiscard]] ImageDownload BuildDownload(const Image& image) const;
 	void UploadImage(Image& image, Buffer& source, uint64_t source_offset);
-	void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-	                       uint64_t destination_size, DownloadPlan plan);
+	void DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset,
+	                       uint64_t destination_size, ImageDownload transfer);
 	void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
 	void CommitGpuWrite(Image& image);
 	// Caller holds m_lock. Volume layer ranges select depth slices.
@@ -153,7 +171,7 @@ private:
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
 	void               InvalidateCpuAliases(uint64_t address, uint64_t size);
-	[[nodiscard]] bool TryDownloadImage(ImageId id);
+	[[nodiscard]] bool DownloadImageMemory(ImageId id);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -203,17 +203,14 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
 		     " layer=%u/%u\n",
 		     rt.attrib3.dimension, rt.attrib3.depth, view.base_layer, view.image_layers);
 	}
-	if (tile) {
-		if (volume || texture_tile) {
-			pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
-		} else {
-			pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
-		}
-		if (pitch == 0) {
-			EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element);
-		}
+	// PPSA28068: the linear EULA target and its sampled view must share the padded pitch.
+	if (!tile || volume || texture_tile) {
+		pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
 	} else {
-		pitch = width;
+		pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
+	}
+	if (pitch == 0) {
+		EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element);
 	}
 
 	TileSizeOffset    mip_sizes[16] {};

src/graphics/host_gpu/renderer/context.cpp

@@ -61,11 +61,11 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0,
 }
 
 void CommandBuffer::BeginRendering(const RenderState& state) const {
-	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
-	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);
 	if (m_rendering && m_render_state == state) {
 		return;
 	}
+	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
+	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);
 	EndRendering();
 
 	std::array<vk::RenderingAttachmentInfo, RENDER_COLOR_ATTACHMENTS_MAX> colors {};

src/graphics/host_gpu/renderer/debug.cpp

@@ -4,7 +4,6 @@
 #include "common/common.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
-#include "common/stringUtils.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/renderer/render.h"
@@ -57,7 +56,7 @@ uint32_t render_target_first_bound_slot(const CommandBuffer& buffer) {
 
 bool graphics_debug_dump_enabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
-	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
+	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
 }
 
 void uc_print(const char* func, const HW::UserConfig& uc) {
@@ -89,91 +88,75 @@ void uc_check(const HW::UserConfig& uc) {
 	EXIT_NOT_IMPLEMENTED(user_en.vgpr3 != false);
 }
 
-void sh_print(const char* func, const HW::Shader& /*uc*/) {
-	LOGF("%s\n", func);
-}
+std::string rt_print(const char* func, const HW::RenderTarget& rt) {
+	std::string dst;
+	dst.reserve(4096);
 
-std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt) {
-	std::vector<std::string> dst;
-	dst.reserve(40);
+	dst += fmt::format("{}\n", func);
 
-	dst.push_back(fmt::format("{}\n", func));
+	dst += fmt::format("\t base.addr                       = 0x{:016x}\n", rt.base.addr);
+	dst += fmt::format("\t view.base_array_slice_index     = 0x{:08x}\n",
+	                  rt.view.base_array_slice_index);
+	dst += fmt::format("\t view.last_array_slice_index     = 0x{:08x}\n",
+	                  rt.view.last_array_slice_index);
+	dst += fmt::format("\t view.current_mip_level          = 0x{:08x}\n", rt.view.current_mip_level);
+	dst += fmt::format("\t info.fmask_compression_enable   = {}\n",
+	                  rt.info.fmask_compression_enable ? "true" : "false");
 
-	dst.push_back(fmt::format("\t base.addr                       = 0x{:016x}\n", rt.base.addr));
-	dst.push_back(fmt::format("\t view.base_array_slice_index     = 0x{:08x}\n",
-	                          rt.view.base_array_slice_index));
-	dst.push_back(fmt::format("\t view.last_array_slice_index     = 0x{:08x}\n",
-	                          rt.view.last_array_slice_index));
-	dst.push_back(
-	    fmt::format("\t view.current_mip_level          = 0x{:08x}\n", rt.view.current_mip_level));
-	dst.push_back(fmt::format("\t info.fmask_compression_enable   = {}\n",
-	                          rt.info.fmask_compression_enable ? "true" : "false"));
+	dst += fmt::format("\t info.fmask_data_compression_disable = {}\n",
+	                  rt.info.fmask_data_compression_disable ? "true" : "false");
+	dst += fmt::format("\t info.fmask_one_frag_mode        = {}\n",
+	                  rt.info.fmask_one_frag_mode ? "true" : "false");
 
-	// dst.push_back(fmt::format("\t info.fmask_compression_mode     = 0x{:08x}\n", //
-	// rt.info.fmask_compression_mode));
-	dst.push_back(fmt::format("\t info.fmask_data_compression_disable = {}\n",
-	                          rt.info.fmask_data_compression_disable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.fmask_one_frag_mode        = {}\n",
-	                          rt.info.fmask_one_frag_mode ? "true" : "false"));
-
-	dst.push_back(fmt::format("\t info.cmask_fast_clear_enable    = {}\n",
-	                          rt.info.cmask_fast_clear_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.dcc_compression_enable     = {}\n",
-	                          rt.info.dcc_compression_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.format                     = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.format)));
-	dst.push_back(fmt::format("\t info.channel_type               = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.channel_type)));
-	dst.push_back(fmt::format("\t info.channel_order              = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.channel_order)));
-	dst.push_back(fmt::format("\t info.blend_bypa                 = {}\n",
-	                          rt.info.blend_bypass ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.blend_clamp                = {}\n",
-	                          rt.info.blend_clamp ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.round_mode                 = {}\n",
-	                          rt.info.round_mode ? "true" : "false"));
-	dst.push_back(fmt::format("\t attrib.force_dest_alpha_to_one  = {}\n",
-	                          rt.attrib.force_dest_alpha_to_one ? "true" : "false"));
-	dst.push_back(
-	    fmt::format("\t attrib.num_samples              = 0x{:08x}\n", rt.attrib.num_samples));
-	dst.push_back(
-	    fmt::format("\t attrib.num_fragments            = 0x{:08x}\n", rt.attrib.num_fragments));
-	dst.push_back(fmt::format("\t attrib2.width                   = 0x{:08x}\n", rt.attrib2.width));
-	dst.push_back(
-	    fmt::format("\t attrib2.height                  = 0x{:08x}\n", rt.attrib2.height));
-	dst.push_back(
-	    fmt::format("\t attrib2.num_mip_levels          = 0x{:08x}\n", rt.attrib2.num_mip_levels));
-	dst.push_back(fmt::format("\t attrib3.depth                   = 0x{:08x}\n", rt.attrib3.depth));
-	dst.push_back(fmt::format("\t attrib3.tile_mode               = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.attrib3.tile_mode)));
-	dst.push_back(
-	    fmt::format("\t attrib3.dimension               = 0x{:08x}\n", rt.attrib3.dimension));
-	dst.push_back(fmt::format("\t attrib3.metadata_pipe_aligned   = {}\n",
-	                          rt.attrib3.metadata_pipe_aligned ? "true" : "false"));
-	dst.push_back(fmt::format("\t attrib3.write_vrs_rate_hint_to_cmask = {}\n",
-	                          rt.attrib3.write_vrs_rate_hint_to_cmask ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n",
-	                          rt.dcc.max_uncompressed_block_size));
-	dst.push_back(fmt::format("\t dcc.max_compressed_block_size   = 0x{:08x}\n",
-	                          rt.dcc.max_compressed_block_size));
-	dst.push_back(
-	    fmt::format("\t dcc.color_transform             = 0x{:08x}\n", rt.dcc.color_transform));
-	dst.push_back(fmt::format("\t dcc.overwrite_combiner_disable  = {}\n",
-	                          rt.dcc.overwrite_combiner_disable ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.independent_block_size      = 0x{:02x}\n",
-	                          static_cast<uint8_t>(rt.dcc.independent_block_size)));
-	dst.push_back(fmt::format("\t data_write_on_dcc_clear_to_reg  = {}\n",
-	                          rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.dcc_clear_key_enable        = {}\n",
-	                          rt.dcc.dcc_clear_key_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t cmask.addr                      = 0x{:016x}\n", rt.cmask.addr));
-	dst.push_back(fmt::format("\t fmask.addr                      = 0x{:016x}\n", rt.fmask.addr));
-	dst.push_back(
-	    fmt::format("\t clear_word0.word0               = 0x{:08x}\n", rt.clear_word0.word0));
-	dst.push_back(
-	    fmt::format("\t clear_word1.word1               = 0x{:08x}\n", rt.clear_word1.word1));
-	dst.push_back(
-	    fmt::format("\t dcc_addr.addr                   = 0x{:016x}\n", rt.dcc_addr.addr));
+	dst += fmt::format("\t info.cmask_fast_clear_enable    = {}\n",
+	                  rt.info.cmask_fast_clear_enable ? "true" : "false");
+	dst += fmt::format("\t info.dcc_compression_enable     = {}\n",
+	                  rt.info.dcc_compression_enable ? "true" : "false");
+	dst += fmt::format("\t info.format                     = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.format));
+	dst += fmt::format("\t info.channel_type               = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.channel_type));
+	dst += fmt::format("\t info.channel_order              = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.channel_order));
+	dst += fmt::format("\t info.blend_bypa                 = {}\n",
+	                  rt.info.blend_bypass ? "true" : "false");
+	dst += fmt::format("\t info.blend_clamp                = {}\n",
+	                  rt.info.blend_clamp ? "true" : "false");
+	dst += fmt::format("\t info.round_mode                 = {}\n",
+	                  rt.info.round_mode ? "true" : "false");
+	dst += fmt::format("\t attrib.force_dest_alpha_to_one  = {}\n",
+	                  rt.attrib.force_dest_alpha_to_one ? "true" : "false");
+	dst += fmt::format("\t attrib.num_samples              = 0x{:08x}\n", rt.attrib.num_samples);
+	dst += fmt::format("\t attrib.num_fragments            = 0x{:08x}\n", rt.attrib.num_fragments);
+	dst += fmt::format("\t attrib2.width                   = 0x{:08x}\n", rt.attrib2.width);
+	dst += fmt::format("\t attrib2.height                  = 0x{:08x}\n", rt.attrib2.height);
+	dst += fmt::format("\t attrib2.num_mip_levels          = 0x{:08x}\n", rt.attrib2.num_mip_levels);
+	dst += fmt::format("\t attrib3.depth                   = 0x{:08x}\n", rt.attrib3.depth);
+	dst += fmt::format("\t attrib3.tile_mode               = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.attrib3.tile_mode));
+	dst += fmt::format("\t attrib3.dimension               = 0x{:08x}\n", rt.attrib3.dimension);
+	dst += fmt::format("\t attrib3.metadata_pipe_aligned   = {}\n",
+	                  rt.attrib3.metadata_pipe_aligned ? "true" : "false");
+	dst += fmt::format("\t attrib3.write_vrs_rate_hint_to_cmask = {}\n",
+	                  rt.attrib3.write_vrs_rate_hint_to_cmask ? "true" : "false");
+	dst += fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n",
+	                  rt.dcc.max_uncompressed_block_size);
+	dst += fmt::format("\t dcc.max_compressed_block_size   = 0x{:08x}\n",
+	                  rt.dcc.max_compressed_block_size);
+	dst += fmt::format("\t dcc.color_transform             = 0x{:08x}\n", rt.dcc.color_transform);
+	dst += fmt::format("\t dcc.overwrite_combiner_disable  = {}\n",
+	                  rt.dcc.overwrite_combiner_disable ? "true" : "false");
+	dst += fmt::format("\t dcc.independent_block_size      = 0x{:02x}\n",
+	                  static_cast<uint8_t>(rt.dcc.independent_block_size));
+	dst += fmt::format("\t data_write_on_dcc_clear_to_reg  = {}\n",
+	                  rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false");
+	dst += fmt::format("\t dcc.dcc_clear_key_enable        = {}\n",
+	                  rt.dcc.dcc_clear_key_enable ? "true" : "false");
+	dst += fmt::format("\t cmask.addr                      = 0x{:016x}\n", rt.cmask.addr);
+	dst += fmt::format("\t fmask.addr                      = 0x{:016x}\n", rt.fmask.addr);
+	dst += fmt::format("\t clear_word0.word0               = 0x{:08x}\n", rt.clear_word0.word0);
+	dst += fmt::format("\t clear_word1.word1               = 0x{:08x}\n", rt.clear_word1.word1);
+	dst += fmt::format("\t dcc_addr.addr                   = 0x{:016x}\n", rt.dcc_addr.addr);
 
 	return dst;
 }
@@ -504,16 +487,7 @@ static void BcPrint(const char* func, const HW::BlendControl& c, const HW::Blend
 	     color.red, color.green, color.blue, color.alpha, cc.mode, cc.op);
 }
 
-static void BcCheck(const HW::BlendControl& /*c*/, const HW::BlendColor& color,
-                    const HW::ColorControl& cc) {
-	// EXIT_NOT_IMPLEMENTED(c.color_srcblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.color_comb_fcn != 0);
-	// EXIT_NOT_IMPLEMENTED(c.color_destblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_srcblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_comb_fcn != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_destblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.separate_alpha_blend != false);
-	// EXIT_NOT_IMPLEMENTED(c.enable != false);
+static void BcCheck(const HW::BlendColor& color, const HW::ColorControl& cc) {
 	if (color.red != 0.0f || color.green != 0.0f || color.blue != 0.0f || color.alpha != 0.0f) {
 		static bool logged = false;
 		if (!logged) {
@@ -576,29 +550,6 @@ static void DPrint(const char* func, const HW::DepthControl& c, const HW::Stenci
 	     sm.stencil_opval_bf);
 }
 
-static void DCheck(const HW::DepthControl& c, const HW::StencilControl& s,
-                   const HW::StencilMask& /*sm*/) {
-	// EXIT_NOT_IMPLEMENTED(c.stencil_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.z_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.z_write_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.zfunc != 0);
-	// Back-face stencil state is handled separately when enabled.
-	// EXIT_NOT_IMPLEMENTED(c.stencilfunc != 0);
-	// EXIT_NOT_IMPLEMENTED(c.stencilfunc_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_fail != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_zpass != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_zfail != 0);
-	// Back-face stencil ops may legitimately differ from the front-face ops.
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_testval != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_mask != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_writemask != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_opval != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_testval_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_mask_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_writemask_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_opval_bf != 0);
-}
-
 static void EqaaPrint(const char* func, const HW::EqaaControl& c) {
 	LOGF("%s\n", func);
 
@@ -904,14 +855,10 @@ void hw_check(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
-	const auto& bc      = hw.GetBlendControl(rt_slot);
 	const auto& bclr    = hw.GetBlendColor();
 	const auto& vp      = hw.GetScreenViewport();
 	const auto& c       = hw.GetClipControl();
 	const auto& rc      = hw.GetRenderControl();
-	const auto& d       = hw.GetDepthControl();
-	const auto& s       = hw.GetStencilControl();
-	const auto& sm      = hw.GetStencilMask();
 	const auto& mc      = hw.GetModeControl();
 	const auto& eqaa    = hw.GetEqaaControl();
 	const auto& cc      = hw.GetColorControl();
@@ -937,11 +884,10 @@ void hw_check(const CommandBuffer& buffer) {
 	log_phase("rc");
 	RcCheck(rc);
 	log_phase("depth");
-	DCheck(d, s, sm);
 	log_phase("mode");
 	McCheck(mc);
 	log_phase("blend");
-	BcCheck(bc, bclr, cc);
+	BcCheck(bclr, cc);
 	log_phase("eqaa");
 	EqaaCheck(eqaa, ac);
 	log_phase("aa");
@@ -1000,7 +946,7 @@ void hw_print(const CommandBuffer& buffer) {
 		     hw.GetRenderTargetMask(), hw.GetDepthClearValue(), hw.GetStencilClearValue(),
 		     hw.GetLineWidth(), hw.GetPrimitiveResetIndex());
 
-		LOGF("%s", Common::Concat(rt_print("RenderTraget:", rt), "").c_str());
+		LOGF("%s", rt_print("RenderTraget:", rt).c_str());
 
 		ZPrint("DepthRenderTraget:", z);
 		VpPrint("ScreenViewport:", vp, smc);

src/graphics/host_gpu/renderer/debug.h

@@ -5,7 +5,6 @@
 
 #include <cstdint>
 #include <string>
-#include <vector>
 
 namespace Libs::Graphics {
 
@@ -13,7 +12,6 @@ class CommandBuffer;
 
 namespace HW {
 class Context;
-class Shader;
 class UserConfig;
 struct RenderTarget;
 struct ScanModeControl;
@@ -32,8 +30,7 @@ uint32_t                 render_target_first_bound_slot(const CommandBuffer& buf
 bool                     graphics_debug_dump_enabled();
 void                     uc_print(const char* func, const HW::UserConfig& uc);
 void                     uc_check(const HW::UserConfig& uc);
-void                     sh_print(const char* func, const HW::Shader& uc);
-std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt);
+std::string              rt_print(const char* func, const HW::RenderTarget& rt);
 bool                     RenderIsColorTileModeLinear(Prospero::TileMode tile_mode);
 void                     hw_print(const CommandBuffer& buffer);
 void                     hw_check(const CommandBuffer& buffer);

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -10,6 +10,7 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
@@ -96,51 +97,17 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
 	return vk::Format::eUndefined;
 }
 
-// NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepthInfo& r) {
-	KYTY_PROFILER_FUNCTION();
-	const auto& hw          = buffer.GetRegisters();
-	const auto& z           = hw.GetDepthRenderTarget();
-	const auto& rc          = hw.GetRenderControl();
-	const auto& dc          = hw.GetDepthControl();
-	const auto& sc          = hw.GetStencilControl();
-	const auto& sm          = hw.GetStencilMask();
-	const bool  has_stencil = z.stencil_info.format != Prospero::StencilFormat::kInvalid;
-	const bool depth_active = dc.z_enable || dc.depth_bounds_enable ||
-	                          rc.depth_clear_enable || rc.copy_depth_to_color;
-	const bool stencil_active =
-	    has_stencil && (dc.stencil_enable || rc.stencil_clear_enable || rc.copy_stencil_to_color);
-	if (!depth_active && !stencil_active) {
-		return;
-	}
+static TextureCache::ImageDesc MakeDepthTargetDesc(const CommandBuffer& buffer,
+                                                  const HW::DepthRenderTarget& z,
+                                                  bool write_buffer = false) {
+	const bool has_stencil = z.stencil_info.format != Prospero::StencilFormat::kInvalid;
+	const auto depth_address = write_buffer ? z.z_write_base_addr : z.z_read_base_addr;
+	const auto stencil_address =
+	    write_buffer ? z.stencil_write_base_addr : z.stencil_read_base_addr;
 	if (!z.z_info.HasValidTextureCompatibility() ||
 	    !z.stencil_info.HasValidTextureCompatibility()) {
 		DepthFatal("invalid PS5 depth texture-compatibility encoding");
 	}
-	const bool attachment_unbound =
-	    z.z_info.format == Prospero::DepthFormat::kInvalid &&
-	    z.stencil_info.format == Prospero::StencilFormat::kInvalid && z.z_info.num_samples == 0 &&
-	    z.z_info.texture_compatibility == Prospero::TextureCompatiblePlaneCompression::kDisable &&
-	    !z.z_info.expclear_enabled && !z.z_info.partially_resident && z.z_info.max_mip_level == 0 &&
-	    z.stencil_info.texture_compatibility == Prospero::TextureCompatibleStencil::kDisable &&
-	    !z.stencil_info.expclear_enabled && !z.stencil_info.partially_resident &&
-	    z.depth_view.slice_start == 0 && z.depth_view.slice_max == 0 &&
-	    z.depth_view.current_mip_level == 0 && !z.depth_view.depth_write_disable &&
-	    !z.depth_view.stencil_write_disable && z.z_read_base_addr == 0 && z.z_write_base_addr == 0 &&
-	    z.stencil_read_base_addr == 0 && z.stencil_write_base_addr == 0 &&
-	    z.htile_data_base_addr == 0 &&
-	    // DB_DEPTH_SIZE_XY is independent state and may remain programmed after the attachment
-	    // formats and addresses are unbound. A zero encoding is the valid 1x1 value, so its
-	    // presence alone must not manufacture a depth attachment.
-	    !z.z_info.htile_acceleration && z.shading_rate_encoding == 0 && z.size.x_max == 0 &&
-	    z.size.y_max == 0;
-	if (attachment_unbound) {
-		static std::atomic_bool logged = false;
-		if (!logged.exchange(true, std::memory_order_relaxed)) {
-			LOGF("DepthTarget: ignoring enabled depth/stencil state without a bound attachment\n");
-		}
-		return;
-	}
 	const bool has_htile = z.z_info.htile_acceleration;
 	const bool unsupported_shading_rate_encoding =
 	    z.shading_rate_encoding > 1 || (z.shading_rate_encoding != 0 && !has_htile);
@@ -158,23 +125,16 @@ void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepth
 			DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
 			           z.depth_view.slice_max);
 	}
-	if (rc.copy_depth_to_color || rc.copy_stencil_to_color || rc.copy_centroid ||
-	    rc.copy_sample != 0 || z.z_info.expclear_enabled ||
-	    z.stencil_info.expclear_enabled || z.z_info.partially_resident ||
+	if (z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
+	    z.z_info.partially_resident ||
 	    z.stencil_info.partially_resident || z.z_info.max_mip_level != 0 ||
 	    z.depth_view.current_mip_level != 0 || unsupported_shading_rate_encoding ||
-	    z.z_read_base_addr == 0 ||
-	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
-	    (z.z_read_base_addr & 0xffffu) != 0 ||
-	    dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways)) {
+	    depth_address == 0 || (depth_address & 0xffffu) != 0) {
 		DepthFatal("unsupported depth register state");
 	}
 	if (has_stencil) {
 		if (z.stencil_info.format != Prospero::StencilFormat::k8UInt || !htile_stencil_compat ||
-		    z.stencil_read_base_addr == 0 ||
-		    (!z.depth_view.stencil_write_disable &&
-		     z.stencil_write_base_addr != z.stencil_read_base_addr) ||
-		    (z.stencil_read_base_addr & 0xffffu) != 0) {
+		    stencil_address == 0 || (stencil_address & 0xffffu) != 0) {
 			DepthFatal("unsupported stencil attachment state");
 		}
 	} else if (z.stencil_read_base_addr != 0 || z.stencil_write_base_addr != 0) {
@@ -227,11 +187,92 @@ void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepth
 	const auto depth_backing_size   = depth_size.size * view.image_layers;
 	const auto stencil_backing_size = stencil_size.size * view.image_layers;
 	const auto htile_backing_size   = htile_size.size * view.image_layers;
-	if (!GuestRange {z.z_read_base_addr, depth_backing_size}.Valid() ||
-	    (has_stencil && !GuestRange {z.stencil_read_base_addr, stencil_backing_size}.Valid()) ||
+	if (!GuestRange {depth_address, depth_backing_size}.Valid() ||
+	    (has_stencil && !GuestRange {stencil_address, stencil_backing_size}.Valid()) ||
 	    (has_htile && !GuestRange {z.htile_data_base_addr, htile_backing_size}.Valid())) {
 		DepthFatal("layered depth backing range is invalid");
 	}
+	TextureCache::ImageDesc desc {};
+	desc.type                 = TextureCache::BindingType::DepthTarget;
+	desc.info.data            = {depth_address, depth_backing_size};
+	desc.info.stencil =
+	    has_stencil ? GuestRange {stencil_address, stencil_backing_size} : GuestRange {};
+	desc.info.pixel_format    = format;
+	desc.info.guest_format    = guest_format;
+	desc.info.type            = Prospero::ImageType::kColor2D;
+	desc.info.extent          = {width, height, 1};
+	desc.info.resources       = {1, view.image_layers};
+	desc.info.pitch           = pitch;
+	desc.info.bytes_per_block = bytes;
+	desc.info.samples         = samples;
+	desc.info.tile_mode       = Prospero::TileMode::kDepth;
+	desc.info.mip_layout[0]   = {0, depth_backing_size, pitch, height};
+	desc.info.metadata.range =
+	    has_htile ? GuestRange {z.htile_data_base_addr, htile_backing_size} : GuestRange {};
+	desc.info.metadata.kind   = has_htile ? ImageMetadataKind::Htile : ImageMetadataKind::None;
+	desc.info.metadata.stencil_compressed =
+	    has_stencil && has_htile && !z.stencil_info.htile_stencil_disabled;
+	desc.view_info.format = format;
+	desc.view_info.type =
+	    view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
+	desc.view_info.aspect      = ImageViewOps::DepthAspectMask(format);
+	desc.view_info.base_level  = 0;
+	desc.view_info.level_count = 1;
+	desc.view_info.base_layer  = view.base_layer;
+	desc.view_info.layer_count = view.layer_count;
+	desc.view_info.usage       = vk::ImageUsageFlagBits::eDepthStencilAttachment;
+	return desc;
+}
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepthInfo& r) {
+	KYTY_PROFILER_FUNCTION();
+	const auto& hw          = buffer.GetRegisters();
+	const auto& z           = hw.GetDepthRenderTarget();
+	const auto& rc          = hw.GetRenderControl();
+	const auto& dc          = hw.GetDepthControl();
+	const auto& sc          = hw.GetStencilControl();
+	const auto& sm          = hw.GetStencilMask();
+	const bool  has_stencil = z.stencil_info.format != Prospero::StencilFormat::kInvalid;
+	const bool depth_active = dc.z_enable || dc.depth_bounds_enable ||
+	                          rc.depth_clear_enable || rc.copy_depth_to_color;
+	const bool stencil_active =
+	    has_stencil && (dc.stencil_enable || rc.stencil_clear_enable || rc.copy_stencil_to_color);
+	if (!depth_active && !stencil_active) {
+		return;
+	}
+	const bool attachment_unbound =
+	    z.z_info.format == Prospero::DepthFormat::kInvalid &&
+	    z.stencil_info.format == Prospero::StencilFormat::kInvalid && z.z_info.num_samples == 0 &&
+	    z.z_info.texture_compatibility == Prospero::TextureCompatiblePlaneCompression::kDisable &&
+	    !z.z_info.expclear_enabled && !z.z_info.partially_resident && z.z_info.max_mip_level == 0 &&
+	    z.stencil_info.texture_compatibility == Prospero::TextureCompatibleStencil::kDisable &&
+	    !z.stencil_info.expclear_enabled && !z.stencil_info.partially_resident &&
+	    z.depth_view.slice_start == 0 && z.depth_view.slice_max == 0 &&
+	    z.depth_view.current_mip_level == 0 && !z.depth_view.depth_write_disable &&
+	    !z.depth_view.stencil_write_disable && z.z_read_base_addr == 0 && z.z_write_base_addr == 0 &&
+	    z.stencil_read_base_addr == 0 && z.stencil_write_base_addr == 0 &&
+	    z.htile_data_base_addr == 0 &&
+	    // DB_DEPTH_SIZE_XY is independent state and may remain programmed after the attachment
+	    // formats and addresses are unbound. A zero encoding is the valid 1x1 value, so its
+	    // presence alone must not manufacture a depth attachment.
+	    !z.z_info.htile_acceleration && z.shading_rate_encoding == 0 && z.size.x_max == 0 &&
+	    z.size.y_max == 0;
+	if (attachment_unbound) {
+		static std::atomic_bool logged = false;
+		if (!logged.exchange(true, std::memory_order_relaxed)) {
+			LOGF("DepthTarget: ignoring enabled depth/stencil state without a bound attachment\n");
+		}
+		return;
+	}
+	if (rc.copy_depth_to_color || rc.copy_stencil_to_color || rc.copy_centroid ||
+	    rc.copy_sample != 0 || dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
+	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
+	    (has_stencil && !z.depth_view.stencil_write_disable &&
+	     z.stencil_write_base_addr != z.stencil_read_base_addr)) {
+		DepthFatal("unsupported depth register state");
+	}
+	r.desc = MakeDepthTargetDesc(buffer, z);
 	r.depth_clear_enable      = rc.depth_clear_enable;
 	r.depth_load_clear_enable = r.depth_clear_enable;
 	r.depth_clear_value       = hw.GetDepthClearValue();
@@ -282,41 +323,73 @@ void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepth
 			r.stencil_dynamic_back = r.stencil_dynamic_front;
 		}
 	}
-	TextureCache::ImageDesc desc {};
-	desc.type                 = TextureCache::BindingType::DepthTarget;
-	desc.info.data            = {z.z_read_base_addr, depth_backing_size};
-	desc.info.stencil =
-	    has_stencil ? GuestRange {z.stencil_read_base_addr, stencil_backing_size} : GuestRange {};
-	desc.info.pixel_format    = format;
-	desc.info.guest_format    = guest_format;
-	desc.info.type            = Prospero::ImageType::kColor2D;
-	desc.info.extent          = {width, height, 1};
-	desc.info.resources       = {1, view.image_layers};
-	desc.info.pitch           = pitch;
-	desc.info.bytes_per_block = bytes;
-	desc.info.samples         = samples;
-	desc.info.tile_mode       = Prospero::TileMode::kDepth;
-	desc.info.mip_layout[0]   = {0, depth_backing_size, pitch, height};
-	desc.info.metadata.range =
-	    has_htile ? GuestRange {z.htile_data_base_addr, htile_backing_size} : GuestRange {};
-	desc.info.metadata.kind   = has_htile ? ImageMetadataKind::Htile : ImageMetadataKind::None;
-	desc.info.metadata.stencil_compressed =
-	    has_stencil && has_htile && !z.stencil_info.htile_stencil_disabled;
-	desc.view_info.format = format;
-	desc.view_info.type =
-	    view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
-	desc.view_info.aspect      = ImageViewOps::DepthAspectMask(format);
-	desc.view_info.base_level  = 0;
-	desc.view_info.level_count = 1;
-	desc.view_info.base_layer  = view.base_layer;
-	desc.view_info.layer_count = view.layer_count;
-	desc.view_info.usage       = vk::ImageUsageFlagBits::eDepthStencilAttachment;
-	r.desc                     = std::move(desc);
-	auto& cache                = m_context.GetTextureCache();
-	r.image_id                 = cache.FindImage(r.desc);
+	auto& cache = m_context.GetTextureCache();
+	r.image_id = cache.FindImage(r.desc);
 	BindRenderTarget(r.image_id);
 }
 
+bool RenderExecutor::DepthStencilCopy(CommandBuffer& buffer) {
+	const auto& hw       = buffer.GetRegisters();
+	const auto& z        = hw.GetDepthRenderTarget();
+	const auto& override = hw.GetDepthRenderOverride();
+	if (hw.GetColorControl().mode != 0) {
+		return false;
+	}
+	const bool depth_copy = override.force_z_dirty && override.force_z_valid &&
+	                        z.z_info.format != Prospero::DepthFormat::kInvalid &&
+	                        z.z_read_base_addr != 0 && z.z_write_base_addr != 0 &&
+	                        z.z_read_base_addr != z.z_write_base_addr;
+	const bool stencil_copy = override.force_stencil_dirty && override.force_stencil_valid &&
+	                          z.stencil_info.format != Prospero::StencilFormat::kInvalid &&
+	                          z.stencil_read_base_addr != 0 && z.stencil_write_base_addr != 0 &&
+	                          z.stencil_read_base_addr != z.stencil_write_base_addr;
+	if (!depth_copy && !stencil_copy) {
+		return false;
+	}
+
+	auto  read_desc  = MakeDepthTargetDesc(buffer, z);
+	auto  write_desc = MakeDepthTargetDesc(buffer, z, true);
+	auto& cache      = m_context.GetTextureCache();
+	const auto read_id = cache.FindImage(read_desc);
+	BindRenderTarget(read_id);
+	const auto write_id = cache.FindImage(write_desc);
+	BindRenderTarget(write_id);
+	cache.UpdateImage(read_id);
+	cache.UpdateImage(write_id);
+	cache.MarkGpuWritten(write_id);
+	auto& source      = cache.GetImage(read_id);
+	auto& destination = cache.GetImage(write_id);
+	EXIT_IF(read_id == write_id || source.backing.format != destination.backing.format);
+
+	auto& scheduler = m_context.GetCommandScheduler();
+	scheduler.EndRendering();
+	const auto command = scheduler.Current().Handle();
+	const ImageSubresourceRange range {read_desc.view_info.base_level, 1,
+	                                  read_desc.view_info.base_layer,
+	                                  read_desc.view_info.layer_count};
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
+	               range, command);
+	destination.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
+	                    range, command);
+	std::array<vk::ImageCopy, 2> regions {};
+	uint32_t                   count = 0;
+	for (const auto aspect: {vk::ImageAspectFlagBits::eDepth, vk::ImageAspectFlagBits::eStencil}) {
+		if ((aspect == vk::ImageAspectFlagBits::eDepth && !depth_copy) ||
+		    (aspect == vk::ImageAspectFlagBits::eStencil && !stencil_copy)) {
+			continue;
+		}
+		auto& region = regions[count++];
+		region.srcSubresource = {aspect, range.base_level, range.base_layer,
+		                         range.layer_count};
+		region.dstSubresource = region.srcSubresource;
+		region.extent = write_desc.info.extent;
+	}
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
+	                  destination.backing.image, vk::ImageLayout::eTransferDstOptimal,
+	                  count, regions.data());
+	return true;
+}
+
 vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const {
 	const auto format = desc.view_info.format;
 	if (format == vk::Format::eUndefined) {

src/graphics/host_gpu/renderer/image/image.cpp

@@ -693,11 +693,12 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 }
 
 uint64_t Image::HashGuestEdges() const {
-	constexpr uint64_t                         page_mask = TRACKER_PAGE_SIZE - 1;
 	std::array<uint8_t, TRACKER_PAGE_SIZE * 2> bytes {};
 	const auto                                 range = info.data;
-	const uint64_t head_end     = std::min(range.End(), (range.address + page_mask) & ~page_mask);
-	const uint64_t tail_begin   = std::max(range.address, range.End() & ~page_mask);
+	const uint64_t head_end =
+	    std::min(range.End(), Common::AlignUp(range.address, TRACKER_PAGE_SIZE));
+	const uint64_t tail_begin =
+	    std::max(range.address, Common::AlignDown(range.End(), TRACKER_PAGE_SIZE));
 	const uint64_t head_size    = head_end - range.address;
 	const uint64_t tail_address = tail_begin < head_end ? head_end : tail_begin;
 	const uint64_t tail_size    = range.End() - tail_address;

src/graphics/host_gpu/renderer/image/image.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/slotVector.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -133,7 +134,7 @@ public:
 	}
 	[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
 	[[nodiscard]] uint64_t AccountedSize() const noexcept {
-		return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
+		return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024);
 	}
 	[[nodiscard]] uint64_t HashGuestEdges() const;
 

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -82,9 +82,6 @@ struct ImageInfo {
 	[[nodiscard]] constexpr bool IsVolume() const noexcept {
 		return type == Prospero::ImageType::kColor3D;
 	}
-	[[nodiscard]] constexpr bool IsLayered() const noexcept {
-		return !IsVolume() && resources.layers > 1;
-	}
 	[[nodiscard]] constexpr uint32_t TransferLayers() const noexcept {
 		return IsVolume() ? extent.depth : resources.layers;
 	}
@@ -342,14 +339,6 @@ ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t
 	}
 }
 
-[[nodiscard]] inline constexpr bool
-CanUseVideoOutNativeWithoutUpload(VideoOutCompression compression, bool render_target,
-                                  bool gpu_modified, bool guest_modified) noexcept {
-	return compression != VideoOutCompression::Uncompressed &&
-	       compression != VideoOutCompression::Unsupported && !guest_modified &&
-	       (render_target || gpu_modified);
-}
-
 struct VideoOutPixelFormatInfo {
 	vk::Format             format            = vk::Format::eUndefined;
 	Prospero::BufferFormat guest_format      = Prospero::BufferFormat::kInvalid;
@@ -400,32 +389,6 @@ inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {
 	return false;
 }
 
-[[nodiscard]] inline constexpr bool
-IsSupportedDisplayRenderTargetTileMode(Prospero::TileMode tile_mode) noexcept {
-	return tile_mode == Prospero::TileMode::kRenderTarget;
-}
-
-[[nodiscard]] inline constexpr bool IsSupportedStandard64RenderTarget(const ImageInfo& info) {
-	if (info.tile_mode != Prospero::TileMode::kStandard64KB || info.data.address == 0 ||
-	    (info.data.address & 0xffffu) != 0 || info.extent.width == 0 || info.extent.height == 0 ||
-	    info.bytes_per_block != 4 || info.resources.levels != 1 || info.resources.layers != 1 ||
-	    info.samples != 1) {
-		return false;
-	}
-	const auto expected_pitch =
-	    (static_cast<uint64_t>(info.extent.width) + 127u) & ~uint64_t {127u};
-	const auto padded_height =
-	    (static_cast<uint64_t>(info.extent.height) + 127u) & ~uint64_t {127u};
-	return expected_pitch <= UINT32_MAX && info.pitch == expected_pitch &&
-	       expected_pitch <= UINT64_MAX / padded_height / info.bytes_per_block &&
-	       info.data.size == expected_pitch * padded_height * info.bytes_per_block;
-}
-
-[[nodiscard]] inline constexpr bool IsTiledRenderTarget(const ImageInfo& info) noexcept {
-	return info.tile_mode == Prospero::TileMode::kRenderTarget ||
-	       IsSupportedStandard64RenderTarget(info);
-}
-
 [[nodiscard]] inline bool DecodePackedColorClear(vk::Format format, uint32_t packed,
                                                  vk::ClearColorValue& clear) {
 	vk::ClearColorValue next {};

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -322,6 +322,12 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		normalized.aspect = vk::ImageAspectFlagBits::eStencil;
 	}
 	normalized.usage = is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
+	for (const auto& cached: views) {
+		if (cached.info == normalized) {
+			return cached.view;
+		}
+	}
+
 	const bool format_compatible = normalized.format != vk::Format::eUndefined &&
 	                               ImageViewOps::FormatsCompatible(image.format, normalized.format);
 	const bool slice_view =
@@ -351,12 +357,6 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     image.layers);
 	}
 
-	for (const auto& cached: views) {
-		if (cached.info == normalized) {
-			return cached.view;
-		}
-	}
-
 	vk::ImageViewUsageCreateInfo usage {};
 	usage.usage = image.usage;
 	if (!is_storage) {

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/gpu_format.h"
@@ -8,9 +9,8 @@
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
-#include <bit>
+#include <array>
 #include <cinttypes>
-#include <cstring>
 
 namespace Libs::Graphics {
 namespace {
@@ -28,13 +28,13 @@ constexpr Prospero::ColorComponentMapping kRenderTargetColorMappings[4][4] = {
      Prospero::ColorMappingArgb},
 };
 
-struct RenderTargetHostFormatInfo {
+struct HostFormatInfo {
 	vk::Format                      format = vk::Format::eUndefined;
 	Prospero::ColorComponentMapping host_to_storage;
 };
 
-RenderTargetHostFormatInfo ResolveRenderTargetHostFormat(Prospero::BufferFormat guest_format,
-                                                         Prospero::ChannelOrder order) {
+HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
+                                 Prospero::ChannelOrder order) {
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k8_8_8_8UNorm:
@@ -48,17 +48,77 @@ RenderTargetHostFormatInfo ResolveRenderTargetHostFormat(Prospero::BufferFormat
 			default: break;
 		}
 	}
+	const auto format = VulkanFormat(guest_format);
 	switch (guest_format) {
-		case Prospero::BufferFormat::k5_5_5_1UNorm:
-			return {vk::Format::eA1R5G5B5UnormPack16, Prospero::ColorMappingBgra};
+		case Prospero::BufferFormat::k5_5_5_1UNorm: return {format, Prospero::ColorMappingBgra};
 		case Prospero::BufferFormat::k1_5_5_5UNorm:
-			return {vk::Format::eR5G5B5A1UnormPack16, Prospero::ColorMappingAbgr};
-		case Prospero::BufferFormat::k4_4_4_4UNorm:
-			return {vk::Format::eR4G4B4A4UnormPack16, Prospero::ColorMappingAbgr};
-		default: return {VulkanFormat(guest_format), {}};
+		case Prospero::BufferFormat::k4_4_4_4UNorm: return {format, Prospero::ColorMappingAbgr};
+		default: return {format, {}};
 	}
 }
 
+uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) {
+	return volume_texture ? std::max(depth >> level, 1u) : depth;
+}
+
+size_t GetTextureRegionCount(uint32_t depth, uint32_t levels, bool volume_texture) {
+	size_t count = 0;
+	for (uint32_t level = 0; level < levels; level++) {
+		count += GetTextureLevelDepth(depth, level, volume_texture);
+	}
+	return count;
+}
+
+uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint32_t levels,
+                                uint64_t total_size, uint32_t depth) {
+	uint64_t stride = 0;
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		stride = std::max(stride, mips[mip_level].offset + mips[mip_level].size);
+	}
+
+	if (depth > 1 && total_size != 0 && total_size % depth == 0) {
+		stride = std::max(stride, total_size / depth);
+	}
+
+	return stride;
+}
+
+uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch,
+                                   uint32_t height) {
+	const uint32_t element_columns =
+	    std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u);
+	const uint32_t element_rows =
+	    std::max((height + element.texel_height - 1u) / element.texel_height, 1u);
+	return static_cast<uint64_t>(element_columns) * element_rows * element.bytes;
+}
+
+uint64_t SetLinearUploadLevels(TextureUploadMipLayout*         mips,
+                               const TileTextureElementLayout& element, uint32_t height,
+                               uint32_t levels, uint32_t base_pitch) {
+	uint64_t offset     = 0;
+	auto     mip_pitch  = base_pitch;
+	auto     mip_height = height;
+
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		const auto size = CalcLinearUploadLevelSize(element, mip_pitch, mip_height);
+		mips[mip_level] = {offset, size};
+
+		offset += size;
+		if (mip_pitch > 1) {
+			mip_pitch /= 2;
+		}
+		if (mip_height > 1) {
+			mip_height /= 2;
+		}
+	}
+
+	return offset;
+}
+
+bool FitsBufferRange(uint64_t offset, uint64_t size, uint64_t capacity) {
+	return offset <= capacity && size <= capacity - offset;
+}
+
 } // namespace
 
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
@@ -66,131 +126,61 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layo
                                                     Prospero::ChannelOrder  order) {
 	const auto encoding = Prospero::ResolveRenderTargetFormat(layout, type);
 	if (encoding.IsValid() && encoding.SupportsOrder(order)) {
-		const auto host  = ResolveRenderTargetHostFormat(encoding.buffer_format, order);
-		const auto bytes = Prospero::RenderTargetBytesPerElement(encoding.buffer_format);
-		if (host.format != vk::Format::eUndefined && bytes != 0) {
+		const auto host_format = ResolveHostFormat(encoding.buffer_format, order);
+		const auto bytes       = Prospero::RenderTargetBytesPerElement(encoding.buffer_format);
+		if (host_format.format != vk::Format::eUndefined && bytes != 0) {
 			const auto order_mapping =
 			    kRenderTargetColorMappings[static_cast<size_t>(order)][encoding.components - 1u];
-			return {host.format, bytes, host.host_to_storage.Then(order_mapping)};
+			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping)};
 		}
 	}
 	EXIT("unsupported render-target format combination: layout=%u type=%u order=%u\n",
 	     static_cast<uint32_t>(layout), static_cast<uint32_t>(type), static_cast<uint32_t>(order));
 }
 
-namespace {
-
-static uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) {
-	return volume_texture ? std::max(depth >> level, 1u) : depth;
-}
-
-static size_t GetTextureRegionCount(uint32_t depth, uint64_t levels, bool volume_texture) {
-	size_t count = 0;
-	for (uint32_t level = 0; level < levels; level++) {
-		count += GetTextureLevelDepth(depth, level, volume_texture);
+vk::ComponentMapping TextureGetComponentMapping(uint32_t                        swizzle,
+                                                Prospero::ColorComponentMapping host_to_storage) {
+	constexpr std::array host_components {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eG,
+	                                      vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eA};
+	std::array<vk::ComponentSwizzle, 4> storage_components {};
+	for (uint32_t host_component = 0; host_component < host_components.size(); ++host_component) {
+		storage_components[host_to_storage.Map(host_component)] = host_components[host_component];
 	}
-	return count;
-}
-
-vk::ComponentSwizzle TextureGetComponentSwizzle(uint8_t s) {
-	switch (static_cast<Prospero::CompSwizzle>(s)) {
-		case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero;
-		case Prospero::CompSwizzle::kOne: return vk::ComponentSwizzle::eOne;
-		case Prospero::CompSwizzle::kRed: return vk::ComponentSwizzle::eR;
-		case Prospero::CompSwizzle::kGreen: return vk::ComponentSwizzle::eG;
-		case Prospero::CompSwizzle::kBlue: return vk::ComponentSwizzle::eB;
-		case Prospero::CompSwizzle::kAlpha: return vk::ComponentSwizzle::eA;
-		default: EXIT("unknown swizzle: %d\n", static_cast<int>(s));
-	}
-	return vk::ComponentSwizzle::eIdentity;
-}
-
-} // namespace
-
-vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) {
-	vk::ComponentMapping components {};
-	components.r = TextureGetComponentSwizzle(GetDstSel(swizzle, 0));
-	components.g = TextureGetComponentSwizzle(GetDstSel(swizzle, 1));
-	components.b = TextureGetComponentSwizzle(GetDstSel(swizzle, 2));
-	components.a = TextureGetComponentSwizzle(GetDstSel(swizzle, 3));
-	return components;
+	const auto resolve = [&](uint32_t component) {
+		const auto selector = static_cast<Prospero::CompSwizzle>(GetDstSel(swizzle, component));
+		switch (selector) {
+			case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero;
+			case Prospero::CompSwizzle::kOne: return vk::ComponentSwizzle::eOne;
+			case Prospero::CompSwizzle::kRed: return storage_components[0];
+			case Prospero::CompSwizzle::kGreen: return storage_components[1];
+			case Prospero::CompSwizzle::kBlue: return storage_components[2];
+			case Prospero::CompSwizzle::kAlpha: return storage_components[3];
+			default: EXIT("unknown swizzle: %u\n", static_cast<uint32_t>(selector));
+		}
+	};
+	return {resolve(0), resolve(1), resolve(2), resolve(3)};
 }
 
 SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
-	const auto backing_format    = Prospero::RemapTextureFormat(format);
-	const auto vk_format         = VulkanFormat(backing_format);
+	const auto backing_format = Prospero::RemapTextureFormat(format);
+	const auto host_format = ResolveHostFormat(backing_format, Prospero::ChannelOrder::kStandard);
 	const auto conversion_format =
 	    backing_format != format ? format : Prospero::BufferFormat::kInvalid;
-	if (vk_format != vk::Format::eUndefined) {
-		return SurfaceFormatInfo(vk_format, conversion_format);
+	if (host_format.format != vk::Format::eUndefined) {
+		return {host_format.format, conversion_format, host_format.host_to_storage};
 	}
 	EXIT("unknown format: fmt = %u\n", static_cast<uint32_t>(format));
-	return SurfaceFormatInfo(vk::Format::eUndefined, Prospero::BufferFormat::kInvalid);
 }
 
-namespace {
-
-uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint64_t levels,
-                                uint64_t total_size, uint32_t depth) {
-	uint64_t stride = 0;
-	for (uint32_t i = 0; i < levels; i++) {
-		stride = std::max(stride, mips[i].offset + mips[i].size);
-	}
-
-	if (depth > 1 && total_size != 0 && total_size % depth == 0) {
-		const auto guest_stride = total_size / depth;
-		if (guest_stride >= stride) {
-			stride = guest_stride;
-		}
-	}
-
-	return stride;
-}
-
-uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch,
-                                   uint32_t height) {
-	const uint32_t elements_w =
-	    std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u);
-	const uint32_t elements_h =
-	    std::max((height + element.texel_height - 1u) / element.texel_height, 1u);
-	return static_cast<uint64_t>(elements_w) * elements_h * element.bytes;
-}
-
-uint64_t SetLinearUploadLevels(TextureUploadMipLayout*         mips,
-                               const TileTextureElementLayout& element, uint64_t height,
-                               uint64_t levels, uint32_t base_pitch) {
-	uint64_t offset = 0;
-	auto     pitch  = base_pitch;
-	auto     h      = static_cast<uint32_t>(height);
-
-	for (uint32_t i = 0; i < levels; i++) {
-		const auto size = CalcLinearUploadLevelSize(element, pitch, h);
-		mips[i].size    = size;
-		mips[i].offset  = offset;
-
-		offset += size;
-		if (pitch > 1) {
-			pitch /= 2;
-		}
-		if (h > 1) {
-			h /= 2;
-		}
-	}
-
-	return offset;
-}
-
-} // namespace
-
 TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                             uint32_t height, uint32_t levels, uint32_t depth,
-                                            Prospero::TileMode tile, uint64_t upload_size,
+                                            Prospero::TileMode tile_mode, uint64_t upload_size,
                                             bool allow_depth_tile, bool volume_texture,
                                             const char* owner) {
 	TextureUploadLayout layout {};
 	layout.surface.description = {
 	    format,
-	    tile,
+	    tile_mode,
 	    volume_texture ? TileSurfaceDimension::Dim3D : TileSurfaceDimension::Dim2D,
 	    width,
 	    height,
@@ -199,7 +189,6 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 	    volume_texture ? 1u : depth,
 	};
 	const auto&              description = layout.surface.description;
-	const auto               tile_mode   = description.tile_mode;
 	TileTextureElementLayout element {};
 
 	if (format == Prospero::BufferFormat::kInvalid) {
@@ -209,34 +198,19 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 		     levels);
 	}
 
-	switch (tile_mode) {
-		case Prospero::TileMode::kLinear:
-			if (!TileGetTextureElementLayout(format, element)) {
-				EXIT("%s: unsupported linear texture format: fmt=%u\n", owner,
-				     static_cast<uint32_t>(format));
-			}
-			break;
-		case Prospero::TileMode::kDepth:
-			if (!allow_depth_tile || !TileGetTiledTextureLayout(description, layout.surface)) {
-				EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
-				     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
-				     owner, static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(description.tile_mode), upload_size, width, height,
-				     levels);
-			}
-			break;
-		default:
-			if (!TileGetTiledTextureLayout(description, layout.surface)) {
-				EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
-				     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
-				     owner, static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(description.tile_mode), upload_size, width, height,
-				     levels);
-			}
-			break;
-	}
-
-	if (tile_mode != Prospero::TileMode::kLinear) {
+	if (tile_mode == Prospero::TileMode::kLinear) {
+		if (!TileGetTextureElementLayout(format, element)) {
+			EXIT("%s: unsupported linear texture format: fmt=%u\n", owner,
+			     static_cast<uint32_t>(format));
+		}
+	} else {
+		if ((tile_mode == Prospero::TileMode::kDepth && !allow_depth_tile) ||
+		    !TileGetTiledTextureLayout(description, layout.surface)) {
+			EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
+			     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
+			     owner, static_cast<uint32_t>(format), static_cast<uint32_t>(tile_mode),
+			     upload_size, width, height, levels);
+		}
 		element = {layout.surface.texture.block.bytes_per_element,
 		           layout.surface.texture.texel_width, layout.surface.texture.texel_height};
 	}
@@ -269,40 +243,33 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout) {
 	const auto& description    = layout.surface.description;
 	const bool  volume_texture = description.dimension == TileSurfaceDimension::Dim3D;
+	const bool  linear_texture = description.tile_mode == Prospero::TileMode::kLinear;
 	const auto  depth          = volume_texture ? description.depth : description.layers;
 	uint32_t    mip_width      = description.width;
 	uint32_t    mip_height     = description.height;
-	uint32_t    mip_pitch = volume_texture && description.tile_mode != Prospero::TileMode::kLinear
-	                            ? description.width
-	                            : layout.pitch;
+	uint32_t    mip_pitch = volume_texture && !linear_texture ? description.width : layout.pitch;
 
 	std::vector<vk::BufferImageCopy> regions;
 	regions.reserve(GetTextureRegionCount(depth, description.levels, volume_texture));
-	for (uint32_t i = 0; i < description.levels; i++) {
-		EXIT_NOT_IMPLEMENTED(layout.mips[i].size == 0);
-
-		const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture);
-
-		for (uint32_t z = 0; z < mip_depth; z++) {
-			const auto          slice_offset = z * layout.slice_stride;
-			vk::BufferImageCopy region {};
-			region.bufferOffset     = layout.mips[i].offset + slice_offset;
-			region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i, volume_texture ? 0u : z,
-			                           1};
-			region.imageOffset.z    = volume_texture ? static_cast<int>(z) : 0;
-			region.imageExtent      = {mip_width, mip_height, 1};
-			const bool linear       = description.tile_mode == Prospero::TileMode::kLinear;
-			if (linear) {
-				region.bufferRowLength   = layout.mips[i].row_length;
-				region.bufferImageHeight = layout.mips[i].image_height;
-			} else {
-				const auto texel_width = layout.surface.texture.texel_width;
-				const auto align       = [](uint32_t value, uint32_t block) {
-					return ((value + block - 1u) / block) * block;
-				};
-				const auto pitch       = align(mip_pitch, texel_width);
-				region.bufferRowLength = pitch > align(mip_width, texel_width) ? pitch : 0;
-			}
+	for (uint32_t mip_level = 0; mip_level < description.levels; ++mip_level) {
+		EXIT_NOT_IMPLEMENTED(layout.mips[mip_level].size == 0);
+		const auto          mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture);
+		vk::BufferImageCopy region {};
+		region.imageSubresource = {vk::ImageAspectFlagBits::eColor, mip_level, 0, 1};
+		region.imageExtent      = {mip_width, mip_height, 1};
+		if (linear_texture) {
+			region.bufferRowLength   = layout.mips[mip_level].row_length;
+			region.bufferImageHeight = layout.mips[mip_level].image_height;
+		} else {
+			const auto texel_width   = layout.surface.texture.texel_width;
+			const auto aligned_pitch = Common::AlignUp(mip_pitch, texel_width);
+			region.bufferRowLength =
+			    aligned_pitch > Common::AlignUp(mip_width, texel_width) ? aligned_pitch : 0;
+		}
+		for (uint32_t slice_index = 0; slice_index < mip_depth; ++slice_index) {
+			region.bufferOffset = layout.mips[mip_level].offset + slice_index * layout.slice_stride;
+			region.imageSubresource.baseArrayLayer = volume_texture ? 0u : slice_index;
+			region.imageOffset.z = volume_texture ? static_cast<int>(slice_index) : 0;
 			regions.push_back(region);
 		}
 
@@ -320,17 +287,9 @@ std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayo
 	return regions;
 }
 
-static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity, uint64_t& size) {
-	if (offset > capacity || length > capacity - offset) {
-		return false;
-	}
-	size = length;
-	return true;
-}
-
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,
-                              std::vector<GpuTileInfo>& out_infos) {
+                              std::vector<GpuTileInfo>& out_tile_infos) {
 	const auto& description    = layout.surface.description;
 	const bool  volume_texture = description.dimension == TileSurfaceDimension::Dim3D;
 	const auto  depth          = volume_texture ? description.depth : description.layers;
@@ -348,111 +307,69 @@ bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferI
 		return false;
 	}
 
-	std::vector<GpuTileInfo> infos;
-	infos.reserve(regions.size());
-	if (volume_texture) {
-		size_t region_base = 0;
-		for (uint32_t level = 0; level < levels; ++level) {
-			const uint32_t mip_depth     = GetTextureLevelDepth(depth, level, true);
-			const bool     tail          = level >= surface.first_tail_level;
-			const uint64_t linear_stride = layout.slice_stride;
-			for (uint32_t z = 0; z < mip_depth; z += block.block_depth) {
-				const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z);
-				const auto&    region     = regions[region_base + z];
-				const auto     pitch =
-				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
-				const auto  logical_height = region.bufferImageHeight != 0
-				                                 ? region.bufferImageHeight
-				                                 : region.imageExtent.height;
-				GpuTileInfo info {};
-				info.family            = block.family;
-				info.bytes_per_element = block.bytes_per_element;
-				info.linear_offset     = region.bufferOffset;
-				info.tiled_offset =
-				    static_cast<uint64_t>(z / block.block_depth) * surface.block_slice_size +
-				    surface.mips[level].offset;
-				const uint64_t linear_span =
-				    static_cast<uint64_t>(copy_depth - 1u) * linear_stride +
-				    layout.mips[level].size;
-				if (!SetGpuTileSize(info.linear_offset, linear_span, UINT64_MAX,
-				                    info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, surface.mips[level].size, tiled_size,
-				                    info.tiled_size)) {
-					return false;
-				}
-				info.linear_slice_stride = linear_stride;
-				info.width  = std::max((region.imageExtent.width + texture.texel_width - 1u) /
-				                           texture.texel_width,
-				                       1u);
-				info.height = std::max(
-				    (logical_height + texture.texel_height - 1u) / texture.texel_height, 1u);
-				info.depth = copy_depth;
-				info.surface_z =
+	std::vector<GpuTileInfo> tile_infos;
+	tile_infos.reserve(regions.size());
+	const auto slices_per_copy = volume_texture ? block.block_depth : 1u;
+	size_t     region_base     = 0;
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		const auto& mip       = surface.mips[mip_level];
+		const auto  mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture);
+		for (uint32_t slice_index = 0; slice_index < mip_depth; slice_index += slices_per_copy) {
+			const auto& region = regions[region_base + slice_index];
+			GpuTileInfo tile_info {};
+			tile_info.family            = block.family;
+			tile_info.bytes_per_element = block.bytes_per_element;
+			tile_info.linear_offset     = region.bufferOffset;
+			tile_info.linear_size       = layout.mips[mip_level].size;
+			tile_info.tiled_size        = mip.size;
+			if (volume_texture) {
+				tile_info.depth               = std::min(slices_per_copy, mip_depth - slice_index);
+				tile_info.linear_slice_stride = layout.slice_stride;
+				tile_info.linear_size += (tile_info.depth - 1u) * layout.slice_stride;
+				tile_info.tiled_offset =
+				    mip.offset +
+				    static_cast<uint64_t>(slice_index / slices_per_copy) * surface.block_slice_size;
+				tile_info.surface_z =
 				    block.block_depth == 1 ? static_cast<uint32_t>(region.imageOffset.z) : 0;
-				info.pitch = std::max((pitch + texture.texel_width - 1u) / texture.texel_width, 1u);
-				info.tail_x       = tail ? surface.mips[level].tail_x : 0;
-				info.tail_y       = tail ? surface.mips[level].tail_y : 0;
-				info.tail         = tail;
-				info.tiled_width  = surface.mips[level].padded_width;
-				info.tiled_height = surface.mips[level].padded_height;
-				infos.push_back(info);
-			}
-			region_base += mip_depth;
-		}
-	} else {
-		size_t region_index = 0;
-		for (uint32_t level = 0; level < levels; level++) {
-			const auto& level_size  = layout.mips[level];
-			const auto& mip         = surface.mips[level];
-			const bool  tail        = level >= surface.first_tail_level;
-			const auto  level_depth = GetTextureLevelDepth(depth, level, false);
-			for (uint32_t z = 0; z < level_depth; z++) {
-				const auto& region = regions[region_index++];
-				const auto  pitch =
-				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
-				const auto  logical_height = region.bufferImageHeight != 0
-				                                 ? region.bufferImageHeight
-				                                 : region.imageExtent.height;
-				GpuTileInfo info {};
-				info.family              = block.family;
-				info.bytes_per_element   = block.bytes_per_element;
-				info.linear_offset       = region.bufferOffset;
+			} else {
 				const auto source_stride = layout.source_slice_stride != 0
 				                               ? layout.source_slice_stride
 				                               : surface.block_slice_size;
-				if (z > (UINT64_MAX - mip.offset) / source_stride) {
+				if (slice_index > (UINT64_MAX - mip.offset) / source_stride) {
 					return false;
 				}
-				info.tiled_offset = mip.offset + static_cast<uint64_t>(z) * source_stride;
-				if (!SetGpuTileSize(info.linear_offset, level_size.size, UINT64_MAX,
-				                    info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, mip.size, tiled_size, info.tiled_size)) {
-					return false;
-				}
-				info.width  = std::max((region.imageExtent.width + texture.texel_width - 1u) /
-				                           texture.texel_width,
-				                       1u);
-				info.height = std::max(
-				    (logical_height + texture.texel_height - 1u) / texture.texel_height, 1u);
-				info.surface_z = block.family == TileBlockFamily::RenderTarget64KB ||
-				                         block.family == TileBlockFamily::Depth64KB
-				                     ? region.imageSubresource.baseArrayLayer
-				                     : 0;
-				info.pitch = std::max((pitch + texture.texel_width - 1u) / texture.texel_width, 1u);
-				info.tail  = tail;
-				info.tail_x       = tail ? mip.tail_x : 0;
-				info.tail_y       = tail ? mip.tail_y : 0;
-				info.tiled_width  = mip.padded_width;
-				info.tiled_height = mip.padded_height;
-				infos.push_back(info);
+				tile_info.tiled_offset =
+				    mip.offset + static_cast<uint64_t>(slice_index) * source_stride;
+				tile_info.surface_z = block.family == TileBlockFamily::RenderTarget64KB ||
+				                              block.family == TileBlockFamily::Depth64KB
+				                          ? region.imageSubresource.baseArrayLayer
+				                          : 0;
 			}
+			if (!FitsBufferRange(tile_info.linear_offset, tile_info.linear_size, UINT64_MAX) ||
+			    !FitsBufferRange(tile_info.tiled_offset, tile_info.tiled_size, tiled_size)) {
+				return false;
+			}
+			const auto row_length =
+			    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
+			const auto image_height = region.bufferImageHeight != 0 ? region.bufferImageHeight
+			                                                        : region.imageExtent.height;
+			tile_info.width         = std::max(
+			    (region.imageExtent.width + texture.texel_width - 1u) / texture.texel_width, 1u);
+			tile_info.height =
+			    std::max((image_height + texture.texel_height - 1u) / texture.texel_height, 1u);
+			tile_info.pitch =
+			    std::max((row_length + texture.texel_width - 1u) / texture.texel_width, 1u);
+			tile_info.tail         = mip_level >= surface.first_tail_level;
+			tile_info.tail_x       = tile_info.tail ? mip.tail_x : 0;
+			tile_info.tail_y       = tile_info.tail ? mip.tail_y : 0;
+			tile_info.tiled_width  = mip.padded_width;
+			tile_info.tiled_height = mip.padded_height;
+			tile_infos.push_back(tile_info);
 		}
+		region_base += mip_depth;
 	}
 
-	if (infos.empty()) {
-		return false;
-	}
-	out_infos = std::move(infos);
+	out_tile_infos = std::move(tile_infos);
 	return true;
 }
 

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -1,9 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 
-#include "common/abi.h"
-#include "common/common.h"
-#include "graphics/guest_gpu/gpu_format.h"
+#include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
@@ -20,12 +18,9 @@ struct RenderTargetFormatInfo {
 };
 
 struct SurfaceFormatInfo {
-	explicit constexpr SurfaceFormatInfo(vk::Format format,
-	                                     Prospero::BufferFormat conversion_format)
-	    : vk_format(format), conversion_format(conversion_format) {}
-
-	vk::Format             vk_format;
-	Prospero::BufferFormat conversion_format;
+	vk::Format                      vk_format;
+	Prospero::BufferFormat          conversion_format;
+	Prospero::ColorComponentMapping host_to_storage;
 };
 
 struct TextureUploadMipLayout {
@@ -43,20 +38,21 @@ struct TextureUploadLayout {
 	TextureUploadMipLayout mips[16] = {};
 };
 
-vk::ComponentMapping   TextureGetComponentMapping(uint32_t swizzle);
+vk::ComponentMapping   TextureGetComponentMapping(uint32_t                        swizzle,
+                                                  Prospero::ColorComponentMapping host_to_storage);
 SurfaceFormatInfo      TextureGetSurfaceFormatInfo(Prospero::BufferFormat format);
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
                                                     Prospero::ChannelType   type,
                                                     Prospero::ChannelOrder  order);
 TextureUploadLayout    TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                                uint32_t height, uint32_t levels, uint32_t depth,
-                                               Prospero::TileMode tile, uint64_t upload_size,
+                                               Prospero::TileMode tile_mode, uint64_t upload_size,
                                                bool allow_depth_tile, bool volume_texture,
                                                const char* owner);
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout);
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,
-                              std::vector<GpuTileInfo>& infos);
+                              std::vector<GpuTileInfo>& out_tile_infos);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/image/tiler.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_depth_spv.h"
@@ -221,7 +222,7 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 
 	const uint64_t uniform_alignment =
 	    std::max<uint64_t>(limits.minUniformBufferOffsetAlignment, 1);
-	const uint64_t stride = (sizeof(Push) + uniform_alignment - 1) & ~(uniform_alignment - 1);
+	const uint64_t stride = Common::AlignUp<uint64_t>(sizeof(Push), uniform_alignment);
 	EXIT_NOT_IMPLEMENTED(dispatches.size() > UINT64_MAX / stride);
 	const uint64_t bytes  = dispatches.size() * stride;
 	auto [mapped, offset] = m_stream_buffer.Map(bytes, uniform_alignment);
@@ -287,12 +288,12 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset,
 	const auto&    limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	const uint64_t descriptor_alignment =
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
-	const uint64_t source_descriptor_offset = source_offset & ~(descriptor_alignment - 1);
-	const uint64_t target_descriptor_offset = target_offset & ~(descriptor_alignment - 1);
+	const uint64_t source_descriptor_offset = Common::AlignDown(source_offset, descriptor_alignment);
+	const uint64_t target_descriptor_offset = Common::AlignDown(target_offset, descriptor_alignment);
 	const uint64_t source_base              = source_offset - source_descriptor_offset;
 	const uint64_t target_base              = target_offset - target_descriptor_offset;
-	const uint64_t source_range             = (source_base + source_capacity + 3u) & ~uint64_t {3};
-	const uint64_t target_range             = (target_base + target_capacity + 3u) & ~uint64_t {3};
+	const uint64_t source_range             = Common::AlignUp(source_base + source_capacity, 4);
+	const uint64_t target_range             = Common::AlignUp(target_base + target_capacity, 4);
 	EXIT_NOT_IMPLEMENTED(source_range > limits.maxStorageBufferRange ||
 	                     target_range > limits.maxStorageBufferRange || target_offset % 4 != 0 ||
 	                     target_capacity % 4 != 0);
@@ -372,7 +373,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
 	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
-	auto scratch = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3});
+	auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4));
 	DeferDestroy(scratch);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
@@ -406,7 +407,7 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	// Reserve all stream parameters before creating a scheduler-lived scratch dependency:
 	// StreamBuffer::Map is allowed to submit the current tick when it wraps.
 	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
-	auto linear = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3});
+	auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4));
 	DeferDestroy(linear);
 	image.Download(regions, linear.buffer, 0, linear.size);
 	Result source {linear.buffer, 0, linear.size};
@@ -418,7 +419,7 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 }
 
 TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
-	auto scratch = AllocateScratch((size + 3u) & ~uint64_t {3});
+	auto scratch = AllocateScratch(Common::AlignUp(size, 4));
 	DeferDestroy(scratch);
 	return {scratch.buffer, 0, scratch.size};
 }
@@ -426,11 +427,11 @@ TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
 TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const {
 	const auto& limits            = m_graphics.GetPhysicalDeviceProperties().limits;
 	const auto  alignment         = std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
-	const auto  descriptor_offset = buffer.offset - buffer.offset % alignment;
+	const auto  descriptor_offset = Common::AlignDown(buffer.offset, alignment);
 	const auto  base              = buffer.offset - descriptor_offset;
 	EXIT_IF(buffer.buffer == nullptr || size == 0 || buffer.size < size || base > UINT32_MAX ||
 	        size > UINT64_MAX - base || base + size > UINT64_MAX - 3);
-	const auto range = (base + size + 3) & ~uint64_t {3};
+	const auto range = Common::AlignUp(base + size, 4);
 	EXIT_IF(range > limits.maxStorageBufferRange || range > UINT32_MAX);
 	return {{buffer.buffer, descriptor_offset, range}, static_cast<uint32_t>(base)};
 }
@@ -512,8 +513,8 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	const auto target_required = required(layout.height, layout.layers, layout.target_row_stride,
 	                                      layout.target_slice_stride, target_active);
 	EXIT_IF(source_required > UINT64_MAX - 3 || target_required > UINT64_MAX - 3);
-	const auto source_barrier_size = (source_required + 3) & ~uint64_t {3};
-	const auto target_barrier_size = (target_required + 3) & ~uint64_t {3};
+	const auto source_barrier_size = Common::AlignUp(source_required, 4);
+	const auto target_barrier_size = Common::AlignUp(target_required, 4);
 	EXIT_IF(source.size < source_barrier_size || target.size < target_barrier_size);
 
 	m_scheduler.EndRendering();

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp

@@ -20,7 +20,7 @@ constexpr std::array DescriptorPoolSizes = {
 
 DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore)
     : m_graphics(graphics), m_master_semaphore(master_semaphore) {
-	CreatePool();
+	CreateDescriptorPool();
 }
 
 DescriptorHeap::~DescriptorHeap() {
@@ -49,7 +49,7 @@ vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) {
 		m_pending_pools.pop_front();
 		EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess);
 	} else {
-		CreatePool();
+		CreateDescriptorPool();
 	}
 
 	m_sets.clear();
@@ -82,7 +82,7 @@ bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) {
 	}
 }
 
-void DescriptorHeap::CreatePool() {
+void DescriptorHeap::CreateDescriptorPool() {
 	vk::DescriptorPoolCreateInfo create {};
 	create.maxSets       = DescriptorHeapCount;
 	create.poolSizeCount = static_cast<uint32_t>(DescriptorPoolSizes.size());

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h

@@ -31,7 +31,7 @@ private:
 	};
 
 	[[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch);
-	void               CreatePool();
+	void               CreateDescriptorPool();
 
 	GraphicContext&                                     m_graphics;
 	MasterSemaphore&                                    m_master_semaphore;

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/file.h"
@@ -145,7 +146,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	}
 	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
 	                                                              resource.formatted, id);
-	const auto aligned_offset = offset - offset % alignment;
+	const auto aligned_offset = Common::AlignDown(offset, alignment);
 	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
 	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
@@ -169,34 +170,6 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	return result;
 }
 
-static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) {
-	bool supported_dimension = false;
-	switch (resource.dimension) {
-		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim1DArray:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
-			supported_dimension = true;
-			break;
-		default: break;
-	}
-	return resource.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled &&
-	       resource.numeric_class != Prospero::TextureNumericClass::Unsupported &&
-	       supported_dimension && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
-	       resource.read && !resource.written && !resource.atomic && !resource.depth_compare;
-}
-
-bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
-                                    const ShaderTextureResource& descriptor, const Image& image) {
-	return image.usage.video_out && image.info.resources.layers == 1 &&
-	       IsSupportedSampledColorResource(resource) &&
-	       resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	       descriptor.Type() == Prospero::ImageType::kColor2D && descriptor.Depth() == 0 &&
-	       descriptor.BaseArray5() == 0;
-}
-
 bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bool r128) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
@@ -483,17 +456,18 @@ static void PopulateTextureMipLayout(ImageInfo& info) {
 
 static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource,
                                      const ShaderTextureResource& descriptor, vk::Format format,
-                                     bool shader_conversion, bool storage, uint32_t view_levels,
-                                     uint32_t image_layers) {
+                                     const SurfaceFormatInfo& surface_format, bool storage,
+                                     uint32_t view_levels, uint32_t image_layers) {
 	ImageViewInfo view {};
 	view.format      = format;
 	view.aspect      = vk::ImageAspectFlagBits::eColor;
 	view.base_level  = descriptor.BaseLevel();
 	view.level_count = view_levels;
-	view.usage   = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
-	view.mapping = storage || shader_conversion
-	                   ? vk::ComponentMapping {}
-	                   : TextureGetComponentMapping(descriptor.DstSelXYZW());
+	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
+	view.mapping =
+	    storage || surface_format.conversion_format != Prospero::BufferFormat::kInvalid
+	        ? vk::ComponentMapping {}
+	        : TextureGetComponentMapping(descriptor.DstSelXYZW(), surface_format.host_to_storage);
 	switch (resource.dimension) {
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
 			view.type       = vk::ImageViewType::e1D;
@@ -672,7 +646,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	} else {
 		PopulateTextureMipLayout(desc.info);
 	}
-	desc.view_info = TextureViewInfo(resource, descriptor, view_format, shader_conversion, storage,
+	desc.view_info = TextureViewInfo(resource, descriptor, view_format, surface_format, storage,
 	                                 view_levels, desc.info.resources.layers);
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
 
@@ -720,13 +694,6 @@ static vk::DescriptorBufferInfo NativeUpload(RenderContext&            context,
 	return {buffer.Handle(), offset, data.size_bytes()};
 }
 
-void RenderExecutor::TrackImageBinding(ImageId id) {
-	EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr);
-	if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) {
-		m_bound_images.push_back(id);
-	}
-}
-
 void RenderExecutor::BindImage(ImageId id, bool storage) {
 	auto& image = m_context.GetTextureCache().GetImage(id);
 	if (image.info.data.Empty()) {
@@ -737,13 +704,13 @@ void RenderExecutor::BindImage(ImageId id, bool storage) {
 	}
 	image.binding.is_bound = true;
 	image.binding.shader_write |= storage;
-	TrackImageBinding(id);
+	m_bound_images.push_back(id);
 }
 
 void RenderExecutor::BindRenderTarget(ImageId id) {
 	auto& image             = m_context.GetTextureCache().GetImage(id);
 	image.binding.is_target = true;
-	TrackImageBinding(id);
+	m_bound_images.push_back(id);
 }
 
 void RenderExecutor::ResetBindings() {
@@ -796,10 +763,7 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[i]);
 		const auto address = descriptor.Base48();
-		const auto stride  = descriptor.Stride();
-		const auto records = descriptor.NumRecords();
-		// The descriptor has a 14-bit stride and 32-bit record count, so the product fits u64.
-		const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+		const auto requested_size = descriptor.GetSize();
 		if (address == 0 || requested_size == 0) {
 			prepared.buffer_sources.push_back({});
 			continue;
@@ -908,7 +872,7 @@ RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
 	}
 	if (bindings.vertex.runtime->program->info.uses_dma ||
 	    (bindings.pixel && bindings.pixel->runtime->program->info.uses_dma)) {
-		m_context.GetGpuResources().PrepareBda();
+		m_context.PrepareBda();
 	}
 	RebindBuffers(bindings.vertex);
 	if (bindings.pixel) {

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -66,9 +66,6 @@ template <typename T>
 
 [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
                                                    bool r128 = false);
-[[nodiscard]] bool
-IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
-                               const ShaderTextureResource& descriptor, const Image& image);
 void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
                             const ShaderTextureResource& descriptor, uint64_t size);
 

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -258,10 +258,6 @@ struct PipelineCache::ProgramCache {
 		                     "create recompiled shader module");
 		EXIT_IF(module == nullptr);
 		if (options.dump_ir) {
-			if (!options.early_dump) {
-				LOGF("%s decoded RDNA2:\n%s", options.dump_label, result.decoded_dump.c_str());
-				LOGF("%s IR:\n%s", options.dump_label, result.ir_dump.c_str());
-			}
 			LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", options.dump_label,
 			     static_cast<uint64_t>(result.spirv.size()), options.wave_size);
 		}
@@ -338,17 +334,16 @@ struct PipelineCache::ProgramCache {
 		options.shader_hash = params.hash;
 		options.user_data   = params.user_data;
 		options.back_code      = params.back_code;
-		options.dump_ir     = Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent;
+		options.dump_ir     = Config::GetShaderLogDirection() != Config::LogDirection::Silent;
 		options.early_dump  = options.dump_ir;
 		options.dump_label  = label;
 		options.input_info  = stage_input;
-		options.scratch_dwords = input_info.scratch_size_dwords;
+
 		if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
 			options.user_data_base = 8;
 			if (stage == ShaderType::Mesh) {
 				options.user_data_base = 0;
 				options.wave_size      = input_info.mesh.wave_size;
-				options.scratch_dwords = input_info.mesh.scratch_size_dwords;
 			}
 		} else if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
 			options.wave_size = input_info.wave_size;
@@ -633,7 +628,7 @@ bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other)
 	return std::memcmp(this, &other, sizeof(*this)) == 0;
 }
 
-PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
+PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
     std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
     const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
     const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
@@ -673,6 +668,16 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 			EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
 			     attachment_samples, colors[i].desc.info.samples);
 		}
+		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
+		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
+		static_params.color_srcblend[i]       = bc.color_srcblend;
+		static_params.color_comb_fcn[i]       = bc.color_comb_fcn;
+		static_params.color_destblend[i]      = bc.color_destblend;
+		static_params.alpha_srcblend[i]       = bc.alpha_srcblend;
+		static_params.alpha_comb_fcn[i]       = bc.alpha_comb_fcn;
+		static_params.alpha_destblend[i]      = bc.alpha_destblend;
+		static_params.separate_alpha_blend[i] = bc.separate_alpha_blend;
+		static_params.blend_enable[i]         = bc.enable && !rt.info.blend_bypass;
 	}
 	const bool with_depth =
 	    depth.desc.view_info.format != vk::Format::eUndefined && static_cast<bool>(depth.image_id);
@@ -732,19 +737,6 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 	static_params.polygon_mode =
 	    ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back);
 
-	for (uint32_t i = 0; i < color_count; i++) {
-		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
-		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		static_params.color_srcblend[i]       = bc.color_srcblend;
-		static_params.color_comb_fcn[i]       = bc.color_comb_fcn;
-		static_params.color_destblend[i]      = bc.color_destblend;
-		static_params.alpha_srcblend[i]       = bc.alpha_srcblend;
-		static_params.alpha_comb_fcn[i]       = bc.alpha_comb_fcn;
-		static_params.alpha_destblend[i]      = bc.alpha_destblend;
-		static_params.separate_alpha_blend[i] = bc.separate_alpha_blend;
-		static_params.blend_enable[i]         = bc.enable;
-		static_params.blend_bypass[i]         = rt.info.blend_bypass;
-	}
 	if (vs_input_info.stage.program->stage != ShaderType::Mesh) {
 		EXIT_IF(vs_input_info.buffers_num < 0 ||
 		        vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX ||
@@ -803,8 +795,8 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 }
 
 PipelineCache::Pipeline&
-PipelineCache::CreateComputePipeline(const ShaderComputeInputInfo& input_info,
-                                     const ShaderProgram&          compute_program) {
+PipelineCache::GetComputePipeline(const ShaderComputeInputInfo& input_info,
+                                  const ShaderProgram&          compute_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100);
 
 	EXIT_IF(!compute_program);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -59,7 +59,6 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
-	bool                       blend_bypass[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
 
 	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
@@ -69,7 +68,7 @@ struct PipelineStaticParameters {
 static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
 static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 166);
+static_assert(sizeof(PipelineStaticParameters) == 158);
 
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};
@@ -138,13 +137,13 @@ public:
 	                                ShaderComputeInputInfo&      input_info);
 
 	Pipeline&
-	CreateGraphicsPipeline(std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
-	                       const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
-	                       const ShaderPixelInputInfo* ps_input_info,
-	                       vk::PrimitiveTopology topology, bool primitive_restart_enable,
-	                       const ShaderProgram& vertex_program, const ShaderProgram& pixel_program);
-	Pipeline& CreateComputePipeline(const ShaderComputeInputInfo& input_info,
-	                                const ShaderProgram&          compute_program);
+	GetGraphicsPipeline(std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
+	                    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+	                    const ShaderPixelInputInfo* ps_input_info,
+	                    vk::PrimitiveTopology topology, bool primitive_restart_enable,
+	                    const ShaderProgram& vertex_program, const ShaderProgram& pixel_program);
+	Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info,
+	                             const ShaderProgram&          compute_program);
 
 private:
 	struct ProgramCache;

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -12,6 +12,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
@@ -115,227 +116,13 @@ static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format,
 		}
 		format = vk::Format::eR16G16Sfloat;
 		size   = 2;
-		if (NarrowInputFormat(format, size, used_components)) {
-			LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", raw_format,
-			     vk::to_string(format).c_str(), used_components);
-		}
 		return;
 	}
 
-	switch (fmt) {
-		case Prospero::BufferFormat::k32_32_32_32Float:
-			format = vk::Format::eR32G32B32A32Sfloat;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32_32SInt:
-			format = vk::Format::eR32G32B32A32Sint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32_32UInt:
-			format = vk::Format::eR32G32B32A32Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32Float:
-			format = vk::Format::eR32G32B32Sfloat;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k32_32_32SInt:
-			format = vk::Format::eR32G32B32Sint;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k32_32_32UInt:
-			format = vk::Format::eR32G32B32Uint;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16Float:
-			format = vk::Format::eR16G16B16A16Sfloat;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SInt:
-			format = vk::Format::eR16G16B16A16Sint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UInt:
-			format = vk::Format::eR16G16B16A16Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SScaled:
-			format = vk::Format::eR16G16B16A16Sscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UScaled:
-			format = vk::Format::eR16G16B16A16Uscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SNorm:
-			format = vk::Format::eR16G16B16A16Snorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UNorm:
-			format = vk::Format::eR16G16B16A16Unorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32Float:
-			format = vk::Format::eR32G32Sfloat;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32_32SInt:
-			format = vk::Format::eR32G32Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32_32UInt:
-			format = vk::Format::eR32G32Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UInt:
-			format = vk::Format::eR8G8B8A8Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8SScaled:
-			format = vk::Format::eR8G8B8A8Sscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UScaled:
-			format = vk::Format::eR8G8B8A8Uscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8SNorm:
-			format = vk::Format::eR8G8B8A8Snorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UNorm:
-			format = vk::Format::eR8G8B8A8Unorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k10_10_10_2UNorm:
-			format = vk::Format::eA2B10G10R10UnormPack32;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k10_10_10_2SNorm:
-			format = vk::Format::eA2B10G10R10SnormPack32;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16Float:
-			format = vk::Format::eR16G16Sfloat;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SInt:
-			format = vk::Format::eR16G16Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UInt:
-			format = vk::Format::eR16G16Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SScaled:
-			format = vk::Format::eR16G16Sscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UScaled:
-			format = vk::Format::eR16G16Uscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SNorm:
-			format = vk::Format::eR16G16Snorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UNorm:
-			format = vk::Format::eR16G16Unorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32Float:
-			format = vk::Format::eR32Sfloat;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k32SInt:
-			format = vk::Format::eR32Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k32UInt:
-			format = vk::Format::eR32Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8_8SInt:
-			format = vk::Format::eR8G8Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UInt:
-			format = vk::Format::eR8G8Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8SScaled:
-			format = vk::Format::eR8G8Sscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UScaled:
-			format = vk::Format::eR8G8Uscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8SNorm:
-			format = vk::Format::eR8G8Snorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UNorm:
-			format = vk::Format::eR8G8Unorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16Float:
-			format = vk::Format::eR16Sfloat;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SInt:
-			format = vk::Format::eR16Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UInt:
-			format = vk::Format::eR16Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SScaled:
-			format = vk::Format::eR16Sscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UScaled:
-			format = vk::Format::eR16Uscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SNorm:
-			format = vk::Format::eR16Snorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UNorm:
-			format = vk::Format::eR16Unorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SInt:
-			format = vk::Format::eR8Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UInt:
-			format = vk::Format::eR8Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SScaled:
-			format = vk::Format::eR8Sscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UScaled:
-			format = vk::Format::eR8Uscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SNorm:
-			format = vk::Format::eR8Snorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UNorm:
-			format = vk::Format::eR8Unorm;
-			size   = 1;
-			break;
-		default:
-			EXIT("unknown format: fmt = %u\n", raw_format);
-			format = vk::Format::eUndefined;
-			size   = 4;
-			break;
+	format = VulkanFormat(fmt);
+	size   = ShaderRecompiler::Format::GetFormatInfo(fmt).component_count;
+	if (format == vk::Format::eUndefined || size == 0) {
+		EXIT("unknown vertex format: fmt = %u\n", raw_format);
 	}
 
 	if (NarrowInputFormat(format, size, used_components)) {
@@ -468,35 +255,23 @@ void CreatePipelineInternal(
 	EXIT_NOT_IMPLEMENTED(
 	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
 
-	vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
-	vert_shader_stage_info.stage  = vertex_stage;
-	vert_shader_stage_info.module = vertex_program.module;
-	vert_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
-	frag_shader_stage_info.stage  = vk::ShaderStageFlagBits::eFragment;
-	frag_shader_stage_info.module = pixel_program.module;
-	frag_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
-	tess_control_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationControl;
-	tess_control_shader_stage_info.module = tess_control_shader_module;
-	tess_control_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo tess_eval_shader_stage_info {};
-	tess_eval_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationEvaluation;
-	tess_eval_shader_stage_info.module = tess_eval_shader_module;
-	tess_eval_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo shader_stages[4]   = {};
-	uint32_t                          shader_stage_count = 0;
-	shader_stages[shader_stage_count++]                  = vert_shader_stage_info;
+	vk::PipelineShaderStageCreateInfo shader_stages[4] = {
+	    {.stage = vertex_stage, .module = vertex_program.module, .pName = "main"}};
+	uint32_t shader_stage_count = 1;
 	if (rect_list) {
-		shader_stages[shader_stage_count++] = tess_control_shader_stage_info;
-		shader_stages[shader_stage_count++] = tess_eval_shader_stage_info;
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           vk::ShaderStageFlagBits::eTessellationControl,
+		                                       .module = tess_control_shader_module,
+		                                       .pName  = "main"};
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           vk::ShaderStageFlagBits::eTessellationEvaluation,
+		                                       .module = tess_eval_shader_module,
+		                                       .pName  = "main"};
 	}
 	if (ps_active) {
-		shader_stages[shader_stage_count++] = frag_shader_stage_info;
+		shader_stages[shader_stage_count++] = {.stage  = vk::ShaderStageFlagBits::eFragment,
+		                                       .module = pixel_program.module,
+		                                       .pName  = "main"};
 	}
 
 	vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX] {};
@@ -619,8 +394,7 @@ void CreatePipelineInternal(
 		EXIT_NOT_IMPLEMENTED((static_params.color_mask[i] & ~0x0fu) != 0);
 		color_blend_attachment[i].colorWriteMask =
 		    vk::ColorComponentFlags {static_params.color_mask[i]};
-		color_blend_attachment[i].blendEnable =
-		    (static_params.blend_enable[i] && !static_params.blend_bypass[i]) ? VK_TRUE : VK_FALSE;
+		color_blend_attachment[i].blendEnable = static_params.blend_enable[i] ? VK_TRUE : VK_FALSE;
 		color_blend_attachment[i].srcColorBlendFactor =
 		    GetBlendFactor(static_params.color_srcblend[i]);
 		color_blend_attachment[i].dstColorBlendFactor =

src/graphics/host_gpu/renderer/render.h

@@ -183,15 +183,15 @@ private:
 	                              uint32_t render_target_slice_offset, uint32_t render_target_slot,
 	                              bool ignore_target_mask = false, bool exact_format = false);
 	void ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepthInfo& target);
+	[[nodiscard]] bool DepthStencilCopy(CommandBuffer& buffer);
 	[[nodiscard]] bool PrepareDrawRenderState(CommandBuffer& buffer,
 	                                          const DrawCallInfo& draw,
 	                                          uint32_t            render_target_slice_offset,
-	                                          bool log_setup_phases, DrawRenderState& state);
+	                                          DrawRenderState& state);
 	void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer, const DrawCallInfo& draw,
 	                         DrawRenderState& state, vk::PrimitiveTopology topology,
 	                         const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
-	                         bool primitive_restart_enable, bool log_pipeline_phase,
-	                         bool set_bind_debug, bool set_auto_debug);
+	                         bool primitive_restart_enable);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth,
 	                                               const std::optional<PreparedBindings>& pixel = std::nullopt);
@@ -199,7 +199,6 @@ private:
 	                                              uint32_t render_target_slice_offset);
 	void                      BindImage(ImageId id, bool storage);
 	void                      BindRenderTarget(ImageId id);
-	void                      TrackImageBinding(ImageId id);
 	void                      ResetBindings();
 	[[nodiscard]] bool        TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
 	                                                     const CommandBuffer&          buffer);

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -36,18 +36,12 @@
 #include <vector>
 
 namespace Libs::Graphics {
-static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) {
-	const uint64_t records = descriptor.NumRecords();
-	const uint64_t stride  = descriptor.Stride();
-	return stride == 0 ? records : records * stride;
-}
-
 static bool FillSourcesDisjoint(std::span<const ShaderRecompiler::IR::DescriptorValue> sources,
                                  GuestRange destination, uint32_t output_buffer = UINT32_MAX) {
 	for (uint32_t i = 0; i < sources.size(); ++i) {
 		if (i == output_buffer) continue;
 		const auto source = DecodeNativeDescriptor<ShaderBufferResource>(sources[i]);
-		const auto bytes  = BufferDescriptorSize(source);
+		const auto bytes  = source.GetSize();
 		if (source.Base48() < destination.End() && destination.address < source.Base48() + bytes)
 			return false;
 	}
@@ -67,7 +61,7 @@ bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& in
 		const auto  descriptor = DecodeNativeDescriptor<ShaderBufferResource>(resources.buffers[i]);
 		// A metadata resource that is also read is not proven to be a full overwrite. Execute it
 		// conservatively instead of replacing the dispatch with a coarse full-surface clear.
-		if (cache.IsMeta(descriptor.Base48()) && (!resource.written || resource.read)) {
+		if ((!resource.written || resource.read) && cache.IsMeta(descriptor.Base48())) {
 			return false;
 		}
 	}
@@ -115,7 +109,7 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	const uint64_t invocations = input.dispatch_thread_dimensions
 	                                 ? group_x
 	                                 : static_cast<uint64_t>(group_x) * input.threads_num[0];
-	const auto     size        = BufferDescriptorSize(descriptor);
+	const auto     size        = descriptor.GetSize();
 	if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
 	    (input.dispatch_thread_dimensions &&
 	     (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
@@ -264,9 +258,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		input_info.dispatch_threads_num[2]    = thread_group_z;
 	}
 
-	const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
-	const bool     large_workgroup =
-	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const auto& program   = *input_info.stage.program;
 	const auto& resources = input_info.stage.resources;
 	if (TryConsumeComputeMetaClear(input_info, buffer)) {
@@ -278,14 +269,17 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		ResetBindings();
 		return;
 	}
-	const auto sampled_images = std::count_if(
-	    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
-		    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
-	    });
+	const bool large_workgroup =
+	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const bool                   has_sampler = !program.info.samplers.empty();
 	static std::atomic<uint32_t> dispatch_log_count {0};
 	if ((large_workgroup || has_sampler) &&
 	    dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) {
+		const auto sampled_images = std::count_if(
+		    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
+			    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
+		    });
+		const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
 		LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64
 		     " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u "
 		     "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n",
@@ -374,11 +368,11 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 
 	buffer.EndRendering();
 	auto& pipeline =
-	    m_context.GetPipelineCache().CreateComputePipeline(input_info, compute_program);
+	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto bindings = PrepareBindings(input_info.stage);
 	FindBuffers(bindings);
 	if (program.info.uses_dma) {
-		m_context.GetGpuResources().PrepareBda();
+		m_context.PrepareBda();
 	}
 	RebindBuffers(bindings);
 	RebindImages(bindings);

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -14,7 +14,8 @@ RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
       m_pipeline_cache(graphics), m_sampler_cache(graphics),
-      m_gpu_resources(graphics, m_command_scheduler) {
+      m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
+      m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -27,12 +28,10 @@ void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) {
 	EXIT_IF(m_gpu != nullptr);
 	m_video_out = video_out;
 	m_gpu       = std::make_unique<GuestGpu>(*this);
-	m_gpu_resources.SetGpu(m_gpu.get());
 }
 
 void RenderContext::ShutdownGpu() {
 	if (m_gpu != nullptr) {
-		m_gpu_resources.SetGpu(nullptr);
 		m_gpu->Shutdown();
 		m_gpu.reset();
 	}
@@ -55,6 +54,88 @@ VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const {
 	return *m_video_out;
 }
 
+bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
+	// The host reports the faulting byte, not the instruction's access width. Both caches
+	// resolve its page; guessing a width can cross the end of a valid guest mapping.
+	constexpr uint64_t fault_size = 1;
+	if (!IsMapped(fault_vaddr, fault_size)) {
+		return false;
+	}
+	if (access == PageFaultAccess::Write) {
+		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
+		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+	} else {
+		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
+	}
+	return true;
+}
+
+bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (!IsMapped(vaddr, size)) {
+		return false;
+	}
+	m_buffer_cache.InvalidateMemory(vaddr, size);
+	m_texture_cache.InvalidateMemory(vaddr, size);
+	return true;
+}
+
+bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
+	if (!GuestRange {vaddr, size}.Valid()) {
+		return false;
+	}
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	return m_mapped_ranges.Contains(vaddr, size);
+}
+
+void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) {
+	std::lock_guard lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.Add(vaddr, size);
+}
+
+void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	const auto unmap = [this, vaddr, size] {
+		if (m_command_scheduler.Active()) {
+			const auto tick = m_command_scheduler.CurrentTick();
+			m_command_scheduler.Finish();
+			m_command_scheduler.WaitPriorityOperations(tick);
+		}
+		m_buffer_cache.InvalidateMemory(vaddr, size);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Subtract(vaddr, size);
+	};
+	// Shutdown still owns the GPU while queued rendering drains, but its command lane no
+	// longer accepts external work. Use the guest GPU's state for the teardown route.
+	if (m_gpu == nullptr || m_gpu->IsStopping()) {
+		unmap();
+		return;
+	}
+	m_gpu->SendCommandSync(unmap);
+}
+
+void RenderContext::PrepareBda() {
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
+		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
+	});
+	m_fault_process_pending = true;
+}
+
+void RenderContext::RunGarbageCollector() {
+	if (m_fault_process_pending) {
+		m_fault_process_pending = false;
+		m_buffer_cache.ProcessFaultBuffer();
+	}
+	m_texture_cache.ProcessDownloadImages();
+	m_texture_cache.RunGarbageCollector();
+	m_buffer_cache.RunGarbageCollector();
+}
+
 void RenderContext::AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
 	Common::LockGuard lock(m_interrupt_mutex);
 

src/graphics/host_gpu/renderer/renderContext.h

@@ -5,8 +5,9 @@
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/threads.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/rangeSet.h"
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
-#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
@@ -15,6 +16,7 @@
 #include "kernel/eventQueue.h"
 
 #include <memory>
+#include <shared_mutex>
 #include <vector>
 
 namespace Libs::VideoOut {
@@ -42,11 +44,18 @@ public:
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
 	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
-	GpuResourceManager& GetGpuResources() { return m_gpu_resources; }
-	BufferCache&        GetBufferCache() { return m_gpu_resources.GetBufferCache(); }
-	TextureCache&       GetTextureCache() { return m_gpu_resources.GetTextureCache(); }
+	BufferCache&        GetBufferCache() { return m_buffer_cache; }
+	TextureCache&       GetTextureCache() { return m_texture_cache; }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
 
+	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
+	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
+	void               MapMemory(uint64_t vaddr, uint64_t size);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
+	void               PrepareBda();
+	void               RunGarbageCollector();
+
 	void AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
 	void DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
 	void TriggerInterrupt(int event_id, uint32_t context_id);
@@ -64,9 +73,14 @@ private:
 	DescriptorHeap            m_descriptor_heap;
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
-	GpuResourceManager        m_gpu_resources;
+	PageManager               m_page_manager;
+	BufferCache               m_buffer_cache;
+	TextureCache              m_texture_cache;
+	mutable std::shared_mutex m_mapped_ranges_mutex;
+	RangeSet                  m_mapped_ranges;
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
+	bool                      m_fault_process_pending = false;
 
 	Common::Mutex                        m_interrupt_mutex;
 	std::vector<InterruptEqRegistration> m_interrupt_eqs;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -46,42 +46,34 @@
 
 namespace Libs::Graphics {
 
-int32_t ResolveVertexOffset(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info) {
-	if (index_offset != 0 || !vs_input_info.fetch_embedded) {
-		return static_cast<int32_t>(index_offset);
+std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
+	                                           const ShaderVertexInputInfo& vs_input_info) {
+	auto     vertex_offset   = static_cast<int32_t>(index_offset);
+	uint32_t instance_offset = 0;
+	if (!vs_input_info.fetch_embedded) {
+		return {vertex_offset, instance_offset};
 	}
 
 	EXIT_IF(!vs_input_info.stage);
 	const auto& program   = *vs_input_info.stage.program;
 	const auto& resources = vs_input_info.stage.resources;
-	if (program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
+	if (index_offset == 0 &&
+	    program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
 		    static_cast<uint32_t>(program.info.vertex_offset_sgpr) - program.user_data_base;
 		if (index < resources.user_data.size()) {
-			return static_cast<int32_t>(resources.user_data[index]);
+			vertex_offset = static_cast<int32_t>(resources.user_data[index]);
 		}
 	}
-
-	return 0;
-}
-
-uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info) {
-	if (!vs_input_info.fetch_embedded) {
-		return 0;
-	}
-
-	EXIT_IF(!vs_input_info.stage);
-	const auto& program   = *vs_input_info.stage.program;
-	const auto& resources = vs_input_info.stage.resources;
 	if (program.info.instance_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
 		    static_cast<uint32_t>(program.info.instance_offset_sgpr) - program.user_data_base;
 		if (index < resources.user_data.size()) {
-			return resources.user_data[index];
+			instance_offset = resources.user_data[index];
 		}
 	}
 
-	return 0;
+	return {vertex_offset, instance_offset};
 }
 
 static std::atomic<uint32_t> g_draw_state_log_count   = 0;
@@ -467,11 +459,13 @@ struct DrawRenderState {
 };
 
 struct DrawCallInfo {
-	const char*          name           = nullptr;
 	CommandBufferDebugOp debug_op       = CommandBufferDebugOp::DrawIndex;
 	uint32_t             index_count    = 0;
 	uint32_t             instance_count = 0;
 	uint32_t             first_instance = 0;
+
+	[[nodiscard]] bool IsIndexed() const { return debug_op == CommandBufferDebugOp::DrawIndex; }
+	[[nodiscard]] const char* Name() const { return IsIndexed() ? "DrawIndex" : "DrawIndexAuto"; }
 };
 
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
@@ -662,7 +656,6 @@ static bool ConsumeMetadataColorOperation(const CommandBuffer& buffer) {
 }
 
 struct DrawEmitInfo {
-	bool     indexed       = false;
 	int32_t  vertex_offset = 0;
 	uint32_t first_vertex  = 0;
 	uint32_t first_instance = 0;
@@ -904,14 +897,14 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
 
 bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
-                                            bool log_setup_phases, DrawRenderState& state) {
+	                                        DrawRenderState& state) {
 	auto& ctx = buffer.GetRegisters();
 
 	if (ResolveColorTargets(buffer, render_target_slice_offset)) {
 		return false;
 	}
-	if (log_setup_phases) {
-		LogDrawPhase(draw.name, "ResolveRenderColorTarget");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}
 	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
 		if (slot == 0 || (render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0 &&
@@ -923,14 +916,14 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 			}
 		}
 	}
-	if (log_setup_phases) {
-		LogDrawPhase(draw.name, "ResolveRenderDepthTarget");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "ResolveRenderDepthTarget");
 	}
 	ResolveRenderDepthTarget(buffer, state.depth_info);
 
 	state.ps_active       = DrawHasActivePixelShader(buffer);
 	if (state.color_count == 0 && !state.depth_info.image_id && !state.ps_active) {
-		LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer,
+		LogFramebufferSkip(draw.Name(), state.color_info[0], state.depth_info, buffer,
 		                   draw.index_count, 0);
 		return false;
 	}
@@ -938,7 +931,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 	return true;
 }
 
-static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
+static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
                            DrawRenderState& state) {
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
@@ -955,8 +948,8 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool
 		target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
 	}
 	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
-	if (log_phases) {
-		LogDrawPhase(draw.name, "GetGraphicsPrograms");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
 	}
 	state.programs = pipeline_cache.GetGraphicsPrograms(
 	    vertex_shader_info, pixel_shader_info, shader_regs, ctx, buffer.GetUserConfig(),
@@ -974,17 +967,14 @@ static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffe
 		auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
 		prepared.offset = stream.Copy(source.host_data, source.size, 16);
 		prepared.buffer = stream.Handle();
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
+		                     static_cast<uint32_t>(source.type));
 	} else {
 		auto [buffer_ptr, offset] =
 		    buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
 		prepared.buffer = buffer_ptr->Handle();
 		prepared.offset = offset;
-	}
-	if (source.host_data != nullptr) {
-		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
-		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
-		                     static_cast<uint32_t>(source.type));
-	} else {
 		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
 		                     "Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
 		                     source.address, source.size, static_cast<uint32_t>(source.type));
@@ -1011,19 +1001,19 @@ static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBu
 }
 
 static void LogDrawStateIfNeeded(const CommandBuffer& buffer, const DrawCallInfo& draw,
-                                 const DrawRenderState& state, bool always_log,
-                                 bool force_legacy_rect_log, uint32_t index_type_and_size,
+	                             const DrawRenderState& state, uint32_t index_type_and_size,
                                  const void* index_addr) {
 	if (!graphics_debug_dump_enabled()) {
 		return;
 	}
 
-	if (!always_log && !force_legacy_rect_log) {
+	if (!draw.IsIndexed() &&
+	    buffer.GetUserConfig().GetPrimType() != Prospero::PrimitiveType::kRectListLegacy) {
 		return;
 	}
 
 	if (state.ps_active) {
-		LogDrawTargetState(draw.name, state.color_info[0], state.depth_info, buffer,
+		LogDrawTargetState(draw.Name(), state.color_info[0], state.depth_info, buffer,
 		                   state.ps_input_info, draw.index_count, 0);
 	}
 	LogDrawInputState(buffer, state.color_info[0], state.vs_input_info, index_type_and_size,
@@ -1041,7 +1031,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
 		case Prospero::PrimitiveType::kRectList:
-			if (emit.indexed) {
+			if (draw.IsIndexed()) {
 				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
 				                      emit.first_instance);
 			} else {
@@ -1050,7 +1040,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 			}
 			break;
 		case Prospero::PrimitiveType::kRectListLegacy:
-			if (emit.indexed) {
+			if (draw.IsIndexed()) {
 				EXIT("unknown primitive type: %u\n", static_cast<uint32_t>(ucfg.GetPrimType()));
 			}
 			// Sarah
@@ -1060,7 +1050,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kQuadListLegacy:
 			EXIT_NOT_IMPLEMENTED((draw.index_count & 0x3u) != 0);
 			for (uint32_t i = 0; i < draw.index_count; i += 4) {
-				if (emit.indexed) {
+				if (draw.IsIndexed()) {
 					vk_buffer.drawIndexed(4, draw.instance_count, i, emit.vertex_offset,
 					                      emit.first_instance);
 				} else {
@@ -1077,8 +1067,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
                                          const DrawCallInfo& draw, DrawRenderState& state,
                                          vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
                                          const DrawIndexBufferSource& index_source,
-                                         bool primitive_restart_enable, bool log_pipeline_phase,
-                                         bool set_bind_debug, bool set_auto_debug) {
+	                                     bool primitive_restart_enable) {
 	auto& ucfg = buffer.GetUserConfig();
 	const bool mesh_active = state.vs_input_info.stage.program->stage == ShaderType::Mesh;
 	uint32_t   mesh_groups = 0;
@@ -1086,7 +1075,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		const auto& mesh = state.vs_input_info.mesh;
 		if (primitive_restart_enable || mesh.primitives_per_group == 0) {
 			EXIT("unsupported mesh draw: primitive=%u indexed=%u restart=%u\n",
-			     static_cast<uint32_t>(ucfg.GetPrimType()), emit.indexed, primitive_restart_enable);
+			     static_cast<uint32_t>(ucfg.GetPrimType()), draw.IsIndexed(), primitive_restart_enable);
 		}
 		const auto primitives = mesh.InputPrimitiveCount(draw.index_count);
 		if (primitives == 0 || draw.instance_count == 0) {
@@ -1103,20 +1092,20 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		}
 	}
 
-	if (mesh_active && emit.indexed) {
+	if (mesh_active && draw.IsIndexed()) {
 		// Register the original guest indices for shader reads; PrepareGraphicsBindings
 		// synchronizes registered BDA ranges before any draw commands are committed.
 		(void)m_context.GetBufferCache().FindBuffer(
 		    index_source.address, static_cast<uint64_t>(draw.index_count) *
 		                              index_source.guest_element_size);
 	}
-	LogDrawPhase(draw.name, "PrepareBindings");
+	LogDrawPhase(draw.Name(), "PrepareBindings");
 	auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage,
 	                                        state.ps_active);
 	PreparedVertexBuffers vertex_bindings;
 	PreparedIndexBuffer   index_binding;
 	if (!mesh_active) {
-		LogDrawPhase(draw.name, "PrepareVertexBuffers");
+		LogDrawPhase(draw.Name(), "PrepareVertexBuffers");
 		vertex_bindings = AcquireVertexBuffers(buffer, state.vs_input_info);
 		index_binding   = PrepareIndexBuffer(buffer, index_source);
 	}
@@ -1124,10 +1113,10 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
 	                         bindings.pixel);
 
-	if (log_pipeline_phase) {
-		LogDrawPhase(draw.name, "CreatePipeline");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "CreatePipeline");
 	}
-	auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline(
+	auto& pipeline = m_context.GetPipelineCache().GetGraphicsPipeline(
 	    std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer,
 	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
 	    state.programs.vertex, state.programs.pixel);
@@ -1136,19 +1125,12 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	// point onward, every operation targets the current command buffer and cannot touch guest
 	// memory.
 	auto vk_buffer = buffer.Handle();
-	if (set_bind_debug) {
-		SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
-	}
-	if (set_auto_debug) {
-		SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
-	}
+	SetDrawDebugPhase(buffer, submit_id, draw, draw.IsIndexed() ? 0x100u : 0x200u);
 	if (!mesh_active) {
 		CommitVertexBuffers(vk_buffer, vertex_bindings);
 	}
-	if (bindings.pixel.has_value()) {
-		if (set_auto_debug) {
-			SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
-		}
+	if (bindings.pixel && !draw.IsIndexed()) {
+		SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 	}
 	std::array<PreparedBindings*, 2> descriptor_stages {&bindings.vertex, nullptr};
 	const size_t                     descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u;
@@ -1160,7 +1142,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	if (mesh_active) {
 		const uint32_t draw_data[] {
 		    draw.index_count,
-		    emit.indexed ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
+		    draw.IsIndexed() ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
 		    emit.first_instance, index_source.guest_element_size,
 		    static_cast<uint32_t>(index_source.address),
 		    static_cast<uint32_t>(index_source.address >> 32u)};
@@ -1183,13 +1165,13 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		        : vk::ImageAspectFlags {});
 	}
 
-	LogDrawPhase(draw.name, "BeginRendering");
-	if (set_auto_debug) {
+	LogDrawPhase(draw.Name(), "BeginRendering");
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
 	}
 	m_context.GetCommandScheduler().BeginRendering(rendering);
 	vk_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline.pipeline);
-	if (set_auto_debug) {
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
 	}
 	if (mesh_active) {
@@ -1198,7 +1180,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		EmitDrawPrimitives(ucfg, vk_buffer, state.vs_input_info, draw, emit);
 	}
 
-	if (set_auto_debug) {
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
 	}
 	vk::PipelineStageFlags shader_write_stages = {};
@@ -1213,8 +1195,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		m_context.GetCommandScheduler().EndRendering();
 		ShaderWriteBarrier(vk_buffer, shader_write_stages);
 	}
-	LogDrawPhase(draw.name, "DrawComplete");
-	if (set_auto_debug) {
+	LogDrawPhase(draw.Name(), "DrawComplete");
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
 	}
 }
@@ -1238,7 +1220,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	if (ConsumeMetadataColorOperation(buffer)) {
+	if (ConsumeMetadataColorOperation(buffer) || DepthStencilCopy(buffer)) {
 		ResetBindings();
 		return;
 	}
@@ -1248,7 +1230,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	}
 
 	if (graphics_debug_dump_enabled()) {
-		sh_print("GraphicsRenderDrawIndex():Shader:", sh_ctx);
+		LOGF("GraphicsRenderDrawIndex():Shader:\n");
 		uc_print("GraphicsRenderDrawIndex():UserConfig:", ucfg);
 		hw_print(buffer);
 
@@ -1304,34 +1286,30 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		index_source.size      = expanded_indices.size() * sizeof(uint16_t);
 	}
 
-	const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex, args.index_count,
+	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndex, args.index_count,
 	                        args.instance_count, args.first_instance};
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, true,
-	                            state)) {
+	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return;
 	}
 
-	RefreshShaders(buffer, draw, true, state);
+	RefreshShaders(buffer, draw, state);
 
-	LogDrawStateIfNeeded(buffer, draw, state, true, false, args.index_type_and_size,
+	LogDrawStateIfNeeded(buffer, draw, state, args.index_type_and_size,
 	                     args.index_addr);
 
 	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto vertex_offset =
-	    indirect
-	        ? args.base_vertex
-	        : ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) + args.base_vertex;
+	const auto [vertex_offset, instance_offset] =
+	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
+	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vs_input_info);
 
 	DrawEmitInfo emit {};
-	emit.indexed       = true;
-	emit.vertex_offset = vertex_offset;
-	emit.first_instance =
-	    indirect ? args.first_instance : ResolveInstanceOffset(state.vs_input_info);
+	emit.vertex_offset  = vertex_offset + args.base_vertex;
+	emit.first_instance = instance_offset;
 
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
-	                    primitive_restart, true, true, false);
+	                    primitive_restart);
 	ResetBindings();
 }
 
@@ -1354,7 +1332,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		return;
 	}
 
-	if (ConsumeMetadataColorOperation(buffer)) {
+	if (ConsumeMetadataColorOperation(buffer) || DepthStencilCopy(buffer)) {
 		ResetBindings();
 		return;
 	}
@@ -1364,7 +1342,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	}
 
 	if (graphics_debug_dump_enabled()) {
-		sh_print("GraphicsRenderDrawIndexAuto():Shader:", sh_ctx);
+		LOGF("GraphicsRenderDrawIndexAuto():Shader:\n");
 		uc_print("GraphicsRenderDrawIndexAuto():UserConfig:", ucfg);
 		hw_print(buffer);
 
@@ -1380,12 +1358,11 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 
 	hw_check(buffer);
 
-	const DrawCallInfo draw {"DrawIndexAuto", CommandBufferDebugOp::DrawIndexAuto,
+	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndexAuto,
 	                         args.vertex_count, args.instance_count, args.first_instance};
 
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, false,
-	                            state)) {
+	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return;
 	}
@@ -1395,7 +1372,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		ResetBindings();
 		return;
 	}
-	RefreshShaders(buffer, draw, false, state);
+	RefreshShaders(buffer, draw, state);
 
 	const bool rect_list = topology == vk::PrimitiveTopology::ePatchList;
 	if (rect_list && state.vs_input_info.buffers_num == 0 &&
@@ -1411,23 +1388,18 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		return;
 	}
 
-	LogDrawStateIfNeeded(buffer, draw, state, false,
-	                     ucfg.GetPrimType() == Prospero::PrimitiveType::kRectListLegacy, 0,
-	                     nullptr);
+	LogDrawStateIfNeeded(buffer, draw, state, 0, nullptr);
 
 	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto vertex_offset =
-	    indirect ? static_cast<int32_t>(args.first_vertex)
-	             : ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) +
-	                   static_cast<int32_t>(args.first_vertex);
+	const auto [vertex_offset, instance_offset] =
+	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
+	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vs_input_info);
 	DrawEmitInfo emit {};
-	emit.first_vertex = static_cast<uint32_t>(vertex_offset);
-	emit.first_instance =
-	    indirect ? args.first_instance : ResolveInstanceOffset(state.vs_input_info);
+	emit.first_vertex = static_cast<uint32_t>(vertex_offset + static_cast<int32_t>(args.first_vertex));
+	emit.first_instance = instance_offset;
 
 	DrawIndexBufferSource index_source {};
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false,
-	                    false, true);
+	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false);
 	ResetBindings();
 }
 

src/graphics/host_gpu/renderer/renderDraw.h

@@ -2,14 +2,14 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 
 #include <cstdint>
+#include <utility>
 
 namespace Libs::Graphics {
 
 struct ShaderVertexInputInfo;
 
-[[nodiscard]] int32_t  ResolveVertexOffset(uint32_t                     index_offset,
-                                           const ShaderVertexInputInfo& vs_input_info);
-[[nodiscard]] uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info);
+[[nodiscard]] std::pair<int32_t, uint32_t>
+ResolveDrawOffsets(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/renderTarget.h

@@ -84,14 +84,6 @@ struct PipelineStencilDynamicState {
 
 #pragma pack(pop)
 
-inline constexpr bool stencil_face_accesses_attachment(const PipelineStencilStaticState&  state,
-                                                       const PipelineStencilDynamicState& dynamic) {
-	return state.compareOp != vk::CompareOp::eAlways ||
-	       (dynamic.writeMask != 0 &&
-	        (state.failOp != vk::StencilOp::eKeep || state.passOp != vk::StencilOp::eKeep ||
-	         state.depthFailOp != vk::StencilOp::eKeep));
-}
-
 static_assert(std::is_trivially_copyable_v<PipelineStencilStaticState>);
 static_assert(std::is_standard_layout_v<PipelineStencilStaticState>);
 static_assert(alignof(PipelineStencilStaticState) == 1);

src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc

@@ -1,29 +1,17 @@
+#include "gpu_tiler_standard.inc"
 uint standard64_offset(uvec2 p) {
 	uint x = p.x, y = p.y;
+	uint offset = standard_offset(p);
 	switch (ELEMENT_BYTES) {
-	case 1: return (x & 0x0fu) ^ ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^
-	               ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
-	               ((y << 4) & 0x1f0u) ^ ((y << 5) & 0x400u) ^
-	               ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u);
-	case 2: return ((x << 1) & 0x00eu) ^ ((x << 4) & 0x080u) ^
-	               ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^
-	               ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
-	               ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^
-	               ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
-	case 4: return ((x << 2) & 0x00cu) ^ ((x << 5) & 0x080u) ^
-	               ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^
-	               ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
-	               ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^
-	               ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
-	case 8: return ((x << 3) & 0x008u) ^ ((x << 5) & 0x0c0u) ^
-	               ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^
-	               ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
-	               ((y << 4) & 0x030u) ^ ((y << 6) & 0x100u) ^
-	               ((y << 7) & 0x400u) ^ ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
-	default: return ((x << 6) & 0x0c0u) ^ ((x << 7) & 0x200u) ^
-	                ((x << 8) & 0x800u) ^ ((x << 9) & 0x2000u) ^
-	                ((x << 10) & 0x8000u) ^ ((y << 4) & 0x030u) ^
-	                ((y << 6) & 0x100u) ^ ((y << 7) & 0x400u) ^
-	                ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
+	case 1: return offset ^ ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
+	                       ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u);
+	case 2: return offset ^ ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
+	                       ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
+	case 4: return offset ^ ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
+	                       ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
+	case 8: return offset ^ ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
+	                       ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
+	default: return offset ^ ((x << 9) & 0x2000u) ^ ((x << 10) & 0x8000u) ^
+	                        ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
 	}
 }

src/graphics/host_gpu/vulkanCommon.cpp

@@ -17,14 +17,22 @@ struct FormatMapping {
 
 constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k8UNorm, vk::Format::eR8Unorm},
+    {Prospero::BufferFormat::k8SNorm, vk::Format::eR8Snorm},
+    {Prospero::BufferFormat::k8UScaled, vk::Format::eR8Uscaled},
+    {Prospero::BufferFormat::k8SScaled, vk::Format::eR8Sscaled},
     {Prospero::BufferFormat::k8UInt, vk::Format::eR8Uint},
+    {Prospero::BufferFormat::k8SInt, vk::Format::eR8Sint},
     {Prospero::BufferFormat::k16UNorm, vk::Format::eR16Unorm},
     {Prospero::BufferFormat::k16SNorm, vk::Format::eR16Snorm},
+    {Prospero::BufferFormat::k16UScaled, vk::Format::eR16Uscaled},
+    {Prospero::BufferFormat::k16SScaled, vk::Format::eR16Sscaled},
     {Prospero::BufferFormat::k16UInt, vk::Format::eR16Uint},
     {Prospero::BufferFormat::k16SInt, vk::Format::eR16Sint},
     {Prospero::BufferFormat::k16Float, vk::Format::eR16Sfloat},
     {Prospero::BufferFormat::k8_8UNorm, vk::Format::eR8G8Unorm},
     {Prospero::BufferFormat::k8_8SNorm, vk::Format::eR8G8Snorm},
+    {Prospero::BufferFormat::k8_8UScaled, vk::Format::eR8G8Uscaled},
+    {Prospero::BufferFormat::k8_8SScaled, vk::Format::eR8G8Sscaled},
     {Prospero::BufferFormat::k8_8UInt, vk::Format::eR8G8Uint},
     {Prospero::BufferFormat::k8_8SInt, vk::Format::eR8G8Sint},
     {Prospero::BufferFormat::k32UInt, vk::Format::eR32Uint},
@@ -32,14 +40,19 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32Float, vk::Format::eR32Sfloat},
     {Prospero::BufferFormat::k16_16UNorm, vk::Format::eR16G16Unorm},
     {Prospero::BufferFormat::k16_16SNorm, vk::Format::eR16G16Snorm},
+    {Prospero::BufferFormat::k16_16UScaled, vk::Format::eR16G16Uscaled},
+    {Prospero::BufferFormat::k16_16SScaled, vk::Format::eR16G16Sscaled},
     {Prospero::BufferFormat::k16_16UInt, vk::Format::eR16G16Uint},
     {Prospero::BufferFormat::k16_16SInt, vk::Format::eR16G16Sint},
     {Prospero::BufferFormat::k16_16Float, vk::Format::eR16G16Sfloat},
     {Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
     {Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
+    {Prospero::BufferFormat::k10_10_10_2SNorm, vk::Format::eA2B10G10R10SnormPack32},
     {Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
     {Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
     {Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},
+    {Prospero::BufferFormat::k8_8_8_8UScaled, vk::Format::eR8G8B8A8Uscaled},
+    {Prospero::BufferFormat::k8_8_8_8SScaled, vk::Format::eR8G8B8A8Sscaled},
     {Prospero::BufferFormat::k8_8_8_8UInt, vk::Format::eR8G8B8A8Uint},
     {Prospero::BufferFormat::k8_8_8_8SInt, vk::Format::eR8G8B8A8Sint},
     {Prospero::BufferFormat::k32_32UInt, vk::Format::eR32G32Uint},
@@ -47,6 +60,8 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32_32Float, vk::Format::eR32G32Sfloat},
     {Prospero::BufferFormat::k16_16_16_16UNorm, vk::Format::eR16G16B16A16Unorm},
     {Prospero::BufferFormat::k16_16_16_16SNorm, vk::Format::eR16G16B16A16Snorm},
+    {Prospero::BufferFormat::k16_16_16_16UScaled, vk::Format::eR16G16B16A16Uscaled},
+    {Prospero::BufferFormat::k16_16_16_16SScaled, vk::Format::eR16G16B16A16Sscaled},
     {Prospero::BufferFormat::k16_16_16_16UInt, vk::Format::eR16G16B16A16Uint},
     {Prospero::BufferFormat::k16_16_16_16SInt, vk::Format::eR16G16B16A16Sint},
     {Prospero::BufferFormat::k16_16_16_16Float, vk::Format::eR16G16B16A16Sfloat},
@@ -63,7 +78,8 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k8_8_8_8Srgb, vk::Format::eR8G8B8A8Srgb},
     {Prospero::BufferFormat::k9_9_9_5Float, vk::Format::eE5B9G9R9UfloatPack32},
     {Prospero::BufferFormat::k5_6_5UNorm, vk::Format::eB5G6R5UnormPack16},
-    {Prospero::BufferFormat::k5_5_5_1UNorm, vk::Format::eR5G5B5A1UnormPack16},
+    {Prospero::BufferFormat::k5_5_5_1UNorm, vk::Format::eA1R5G5B5UnormPack16},
+    {Prospero::BufferFormat::k1_5_5_5UNorm, vk::Format::eR5G5B5A1UnormPack16},
     {Prospero::BufferFormat::k4_4_4_4UNorm, vk::Format::eR4G4B4A4UnormPack16},
     {Prospero::BufferFormat::kBc1UNorm, vk::Format::eBc1RgbaUnormBlock},
     {Prospero::BufferFormat::kBc1Srgb, vk::Format::eBc1RgbaSrgbBlock},

src/graphics/host_gpu/vulkanCommon.h

@@ -38,7 +38,6 @@ void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<A
 
 	const auto                      name = fmt::format(format, std::forward<Args>(args)...);
 	vk::DebugUtilsObjectNameInfoEXT info {};
-	info.sType        = vk::StructureType::eDebugUtilsObjectNameInfoEXT;
 	info.objectType   = Handle::objectType;
 	info.objectHandle = static_cast<uint64_t>(
 	    reinterpret_cast<uintptr_t>(static_cast<typename Handle::CType>(handle)));

src/graphics/presentation/videoOut.cpp

@@ -1152,7 +1152,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 	Graphics::RenderDocOnGuestFlip(m_presenter.Renderer());
 
 	if (Config::GraphicsDebugDumpEnabled() &&
-	    Config::GetPrintfDirection() != Config::OutputDirection::Silent) {
+	    Config::GetPrintfDirection() != Config::LogDirection::Silent) {
 		LOGF("Flip done: %d\n", r.index);
 	}
 

src/graphics/presentation/window/vulkanWindow.cpp

@@ -64,7 +64,6 @@ struct VulkanExtensions {
 
 vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
 	vk::PhysicalDeviceVulkan12Features features {};
-	features.sType                     = vk::StructureType::ePhysicalDeviceVulkan12Features;
 	features.samplerMirrorClampToEdge  = VK_TRUE;
 	features.timelineSemaphore         = VK_TRUE;
 	features.shaderOutputLayer         = VK_TRUE;
@@ -76,7 +75,6 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 
 vk::PhysicalDeviceVulkan13Features WindowContext::RequiredVulkan13Features() noexcept {
 	vk::PhysicalDeviceVulkan13Features features {};
-	features.sType            = vk::StructureType::ePhysicalDeviceVulkan13Features;
 	features.dynamicRendering = VK_TRUE;
 	features.synchronization2 = VK_TRUE;
 	return features;
@@ -201,35 +199,24 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		vk::PhysicalDeviceFeatures2 device_features2 {};
 
 		vk::PhysicalDeviceVulkan13Features features13 {};
-		features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-		features13.pNext = nullptr;
 
 		vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {};
-		color_write_ext.sType = vk::StructureType::ePhysicalDeviceColorWriteEnableFeaturesEXT;
-		color_write_ext.pNext = nullptr;
 
 		vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip_enable {};
-		depth_clip_enable.sType = vk::StructureType::ePhysicalDeviceDepthClipEnableFeaturesEXT;
 		depth_clip_enable.pNext = &color_write_ext;
 
 		vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
-		depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
 		depth_clip_control.pNext = &depth_clip_enable;
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
-		features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
 #if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
 #else
 		vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
-		fragment_barycentric.sType =
-		    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
 		features13.pNext = &features12;
-
-		device_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
 		device_features2.pNext = &features13;
 
 		device.getFeatures2(&device_features2);
@@ -506,83 +493,31 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 	}
 }
 
-static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device,
-                                          GraphicContext&    graphics) {
-	EXIT_IF(physical_device == nullptr);
-
-	vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {};
-	subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlProperties;
-	subgroup_size_control.pNext = nullptr;
-
-	vk::PhysicalDeviceVulkan11Properties properties11 {};
-	properties11.sType = vk::StructureType::ePhysicalDeviceVulkan11Properties;
-	properties11.pNext = &subgroup_size_control;
-
-	vk::PhysicalDeviceProperties2 properties2 {};
-	properties2.sType = vk::StructureType::ePhysicalDeviceProperties2;
-	properties2.pNext = &properties11;
-
-	physical_device.getProperties2(&properties2);
-
-	vk::PhysicalDeviceVulkan13Features features13 {};
-	features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-	features13.pNext = nullptr;
-
-	vk::PhysicalDeviceFeatures2 features2 {};
-	features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
-	features2.pNext = &features13;
-
-	physical_device.getFeatures2(&features2);
-
-	graphics.subgroup_size                 = properties11.subgroupSize;
-	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
-	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
-	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
-	graphics.compute_subgroup_size_control_enabled =
-	    features13.subgroupSizeControl == VK_TRUE &&
-	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
-	    subgroup_size_control.minSubgroupSize <= 64 &&
-	    subgroup_size_control.maxSubgroupSize >= 64;
-
-	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
-	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
-	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
-	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
-	     graphics.SupportsComputeWave64() ? "true" : "false");
-}
-
-static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r,
-                                     uint32_t                        queue_family,
-                                     const std::vector<const char*>& device_extensions,
-                                     GraphicContext&                 graphics) {
+static vk::Device VulkanCreateDevice(GraphicContext& graphics,
+	                                 const std::vector<const char*>& device_extensions) {
+	const auto physical_device = graphics.physical_device;
+	const auto queue_family    = graphics.queue_family;
 	EXIT_IF(physical_device == nullptr);
 	EXIT_IF(queue_family == static_cast<uint32_t>(-1));
 
 	const float               queue_priority = 1.0f;
 	vk::DeviceQueueCreateInfo queue_create_info {};
-	queue_create_info.sType            = vk::StructureType::eDeviceQueueCreateInfo;
 	queue_create_info.queueFamilyIndex = queue_family;
 	queue_create_info.queueCount       = 1;
 	queue_create_info.pQueuePriorities = &queue_priority;
 
 	vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {};
-	color_write_ext.sType = vk::StructureType::ePhysicalDeviceColorWriteEnableFeaturesEXT;
-	color_write_ext.pNext = nullptr;
 	color_write_ext.colorWriteEnable = VK_TRUE;
 
 	vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip_enable {};
-	depth_clip_enable.sType = vk::StructureType::ePhysicalDeviceDepthClipEnableFeaturesEXT;
 	depth_clip_enable.pNext = &color_write_ext;
 	depth_clip_enable.depthClipEnable = VK_TRUE;
 
 	vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
-	depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
 	// MoltenVK lacks VK_EXT_depth_clip_enable and VK_EXT_color_write_enable, so drop those
 	// feature structs from the chain on macOS (the renderer falls back to default depth
 	// clipping and static color-write masks).
-#if defined(__APPLE__)
-	depth_clip_control.pNext = nullptr;
-#else
+#if !defined(__APPLE__)
 	depth_clip_control.pNext = &depth_clip_enable;
 #endif
 	depth_clip_control.depthClipControl = VK_TRUE;
@@ -591,40 +526,19 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	features12.pNext = &depth_clip_control;
 
 	vk::PhysicalDeviceVulkan13Features supported_features13 {};
-	supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-
-	vk::PhysicalDeviceVulkan12Features supported_features12 {};
-	supported_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
-	supported_features12.pNext = nullptr;
-	supported_features13.pNext = &supported_features12;
-
-#if !defined(__APPLE__)
-	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_fragment_barycentric {};
-	supported_fragment_barycentric.sType =
-	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
-	supported_fragment_barycentric.pNext = nullptr;
-	supported_features12.pNext           = &supported_fragment_barycentric;
-#endif
 
 	const auto robustness2_ext_enabled =
 	    HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT supported_robustness2 {};
-	supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
-	supported_robustness2.pNext = nullptr;
 	if (robustness2_ext_enabled) {
-#if defined(__APPLE__)
-		supported_features12.pNext = &supported_robustness2;
-#else
-		supported_fragment_barycentric.pNext = &supported_robustness2;
-#endif
+		supported_features13.pNext = &supported_robustness2;
 	}
 
 	const bool mesh_extension = HasExtension(device_extensions, VK_EXT_MESH_SHADER_EXTENSION_NAME);
 	vk::PhysicalDeviceMeshShaderFeaturesEXT supported_mesh {};
 	supported_mesh.pNext = &supported_features13;
 	vk::PhysicalDeviceFeatures2 supported_features2 {};
-	supported_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
 	supported_features2.pNext = mesh_extension ? static_cast<void*>(&supported_mesh)
 	                                           : static_cast<void*>(&supported_features13);
 	const bool feedback_extensions =
@@ -645,53 +559,53 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		supported_features2.pNext = &provoking_vertex;
 	}
 	physical_device.getFeatures2(&supported_features2);
+	graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader;
+
+	vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {};
+
+	vk::PhysicalDeviceVulkan11Properties properties11 {};
+	properties11.pNext = &subgroup_size_control;
+
+	vk::PhysicalDeviceProperties2 properties2 {};
+	properties2.pNext = &properties11;
+
+	if (graphics.mesh_shader_enabled) {
+		subgroup_size_control.pNext = &graphics.mesh_shader_properties;
+	}
+	physical_device.getProperties2(&properties2);
+
+	graphics.subgroup_size                 = properties11.subgroupSize;
+	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
+	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
+	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
+	graphics.compute_subgroup_size_control_enabled =
+	    supported_features13.subgroupSizeControl == VK_TRUE &&
+	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
+	    subgroup_size_control.minSubgroupSize <= 64 &&
+	    subgroup_size_control.maxSubgroupSize >= 64;
+
+	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
+	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
+	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
+	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
+	     graphics.SupportsComputeWave64() ? "true" : "false");
 	graphics.provoking_vertex_last_enabled = provoking_extension && provoking_vertex.provokingVertexLast;
 	graphics.attachment_feedback_loop_enabled =
 	    feedback_extensions && feedback_layout.attachmentFeedbackLoopLayout &&
 	    feedback_dynamic.attachmentFeedbackLoopDynamicState;
 	LOGF("Vulkan depth feedback support: %s\n",
 	     graphics.attachment_feedback_loop_enabled ? "true" : "false");
-	graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader;
 	if (graphics.mesh_shader_enabled) {
-		vk::PhysicalDeviceProperties2 properties {};
-		properties.pNext = &graphics.mesh_shader_properties;
-		physical_device.getProperties2(&properties);
 		LOGF("Vulkan MeshEXT: invocations=%u vertices=%u primitives=%u shared=%u\n",
 		     graphics.mesh_shader_properties.maxMeshWorkGroupInvocations,
 		     graphics.mesh_shader_properties.maxMeshOutputVertices,
 		     graphics.mesh_shader_properties.maxMeshOutputPrimitives,
 		     graphics.mesh_shader_properties.maxMeshSharedMemorySize);
 	}
-	const auto required_features12 = WindowContext::RequiredVulkan12Features();
-	const auto required_features13 = WindowContext::RequiredVulkan13Features();
-	EXIT_NOT_IMPLEMENTED(required_features12.samplerMirrorClampToEdge == VK_TRUE &&
-	                     supported_features12.samplerMirrorClampToEdge != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.timelineSemaphore == VK_TRUE &&
-	                     supported_features12.timelineSemaphore != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features13.dynamicRendering == VK_TRUE &&
-	                     supported_features13.dynamicRendering != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features13.synchronization2 == VK_TRUE &&
-	                     supported_features13.synchronization2 != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.multiViewport != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.fillModeNonSolid != VK_TRUE);
+	// VulkanFindPhysicalDevice already checked the required creation features. These two
+	// requirements are specific to this creation path and are not part of device selection.
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderOutputLayer == VK_TRUE &&
-	                     supported_features12.shaderOutputLayer != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderOutputViewportIndex == VK_TRUE &&
-	                     supported_features12.shaderOutputViewportIndex != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.bufferDeviceAddress == VK_TRUE &&
-	                     supported_features12.bufferDeviceAddress != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderBufferInt64Atomics == VK_TRUE &&
-	                     supported_features12.shaderBufferInt64Atomics != VK_TRUE);
-#if !defined(__APPLE__)
-	EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
-#endif
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
 	device_features.samplerAnisotropy        = VK_TRUE;
@@ -715,13 +629,10 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	device_features.shaderInt64 = VK_TRUE;
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
-	robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
 #if defined(__APPLE__)
 	robustness2.pNext = &features12;
 #else
 	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
-	fragment_barycentric.sType =
-	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
 	fragment_barycentric.pNext                     = &features12;
 	fragment_barycentric.fragmentShaderBarycentric = VK_TRUE;
 	robustness2.pNext                              = &fragment_barycentric;
@@ -732,11 +643,7 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		robustness2.nullDescriptor      = supported_robustness2.nullDescriptor;
 	}
 
-	const bool subgroup_size_control_enabled =
-	    graphics.compute_subgroup_size_control_enabled &&
-	    supported_features13.subgroupSizeControl == VK_TRUE;
-
-	auto features13 = required_features13;
+	auto features13 = WindowContext::RequiredVulkan13Features();
 #if defined(__APPLE__)
 	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
 	                                           : static_cast<void*>(&features12);
@@ -745,7 +652,8 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	                                           : static_cast<void*>(&fragment_barycentric);
 #endif
 	features13.robustImageAccess   = supported_features13.robustImageAccess;
-	features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
+	features13.subgroupSizeControl =
+	    graphics.compute_subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
 
 	LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
 	     features13.robustImageAccess == VK_TRUE ? "true" : "false",
@@ -755,7 +663,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	vk::PhysicalDeviceMeshShaderFeaturesEXT mesh_features {};
 	mesh_features.pNext                 = &features13;
 	mesh_features.meshShader            = graphics.mesh_shader_enabled;
-	create_info.sType                   = vk::StructureType::eDeviceCreateInfo;
 	feedback_dynamic.pNext =
 	    mesh_extension ? static_cast<void*>(&mesh_features) : static_cast<void*>(&features13);
 	create_info.pNext = graphics.attachment_feedback_loop_enabled
@@ -766,7 +673,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		provoking_vertex.transformFeedbackPreservesProvokingVertex = VK_FALSE;
 		create_info.pNext = &provoking_vertex;
 	}
-	create_info.flags                   = {};
 	create_info.pQueueCreateInfos       = &queue_create_info;
 	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
@@ -796,8 +702,6 @@ static void VulkanGetExtensions(SDL_Window* window, VulkanExtensions& r) {
 
 	r.required_extensions =
 	    std::vector<const char*>(required_extensions_count); // @suppress("Ambiguous problem")
-	std::memset(r.required_extensions.data(), 0,
-	            sizeof(const char*) * r.required_extensions.size());
 
 	sdl_result = SDL_Vulkan_GetInstanceExtensions(window, &required_extensions_count,
 	                                              r.required_extensions.data());
@@ -989,8 +893,6 @@ void WindowContext::CreateVulkan() {
 	VulkanCheckInstanceVersion();
 
 	vk::ApplicationInfo app_info {};
-	app_info.sType              = vk::StructureType::eApplicationInfo;
-	app_info.pNext              = nullptr;
 	app_info.pApplicationName   = "Kyty";
 	app_info.applicationVersion = 1;
 	app_info.pEngineName        = "Kyty";
@@ -1019,17 +921,11 @@ void WindowContext::CreateVulkan() {
 	}
 
 	vk::ValidationFeaturesEXT validation_features {};
-	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
-	validation_features.pNext                          = nullptr;
 	validation_features.enabledValidationFeatureCount  = enabled_features_count;
 	validation_features.pEnabledValidationFeatures     = enabled_features;
-	validation_features.disabledValidationFeatureCount = 0;
-	validation_features.pDisabledValidationFeatures    = nullptr;
 
 	vk::DebugUtilsMessengerCreateInfoEXT dbg_create_info {};
-	dbg_create_info.sType           = vk::StructureType::eDebugUtilsMessengerCreateInfoEXT;
 	dbg_create_info.pNext           = &validation_features;
-	dbg_create_info.flags           = {};
 	dbg_create_info.messageSeverity = vk::DebugUtilsMessageSeverityFlagBitsEXT::eVerbose |
 	                                  vk::DebugUtilsMessageSeverityFlagBitsEXT::eInfo |
 	                                  vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning |
@@ -1038,12 +934,9 @@ void WindowContext::CreateVulkan() {
 	                                  vk::DebugUtilsMessageTypeFlagBitsEXT::eValidation |
 	                                  vk::DebugUtilsMessageTypeFlagBitsEXT::ePerformance;
 	dbg_create_info.pfnUserCallback = VulkanDebugMessengerCallback;
-	dbg_create_info.pUserData       = nullptr;
 
 	vk::InstanceCreateInfo inst_info {};
-	inst_info.sType                   = vk::StructureType::eInstanceCreateInfo;
 	inst_info.pNext                   = (r.enable_validation_layers ? &dbg_create_info : nullptr);
-	inst_info.flags                   = {};
 #if defined(__APPLE__)
 	// MoltenVK requires VK_KHR_portability_enumeration + flag to surface
 	// portability devices. Without this, enumeratePhysicalDevices hides the
@@ -1110,10 +1003,8 @@ void WindowContext::CreateVulkan() {
 	}
 #endif
 
-	uint32_t queue_family = static_cast<uint32_t>(-1);
-
 	VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, surface_capabilities,
-	                         graphic_ctx.physical_device, queue_family);
+	                         graphic_ctx.physical_device, graphic_ctx.queue_family);
 
 	if (graphic_ctx.physical_device == nullptr) {
 		EXIT("Could not find suitable device");
@@ -1121,9 +1012,6 @@ void WindowContext::CreateVulkan() {
 
 	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
 	vk::PhysicalDeviceProperties2              physical_device_properties {};
-	push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties;
-	push_descriptor_properties.pNext = nullptr;
-	physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2;
 	physical_device_properties.pNext = &push_descriptor_properties;
 	graphic_ctx.physical_device.getProperties2(&physical_device_properties);
 	graphic_ctx.physical_device_properties = physical_device_properties.properties;
@@ -1172,16 +1060,12 @@ void WindowContext::CreateVulkan() {
 		}
 	}
 
-	VulkanInitSubgroupSizeControl(graphic_ctx.physical_device, graphic_ctx);
-
-	graphic_ctx.device = VulkanCreateDevice(graphic_ctx.physical_device, r, queue_family,
-	                                        device_extensions, graphic_ctx);
+	graphic_ctx.device = VulkanCreateDevice(graphic_ctx, device_extensions);
 	if (graphic_ctx.device == nullptr) {
 		EXIT("Could not create device");
 	}
 	VULKAN_HPP_DEFAULT_DISPATCHER.init(graphic_ctx.device);
-	graphic_ctx.queue_family = queue_family;
-	graphic_ctx.device.getQueue(queue_family, 0, &graphic_ctx.queue);
+	graphic_ctx.device.getQueue(graphic_ctx.queue_family, 0, &graphic_ctx.queue);
 	EXIT_IF(graphic_ctx.queue == nullptr);
 
 	if (!graphic_ctx.CreateAllocator()) {
@@ -1189,7 +1073,7 @@ void WindowContext::CreateVulkan() {
 	}
 
 	render_context = std::make_unique<RenderContext>(graphic_ctx);
-	LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
+	LibKernel::Memory::InstallGpuResources(render_context.get());
 	presenter = std::make_unique<Presenter>(*this);
 }
 

src/graphics/rt/hardware.cpp

@@ -1,77 +0,0 @@
-#include "graphics/rt/hardware.h"
-
-#include "common/assert.h"
-#include "common/logging/log.h"
-
-#include <algorithm>
-#include <cstring>
-
-namespace Libs::Graphics {
-namespace {
-
-bool ExtensionAvailable(const std::vector<vk::ExtensionProperties>& available, const char* name) {
-	return std::ranges::any_of(available, [name](const auto& extension) {
-		return std::strcmp(extension.extensionName, name) == 0;
-	});
-}
-
-void AppendExtensionIfMissing(std::vector<const char*>& extensions, const char* name) {
-	if (std::ranges::none_of(extensions, [name](const char* extension) {
-		    return std::strcmp(extension, name) == 0;
-	    })) {
-		extensions.push_back(name);
-	}
-}
-
-} // namespace
-
-void GraphicContext::AppendHardwareRayTracingDeviceExtensions(
-    const std::vector<vk::ExtensionProperties>& available_extensions,
-    std::vector<const char*>&                   device_extensions) {
-
-	const char* required[] = {
-	    VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME,
-	    VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME,
-	    VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME,
-	    VK_KHR_RAY_QUERY_EXTENSION_NAME,
-	};
-	for (const auto* extension: required) {
-		if (!ExtensionAvailable(available_extensions, extension)) {
-			LOGF("Vulkan RT: extension %s is unavailable; hardware ray tracing disabled\n",
-			     extension);
-			rt_extensions_enabled = false;
-			return;
-		}
-	}
-	for (const auto* extension: required) {
-		AppendExtensionIfMissing(device_extensions, extension);
-	}
-	if (ExtensionAvailable(available_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME)) {
-		AppendExtensionIfMissing(device_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME);
-	}
-	if (ExtensionAvailable(available_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME)) {
-		AppendExtensionIfMissing(device_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
-	}
-
-	rt_extensions_enabled = true;
-	LOGF("Vulkan RT: enabling hardware ray query extensions\n");
-}
-
-void GraphicContext::LoadHardwareRayTracingFunctions() const {
-	EXIT_IF(device == nullptr);
-	if (!rt_extensions_enabled) {
-		return;
-	}
-
-	const auto& dispatcher = VULKAN_HPP_DEFAULT_DISPATCHER;
-	if (dispatcher.vkGetBufferDeviceAddressKHR == nullptr ||
-	    dispatcher.vkCreateAccelerationStructureKHR == nullptr ||
-	    dispatcher.vkDestroyAccelerationStructureKHR == nullptr ||
-	    dispatcher.vkGetAccelerationStructureBuildSizesKHR == nullptr ||
-	    dispatcher.vkCmdBuildAccelerationStructuresKHR == nullptr ||
-	    dispatcher.vkGetAccelerationStructureDeviceAddressKHR == nullptr) {
-		EXIT("Vulkan RT: failed to load required device functions\n");
-	}
-}
-
-} // namespace Libs::Graphics

src/graphics/rt/hardware.h

@@ -1,10 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_
-
-#include "graphics/host_gpu/graphicContext.h"
-
-#include <vector>
-
-namespace Libs::Graphics {} // namespace Libs::Graphics
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_ */

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -53,8 +53,7 @@ const char* StageName(ShaderType stage) {
 	}
 }
 
-void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase,
-                           const std::string& reason) {
+void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase) {
 	const auto* block        = cfg.FindBlock(cfg.failure_block);
 	const auto  start        = block != nullptr ? block->start_pc : UINT32_MAX;
 	const auto  end          = block != nullptr ? block->end_pc : UINT32_MAX;
@@ -68,7 +67,7 @@ void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg,
 	     CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
 	     static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
-	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
+	     static_cast<uint64_t>(cfg.back_edges.size()), cfg.unsupported_reason.c_str());
 }
 
 enum class EmbeddedFetchValueType {
@@ -130,20 +129,6 @@ uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) {
 	return inst.opcode == Decoder::Opcode::V_MAD_U64_U32 ? 2u : DecodedDstSize(inst);
 }
 
-bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::S_SETPC_B64:
-		case Decoder::Opcode::S_BRANCH:
-		case Decoder::Opcode::S_CBRANCH_SCC0:
-		case Decoder::Opcode::S_CBRANCH_SCC1:
-		case Decoder::Opcode::S_CBRANCH_VCCZ:
-		case Decoder::Opcode::S_CBRANCH_VCCNZ:
-		case Decoder::Opcode::S_CBRANCH_EXECZ:
-		case Decoder::Opcode::S_CBRANCH_EXECNZ: return true;
-		default: return false;
-	}
-}
-
 void ClearEmbeddedFetchSgprs(std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
                              const Decoder::Operand& dst, uint32_t size) {
 	if (!IsDecodedSgpr(dst)) {
@@ -242,7 +227,10 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 	EmbeddedFetchVectorLanes               vector_lanes;
 	const bool                             track_vector_lanes =
 	    std::none_of(decoded.instructions.begin(), decoded.instructions.end(),
-	                 [](const auto& inst) { return EmbeddedFetchHasBranch(inst.opcode); });
+	                 [](const auto& inst) {
+		                 return Decoder::IsDirectBranch(inst.opcode) ||
+		                        inst.opcode == Decoder::Opcode::S_SETPC_B64;
+	                 });
 
 	if (attrib_reg >= 0 && attrib_reg < static_cast<int>(sgprs.size())) {
 		sgprs[attrib_reg].type = EmbeddedFetchValueType::AttribTable;
@@ -561,27 +549,13 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
 	     static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
-	bool        dispatcher_fallback = false;
-	std::string dispatcher_reason;
 	if (cfg.irreducible) {
-		dispatcher_fallback = true;
-		dispatcher_reason   = cfg.unsupported_reason;
-		LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
+		LogDispatcherFallback(options, cfg, "build");
 	} else {
-		const auto unstructured_cfg = cfg;
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
 		if (!CFG::Structurize(cfg)) {
-			dispatcher_fallback      = true;
-			dispatcher_reason        = cfg.unsupported_reason;
-			const auto failure_kind  = cfg.failure_kind;
-			const auto failure_block = cfg.failure_block;
-			LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
-			cfg                    = unstructured_cfg;
-			cfg.unsupported        = true;
-			cfg.failure_kind       = failure_kind;
-			cfg.failure_block      = failure_block;
-			cfg.unsupported_reason = dispatcher_reason;
+			LogDispatcherFallback(options, cfg, "structurize");
 		} else {
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(cfg.blocks.size()));
@@ -593,45 +567,25 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		     static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
 	}
 
-	const ShaderVertexInputInfo*  vertex  = nullptr;
-	const ShaderPixelInputInfo*   pixel   = nullptr;
-	const ShaderComputeInputInfo* compute = nullptr;
-	switch (options.stage) {
-		case ShaderType::Vertex:
-		case ShaderType::Mesh: vertex = options.input_info.vertex; break;
-		case ShaderType::Pixel:
-			pixel = options.input_info.pixel;
-			break;
-		case ShaderType::Compute:
-			compute = options.input_info.compute;
-			break;
-		default: break;
-	}
 	EmbeddedFetchData embedded_fetch;
-	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
-		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base,
-		                                           static_cast<uint32_t>(options.user_data.size()),
-		                                           options.wave_size);
+	if (options.stage == ShaderType::Vertex && options.input_info.vertex != nullptr &&
+	    options.input_info.vertex->fetch_embedded) {
+		embedded_fetch = DetectEmbeddedVertexFetch(
+		    decoded, options.input_info.vertex, options.user_data_base,
+		    static_cast<uint32_t>(options.user_data.size()), options.wave_size);
 		if (!embedded_fetch.loads.empty()) {
 			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(embedded_fetch.loads.size()));
 		}
 	}
 	Frontend::TranslateOptions translate_options {
-	    .stage               = options.stage,
-	    .wave_size           = options.wave_size,
-	    .shader_hash         = options.shader_hash,
-	    .user_data_base      = options.user_data_base,
-	    .user_data_count     = static_cast<uint32_t>(options.user_data.size()),
-	    .scratch_dwords      = options.scratch_dwords,
-	    .dispatcher_fallback = dispatcher_fallback,
-	    .cfg_failure_kind    = cfg.failure_kind,
-	    .fallback_reason     = dispatcher_reason.empty() ? cfg.unsupported_reason
-	                                                    : dispatcher_reason,
-	    .vertex              = vertex,
-	    .pixel               = pixel,
-	    .compute             = compute,
-	    .embedded_fetch      = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
+	    .stage            = options.stage,
+	    .wave_size        = options.wave_size,
+	    .shader_hash      = options.shader_hash,
+	    .user_data_base   = options.user_data_base,
+	    .user_data_count  = static_cast<uint32_t>(options.user_data.size()),
+	    .input_info       = options.input_info,
+	    .embedded_fetch   = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
 	};
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
@@ -676,24 +630,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
 
-	const ShaderVertexInputInfo*  vertex  = nullptr;
-	const ShaderPixelInputInfo*   pixel   = nullptr;
-	const ShaderComputeInputInfo* compute = nullptr;
-	switch (options.stage) {
-		case ShaderType::Vertex:
-		case ShaderType::Mesh: vertex = options.input_info.vertex; break;
-		case ShaderType::Pixel: pixel = options.input_info.pixel; break;
-		case ShaderType::Compute: compute = options.input_info.compute; break;
-		default: EXIT("invalid shader stage\n");
-	}
-
-	IR::ShaderInfoOptions info_options;
-	info_options.vertex  = vertex;
-	info_options.pixel   = pixel;
-	info_options.compute = compute;
-	IR::CollectShaderInfo(ir, info_options);
+	IR::CollectShaderInfo(ir, options.input_info);
 	IR::AllocateBindings(ir, push_data_start_dword);
-	Spirv::AnalyzeProgramRequirements(ir);
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(translated.cfg_dump, ir);
@@ -703,11 +641,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	}
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
 	auto spirv = Spirv::EmitProgram(ir, options.input_info);
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,
 	     static_cast<uint64_t>(spirv.size()),
 	     static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
 	                               std::chrono::steady_clock::now() - emit_begin)

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -15,7 +15,6 @@ struct CompileOptions {
 	ShaderType                  stage           = ShaderType::Compute;
 	uint32_t                    wave_size       = 64;
 	uint32_t                    user_data_base  = 0;
-	uint32_t                    scratch_dwords  = 0;
 	uint64_t                    shader_hash     = 0;
 	bool                        dump_ir                    = true;
 	bool                        early_dump                 = false;

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 
-#include "common/debug.h"
+#include "common/assert.h"
 
 #include <algorithm>
 #include <cstring>
@@ -17,7 +17,7 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
 	}
 	const auto opcode     = words[0];
 	const auto word_count = static_cast<uint32_t>(words_num);
-	section.push_back((word_count << 16u) | opcode);
+	section.push_back((word_count << spv::WordCountShift) | opcode);
 	section.insert(section.end(), words + 1, words + words_num);
 }
 
@@ -36,15 +36,17 @@ void Builder::RequireVersion(uint32_t version) {
 	m_version = std::max(m_version, version);
 }
 
-void Builder::RequireCapability(uint32_t capability) {
+void Builder::RequireCapability(spv::Capability capability) {
 	if (m_required_capabilities.insert(capability).second) {
-		AddCapability({capability});
+		AppendInstruction(m_capabilities, spv::OpCapability, {capability});
 	}
 }
 
 void Builder::RequireExtension(const char* name) {
 	if (m_required_extensions.emplace(name).second) {
-		AddExtension(name);
+		std::vector<uint32_t> operands;
+		AppendString(operands, name);
+		AppendInstruction(m_extensions, spv::OpExtension, operands);
 	}
 }
 
@@ -54,15 +56,17 @@ uint32_t Builder::Import(const char* name) {
 	}
 	const auto id = AllocateId();
 	m_import_ids.emplace(name, id);
-	AddExtInstImport(id, name);
+	std::vector<uint32_t> operands = {id};
+	AppendString(operands, name);
+	AppendInstruction(m_ext_inst_imports, spv::OpExtInstImport, operands);
 	return id;
 }
 
-uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) {
+uint32_t Builder::Type(spv::Op opcode, std::initializer_list<uint32_t> operands) {
 	return Type(opcode, std::vector<uint32_t>(operands));
 }
 
-uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
+uint32_t Builder::Type(spv::Op opcode, const std::vector<uint32_t>& operands) {
 	std::vector<uint32_t> key;
 	key.reserve(operands.size() + 2u);
 	key.push_back(opcode);
@@ -79,7 +83,7 @@ uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
 	return id;
 }
 
-uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+uint32_t Builder::DecoratedType(spv::Op opcode, std::initializer_list<uint32_t> operands,
                                 std::initializer_list<TypeAnnotation> annotations) {
 	if (annotations.size() == 0) {
 		return Type(opcode, operands);
@@ -108,12 +112,12 @@ uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t>
 	return id;
 }
 
-uint32_t Builder::Constant(uint32_t opcode, uint32_t type,
+uint32_t Builder::Constant(spv::Op opcode, uint32_t type,
                            std::initializer_list<uint32_t> operands) {
 	return Constant(opcode, type, std::vector<uint32_t>(operands));
 }
 
-uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) {
+uint32_t Builder::Constant(spv::Op opcode, uint32_t type, const std::vector<uint32_t>& operands) {
 	std::vector<uint32_t> key;
 	key.reserve(operands.size() + 2u);
 	key.push_back(opcode);
@@ -130,14 +134,15 @@ uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uin
 	return id;
 }
 
-uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) {
+uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class) {
 	const auto id = AllocateId();
 	DefineGlobalVariable(id, pointer_type, storage_class);
 	return id;
 }
 
-void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) {
-	AddType({59u, pointer_type, id, storage_class});
+void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type,
+                                   spv::StorageClass storage_class) {
+	AppendInstruction(m_declarations, spv::OpVariable, {pointer_type, id, storage_class});
 }
 
 void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
@@ -156,49 +161,33 @@ void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
 	}
 }
 
-void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+void Builder::AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
                                 const std::vector<uint32_t>& operands) {
 	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
-	section.push_back((word_count << 16u) | opcode);
+	section.push_back((word_count << spv::WordCountShift) | opcode);
 	section.insert(section.end(), operands.begin(), operands.end());
 }
 
-void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+void Builder::AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
                                 std::initializer_list<uint32_t> operands) {
 	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
-	section.push_back((word_count << 16u) | opcode);
+	section.push_back((word_count << spv::WordCountShift) | opcode);
 	section.insert(section.end(), operands.begin(), operands.end());
 }
 
-void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_capabilities, 17u, operands);
-}
-
-void Builder::AddExtension(const char* name) {
-	std::vector<uint32_t> operands;
-	AppendString(operands, name);
-	AppendInstruction(m_extensions, 10u, operands);
-}
-
-void Builder::AddExtInstImport(uint32_t id, const char* name) {
-	std::vector<uint32_t> operands = {id};
-	AppendString(operands, name);
-	AppendInstruction(m_ext_inst_imports, 11u, operands);
-}
-
 void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_memory_model, 14u, operands);
+	AppendInstruction(m_memory_model, spv::OpMemoryModel, operands);
 }
 
-void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-                            const std::vector<uint32_t>& interfaces) {
+void Builder::AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point,
+                            const char* name, const std::vector<uint32_t>& interfaces) {
 	std::vector<uint32_t> operands = {execution_model, entry_point};
 	AppendString(operands, name);
 	operands.insert(operands.end(), interfaces.begin(), interfaces.end());
 	if (m_version >= 0x00010400u) {
 		for (size_t offset = 0; offset < m_declarations.size();) {
-			const auto count = m_declarations[offset] >> 16u;
-			if ((m_declarations[offset] & 0xffffu) == 59u) {
+			const auto count = m_declarations[offset] >> spv::WordCountShift;
+			if ((m_declarations[offset] & spv::OpCodeMask) == spv::OpVariable) {
 				const auto id = m_declarations[offset + 2u];
 				if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) {
 					operands.push_back(id);
@@ -207,27 +196,23 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons
 			offset += count;
 		}
 	}
-	AppendInstruction(m_entry_points, 15u, operands);
+	AppendInstruction(m_entry_points, spv::OpEntryPoint, operands);
 }
 
 void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_execution_modes, 16u, operands);
+	AppendInstruction(m_execution_modes, spv::OpExecutionMode, operands);
 }
 
 void Builder::AddName(uint32_t target, const char* name) {
 	std::vector<uint32_t> operands = {target};
 	AppendString(operands, name);
-	AppendInstruction(m_debug, 5u, operands);
+	AppendInstruction(m_debug, spv::OpName, operands);
 }
 
 void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
 	AppendInstructionWords(m_annotations, words.begin(), words.size());
 }
 
-void Builder::AddType(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_declarations, words.begin(), words.size());
-}
-
 void Builder::AddFunction(std::initializer_list<uint32_t> words) {
 	AppendInstructionWords(m_functions, words.begin(), words.size());
 }
@@ -237,16 +222,18 @@ void Builder::AddFunction(const std::vector<uint32_t>& words) {
 }
 
 DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
-	std::vector<uint32_t> words {245u, type, result};
+	std::vector<uint32_t> words {spv::OpPhi, type, result};
 	words.resize(words.size() + incoming_count * 2u);
-	const DeferredPhi phi {m_functions.size(), incoming_count};
+	const DeferredPhi phi {m_functions.size()};
 	AddFunction(words);
 	m_unpatched_phi_incomings += incoming_count;
 	return phi;
 }
 
 void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
-	EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0);
+	const auto incoming_count =
+	    ((m_functions.at(phi.word_offset) >> spv::WordCountShift) - 3u) / 2u;
+	EXIT_IF(incoming >= incoming_count || value == 0 || parent == 0);
 	const auto value_word  = phi.word_offset + 3u + incoming * 2u;
 	const auto parent_word = value_word + 1u;
 	EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
@@ -264,7 +251,7 @@ std::vector<uint32_t> Builder::Build() const {
 	               m_debug.size() + m_annotations.size() + m_declarations.size() +
 	               m_functions.size());
 
-	module.push_back(0x07230203u);
+	module.push_back(spv::MagicNumber);
 	module.push_back(m_version);
 	module.push_back(0u);
 	module.push_back(m_next_id);

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -6,19 +6,19 @@
 #include <initializer_list>
 #include <map>
 #include <set>
+#include <spirv/unified1/spirv.hpp>
 #include <string>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
 struct TypeAnnotation {
-	uint32_t              opcode = 0;
+	spv::Op               opcode = spv::OpNop;
 	std::vector<uint32_t> operands;
 };
 
 struct DeferredPhi {
 	size_t word_offset    = 0;
-	size_t incoming_count = 0;
 };
 
 class Builder {
@@ -29,22 +29,21 @@ public:
 
 	uint32_t AllocateId();
 	void     RequireVersion(uint32_t version);
-	void     RequireCapability(uint32_t capability);
+	void     RequireCapability(spv::Capability capability);
 	void     RequireExtension(const char* name);
 	uint32_t Import(const char* name);
-	uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {});
-	uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands);
-	uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+	uint32_t Type(spv::Op opcode, std::initializer_list<uint32_t> operands = {});
+	uint32_t Type(spv::Op opcode, const std::vector<uint32_t>& operands);
+	uint32_t DecoratedType(spv::Op opcode, std::initializer_list<uint32_t> operands,
 	                       std::initializer_list<TypeAnnotation> annotations);
-	uint32_t Constant(uint32_t opcode, uint32_t type,
-	                  std::initializer_list<uint32_t> operands = {});
-	uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands);
-	uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class);
-	void     DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
+	uint32_t Constant(spv::Op opcode, uint32_t type, std::initializer_list<uint32_t> operands = {});
+	uint32_t Constant(spv::Op opcode, uint32_t type, const std::vector<uint32_t>& operands);
+	uint32_t DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class);
+	void DefineGlobalVariable(uint32_t id, uint32_t pointer_type, spv::StorageClass storage_class);
 
 	void        AddMemoryModel(std::initializer_list<uint32_t> operands);
-	void        AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-	                          const std::vector<uint32_t>& interfaces);
+	void AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point, const char* name,
+	                   const std::vector<uint32_t>& interfaces);
 	void        AddExecutionMode(std::initializer_list<uint32_t> operands);
 	void        AddName(uint32_t target, const char* name);
 	void        AddAnnotation(std::initializer_list<uint32_t> words);
@@ -56,15 +55,11 @@ public:
 	[[nodiscard]] std::vector<uint32_t> Build() const;
 
 private:
-	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+	static void AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
 	                              const std::vector<uint32_t>& operands);
-	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+	static void AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
 	                              std::initializer_list<uint32_t> operands);
 	static void AppendString(std::vector<uint32_t>& words, const char* text);
-	void        AddCapability(std::initializer_list<uint32_t> operands);
-	void        AddExtension(const char* name);
-	void        AddExtInstImport(uint32_t id, const char* name);
-	void        AddType(std::initializer_list<uint32_t> words);
 
 	uint32_t                                  m_next_id = 1;
 	uint32_t                                  m_version = 0;
@@ -78,7 +73,7 @@ private:
 	std::vector<uint32_t>                     m_annotations;
 	std::vector<uint32_t>                     m_declarations;
 	std::vector<uint32_t>                     m_functions;
-	std::set<uint32_t>                        m_required_capabilities;
+	std::set<spv::Capability>                 m_required_capabilities;
 	std::set<std::string>                     m_required_extensions;
 	std::map<std::string, uint32_t>           m_import_ids;
 	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -183,10 +183,8 @@ void ValidateNativeProgram(const IR::Program& program) {
 
 } // namespace
 
-void AnalyzeProgramRequirements(IR::Program& program) {
-	program.spirv_requirements.reset();
-	IR::SpirvRequirements requirements {};
-	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
+Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) {
+	SpirvRequirements requirements {};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
@@ -242,17 +240,17 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 				}
 				if (shared_access == IR::SharedAccess::Append ||
 				    shared_access == IR::SharedAccess::Consume) {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 				}
 			}
 			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::Ballot: MarkBallot(); break;
+				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
 				case IR::ValueOpcode::ReadFirstLane:
 				case IR::ValueOpcode::ReadLane: {
-					MarkBallot();
+					requirements.subgroup_ballot  = true;
 					requirements.subgroup_shuffle = true;
 					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
 						requirements.subgroup_local_invocation_id = true;
@@ -261,19 +259,19 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 				}
 				case IR::ValueOpcode::DppUpdateU32:
 				case IR::ValueOpcode::WriteLane: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::Permlane16U32: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::SwizzleU32:
 				case IR::ValueOpcode::BpermuteU32: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
@@ -300,7 +298,7 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 			}
 		}
 	}
-	program.spirv_requirements.emplace(requirements);
+	return requirements;
 }
 
 std::vector<uint32_t> EmitProgram(const IR::Program& program,
@@ -312,35 +310,23 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
 		Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
 	}
 	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete ||
-	    !program.spirv_requirements.has_value()) {
+	    !program.shader_info_complete || !program.binding_layout_complete) {
 		Fail(program, "SPIR-V emitter requires a fully planned native shader program");
 	}
 	ValidateNativeProgram(program);
 	IR::ValidateProgram(program, true);
 	EmitterState state(program, input_info);
-	state.stage = program.stage;
 	const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
 	state.lane_count =
 	    workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u
 	        ? 2u
 	        : 1u;
-	state.inputs.reserve(program.info.inputs.size());
-	state.outputs.reserve(program.info.outputs.size());
-	state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
-	CopyProgramInputsAndOutputs(state, program);
-	AllocateInputVariables(state);
-	AllocateOutputVariables(state);
 	DefineModule(state);
 	EmitProgram(state);
-	state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
-	                            state.interface_variables);
+	state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func,
+	                            "main", state.interface_variables);
 
-	auto binary = state.builder.Build();
-	if (binary.empty()) {
-		Fail(program, "SPIR-V builder returned an empty module");
-	}
-	return binary;
+	return state.builder.Build();
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,8 +9,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-void AnalyzeProgramRequirements(IR::Program& program);
-
 std::vector<uint32_t> EmitProgram(const IR::Program& program,
                                   ShaderStageInputInfo input_info);
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -2,97 +2,17 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
-                            uint32_t false_value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value});
-	return ret;
-}
-
-uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
-
-void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
-                                   uint32_t& out_low, uint32_t& out_high) {
-	const auto amount           = EmitAndConstant(state, shift, 63u);
-	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
-	const auto less32           = state.builder.AllocateId();
-	const auto non_zero         = state.builder.AllocateId();
-	const auto carry_count_base = state.builder.AllocateId();
-	const auto carry_count      = state.builder.AllocateId();
-	const auto low_part         = state.builder.AllocateId();
-	const auto high_part        = state.builder.AllocateId();
-	const auto carry_part       = state.builder.AllocateId();
-	const auto carry            = state.builder.AllocateId();
-	const auto high_less        = state.builder.AllocateId();
-	const auto high_ge          = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
-	state.builder.AddFunction(
-	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
-	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
-	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
-}
-
-void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
-                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
-	const auto amount           = EmitAndConstant(state, shift, 63u);
-	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
-	const auto less32           = state.builder.AllocateId();
-	const auto non_zero         = state.builder.AllocateId();
-	const auto carry_count_base = state.builder.AllocateId();
-	const auto carry_count      = state.builder.AllocateId();
-	const auto low_part         = state.builder.AllocateId();
-	const auto high_part        = state.builder.AllocateId();
-	const auto carry_part       = state.builder.AllocateId();
-	const auto carry            = state.builder.AllocateId();
-	const auto low_less         = state.builder.AllocateId();
-	const auto low_ge           = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
-	state.builder.AddFunction(
-	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
-	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
-	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
-}
-
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
 	return ret;
 }
 
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
-	return ret;
-}
-
-uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
+	    {spv::OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
 	return ret;
 }
 
@@ -105,13 +25,14 @@ uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t v
 
 uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
+	state.builder.AddFunction(
+	    {spv::OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
 	return ret;
 }
 
 uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32});
+	state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, f32});
 
 	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
 	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
@@ -122,17 +43,18 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
 	const auto normal_payload = state.builder.AllocateId();
 	const auto normal         = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
+	    {spv::OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
+	    {spv::OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
 
 	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
 	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
-	const auto exp_lt_103       = EmitCompareU32Constant(state, OpULessThan, exp, 103);
-	const auto exp_gt_112       = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112);
+	const auto exp_lt_103       = EmitCompareU32Constant(state, spv::OpULessThan, exp, 103);
+	const auto exp_gt_112       = EmitCompareU32Constant(state, spv::OpUGreaterThan, exp, 112);
 	const auto sub_shift_low =
 	    EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
 	const auto sub_shift =
@@ -140,8 +62,8 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto sub_mant  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
+	    {spv::OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
 
 	const auto nan_payload =
 	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
@@ -149,12 +71,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	    EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
 	const auto inf        = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
 	const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
-	const auto mant_zero  = EmitCompareU32Constant(state, OpIEqual, mant, 0);
+	const auto mant_zero  = EmitCompareU32Constant(state, spv::OpIEqual, mant, 0);
 	const auto special    = EmitSelectValueU32(state, mant_zero, inf, nan);
 
-	const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112);
-	const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143);
-	const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255);
+	const auto exp_le_112 = EmitCompareU32Constant(state, spv::OpULessThanEqual, exp, 112);
+	const auto exp_ge_143 = EmitCompareU32Constant(state, spv::OpUGreaterThanEqual, exp, 143);
+	const auto exp_eq_255 = EmitCompareU32Constant(state, spv::OpIEqual, exp, 255);
 	const auto finite0    = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
 	const auto finite1    = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
 	const auto finite2    = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
@@ -165,8 +87,8 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
+	    {max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs});
 	return ret;
 }
 
@@ -174,44 +96,8 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value});
+	    {max_value ? spv::OpSGreaterThan : spv::OpSLessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs});
 	return ret;
 }
 
@@ -221,10 +107,11 @@ F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
 	const auto abs_bits      = EmitAndConstant(state, cls.bits, 0x7fffffffu);
 	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
-	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
-	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
+	const auto exponent_max =
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
+	const auto mantissa_nonzero = EmitCompareU32Constant(state, spv::OpINotEqual, mantissa_bits, 0);
 	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
-	cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	cls.zero                    = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 	return cls;
 }
 
@@ -235,7 +122,7 @@ F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
 uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
                                uint32_t class_match) {
 	const auto selected =
-	    EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
+	    EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
 	return EmitLogicalAndBool(state, selected, class_match);
 }
 
@@ -247,15 +134,16 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
 	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
 
-	const auto sign             = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0);
+	const auto sign             = EmitCompareU32Constant(state, spv::OpINotEqual, sign_bits, 0);
 	const auto positive         = EmitLogicalNotBool(state, sign);
-	const auto exponent_zero    = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0);
+	const auto exponent_zero    = EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0);
 	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
-	const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto exponent_inf =
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
 	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
-	const auto mantissa_zero    = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0);
+	const auto mantissa_zero    = EmitCompareU32Constant(state, spv::OpIEqual, mantissa_bits, 0);
 	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
-	const auto quiet            = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0);
+	const auto quiet            = EmitCompareU32Constant(state, spv::OpINotEqual, quiet_bits, 0);
 
 	const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
 	const auto snan       = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
@@ -263,7 +151,7 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto inf        = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
 	const auto normal     = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
 	const auto denorm     = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
-	const auto zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	const auto zero       = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 
 	uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
 	match          = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
@@ -296,8 +184,8 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto rhs_class = EmitClassifyF32(state, rhs);
 
 	const auto numeric_cond = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state),
-	                           numeric_cond, lhs, rhs});
+	state.builder.AddFunction({max_value ? spv::OpFOrdGreaterThanEqual : spv::OpFOrdLessThan,
+	                           TypeBool(state), numeric_cond, lhs, rhs});
 	const auto ordered_bits =
 	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
 
@@ -312,13 +200,6 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	return EmitBitcastU32ToF32(state, result_bits);
 }
 
-uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value});
-	return ret;
-}
-
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto bits      = state.builder.AllocateId();
 	const auto abs_bits  = state.builder.AllocateId();
@@ -328,18 +209,18 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	    {spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
+	    {spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
 	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
+	    {spv::OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
-	state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
-	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected});
+	    {spv::OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
+	state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
+	state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
+	state.builder.AddFunction({spv::OpBitcast, TypeF32(state), ret, selected});
 	return ret;
 }
 
@@ -352,45 +233,34 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
 	const auto large_finite = state.builder.AllocateId();
 	const auto reduced      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src});
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src});
+	    {spv::OpExtInst, TypeF32(state), fract, GlslStd450(state), GLSLstd450Fract, src});
+	state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, src});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	    {spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	state.builder.AddFunction({spv::OpUGreaterThanEqual, TypeBool(state), large, abs_bits,
+	                           ConstantU32(state, 0x4b000000u)});
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)});
+	    {spv::OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
+	state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), large_finite, large, finite});
 	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
-	state.builder.AddFunction(
-	    {OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
+	    {spv::OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
 	if (!preserve_signed_zero) {
 		return reduced;
 	}
 	const auto zero = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced});
-	return ret;
-}
-
-uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value});
+	state.builder.AddFunction(
+	    {spv::OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpSelect, TypeF32(state), ret, zero, src, reduced});
 	return ret;
 }
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
-	                           GlslUnpackHalf2x16, bits});
-	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
+	state.builder.AddFunction({spv::OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
+	                           GLSLstd450UnpackHalf2x16, bits});
+	state.builder.AddFunction({spv::OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -14,42 +14,22 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
 			active_inputs[active_count++] = input.location;
 		}
 	}
-	return state.stage == ShaderType::Pixel
+	return state.program.stage == ShaderType::Pixel
 	           ? ShaderPixelParameterLocation(*state.input_info.pixel,
 	                                          {active_inputs.data(), active_count}, attr)
 	           : attr;
 }
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
-	return state.stage == ShaderType::Pixel &&
+	return state.program.stage == ShaderType::Pixel &&
 	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
 }
 
 bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
-	return state.stage == ShaderType::Pixel &&
+	return state.program.stage == ShaderType::Pixel &&
 	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
 }
 
-bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
-               uint32_t index) {
-	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
-		return binding.kind == kind && binding.index == index;
-	});
-}
-
-void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
-	for (const auto& input: program.info.inputs) {
-		state.inputs.push_back({input.kind, input.location, input.component_count, 0,
-		                        input.debug_name, input.per_vertex});
-	}
-	for (const auto& output: program.info.outputs) {
-		if (HasOutput(state.outputs, output.kind, output.index)) {
-			continue;
-		}
-		state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name});
-	}
-}
-
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
 	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
 		return state.per_vertex_variable;
@@ -74,7 +54,7 @@ uint32_t          ConstantU32(EmitterState& state, uint32_t value);
                                                const char* reason) {
 	EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
 	     " stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
-	     state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
+	     state.program.shader_hash, static_cast<unsigned>(state.program.stage), resource,
 	     static_cast<unsigned>(kind), reason);
 	std::abort();
 }
@@ -102,8 +82,8 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
 		ExitDescriptorBindingFailure(state, kind, resource, variable_name);
 	}
 	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
+	state.builder.AddFunction({spv::OpAccessChain, result_ptr_type, pointer, variable_id,
+	                           ConstantU32(state, array_index)});
 	return pointer;
 }
 
@@ -129,12 +109,13 @@ uint32_t ImageScalarType(EmitterState& state, Prospero::TextureNumericClass nume
 
 uint32_t ImageVectorType(EmitterState& state, Prospero::TextureNumericClass numeric_class,
                          uint32_t components) {
-	return state.builder.Type(OpTypeVector, {ImageScalarType(state, numeric_class), components});
+	return state.builder.Type(spv::OpTypeVector,
+	                          {ImageScalarType(state, numeric_class), components});
 }
 
 uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 	uint32_t sampled = 0;
-	uint32_t format  = ImageFormatUnknown;
+	uint32_t format  = spv::ImageFormatUnknown;
 	if (image.resource_class == IR::ImageResourceClass::Sampled) {
 		EXIT_IF(image.atomic);
 		sampled = 1;
@@ -144,16 +125,16 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 		sampled = 2;
 		if (image.atomic) {
 			EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint);
-			format = ImageFormatR32ui;
+			format = spv::ImageFormatR32ui;
 		}
 	} else {
 		EXIT("invalid image resource class");
 	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
-	return state.builder.Type(OpTypeImage,
+	return state.builder.Type(spv::OpTypeImage,
 	                          {ImageScalarType(state, image.numeric_class), info.spirv_dimension,
-	                           image.depth_compare ? 1u : 0u,
-	                           info.arrayed, info.multisampled, sampled, format});
+	                           image.depth_compare ? 1u : 0u, info.arrayed, info.multisampled,
+	                           sampled, format});
 }
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
@@ -173,26 +154,26 @@ uint32_t LoadSampledImageDescriptor(EmitterState& state, uint32_t resource) {
 	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
 	const auto variable     = state.image_variables[IR::ImageBindingIndex(*kind)];
 	const auto pointer_type = state.builder.Type(
-	    OpTypePointer, {StorageClassUniformConstant, ImageType(state, image_resource)});
+	    spv::OpTypePointer, {spv::StorageClassUniformConstant, ImageType(state, image_resource)});
 	const auto pointer =
 	    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 	                             "sampled image descriptor array was not emitted");
 	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, ImageType(state, image_resource), image, pointer});
+	state.builder.AddFunction({spv::OpLoad, ImageType(state, image_resource), image, pointer});
 	return image;
 }
 
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
 	const auto array_index =
 	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
-	const auto sampler_type = state.builder.Type(OpTypeSampler);
+	const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
 	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type});
+	    state.builder.Type(spv::OpTypePointer, {spv::StorageClassUniformConstant, sampler_type});
 	const auto pointer = DescriptorElementPointer(
 	    state, pointer_type, state.sampler_variable, array_index,
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer});
+	state.builder.AddFunction({spv::OpLoad, sampler_type, sampler_id, pointer});
 	return sampler_id;
 }
 
@@ -201,9 +182,10 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
 	const auto  image          = LoadSampledImageDescriptor(state, resource);
 	const auto  sampler_id     = LoadSamplerDescriptor(state, sampler);
 	const auto  sampled_image = state.builder.AllocateId();
-	const auto  sampled_type  =
-	    state.builder.Type(OpTypeSampledImage, {ImageType(state, image_resource)});
-	state.builder.AddFunction({OpSampledImage, sampled_type, sampled_image, image, sampler_id});
+	const auto  sampled_type =
+	    state.builder.Type(spv::OpTypeSampledImage, {ImageType(state, image_resource)});
+	state.builder.AddFunction(
+	    {spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id});
 	return sampled_image;
 }
 
@@ -213,8 +195,8 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource) {
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
 	const auto array_index = ResourceForDescriptor(state, *kind, resource);
-	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, ImageType(state, image)});
+	const auto pointer_type = state.builder.Type(
+	    spv::OpTypePointer, {spv::StorageClassUniformConstant, ImageType(state, image)});
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 	                                "storage image descriptor array was not emitted");
@@ -225,25 +207,25 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
 	if (!image.atomic) {
-		state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat);
+		state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
 	}
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
 	const auto array_index = ResourceForDescriptor(state, *kind, resource);
 	const auto image_type  = ImageType(state, image);
 	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, image_type});
+	    state.builder.Type(spv::OpTypePointer, {spv::StorageClassUniformConstant, image_type});
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	const auto LoadAt   = [&](uint32_t array_index) {
 		const auto pointer =
 		    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 		                             "storage image descriptor array was not emitted");
 		const auto descriptor = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, image_type, descriptor, pointer});
+		state.builder.AddFunction({spv::OpLoad, image_type, descriptor, pointer});
 		return descriptor;
 	};
 	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
-		state.builder.AddFunction({OpImageWrite, LoadAt(array_index), coord, texel});
+		state.builder.AddFunction({spv::OpImageWrite, LoadAt(array_index), coord, texel});
 		return;
 	}
 	if (image.mip_count == 0u) {
@@ -253,28 +235,28 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 
 	const auto            merge_label = state.builder.AllocateId();
 	std::vector<uint32_t> labels(image.mip_count);
-	std::vector<uint32_t> words {OpSwitch, mip_lod, merge_label};
+	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
 		labels[mip] = state.builder.AllocateId();
 		words.push_back(mip);
 		words.push_back(labels[mip]);
 	}
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
 	state.builder.AddFunction(words);
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
 		EmitLabel(state, labels[mip]);
-		state.builder.AddFunction({OpImageWrite, LoadAt(array_index + mip), coord, texel});
-		state.builder.AddFunction({OpBranch, merge_label});
+		state.builder.AddFunction({spv::OpImageWrite, LoadAt(array_index + mip), coord, texel});
+		state.builder.AddFunction({spv::OpBranch, merge_label});
 	}
 	EmitLabel(state, merge_label);
 }
 
-uint32_t ExecutionModelForStage(ShaderType stage) {
+spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {
 	switch (stage) {
-		case ShaderType::Vertex: return ExecutionModelVertex;
-		case ShaderType::Mesh: return 5365u; // MeshEXT
-		case ShaderType::Pixel: return ExecutionModelFragment;
-		default: return ExecutionModelGLCompute;
+		case ShaderType::Vertex: return spv::ExecutionModelVertex;
+		case ShaderType::Mesh: return spv::ExecutionModelMeshEXT; // MeshEXT
+		case ShaderType::Pixel: return spv::ExecutionModelFragment;
+		default: return spv::ExecutionModelGLCompute;
 	}
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include <algorithm>
 
@@ -16,48 +16,27 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
 	return true;
 }
 
-uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
-	const auto shifted_one = state.builder.AllocateId();
-	const auto merged_one  = state.builder.AllocateId();
-	const auto shifted_two = state.builder.AllocateId();
-	const auto merged_two  = state.builder.AllocateId();
-	const auto quad_bits   = state.builder.AllocateId();
-	const auto result      = state.builder.AllocateId();
-	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
-	                           ConstantU64(state, 0x0000000100000001ull)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
-	                           ConstantU64(state, 0x0000000200000002ull)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
-	                           ConstantU64(state, 0x1111111111111111ull)});
-	state.builder.AddFunction(
-	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
-	return result;
-}
-
-uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
-	auto& state = ctx.state;
+uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
 	if (kind == IR::StageInputKind::LocalInvocationIndex) {
 		return EmitLocalInvocationIndex(state);
 	}
 	if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId ||
 	                              kind == IR::StageInputKind::GlobalInvocationId)) {
-		const auto* cs      = ShaderWorkgroupInput(state.stage, state.input_info);
+		const auto* cs      = ShaderWorkgroupInput(state.program.stage, state.input_info);
 		uint32_t    divisor = 1;
 		for (uint32_t axis = 0; axis < component; axis++) {
 			divisor *= std::max(cs->threads_num[axis], 1u);
 		}
 		const auto size    = std::max(cs->threads_num[component], 1u);
-		const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state),
+		const auto divided = EmitBinaryU32(state, spv::OpUDiv, EmitLocalInvocationIndex(state),
 		                                   ConstantU32(state, divisor));
-		const auto local   = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size));
+		const auto local   = EmitBinaryU32(state, spv::OpUMod, divided, ConstantU32(state, size));
 		if (kind == IR::StageInputKind::LocalInvocationId) {
 			return local;
 		}
 		const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component);
 		return EmitAddU32(state, local,
-		                  EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size)));
+		                  EmitBinaryU32(state, spv::OpIMul, group, ConstantU32(state, size)));
 	}
 	const auto variable = InputVariableForKind(state, kind);
 	if (variable == 0) {
@@ -66,28 +45,28 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 	if (kind == IR::StageInputKind::FrontFacing) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, TypeBool(state), value, variable});
-		state.builder.AddFunction(
-		    {OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		state.builder.AddFunction({spv::OpLoad, TypeBool(state), value, variable});
+		state.builder.AddFunction({spv::OpSelect, TypeU32(state), bits, value,
+		                           ConstantU32(state, 1), ConstantU32(state, 0)});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
 	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, TypeI32(state), value, variable});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction({spv::OpLoad, TypeI32(state), value, variable});
+		state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::FragCoord) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain,
-		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		                           variable, ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction({spv::OpAccessChain,
+		                           TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                           pointer, variable, ConstantU32(state, component)});
+		state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::BaryCoordSmooth ||
@@ -95,11 +74,11 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain,
-		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		                           variable, ConstantU32(state, component + 1u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction({spv::OpAccessChain,
+		                           TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                           pointer, variable, ConstantU32(state, component + 1u)});
+		state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	return EmitInputComponentU32(state, kind, component);
@@ -121,54 +100,55 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	const auto row_ok     = state.builder.AllocateId();
 	const auto masks_ok   = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
+	    {spv::OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
+	    {spv::OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
+	    {spv::OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
+	    {spv::OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
+	    {spv::OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
+	    {spv::OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
+	state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), bank_hit,
+	                           ConstantU32(state, flags.bank_mask), bank_bit});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
+	    {spv::OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
+	    {spv::OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
+	    {spv::OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
 	uint32_t writable = masks_ok;
 	if (!flags.bound_control) {
 		const auto target  = EmitDppTargetLane(state, flags.control);
 		const auto bounded = state.builder.AllocateId();
-		state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
+		state.builder.AddFunction(
+		    {spv::OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
 		writable = bounded;
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable});
+	state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), result, exec, writable});
 	return result;
 }
 
-uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
-	auto&       state = ctx.state;
+uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
 	const auto* input = InputBindingForParameter(state, attr);
 	if (input == nullptr || input->variable_id == 0) {
 		return ConstantU32(state, 0);
 	}
-	if (state.stage == ShaderType::Vertex) {
+	if (state.program.stage == ShaderType::Vertex) {
 		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
 	}
 	const auto load_per_vertex = [&](uint32_t vertex) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({spv::OpAccessChain,
+		                           TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                           pointer, input->variable_id, ConstantU32(state, vertex),
+		                           ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
 		return value;
 	};
 	if (input->per_vertex) {
@@ -181,30 +161,31 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
 			const auto pointer = state.builder.AllocateId();
 			const auto weight  = state.builder.AllocateId();
 			const auto product = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain,
-			                           TypePointer(state, StorageClassInput, TypeF32(state)),
+			state.builder.AddFunction({spv::OpAccessChain,
+			                           TypePointer(state, spv::StorageClassInput, TypeF32(state)),
 			                           pointer, barycentric, ConstantU32(state, vertex)});
-			state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer});
+			state.builder.AddFunction({spv::OpLoad, TypeF32(state), weight, pointer});
 			state.builder.AddFunction(
-			    {OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
+			    {spv::OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
 			if (vertex == 0u) {
 				sum = product;
 			} else {
 				const auto next = state.builder.AllocateId();
-				state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product});
+				state.builder.AddFunction({spv::OpFAdd, TypeF32(state), next, sum, product});
 				sum = next;
 			}
 		}
 		const auto bits = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum});
+		state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, sum});
 		return bits;
 	}
 	const auto vector    = state.builder.AllocateId();
 	const auto component = state.builder.AllocateId();
 	const auto bits      = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
-	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component});
+	state.builder.AddFunction({spv::OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
+	state.builder.AddFunction(
+	    {spv::OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
+	state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, component});
 	return bits;
 }
 
@@ -213,15 +194,16 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	auto&       state = ctx.state;
 	const auto* input = InputBindingForParameter(state, attr);
 	if (!input->per_vertex) {
-		return EmitAttribute(ctx, attr, chan);
+		return EmitAttribute(ctx.state, attr, chan);
 	}
 	const auto load_vertex = [&](uint32_t vertex) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({spv::OpAccessChain,
+		                           TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                           pointer, input->variable_id, ConstantU32(state, vertex),
+		                           ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
 		return value;
 	};
 
@@ -229,16 +211,16 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	uint32_t   value           = load_vertex(selected_vertex);
 	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
 		const auto delta = state.builder.AllocateId();
-		state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)});
+		state.builder.AddFunction({spv::OpFSub, TypeF32(state), delta, value, load_vertex(0)});
 		value = delta;
 	}
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
 	return bits;
 }
 
 uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
-	if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
+	if (state.program.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
 	    exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
 		return 0;
 	}
@@ -248,7 +230,7 @@ uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
 uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
 	const auto value = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(
-	    {OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
+	    {spv::OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
 	return value;
 }
 
@@ -257,7 +239,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	auto& state = ctx.state;
 	if (exp.compr && !uint_output) {
 		const auto unpack =
-		    MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
+		    MrtOutputMode(state, exp) == 5u ? GLSLstd450UnpackUnorm2x16 : GLSLstd450UnpackHalf2x16;
 		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
 		                   ConstantF32(state, 0x3f800000u)};
 		for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -266,21 +248,22 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 			}
 			const auto packed   = state.builder.AllocateId();
 			const auto unpacked = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
 			state.builder.AddFunction(
-			    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed});
+			    {spv::OpCompositeExtract, TypeU32(state), packed, data, pair});
+			state.builder.AddFunction({spv::OpExtInst, TypeF32Vector(state, 2), unpacked,
+			                           GlslStd450(state), unpack, packed});
 			for (uint32_t lane = 0; lane < 2u; lane++) {
 				const auto component = pair * 2u + lane;
 				if (((exp.en >> component) & 1u) != 0u) {
 					f32[component] = state.builder.AllocateId();
 					state.builder.AddFunction(
-					    {OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
+					    {spv::OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
 				}
 			}
 		}
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
-		                           f32[1], f32[2], f32[3]});
+		state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector,
+		                           f32[0], f32[1], f32[2], f32[3]});
 		return vector;
 	}
 	uint32_t raw[4] = {
@@ -301,7 +284,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 					continue;
 				}
 				raw[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component],
+				state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(state), raw[component],
 				                           packed, ConstantU32(state, lane * 16u),
 				                           ConstantU32(state, 16)});
 			}
@@ -315,18 +298,18 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	}
 	if (uint_output) {
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0],
-		                           raw[1], raw[2], raw[3]});
+		state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(state, 4), vector,
+		                           raw[0], raw[1], raw[2], raw[3]});
 		return vector;
 	}
 	uint32_t f32[4] {};
 	for (uint32_t component = 0; component < 4u; component++) {
 		f32[component] = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]});
+		state.builder.AddFunction({spv::OpBitcast, TypeF32(state), f32[component], raw[component]});
 	}
 	const auto vector = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]});
+	state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
+	                           f32[1], f32[2], f32[3]});
 	return vector;
 }
 
@@ -342,19 +325,19 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 			const auto raw = ExportRawComponent(ctx, data, component);
 			if (output.layer) {
 				const auto layer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
-				                           ConstantU32(state, 0x7ffu)});
-				const auto pointer = state.stage == ShaderType::Mesh
+				state.builder.AddFunction(
+				    {spv::OpBitwiseAnd, TypeU32(state), layer, raw, ConstantU32(state, 0x7ffu)});
+				const auto pointer = state.program.stage == ShaderType::Mesh
 				                         ? MeshOutputPointer(state, IR::StageOutputKind::Layer)
 				                         : state.layer_variable;
-				state.builder.AddFunction({OpStore, pointer, layer});
+				state.builder.AddFunction({spv::OpStore, pointer, layer});
 			}
 			if (output.viewport) {
 				// GFX10 MISC.z packs the viewport index in bits 16..19 alongside the layer.
 				const auto viewport = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), viewport, raw,
+				state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(state), viewport, raw,
 				                           ConstantU32(state, 16), ConstantU32(state, 4)});
-				state.builder.AddFunction({OpStore, state.viewport_index_variable, viewport});
+				state.builder.AddFunction({spv::OpStore, state.viewport_index_variable, viewport});
 			}
 			continue;
 		}
@@ -365,9 +348,9 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 
 		const auto raw = ExportRawComponent(ctx, data, component);
 		const auto f32 = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+		state.builder.AddFunction({spv::OpBitcast, TypeF32(state), f32, raw});
 		if (output.point_size) {
-			state.builder.AddFunction({OpStore, state.point_size_variable, f32});
+			state.builder.AddFunction({spv::OpStore, state.point_size_variable, f32});
 			continue;
 		}
 		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
@@ -375,10 +358,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 				return;
 			}
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer,
-			     variable, ConstantU32(state, index)});
-			state.builder.AddFunction({OpStore, pointer, f32});
+			state.builder.AddFunction({spv::OpAccessChain,
+			                           TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+			                           pointer, variable, ConstantU32(state, index)});
+			state.builder.AddFunction({spv::OpStore, pointer, f32});
 		};
 		StoreDistance(state.clip_distance_variable, output.clip_distance);
 		StoreDistance(state.cull_distance_variable, output.cull_distance);
@@ -392,13 +375,13 @@ uint32_t ConvertClipCoordinate(EmitterState& state, uint32_t coordinate, float s
 	const auto divided = state.builder.AllocateId();
 	const auto ndc     = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpFMul, TypeF32(state), window, coordinate, ConstantF32Value(state, scale)});
+	    {spv::OpFMul, TypeF32(state), window, coordinate, ConstantF32Value(state, scale)});
 	state.builder.AddFunction(
-	    {OpFAdd, TypeF32(state), biased, window, ConstantF32Value(state, offset)});
+	    {spv::OpFAdd, TypeF32(state), biased, window, ConstantF32Value(state, offset)});
 	state.builder.AddFunction(
-	    {OpFDiv, TypeF32(state), divided, biased, ConstantF32Value(state, half_extent)});
+	    {spv::OpFDiv, TypeF32(state), divided, biased, ConstantF32Value(state, half_extent)});
 	state.builder.AddFunction(
-	    {OpFSub, TypeF32(state), ndc, divided, ConstantF32Value(state, 1.0f)});
+	    {spv::OpFSub, TypeF32(state), ndc, divided, ConstantF32Value(state, 1.0f)});
 	return ndc;
 }
 
@@ -408,28 +391,50 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) {
 	for (uint32_t i = 0; i < 4; i++) {
 		components[i] = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpCompositeExtract, TypeF32(state), components[i], position, i});
+		    {spv::OpCompositeExtract, TypeF32(state), components[i], position, i});
 	}
 	components[0] = ConvertClipCoordinate(state, components[0], transform.scale[0],
 	                                      transform.offset[0], transform.half_extent[0]);
 	components[1] = ConvertClipCoordinate(state, components[1], transform.scale[1],
 	                                      transform.offset[1], transform.half_extent[1]);
 	const auto converted = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), converted,
+	state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 4), converted,
 	                           components[0], components[1], components[2], components[3]});
 	return converted;
 }
 
-void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
+} // namespace
+uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
+	                           ConstantU64(state, 0x0000000100000001ull)});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
+	state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                           ConstantU64(state, 0x0000000200000002ull)});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                           ConstantU64(state, 0x1111111111111111ull)});
+	state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits,
+	                           ConstantU64(state, 0x0000000f0000000full)});
+	return result;
+}
+
+void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&       state = ctx.state;
 	const auto& exp   = ctx.Export(inst);
 	const auto  exec  = ctx.Arg(inst, 1);
-	if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
+	if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
 	    state.pixel_valid_mask_variable != 0) {
 		const auto value = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
-		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
+		state.builder.AddFunction({spv::OpSelect, TypeU32(state), value, exec,
+		                           ConstantU32(state, 1), ConstantU32(state, 0)});
+		state.builder.AddFunction({spv::OpStore, state.pixel_valid_mask_variable, value});
 	}
 	if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
 		return;
@@ -437,9 +442,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	EmitIfCondition(state, exec, [&]() {
 		const auto data = ctx.Arg(inst, 0);
 		if (exp.kind == IR::ExportTargetKind::Primitive) {
-			if (state.stage == ShaderType::Mesh) {
+			if (state.program.stage == ShaderType::Mesh) {
 				state.builder.AddFunction(
-				    {OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
+				    {spv::OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
 			}
 			return;
 		}
@@ -451,35 +456,36 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
 				const auto raw = ExportRawComponent(ctx, data, 0);
 				const auto f32 = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
-				state.builder.AddFunction({OpStore, state.depth_variable, f32});
+				state.builder.AddFunction({spv::OpBitcast, TypeF32(state), f32, raw});
+				state.builder.AddFunction({spv::OpStore, state.depth_variable, f32});
 			}
 			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
 				const auto raw     = ExportRawComponent(ctx, data, 2);
 				const auto value   = state.builder.AllocateId();
 				const auto pointer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw});
+				state.builder.AddFunction({spv::OpBitcast, TypeI32(state), value, raw});
 				state.builder.AddFunction(
-				    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer,
+				    {spv::OpAccessChain,
+				     TypePointer(state, spv::StorageClassOutput, TypeI32(state)), pointer,
 				     state.sample_mask_variable, ConstantU32(state, 0)});
-				state.builder.AddFunction({OpStore, pointer, value});
+				state.builder.AddFunction({spv::OpStore, pointer, value});
 			}
 			return;
 		}
 		const auto variable =
-		    state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
-		if (state.stage != ShaderType::Mesh && variable == 0) {
+		    state.program.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
+		if (state.program.stage != ShaderType::Mesh && variable == 0) {
 			return;
 		}
 		const bool uint_output = MrtOutputMode(state, exp) == 7u;
 		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
-		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
 		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
 			const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];
 			if (!mapping.IsIdentity()) {
 				const auto mapped = state.builder.AllocateId();
-				state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
+				state.builder.AddFunction({spv::OpVectorShuffle, vector_type, mapped, value, value,
 				                           mapping.Map(0), mapping.Map(1), mapping.Map(2),
 				                           mapping.Map(3)});
 				value = mapped;
@@ -489,192 +495,184 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		    state.input_info.vertex->clip_space.enabled) {
 			value = ConvertPositionToClipSpace(state, value);
 		}
-		if (state.stage == ShaderType::Mesh) {
+		if (state.program.stage == ShaderType::Mesh) {
 			const auto kind = exp.kind == IR::ExportTargetKind::Position
 			                      ? IR::StageOutputKind::Position
 			                      : IR::StageOutputKind::Parameter;
-			state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value});
+			state.builder.AddFunction(
+			    {spv::OpStore, MeshOutputPointer(state, kind, exp.index), value});
 		} else if (exp.kind == IR::ExportTargetKind::Position) {
 			const auto pointer = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
-			     pointer, variable, ConstantU32(state, 0)});
-			state.builder.AddFunction({OpStore, pointer, value});
+			    {spv::OpAccessChain,
+			     TypePointer(state, spv::StorageClassOutput, TypeF32Vector(state, 4)), pointer,
+			     variable, ConstantU32(state, 0)});
+			state.builder.AddFunction({spv::OpStore, pointer, value});
 		} else {
-			state.builder.AddFunction({OpStore, variable, value});
+			state.builder.AddFunction({spv::OpStore, variable, value});
 		}
 	});
 }
 
-} // namespace
+uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
+	return value;
+}
 
-bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
-	auto& state = ctx.state;
-	switch (inst.GetOpcode()) {
-		case IR::ValueOpcode::Identity: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
-		case IR::ValueOpcode::Void:
-		case IR::ValueOpcode::Reference:
-		case IR::ValueOpcode::ReferenceU32:
-		case IR::ValueOpcode::ControlNop:
-		case IR::ValueOpcode::Waitcnt:
-		case IR::ValueOpcode::Sendmsg:
-		case IR::ValueOpcode::TtraceData:
-		case IR::ValueOpcode::InstPrefetch: return true;
-		case IR::ValueOpcode::Barrier: {
-			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
-			state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
-			                           ConstantU32(state, ScopeWorkgroup),
-			                           ConstantU32(state, semantics)});
-			return true;
-		}
-		case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true;
-		case IR::ValueOpcode::MeshDrawParameter: {
-			const auto index = inst.Arg(0).U32();
-			if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
-				ctx.Fail(inst, "invalid mesh draw parameter");
-			}
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state),
-			                           pointer, state.push_constant_variable, ConstantU32(state, 0),
-			                           ConstantU32(state, index)});
-			state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer});
-			return true;
-		}
-		case IR::ValueOpcode::GetUserData: {
-			const auto reg   = inst.Arg(0).ScalarRegister();
-			uint32_t   dword = 0;
-			if (!UserDataDwordIndex(state, reg, dword)) {
-				ctx.Define(inst, ConstantU32(state, 0));
-			} else {
-				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
-			}
-			return true;
-		}
-		case IR::ValueOpcode::GetBuiltin:
-			ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
-			                                inst.Arg(1).U32()));
-			return true;
-		case IR::ValueOpcode::UndefU1:
-		case IR::ValueOpcode::UndefU8:
-		case IR::ValueOpcode::UndefU16:
-		case IR::ValueOpcode::UndefU32:
-		case IR::ValueOpcode::UndefU64:
-			state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
-			return true;
-		case IR::ValueOpcode::DppMoveU32: {
-			const auto flags    = inst.Flags<IR::DppMoveFlags>();
-			const auto target   = EmitDppTargetLane(state, flags.control);
-			const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
-			if (flags.fetch_inactive) {
-				ctx.Define(inst, shuffled);
-				return true;
-			}
-			const auto ballot        = ctx.Ballot(inst.Arg(1));
-			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
-			const auto can_fetch     = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
-			return true;
-		}
-		case IR::ValueOpcode::DppUpdateU32: {
-			const auto flags = inst.Flags<IR::DppMoveFlags>();
-			const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
-			return true;
-		}
-		case IR::ValueOpcode::WqmU64:
-			ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
-			return true;
-		case IR::ValueOpcode::LaneId:
-			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
-			return true;
-		case IR::ValueOpcode::Ballot: ctx.Define(inst, ctx.Ballot(inst.Arg(0))); return true;
-		case IR::ValueOpcode::ReadFirstLane: {
-			const auto ballot = ctx.Ballot(inst.Arg(1));
-			const auto lane   = ctx.FirstLane(ballot);
-			ctx.Define(inst, ctx.Shuffle(inst, 0, lane));
-			return true;
-		}
-		case IR::ValueOpcode::ReadLane:
-			ctx.Define(inst, ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::WriteLane: {
-			const auto hit = state.builder.AllocateId();
-			state.builder.AddFunction({OpIEqual, TypeBool(state), hit,
-			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
-			return true;
-		}
-		case IR::ValueOpcode::Permlane16U32: {
-			const auto flags     = inst.Flags<IR::PermlaneFlags>();
-			const auto subid     = EmitSubgroupLocalInvocationId(state);
-			const auto row       = state.builder.AllocateId();
-			const auto row_value = state.builder.AllocateId();
-			const auto lane      = state.builder.AllocateId();
-			const auto lane8     = state.builder.AllocateId();
-			const auto shift     = state.builder.AllocateId();
-			const auto upper     = state.builder.AllocateId();
-			const auto selected  = state.builder.AllocateId();
-			const auto shifted   = state.builder.AllocateId();
-			const auto index     = state.builder.AllocateId();
-			const auto target    = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-			if (flags.x16) {
-				state.builder.AddFunction(
-				    {OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
-			} else {
-				state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row});
-			}
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
-			state.builder.AddFunction(
-			    {OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
-			state.builder.AddFunction(
-			    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
-			state.builder.AddFunction(
-			    {OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
-			state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
-			const auto shuffled = ctx.Shuffle(inst, 0, target);
-			uint32_t result = shuffled;
-			if (!flags.fetch_inactive) {
-				const auto source_exec = ctx.Shuffle(inst, 3, target);
-				result = state.builder.AllocateId();
-				state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
-				                           ConstantU32(state, 0)});
-			}
-			ctx.Define(inst, result);
-			return true;
-		}
-		case IR::ValueOpcode::GetAttribute:
-			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
-			return true;
-		case IR::ValueOpcode::GetInterpolationParameter:
-			ctx.Define(inst, EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(),
-			                                            inst.Arg(2).U32()));
-			return true;
-		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
-		case IR::ValueOpcode::GetShaderBase:
-			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
-			// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
-			ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
-			return true;
-		case IR::ValueOpcode::GetSrtResource:
-		case IR::ValueOpcode::GetBufferResource:
-		case IR::ValueOpcode::GetAddressResource:
-		case IR::ValueOpcode::GetScratchResource:
-		case IR::ValueOpcode::GetImageResource:
-		case IR::ValueOpcode::GetSamplerResource:
-		case IR::ValueOpcode::MakeImageAddress: return true;
-		default: return false;
+void EmitVoid(ValueEmitContext&) {}
+
+void EmitBarrier(EmitterState& state) {
+	const auto semantics =
+	    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
+	state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
+	                           ConstantU32(state, spv::ScopeWorkgroup),
+	                           ConstantU32(state, semantics)});
+}
+
+uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state  = ctx.state;
+	const auto result = state.builder.AllocateId();
+	const auto index  = inst.Arg(0).U32();
+	if (state.program.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
+		ctx.Fail(inst, "invalid mesh draw parameter");
+	}
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction({spv::OpAccessChain, TypePushConstantElementPointer(state), pointer,
+	                           state.push_constant_variable, ConstantU32(state, 0),
+	                           ConstantU32(state, index)});
+	state.builder.AddFunction({spv::OpLoad, TypeU32(state), result, pointer});
+	return result;
+}
+
+uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg) {
+
+	uint32_t dword = 0;
+	if (!UserDataDwordIndex(state, reg, dword)) {
+		return ConstantU32(state, 0);
+	} else {
+		return EmitShaderDataDwordLoad(state, dword);
 	}
 }
 
+uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) {
+	return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
+}
+
+uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({spv::OpUndef, TypeId(state, inst.GetType()), result});
+	return result;
+}
+
+uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state    = ctx.state;
+	const auto flags    = inst.Flags<IR::DppMoveFlags>();
+	const auto target   = EmitDppTargetLane(state, flags.control);
+	const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
+	if (flags.fetch_inactive) {
+		return shuffled;
+	}
+	const auto ballot        = ctx.Ballot(inst.Arg(1));
+	const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
+	const auto can_fetch     = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, can_fetch, shuffled,
+	                                                ConstantU32(state, 0));
+}
+
+uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto flags = inst.Flags<IR::DppMoveFlags>();
+	const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, write, ctx.Arg(inst, 0),
+	                                                ctx.Arg(inst, 1));
+}
+
+uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
+	return ctx.Ballot(predicate);
+}
+
+uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto ballot = ctx.Ballot(inst.Arg(1));
+	const auto lane   = ctx.FirstLane(ballot);
+	return ctx.Shuffle(inst, 0, lane);
+}
+
+uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1));
+}
+
+uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state = ctx.state;
+	const auto hit   = state.builder.AllocateId();
+	state.builder.AddFunction({spv::OpIEqual, TypeBool(state), hit,
+	                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, hit, ctx.Arg(inst, 1),
+	                                                ctx.Arg(inst, 0));
+}
+
+uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state     = ctx.state;
+	const auto flags     = inst.Flags<IR::PermlaneFlags>();
+	const auto subid     = EmitSubgroupLocalInvocationId(state);
+	const auto row       = state.builder.AllocateId();
+	const auto row_value = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto lane8     = state.builder.AllocateId();
+	const auto shift     = state.builder.AllocateId();
+	const auto upper     = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto shifted   = state.builder.AllocateId();
+	const auto index     = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	if (flags.x16) {
+		state.builder.AddFunction(
+		    {spv::OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
+	} else {
+		state.builder.AddFunction({spv::OpCopyObject, TypeU32(state), row_value, row});
+	}
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
+	state.builder.AddFunction(
+	    {spv::OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
+	state.builder.AddFunction(
+	    {spv::OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
+	state.builder.AddFunction(
+	    {spv::OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
+	state.builder.AddFunction({spv::OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, row_value, index});
+	const auto shuffled = ctx.Shuffle(inst, 0, target);
+	uint32_t   result   = shuffled;
+	if (!flags.fetch_inactive) {
+		const auto source_exec = ctx.Shuffle(inst, 3, target);
+		result                 = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {spv::OpSelect, TypeU32(state), result, source_exec, shuffled, ConstantU32(state, 0)});
+	}
+	return result;
+}
+
+uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32());
+}
+
+uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(), inst.Arg(2).U32());
+}
+
+uint32_t EmitGetShaderBase(ValueEmitContext& ctx) {
+	// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
+	// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
+	return ctx.Def(IR::Value(uint64_t {0}));
+}
+
+void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) {
+	ctx.Fail(inst, "must be lowered before SPIR-V emission");
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -10,15 +10,16 @@ DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto selected  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
+	    {spv::OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
+	    {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
+	    {spv::OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected});
+	    {spv::OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, quad_base, selected});
 	return {target, ConstantBool(state, true)};
 }
 
@@ -30,20 +31,21 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto target       = state.builder.AllocateId();
 	const auto valid        = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	    {spv::OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction(
+	    {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 	if (left) {
 		state.builder.AddFunction(
-		    {OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
+		    {spv::OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
 		state.builder.AddFunction(
-		    {OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
+		    {spv::OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
 	} else {
 		state.builder.AddFunction(
-		    {OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
+		    {spv::OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
 		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
+		    {spv::OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
 	}
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
 	return {target, valid};
 }
 
@@ -57,15 +59,17 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
 	const auto selected = state.builder.AllocateId();
 	const auto target   = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	    {spv::OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
+	    {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 	state.builder.AddFunction(
-	    {OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
-	state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected});
+	    {spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction(
+	    {spv::OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction(
+	    {spv::OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
+	state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, in_high, minus, plus});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, row, selected});
 	return {target, ConstantBool(state, true)};
 }
 
@@ -77,12 +81,12 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
 	const auto mirrored  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
+	    {spv::OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
+	    {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
 	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored});
+	    {spv::OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, base, mirrored});
 	return {target, ConstantBool(state, true)};
 }
 
@@ -109,7 +113,7 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
 	if (control >= 0x160u && control <= 0x16fu) {
 		const auto target = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpBitwiseXor, TypeU32(state), target, subid, ConstantU32(state, control & 0xfu)});
+		    {spv::OpBitwiseXor, TypeU32(state), target, subid, ConstantU32(state, control & 0xfu)});
 		return {target, ConstantBool(state, true)};
 	}
 	return {subid, ConstantBool(state, true)};
@@ -121,7 +125,7 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
 	}
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
+	    {spv::OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
 	return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32));
 }
 
@@ -150,9 +154,10 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
 	}
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)),
-	                           pointer, variable, ConstantU32(state, component)});
-	state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+	state.builder.AddFunction({spv::OpAccessChain,
+	                           TypePointer(state, spv::StorageClassInput, TypeU32(state)), pointer,
+	                           variable, ConstantU32(state, component)});
+	state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
 	return value;
 }
 
@@ -162,9 +167,10 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
 		return ConstantU32(state, 0);
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
+	state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, variable});
 	if (state.lane_count == 2) {
-		const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u));
+		const auto wave_base =
+		    EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, ~31u));
 		return EmitAddU32(state, EmitAddU32(state, value, wave_base),
 		                  ConstantU32(state, state.lane_half * 32));
 	}
@@ -178,13 +184,13 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	const auto scalar_type = VertexParameterScalarType(state, kind);
 	uint32_t   raw         = state.builder.AllocateId();
 	if (count == 1u) {
-		state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
+		state.builder.AddFunction({spv::OpLoad, scalar_type, raw, input.variable_id});
 	} else {
-		const auto pointer_type = TypePointer(state, StorageClassInput, scalar_type);
+		const auto pointer_type = TypePointer(state, spv::StorageClassInput, scalar_type);
 		const auto pointer      = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
+		state.builder.AddFunction({spv::OpAccessChain, pointer_type, pointer, input.variable_id,
 		                           ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
+		state.builder.AddFunction({spv::OpLoad, scalar_type, raw, pointer});
 	}
 
 	if (kind == VertexInputScalarKind::Uint) {
@@ -192,47 +198,49 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	}
 
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw});
+	state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, raw});
 	return bits;
 }
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
 	const auto active_ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
-	                           ConstantU32(state, ScopeSubgroup), ConstantBool(state, true)});
+	state.builder.AddFunction({spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
+	                           ConstantU32(state, spv::ScopeSubgroup), ConstantBool(state, true)});
 	return EmitBallotLaneActiveBool(state, active_ballot, lane);
 }
 uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
 	const auto low = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
+	state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
 	uint32_t mask = low;
 	if (state.program.wave_size == 64u) {
 		const auto high     = state.builder.AllocateId();
 		const auto in_high  = state.builder.AllocateId();
 		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
 		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low});
+		    {spv::OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
+		state.builder.AddFunction(
+		    {spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, in_high, high, low});
 		mask = selected;
 	}
 
 	const auto lane_low = state.builder.AllocateId();
 	const auto bit      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
+	    {spv::OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
+	    {spv::OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
 
 	const auto hit      = state.builder.AllocateId();
 	const auto active   = state.builder.AllocateId();
 	const auto in_range = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), hit, mask, bit});
 	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.program.wave_size)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range});
+	    {spv::OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpULessThan, TypeBool(state), in_range, lane,
+	                           ConstantU32(state, state.program.wave_size)});
+	state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), ret, active, in_range});
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -1,5 +1,5 @@
 #include "graphics/guest_gpu/gpu_format.h"
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
 
 #include <algorithm>
@@ -44,10 +44,10 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	auto value = ctx.Def(address.Arg(packed));
 	if (layout.bit_width == 16u) {
 		if ((layout.bit_offset & 31u) != 0u) {
-			value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value,
+			value = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), value,
 			               ConstantU32(ctx.state, 16));
 		}
-		value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value,
+		value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), value,
 		               ConstantU32(ctx.state, 0xffffu));
 	}
 	return value;
@@ -58,7 +58,7 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	const auto value = AddressU32(ctx, mem, address, component);
 	return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u
 	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value);
+	           : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
 }
 
 ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) {
@@ -85,19 +85,19 @@ uint32_t ZeroF32(EmitterState& state) {
 }
 
 uint32_t CubeAxis(EmitterState& state, uint32_t value) {
-	return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
+	return Binary(state, spv::OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
 }
 
 uint32_t CubeLayer(EmitterState& state, uint32_t value) {
 	const auto guest = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value});
-	const auto padding =
-	    Binary(state, OpShiftLeftLogical, TypeU32(state),
-	           Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
-	           ConstantU32(state, 1));
-	const auto host   = Binary(state, OpISub, TypeU32(state), guest, padding);
+	state.builder.AddFunction({spv::OpConvertFToU, TypeU32(state), guest, value});
+	const auto padding = Binary(
+	    state, spv::OpShiftLeftLogical, TypeU32(state),
+	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
+	    ConstantU32(state, 1));
+	const auto host   = Binary(state, spv::OpISub, TypeU32(state), guest, padding);
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host});
+	state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), result, host});
 	return result;
 }
 
@@ -119,10 +119,10 @@ uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		             : ZeroF32(ctx.state);
 		if (cube) z = CubeLayer(ctx.state, z);
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
+		    {spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
+		    {spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
 	}
 	return result;
 }
@@ -139,10 +139,10 @@ uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
 		                                                 : ConstantU32(ctx.state, 0);
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
+		    {spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
+		    {spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
 	}
 	return result;
 }
@@ -156,7 +156,7 @@ uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst
 }
 
 uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
-	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+	return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
 }
 
 uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
@@ -164,7 +164,7 @@ uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
 	if (numeric_class == Prospero::TextureNumericClass::Uint) {
 		return value;
 	}
-	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+	return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
 }
 
 uint32_t SampledComponentZero(EmitterState& state, Prospero::TextureNumericClass numeric_class) {
@@ -193,7 +193,7 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 			if (dref) return value;
 			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
 			const auto result    = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpCompositeExtract,
+			ctx.state.builder.AddFunction({spv::OpCompositeExtract,
 			                               ImageScalarType(ctx.state, value_class), result, value,
 			                               component});
 			return result;
@@ -209,24 +209,26 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 				const auto low_bits  = SampledComponentBits(ctx, low, value_class);
 				const auto high_bits = SampledComponentBits(ctx, high, value_class);
 				const auto mask      = ConstantU32(ctx.state, 0xffffu);
-				packed[word]         = Binary(
-				    ctx.state, OpBitwiseOr, TypeU32(ctx.state),
-				    Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
-				    Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
-				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high_bits, mask),
-				           ConstantU32(ctx.state, 16u)));
+				packed[word] =
+				    Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state),
+				           Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
+				           Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
+				                  Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
+				                         high_bits, mask),
+				                  ConstantU32(ctx.state, 16u)));
 			} else {
 				const auto pair = ctx.state.builder.AllocateId();
 				ctx.state.builder.AddFunction(
-				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				    {spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
 				packed[word] = ctx.state.builder.AllocateId();
-				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
-				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+				ctx.state.builder.AddFunction({spv::OpExtInst, TypeU32(ctx.state), packed[word],
+				                               GlslStd450(ctx.state), GLSLstd450PackHalf2x16,
+				                               pair});
 			}
 		}
 		const auto result = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-		                               packed[0], packed[1], packed[2], packed[3]});
+		ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4),
+		                               result, packed[0], packed[1], packed[2], packed[3]});
 		return result;
 	}
 	uint32_t component[4] {};
@@ -236,12 +238,13 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 			continue;
 		}
 		const auto scalar = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, ImageScalarType(ctx.state, value_class), scalar, value, index});
+		ctx.state.builder.AddFunction({spv::OpCompositeExtract,
+		                               ImageScalarType(ctx.state, value_class), scalar, value,
+		                               index});
 		component[index] = SampledComponentBits(ctx, scalar, value_class);
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
 	                               component[0], component[1], component[2], component[3]});
 	return result;
 }
@@ -254,10 +257,11 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 	const auto  size      = ctx.state.builder.AllocateId();
 	if (info.multisampled != 0u) {
 		ctx.state.builder.AddFunction(
-		    {OpImageQuerySize, ImageViewSizeType(ctx.state, dimension), size, image});
+		    {spv::OpImageQuerySize, ImageViewSizeType(ctx.state, dimension), size, image});
 	} else {
-		ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, dimension),
-		                               size, image, AddressU32(ctx, mem, address, 0)});
+		ctx.state.builder.AddFunction({spv::OpImageQuerySizeLod,
+		                               ImageViewSizeType(ctx.state, dimension), size, image,
+		                               AddressU32(ctx, mem, address, 0)});
 	}
 	const auto components = info.coordinate_components;
 	uint32_t   result[4]  = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
@@ -268,15 +272,16 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 		for (uint32_t index = 0; index < components; index++) {
 			result[index] = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
+			    {spv::OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
 		}
 	}
 	if (info.multisampled == 0u) {
 		result[3] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
+		ctx.state.builder.AddFunction(
+		    {spv::OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
 	}
 	const auto vector = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
+	ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
 	                               result[0], result[1], result[2], result[3]});
 	return vector;
 }
@@ -290,30 +295,30 @@ uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR
 		const auto result = ctx.state.builder.AllocateId();
 		if (components == 3u) {
 			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
+			    {spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
 		} else {
 			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
+			    {spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
 		}
 		return result;
 	}
-	const auto packed    = Unary(ctx.state, OpBitcast, TypeI32(ctx.state),
+	const auto packed    = Unary(ctx.state, spv::OpBitcast, TypeI32(ctx.state),
 	                             AddressU32(ctx, mem, address, layout.offset));
 	uint32_t   values[3] = {zero, zero, zero};
 	for (uint32_t index = 0; index < components; index++) {
 		values[index] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index],
+		ctx.state.builder.AddFunction({spv::OpBitFieldSExtract, TypeI32(ctx.state), values[index],
 		                               packed, ConstantU32(ctx.state, index * 8u),
 		                               ConstantU32(ctx.state, 6)});
 	}
 	if (components == 1u) return values[0];
 	const auto result = ctx.state.builder.AllocateId();
 	if (components == 3u) {
-		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result,
-		                               values[0], values[1], values[2]});
+		ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3),
+		                               result, values[0], values[1], values[2]});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
+		    {spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
 	}
 	return result;
 }
@@ -322,18 +327,18 @@ uint32_t HorizontalOffsets(EmitterState& state, ImageDimension dimension) {
 	const auto components   = ImageDimensionInfoFor(dimension).spatial_components;
 	const auto count        = ConstantU32(state, 4);
 	const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
-	const auto array_type   = state.builder.Type(OpTypeArray, {element_type, count});
+	const auto array_type   = state.builder.Type(spv::OpTypeArray, {element_type, count});
 	uint32_t   offsets[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
 		if (components == 1u) {
 			offsets[index] = x;
 		} else {
-			offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2),
-			                                        {x, ConstantI32(state, 0)});
+			offsets[index] = state.builder.Constant(
+			    spv::OpConstantComposite, TypeI32Vector(state, 2), {x, ConstantI32(state, 0)});
 		}
 	}
-	return state.builder.Constant(OpConstantComposite, array_type,
+	return state.builder.Constant(spv::OpConstantComposite, array_type,
 	                              {offsets[0], offsets[1], offsets[2], offsets[3]});
 }
 
@@ -362,12 +367,12 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
 
 	const auto packed = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
+	ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
 	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
 	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		components[component] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state),
+		ctx.state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(ctx.state),
 		                               components[component], packed,
 		                               ConstantU32(ctx.state, info.component_bit_offset[component]),
 		                               ConstantU32(ctx.state, info.component_bits[component])});
@@ -389,7 +394,7 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 		}
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
 	                               selected[0], selected[1], selected[2], selected[3]});
 	return result;
 }
@@ -403,7 +408,7 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
 	if (selector < 4u) {
 		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
-		return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4),
+		return ctx.state.builder.Constant(spv::OpConstantComposite, TypeU32Vector(ctx.state, 4),
 		                                  {value, value, value, value});
 	}
 
@@ -411,15 +416,16 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 	for (uint32_t lane = 0; lane < 4u; lane++) {
 		const auto packed = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
+		    {spv::OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
 		values[lane]        = ctx.state.builder.AllocateId();
 		const auto physical = (selector - 4u) % info.component_count;
-		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed,
+		ctx.state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(ctx.state), values[lane],
+		                               packed,
 		                               ConstantU32(ctx.state, info.component_bit_offset[physical]),
 		                               ConstantU32(ctx.state, info.component_bits[physical])});
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
 	                               values[0], values[1], values[2], values[3]});
 	return result;
 }
@@ -427,17 +433,18 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                                     uint32_t coord, Prospero::TextureNumericClass numeric_class) {
 	auto& state = ctx.state;
-	state.builder.RequireCapability(CapabilityImageQuery);
+	state.builder.RequireCapability(spv::CapabilityImageQuery);
 	const auto image = LoadSampledImageDescriptor(state, mem.resource);
 	const auto width = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)});
+	    {spv::OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)});
 	const auto width_f32 = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), width_f32, width});
+	state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), width_f32, width});
 	const auto left = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32(state), left, GlslStd450(state), GlslFloor,
-	                           Binary(state, OpFSub, TypeF32(state),
-	                                  Binary(state, OpFMul, TypeF32(state), coord, width_f32),
+	state.builder.AddFunction({spv::OpExtInst, TypeF32(state), left, GlslStd450(state),
+	                           GLSLstd450Floor,
+	                           Binary(state, spv::OpFSub, TypeF32(state),
+	                                  Binary(state, spv::OpFMul, TypeF32(state), coord, width_f32),
 	                                  ConstantF32(state, 0x3f000000u))});
 
 	const auto sampled     = MakeSampledImage(state, mem.resource, mem.sampler);
@@ -450,20 +457,20 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 	uint32_t values[2] {};
 	for (uint32_t index = 0; index < 2u; index++) {
 		const auto sample_coord =
-		    Binary(state, OpFDiv, TypeF32(state),
-		           Binary(state, OpFAdd, TypeF32(state), left,
+		    Binary(state, spv::OpFDiv, TypeF32(state),
+		           Binary(state, spv::OpFAdd, TypeF32(state), left,
 		                  ConstantF32(state, index == 0u ? 0x3f000000u : 0x3fc00000u)),
 		           width_f32);
 		const auto texel = state.builder.AllocateId();
-		state.builder.AddFunction({OpImageSampleExplicitLod, vector_type, texel, sampled,
-		                           sample_coord, ImageOperandsLodMask, ZeroF32(state)});
+		state.builder.AddFunction({spv::OpImageSampleExplicitLod, vector_type, texel, sampled,
+		                           sample_coord, spv::ImageOperandsLodMask, ZeroF32(state)});
 		values[index] = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpCompositeExtract, scalar_type, values[index], texel, component});
+		    {spv::OpCompositeExtract, scalar_type, values[index], texel, component});
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpCompositeConstruct, vector_type, result, values[0], values[1], values[1], values[0]});
+	state.builder.AddFunction({spv::OpCompositeConstruct, vector_type, result, values[0], values[1],
+	                           values[1], values[0]});
 	return result;
 }
 
@@ -475,26 +482,27 @@ uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		const auto value = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
+		    {spv::OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
 		const auto maximum =
 		    ConstantU32(ctx.state, info.component_bits[component] == 32u
 		                               ? UINT32_MAX
 		                               : (1u << info.component_bits[component]) - 1u);
-		const auto within  = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum);
+		const auto within =
+		    Binary(ctx.state, spv::OpULessThan, TypeBool(ctx.state), value, maximum);
 		const auto clamped = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction(
-		    {OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
+		    {spv::OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
 		const auto shifted =
 		    info.component_bit_offset[component] == 0u
 		        ? clamped
-		        : Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped,
+		        : Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), clamped,
 		                 ConstantU32(ctx.state, info.component_bit_offset[component]));
-		packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
+		packed = Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
 	}
 	const auto zero   = ConstantU32(ctx.state, 0u);
 	const auto result = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(
-	    {OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
+	    {spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
 	return result;
 }
 
@@ -509,82 +517,80 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 			const auto packed_index = DmaskComponentIndex(dmask, source);
 			raw                     = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			    {spv::OpCompositeExtract, TypeU32(ctx.state), raw, data,
 			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
 			if (mem.data_bits == 16u) {
 				if ((packed_index & 1u) != 0u) {
-					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					raw = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), raw,
 					             ConstantU32(ctx.state, 16u));
 				}
-				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				raw = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), raw,
 				             ConstantU32(ctx.state, 0xffffu));
 			}
 		}
 		values[component] = integer ? raw
 		                    : mem.data_bits == 16u
 		                        ? EmitF16BitsToF32(ctx.state, raw)
-		                        : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
+		                        : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), raw);
 	}
 	const auto texel = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(
-	    {OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
-	     texel, values[0], values[1], values[2], values[3]});
+	    {spv::OpCompositeConstruct,
+	     integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4), texel, values[0],
+	     values[1], values[2], values[3]});
 	return PackImageTexel(ctx, mem, texel);
 }
 
 uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
 	switch (opcode) {
-		case IR::ValueOpcode::ImageAtomicSwap32: return OpAtomicExchange;
-		case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
-		case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
-		case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
-		case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
-		case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
-		case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
+		case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
+		case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
+		case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
 		default: return 0;
 	}
 }
 
 } // namespace
 
-bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
+void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto op         = inst.GetOpcode();
 	const auto image_info = IR::ImageOpcodeInfoOf(op);
-	if (image_info.access == IR::ImageAccess::None) {
-		return false;
-	}
 	auto&       state     = ctx.state;
 	const auto& mem       = ctx.Memory(inst);
 	const auto  image_arg = inst.Arg(0);
 	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
 	const auto& image   = state.program.info.images.at(mem.resource);
 	const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
-	if (address == nullptr) return true;
 	if (op == IR::ValueOpcode::ImageQueryDimensions) {
-		state.builder.RequireCapability(CapabilityImageQuery);
+		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		ctx.Define(inst, QueryDimensions(ctx, mem, *address));
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageQueryLod) {
-		state.builder.RequireCapability(CapabilityImageQuery);
+		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		const auto dimension = image.dimension;
 		const auto sampled   = MakeSampledImage(state, mem.resource, mem.sampler);
 		const auto lod       = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
+		    {spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
 		     CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components)});
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
 			const auto component = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index});
+			state.builder.AddFunction(
+			    {spv::OpCompositeExtract, TypeF32(state), component, lod, index});
 			values[index] = FloatBits(ctx, component);
 		}
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0],
-		                           values[1], values[2], values[3]});
+		state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(state, 4), result,
+		                           values[0], values[1], values[2], values[3]});
 		ctx.Define(inst, result);
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageRead) {
 		const auto  dimension      = image.dimension;
@@ -601,19 +607,19 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			        const auto coord      = CoordU32(ctx, mem, *address, dimension);
 			        if (dimension_info.multisampled != 0u) {
 				        state.builder.AddFunction(
-				            {OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
-				             descriptor, coord, ImageOperandsSampleMask,
+				            {spv::OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
+				             descriptor, coord, spv::ImageOperandsSampleMask,
 				             AddressU32(ctx, mem, *address, dimension_info.coordinate_components)});
 			        } else {
-				        state.builder.AddFunction({OpImageFetch,
+				        state.builder.AddFunction({spv::OpImageFetch,
 				                                   ImageVectorType(state, numeric_class, 4), color,
-				                                   descriptor, coord, ImageOperandsLodMask,
+				                                   descriptor, coord, spv::ImageOperandsLodMask,
 				                                   LodU32(ctx, mem, *address, dimension)});
 			        }
 			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class,
 			                            false, mem);
 		        }));
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageWrite) {
 		const bool uint_image = image.numeric_class == Prospero::TextureNumericClass::Uint;
@@ -627,7 +633,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
 			EmitStorageImageWrite(state, mem.resource, mip_lod, coord, texel);
 		});
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
 		const auto  dimension      = image.dimension;
@@ -638,7 +644,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (dref && state.program.info.images[mem.resource].conversion_format !=
 		                Prospero::BufferFormat::kInvalid) {
 			ctx.Fail(inst, "uses depth comparison with a packed integer image");
-			return true;
+			return;
 		}
 		const auto coord =
 		    CoordF32(ctx, mem, *address, layout.coord, dimension_info.coordinate_components);
@@ -648,16 +654,16 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				    HasFlag(mem, Decoder::ImageSampleFlagOffset) ||
 				    HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
 					ctx.Fail(inst, "has an unsupported 1D gather variant");
-					return true;
+					return;
 				}
 				const auto sample = EmitOneDimensionalGatherLz(ctx, mem, coord, numeric_class);
 				ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 				                              numeric_class, false, mem, true));
-				return true;
+				return;
 			}
 			if (dimension == ImageDimension::Dim1DArray) {
 				ctx.Fail(inst, "has an unsupported 1D-array gather");
-				return true;
+				return;
 			}
 			const auto            sampled = MakeSampledImage(state, mem.resource, mem.sampler);
 			const auto            sample  = state.builder.AllocateId();
@@ -667,22 +673,29 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				if (layout.dref != NoImageComponent) {
 					dref_value = AddressF32(ctx, mem, *address, layout.dref);
 				}
-				words = {OpImageDrefGather, TypeF32Vector(state, 4), sample, sampled, coord,
+				words = {spv::OpImageDrefGather,
+				         TypeF32Vector(state, 4),
+				         sample,
+				         sampled,
+				         coord,
 				         dref_value};
 			} else {
 				uint32_t component = 0;
 				if (ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid) {
 					component = ImageGatherComponent(mem.dmask);
 				}
-				words = {OpImageGather, ImageVectorType(state, numeric_class, 4),
-				         sample,        sampled,
-				         coord,         ConstantU32(state, component)};
+				words = {spv::OpImageGather,
+				         ImageVectorType(state, numeric_class, 4),
+				         sample,
+				         sampled,
+				         coord,
+				         ConstantU32(state, component)};
 			}
 			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
-				words.push_back(ImageOperandsConstOffsetsMask);
+				words.push_back(spv::ImageOperandsConstOffsetsMask);
 				words.push_back(HorizontalOffsets(state, dimension));
 			} else if (layout.offset != NoImageComponent) {
-				words.push_back(ImageOperandsOffsetMask);
+				words.push_back(spv::ImageOperandsOffsetMask);
 				words.push_back(PackedOffset(ctx, mem, *address, layout, dimension));
 			}
 			state.builder.AddFunction(words);
@@ -692,17 +705,17 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			}
 			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 			                              result_numeric_class, false, mem, true));
-			return true;
+			return;
 		}
 		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
 		                          HasFlag(mem, Decoder::ImageSampleFlagLod) ||
 		                          HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
-		                          state.stage != ShaderType::Pixel;
-		uint32_t opcode = OpImageSampleImplicitLod;
+		                          state.program.stage != ShaderType::Pixel;
+		uint32_t   opcode       = spv::OpImageSampleImplicitLod;
 		if (explicit_lod) {
-			opcode = dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
+			opcode = dref ? spv::OpImageSampleDrefExplicitLod : spv::OpImageSampleExplicitLod;
 		} else if (dref) {
-			opcode = OpImageSampleDrefImplicitLod;
+			opcode = spv::OpImageSampleDrefImplicitLod;
 		}
 		uint32_t result_type = ImageVectorType(state, numeric_class, 4);
 		uint32_t dref_value  = 0;
@@ -716,20 +729,20 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		uint32_t              operand_mask = 0;
 		std::vector<uint32_t> operands;
 		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
-			operand_mask |= ImageOperandsGradMask;
+			operand_mask |= spv::ImageOperandsGradMask;
 			operands.push_back(
 			    CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components));
 			operands.push_back(
 			    CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components));
 		} else if (explicit_lod) {
-			operand_mask |= ImageOperandsLodMask;
+			operand_mask |= spv::ImageOperandsLodMask;
 			auto lod = ZeroF32(state);
 			if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) {
 				lod = AddressF32(ctx, mem, *address, layout.lod);
 			}
 			operands.push_back(lod);
 		} else if (layout.bias != NoImageComponent) {
-			operand_mask |= ImageOperandsBiasMask;
+			operand_mask |= spv::ImageOperandsBiasMask;
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
 		const auto EmitSample = [&](uint32_t resource) {
@@ -753,7 +766,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				result = UnpackImageTexel(ctx, mem, sample);
 			}
 			ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-			return true;
+			return;
 		}
 		const auto* handle = image_arg.ResolveInstruction();
 		const auto* source = image.source < state.program.descriptor_sources.size()
@@ -762,21 +775,21 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
 		    source->indirect_image->key_arg >= handle->NumArgs()) {
 			ctx.Fail(inst, "has invalid indirect image key provenance");
-			return true;
+			return;
 		}
 		const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg));
 		if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u ||
 		    image.indirect_resources.size() < 2u) {
 			ctx.Fail(inst, "has no indirect image runtime mapping");
-			return true;
+			return;
 		}
 		const auto LoadMapping = [&](uint32_t index) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state),
+			state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state),
 			                           pointer, state.flattened_srt_variable, ConstantU32(state, 0),
 			                           index});
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+			state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
 			return value;
 		};
 		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
@@ -785,61 +798,63 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		auto       selected = ConstantU32(state, 0u);
 		for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
 		     iteration++) {
-			const auto active = Binary(state, OpULessThan, TypeBool(state), low, high);
-			const auto mid =
-			    Binary(state, OpShiftRightLogical, TypeU32(state),
-			           Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u));
+			const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
+			const auto mid    = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+			                           Binary(state, spv::OpIAdd, TypeU32(state), low, high),
+			                           ConstantU32(state, 1u));
 			const auto probe = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
-			const auto entry      = Binary(state, OpIAdd, TypeU32(state), mapping,
-			                               Binary(state, OpIAdd, TypeU32(state),
-			                                      Binary(state, OpShiftLeftLogical, TypeU32(state),
-			                                             probe, ConstantU32(state, 1u)),
-			                                      ConstantU32(state, 1u)));
+			    {spv::OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
+			const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
+			                          Binary(state, spv::OpIAdd, TypeU32(state),
+			                                 Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
+			                                        probe, ConstantU32(state, 1u)),
+			                                 ConstantU32(state, 1u)));
 			const auto mapped_key = LoadMapping(entry);
-			const auto candidate =
-			    LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
-			const auto equal         = Binary(state, OpIEqual, TypeBool(state), mapped_key, key);
-			const auto match         = Binary(state, OpLogicalAnd, TypeBool(state), active, equal);
+			const auto candidate  = LoadMapping(
+			    Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
+			const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
+			const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
 			const auto next_selected = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), next_selected, match, candidate, selected});
+			    {spv::OpSelect, TypeU32(state), next_selected, match, candidate, selected});
 			selected              = next_selected;
-			const auto less       = Binary(state, OpULessThan, TypeBool(state), mapped_key, key);
-			const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less);
-			const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active,
-			                               Unary(state, OpLogicalNot, TypeBool(state), less));
+			const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
+			const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
+			const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
+			                               Unary(state, spv::OpLogicalNot, TypeBool(state), less));
 			const auto next_low   = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), next_low, take_upper,
-			     Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
+			    {spv::OpSelect, TypeU32(state), next_low, take_upper,
+			     Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
 			low                  = next_low;
 			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high});
+			state.builder.AddFunction(
+			    {spv::OpSelect, TypeU32(state), next_high, take_lower, mid, high});
 			high = next_high;
 		}
 		const auto            default_label = state.builder.AllocateId();
 		const auto            merge_label   = state.builder.AllocateId();
 		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
-		std::vector<uint32_t> switch_words {OpSwitch, selected, default_label};
+		std::vector<uint32_t> switch_words {spv::OpSwitch, selected, default_label};
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
 			labels[candidate - 1u] = state.builder.AllocateId();
 			switch_words.push_back(candidate);
 			switch_words.push_back(labels[candidate - 1u]);
 		}
-		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+		state.builder.AddFunction(
+		    {spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
 		state.builder.AddFunction(switch_words);
-		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
+		std::vector<uint32_t> phi_words {spv::OpPhi, result_type, state.builder.AllocateId()};
 		EmitLabel(state, default_label);
 		phi_words.push_back(EmitSample(image.indirect_resources[0]));
 		phi_words.push_back(default_label);
-		state.builder.AddFunction({OpBranch, merge_label});
+		state.builder.AddFunction({spv::OpBranch, merge_label});
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
 			EmitLabel(state, labels[candidate - 1u]);
 			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
 			phi_words.push_back(labels[candidate - 1u]);
-			state.builder.AddFunction({OpBranch, merge_label});
+			state.builder.AddFunction({spv::OpBranch, merge_label});
 		}
 		EmitLabel(state, merge_label);
 		state.builder.AddFunction(phi_words);
@@ -848,30 +863,30 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			result = UnpackImageTexel(ctx, mem, result);
 		}
 		ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-		return true;
+		return;
 	}
 	const auto atomic_opcode = ImageAtomicOpcode(op);
 	if (atomic_opcode != 0u) {
 		const auto dimension = image.dimension;
 		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
 			           const auto pointer = state.builder.AllocateId();
-			           const auto pointer_type =
-			               state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)});
+			           const auto pointer_type = state.builder.Type(
+			               spv::OpTypePointer, {spv::StorageClassImage, TypeU32(state)});
 			           state.builder.AddFunction(
-			               {OpImageTexelPointer, pointer_type, pointer,
+			               {spv::OpImageTexelPointer, pointer_type, pointer,
 			                StorageImageDescriptorPointer(state, mem.resource),
 			                CoordU32(ctx, mem, *address, dimension), ConstantU32(state, 0)});
 			           const auto old = state.builder.AllocateId();
 			           state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
-			                                      ConstantU32(state, ScopeDevice),
-			                                      ConstantU32(state, MemorySemanticsNone),
+			                                      ConstantU32(state, spv::ScopeDevice),
+			                                      ConstantU32(state, spv::MemorySemanticsMaskNone),
 			                                      ctx.Arg(inst, 2)});
 			           EmitDeviceAtomicMemoryBarrier(state);
 			           return old;
 		           }));
-		return true;
+		return;
 	}
-	return false;
+	ctx.Fail(inst, "has no image SPIR-V emitter");
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -0,0 +1,301 @@
+#pragma once
+
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+// Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
+// (IR::Value), and instructions whose metadata or operand loads must stay lazy.
+#define EMIT_NATIVE(name, opcode, type, ...)                                                       \
+	inline constexpr auto Emit##name = EmitNative<spv::opcode, IR::Type::type, __VA_ARGS__>;
+EMIT_NATIVE(BitCastU16F16, OpBitcast, U32, uint32_t)
+inline constexpr auto EmitBitCastF16U16 = EmitBitCastU16F16;
+inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16;
+inline constexpr auto EmitConvertU32U8  = EmitBitCastU16F16;
+inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16;
+EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t)
+uint32_t              EmitConvertU16U32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertU8U32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertF16F32(EmitterState& state, uint32_t arg0);
+inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32;
+uint32_t              EmitConvertS32F32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertU32F32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertF32S32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
+EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
+            uint32_t)
+uint32_t              EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
+inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
+inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
+inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
+inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
+inline constexpr auto EmitPackSnorm2x16 =
+    EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
+inline constexpr auto EmitPackUnorm2x16 =
+    EmitGlsl<GLSLstd450PackUnorm2x16, IR::Type::U32, uint32_t>;
+uint32_t              EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1);
+inline constexpr auto EmitFPAbs32 = EmitFAbsValue;
+inline constexpr auto EmitFPNeg32 = EmitFNegateValue;
+uint32_t              EmitFPSaturate32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(BitFieldInsert, OpBitFieldInsert, U32, uint32_t, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(BitFieldUExtract, OpBitFieldUExtract, U32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectU1, OpSelect, U1, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectU32, OpSelect, U32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectF32, OpSelect, F32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
+EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
+EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
+EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
+uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
+uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
+EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
+EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
+uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
+EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
+EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
+EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
+uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
+EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
+uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
+uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0);
+uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+EMIT_NATIVE(SLessThan32, OpSLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThan32, OpULessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(IEqual32, OpIEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SLessThanEqual32, OpSLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThanEqual32, OpULessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SGreaterThan32, OpSGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
+uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
+EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
+EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t)
+EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
+uint32_t              EmitFPIsNan32(EmitterState& state, uint32_t arg0);
+inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
+EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
+inline constexpr auto EmitFPFma32 =
+    EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
+uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1);
+inline constexpr auto EmitFPRoundEven32 = EmitGlsl<GLSLstd450RoundEven, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFloor32     = EmitGlsl<GLSLstd450Floor, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPCeil32      = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPTrunc32     = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFract32     = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
+uint32_t              EmitFPSin(EmitterState& state, uint32_t arg0);
+uint32_t              EmitFPCos(EmitterState& state, uint32_t arg0);
+#undef EMIT_NATIVE
+
+uint32_t              EmitIdentity(ValueEmitContext& ctx, uint32_t value);
+void                  EmitVoid(ValueEmitContext&);
+inline constexpr auto EmitReference    = EmitVoid;
+inline constexpr auto EmitReferenceU32 = EmitVoid;
+inline constexpr auto EmitControlNop   = EmitVoid;
+inline constexpr auto EmitWaitcnt      = EmitVoid;
+inline constexpr auto EmitSendmsg      = EmitVoid;
+inline constexpr auto EmitTtraceData   = EmitVoid;
+inline constexpr auto EmitInstPrefetch = EmitVoid;
+void                  EmitBarrier(EmitterState& state);
+void                  EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
+uint32_t              EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
+uint32_t              EmitUndefU1(EmitterState& state, const IR::Inst& inst);
+inline constexpr auto EmitUndefU8  = EmitUndefU1;
+inline constexpr auto EmitUndefU16 = EmitUndefU1;
+inline constexpr auto EmitUndefU32 = EmitUndefU1;
+inline constexpr auto EmitUndefU64 = EmitUndefU1;
+uint32_t              EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitWqmU64(EmitterState& state, uint32_t value);
+inline constexpr auto EmitLaneId = EmitSubgroupLocalInvocationId;
+uint32_t              EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
+uint32_t              EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetShaderBase(ValueEmitContext& ctx);
+inline constexpr auto EmitGetSrtResource     = EmitVoid;
+inline constexpr auto EmitGetBufferResource  = EmitGetSrtResource;
+inline constexpr auto EmitGetAddressResource = EmitGetSrtResource;
+inline constexpr auto EmitGetScratchResource = EmitGetSrtResource;
+inline constexpr auto EmitGetImageResource   = EmitGetSrtResource;
+inline constexpr auto EmitGetSamplerResource = EmitGetSrtResource;
+inline constexpr auto EmitMakeImageAddress   = EmitGetSrtResource;
+void                  EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);
+inline constexpr auto EmitLoadAddressU8         = EmitLoadMemory;
+inline constexpr auto EmitLoadAddressU16        = EmitLoadMemory;
+inline constexpr auto EmitLoadAddressU32        = EmitLoadMemory;
+inline constexpr auto EmitStoreAddressU8        = EmitStoreMemory;
+inline constexpr auto EmitStoreAddressU16       = EmitStoreMemory;
+inline constexpr auto EmitStoreAddressU32       = EmitStoreMemory;
+inline constexpr auto EmitLoadBufferU8          = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU16         = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32         = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32x2       = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32x3       = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32x4       = EmitLoadMemory;
+inline constexpr auto EmitStoreBufferU8         = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU16        = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32        = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32x2      = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32x3      = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32x4      = EmitStoreMemory;
+inline constexpr auto EmitBufferAtomicSwap32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicSwap64    = EmitBufferAtomic64;
+inline constexpr auto EmitBufferAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicISub32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicSMin32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
+inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicFMin32    = EmitBufferFloatAtomic;
+inline constexpr auto EmitBufferAtomicFMax32    = EmitBufferFloatAtomic;
+inline constexpr auto EmitLoadSharedU8          = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU16         = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32         = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32x2       = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32x3       = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32x4       = EmitLoadMemory;
+inline constexpr auto EmitWriteSharedU8         = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU16        = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32        = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32x2      = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32x3      = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32x4      = EmitStoreMemory;
+inline constexpr auto EmitSharedAtomicFMin32    = EmitSharedFloatAtomic;
+inline constexpr auto EmitSharedAtomicFMax32    = EmitSharedFloatAtomic;
+inline constexpr auto EmitSharedAtomicSwap32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicISub32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicInc32     = EmitSharedIncDec;
+inline constexpr auto EmitSharedAtomicDec32     = EmitSharedIncDec;
+inline constexpr auto EmitSharedAtomicSMin32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicUMin32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicXor32     = EmitAtomic32;
+inline constexpr auto EmitDataAppend            = EmitAppendConsume;
+inline constexpr auto EmitDataConsume           = EmitAppendConsume;
+uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
+inline constexpr auto EmitImageQueryDimensions = EmitImage;
+inline constexpr auto EmitImageQueryLod        = EmitImage;
+inline constexpr auto EmitImageRead            = EmitImage;
+inline constexpr auto EmitImageWrite           = EmitImage;
+inline constexpr auto EmitImageSampleRaw       = EmitImage;
+inline constexpr auto EmitImageGatherRaw       = EmitImage;
+inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
+inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
+inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
+inline constexpr auto EmitImageAtomicOr32      = EmitImage;
+inline constexpr auto EmitImageAtomicXor32     = EmitImage;
+void                  EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);
+inline constexpr auto EmitPhi                        = EmitUnreachable;
+inline constexpr auto EmitGetThreadBitScalarRegister = EmitUnreachable;
+inline constexpr auto EmitSetThreadBitScalarRegister = EmitUnreachable;
+inline constexpr auto EmitGetScalarMaskTag           = EmitUnreachable;
+inline constexpr auto EmitSetScalarMaskTag           = EmitUnreachable;
+inline constexpr auto EmitGetScalarRegister          = EmitUnreachable;
+inline constexpr auto EmitSetScalarRegister          = EmitUnreachable;
+inline constexpr auto EmitGetVectorRegister          = EmitUnreachable;
+inline constexpr auto EmitSetVectorRegister          = EmitUnreachable;
+inline constexpr auto EmitGetGotoVariable            = EmitUnreachable;
+inline constexpr auto EmitSetGotoVariable            = EmitUnreachable;
+inline constexpr auto EmitGetScc                     = EmitUnreachable;
+inline constexpr auto EmitSetScc                     = EmitUnreachable;
+inline constexpr auto EmitGetExec                    = EmitUnreachable;
+inline constexpr auto EmitSetExec                    = EmitUnreachable;
+inline constexpr auto EmitGetExecLo                  = EmitUnreachable;
+inline constexpr auto EmitSetExecLo                  = EmitUnreachable;
+inline constexpr auto EmitGetExecHi                  = EmitUnreachable;
+inline constexpr auto EmitSetExecHi                  = EmitUnreachable;
+inline constexpr auto EmitGetVcc                     = EmitUnreachable;
+inline constexpr auto EmitSetVcc                     = EmitUnreachable;
+inline constexpr auto EmitGetVccLo                   = EmitUnreachable;
+inline constexpr auto EmitSetVccLo                   = EmitUnreachable;
+inline constexpr auto EmitGetVccHi                   = EmitUnreachable;
+inline constexpr auto EmitSetVccHi                   = EmitUnreachable;
+inline constexpr auto EmitGetM0                      = EmitUnreachable;
+inline constexpr auto EmitSetM0                      = EmitUnreachable;
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -17,6 +17,7 @@
 #include <cstring>
 #include <iterator>
 #include <map>
+#include <spirv/unified1/GLSL.std.450.h>
 #include <string>
 #include <unordered_map>
 #include <utility>
@@ -24,307 +25,13 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-enum : uint32_t {
-	ExecutionModelVertex                     = 0,
-	ExecutionModelFragment                   = 4,
-	ExecutionModelGLCompute                  = 5,
-	ExecutionModeOriginUpperLeft             = 7,
-	ExecutionModeEarlyFragmentTests          = 9,
-	ExecutionModeDepthReplacing              = 12,
-	ExecutionModeLocalSize                   = 17,
-	ExecutionModeSignedZeroInfNanPreserve    = 4461,
-	ExecutionModeDerivativeGroupQuadsKHR     = 5289,
-	AddressingModelLogical                   = 0,
-	AddressingModelPhysicalStorageBuffer64   = 5348,
-	MemoryModelGLSL450                       = 1,
-	CapabilityShader                         = 1,
-	CapabilityInt64                          = 11,
-	CapabilityInt64Atomics                   = 12,
-	CapabilityImageGatherExtended            = 25,
-	CapabilityClipDistance                   = 32,
-	CapabilityCullDistance                   = 33,
-	CapabilitySampleRateShading              = 35,
-	CapabilitySampled1D                      = 43,
-	CapabilityImage1D                        = 44,
-	CapabilityImageQuery                     = 50,
-	CapabilityStorageImageWriteWithoutFormat = 56,
-	CapabilityGroupNonUniform                = 61,
-	CapabilityGroupNonUniformBallot          = 64,
-	CapabilityGroupNonUniformShuffle         = 65,
-	CapabilityShaderLayer                    = 69,
-	CapabilityShaderViewportIndex            = 70,
-	CapabilitySignedZeroInfNanPreserve       = 4466,
-	CapabilityFragmentBarycentricKHR         = 5284,
-	CapabilityComputeDerivativeGroupQuadsKHR = 5288,
-	CapabilityPhysicalStorageBufferAddresses = 5347,
-	StorageClassUniformConstant              = 0,
-	StorageClassInput                        = 1,
-	StorageClassOutput                       = 3,
-	StorageClassWorkgroup                    = 4,
-	StorageClassPrivate                      = 6,
-	StorageClassFunction                     = 7,
-	StorageClassPushConstant                 = 9,
-	StorageClassImage                        = 11,
-	StorageClassStorageBuffer                = 12,
-	StorageClassPhysicalStorageBuffer        = 5349,
-	FunctionControlNone                      = 0,
-	SelectionControlNone                     = 0,
-	LoopControlNone                          = 0,
+struct InputBinding : IR::StageInput {
+	uint32_t variable_id = 0;
 };
 
-enum : uint32_t {
-	DecorationBlock         = 2,
-	DecorationBuiltIn       = 11,
-	DecorationNoPerspective = 13,
-	DecorationFlat          = 14,
-	DecorationAliased       = 20,
-	DecorationLocation      = 30,
-	DecorationArrayStride   = 6,
-	DecorationBinding       = 33,
-	DecorationDescriptorSet = 34,
-	DecorationOffset        = 35,
-	DecorationPerVertexKHR  = 5285,
-};
-
-enum : uint32_t {
-	BuiltInPosition                  = 0,
-	BuiltInPointSize                 = 1,
-	BuiltInClipDistance              = 3,
-	BuiltInCullDistance              = 4,
-	BuiltInLayer                     = 9,
-	BuiltInViewportIndex              = 10,
-	BuiltInFragCoord                 = 15,
-	BuiltInFrontFacing               = 17,
-	BuiltInSampleId                  = 18,
-	BuiltInSampleMask                = 20,
-	BuiltInFragDepth                 = 22,
-	BuiltInWorkgroupId               = 26,
-	BuiltInLocalInvocationId         = 27,
-	BuiltInGlobalInvocationId        = 28,
-	BuiltInLocalInvocationIndex      = 29,
-	BuiltInSubgroupLocalInvocationId = 41,
-	BuiltInVertexIndex               = 42,
-	BuiltInInstanceIndex             = 43,
-	BuiltInBaryCoordKHR              = 5286,
-	BuiltInBaryCoordNoPerspKHR       = 5287,
-};
-
-enum : uint32_t {
-	Dim1D              = 0,
-	Dim3D              = 2,
-	Dim2D              = 1,
-	ImageFormatUnknown = 0,
-	ImageFormatRgba32f = 1,
-	ImageFormatR32ui   = 33,
-};
-
-enum : uint32_t {
-	ImageOperandsBiasMask         = 0x00000001u,
-	ImageOperandsLodMask          = 0x00000002u,
-	ImageOperandsGradMask         = 0x00000004u,
-	ImageOperandsOffsetMask       = 0x00000010u,
-	ImageOperandsConstOffsetsMask = 0x00000020u,
-	ImageOperandsSampleMask       = 0x00000040u,
-};
-
-enum : uint32_t {
-	ScopeDevice                    = 1,
-	ScopeWorkgroup                 = 2,
-	ScopeSubgroup                  = 3,
-	MemorySemanticsNone            = 0,
-	MemorySemanticsAcquireRelease  = 0x00000008u,
-	MemorySemanticsUniformMemory   = 0x00000040u,
-	MemorySemanticsWorkgroupMemory = 0x00000100u,
-	MemorySemanticsImageMemory     = 0x00000800u,
-};
-
-enum : uint32_t {
-	OpExtInst                      = 12,
-	OpTypeVoid                     = 19,
-	OpTypeBool                     = 20,
-	OpTypeInt                      = 21,
-	OpTypeFloat                    = 22,
-	OpTypeVector                   = 23,
-	OpTypeImage                    = 25,
-	OpTypeSampler                  = 26,
-	OpTypeSampledImage             = 27,
-	OpTypeArray                    = 28,
-	OpTypeRuntimeArray             = 29,
-	OpTypeStruct                   = 30,
-	OpTypePointer                  = 32,
-	OpTypeFunction                 = 33,
-	OpConstantTrue                 = 41,
-	OpConstantFalse                = 42,
-	OpConstant                     = 43,
-	OpConstantComposite            = 44,
-	OpUndef                        = 1,
-	OpFunction                     = 54,
-	OpFunctionParameter            = 55,
-	OpFunctionEnd                  = 56,
-	OpFunctionCall                 = 57,
-	OpVariable                     = 59,
-	OpImageTexelPointer            = 60,
-	OpLoad                         = 61,
-	OpStore                        = 62,
-	OpAccessChain                  = 65,
-	OpArrayLength                  = 68,
-	OpDecorate                     = 71,
-	OpMemberDecorate               = 72,
-	OpVectorShuffle                = 79,
-	OpCompositeConstruct           = 80,
-	OpCompositeExtract             = 81,
-	OpCopyObject                   = 83,
-	OpSampledImage                 = 86,
-	OpImageSampleImplicitLod       = 87,
-	OpImageSampleExplicitLod       = 88,
-	OpImageSampleDrefImplicitLod   = 89,
-	OpImageSampleDrefExplicitLod   = 90,
-	OpImageFetch                   = 95,
-	OpImageGather                  = 96,
-	OpImageDrefGather              = 97,
-	OpImageWrite                   = 99,
-	OpImageQuerySizeLod            = 103,
-	OpImageQueryLod                = 105,
-	OpImageQuerySize               = 104,
-	OpImageQueryLevels             = 106,
-	OpConvertFToU                  = 109,
-	OpConvertFToS                  = 110,
-	OpConvertSToF                  = 111,
-	OpConvertUToF                  = 112,
-	OpUConvert                     = 113,
-	OpConvertUToPtr                = 120,
-	OpBitcast                      = 124,
-	OpSNegate                      = 126,
-	OpFNegate                      = 127,
-	OpIAdd                         = 128,
-	OpFAdd                         = 129,
-	OpISub                         = 130,
-	OpFSub                         = 131,
-	OpIMul                         = 132,
-	OpFMul                         = 133,
-	OpUDiv                         = 134,
-	OpUMod                         = 137,
-	OpFDiv                         = 136,
-	OpIAddCarry                    = 149,
-	OpUMulExtended                 = 151,
-	OpSMulExtended                 = 152,
-	OpAny                          = 154,
-	OpAll                          = 155,
-	OpLogicalNotEqual              = 165,
-	OpLogicalOr                    = 166,
-	OpLogicalAnd                   = 167,
-	OpLogicalNot                   = 168,
-	OpSelect                       = 169,
-	OpIEqual                       = 170,
-	OpINotEqual                    = 171,
-	OpUGreaterThan                 = 172,
-	OpSGreaterThan                 = 173,
-	OpUGreaterThanEqual            = 174,
-	OpSGreaterThanEqual            = 175,
-	OpULessThan                    = 176,
-	OpSLessThan                    = 177,
-	OpULessThanEqual               = 178,
-	OpSLessThanEqual               = 179,
-	OpFOrdEqual                    = 180,
-	OpFUnordEqual                  = 181,
-	OpFOrdNotEqual                 = 182,
-	OpFUnordNotEqual               = 183,
-	OpFOrdLessThan                 = 184,
-	OpFUnordLessThan               = 185,
-	OpFOrdGreaterThan              = 186,
-	OpFUnordGreaterThan            = 187,
-	OpFOrdLessThanEqual            = 188,
-	OpFUnordLessThanEqual          = 189,
-	OpFOrdGreaterThanEqual         = 190,
-	OpFUnordGreaterThanEqual       = 191,
-	OpShiftRightLogical            = 194,
-	OpShiftRightArithmetic         = 195,
-	OpShiftLeftLogical             = 196,
-	OpBitwiseOr                    = 197,
-	OpBitwiseXor                   = 198,
-	OpBitwiseAnd                   = 199,
-	OpNot                          = 200,
-	OpBitFieldInsert               = 201,
-	OpBitFieldSExtract             = 202,
-	OpBitFieldUExtract             = 203,
-	OpBitReverse                   = 204,
-	OpBitCount                     = 205,
-	OpControlBarrier               = 224,
-	OpMemoryBarrier                = 225,
-	OpAtomicLoad                   = 227,
-	OpAtomicExchange               = 229,
-	OpAtomicCompareExchange        = 230,
-	OpAtomicIAdd                   = 234,
-	OpAtomicISub                   = 235,
-	OpAtomicSMin                   = 236,
-	OpAtomicUMin                   = 237,
-	OpAtomicSMax                   = 238,
-	OpAtomicUMax                   = 239,
-	OpAtomicAnd                    = 240,
-	OpAtomicOr                     = 241,
-	OpAtomicXor                    = 242,
-	OpPhi                          = 245,
-	OpLoopMerge                    = 246,
-	OpSelectionMerge               = 247,
-	OpLabel                        = 248,
-	OpBranch                       = 249,
-	OpBranchConditional            = 250,
-	OpSwitch                       = 251,
-	OpKill                         = 252,
-	OpReturn                       = 253,
-	OpReturnValue                  = 254,
-	OpGroupNonUniformBallot        = 339,
-	OpGroupNonUniformBallotFindLSB = 343,
-	OpGroupNonUniformShuffle       = 345,
-};
-
-enum : uint32_t {
-	MemoryAccessAlignedMask = 0x2,
-};
-
-enum : uint32_t {
-	GlslRoundEven       = 2,
-	GlslTrunc           = 3,
-	GlslFAbs            = 4,
-	GlslFloor           = 8,
-	GlslCeil            = 9,
-	GlslFract           = 10,
-	GlslSin             = 13,
-	GlslCos             = 14,
-	GlslExp2            = 29,
-	GlslLog2            = 30,
-	GlslSqrt            = 31,
-	GlslInverseSqrt     = 32,
-	GlslFMin            = 37,
-	GlslFMax            = 40,
-	GlslFClamp          = 43,
-	GlslLdexp           = 53,
-	GlslFma             = 50,
-	GlslPackSnorm2x16   = 56,
-	GlslPackUnorm2x16   = 57,
-	GlslPackHalf2x16    = 58,
-	GlslUnpackUnorm2x16 = 61,
-	GlslUnpackHalf2x16  = 62,
-	GlslFindILsb        = 73,
-	GlslFindUMsb        = 75,
-};
-
-struct InputBinding {
-	IR::StageInputKind kind            = IR::StageInputKind::VertexIndex;
-	uint32_t           location        = 0;
-	uint32_t           component_count = 1;
-	uint32_t           variable_id     = 0;
-	std::string        debug_name;
-	bool               per_vertex = false;
-};
-
-struct OutputBinding {
-	IR::StageOutputKind kind        = IR::StageOutputKind::Parameter;
-	uint32_t            index       = 0;
-	uint32_t            location    = 0;
-	uint32_t            variable_id = 0;
-	std::string         debug_name;
-	uint32_t            mesh_data_variable = 0;
+struct OutputBinding : IR::StageOutput {
+	uint32_t variable_id        = 0;
+	uint32_t mesh_data_variable = 0;
 };
 
 using ImageDimension = Decoder::ImageDimension;
@@ -339,27 +46,41 @@ struct ImageDimensionInfo {
 };
 
 constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
-    {ImageDimension::Dim1D, Dim1D, 1, 1, 0, 0},
-    {ImageDimension::Dim1DArray, Dim1D, 2, 1, 1, 0},
-    {ImageDimension::Dim2D, Dim2D, 2, 2, 0, 0},
-    {ImageDimension::Dim2DArray, Dim2D, 3, 2, 1, 0},
-    {ImageDimension::Dim3D, Dim3D, 3, 3, 0, 0},
-    {ImageDimension::Dim2DMsaa, Dim2D, 2, 2, 0, 1},
-    {ImageDimension::Dim2DMsaaArray, Dim2D, 3, 2, 1, 1},
+    {ImageDimension::Dim1D, spv::Dim1D, 1, 1, 0, 0},
+    {ImageDimension::Dim1DArray, spv::Dim1D, 2, 1, 1, 0},
+    {ImageDimension::Dim2D, spv::Dim2D, 2, 2, 0, 0},
+    {ImageDimension::Dim2DArray, spv::Dim2D, 3, 2, 1, 0},
+    {ImageDimension::Dim3D, spv::Dim3D, 3, 3, 0, 0},
+    {ImageDimension::Dim2DMsaa, spv::Dim2D, 2, 2, 0, 1},
+    {ImageDimension::Dim2DMsaaArray, spv::Dim2D, 3, 2, 1, 1},
 }};
 
 const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 
+struct SpirvRequirements {
+	bool subgroup_ballot              = false;
+	bool subgroup_shuffle             = false;
+	bool subgroup_local_invocation_id = false;
+	bool compute_derivatives          = false;
+	bool image_gather_extended        = false;
+	bool function_lds                 = false;
+	bool function_scratch             = false;
+	bool pixel_valid_mask             = false;
+	bool buffer_int64_atomics         = false;
+};
+
+SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
+
 struct EmitterState {
 	EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
 	    : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
-	      program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {}
+	      program(program_), input_info(input_info_),
+	      requirements(AnalyzeProgramRequirements(program_)) {}
 
 	Builder                                          builder;
 	const IR::Program&                               program;
 	ShaderStageInputInfo                             input_info;
-	const IR::SpirvRequirements&                     requirements;
-	ShaderType                                       stage                   = ShaderType::Unknown;
+	const SpirvRequirements                          requirements;
 	uint32_t                                         lane_count              = 1;
 	uint32_t                                         lane_half               = 0;
 	uint32_t                                         storage_buffer_variable = 0;
@@ -394,8 +115,6 @@ struct EmitterState {
 	uint32_t                   cull_distance_variable                = 0;
 	uint32_t                   layer_variable                        = 0;
 	uint32_t                   viewport_index_variable               = 0;
-	uint32_t                   clip_distance_count                   = 0;
-	uint32_t                   cull_distance_count                   = 0;
 	uint32_t                   depth_variable                        = 0;
 	uint32_t                   sample_mask_variable                  = 0;
 	std::vector<InputBinding>  inputs;
@@ -409,7 +128,6 @@ uint32_t TypeBool(EmitterState& state);
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
 uint32_t TypeU64(EmitterState& state);
-uint32_t TypeDeviceAddress(EmitterState& state);
 uint32_t TypeScalarU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
@@ -420,21 +138,37 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
 uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
-uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee);
+uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
 uint32_t TypeFunction(EmitterState& state);
 uint32_t TypeStorageBufferPointer(EmitterState& state);
 uint32_t TypeStorageBufferElementPointer(EmitterState& state);
 uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
 uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
-uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
 uint32_t TypePhysicalU32Pointer(EmitterState& state);
 uint32_t TypePushConstantElementPointer(EmitterState& state);
-uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
-uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
+uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
+uint32_t TypeU32ElementPointer(EmitterState& state, spv::StorageClass storage_class);
 
 inline void EmitLabel(EmitterState& state, uint32_t label) {
 	state.current_label = label;
-	state.builder.AddFunction({OpLabel, label});
+	state.builder.AddFunction({spv::OpLabel, label});
+}
+
+uint32_t TypeId(EmitterState& state, IR::Type type);
+
+// Shared instruction construction; typed aliases add no forwarding functions.
+template <spv::Op opcode, IR::Type type, typename... Args>
+uint32_t EmitNative(EmitterState& state, Args... args) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, TypeId(state, type), result, args...});
+	return result;
+}
+
+uint32_t GlslStd450(EmitterState& state);
+
+template <GLSLstd450 opcode, IR::Type type, typename... Args>
+uint32_t EmitGlsl(EmitterState& state, Args... args) {
+	return EmitNative<spv::OpExtInst, type>(state, GlslStd450(state), opcode, args...);
 }
 
 inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
@@ -444,16 +178,16 @@ inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint3
 }
 
 inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
-                            uint32_t rhs) {
+                       uint32_t rhs) {
 	const auto result = state.builder.AllocateId();
 	state.builder.AddFunction({opcode, type, result, lhs, rhs});
 	return result;
 }
 
-inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition,
-                            uint32_t true_value, uint32_t false_value) {
+inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
+                       uint32_t false_value) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
+	state.builder.AddFunction({spv::OpSelect, type, result, condition, true_value, false_value});
 	return result;
 }
 
@@ -467,9 +201,6 @@ struct ValueEmitContext {
 	uint32_t              FirstLane(uint32_t ballot);
 	uint32_t              Shuffle(const IR::Inst& inst, size_t index, uint32_t lane);
 	uint32_t              Result(const IR::Inst& inst);
-	uint32_t              TypeId(IR::Type type) const;
-	uint32_t              Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
-	                           std::initializer_list<uint32_t> args);
 	uint32_t              Define(const IR::Inst& inst, uint32_t value);
 	uint32_t              ResourceIndex(IR::Value value, IR::ValueOpcode opcode);
 	const IR::Inst*       ImageAddress(IR::Value value);
@@ -525,12 +256,6 @@ uint32_t VertexParameterComponentCount(const InputBinding& input);
 
 uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind);
 
-uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
-                                         uint32_t components);
-
-bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
-
-void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
 
@@ -570,7 +295,7 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource);
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
                            uint32_t texel);
 
-uint32_t ExecutionModelForStage(ShaderType stage);
+spv::ExecutionModel ExecutionModelForStage(ShaderType stage);
 
 uint32_t ConstantU32(EmitterState& state, uint32_t value);
 
@@ -588,24 +313,7 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value);
 
 uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
 
-uint32_t GlslStd450(EmitterState& state);
-
-void AllocateInputVariables(EmitterState& state);
-
-void AllocateOutputVariables(EmitterState& state);
-
-uint32_t BuiltInForInput(IR::StageInputKind kind);
-
-void AddInputAnnotationsAndNames(EmitterState& state);
-
-void AddOutputAnnotationsAndNames(EmitterState& state);
-
-void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name,
-                        IR::DescriptorBindingKind kind);
-
-void AddDescriptorAnnotationsAndNames(EmitterState& state);
-
-void DefineModule(EmitterState& state);
+void     DefineModule(EmitterState& state);
 void     DefineMeshOutputs(EmitterState& state);
 void     EmitMeshEntryPoint(EmitterState& state);
 void     EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -640,7 +348,7 @@ uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane);
 
-uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitAddU32 = EmitNative<spv::OpIAdd, IR::Type::U32, uint32_t, uint32_t>;
 
 uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
 
@@ -665,10 +373,9 @@ struct MemoryResourceAccess {
 
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
-MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
-                                                         const IR::MemoryInfo& mem,
-                                                         uint32_t variable,
-                                                         uint32_t pointer_type);
+MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState&         state,
+                                                        const IR::MemoryInfo& mem,
+                                                        uint32_t variable, uint32_t pointer_type);
 
 uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
                                 uint32_t raw_index);
@@ -679,14 +386,13 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
 uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
                                   uint32_t index);
 
-uint32_t EmitStorageBufferElementPointer(EmitterState& state,
-                                         const MemoryResourceAccess& access, uint32_t index,
-                                         uint32_t pointer_type);
+uint32_t EmitStorageBufferElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                         uint32_t index, uint32_t pointer_type);
 
 uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value);
 
-uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
-                              uint32_t false_value);
+inline constexpr auto EmitTBufferSelectF32 =
+    EmitNative<spv::OpSelect, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
 
 bool IsSignedFormatComponent(Format::ComponentType type);
 
@@ -702,20 +408,14 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
 
-uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
-                            uint32_t false_value);
-
-void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
-                                   uint32_t& out_low, uint32_t& out_high);
-
-void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
-                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high);
+inline constexpr auto EmitSelectValueU32 =
+    EmitNative<spv::OpSelect, IR::Type::U32, uint32_t, uint32_t, uint32_t>;
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
 
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
 
-uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitOrU32 = EmitNative<spv::OpBitwiseOr, IR::Type::U32, uint32_t, uint32_t>;
 
 uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                 uint32_t constant);
@@ -728,17 +428,19 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 
 uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
-uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value);
+inline constexpr auto EmitBitcastF32ToU32 = EmitNative<spv::OpBitcast, IR::Type::U32, uint32_t>;
 
-uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value);
+inline constexpr auto EmitBitcastU32ToF32 = EmitNative<spv::OpBitcast, IR::Type::F32, uint32_t>;
 
-uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitAndU32 = EmitNative<spv::OpBitwiseAnd, IR::Type::U32, uint32_t, uint32_t>;
 
-uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitLogicalAndBool =
+    EmitNative<spv::OpLogicalAnd, IR::Type::U1, uint32_t, uint32_t>;
 
-uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitLogicalOrBool =
+    EmitNative<spv::OpLogicalOr, IR::Type::U1, uint32_t, uint32_t>;
 
-uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value);
+inline constexpr auto EmitLogicalNotBool = EmitNative<spv::OpLogicalNot, IR::Type::U1, uint32_t>;
 
 F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits);
 
@@ -751,26 +453,18 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
 
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
-uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value);
+inline constexpr auto EmitTruncF32Value = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
 
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
 
 uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
 
-uint32_t EmitFNegateValue(EmitterState& state, uint32_t value);
+inline constexpr auto EmitFNegateValue = EmitNative<spv::OpFNegate, IR::Type::F32, uint32_t>;
 
-uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
+inline constexpr auto EmitFAbsValue = EmitGlsl<GLSLstd450FAbs, IR::Type::F32, uint32_t>;
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 
-bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
-
-bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst);
-
-bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
-
-bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
-
 void EmitProgram(EmitterState& state);
 
 void DefineGetBdaPointer(EmitterState& state);
@@ -780,11 +474,11 @@ template <typename Fn>
 void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
 	const auto then_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label});
+	state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
+	state.builder.AddFunction({spv::OpBranchConditional, condition, then_label, merge_label});
 	EmitLabel(state, then_label);
 	fn();
-	state.builder.AddFunction({OpBranch, merge_label});
+	state.builder.AddFunction({spv::OpBranch, merge_label});
 	EmitLabel(state, merge_label);
 }
 
@@ -795,19 +489,19 @@ uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition,
 	const auto then_exit   = state.builder.AllocateId();
 	const auto else_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
+	state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
+	state.builder.AddFunction({spv::OpBranchConditional, condition, then_label, else_label});
 	EmitLabel(state, then_label);
 	const auto then_value = fn();
-	state.builder.AddFunction({OpBranch, then_exit});
+	state.builder.AddFunction({spv::OpBranch, then_exit});
 	EmitLabel(state, then_exit);
-	state.builder.AddFunction({OpBranch, merge_label});
+	state.builder.AddFunction({spv::OpBranch, merge_label});
 	EmitLabel(state, else_label);
-	state.builder.AddFunction({OpBranch, merge_label});
+	state.builder.AddFunction({spv::OpBranch, merge_label});
 	EmitLabel(state, merge_label);
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpPhi, type, value, then_value, then_exit, default_value, else_label});
+	    {spv::OpPhi, type, value, then_value, then_exit, default_value, else_label});
 	return value;
 }
 
@@ -819,12 +513,12 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
 
 template <typename Fn>
 uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
-	const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
+	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto memory = [&] {
 		switch (kind) {
-			case IR::ResourceKind::Lds: return MemorySemanticsWorkgroupMemory;
-			case IR::ResourceKind::Image: return MemorySemanticsImageMemory;
-			default: return MemorySemanticsUniformMemory;
+			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
+			default: return spv::MemorySemanticsUniformMemoryMask;
 		}
 	}();
 	const auto preheader = state.builder.AllocateId();
@@ -834,27 +528,30 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
 	const auto initial   = state.builder.AllocateId();
 	const auto observed  = state.builder.AllocateId();
 	const auto exchanged = state.builder.AllocateId();
-	state.builder.AddFunction({OpBranch, preheader});
+	state.builder.AddFunction({spv::OpBranch, preheader});
 	EmitLabel(state, preheader);
-	state.builder.AddFunction({OpAtomicLoad, TypeU32(state), initial, pointer,
-	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
-	state.builder.AddFunction({OpBranch, header});
+	state.builder.AddFunction({spv::OpAtomicLoad, TypeU32(state), initial, pointer,
+	                           ConstantU32(state, scope),
+	                           ConstantU32(state, spv::MemorySemanticsMaskNone)});
+	state.builder.AddFunction({spv::OpBranch, header});
 	EmitLabel(state, header);
 	state.builder.AddFunction(
-	    {OpPhi, TypeU32(state), observed, initial, preheader, exchanged, cont});
+	    {spv::OpPhi, TypeU32(state), observed, initial, preheader, exchanged, cont});
 	const auto next = desired(observed);
-	state.builder.AddFunction({OpAtomicCompareExchange, TypeU32(state), exchanged, pointer,
-	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
-	                           ConstantU32(state, MemorySemanticsNone), next, observed});
+	state.builder.AddFunction({spv::OpAtomicCompareExchange, TypeU32(state), exchanged, pointer,
+	                           ConstantU32(state, scope),
+	                           ConstantU32(state, spv::MemorySemanticsMaskNone),
+	                           ConstantU32(state, spv::MemorySemanticsMaskNone), next, observed});
 	const auto success = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, TypeBool(state), success, exchanged, observed});
-	state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone});
-	state.builder.AddFunction({OpBranchConditional, success, merge, cont});
+	state.builder.AddFunction({spv::OpIEqual, TypeBool(state), success, exchanged, observed});
+	state.builder.AddFunction({spv::OpLoopMerge, merge, cont, spv::LoopControlMaskNone});
+	state.builder.AddFunction({spv::OpBranchConditional, success, merge, cont});
 	EmitLabel(state, cont);
-	state.builder.AddFunction({OpBranch, header});
+	state.builder.AddFunction({spv::OpBranch, header});
 	EmitLabel(state, merge);
-	state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope),
-	                           ConstantU32(state, MemorySemanticsAcquireRelease | memory)});
+	state.builder.AddFunction(
+	    {spv::OpMemoryBarrier, ConstantU32(state, scope),
+	     ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory)});
 	return observed;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -7,30 +7,24 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
 		dword_index += state.program.bindings.push_data_start_dword;
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer,
-		                           state.push_constant_variable, ConstantU32(state, 0),
+		state.builder.AddFunction({spv::OpAccessChain, TypePushConstantElementPointer(state),
+		                           pointer, state.push_constant_variable, ConstantU32(state, 0),
 		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
 		return value;
 	}
 	if (state.shader_data_storage_variable != 0) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
-		                           state.shader_data_storage_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+		                           pointer, state.shader_data_storage_variable,
+		                           ConstantU32(state, 0), ConstantU32(state, dword_index)});
+		state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
 		return value;
 	}
 	return ConstantU32(state, 0);
 }
 
-uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAdd, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
 uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
@@ -59,13 +53,13 @@ void EmitMemoryOffsets(EmitterState& state) {
 		    EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
 		const auto shift             = ConstantU32(state, (i % 4u) * 8u);
 		state.memory_byte_offsets[i] = EmitBinaryU32(
-		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    state, spv::OpBitwiseAnd, EmitBinaryU32(state, spv::OpShiftRightLogical, word, shift),
 		    ConstantU32(state, 0xffu));
 	}
 }
 
 uint32_t LdsDwordCount(const EmitterState& state) {
-	const auto* workgroup = ShaderWorkgroupInput(state.stage, state.input_info);
+	const auto* workgroup = ShaderWorkgroupInput(state.program.stage, state.input_info);
 	return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u;
 }
 
@@ -73,12 +67,12 @@ static void EnsureLdsStorage(EmitterState& state) {
 	if (state.lds_variable != 0) {
 		return;
 	}
-	if (ShaderWorkgroupInput(state.stage, state.input_info) == nullptr) {
+	if (ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
 		EXIT("function LDS was not prepared before SPIR-V function emission\n");
 	}
 	state.lds_variable = state.builder.DefineGlobalVariable(
-	    TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)),
-	    StorageClassWorkgroup);
+	    TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsDwordCount(state)),
+	    spv::StorageClassWorkgroup);
 	state.builder.AddName(state.lds_variable, "lds_dwords");
 }
 
@@ -94,12 +88,12 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
 	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
 	MemoryResourceAccess access {.kind = mem.kind};
 	access.object_pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, pointer_type, access.object_pointer, variable,
+	state.builder.AddFunction({spv::OpAccessChain, pointer_type, access.object_pointer, variable,
 	                           ConstantU32(state, array_index)});
 	access.byte_offset = state.memory_byte_offsets[array_index];
 	access.length      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	    {spv::OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
 	return access;
 }
 
@@ -137,9 +131,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 		case IR::ResourceKind::Buffer: {
 			access = PrepareStorageBufferResourceAccess(
 			    state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
-			access.index_offset =
-			    EmitBinaryU32(state, OpShiftRightLogical, access.byte_offset,
-			                  ConstantU32(state, 2u));
+			access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
+			                                    ConstantU32(state, 2u));
 			access.add_index_offset = true;
 			return access;
 		}
@@ -147,7 +140,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 	}
 	access.length = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	    {spv::OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
 	return access;
 }
 
@@ -159,7 +152,7 @@ uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess&
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
                                    uint32_t index) {
 	const auto in_bounds = state.builder.AllocateId();
-	state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length});
+	state.builder.AddFunction({spv::OpULessThan, TypeBool(state), in_bounds, index, access.length});
 	return in_bounds;
 }
 
@@ -167,11 +160,12 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
                                   uint32_t index) {
 	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
 		const auto pointer = state.builder.AllocateId();
-		const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
-		                           : ShaderWorkgroupInput(state.stage, state.input_info) != nullptr
-		                               ? StorageClassWorkgroup
-		                               : StorageClassFunction;
-		state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
+		const auto storage_class =
+		    access.kind == IR::ResourceKind::Scratch ? spv::StorageClassFunction
+		    : ShaderWorkgroupInput(state.program.stage, state.input_info) != nullptr
+		        ? spv::StorageClassWorkgroup
+		        : spv::StorageClassFunction;
+		state.builder.AddFunction({spv::OpAccessChain, TypeU32ElementPointer(state, storage_class),
 		                           pointer, access.object_pointer, index});
 		return pointer;
 	}
@@ -183,21 +177,14 @@ uint32_t EmitStorageBufferElementPointer(EmitterState& state,
                                          const MemoryResourceAccess& access, uint32_t index,
                                          uint32_t pointer_type) {
 	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, pointer_type, pointer, access.object_pointer,
+	state.builder.AddFunction({spv::OpAccessChain, pointer_type, pointer, access.object_pointer,
 	                           ConstantU32(state, 0), index});
 	return pointer;
 }
 
 uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeI32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
-                              uint32_t false_value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, TypeF32(state), ret, condition, true_value, false_value});
+	state.builder.AddFunction({spv::OpBitcast, TypeI32(state), ret, value});
 	return ret;
 }
 
@@ -210,33 +197,34 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
 	const auto mantissa_bits = bits == 11u ? 6u : 5u;
 	const auto mantissa_mask = (1u << mantissa_bits) - 1u;
 	const auto mantissa =
-	    EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
+	    EmitBinaryU32(state, spv::OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
 	const auto exponent = EmitBinaryU32(
-	    state, OpBitwiseAnd,
-	    EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
+	    state, spv::OpBitwiseAnd,
+	    EmitBinaryU32(state, spv::OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
 	    ConstantU32(state, 0x1fu));
 
 	const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
 	const auto exponent_bits =
-	    EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
-	const auto mantissa_bits_32 =
-	    EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits));
-	const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32);
+	    EmitBinaryU32(state, spv::OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
+	const auto mantissa_bits_32 = EmitBinaryU32(state, spv::OpShiftLeftLogical, mantissa,
+	                                            ConstantU32(state, 23u - mantissa_bits));
+	const auto normal_bits =
+	    EmitBinaryU32(state, spv::OpBitwiseOr, exponent_bits, mantissa_bits_32);
 	const auto normal      = EmitBitcastU32ToF32(state, normal_bits);
 
 	const auto special_bits =
-	    EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
+	    EmitBinaryU32(state, spv::OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
 	const auto special = EmitBitcastU32ToF32(state, special_bits);
 
 	const auto mantissa_f32 = state.builder.AllocateId();
 	const auto subnormal    = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
+	state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
 	state.builder.AddFunction(
-	    {OpFMul, TypeF32(state), subnormal, mantissa_f32,
+	    {spv::OpFMul, TypeF32(state), subnormal, mantissa_f32,
 	     ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
 
-	const auto zero_exp    = EmitCompareU32Constant(state, OpIEqual, exponent, 0);
-	const auto special_exp = EmitCompareU32Constant(state, OpIEqual, exponent, 31);
+	const auto zero_exp    = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0);
+	const auto special_exp = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 31);
 	const auto finite      = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
 	const auto result      = EmitTBufferSelectF32(state, special_exp, special, finite);
 	return EmitBitcastF32ToU32(state, result);
@@ -250,22 +238,22 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 		case Format::ComponentType::Sint: return raw;
 		case Format::ComponentType::Uscaled: {
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
+			state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), value, raw});
 			return EmitBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Sscaled: {
 			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
 			const auto value      = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
+			state.builder.AddFunction({spv::OpConvertSToF, TypeF32(state), value, signed_raw});
 			return EmitBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Unorm: {
 			const auto value      = state.builder.AllocateId();
 			const auto normalized = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << bits) - 1u);
-			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
-			state.builder.AddFunction(
-			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
+			state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), value, raw});
+			state.builder.AddFunction({spv::OpFDiv, TypeF32(state), normalized, value,
+			                           ConstantF32Value(state, max_value)});
 			return EmitBitcastF32ToU32(state, normalized);
 		}
 		case Format::ComponentType::Snorm: {
@@ -274,11 +262,11 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			const auto normalized = state.builder.AllocateId();
 			const auto clamped    = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << (bits - 1u)) - 1u);
-			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
-			state.builder.AddFunction(
-			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
-			state.builder.AddFunction({OpExtInst, TypeF32(state), clamped, GlslStd450(state),
-			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			state.builder.AddFunction({spv::OpConvertSToF, TypeF32(state), value, signed_raw});
+			state.builder.AddFunction({spv::OpFDiv, TypeF32(state), normalized, value,
+			                           ConstantF32Value(state, max_value)});
+			state.builder.AddFunction({spv::OpExtInst, TypeF32(state), clamped, GlslStd450(state),
+			                           GLSLstd450FMax, normalized, ConstantF32Value(state, -1.0f)});
 			return EmitBitcastF32ToU32(state, clamped);
 		}
 		case Format::ComponentType::Float:
@@ -294,9 +282,10 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 }
 
 void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
-	const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory;
-	state.builder.AddFunction(
-	    {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
+	const auto semantics =
+	    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask;
+	state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice),
+	                           ConstantU32(state, semantics)});
 }
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
@@ -308,12 +297,12 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	};
 	const auto classify = [&](uint32_t bits) {
 		const auto cls = EmitClassifyF32Bits(state, bits);
-		const auto negative = EmitCompareU32Constant(
-		    state, OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u);
+		const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
+		                                             EmitAndConstant(state, bits, 0x80000000u), 0u);
 		const auto negative_key = state.builder.AllocateId();
-		state.builder.AddFunction({OpNot, TypeU32(state), negative_key, bits});
-		const auto positive_key = EmitBinaryU32(
-		    state, OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
+		state.builder.AddFunction({spv::OpNot, TypeU32(state), negative_key, bits});
+		const auto positive_key =
+		    EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
 		return OrderedBits {
 		    cls.nan, cls.zero,
 		    EmitSelectValueU32(state, negative, negative_key, positive_key)};
@@ -324,8 +313,8 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	    state, EmitLogicalOrBool(state, source_class.nan, old_class.nan),
 	    EmitLogicalAndBool(state, source_class.zero, old_class.zero));
 	const auto compare = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), compare,
-	                           source_class.key, old_class.key});
+	state.builder.AddFunction({max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state),
+	                           compare, source_class.key, old_class.key});
 	return EmitSelectValueU32(
 	    state, EmitLogicalAndBool(state, EmitLogicalNotBool(state, unordered), compare), source,
 	    old);
@@ -354,16 +343,16 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 		const auto base   = state.builder.AllocateId();
 		const auto target = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
+		    {spv::OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
 		state.builder.AddFunction(
-		    {OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
+		    {spv::OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
 		state.builder.AddFunction(
-		    {OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
+		    {spv::OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
+		    {spv::OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
-		state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, perm});
+		    {spv::OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
+		state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, base, perm});
 		return target;
 	}
 
@@ -373,16 +362,17 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
+	    {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
+	    {spv::OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), ored, masked,
+	                           ConstantU32(state, (control >> 5u) & 0x1fu)});
+	state.builder.AddFunction({spv::OpBitwiseXor, TypeU32(state), xored, ored,
+	                           ConstantU32(state, (control >> 10u) & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseXor, TypeU32(state), xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, xored});
+	    {spv::OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
+	state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, base, xored});
 	return target;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -3,24 +3,24 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
 
-uint32_t MeshArray(EmitterState& state, uint32_t storage, uint32_t type, uint32_t count) {
-	const auto array = state.builder.Type(OpTypeArray, {type, ConstantU32(state, count)});
+uint32_t MeshArray(EmitterState& state, spv::StorageClass storage, uint32_t type, uint32_t count) {
+	const auto array = state.builder.Type(spv::OpTypeArray, {type, ConstantU32(state, count)});
 	return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage);
 }
 
-uint32_t MeshElement(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type,
-                     uint32_t index) {
+uint32_t MeshElement(EmitterState& state, uint32_t variable, spv::StorageClass storage,
+                     uint32_t type, uint32_t index) {
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpAccessChain, TypePointer(state, storage, type), pointer, variable, index});
+	    {spv::OpAccessChain, TypePointer(state, storage, type), pointer, variable, index});
 	return pointer;
 }
 
-uint32_t MeshLoad(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type,
+uint32_t MeshLoad(EmitterState& state, uint32_t variable, spv::StorageClass storage, uint32_t type,
                   uint32_t index) {
 	const auto pointer = MeshElement(state, variable, storage, type, index);
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, type, value, pointer});
+	state.builder.AddFunction({spv::OpLoad, type, value, pointer});
 	return value;
 }
 
@@ -40,40 +40,44 @@ void DefineMeshOutputs(EmitterState& state) {
 		}
 		const auto type    = MeshOutputType(state, output.kind);
 		output.variable_id = MeshArray(
-		    state, StorageClassOutput, type,
+		    state, spv::StorageClassOutput, type,
 		    output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
 		// Only Layer is read by another invocation, through the primitive's provoking vertex.
 		const bool shared = output.kind == IR::StageOutputKind::Layer;
-		output.mesh_data_variable = MeshArray(
-		    state, shared ? StorageClassWorkgroup : StorageClassPrivate, type,
-		    shared ? mesh.max_vertices : state.lane_count);
+		output.mesh_data_variable =
+		    MeshArray(state, shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, type,
+		              shared ? mesh.max_vertices : state.lane_count);
 		state.interface_variables.push_back(output.variable_id);
 		state.builder.AddName(output.variable_id, output.debug_name.c_str());
 		if (output.kind == IR::StageOutputKind::Parameter) {
 			state.builder.AddAnnotation(
-			    {OpDecorate, output.variable_id, DecorationLocation, output.location});
+			    {spv::OpDecorate, output.variable_id, spv::DecorationLocation, output.location});
 		} else {
 			state.builder.AddAnnotation(
-			    {OpDecorate, output.variable_id, DecorationBuiltIn,
-			     output.kind == IR::StageOutputKind::Layer ? BuiltInLayer : BuiltInPosition});
+			    {spv::OpDecorate, output.variable_id, spv::DecorationBuiltIn,
+			     output.kind == IR::StageOutputKind::Layer ? spv::BuiltInLayer
+			                                               : spv::BuiltInPosition});
 		}
 		if (output.kind == IR::StageOutputKind::Layer) {
-			state.builder.AddAnnotation({OpDecorate, output.variable_id, 5271u}); // PerPrimitiveEXT
+			state.builder.AddAnnotation({spv::OpDecorate, output.variable_id,
+			                             spv::DecorationPerPrimitiveEXT}); // PerPrimitiveEXT
 		}
 	}
-	state.mesh_allocation = MeshArray(state, StorageClassWorkgroup, TypeU32(state), 2);
+	state.mesh_allocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2);
 	state.mesh_primitive_data =
-	    MeshArray(state, StorageClassPrivate, TypeU32(state), state.lane_count);
+	    MeshArray(state, spv::StorageClassPrivate, TypeU32(state), state.lane_count);
 	state.mesh_primitives =
-	    MeshArray(state, StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
-	state.mesh_cull = MeshArray(state, StorageClassOutput, TypeBool(state), mesh.max_primitives);
+	    MeshArray(state, spv::StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
+	state.mesh_cull =
+	    MeshArray(state, spv::StorageClassOutput, TypeBool(state), mesh.max_primitives);
 	state.interface_variables.push_back(state.mesh_primitives);
 	state.interface_variables.push_back(state.mesh_cull);
-	state.builder.AddAnnotation({OpDecorate, state.mesh_primitives, DecorationBuiltIn,
-	                             5296u}); // PrimitiveTriangleIndicesEXT
 	state.builder.AddAnnotation(
-	    {OpDecorate, state.mesh_cull, DecorationBuiltIn, 5299u}); // CullPrimitiveEXT
-	state.builder.AddAnnotation({OpDecorate, state.mesh_cull, 5271u});
+	    {spv::OpDecorate, state.mesh_primitives, spv::DecorationBuiltIn,
+	     spv::BuiltInPrimitiveTriangleIndicesEXT}); // PrimitiveTriangleIndicesEXT
+	state.builder.AddAnnotation({spv::OpDecorate, state.mesh_cull, spv::DecorationBuiltIn,
+	                             spv::BuiltInCullPrimitiveEXT}); // CullPrimitiveEXT
+	state.builder.AddAnnotation({spv::OpDecorate, state.mesh_cull, spv::DecorationPerPrimitiveEXT});
 }
 
 uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) {
@@ -85,113 +89,118 @@ uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32
 		     index);
 	}
 	const bool shared = kind == IR::StageOutputKind::Layer;
-	return MeshElement(state, output->mesh_data_variable,
-	                   shared ? StorageClassWorkgroup : StorageClassPrivate,
-	                   MeshOutputType(state, kind),
-	                   shared ? EmitLocalInvocationIndex(state)
-	                          : ConstantU32(state, state.lane_half));
+	return MeshElement(
+	    state, output->mesh_data_variable,
+	    shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, MeshOutputType(state, kind),
+	    shared ? EmitLocalInvocationIndex(state) : ConstantU32(state, state.lane_half));
 }
 
 uint32_t MeshPrimitivePointer(EmitterState& state) {
-	return MeshElement(state, state.mesh_primitive_data, StorageClassPrivate, TypeU32(state),
+	return MeshElement(state, state.mesh_primitive_data, spv::StorageClassPrivate, TypeU32(state),
 	                   ConstantU32(state, state.lane_half));
 }
 
 void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&      state = ctx.state;
 	const auto first = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpIEqual, TypeBool(state), first, EmitLocalInvocationIndex(state), ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpIEqual, TypeBool(state), first,
+	                           EmitLocalInvocationIndex(state), ConstantU32(state, 0)});
 	EmitIfCondition(state, first, [&] {
 		const auto allocation = ctx.Arg(inst, 0);
 		for (uint32_t field = 0; field < 2; field++) {
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), value, allocation,
+			state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(state), value, allocation,
 			                           ConstantU32(state, field * 12u),
 			                           ConstantU32(state, field == 0 ? 10u : 11u)});
-			const auto pointer = MeshElement(state, state.mesh_allocation, StorageClassWorkgroup,
-			                                 TypeU32(state), ConstantU32(state, field));
-			state.builder.AddFunction({OpStore, pointer, value});
+			const auto pointer =
+			    MeshElement(state, state.mesh_allocation, spv::StorageClassWorkgroup,
+			                TypeU32(state), ConstantU32(state, field));
+			state.builder.AddFunction({spv::OpStore, pointer, value});
 		}
 	});
 }
 
 void EmitMeshEntryPoint(EmitterState& state) {
-	state.builder.AddFunction(
-	    {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
+	state.builder.AddFunction({spv::OpFunction, TypeVoid(state), state.main_func,
+	                           spv::FunctionControlMaskNone, TypeFunction(state)});
 	EmitLabel(state, state.builder.AllocateId());
 	state.builder.AddFunction(
-	    {OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func});
+	    {spv::OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func});
 	// All guest waves finish before the uniform Vulkan allocation and output stores.
-	state.builder.AddFunction(
-	    {OpControlBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, ScopeWorkgroup),
-	     ConstantU32(state, MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory)});
-	const auto vertices = MeshLoad(state, state.mesh_allocation, StorageClassWorkgroup,
-	                               TypeU32(state), ConstantU32(state, 0));
-	const auto primitives = MeshLoad(state, state.mesh_allocation, StorageClassWorkgroup,
+	state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
+	                           ConstantU32(state, spv::ScopeWorkgroup),
+	                           ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask |
+	                                                  spv::MemorySemanticsWorkgroupMemoryMask)});
+	const auto vertices   = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
+	                                 TypeU32(state), ConstantU32(state, 0));
+	const auto primitives = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
 	                                 TypeU32(state), ConstantU32(state, 1));
-	state.builder.AddFunction({5295u, vertices, primitives}); // OpSetMeshOutputsEXT
+	state.builder.AddFunction(
+	    {spv::OpSetMeshOutputsEXT, vertices, primitives}); // OpSetMeshOutputsEXT
 	for (uint32_t half = 0; half < state.lane_count; half++) {
 		state.lane_half      = half;
 		const auto index     = EmitLocalInvocationIndex(state);
 		const auto is_vertex = state.builder.AllocateId();
-		state.builder.AddFunction({OpULessThan, TypeBool(state), is_vertex, index, vertices});
+		state.builder.AddFunction({spv::OpULessThan, TypeBool(state), is_vertex, index, vertices});
 		EmitIfCondition(state, is_vertex, [&] {
 			for (const auto& output: state.outputs) {
 				if (output.kind == IR::StageOutputKind::Layer) {
 					continue;
 				}
 				const auto type  = MeshOutputType(state, output.kind);
-				const auto value = MeshLoad(state, output.mesh_data_variable, StorageClassPrivate,
-				                            type, ConstantU32(state, half));
+				const auto value =
+				    MeshLoad(state, output.mesh_data_variable, spv::StorageClassPrivate, type,
+				             ConstantU32(state, half));
 				const auto pointer =
-				    MeshElement(state, output.variable_id, StorageClassOutput, type, index);
-				state.builder.AddFunction({OpStore, pointer, value});
+				    MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				state.builder.AddFunction({spv::OpStore, pointer, value});
 			}
 		});
 		const auto is_primitive = state.builder.AllocateId();
-		state.builder.AddFunction({OpULessThan, TypeBool(state), is_primitive, index, primitives});
+		state.builder.AddFunction(
+		    {spv::OpULessThan, TypeBool(state), is_primitive, index, primitives});
 		EmitIfCondition(state, is_primitive, [&] {
-			const auto packed = MeshLoad(state, state.mesh_primitive_data, StorageClassPrivate,
+			const auto packed = MeshLoad(state, state.mesh_primitive_data, spv::StorageClassPrivate,
 			                             TypeU32(state), ConstantU32(state, half));
 			uint32_t   vertex[3] {};
 			for (uint32_t component = 0; component < 3; component++) {
 				vertex[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), vertex[component],
-				                           packed, ConstantU32(state, component * 10u),
-				                           ConstantU32(state, 10)});
+				state.builder.AddFunction(
+				    {spv::OpBitFieldUExtract, TypeU32(state), vertex[component], packed,
+				     ConstantU32(state, component * 10u), ConstantU32(state, 10)});
 			}
 			const auto triangle = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
+			state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
 			                           vertex[0], vertex[1], vertex[2]});
-			const auto triangle_pointer = MeshElement(
-			    state, state.mesh_primitives, StorageClassOutput, TypeU32Vector(state, 3), index);
-			state.builder.AddFunction({OpStore, triangle_pointer, triangle});
+			const auto triangle_pointer =
+			    MeshElement(state, state.mesh_primitives, spv::StorageClassOutput,
+			                TypeU32Vector(state, 3), index);
+			state.builder.AddFunction({spv::OpStore, triangle_pointer, triangle});
 			const auto null_bit =
-			    EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
+			    EmitBinaryU32(state, spv::OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
 			const auto culled = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)});
-			state.builder.AddFunction(
-			    {OpStore,
-			     MeshElement(state, state.mesh_cull, StorageClassOutput, TypeBool(state), index),
-			     culled});
+			    {spv::OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)});
+			state.builder.AddFunction({spv::OpStore,
+			                           MeshElement(state, state.mesh_cull, spv::StorageClassOutput,
+			                                       TypeBool(state), index),
+			                           culled});
 			for (const auto& output: state.outputs) {
 				if (output.kind != IR::StageOutputKind::Layer) {
 					continue;
 				}
-				const auto layer = MeshLoad(
-				    state, output.mesh_data_variable, StorageClassWorkgroup, TypeU32(state),
-				    vertex[state.input_info.vertex->mesh.provoking_vertex]);
-				const auto pointer = MeshElement(state, output.variable_id, StorageClassOutput,
+				const auto layer = MeshLoad(state, output.mesh_data_variable,
+				                            spv::StorageClassWorkgroup, TypeU32(state),
+				                            vertex[state.input_info.vertex->mesh.provoking_vertex]);
+				const auto pointer = MeshElement(state, output.variable_id, spv::StorageClassOutput,
 				                                 TypeU32(state), index);
-				state.builder.AddFunction({OpStore, pointer, layer});
+				state.builder.AddFunction({spv::OpStore, pointer, layer});
 			}
 		});
 	}
 	state.lane_half = 0;
-	state.builder.AddFunction({OpReturn});
-	state.builder.AddFunction({OpFunctionEnd});
+	state.builder.AddFunction({spv::OpReturn});
+	state.builder.AddFunction({spv::OpFunctionEnd});
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include "common/assert.h"
 
@@ -6,7 +6,9 @@
 #include <bit>
 #include <functional>
 #include <optional>
+#include <type_traits>
 #include <unordered_set>
+#include <utility>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -15,11 +17,11 @@ void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
 	const auto kill_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	const auto inactive    = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, TypeBool(state), inactive, active});
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
+	state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), inactive, active});
+	state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
+	state.builder.AddFunction({spv::OpBranchConditional, inactive, kill_label, merge_label});
 	EmitLabel(state, kill_label);
-	state.builder.AddFunction({OpKill});
+	state.builder.AddFunction({spv::OpKill});
 	EmitLabel(state, merge_label);
 }
 
@@ -31,15 +33,15 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
 	const auto mask_value = state.builder.AllocateId();
 	const auto active     = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
+	    {spv::OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
 	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
+	    {spv::OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
 	EmitKillIfBoolFalse(state, active);
 }
 
 uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
-	const auto value_type = ctx.TypeId(type);
-	return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type);
+	const auto value_type = TypeId(ctx.state, type);
+	return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type);
 }
 
 struct DeferredPhiPatch {
@@ -74,7 +76,7 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
 		for (size_t index = 0; index < phi.NumArgs(); index++) {
 			if (phi.PhiBlock(index) == from) {
 				ctx.state.builder.AddFunction(
-				    {OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
+				    {spv::OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
 				break;
 			}
 		}
@@ -92,11 +94,13 @@ const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
 
 void EmitReturn(ValueEmitContext& ctx) {
 	EmitKillIfPixelValidMaskInactive(ctx.state);
-	ctx.state.builder.AddFunction({OpReturn});
+	ctx.state.builder.AddFunction({spv::OpReturn});
 }
 
 uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
+	// Scalar-instruction conditions already test the full wave's raw register values.
 	if (ctx.other_half == nullptr ||
+	    info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
 	    info.terminator.condition == CFG::BranchCondition::GotoVariable) {
 		return ctx.Def(info.condition);
 	}
@@ -104,15 +108,16 @@ uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
 	const auto low    = ctx.state.builder.AllocateId();
 	const auto high   = ctx.state.builder.AllocateId();
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
+	ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
+	ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
 	const auto kind     = info.terminator.condition;
 	const bool zero     = kind == CFG::BranchCondition::ExecZero ||
 	                      kind == CFG::BranchCondition::VccZero ||
 	                      kind == CFG::BranchCondition::SccZero;
-	const auto combined = EmitBinaryU32(ctx.state, zero ? OpBitwiseAnd : OpBitwiseOr, low, high);
-	ctx.state.builder.AddFunction({zero ? OpIEqual : OpINotEqual, TypeBool(ctx.state), result,
-	                               combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
+	const auto combined =
+	    EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
+	ctx.state.builder.AddFunction({zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
+	                               result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
 	return result;
 }
 
@@ -125,14 +130,14 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 			const auto* merge = TargetBlock(program, term.merge_block);
 			const auto* cont  = TargetBlock(program, term.continue_block);
 			if (merge != nullptr && cont != nullptr) {
-				ctx.state.builder.AddFunction(
-				    {OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone});
+				ctx.state.builder.AddFunction({spv::OpLoopMerge, ctx.Label(merge), ctx.Label(cont),
+				                               spv::LoopControlMaskNone});
 			}
 		} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
 		           term.merge_block != UINT32_MAX) {
 			if (const auto* merge = TargetBlock(program, term.merge_block); merge != nullptr) {
 				ctx.state.builder.AddFunction(
-				    {OpSelectionMerge, ctx.Label(merge), SelectionControlNone});
+				    {spv::OpSelectionMerge, ctx.Label(merge), spv::SelectionControlMaskNone});
 			}
 		}
 	};
@@ -145,7 +150,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				return;
 			}
 			emit_merge();
-			ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
+			ctx.state.builder.AddFunction({spv::OpBranch, ctx.Label(target)});
 			return;
 		}
 		case CFG::TerminatorKind::ConditionalBranch: {
@@ -157,8 +162,8 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 			}
 			const auto condition = BranchCondition(ctx, info);
 			emit_merge();
-			ctx.state.builder.AddFunction(
-			    {OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
+			ctx.state.builder.AddFunction({spv::OpBranchConditional, condition,
+			                               ctx.Label(true_block), ctx.Label(false_block)});
 			return;
 		}
 		default: EmitReturn(ctx); return;
@@ -187,7 +192,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
 			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
+			ctx.state.builder.AddFunction({spv::OpSelect, TypeU32(ctx.state), selected,
 			                               BranchCondition(ctx, info),
 			                               ConstantU32(ctx.state, term.true_block),
 			                               ConstantU32(ctx.state, term.false_block)});
@@ -209,9 +214,10 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
 					const auto match = ctx.state.builder.AllocateId();
 					const auto next  = ctx.state.builder.AllocateId();
-					ctx.state.builder.AddFunction({OpIEqual, TypeBool(ctx.state), match, selector,
+					ctx.state.builder.AddFunction({spv::OpIEqual, TypeBool(ctx.state), match,
+					                               selector,
 					                               ConstantU32(ctx.state, values[index])});
-					ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), next, match,
+					ctx.state.builder.AddFunction({spv::OpSelect, TypeU32(ctx.state), next, match,
 					                               ConstantU32(ctx.state, targets[index]),
 					                               selected});
 					selected = next;
@@ -223,18 +229,60 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 	}
 }
 
+template <typename T>
+decltype(auto) Arg(ValueEmitContext& ctx, const IR::Inst& inst, size_t index) {
+	if constexpr (std::is_same_v<T, const IR::Inst&>) {
+		return inst;
+	} else if constexpr (std::is_same_v<T, IR::Value>) {
+		return inst.Arg(index);
+	} else if constexpr (std::is_same_v<T, IR::ScalarReg>) {
+		return inst.Arg(index).ScalarRegister();
+	} else {
+		static_assert(std::is_same_v<T, uint32_t>);
+		return ctx.Def(inst.Arg(index));
+	}
+}
+
+template <typename Context, typename Return, typename... Args>
+void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::Inst& inst) {
+	// A full instruction keeps metadata and predicated/lane operand loads lazy.
+	static_assert(std::is_same_v<Context, ValueEmitContext> ||
+	              std::is_same_v<Context, EmitterState>);
+	auto& context = [&]() -> Context& {
+		if constexpr (std::is_same_v<Context, EmitterState>)
+			return ctx.state;
+		else
+			return ctx;
+	}();
+	constexpr bool has_inst = (std::is_same_v<Args, const IR::Inst&> || ...);
+	[&]<size_t... I>(std::index_sequence<I...>) {
+		static_assert(((!std::is_same_v<Args, const IR::Inst&> || I == 0) && ...));
+		const auto call = [&] {
+			return emit(context, Arg<Args>(ctx, inst, I - (has_inst && I != 0))...);
+		};
+		if constexpr (std::is_void_v<Return>) {
+			call();
+		} else {
+			static_assert(std::is_same_v<Return, uint32_t>);
+			ctx.Define(inst, call());
+		}
+	}(std::index_sequence_for<Args...> {});
+}
+
 void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
-	if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
-	    EmitValueImage(ctx, inst)) {
-		return;
+	switch (inst.GetOpcode()) {
+#define VALUE_OPCODE(name, ...)                                                                    \
+	case IR::ValueOpcode::name: return Invoke(Emit##name, ctx, inst);
+#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc"
+#undef VALUE_OPCODE
+		default: ctx.Fail(inst, "has no direct SPIR-V emitter");
 	}
-	ctx.Fail(inst, "has no direct SPIR-V emitter");
 }
 
 void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
                                const IR::Inst& inst) {
 	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
-		const auto type = ctx.TypeId(inst.GetType());
+		const auto type = TypeId(ctx.state, inst.GetType());
 		if (type == 0 || inst.NumArgs() == 0) {
 			ctx.Fail(inst, "has no native SPIR-V representation");
 		}
@@ -255,19 +303,19 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
 void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
                                const IR::Inst& inst) {
 	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
-		const auto type = ctx.TypeId(inst.GetType());
+		const auto type = TypeId(ctx.state, inst.GetType());
 		if (type == 0) {
 			ctx.Fail(inst, "cannot be loaded by the dispatcher");
 		}
 		ctx.state.builder.AddFunction(
-		    {OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
+		    {spv::OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
 		return;
 	}
 	EmitDirectInstruction(ctx, inst);
 	if (const auto found = dispatcher.spills[ctx.half].find(&inst);
 	    found != dispatcher.spills[ctx.half].end()) {
 		ctx.state.builder.AddFunction(
-		    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
+		    {spv::OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
 	}
 }
 
@@ -314,7 +362,7 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
 void EmitStructuredFunction(ValueEmitContext& ctx) {
 	const auto& program = ctx.state.program;
 	StructuredFunctionState structured;
-	ctx.state.builder.AddFunction({OpBranch, ctx.Label(program.blocks.front())});
+	ctx.state.builder.AddFunction({spv::OpBranch, ctx.Label(program.blocks.front())});
 	for (size_t index = 0; index < program.blocks.size(); index++) {
 		const auto* block = program.blocks[index];
 		EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
@@ -329,38 +377,38 @@ void EmitStructuredFunction(ValueEmitContext& ctx) {
 void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
 	auto&       state = ctx.state;
 	const auto* entry = state.program.blocks.front();
-	state.builder.AddFunction({OpBranch, ctx.Label(entry)});
+	state.builder.AddFunction({spv::OpBranch, ctx.Label(entry)});
 	EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
 		EmitDispatcherInstruction(lane, dispatcher, inst);
 	});
 	const auto initial_pc =
 	    EmitDispatcherNextPc(ctx, dispatcher, entry, state.program.block_info.front());
 	const auto initial_parent = state.current_label;
-	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	state.builder.AddFunction({spv::OpBranch, dispatcher.header_label});
 
 	EmitLabel(state, dispatcher.header_label);
 	const auto pc      = state.builder.AllocateId();
 	const auto next_pc = state.builder.AllocateId();
-	state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
+	state.builder.AddFunction({spv::OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
 	                           dispatcher.continue_label});
 	const auto done = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
+	    {spv::OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
+	state.builder.AddFunction({spv::OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label,
+	                           spv::LoopControlMaskNone});
 	state.builder.AddFunction(
-	    {OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label, LoopControlNone});
-	state.builder.AddFunction(
-	    {OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
+	    {spv::OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
 
 	EmitLabel(state, dispatcher.select_label);
 	state.builder.AddFunction(
-	    {OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone});
-	std::vector<uint32_t> words {OpSwitch, pc, dispatcher.after_switch_label};
+	    {spv::OpSelectionMerge, dispatcher.after_switch_label, spv::SelectionControlMaskNone});
+	std::vector<uint32_t> words {spv::OpSwitch, pc, dispatcher.after_switch_label};
 	for (size_t index = 1; index < state.program.blocks.size(); index++) {
 		words.push_back(state.program.block_info[index].id);
 		words.push_back(ctx.Label(state.program.blocks[index]));
 	}
 	state.builder.AddFunction(words);
-	std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc,
+	std::vector<uint32_t> next_pc_words {spv::OpPhi, TypeU32(state), next_pc,
 	                                     ConstantU32(state, UINT32_MAX), dispatcher.select_label};
 
 	for (size_t index = 1; index < state.program.blocks.size(); index++) {
@@ -372,20 +420,20 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
 		                                           state.program.block_info[index]);
 		next_pc_words.push_back(selected);
 		next_pc_words.push_back(state.current_label);
-		state.builder.AddFunction({OpBranch, dispatcher.after_switch_label});
+		state.builder.AddFunction({spv::OpBranch, dispatcher.after_switch_label});
 	}
 	EmitLabel(state, dispatcher.after_switch_label);
 	state.builder.AddFunction(next_pc_words);
-	state.builder.AddFunction({OpBranch, dispatcher.continue_label});
+	state.builder.AddFunction({spv::OpBranch, dispatcher.continue_label});
 	EmitLabel(state, dispatcher.continue_label);
-	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	state.builder.AddFunction({spv::OpBranch, dispatcher.header_label});
 	EmitLabel(state, dispatcher.merge_label);
 	EmitReturn(ctx);
 }
 
 } // namespace
 
-uint32_t ValueEmitContext::TypeId(IR::Type type) const {
+uint32_t TypeId(EmitterState& state, IR::Type type) {
 	switch (type) {
 		case IR::Type::U1: return TypeBool(state);
 		case IR::Type::U8:
@@ -430,7 +478,7 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
 				return loaded->second.second;
 			}
 			const auto id = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
+			state.builder.AddFunction({spv::OpLoad, TypeId(state, inst->GetType()), id, found->second});
 			dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
 			return id;
 		}
@@ -448,10 +496,10 @@ uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t
 
 uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
 	const auto ballot_type = TypeU32Vector(state, 4);
-	const auto scope       = ConstantU32(state, ScopeSubgroup);
+	const auto scope       = ConstantU32(state, spv::ScopeSubgroup);
 	const auto low         = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpGroupNonUniformBallot, ballot_type, low, scope,
+	    {spv::OpGroupNonUniformBallot, ballot_type, low, scope,
 	     other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)});
 	if (other_half == nullptr) {
 		return low;
@@ -460,11 +508,11 @@ uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
 	const auto low_word  = state.builder.AllocateId();
 	const auto high_word = state.builder.AllocateId();
 	const auto ballot    = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, ballot_type, high, scope,
+	state.builder.AddFunction({spv::OpGroupNonUniformBallot, ballot_type, high, scope,
 	                           half == 1 ? Def(predicate) : other_half->Def(predicate)});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low_word, low, 0});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high_word, high, 0});
-	state.builder.AddFunction({OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
+	state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low_word, low, 0});
+	state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high_word, high, 0});
+	state.builder.AddFunction({spv::OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
 	                           ConstantU32(state, 0), ConstantU32(state, 0)});
 	return ballot;
 }
@@ -472,8 +520,8 @@ uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
 uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
 	if (other_half == nullptr) {
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
-		                           ConstantU32(state, ScopeSubgroup), ballot});
+		state.builder.AddFunction({spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
+		                           ConstantU32(state, spv::ScopeSubgroup), ballot});
 		return result;
 	}
 	const auto low        = state.builder.AllocateId();
@@ -482,40 +530,42 @@ uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
 	const auto high_first = state.builder.AllocateId();
 	const auto low_active = state.builder.AllocateId();
 	const auto result     = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
+	state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
+	state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
 	state.builder.AddFunction(
-	    {OpExtInst, TypeU32(state), low_first, GlslStd450(state), GlslFindILsb, low});
+	    {spv::OpExtInst, TypeU32(state), low_first, GlslStd450(state), GLSLstd450FindILsb, low});
 	state.builder.AddFunction(
-	    {OpExtInst, TypeU32(state), high_first, GlslStd450(state), GlslFindILsb, high});
+	    {spv::OpExtInst, TypeU32(state), high_first, GlslStd450(state), GLSLstd450FindILsb, high});
 	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, TypeU32(state), result, low_active, low_first,
+	    {spv::OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpSelect, TypeU32(state), result, low_active, low_first,
 	                           EmitAddU32(state, high_first, ConstantU32(state, 32))});
 	return result;
 }
 
 uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
-	const auto type  = TypeId(inst.Arg(index).GetType());
-	const auto scope = ConstantU32(state, ScopeSubgroup);
+	const auto type  = TypeId(state, inst.Arg(index).GetType());
+	const auto scope = ConstantU32(state, spv::ScopeSubgroup);
 	const auto low   = state.builder.AllocateId();
 	if (other_half == nullptr) {
 		state.builder.AddFunction(
-		    {OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
+		    {spv::OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
 		return low;
 	}
-	const auto physical_lane = EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 31));
+	const auto physical_lane =
+	    EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31));
 	const auto high          = state.builder.AllocateId();
 	const auto in_high       = state.builder.AllocateId();
 	const auto value         = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
+	    {spv::OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
 	state.builder.AddFunction(
-	    {OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
-	state.builder.AddFunction({OpINotEqual, TypeBool(state), in_high,
-	                           EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 32)),
-	                           ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, type, value, in_high, high, low});
+	    {spv::OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
+	state.builder.AddFunction(
+	    {spv::OpINotEqual, TypeBool(state), in_high,
+	     EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 32)),
+	     ConstantU32(state, 0)});
+	state.builder.AddFunction({spv::OpSelect, type, value, in_high, high, low});
 	return value;
 }
 
@@ -528,20 +578,11 @@ uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
 	return id;
 }
 
-uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
-                                std::initializer_list<uint32_t> args) {
-	const auto type_id = TypeId(type);
-	const auto result  = Result(inst);
-	std::vector<uint32_t> words {opcode, type_id, result};
-	words.insert(words.end(), args.begin(), args.end());
-	state.builder.AddFunction(words);
-	return result;
-}
-
 uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
 	if (const auto found = definitions.find(&inst); found != definitions.end()) {
 		if (found->second != value) {
-			state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value});
+			state.builder.AddFunction(
+			    {spv::OpCopyObject, TypeId(state, inst.GetType()), found->second, value});
 		}
 		return found->second;
 	}
@@ -600,7 +641,7 @@ void EmitProgram(EmitterState& state) {
 		high.half       = 1;
 	}
 	std::optional<DispatcherFunctionState> dispatcher;
-	if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
+	if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
 		state.pixel_valid_mask_variable = state.builder.AllocateId();
 		state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
 	}
@@ -677,27 +718,28 @@ void EmitProgram(EmitterState& state) {
 			break;
 		}
 	}
-	state.builder.AddFunction({OpFunction, TypeVoid(state),
+	state.builder.AddFunction({spv::OpFunction, TypeVoid(state),
 	                           state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
-	                           FunctionControlNone, TypeFunction(state)});
+	                           spv::FunctionControlMaskNone, TypeFunction(state)});
 	EmitLabel(state, state.entry_label);
 	if (state.requirements.function_lds) {
 		state.builder.AddFunction(
-		    {OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)),
-		     state.lds_variable, StorageClassFunction});
+		    {spv::OpVariable,
+		     TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)),
+		     state.lds_variable, spv::StorageClassFunction});
 	}
 	if (state.requirements.function_scratch) {
 		for (uint32_t half = 0; half < state.lane_count; half++) {
-			state.builder.AddFunction(
-			    {OpVariable,
-			     TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
-			     state.scratch_variable[half], StorageClassFunction});
+			state.builder.AddFunction({spv::OpVariable,
+			                           TypeU32ArrayPointer(state, spv::StorageClassFunction,
+			                                               state.program.scratch_dwords),
+			                           state.scratch_variable[half], spv::StorageClassFunction});
 		}
 	}
 	if (state.pixel_valid_mask_variable != 0) {
-		state.builder.AddFunction({OpVariable,
-		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
-		                           state.pixel_valid_mask_variable, StorageClassFunction});
+		state.builder.AddFunction({spv::OpVariable,
+		                           TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
+		                           state.pixel_valid_mask_variable, spv::StorageClassFunction});
 	}
 	for (uint32_t half = 0; half < state.lane_count; half++) {
 		auto& lane = half == 0 ? ctx : high;
@@ -706,27 +748,27 @@ void EmitProgram(EmitterState& state) {
 				for (const auto& inst: *block) {
 					if (const auto found = dispatcher->spills[half].find(&inst);
 					    found != dispatcher->spills[half].end()) {
-						state.builder.AddFunction({OpVariable,
+						state.builder.AddFunction({spv::OpVariable,
 						                           SpillPointerType(lane, inst.GetType()),
-						                           found->second, StorageClassFunction});
+						                           found->second, spv::StorageClassFunction});
 					}
 				}
 			}
 		}
 		if (lane.scratch_u32_variable != 0) {
-			state.builder.AddFunction({OpVariable,
-			                           TypePointer(state, StorageClassFunction, TypeU32(state)),
-			                           lane.scratch_u32_variable, StorageClassFunction});
+			state.builder.AddFunction(
+			    {spv::OpVariable, TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
+			     lane.scratch_u32_variable, spv::StorageClassFunction});
 		}
 	}
 	if (state.gds_variable != 0) {
 		state.gds_length = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
+		    {spv::OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
 	}
 	if (state.pixel_valid_mask_variable != 0) {
 		state.builder.AddFunction(
-		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
+		    {spv::OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
 	}
 	EmitMemoryOffsets(state);
 	if (program.blocks.empty()) {
@@ -736,8 +778,8 @@ void EmitProgram(EmitterState& state) {
 	} else {
 		EmitStructuredFunction(ctx);
 	}
-	state.builder.AddFunction({OpFunctionEnd});
-	if (state.stage == ShaderType::Mesh) {
+	state.builder.AddFunction({spv::OpFunctionEnd});
+	if (state.program.stage == ShaderType::Mesh) {
 		EmitMeshEntryPoint(state);
 	}
 }

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -52,26 +52,6 @@ uint32_t ProgramEndPc(const Decoder::Program& program) {
 	return InstructionEndPc(program.instructions.back());
 }
 
-bool IsUnconditionalBranch(Opcode opcode) {
-	return opcode == Opcode::S_BRANCH;
-}
-
-bool IsConditionalBranch(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::S_CBRANCH_SCC0:
-		case Opcode::S_CBRANCH_SCC1:
-		case Opcode::S_CBRANCH_VCCZ:
-		case Opcode::S_CBRANCH_VCCNZ:
-		case Opcode::S_CBRANCH_EXECZ:
-		case Opcode::S_CBRANCH_EXECNZ: return true;
-		default: return false;
-	}
-}
-
-bool IsBranch(Opcode opcode) {
-	return IsUnconditionalBranch(opcode) || IsConditionalBranch(opcode);
-}
-
 BranchCondition ConditionForOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::S_BRANCH: return BranchCondition::Always;
@@ -81,6 +61,8 @@ BranchCondition ConditionForOpcode(Opcode opcode) {
 		case Opcode::S_CBRANCH_VCCNZ: return BranchCondition::VccNonZero;
 		case Opcode::S_CBRANCH_EXECZ: return BranchCondition::ExecZero;
 		case Opcode::S_CBRANCH_EXECNZ: return BranchCondition::ExecNonZero;
+		case Opcode::S_SUBVECTOR_LOOP_BEGIN:
+		case Opcode::S_SUBVECTOR_LOOP_END: return BranchCondition::ScalarInstruction;
 		default: return BranchCondition::Unknown;
 	}
 }
@@ -1925,7 +1907,7 @@ Graph BuildGraph(const Decoder::Program& program) {
 	for (uint32_t i = 0; i < program.instructions.size(); i++) {
 		const auto& inst    = program.instructions[i];
 		const auto  next_pc = InstructionEndPc(inst);
-		if (IsBranch(inst.opcode)) {
+		if (Decoder::IsDirectBranch(inst.opcode)) {
 			if (!IsValidTarget(inst.branch_target, instruction_pcs, first_pc, end_pc)) {
 				ExitBuildFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
 				                 fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}",
@@ -2032,11 +2014,11 @@ Graph BuildGraph(const Decoder::Program& program) {
 				block.terminator.condition  = BranchCondition::Always;
 				block.terminator.true_block = pc_to_block.at(target_info.target);
 			}
-		} else if (IsUnconditionalBranch(last.opcode)) {
+		} else if (last.opcode == Opcode::S_BRANCH) {
 			block.terminator.kind       = TerminatorKind::Branch;
 			block.terminator.condition  = BranchCondition::Always;
 			block.terminator.true_block = pc_to_block.at(last.branch_target);
-		} else if (IsConditionalBranch(last.opcode)) {
+		} else if (Decoder::IsConditionalBranch(last.opcode)) {
 			block.terminator.kind       = TerminatorKind::ConditionalBranch;
 			block.terminator.condition  = ConditionForOpcode(last.opcode);
 			block.terminator.true_block = pc_to_block.at(last.branch_target);
@@ -2211,27 +2193,38 @@ bool StructurizeImpl(Graph& graph) {
 } // namespace
 
 bool Structurize(Graph& graph) {
-	Graph original = graph;
-	if (StructurizeImpl(graph)) {
+	Graph structured = graph;
+	if (StructurizeImpl(structured)) {
+		graph = std::move(structured);
 		return true;
 	}
 
-	Graph failed_graph      = std::move(graph);
-	graph                   = std::move(original);
+	const auto failure_kind = structured.failure_kind;
+	// Structurization inserts and renumbers blocks. Recover source identity for a
+	// semantic block; a synthetic block has no corresponding original diagnostic ID.
+	const auto* failed = structured.FindBlock(structured.failure_block);
+	const auto original = std::ranges::find_if(graph.blocks, [&](const BasicBlock& block) {
+		return failed != nullptr && failed->inst_begin != failed->inst_end &&
+		       block.inst_begin == failed->inst_begin && block.inst_end == failed->inst_end &&
+		       block.start_pc == failed->start_pc && block.end_pc == failed->end_pc;
+	});
+	const auto failure_block = original != graph.blocks.end() ? original->id : UINT32_MAX;
+	auto failure_reason = std::move(structured.unsupported_reason);
+	Graph routed = graph;
 	const auto route_budget = static_cast<uint32_t>(graph.blocks.size());
 	// Apply one route at a time and retry. Eagerly routing every matching diamond can
 	// rewrite unrelated selections that were already structurally valid.
 	for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) {
-		if (!RouteSharedSelectionArm(graph, route_variable)) {
+		if (!RouteSharedSelectionArm(routed, route_variable)) {
 			break;
 		}
-		Graph routed = graph;
-		if (StructurizeImpl(routed)) {
-			graph = std::move(routed);
+		structured = routed;
+		if (StructurizeImpl(structured)) {
+			graph = std::move(structured);
 			return true;
 		}
 	}
-	graph = std::move(failed_graph);
+	SetFailure(graph, failure_kind, failure_block, failure_reason);
 	return false;
 }
 
@@ -2244,6 +2237,7 @@ std::string BranchConditionToString(BranchCondition condition) {
 		case BranchCondition::VccNonZero: return "vccnz";
 		case BranchCondition::ExecZero: return "execz";
 		case BranchCondition::ExecNonZero: return "execnz";
+		case BranchCondition::ScalarInstruction: return "scalar_instruction";
 		case BranchCondition::GotoVariable: return "goto_variable";
 		default: return "unknown";
 	}

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -17,6 +17,7 @@ enum class BranchCondition {
 	VccNonZero,
 	ExecZero,
 	ExecNonZero,
+	ScalarInstruction,
 	GotoVariable,
 	Unknown
 };
@@ -109,6 +110,8 @@ struct Graph {
 };
 
 Graph       BuildGraph(const Decoder::Program& program);
+// Commits structured control flow on success; preserves the original graph with
+// failure diagnostics on failure. failure_block is an original block ID or UINT32_MAX.
 bool        Structurize(Graph& graph);
 std::string BranchConditionToString(BranchCondition condition);
 std::string FailureKindToString(FailureKind kind);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -94,6 +94,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x12u, Opcode::DS_MIN_F32, 1, 32},          {0x13u, Opcode::DS_MAX_F32, 1, 32},
     {0x1eu, Opcode::DS_WRITE_B8, 1, 8},          {0x1fu, Opcode::DS_WRITE_B16, 1, 16},
     {0x20u, Opcode::DS_ADD_RTN_U32, 1, 32},      {0x21u, Opcode::DS_SUB_RTN_U32, 1, 32},
+    {0x23u, Opcode::DS_INC_RTN_U32, 1, 32},      {0x24u, Opcode::DS_DEC_RTN_U32, 1, 32},
     {0x25u, Opcode::DS_MIN_RTN_I32, 1, 32},      {0x26u, Opcode::DS_MAX_RTN_I32, 1, 32},
     {0x27u, Opcode::DS_MIN_RTN_U32, 1, 32},      {0x28u, Opcode::DS_MAX_RTN_U32, 1, 32},
     {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
@@ -165,6 +166,8 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:
+		case Opcode::DS_INC_RTN_U32:
+		case Opcode::DS_DEC_RTN_U32:
 		case Opcode::DS_MIN_I32:
 		case Opcode::DS_MIN_RTN_I32:
 		case Opcode::DS_MAX_I32:

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -33,6 +33,7 @@ constexpr OpcodeMap SOP2_OPCODE_LIST[] = {
     {0x31u, Opcode::S_LSHL4_ADD_U32},   {0x32u, Opcode::S_PACK_LL_B32_B16},
     {0x33u, Opcode::S_PACK_LH_B32_B16}, {0x34u, Opcode::S_PACK_HH_B32_B16},
     {0x35u, Opcode::S_MUL_HI_U32},
+    {0x36u, Opcode::S_MUL_HI_I32},
 };
 
 constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
@@ -41,6 +42,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x06u, Opcode::S_CMOV_B64},
     {0x07u, Opcode::S_NOT_B32},
     {0x08u, Opcode::S_NOT_B64},
+    {0x09u, Opcode::S_WQM_B32},
     {0x0au, Opcode::S_WQM_B64},
     {0x0bu, Opcode::S_BREV_B32},
     {0x0fu, Opcode::S_BCNT1_I32_B32},
@@ -62,6 +64,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x37u, Opcode::S_ANDN1_SAVEEXEC_B64},
     {0x3bu, Opcode::S_BITREPLICATE_B64_B32},
     {0x3cu, Opcode::S_AND_SAVEEXEC_B32},
+    {0x40u, Opcode::S_ORN2_SAVEEXEC_B32},
     {0x44u, Opcode::S_ANDN1_SAVEEXEC_B32},
 };
 
@@ -82,6 +85,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = {
     {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32},    {0x10u, Opcode::S_MULK_I32},
     {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT},    {0x18u, Opcode::S_WAITCNT},
     {0x19u, Opcode::S_WAITCNT},    {0x1au, Opcode::S_WAITCNT},
+    {0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END},
 };
 
 constexpr OpcodeMap SOPP_OPCODE_LIST[] = {
@@ -208,6 +212,11 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 
 	switch (inst.opcode) {
 		case Opcode::S_MOVK_I32: DecodeScalarDestination(sdst, pc, inst.dst); return;
+		case Opcode::S_SUBVECTOR_LOOP_BEGIN:
+		case Opcode::S_SUBVECTOR_LOOP_END:
+			DecodeScalarDestination(sdst, pc, inst.dst);
+			inst.branch_target = pc + 4u + static_cast<uint32_t>(imm * 4);
+			return;
 		case Opcode::S_WAITCNT: {
 			const uint32_t waitcnt = word & 0xffffu;
 			inst.dst.kind          = OperandKind::Null;

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -25,19 +25,6 @@ bool HasLiteral(const Instruction& inst) {
 	       inst.src3.kind == OperandKind::LiteralConstant;
 }
 
-bool IsControlFlowBranch(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::S_BRANCH:
-		case Opcode::S_CBRANCH_SCC0:
-		case Opcode::S_CBRANCH_SCC1:
-		case Opcode::S_CBRANCH_VCCZ:
-		case Opcode::S_CBRANCH_VCCNZ:
-		case Opcode::S_CBRANCH_EXECZ:
-		case Opcode::S_CBRANCH_EXECNZ: return true;
-		default: return false;
-	}
-}
-
 void ApplyLiteral(Operand& operand, uint32_t literal) {
 	if (operand.kind == OperandKind::LiteralConstant) {
 		operand.value      = literal;
@@ -198,6 +185,24 @@ std::string FormatExp(const Instruction& inst) {
 
 } // namespace
 
+bool IsConditionalBranch(Opcode opcode) {
+	switch (opcode) {
+		case Opcode::S_CBRANCH_SCC0:
+		case Opcode::S_CBRANCH_SCC1:
+		case Opcode::S_CBRANCH_VCCZ:
+		case Opcode::S_CBRANCH_VCCNZ:
+		case Opcode::S_CBRANCH_EXECZ:
+		case Opcode::S_CBRANCH_EXECNZ:
+		case Opcode::S_SUBVECTOR_LOOP_BEGIN:
+		case Opcode::S_SUBVECTOR_LOOP_END: return true;
+		default: return false;
+	}
+}
+
+bool IsDirectBranch(Opcode opcode) {
+	return opcode == Opcode::S_BRANCH || IsConditionalBranch(opcode);
+}
+
 const char* ImageDimensionToString(ImageDimension dimension) {
 	switch (dimension) {
 		case ImageDimension::Dim1D: return "1d";
@@ -389,7 +394,7 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 		const auto& inst = program.instructions.back();
 		word_index += inst.word_count;
 
-		if (IsControlFlowBranch(inst.opcode)) {
+		if (IsDirectBranch(inst.opcode)) {
 			const auto target_index = inst.branch_target / sizeof(uint32_t);
 			if (branch_targets.empty()) {
 				branch_targets.resize(code.size());
@@ -482,9 +487,11 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_BITSET0_B64:
 		case Opcode::S_BITSET1_B64:
 		case Opcode::S_NOT_B64:
+		case Opcode::S_WQM_B32:
 		case Opcode::S_WQM_B64:
 		case Opcode::S_QUADMASK_B64:
 		case Opcode::S_AND_SAVEEXEC_B32:
+		case Opcode::S_ORN2_SAVEEXEC_B32:
 		case Opcode::S_ANDN1_SAVEEXEC_B32:
 		case Opcode::S_AND_SAVEEXEC_B64:
 		case Opcode::S_ORN2_SAVEEXEC_B64:
@@ -531,6 +538,11 @@ std::string InstructionToString(const Instruction& inst) {
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc,
 			                                               magic_enum::enum_name(inst.opcode),
 			                                               inst.branch_target));
+		case Opcode::S_SUBVECTOR_LOOP_BEGIN:
+		case Opcode::S_SUBVECTOR_LOOP_END:
+			return WithUnsupportedReason(inst, fmt::format(
+			    "0x{:08x}: {} {}, 0x{:08x}", inst.pc, magic_enum::enum_name(inst.opcode),
+			    OperandToString(inst.dst), inst.branch_target));
 		case Opcode::EXP: return WithUnsupportedReason(inst, FormatExp(inst));
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_STORE:
@@ -626,6 +638,8 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:
+		case Opcode::DS_INC_RTN_U32:
+		case Opcode::DS_DEC_RTN_U32:
 		case Opcode::DS_MIN_I32:
 		case Opcode::DS_MIN_RTN_I32:
 		case Opcode::DS_MAX_I32:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -53,13 +53,17 @@ enum class Opcode {
 	S_QUADMASK_B64,
 	S_GETPC_B64,
 	S_SETPC_B64,
+	S_SUBVECTOR_LOOP_BEGIN,
+	S_SUBVECTOR_LOOP_END,
 	S_AND_SAVEEXEC_B32,
+	S_ORN2_SAVEEXEC_B32,
 	S_ANDN1_SAVEEXEC_B32,
 	S_AND_SAVEEXEC_B64,
 	S_ORN2_SAVEEXEC_B64,
 	S_ANDN1_SAVEEXEC_B64,
 	S_NOT_B32,
 	S_NOT_B64,
+	S_WQM_B32,
 	S_WQM_B64,
 	S_ADD_U32,
 	S_ADDC_U32,
@@ -104,6 +108,7 @@ enum class Opcode {
 	S_ASHR_I32,
 	S_MUL_I32,
 	S_MUL_HI_U32,
+	S_MUL_HI_I32,
 	S_MULK_I32,
 	S_BFE_U32,
 	S_BFE_I32,
@@ -488,6 +493,8 @@ enum class Opcode {
 	DS_ADD_RTN_U32,
 	DS_SUB_U32,
 	DS_SUB_RTN_U32,
+	DS_INC_RTN_U32,
+	DS_DEC_RTN_U32,
 	DS_MIN_I32,
 	DS_MIN_RTN_I32,
 	DS_MAX_I32,
@@ -714,6 +721,8 @@ Family GetInstructionFamily(uint32_t word);
 // The output object must be freshly initialized.
 void DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst);
 void DecodeProgram(std::span<const uint32_t> code, Program& program);
+bool IsConditionalBranch(Opcode opcode);
+bool IsDirectBranch(Opcode opcode);
 
 void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand);
 void DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand);

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -22,29 +22,63 @@ Decoder::Operand ConditionOperand(Decoder::OperandKind kind) {
 
 } // namespace
 
+void Translator::S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin) {
+	const auto zero  = IR::U32(IR::Value(0u));
+	const auto lo    = ir.GetExecLo();
+	const auto hi    = ir.GetExecHi();
+	const auto saved = ReadU32(inst.dst);
+	if (begin) {
+		const auto low_active = ir.INotEqual(lo, zero);
+		instruction_branch_condition = ir.IEqual(ir.BitwiseOr(lo, hi), zero);
+		WriteRawU32(inst.dst, ir.Select(instruction_branch_condition, saved,
+		                               ir.Select(low_active, hi, lo)));
+		// Keep the ISA assignment order: SDST may itself name an EXEC half.
+		WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecHi),
+		            ir.Select(low_active, zero, ir.GetExecHi()));
+	} else {
+		const auto high_active = ir.INotEqual(hi, zero);
+		instruction_branch_condition =
+		    ir.LogicalAnd(ir.LogicalNot(high_active), ir.INotEqual(saved, zero));
+		WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecHi),
+		            ir.Select(instruction_branch_condition, saved, hi));
+		WriteRawU32(inst.dst,
+		            ir.Select(instruction_branch_condition, lo, ReadU32(inst.dst)));
+		WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecLo),
+		            ir.Select(high_active, saved,
+		                      ir.Select(instruction_branch_condition, zero, ir.GetExecLo())));
+	}
+}
+
 void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation,
                             bool negate_exec, bool negate_source, bool write_64) {
+	if (!write_64) {
+		// Read the encoded scalar word and preserve EXEC_HI, including in wave32.
+		const auto old = ir.GetExecLo();
+		const auto src = ReadU32(inst.src0);
+		const auto lhs = negate_exec ? ir.BitwiseNot(old) : old;
+		const auto rhs = negate_source ? ir.BitwiseNot(src) : src;
+		IR::U32 result;
+		switch (operation) {
+			case IR::ValueOpcode::LogicalAnd: result = ir.BitwiseAnd(lhs, rhs); break;
+			case IR::ValueOpcode::LogicalOr: result = ir.BitwiseOr(lhs, rhs); break;
+			default: EXIT("unsupported SAVEEXEC operation");
+		}
+		WriteRawU32(inst.dst, old);
+		WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecLo), result);
+		ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
+		return;
+	}
 	const auto old    = ir.GetExec();
 	const auto src    = ReadMask(inst.src0);
 	const auto lhs    = negate_exec ? ir.LogicalNot(old) : old;
 	const auto rhs    = negate_source ? ir.LogicalNot(src) : src;
-	auto       result = IR::U1(ir.Emit(operation, {lhs, rhs}));
-	if (write_64) {
-		WriteMask(inst.dst, old, true);
-	} else {
-		WriteRawU32(inst.dst, ir.GetExecLo());
-		if (program.wave_size == 64u) {
-			const auto low_half =
-			    ir.ULessThan(IR::U32(ir.Emit(IR::ValueOpcode::LaneId)), IR::U32(IR::Value(32u)));
-			result = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {low_half, result, old}));
-		}
-	}
+	const auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
+	WriteMask(inst.dst, old, true);
 	const auto mask = BallotMask(result);
 	ir.SetExec(result);
 	ir.SetExecLo(mask[0]);
 	ir.SetExecHi(mask[1]);
-	ir.SetScc(
-	    ir.INotEqual(write_64 ? ir.BitwiseOr(mask[0], mask[1]) : mask[0], IR::U32(IR::Value(0u))));
+	ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))));
 }
 
 void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use_carry_in) {
@@ -267,10 +301,19 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
 	}
 }
 
-void Translator::S_WQM_B64(const Decoder::Instruction& inst) {
-	const auto mask_valid  = ReadMaskValid(inst.src0);
-	const auto result =
-	    IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src0, IR::Type::U64)}));
+void Translator::S_WQM(const Decoder::Instruction& inst, bool wide) {
+	const auto source = wide ? ReadU64(inst.src0)
+	                         : ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
+	const auto result = IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {source}));
+	if (!wide) {
+		const auto low = ir.CompositeExtract(result, 0);
+		// A 32-bit write preserves the other EXEC/VCC half and invalidates any
+		// overlapping SGPR-pair mask provenance through the ordinary scalar path.
+		WriteRawU32(inst.dst, low);
+		ir.SetScc(ir.INotEqual(low, IR::U32(IR::Value(0u))));
+		return;
+	}
+	const auto mask_valid = ReadMaskValid(inst.src0);
 	WriteOperand(DestinationOperand(inst), result);
 	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
 		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -111,11 +111,6 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
 	memory.image_sample_flags       = decoded.image_sample_flags;
 	memory.image_dimension          = decoded.image_dimension;
 	memory.image_address_components = decoded.image_address_components;
-	memory.image_nsa_dwords         = decoded.image_nsa_dwords;
-	for (uint32_t index = 0; index < Decoder::MaxImageNsaAddressComponents; index++) {
-		memory.image_nsa_addr[index] = decoded.image_nsa_addr[index];
-	}
-	memory.memory_segment = decoded.memory_segment;
 	memory.address_is_full =
 	    memory.kind == ResourceKind::Flat ||
 	    (memory.kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
@@ -125,8 +120,6 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
 	memory.image_has_mip = decoded.opcode == Decoder::Opcode::IMAGE_LOAD_MIP ||
 	                       decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP;
 	memory.image_r128    = decoded.image_r128;
-	memory.glc           = decoded.glc;
-	memory.slc           = decoded.slc;
 	memory.idxen         = decoded.idxen;
 	memory.offen         = decoded.offen;
 	memory.resource      = ResourceIndexFromOperand(decoded.src1);
@@ -968,6 +961,10 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicISub32, false);
 		case Decoder::Opcode::DS_SUB_RTN_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicISub32, true);
+		case Decoder::Opcode::DS_INC_RTN_U32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicInc32, true);
+		case Decoder::Opcode::DS_DEC_RTN_U32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicDec32, true);
 		case Decoder::Opcode::DS_MIN_I32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSMin32, false);
 		case Decoder::Opcode::DS_MIN_RTN_I32:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -8,9 +8,12 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_MOV_B32:
 		case O::S_MOVK_I32: MOV_B32(inst, false); return true;
 		case O::S_MOV_B64: S_MOV_B64(inst); return true;
-		case O::S_WQM_B64: S_WQM_B64(inst); return true;
+		case O::S_WQM_B32: S_WQM(inst, false); return true;
+		case O::S_WQM_B64: S_WQM(inst, true); return true;
 		case O::S_GETPC_B64: S_GETPC_B64(inst); return true;
 		case O::S_SETPC_B64: return true;
+		case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return true;
+		case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return true;
 		case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true;
 		case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return true;
 		case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return true;
@@ -23,6 +26,9 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_ANDN1_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, false);
 			return true;
+		case O::S_ORN2_SAVEEXEC_B32:
+			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, false);
+			return true;
 		case O::S_AND_SAVEEXEC_B64:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, true);
 			return true;
@@ -132,6 +138,8 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
 			return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false);
 		case O::S_MUL_HI_U32:
 			return SimpleInteger(inst, IR::ValueOpcode::UMulHi, IR::Type::U32, false, false, false);
+		case O::S_MUL_HI_I32:
+			return SimpleInteger(inst, IR::ValueOpcode::SMulHi, IR::Type::U32, false, false, false);
 		case O::S_AND_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitwiseAnd32, IR::Type::U32, false, false,
 			                     true);