Remove global condition waiter registry that deadlocks exception delivery

Co-authored-by: Bipin <bipinkrish4@gmail.com>

CMakeLists.txt

@@ -342,6 +342,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/SDL2/include
 	${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
+	${KYTY_THIRD_PARTY_DIR}/SPIRV-Headers/include
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/xxhash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
@@ -550,6 +551,10 @@ if(BUILD_TESTING)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME shader_recompiler_alignbyte
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
+	add_test(NAME graphics_pipeline_rasterization
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --polygon-mode-only)
+	add_test(NAME graphics_draw_offsets
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --draw-offset-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME guest_red_zone_patcher
@@ -567,6 +572,8 @@ if(BUILD_TESTING)
 
 	add_test(NAME texture_cache_layered_image
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --layered-image-only)
+	add_test(NAME host_image_allocation
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --host-image-allocation-only)
 	add_test(NAME texture_cache_image_views
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-view-cache-only)
 	add_test(NAME texture_cache_storage_sampled

README.md

@@ -90,8 +90,14 @@ starting a large change.
 
 Set up the clang-format hook after cloning:
 
-```powershell
-python -m pip install pre-commit
+Install `pre-commit` using the method appropriate for your platform:
+
+- **Arch Linux / CachyOS:** `sudo pacman -S pre-commit`
+- **Other Linux / macOS / Windows:** `python -m pip install pre-commit`
+
+Then install the Git hook:
+
+```bash
 python -m pre_commit install --install-hooks
 ```
 

src/common/alignment.h

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2014 Jannik Vogel <email@jannikvogel.de>
+// SPDX-License-Identifier: CC0-1.0
+
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+
+namespace Common {
+
+template <typename T>
+[[nodiscard]] constexpr T AlignUp(T value, std::size_t size) {
+	static_assert(std::is_unsigned_v<T>, "T must be an unsigned value.");
+	auto mod{static_cast<T>(value % size)};
+	value -= mod;
+	return static_cast<T>(mod == T{0} ? value : value + size);
+}
+
+template <typename T>
+[[nodiscard]] constexpr T AlignDown(T value, std::size_t size) {
+	static_assert(std::is_unsigned_v<T>, "T must be an unsigned value.");
+	return static_cast<T>(value - value % size);
+}
+
+} // namespace Common

src/common/assert.cpp

@@ -1,6 +1,5 @@
 #include "common/assert.h"
 
-#include "common/debug.h"
 #include "common/logging/log.h"
 #include "common/subsystems.h"
 #include "kytyGitVersion.h"
@@ -12,25 +11,10 @@
 
 namespace Common {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_BUILD == KYTY_BUILD_DEBUG &&                    \
-    KYTY_COMPILER == KYTY_COMPILER_CLANG
-constexpr int PRINT_STACK_FROM = 4;
-#else
-constexpr int PRINT_STACK_FROM = 2;
-#endif
-
 static std::string BuildFatalReport(const char* title, std::string_view text, const char* file,
                                     int line) {
-	DebugStack stack;
-	DebugStack::Trace(&stack);
-
-	std::string report = "--- Build ---\n" KYTY_BUILD_LABEL "\n--- Stack Trace ---\n";
-	for (int i = PRINT_STACK_FROM; i < stack.depth; i++) {
-		report += fmt::format("[{}] {:016x}\n", i - PRINT_STACK_FROM,
-		                      static_cast<uint64_t>(stack.GetAddr(i)));
-	}
-	report += fmt::format("{}\n{} in {}:{}\n", title, text, file, line);
-	return report;
+	return fmt::format("--- Build ---\n{}\n{}\n{} in {}:{}\n", KYTY_BUILD_LABEL, title, text, file,
+	                   line);
 }
 
 static int DbgReport(const char* title, std::string_view text, const char* file, int line) {

src/common/debug.cpp

@@ -1,20 +1,7 @@
 #include "common/debug.h"
 
-#include "common/platform/sysDbg.h"
-
 namespace Common {
 
-void DebugStack::Print(int from, [[maybe_unused]] bool with_name) const {
-	for (int i = from; i < depth; i++) {
-		printf("[%d] %016" PRIx64 "\n", i - from, static_cast<uint64_t>(GetAddr(i)));
-	}
-}
-
-void DebugStack::Trace(DebugStack* stack) {
-	stack->depth = DEBUG_MAX_STACK_DEPTH;
-	SysStackWalk(stack->stack.data(), &stack->depth);
-}
-
 std::string Debug::GetCompiler() {
 #if KYTY_COMPILER == KYTY_COMPILER_CLANG
 	return "clang";

src/common/debug.h

@@ -2,42 +2,17 @@
 #define KYTY_COMMON_DEBUG_H_
 
 #include "common/common.h"
-#include "common/stringUtils.h"
 
-#include <array>
-#include <cstring>
+#include <string>
 
 namespace Common {
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_BUILD == KYTY_BUILD_DEBUG &&                    \
-    KYTY_COMPILER == KYTY_COMPILER_CLANG
-constexpr int DEBUG_MAX_STACK_DEPTH = 20;
-#else
-constexpr int DEBUG_MAX_STACK_DEPTH = 15;
-#endif
-
 namespace Debug {
 std::string GetCompiler();
 std::string GetLinker();
 std::string GetBitness();
 } // namespace Debug
 
-struct DebugStack {
-	int                                      depth {0};
-	std::array<void*, DEBUG_MAX_STACK_DEPTH> stack {};
-
-	[[nodiscard]] uintptr_t GetAddr(int i) const { return reinterpret_cast<uintptr_t>(stack[i]); }
-
-	void Print(int from, bool with_name = true) const;
-
-	void CopyTo(DebugStack* s) const {
-		std::memcpy(s->stack.data(), stack.data(), sizeof(void*) * depth);
-		s->depth = depth;
-	}
-
-	static void Trace(DebugStack* stack);
-};
-
 } // namespace Common
 
 #endif /* KYTY_COMMON_DEBUG_H_ */

src/common/emulatorConfig.cpp

@@ -55,6 +55,10 @@ bool FullscreenEnabled() {
 	return g_config->fullscreen_enabled;
 }
 
+bool VrEnabled() {
+	return g_config->vr_enabled;
+}
+
 uint32_t GetVblankFrequency() {
 	return std::clamp(g_config->vblank_frequency, 30u, 360u);
 }
@@ -75,7 +79,7 @@ ShaderOptimizationType GetShaderOptimizationType() {
 	return g_config->shader_optimization_type;
 }
 
-ShaderLogDirection GetShaderLogDirection() {
+LogDirection GetShaderLogDirection() {
 	return g_config->shader_log_direction;
 }
 
@@ -95,7 +99,7 @@ bool GraphicsDebugDumpEnabled() {
 	return g_config->graphics_debug_dump_enabled;
 }
 
-OutputDirection GetPrintfDirection() {
+LogDirection GetPrintfDirection() {
 	return g_config->printf_direction;
 }
 
@@ -103,8 +107,8 @@ std::filesystem::path GetPrintfOutputFile() {
 	return g_config->printf_output_file;
 }
 
-ProfilerDirection GetProfilerDirection() {
-	return g_config->profiler_direction;
+bool ProfilerEnabled() {
+	return g_config->profiler_enabled;
 }
 
 bool SpirvDebugPrintfEnabled() {

src/common/emulatorConfig.h

@@ -21,11 +21,7 @@ struct Lifecycle {
 
 enum class ShaderOptimizationType { None, Size, Performance };
 
-enum class ShaderLogDirection { Silent, Console, File };
-
-enum class ProfilerDirection { None, Network };
-
-enum class OutputDirection { Silent, Console, File };
+enum class LogDirection { Silent, Console, File };
 
 enum class PresentMode { Fifo, Mailbox, Immediate };
 
@@ -47,22 +43,23 @@ struct ConfigOptions {
 	uint32_t               screen_height               = 720;
 	std::string            user_name                   = "Kyty";
 	int32_t                user_id                     = DEFAULT_USER_ID;
-	PresentMode            present_mode                = PresentMode::Fifo;
+	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
+	bool                   vr_enabled                  = false;
 	uint32_t               vblank_frequency            = 60;
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
-	ShaderLogDirection     shader_log_direction        = ShaderLogDirection::Silent;
+	LogDirection           shader_log_direction        = LogDirection::Silent;
 	std::filesystem::path  shader_log_folder           = "_Shaders";
 	bool                   command_buffer_dump_enabled = false;
 	std::filesystem::path  command_buffer_dump_folder  = "_Buffers";
 	bool                   graphics_debug_dump_enabled = false;
-	OutputDirection        printf_direction            = OutputDirection::Silent;
+	LogDirection           printf_direction            = LogDirection::Silent;
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
-	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
+	bool                   profiler_enabled            = false;
 	bool                   spirv_debug_printf_enabled  = false;
 	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
@@ -83,13 +80,14 @@ int32_t  GetUserId();
 PresentMode GetPresentMode();
 int32_t GetGpuIndex();
 bool     FullscreenEnabled();
+bool     VrEnabled();
 uint32_t GetVblankFrequency();
 uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
 ShaderOptimizationType GetShaderOptimizationType();
-ShaderLogDirection     GetShaderLogDirection();
+LogDirection           GetShaderLogDirection();
 std::filesystem::path  GetShaderLogFolder();
 
 bool                  CommandBufferDumpEnabled();
@@ -97,10 +95,10 @@ std::filesystem::path GetCommandBufferDumpFolder();
 
 bool GraphicsDebugDumpEnabled();
 
-OutputDirection       GetPrintfDirection();
+LogDirection          GetPrintfDirection();
 std::filesystem::path GetPrintfOutputFile();
 
-ProfilerDirection GetProfilerDirection();
+bool ProfilerEnabled();
 
 bool SpirvDebugPrintfEnabled();
 

src/common/logging/log.cpp

@@ -152,9 +152,9 @@ void WriteFatal(fmt::text_style style, std::string_view text) {
 void Initialize() {
 	g_initialized = true;
 	switch (Config::GetPrintfDirection()) {
-		case Config::OutputDirection::Silent: g_direction = Direction::Silent; break;
-		case Config::OutputDirection::Console: g_direction = Direction::Console; break;
-		case Config::OutputDirection::File: g_direction = Direction::File; break;
+		case Config::LogDirection::Silent: g_direction = Direction::Silent; break;
+		case Config::LogDirection::Console: g_direction = Direction::Console; break;
+		case Config::LogDirection::File: g_direction = Direction::File; break;
 	}
 	g_output_file =
 	    (g_direction == Direction::File ? Config::GetPrintfOutputFile() : std::filesystem::path {});

src/common/platform/sysDbg.h

@@ -30,8 +30,6 @@ struct sys_dbg_stack_info_t {
 #endif
 };
 
-void SysStackWalk(void** stack, int* depth);
-void SysStackUsage(sys_dbg_stack_info_t& s);          // NOLINT(google-runtime-references)
-void SysStackUsagePrint(sys_dbg_stack_info_t& stack); // NOLINT(google-runtime-references)
+void SysStackUsage(sys_dbg_stack_info_t& s); // NOLINT(google-runtime-references)
 
 #endif /* KYTY_COMMON_PLATFORM_SYSDBG_H_ */

src/common/platform/sysLinuxDbg.cpp

@@ -8,7 +8,6 @@
 
 #include <cstdlib>
 #include <cstring>
-#include <execinfo.h>
 #include <pthread.h>
 #include <sys/param.h>
 #include <sys/types.h>
@@ -17,56 +16,6 @@
 #include <libgen.h> // POSIX basename() lives here on macOS, not in <cstring>
 #endif
 
-// Avoid unwinding a guest-owned stack.
-static bool OnOwnStack() {
-	const char* probe = reinterpret_cast<const char*>(&probe);
-
-	pthread_attr_t attr {};
-#if defined(__APPLE__)
-	const auto* top  = static_cast<const char*>(pthread_get_stackaddr_np(pthread_self()));
-	const auto  size = pthread_get_stacksize_np(pthread_self());
-	(void)attr;
-	return top != nullptr && size != 0 && probe < top && probe >= top - size;
-#else
-	if (pthread_getattr_np(pthread_self(), &attr) != 0) {
-		return false;
-	}
-	void*      base = nullptr;
-	size_t     size = 0;
-	const bool ok = pthread_attr_getstack(&attr, &base, &size) == 0 && base != nullptr && size != 0;
-	pthread_attr_destroy(&attr);
-	if (!ok) {
-		return false;
-	}
-	const auto* low = static_cast<const char*>(base);
-	return probe >= low && probe < low + size;
-#endif
-}
-
-void SysStackWalk(void** stack, int* depth) {
-	if (stack == nullptr || depth == nullptr || *depth <= 0) {
-		if (depth != nullptr) {
-			*depth = 0;
-		}
-		return;
-	}
-
-	if (!OnOwnStack()) {
-		*depth = 0;
-		return;
-	}
-
-	const int n = ::backtrace(stack, *depth);
-	*depth      = (n < 0 ? 0 : n);
-}
-
-void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
-	printf("stack: (0x%" PRIx64 ", %" PRIu64 ")\n", static_cast<uint64_t>(stack.commited_addr),
-	       static_cast<uint64_t>(stack.commited_size));
-	printf("code: (0x%" PRIx64 ", %" PRIu64 ")\n", static_cast<uint64_t>(stack.code_addr),
-	       static_cast<uint64_t>(stack.code_size));
-}
-
 void SysStackUsage(sys_dbg_stack_info_t& s) {
 	pid_t pid = getpid();
 

src/common/platform/sysLinuxVirtual.cpp

@@ -31,8 +31,7 @@
 namespace Common {
 
 static pthread_mutex_t              g_virtual_mutex {};
-static std::map<uintptr_t, size_t>* g_allocs   = nullptr;
-static std::map<uintptr_t, int>*    g_protects = nullptr;
+static std::map<uintptr_t, size_t>* g_allocs = nullptr;
 
 void SysVirtualInit() {
 	pthread_mutexattr_t attr {};
@@ -46,8 +45,7 @@ void SysVirtualInit() {
 	pthread_mutex_init(&g_virtual_mutex, &attr);
 	pthread_mutexattr_destroy(&attr);
 
-	g_allocs   = new std::map<uintptr_t, size_t>;
-	g_protects = new std::map<uintptr_t, int>;
+	g_allocs = new std::map<uintptr_t, size_t>;
 }
 
 static int get_protection_flag(VirtualMemory::Mode mode) {
@@ -68,21 +66,6 @@ static int get_protection_flag(VirtualMemory::Mode mode) {
 	return protect;
 }
 
-static VirtualMemory::Mode get_protection_flag(int mode) {
-	switch (mode) {
-		case PROT_NONE: return VirtualMemory::Mode::NoAccess;
-		case PROT_READ: return VirtualMemory::Mode::Read;
-		case PROT_WRITE: return VirtualMemory::Mode::Write;
-		case PROT_READ | PROT_WRITE: return VirtualMemory::Mode::ReadWrite; // NOLINT
-		case PROT_EXEC: return VirtualMemory::Mode::Execute;
-		case PROT_EXEC | PROT_WRITE: return VirtualMemory::Mode::ExecuteWrite; // NOLINT
-		case PROT_EXEC | PROT_READ: return VirtualMemory::Mode::ExecuteRead;   // NOLINT
-		case PROT_EXEC | PROT_WRITE | PROT_READ:
-			return VirtualMemory::Mode::ExecuteReadWrite; // NOLINT
-		default: return VirtualMemory::Mode::NoAccess;
-	}
-}
-
 // Keep automatic mappings inside the guest and GPU-addressable low window.
 #ifdef KYTY_FIXED_NOREPLACE
 static constexpr uintptr_t LOW_ARENA_LIMIT = 0x000000FC00000000ULL; // libc mspace window ceiling
@@ -161,11 +144,6 @@ uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mo
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start = ret_addr >> 12u;
-		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = protect;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 	}
 
@@ -251,11 +229,6 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
 
 	pthread_mutex_lock(&g_virtual_mutex);
 	record_alloc(ret_addr, size);
-	uintptr_t page_start = ret_addr >> 12u;
-	uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-	for (uintptr_t page = page_start; page <= page_end; page++) {
-		(*g_protects)[page] = protect;
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 
 	return ret_addr;
@@ -337,11 +310,6 @@ bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode m
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start = ret_addr >> 12u;
-		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = protect;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 
 		return true;
@@ -516,18 +484,7 @@ bool SysVirtualFree(uint64_t address) {
 		return false;
 	}
 
-	if (munmap(reinterpret_cast<void*>(addr), size) == 0) {
-		uintptr_t page_start = addr >> 12u;
-		uintptr_t page_end   = (addr + size - 1) >> 12u;
-		pthread_mutex_lock(&g_virtual_mutex);
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			g_protects->erase(page);
-		}
-		pthread_mutex_unlock(&g_virtual_mutex);
-		return true;
-	}
-
-	return false;
+	return munmap(reinterpret_cast<void*>(addr), size) == 0;
 }
 
 bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
@@ -582,40 +539,16 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 	if (end < alloc_end) {
 		(*g_allocs)[end] = alloc_end - end;
 	}
-	for (uintptr_t page = addr >> 12u; page <= (end - 1u) >> 12u; page++) {
-		g_protects->erase(page);
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 	return true;
 }
 
-bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                       VirtualMemory::Mode* old_mode) {
-	auto addr = static_cast<uintptr_t>(address);
-
-	pthread_mutex_lock(&g_virtual_mutex);
-	if (old_mode != nullptr) {
-		if (auto s = g_protects->find(addr >> 12u); s != g_protects->end()) {
-			*old_mode = get_protection_flag(s->second);
-		} else {
-			*old_mode = VirtualMemory::Mode::NoAccess;
-		}
-	}
-	pthread_mutex_unlock(&g_virtual_mutex);
-
-	uintptr_t page_start = addr >> 12u;
-	uintptr_t page_end   = (addr + size - 1) >> 12u;
-	if (mprotect(reinterpret_cast<void*>(page_start << 12u), (page_end - page_start + 1) << 12u,
-	             get_protection_flag(mode)) == 0) {
-		pthread_mutex_lock(&g_virtual_mutex);
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = get_protection_flag(mode);
-		}
-		pthread_mutex_unlock(&g_virtual_mutex);
-		return true;
-	}
-
-	return false;
+bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
+	const auto addr       = static_cast<uintptr_t>(address);
+	const auto page_start = addr >> 12u;
+	const auto page_end   = (addr + size - 1) >> 12u;
+	return mprotect(reinterpret_cast<void*>(page_start << 12u), (page_end - page_start + 1) << 12u,
+	                get_protection_flag(mode)) == 0;
 }
 
 bool SysVirtualFlushInstructionCache(uint64_t /*address*/, uint64_t /*size*/) {

src/common/platform/sysVirtual.h

@@ -19,8 +19,7 @@ bool     SysVirtualReserveFixed(uint64_t address, uint64_t size);
 bool     SysVirtualDecommit(uint64_t address, uint64_t size);
 bool     SysVirtualFree(uint64_t address);
 bool     SysVirtualFreeRange(uint64_t address, uint64_t size);
-bool     SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                           VirtualMemory::Mode* old_mode = nullptr);
+bool     SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode);
 bool     SysVirtualFlushInstructionCache(uint64_t address, uint64_t size);
 
 } // namespace Common

src/common/platform/sysWindowsDbg.cpp

@@ -15,18 +15,6 @@
 #include "common/assert.h"
 #include "common/platform/sysDbg.h"
 
-void SysStackWalk(void** stack, int* depth) {
-	*depth = static_cast<int>(CaptureStackBackTrace(0, static_cast<DWORD>(*depth), stack, nullptr));
-}
-
-void SysStackUsagePrint(sys_dbg_stack_info_t& stack) {
-	printf("stack: (0x%" PRIx64 ", %" PRIu64 ") + (0x%" PRIx64 ", %" PRIu64 ") + (0x%" PRIx64
-	       ", %" PRIu64 ")\n",
-	       static_cast<uint64_t>(stack.reserved_addr), static_cast<uint64_t>(stack.reserved_size),
-	       static_cast<uint64_t>(stack.guard_addr), static_cast<uint64_t>(stack.guard_size),
-	       static_cast<uint64_t>(stack.commited_addr), static_cast<uint64_t>(stack.commited_size));
-}
-
 void SysStackUsage(sys_dbg_stack_info_t& s) {
 	MEMORY_BASIC_INFORMATION mbi {};
 	[[maybe_unused]] size_t  ss = VirtualQuery(&mbi, &mbi, sizeof(mbi));

src/common/platform/sysWindowsVirtual.cpp

@@ -45,18 +45,6 @@ static DWORD GetProtectionFlag(VirtualMemory::Mode mode) {
 	return protect;
 }
 
-static VirtualMemory::Mode GetProtectionFlag(DWORD mode) {
-	switch (mode) {
-		case PAGE_NOACCESS: return VirtualMemory::Mode::NoAccess;
-		case PAGE_READONLY: return VirtualMemory::Mode::Read;
-		case PAGE_READWRITE: return VirtualMemory::Mode::ReadWrite;
-		case PAGE_EXECUTE: return VirtualMemory::Mode::Execute;
-		case PAGE_EXECUTE_READ: return VirtualMemory::Mode::ExecuteRead;
-		case PAGE_EXECUTE_READWRITE: return VirtualMemory::Mode::ExecuteReadWrite;
-		default: return VirtualMemory::Mode::NoAccess;
-	}
-}
-
 void SysVirtualInit() {}
 
 uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
@@ -328,17 +316,13 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 	return current - address == size && SysVirtualFree(address);
 }
 
-bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                       VirtualMemory::Mode* old_mode) {
+bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
 	DWORD old_protect = 0;
 	if (VirtualProtect(reinterpret_cast<LPVOID>(static_cast<uintptr_t>(address)), size,
 	                   GetProtectionFlag(mode), &old_protect) == 0) {
 		printf("VirtualProtect() failed: 0x%08" PRIx32 "\n", static_cast<uint32_t>(GetLastError()));
 		return false;
 	}
-	if (old_mode != nullptr) {
-		*old_mode = GetProtectionFlag(old_protect);
-	}
 	return true;
 }
 

src/common/profiler.cpp

@@ -55,19 +55,13 @@ void SetThreadName(const char* name) {
 }
 
 void Initialize() {
-	switch (Config::GetProfilerDirection()) {
-		case Config::ProfilerDirection::Network:
-			if (!tracy::ProfilerAvailable()) {
-				tracy::StartupProfiler();
-				TracySetProgramName("KytyPS5");
-				::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, "
-				         "broadcast %u, connect to 127.0.0.1:8086\n",
-				         tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch,
-				         tracy::ProtocolVersion, tracy::BroadcastVersion);
-			}
-			break;
-		case Config::ProfilerDirection::None:
-		default: break;
+	if (Config::ProfilerEnabled() && !tracy::ProfilerAvailable()) {
+		tracy::StartupProfiler();
+		TracySetProgramName("KytyPS5");
+		::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, "
+		         "broadcast %u, connect to 127.0.0.1:8086\n",
+		         tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch,
+		         tracy::ProtocolVersion, tracy::BroadcastVersion);
 	}
 }
 

src/common/threads.cpp

@@ -1,7 +1,6 @@
 #include "common/threads.h"
 
 #include "common/assert.h"
-#include "common/debug.h"
 
 #include <algorithm>
 #include <atomic>
@@ -9,7 +8,6 @@
 #include <chrono>             // IWYU pragma: keep
 #include <condition_variable> // IWYU pragma: keep
 #include <mutex>
-#include <vector>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_COMPILER == KYTY_COMPILER_CLANG
 #define KYTY_WIN_CS
@@ -198,37 +196,7 @@ struct CondVarPrivate {
 #endif
 };
 
-static std::recursive_mutex                         g_cond_waiters_mutex;
-static std::vector<std::pair<int, CondVarPrivate*>> g_cond_waiters;
-static wait_poll_func_t                             g_cond_wait_poll_callback = nullptr;
-
-static void WakeCondVar(CondVarPrivate* cond_var) {
-#ifdef KYTY_WIN_CS
-	static auto func = ResolveWakeAllConditionVariable();
-	EXIT_NOT_IMPLEMENTED(func == nullptr);
-	func(&cond_var->m_cv);
-#else
-	cond_var->m_cv.notify_all();
-#endif
-}
-
-static void RegisterCondWaiter(CondVarPrivate* cond_var) {
-	std::lock_guard lock(g_cond_waiters_mutex);
-	g_cond_waiters.emplace_back(Thread::GetThreadIdUnique(), cond_var);
-}
-
-static void UnregisterCondWaiter(CondVarPrivate* cond_var) {
-	const auto      thread_id = Thread::GetThreadIdUnique();
-	std::lock_guard lock(g_cond_waiters_mutex);
-
-	const auto it = std::find_if(g_cond_waiters.begin(), g_cond_waiters.end(),
-	                             [thread_id, cond_var](const auto& waiter) {
-		                             return waiter.first == thread_id && waiter.second == cond_var;
-	                             });
-	if (it != g_cond_waiters.end()) {
-		g_cond_waiters.erase(it);
-	}
-}
+static wait_poll_func_t g_cond_wait_poll_callback = nullptr;
 
 struct ThreadPrivate {
 	ThreadPrivate(thread_func_t f, void* a): func(f), arg(a), m_thread(&Run, this) {}
@@ -366,7 +334,6 @@ CondVar::~CondVar() {
 }
 
 void CondVar::Wait(Mutex* mutex) {
-	RegisterCondWaiter(m_cond_var.get());
 #ifndef KYTY_WIN_CS
 	std::unique_lock<std::recursive_mutex> cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t());
 #endif
@@ -407,7 +374,6 @@ void CondVar::Wait(Mutex* mutex) {
 	}
 	cpp_lock.release();
 #endif
-	UnregisterCondWaiter(m_cond_var.get());
 }
 
 void CondVar::SetWaitPollCallback(wait_poll_func_t callback) {
@@ -416,7 +382,6 @@ void CondVar::SetWaitPollCallback(wait_poll_func_t callback) {
 
 bool CondVar::WaitFor(Mutex* mutex, uint32_t micros) {
 	bool ok = false;
-	RegisterCondWaiter(m_cond_var.get());
 #ifndef KYTY_WIN_CS
 	std::unique_lock<std::recursive_mutex> cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t());
 #endif
@@ -431,7 +396,6 @@ bool CondVar::WaitFor(Mutex* mutex, uint32_t micros) {
 	      std::cv_status::no_timeout);
 	cpp_lock.release();
 #endif
-	UnregisterCondWaiter(m_cond_var.get());
 	return ok;
 }
 
@@ -446,16 +410,13 @@ void CondVar::Signal() {
 }
 
 void CondVar::SignalAll() {
-	WakeCondVar(m_cond_var.get());
-}
-
-void CondVar::SignalThread(int thread_id) {
-	std::lock_guard lock(g_cond_waiters_mutex);
-	for (const auto& waiter: g_cond_waiters) {
-		if (waiter.first == thread_id) {
-			WakeCondVar(waiter.second);
-		}
-	}
+#ifdef KYTY_WIN_CS
+	static auto func = ResolveWakeAllConditionVariable();
+	EXIT_NOT_IMPLEMENTED(func == nullptr);
+	func(&m_cond_var->m_cv);
+#else
+	m_cond_var->m_cv.notify_all();
+#endif
 }
 
 int Thread::GetThreadIdUnique() {

src/common/threads.h

@@ -77,7 +77,6 @@ public:
 	void Signal();
 	void SignalAll();
 
-	static void SignalThread(int thread_id);
 	static void SetWaitPollCallback(wait_poll_func_t callback);
 
 	KYTY_CLASS_NO_COPY(CondVar);

src/common/virtualMemory.cpp

@@ -50,8 +50,8 @@ bool FreeRange(uint64_t address, uint64_t size) {
 	return SysVirtualFreeRange(address, size);
 }
 
-bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode) {
-	return SysVirtualProtect(address, size, mode, old_mode);
+bool Protect(uint64_t address, uint64_t size, Mode mode) {
+	return SysVirtualProtect(address, size, mode);
 }
 
 bool FlushInstructionCache(uint64_t address, uint64_t size) {

src/common/virtualMemory.h

@@ -35,7 +35,7 @@ bool     ReserveFixed(uint64_t address, uint64_t size);
 bool     Decommit(uint64_t address, uint64_t size);
 bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
-bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
+bool     Protect(uint64_t address, uint64_t size, Mode mode);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
 
 } // namespace VirtualMemory

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -148,7 +148,6 @@ private:
 	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
 	void                CheckBuffer() const { GetScheduler().CheckActive(); }
-	GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); }
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -315,8 +315,17 @@ static void HwCtxSetDepthBoundsRegister(CommandProcessor& cp, uint32_t cmd_offse
 	}
 }
 
-static void HwCtxIgnoreDepthMetadataRegister([[maybe_unused]] uint32_t cmd_offset,
-                                             [[maybe_unused]] uint32_t value) {}
+static void HwCtxSetDepthMetadataRegister(CommandProcessor& cp, uint32_t cmd_offset,
+                                         uint32_t value) {
+	if (cmd_offset == Pm4::DB_RENDER_OVERRIDE) {
+		HW::DepthRenderOverride control;
+		control.force_z_valid       = (value & 0x20000000u) != 0;
+		control.force_z_dirty       = (value & 0x08000000u) != 0;
+		control.force_stencil_valid = (value & 0x40000000u) != 0;
+		control.force_stencil_dirty = (value & 0x10000000u) != 0;
+		cp.GetCtx().SetDepthRenderOverride(control);
+	}
+}
 
 static void HwCtxIgnoreFovWindow([[maybe_unused]] uint32_t value) {}
 
@@ -389,7 +398,7 @@ KYTY_HW_CTX_PARSER(HwCtxSetDepthMetadataRegisters) {
 	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
 
 	for (uint32_t i = 0; i < num_values; i++) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset + i, buffer[i]);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset + i, buffer[i]);
 	}
 
 	return num_values;
@@ -3003,28 +3012,28 @@ void GraphicsInitJmpTablesCxIndirect() {
 		HwCtxIgnoreCbDccControl(value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_COUNT_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	for (auto cmd_offset = Pm4::DB_SRESULTS_COMPARE_STATE0;
 	     cmd_offset <= Pm4::DB_SRESULTS_COMPARE_STATE1; cmd_offset++) {
 		g_hw_ctx_indirect_func[cmd_offset] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-			HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+			HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 		};
 	}
 	g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_RENDER_OVERRIDE2] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_DFSM_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::DB_RMI_L2_CACHE_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::CB_RMI_GL2_CACHE_CONTROL] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwCtxIgnoreDepthMetadataRegister(cmd_offset, value);
+		HwCtxSetDepthMetadataRegister(cp, cmd_offset, value);
 	};
 	g_hw_ctx_indirect_func[Pm4::TA_BC_BASE_ADDR] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
 		HwCtxIgnoreBorderColorTableAddr(cmd_offset, value);
@@ -3626,6 +3635,11 @@ void GraphicsInitJmpTablesShIndirect() {
 	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
+	// Toolkit state restoration emits the compiler's GS-front allocation
+	// metadata here. Native GS resource registers already carry that allocation.
+	g_hw_sh_indirect_func[0x0ca] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+		HwShIgnoreShaderRegister(cmd_offset, value);
+	};
 	g_hw_sh_indirect_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
@@ -3645,12 +3659,13 @@ void GraphicsInitJmpTablesShIndirect() {
 	g_hw_sh_indirect_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
-	g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
-		HwShIgnoreShaderRegister(cmd_offset, value);
-	};
-	g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
-		HwShIgnoreShaderRegister(cmd_offset, value);
-	};
+	for (uint32_t offset = Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS;
+	     offset <= Pm4::SPI_SHADER_USER_DATA_ADDR_HI_HS; offset++) {
+		g_hw_sh_indirect_func[offset] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+			cp.GetShCtx().SetHsUserDataAddress(cmd_offset - Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS,
+			                                   value);
+		};
+	}
 	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		auto base = cp.GetShCtx().GetVs().hs_regs.data_addr;
 		base &= 0xFFFFFF00000000FFull;
@@ -3809,6 +3824,25 @@ void GraphicsInitJmpTablesUcIndirect() {
 	for (auto& func: g_hw_uc_indirect_func) {
 		func = nullptr;
 	}
+	for (uint32_t i = 0; i < 4; ++i) {
+		g_hw_uc_indirect_func[Pm4::FSR_CONTROL_POINTS_LEFT_X + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrControlPoint(0, cmd_offset - Pm4::FSR_CONTROL_POINTS_LEFT_X, value);
+		};
+		g_hw_uc_indirect_func[Pm4::FSR_CONTROL_POINTS_LEFT_Y + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrControlPoint(1, cmd_offset - Pm4::FSR_CONTROL_POINTS_LEFT_Y, value);
+		};
+	}
+	for (uint32_t i = 0; i < 2; ++i) {
+		g_hw_uc_indirect_func[Pm4::FSR_ALPHA_LEFT_X + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrAlpha(0, cmd_offset - Pm4::FSR_ALPHA_LEFT_X, value);
+		};
+		g_hw_uc_indirect_func[Pm4::FSR_ALPHA_LEFT_Y + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrAlpha(1, cmd_offset - Pm4::FSR_ALPHA_LEFT_Y, value);
+		};
+		g_hw_uc_indirect_func[Pm4::FSR_WINDOW_LEFT + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrWindow(cmd_offset - Pm4::FSR_WINDOW_LEFT, value);
+		};
+	}
 
 	g_hw_uc_indirect_func[Pm4::GE_CNTL] = [](KYTY_HW_UC_INDIRECT_ARGS) {
 		HW::GeControl r;

src/graphics/guest_gpu/graphicsRun.cpp

@@ -77,7 +77,7 @@ private:
 
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
-	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
+	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
 }
 
 GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
@@ -427,7 +427,7 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 	if (!decode_gds(dst_sel, dst_gds)) {
 		EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel);
 	}
-	auto& buffer_cache = GetGpuResources().GetBufferCache();
+	auto& buffer_cache = m_renderer.GetBufferCache();
 	if (src_sel == 2) {
 		buffer_cache.FillBuffer(
 		    dst_address_or_offset, num_bytes,
@@ -603,11 +603,11 @@ bool GuestGpu::Process(Submission& submission) {
 			}
 			if (progressed) {
 				if (complete) {
-					m_renderer.GetGpuResources().RunGarbageCollector();
+					m_renderer.RunGarbageCollector();
 				}
 				cp.BufferFlush();
 			} else if (complete) {
-				m_renderer.GetGpuResources().RunGarbageCollector();
+				m_renderer.RunGarbageCollector();
 			}
 			break;
 		}
@@ -629,16 +629,16 @@ bool GuestGpu::Process(Submission& submission) {
 			           Pm4ProcessResult::Complete;
 			if (submission.command_execution.MadeProgress()) {
 				if (complete) {
-					m_renderer.GetGpuResources().RunGarbageCollector();
+					m_renderer.RunGarbageCollector();
 				}
 				cp.BufferFlush();
 			} else if (complete) {
-				m_renderer.GetGpuResources().RunGarbageCollector();
+				m_renderer.RunGarbageCollector();
 			}
 			break;
 		}
 		case SubmissionType::FlipPreparation:
-			m_renderer.GetGpuResources().RunGarbageCollector();
+			m_renderer.RunGarbageCollector();
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
 	}
@@ -1292,6 +1292,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 								break;
 							case 0x14:
 							case 0x28:
+							case 0x2f:
 								if (event_index == 0x00) {
 									write64(false);
 									return;
@@ -1299,7 +1300,6 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 								break;
 							case 0x2b:
 							case 0x2d:
-							case 0x2f:
 							case 0x30:
 								if (event_index == 0x00 && !with_interrupt) {
 									write64(false);

src/graphics/guest_gpu/hardwareContext.h

@@ -202,6 +202,13 @@ struct RenderControl {
 	uint8_t copy_sample              = 0;
 };
 
+struct DepthRenderOverride {
+	bool force_z_valid       = false;
+	bool force_z_dirty       = false;
+	bool force_stencil_valid = false;
+	bool force_stencil_dirty = false;
+};
+
 struct GdsOaCounter {
 	uint32_t counter = 0;
 	uint32_t address = 0;
@@ -491,7 +498,8 @@ struct LsStageRegisters {
 };
 
 struct HsStageRegisters {
-	uint64_t          data_addr = 0;
+	uint64_t          data_addr      = 0;
+	uint64_t          user_data_addr = 0;
 	HsShaderResource1 rsrc1;
 	HsShaderResource2 rsrc2;
 };
@@ -809,6 +817,12 @@ public:
 	void SetClipControl(const ClipControl& control) { m_clip_control = control; }
 	[[nodiscard]] const RenderControl& GetRenderControl() const { return m_render_control; }
 	void SetRenderControl(const RenderControl& control) { m_render_control = control; }
+	[[nodiscard]] const DepthRenderOverride& GetDepthRenderOverride() const {
+		return m_depth_render_override;
+	}
+	void SetDepthRenderOverride(const DepthRenderOverride& control) {
+		m_depth_render_override = control;
+	}
 	[[nodiscard]] const DepthControl& GetDepthControl() const { return m_depth_control; }
 	void SetDepthControl(const DepthControl& control) { m_depth_control = control; }
 	[[nodiscard]] const ModeControl& GetModeControl() const { return m_mode_control; }
@@ -904,15 +918,16 @@ private:
 
 	uint32_t m_shader_stages = 0;
 
-	DepthRenderTarget m_depth_render_target;
-	RenderControl     m_render_control;
-	DepthControl      m_depth_control;
-	StencilControl    m_stencil_control;
-	StencilMask       m_stencil_mask;
-	float             m_depth_clear_value   = 0.0f;
-	float             m_depth_bounds_min    = 0.0f;
-	float             m_depth_bounds_max    = 1.0f;
-	uint8_t           m_stencil_clear_value = 0;
+	DepthRenderTarget   m_depth_render_target;
+	RenderControl       m_render_control;
+	DepthRenderOverride m_depth_render_override;
+	DepthControl        m_depth_control;
+	StencilControl      m_stencil_control;
+	StencilMask         m_stencil_mask;
+	float               m_depth_clear_value   = 0.0f;
+	float               m_depth_bounds_min    = 0.0f;
+	float               m_depth_bounds_max    = 1.0f;
+	uint8_t             m_stencil_clear_value = 0;
 
 	ModeControl m_mode_control;
 	PolyOffset  m_poly_offset;
@@ -921,6 +936,12 @@ private:
 	ShaderRegisters m_sh_regs;
 };
 
+struct FsrView {
+	uint32_t control_points[2][4] {};
+	uint32_t alphas[2][2] {};
+	uint32_t window[2] {};
+};
+
 class UserConfig {
 public:
 	UserConfig()  = default;
@@ -950,8 +971,17 @@ public:
 	void SetGdsOaCntl(uint32_t value) { m_gds_oa.cntl = value; }
 	void SetGdsOaCounter(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].counter = value; }
 	void SetGdsOaAddress(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].address = value; }
+	void SetFsrControlPoint(uint32_t axis, uint32_t index, uint32_t value) {
+		m_fsr_view.control_points[axis][index] = value;
+	}
+	void SetFsrAlpha(uint32_t axis, uint32_t index, uint32_t value) {
+		m_fsr_view.alphas[axis][index] = value;
+	}
+	void SetFsrWindow(uint32_t index, uint32_t value) { m_fsr_view.window[index] = value; }
+	[[nodiscard]] const FsrView& GetFsrView() const { return m_fsr_view; }
 
 private:
+	FsrView m_fsr_view;
 	Prospero::PrimitiveType m_prim_type               = Prospero::PrimitiveType::kNone;
 	uint32_t                m_index_offset            = 0;
 	uint32_t                m_object_id               = 0;
@@ -974,14 +1004,14 @@ public:
 	void SetEsShaderBase(uint64_t addr) { m_vs.es_regs.data_addr = addr; }
 	void SetLsShaderBase(uint64_t addr) { m_vs.ls_regs.data_addr = addr; }
 	void SetHsShaderBase(uint64_t addr) { m_vs.hs_regs.data_addr = addr; }
+	void SetHsUserDataAddress(uint32_t word, uint32_t value) {
+		SetUserDataAddressWord(m_vs.hs_regs.user_data_addr, word, value);
+	}
 	void SetHsShaderResource1(const HsShaderResource1& rsrc1) { m_vs.hs_regs.rsrc1 = rsrc1; }
 	void SetHsShaderResource2(const HsShaderResource2& rsrc2) { m_vs.hs_regs.rsrc2 = rsrc2; }
 	void SetGsShaderBase(uint64_t addr) { m_vs.gs_regs.data_addr = addr; }
 	void SetGsUserDataAddress(uint32_t word, uint32_t value) {
-		const auto shift   = word * 32u;
-		auto&      address = m_vs.gs_regs.user_data_addr;
-		address = (address & ~(uint64_t {0xffffffffu} << shift)) |
-		          (static_cast<uint64_t>(value) << shift);
+		SetUserDataAddressWord(m_vs.gs_regs.user_data_addr, word, value);
 	}
 	void SetGsShaderResource1(const GsShaderResource1& rsrc1) { m_vs.gs_regs.rsrc1 = rsrc1; }
 	void SetGsShaderResource2(const GsShaderResource2& rsrc2) { m_vs.gs_regs.rsrc2 = rsrc2; }
@@ -1023,6 +1053,12 @@ public:
 	[[nodiscard]] const ComputeShaderInfo& GetCs() const { return m_cs; }
 
 private:
+	static void SetUserDataAddressWord(uint64_t& address, uint32_t word, uint32_t value) {
+		const auto shift = word * 32u;
+		address = (address & ~(uint64_t {0xffffffffu} << shift)) |
+		          (static_cast<uint64_t>(value) << shift);
+	}
+
 	VertexShaderInfo  m_vs;
 	PixelShaderInfo   m_ps;
 	ComputeShaderInfo m_cs;

src/graphics/guest_gpu/pm4.h

@@ -987,6 +987,12 @@ constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR2_MASK  = 0x1;
 constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR3_SHIFT = 2;
 constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR3_MASK  = 0x1;
 
+constexpr uint32_t FSR_WINDOW_LEFT           = 0x2BC;
+constexpr uint32_t FSR_CONTROL_POINTS_LEFT_X = 0x2C4;
+constexpr uint32_t FSR_CONTROL_POINTS_LEFT_Y = 0x2CC;
+constexpr uint32_t FSR_ALPHA_LEFT_X          = 0x2E4;
+constexpr uint32_t FSR_ALPHA_LEFT_Y          = 0x2E8;
+
 constexpr uint32_t TA_CS_BC_BASE_ADDR       = 0x380;
 constexpr uint32_t TA_CS_BC_BASE_ADDR_HI    = 0x381;
 constexpr uint32_t TEXTURE_GRADIENT_FACTORS = 0x383;

src/graphics/guest_gpu/tile.cpp

@@ -1,5 +1,6 @@
 #include "graphics/guest_gpu/tile.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -15,10 +16,6 @@
 
 namespace Libs::Graphics {
 
-static uint32_t AlignUp(uint32_t value, uint32_t alignment) {
-	return (value + alignment - 1u) & ~(alignment - 1u);
-}
-
 static uint32_t ShiftCeil(uint32_t value, uint32_t shift) {
 	return static_cast<uint32_t>((static_cast<uint64_t>(value) + (1ull << shift) - 1ull) >> shift);
 }
@@ -33,7 +30,7 @@ static uint32_t CalcLinearAlignedLevelPitch(uint32_t base_width, uint32_t base_h
 	const uint32_t level_width  = ShiftCeil(base_width, level);
 	const uint32_t level_height = ShiftCeil(base_height, level);
 	const uint32_t padded_width =
-	    AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element));
+	    Common::AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element));
 	const uint64_t size =
 	    static_cast<uint64_t>(padded_width) * std::max(level_height, 1u) * bytes_per_element;
 	EXIT_NOT_IMPLEMENTED(size > 0xffffffffull);
@@ -61,7 +58,7 @@ static uint32_t SetLinearMipChainLayout(uint32_t levels, const uint32_t* mip_pit
 		offset += mip_size[level];
 	}
 
-	return AlignUp(offset, 256u);
+	return Common::AlignUp(offset, 256u);
 }
 
 struct Log2BlockDimensions {
@@ -366,8 +363,8 @@ bool TileGetTiledTextureLayout(const TileSurfaceDescription& description, TileSu
 		    ((description.width >> level) + texture.texel_width - 1u) / texture.texel_width, 1u);
 		mip.height = std::max(
 		    ((description.height >> level) + texture.texel_height - 1u) / texture.texel_height, 1u);
-		mip.padded_width  = AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width);
-		mip.padded_height = AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height);
+		mip.padded_width  = Common::AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width);
+		mip.padded_height = Common::AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height);
 		mip.size = static_cast<uint64_t>(block.block_depth) * mip.padded_width * mip.padded_height *
 		           block.bytes_per_element;
 		result.block_slice_size += mip.size;
@@ -1121,8 +1118,8 @@ static bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign& hti
 	}
 	// Prospero HTile stores one DWORD per depth tile. Its 32 KiB allocation blocks cover
 	// 1024x512 pixels, independently of the attachment's fragment count.
-	const uint64_t size = static_cast<uint64_t>(AlignUp(width, 1024u) / 1024u) *
-	                      (AlignUp(height, 512u) / 512u) * 32768u;
+	const uint64_t size = static_cast<uint64_t>(Common::AlignUp(width, 1024u) / 1024u) *
+	                      (Common::AlignUp(height, 512u) / 512u) * 32768u;
 	if (size == 0 || size > UINT32_MAX) {
 		return false;
 	}
@@ -1154,13 +1151,13 @@ bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch,
 		                                           &stencil_block_height));
 		const uint32_t fragments = 1u << num_fragments_log2;
 		const uint64_t depth_bytes_total =
-		    valid_blocks ? static_cast<uint64_t>(AlignUp(width, depth_block_width)) *
-		                       AlignUp(height, depth_block_height) * depth_bytes * fragments
+		    valid_blocks ? static_cast<uint64_t>(Common::AlignUp(width, depth_block_width)) *
+		                       Common::AlignUp(height, depth_block_height) * depth_bytes * fragments
 		                 : 0;
 		const uint64_t stencil_bytes_total =
 		    stencil_format == Prospero::StencilFormat::k8UInt && valid_blocks
-		        ? static_cast<uint64_t>(AlignUp(width, stencil_block_width)) *
-		              AlignUp(height, stencil_block_height) * fragments
+		        ? static_cast<uint64_t>(Common::AlignUp(width, stencil_block_width)) *
+		              Common::AlignUp(height, stencil_block_height) * fragments
 		        : 0;
 		TileSizeAlign calculated_htile {};
 		const bool    htile_valid = !htile || TileGetHtileSize(width, height, calculated_htile);
@@ -1187,8 +1184,7 @@ uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element,
 	                                                         &block_width, &block_height)) {
 		return 0;
 	}
-	const uint64_t pitch = (static_cast<uint64_t>(width) + block_width - 1u) &
-	                       ~static_cast<uint64_t>(block_width - 1u);
+	const uint64_t pitch = Common::AlignUp(static_cast<uint64_t>(width), block_width);
 	return pitch <= UINT32_MAX ? static_cast<uint32_t>(pitch) : 0;
 }
 
@@ -1209,8 +1205,7 @@ bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
 	    pitch != TileGetRenderTargetPitch(width, bytes_per_element, num_fragments_log2)) {
 		return false;
 	}
-	const uint64_t padded_height = (static_cast<uint64_t>(height) + block_height - 1u) &
-	                               ~static_cast<uint64_t>(block_height - 1u);
+	const uint64_t padded_height = Common::AlignUp(static_cast<uint64_t>(height), block_height);
 	const uint64_t size = static_cast<uint64_t>(pitch) * padded_height * bytes_per_element *
 	                      (1u << num_fragments_log2);
 	if (size == 0 || size > UINT32_MAX) {
@@ -1372,7 +1367,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 			TileTextureElementLayout element {};
 			if (TileGetTextureElementLayout(format, element) && element.texel_width == 1 &&
 			    element.texel_height == 1) {
-				pitch = AlignUp(pitch, CalcLinearBlockWidth(element.bytes));
+				pitch = Common::AlignUp(pitch, CalcLinearBlockWidth(element.bytes));
 			}
 			break;
 		}
@@ -1383,7 +1378,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 		case Prospero::TileMode::kRenderTarget: {
 			TileTextureBlockLayout layout {};
 			if (TileGetTextureBlockLayout(format, tile, false, layout)) {
-				pitch = AlignUp(pitch, layout.block.block_width * layout.texel_width);
+				pitch = Common::AlignUp(pitch, layout.block.block_width * layout.texel_width);
 			}
 			break;
 		}

src/graphics/host_gpu/graphicContext.h

@@ -15,7 +15,6 @@
 namespace Libs::Graphics {
 
 struct VulkanImage;
-struct VulkanMemory;
 
 inline constexpr uint32_t VULKAN_TARGET_API_VERSION = VK_API_VERSION_1_3;
 
@@ -28,10 +27,10 @@ struct GraphicContext {
 	vk::Device                         device                                = nullptr;
 	VmaAllocator                       allocator                             = nullptr;
 	bool                               memory_budget_ext_enabled             = false;
-	bool                               rt_extensions_enabled                 = false;
 	bool                               compute_subgroup_size_control_enabled = false;
 	bool                               sample_rate_shading_enabled           = false;
 	bool                               attachment_feedback_loop_enabled      = false;
+	bool                               provoking_vertex_last_enabled         = false;
 	bool                               supports_block_texel_view              = false;
 	bool                                      mesh_shader_enabled                   = false;
 	vk::PhysicalDeviceMeshShaderPropertiesEXT mesh_shader_properties                = {};
@@ -99,10 +98,6 @@ struct GraphicContext {
 	[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
 	[[nodiscard]] bool     CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
 	void                   DeleteImage(VulkanImage& image);
-	void                   AppendHardwareRayTracingDeviceExtensions(
-	    const std::vector<vk::ExtensionProperties>& available_extensions,
-	    std::vector<const char*>&                   device_extensions);
-	void LoadHardwareRayTracingFunctions() const;
 
 	uint32_t screen_width  = 0;
 	uint32_t screen_height = 0;
@@ -117,12 +112,6 @@ private:
 	    m_image_format_properties;
 };
 
-struct VulkanMemory {
-	vk::MemoryPropertyFlags property           = {};
-	vk::MemoryPropertyFlags preferred_property = {};
-	VmaAllocation           allocation         = nullptr;
-};
-
 struct VulkanImageState {
 	vk::PipelineStageFlags2 pl_stage    = vk::PipelineStageFlagBits2::eAllCommands;
 	vk::AccessFlags2        access_mask = vk::AccessFlagBits2::eNone;
@@ -144,7 +133,7 @@ struct VulkanImage {
 	vk::Image                     image       = nullptr;
 	VulkanImageState              state;
 	std::vector<VulkanImageState> subresource_states;
-	Graphics::VulkanMemory        memory;
+	VmaAllocation                allocation = nullptr;
 };
 
 

src/graphics/host_gpu/memoryTracker.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/memoryTracker.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 
 namespace Libs::Graphics {
@@ -8,9 +9,6 @@ static_assert(std::atomic<void*>::is_always_lock_free);
 
 MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) {
 	m_regions = std::make_unique<std::atomic<RegionManager*>[]>(REGION_COUNT);
-	for (size_t i = 0; i < REGION_COUNT; i++) {
-		m_regions[i].store(nullptr, std::memory_order_relaxed);
-	}
 }
 
 MemoryTracker::~MemoryTracker() = default;
@@ -34,8 +32,8 @@ void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr,
 void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
                                               const char* operation) {
 	ValidateRange(vaddr, size);
-	const auto begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
+	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
 	for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
 		const bool has_dirty_bytes = dirty.Intersects(page, TRACKER_PAGE_SIZE);
 		if (IsRegionGpuModified(page, TRACKER_PAGE_SIZE) != has_dirty_bytes) {

src/graphics/host_gpu/memoryTracker.h

@@ -62,43 +62,18 @@ public:
 	void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {}
 #endif
 
-	template <bool clear, typename Preflight, typename Func>
-	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) {
-		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
-		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
-		CheckNotInUploadCallback();
-		std::vector<RegionManager*> managers;
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
-			managers.push_back(manager);
-		});
-		std::vector<std::unique_lock<TrackingSpinLock>> locks;
-		locks.reserve(managers.size());
-		for (auto* manager: managers) {
-			locks.emplace_back(manager->lock);
-		}
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			const auto address = manager->GetCpuAddr() + offset;
-			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes,
-			                                                                preflight);
-		});
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(
-			    manager->GetCpuAddr() + offset, bytes, func);
-		});
-		if constexpr (clear) {
-			Iterate<false>(vaddr, size,
-			               [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				               const auto address = manager->GetCpuAddr() + offset;
-				               manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
-				                   address, bytes, [](uint64_t, uint64_t) noexcept {});
-			               });
-		}
-	}
-
 	template <bool clear, typename Func>
 	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Func&& func) {
-		ForEachDownloadRange<clear>(
-		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
+		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
+		CheckNotInUploadCallback();
+		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+			std::scoped_lock lock(manager->lock);
+			const auto       address = manager->GetCpuAddr() + offset;
+			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes, func);
+			if constexpr (clear) {
+				manager->template ChangeState<DirtySource::Gpu, false>(address, bytes);
+			}
+		});
 	}
 
 	template <typename RangeFunc, typename UploadFunc>

src/graphics/host_gpu/pageManager.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/pageManager.h"
 
-#include "common/platform/sysDbg.h"
+#include "common/alignment.h"
+#include "common/virtualMemory.h"
 #include "graphics/host_gpu/regionDefinitions.h"
 #include "kernel/memory.h"
 
@@ -21,10 +22,7 @@
 #include <windows.h>
 #undef min
 #undef max
-#elif defined(__APPLE__)
-#include <unistd.h>
 #else
-#include <execinfo.h>
 #include <unistd.h>
 #endif
 
@@ -36,38 +34,12 @@ constexpr uint64_t REGION_SIZE  = TRACKER_REGION_SIZE;
 constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
 constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
 
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-// The tracker reuses Win32 memory-protection tags as internal page-state values.
-// Mirror their canonical numeric values so the shared state-machine logic is identical.
-constexpr uint32_t PAGE_NOACCESS  = 0x01;
-constexpr uint32_t PAGE_READONLY  = 0x02;
-constexpr uint32_t PAGE_READWRITE = 0x04;
-#endif
 constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 
-constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
-constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
-constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-
 [[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
 	std::fputs("PageManager fail-fast: ", stderr);
 	std::fputs(reason != nullptr ? reason : "invalid page state", stderr);
 	std::fputc('\n', stderr);
-#if !defined(__APPLE__)
-	void* frames[16] {};
-	int   frame_count = static_cast<int>(std::size(frames));
-	SysStackWalk(frames, &frame_count);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	const auto image_base = reinterpret_cast<uintptr_t>(GetModuleHandleW(nullptr));
-	for (int i = 0; i < frame_count; i++) {
-		const auto address = reinterpret_cast<uintptr_t>(frames[i]);
-		std::fprintf(stderr, "  frame[%d]=0x%016" PRIxPTR " image_rva=0x%016" PRIxPTR "\n", i,
-		             address, address >= image_base ? address - image_base : 0);
-	}
-#else
-	::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
-#endif
-#endif
 	std::fflush(stderr);
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
@@ -86,15 +58,6 @@ constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
 	std::_Exit(322);
 }
 
-Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
-	switch (protection) {
-		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
-		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
-		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
-		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
-	}
-}
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -115,15 +78,6 @@ void ValidateRange(uint64_t vaddr, uint64_t size) {
 	}
 }
 
-uint64_t PageStart(uint64_t vaddr) {
-	return vaddr & ~(PAGE_SIZE - 1);
-}
-
-uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
-	ValidateRange(vaddr, size);
-	return PageStart(vaddr + size - 1) + PAGE_SIZE;
-}
-
 } // namespace
 
 struct PageManager::Impl {
@@ -131,14 +85,14 @@ struct PageManager::Impl {
 		uint8_t write_watchers  : 7 = 0;
 		uint8_t access_watchers : 1 = 0;
 
-		[[nodiscard]] uint32_t Perms() const noexcept {
+		[[nodiscard]] Common::VirtualMemory::Mode Perms() const noexcept {
 			if (access_watchers != 0) {
-				return NO_ACCESS_PROTECTION;
+				return Common::VirtualMemory::Mode::NoAccess;
 			}
 			if (write_watchers != 0) {
-				return READ_ONLY_PROTECTION;
+				return Common::VirtualMemory::Mode::Read;
 			}
-			return READ_WRITE_PROTECTION;
+			return Common::VirtualMemory::Mode::ReadWrite;
 		}
 
 		template <int delta, bool is_read>
@@ -202,9 +156,6 @@ struct PageManager::Impl {
 		}
 #endif
 		regions = std::make_unique<std::atomic<Region*>[]>(REGION_COUNT);
-		for (uint64_t i = 0; i < REGION_COUNT; i++) {
-			regions[i].store(nullptr, std::memory_order_relaxed);
-		}
 	}
 
 	~Impl() {
@@ -239,11 +190,10 @@ struct PageManager::Impl {
 		return ptr;
 	}
 
-	void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept {
-		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
-		                                                     ToMemoryMode(protection))) {
-			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
-			      protection);
+	void Protect(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) noexcept {
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, mode)) {
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", mode=0x%08" PRIx32, vaddr,
+			      static_cast<uint32_t>(mode));
 		}
 	}
 
@@ -300,11 +250,12 @@ struct PageManager::Impl {
 
 	template <bool track, bool is_read>
 	void UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
-		const auto begin = PageStart(vaddr);
-		const auto end   = PageEnd(vaddr, size);
+		ValidateRange(vaddr, size);
+		const auto begin = Common::AlignDown(vaddr, PAGE_SIZE);
+		const auto end   = Common::AlignUp(vaddr + size, PAGE_SIZE);
 		for (auto chunk_begin = begin; chunk_begin < end;) {
-			const auto chunk_end   = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
-			const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE;
+			const auto chunk_end = std::min(end, Common::AlignUp(chunk_begin + 1, REGION_SIZE));
+			const auto region_base = Common::AlignDown(chunk_begin, REGION_SIZE);
 			auto*      region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin);
 			if (region == nullptr) {
 				Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);

src/graphics/host_gpu/rangeSet.h

@@ -6,17 +6,11 @@
 #include <algorithm>
 #include <cstdint>
 #include <map>
-#include <vector>
 
 namespace Libs::Graphics {
 
 class RangeSet final {
 public:
-	struct Range {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-
 	void Add(uint64_t address, uint64_t size) {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.lower_bound(address);
@@ -62,12 +56,6 @@ public:
 		}
 	}
 
-	[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
-		std::vector<Range> result;
-		ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
-		return result;
-	}
-
 	[[nodiscard]] bool Intersects(uint64_t address, uint64_t size) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.lower_bound(address);
@@ -88,7 +76,7 @@ public:
 	}
 
 	template <typename Func>
-	void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
+	void ForEachInRange(uint64_t address, uint64_t size, Func&& func) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.upper_bound(address);
 		if (it != m_ranges.begin()) {
@@ -98,7 +86,7 @@ public:
 			const auto begin = std::max(address, it->first);
 			const auto last  = std::min(end, it->second);
 			if (begin < last) {
-				func(Range {begin, last - begin});
+				func(begin, last);
 			}
 		}
 	}

src/graphics/host_gpu/regionManager.h

@@ -111,56 +111,43 @@ public:
 			bits.UnsetRange(start, end);
 		}
 		if constexpr (source == DirtySource::Cpu) {
-			UpdateCpuProtection<!enable>();
+			UpdateProtection<!enable, false>();
 		} else {
-			UpdateGpuProtection<enable>();
+			UpdateProtection<enable, true>();
 		}
 	}
 
 	template <DirtySource source, bool clear, typename Func>
 	void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
 		const auto [start, end] = GetPageRange(vaddr, size);
-		RegionBits mask(GetBits<source>(), start, end);
+		auto&      bits         = GetBits<source>();
+		RegionBits mask(bits, start, end);
 		if constexpr (clear) {
-			GetBits<source>().UnsetRange(start, end);
+			bits.UnsetRange(start, end);
+			if constexpr (source == DirtySource::Cpu) {
+				UpdateProtection<true, false>();
+			} else {
+				UpdateProtection<false, true>();
+			}
 		}
-		if constexpr (source == DirtySource::Cpu && clear) {
-			UpdateCpuProtection<true>();
-			ForEachRange(mask, std::forward<Func>(func));
-			return;
+		for (const auto [first, last]: mask) {
+			func(m_cpu_addr + first * TRACKER_PAGE_SIZE, (last - first) * TRACKER_PAGE_SIZE);
 		}
-		if constexpr (source == DirtySource::Gpu && clear) {
-			UpdateGpuProtection<false>();
-		}
-		ForEachRange(mask, std::forward<Func>(func));
 	}
 
 	TrackingSpinLock lock;
 
 private:
-	template <bool track>
-	void UpdateCpuProtection() {
-		auto mask  = m_cpu_dirty ^ m_writable;
-		m_writable = m_cpu_dirty;
+	template <bool track, bool is_read>
+	void UpdateProtection() {
+		const auto protection = is_read ? ~m_gpu_dirty : m_cpu_dirty;
+		auto&      previous   = is_read ? m_readable : m_writable;
+		auto       mask       = protection ^ previous;
 		if (mask.None()) {
 			return;
 		}
-		m_page_manager.UpdatePageWatchersForRegion<track>(m_cpu_addr, mask);
-	}
-
-	template <bool track>
-	void UpdateGpuProtection() {
-		auto readable = ~m_gpu_dirty;
-		auto mask     = readable ^ m_readable;
-		m_readable    = readable;
-		if (mask.None()) {
-			return;
-		}
-		if constexpr (track) {
-			m_page_manager.UpdatePageWatchersForRegion<true, true>(m_cpu_addr, mask);
-		} else {
-			m_page_manager.UpdatePageWatchersForRegion<false, true>(m_cpu_addr, mask);
-		}
+		previous = protection;
+		m_page_manager.UpdatePageWatchersForRegion<track, is_read>(m_cpu_addr, mask);
 	}
 
 	template <DirtySource source>
@@ -191,13 +178,6 @@ private:
 		        static_cast<size_t>((offset + size + TRACKER_PAGE_SIZE - 1) / TRACKER_PAGE_SIZE)};
 	}
 
-	template <typename Func>
-	void ForEachRange(const RegionBits& bits, Func&& func) const {
-		for (const auto [start, end]: bits) {
-			func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE);
-		}
-	}
-
 	PageManager& m_page_manager;
 	uint64_t     m_cpu_addr = 0;
 	RegionBits   m_cpu_dirty;

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -42,13 +43,6 @@ void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void*
 	}
 }
 
-struct BufferCache::DownloadCopy {
-	Buffer*  buffer        = nullptr;
-	uint64_t source_offset = 0;
-	uint64_t address       = 0;
-	uint64_t size          = 0;
-};
-
 void BufferCache::Register(BufferId id) {
 	ChangeRegister<true>(id);
 }
@@ -103,8 +97,7 @@ void BufferCache::TouchBuffer(const Buffer& buffer) {
 }
 
 void BufferCache::DeleteBuffer(BufferId id) {
-	auto* buffer = m_slot_buffers.try_get(id);
-	if (buffer == nullptr || buffer->is_deleted) {
+	if (IsBufferInvalid(id)) {
 		return;
 	}
 	Unregister(id);
@@ -115,79 +108,70 @@ void BufferCache::DeleteBuffer(BufferId id) {
 	}
 }
 
-std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
-	if (copy.buffer == nullptr || copy.size == 0 || copy.source_offset > copy.buffer->Size() ||
-	    copy.size > copy.buffer->Size() - copy.source_offset) {
-		EXIT("BufferCache: invalid download copy\n");
+bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size) {
+	std::vector<vk::BufferCopy> copies;
+	uint64_t                    total_size     = 0;
+	const auto                  buffer_address = buffer.CpuAddress();
+	m_memory_tracker.ForEachDownloadRange<false>(
+	    vaddr, size, [&](uint64_t address, uint64_t bytes) noexcept {
+		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
+		                                           "buffer download");
+		    m_gpu_modified_ranges.ForEachInRange(address, bytes, [&](uint64_t start, uint64_t end) {
+			    copies.emplace_back(start - buffer_address, total_size, end - start);
+			    // Keep packed ranges on separate cache lines, as in shadPS4.
+			    total_size += Common::AlignUp(end - start, 64);
+		    });
+		    m_gpu_modified_ranges.Subtract(address, bytes);
+	    });
+	if (copies.empty()) {
+		return false;
 	}
-	const auto begin = copy.source_offset & ~uint64_t {3};
-	if (copy.source_offset > UINT64_MAX - copy.size ||
-	    copy.source_offset + copy.size > UINT64_MAX - 3) {
-		EXIT("BufferCache: download copy alignment overflow\n");
-	}
-	const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3};
-	if (end > copy.buffer->Size()) {
-		EXIT("BufferCache: aligned download copy exceeds its owner\n");
-	}
-	return {begin, end - begin};
-}
 
-void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
-	std::vector<DownloadCopy> batch;
-	batch.reserve(copies.size());
-	uint64_t                  packed_size = 0;
-	auto&                     download    = m_download_buffer;
-	const auto flush = [&] {
-		const auto [mapped, base_offset] = download.Map(packed_size, DOWNLOAD_ALIGNMENT);
-		EXIT_IF(mapped == nullptr);
-		uint64_t cursor = 0;
-		for (const auto& copy: batch) {
-			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-			download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
-			                  envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
-			                  vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
-			                  vk::AccessFlagBits::eHostRead);
-			cursor += AlignDownload(envelope_size);
-		}
-		download.Commit();
-		const auto completion_tick = m_scheduler.CurrentTick();
-		m_scheduler.Finish();
-		m_scheduler.WaitPriorityOperations(completion_tick);
-		cursor = 0;
-		for (const auto& copy: batch) {
-			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-			const auto offset = cursor + copy.source_offset - source_begin;
-			download.Invalidate(base_offset + offset, copy.size);
-			Libs::LibKernel::Memory::WriteBacking(copy.address, mapped + offset, copy.size);
-			cursor += AlignDownload(envelope_size);
-		}
-		batch.clear();
-		packed_size = 0;
-	};
-	for (auto copy: copies) {
-		while (copy.size != 0) {
-			const auto available = download.Size() - packed_size;
-			const auto prefix    = copy.source_offset & 3u;
-			const auto bytes     = std::min(copy.size, available - prefix);
-			DownloadCopy part {copy.buffer, copy.source_offset, copy.address, bytes};
-			const auto [source_begin, envelope_size] = DownloadEnvelope(part);
-			(void)source_begin;
-			packed_size += AlignDownload(envelope_size);
-			batch.push_back(part);
-			copy.source_offset += bytes;
-			copy.address += bytes;
-			copy.size -= bytes;
-			if (packed_size == download.Size()) {
-				flush();
-			}
-		}
+	const auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
+	if (mapped == nullptr) {
+		EXIT("BufferCache: download exceeds 32 MiB staging buffer capacity\n");
 	}
-	if (!batch.empty()) {
-		flush();
-	}
-	for (const auto& copy: copies) {
-		m_gpu_modified_ranges.Subtract(copy.address, copy.size);
+	m_download_buffer.Commit();
+	for (auto& copy: copies) {
+		copy.dstOffset += offset;
 	}
+
+	auto& command = m_scheduler.Current();
+	command.EndRendering();
+	const auto              native = command.Handle();
+	vk::BufferMemoryBarrier before {};
+	before.srcAccessMask       = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite;
+	before.dstAccessMask       = vk::AccessFlagBits::eTransferRead;
+	before.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+	before.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+	before.buffer              = buffer.Handle();
+	before.offset              = 0;
+	before.size                = buffer.Size();
+	native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
+	                       vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &before, 0,
+	                       nullptr);
+	native.copyBuffer(buffer.Handle(), m_download_buffer.Handle(),
+	                  static_cast<uint32_t>(copies.size()), copies.data());
+
+	auto after          = before;
+	after.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+	after.dstAccessMask = vk::AccessFlagBits::eHostRead;
+	after.buffer        = m_download_buffer.Handle();
+	after.offset        = offset;
+	after.size          = total_size;
+	native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+	                       vk::PipelineStageFlagBits::eAllCommands |
+	                           vk::PipelineStageFlagBits::eHost,
+	                       {}, 0, nullptr, 1, &after, 0, nullptr);
+	m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address,
+	                                    copies = std::move(copies)] {
+		m_download_buffer.Invalidate(offset, total_size);
+		for (const auto& copy: copies) {
+			Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset,
+			                                      mapped + (copy.dstOffset - offset), copy.size);
+		}
+	});
+	return true;
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -252,44 +236,29 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 		     vaddr, size);
 	}
-	m_scheduler.Context().GetGpu().SendCommandSync(
-	    [this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); });
-}
+	m_scheduler.Context().GetGpu().SendCommandSync([this, vaddr, size, is_write] {
+		if (is_write && !IsRegionRegistered(vaddr, size)) {
+			return;
+		}
+		auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
 
-void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) {
-	if (is_write && !IsRegionRegistered(vaddr, size)) {
-		return;
-	}
-	auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
+		// Widen nearby CPU reads so they share one GPU drain.
+		constexpr uint64_t WindowSize   = 512 * 1024;
+		const auto         buffer_begin = buffer.CpuAddress();
+		const auto         buffer_end   = buffer_begin + buffer.Size();
+		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
+		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
 
-	// Widen nearby CPU reads so they share one GPU drain.
-	constexpr uint64_t WindowSize   = 512 * 1024;
-	const auto         buffer_begin = buffer.CpuAddress();
-	const auto         buffer_end   = buffer_begin + buffer.Size();
-	const auto         window_begin = std::max(vaddr & ~(WindowSize - 1), buffer_begin);
-	const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
-
-	std::vector<DownloadCopy> copies;
-	m_memory_tracker.ForEachDownloadRange<false>(
-	    window_begin, window_end - window_begin,
-	    [&](uint64_t address, uint64_t bytes) noexcept {
-		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
-		                                           "memory invalidation");
-	    },
-	    [&](uint64_t address, uint64_t bytes) noexcept {
-		    for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
-			    copies.push_back(
-			        {&buffer, buffer.Offset(range.address), range.address, range.size});
-		    }
-	    });
-	if (!copies.empty()) {
-		DownloadBufferMemory(copies);
-		// The enumeration covered whole dirty pages and every exact interval on them.
-		m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
-	}
-	if (is_write) {
-		m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
-	}
+		if (DownloadBufferMemory(buffer, window_begin, window_end - window_begin)) {
+			const auto tick = m_scheduler.CurrentTick();
+			m_scheduler.Wait(tick);
+			m_scheduler.WaitPriorityOperations(tick);
+			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
+		}
+		if (is_write) {
+			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
+		}
+	});
 }
 
 BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
@@ -340,10 +309,12 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t
 		end                       = std::max(end, buffer_end);
 		if (!has_stream_leap && (stream_score += buffer.StreamScore()) > StreamLeapThreshold) {
 			has_stream_leap = true;
-			if (expands_right) {
+			// Reserve space in the incoming stream's direction of growth.
+			// The old buffer extending left of the request predicts growth to the right, and vice versa.
+			if (expands_left) {
 				end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
 			}
-			if (expands_left) {
+			if (expands_right) {
 				const auto minimum = CACHING_PAGESIZE * 2;
 				if (begin > minimum) {
 					begin -= std::min(StreamLeapSize, begin - minimum);
@@ -369,8 +340,8 @@ void BufferCache::JoinOverlap(BufferId new_id, BufferId overlap_id, bool accumul
 
 BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
 	EXIT_IF(m_scheduler.Current().IsInvalid());
-	const auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
-	vaddr &= ~(CACHING_PAGESIZE - 1);
+	const auto end = Common::AlignUp(vaddr + size, CACHING_PAGESIZE);
+	vaddr = Common::AlignDown(vaddr, CACHING_PAGESIZE);
 	size               = end - vaddr;
 	const auto overlap = ResolveOverlaps(vaddr, size);
 
@@ -483,17 +454,16 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
 		}
 	}
 
-	auto* buffer = m_slot_buffers.try_get(id);
-	if (buffer == nullptr || buffer->is_deleted || !buffer->IsInBounds(vaddr, size)) {
-		id     = FindBuffer(vaddr, size);
-		buffer = &m_slot_buffers[id];
+	if (IsBufferInvalid(id) || !m_slot_buffers[id].IsInBounds(vaddr, size)) {
+		id = FindBuffer(vaddr, size);
 	}
-	TouchBuffer(*buffer);
-	(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
+	auto& buffer = m_slot_buffers[id];
+	TouchBuffer(buffer);
+	(void)SynchronizeBuffer(buffer, vaddr, size, is_written, is_texel_buffer);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
-	return {buffer, buffer->Offset(vaddr)};
+	return {&buffer, buffer.Offset(vaddr)};
 }
 
 std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
@@ -545,9 +515,7 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	}
 
 	m_texture_cache.InvalidateMemoryFromGPU(vaddr, size);
-	const auto id          = FindBuffer(vaddr, size);
-	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true, id);
-	EXIT_IF(dst == nullptr);
+	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true);
 	dst->Fill(dst_offset, size, value);
 }
 
@@ -581,10 +549,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	                                    : std::pair {&m_gds_buffer, src_vaddr};
 	auto [dst, dst_offset] = dst_memory ? ObtainBuffer(dst_vaddr, size, true, true, dst_id)
 	                                    : std::pair {&m_gds_buffer, dst_vaddr};
-	EXIT_IF(src == nullptr || dst == nullptr);
-	if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) {
-		EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
-	}
 	dst->CopyFrom(command, *src, src_offset, dst_offset, size);
 }
 
@@ -628,8 +592,7 @@ void BufferCache::RunGarbageCollector() {
 	const size_t   limit      = aggressive ? 64 : 32;
 
 	std::vector<BufferId> dirty_buffers;
-	std::vector<DownloadCopy> copies;
-	size_t                    retire_count = 0;
+	size_t                retire_count = 0;
 	m_lru_cache.ForEachItemBelow(tick - age, [&](BufferId id) {
 		auto& buffer = m_slot_buffers[id];
 		EXIT_IF(buffer.is_deleted);
@@ -640,19 +603,7 @@ void BufferCache::RunGarbageCollector() {
 			return false;
 		}
 		if (dirty) {
-			m_memory_tracker.ForEachDownloadRange<false>(
-			    buffer.CpuAddress(), buffer.Size(),
-			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address,
-				                                           dirty_size, "garbage collection");
-			    },
-			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    m_gpu_modified_ranges.ForEachIntersection(
-				        dirty_address, dirty_size, [&](RangeSet::Range range) {
-					    copies.push_back({&buffer, range.address - buffer.CpuAddress(),
-					                      range.address, range.size});
-				        });
-				});
+			EXIT_IF(!DownloadBufferMemory(buffer, buffer.CpuAddress(), buffer.Size()));
 			dirty_buffers.push_back(id);
 		} else {
 			m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());
@@ -664,8 +615,10 @@ void BufferCache::RunGarbageCollector() {
 		return;
 	}
 
-	EXIT_IF(copies.empty());
-	DownloadBufferMemory(copies);
+	// Publish all queued downloads before releasing their tracked pages and owners.
+	const auto completion_tick = m_scheduler.CurrentTick();
+	m_scheduler.Wait(completion_tick);
+	m_scheduler.WaitPriorityOperations(completion_tick);
 	for (const auto id: dirty_buffers) {
 		auto& buffer = m_slot_buffers[id];
 		m_memory_tracker.UnmarkRegionAsGpuModified(buffer.CpuAddress(), buffer.Size());

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -74,6 +74,11 @@ public:
 private:
 	friend struct BufferCacheTestAccess;
 
+	bool IsBufferInvalid(BufferId id) const {
+		const auto* buffer = m_slot_buffers.try_get(id);
+		return buffer == nullptr || buffer->is_deleted;
+	}
+
 	using BufferMap = std::map<uint64_t, BufferId>;
 	struct OverlapResult {
 		BufferMap::iterator first;
@@ -83,14 +88,8 @@ private:
 		bool                has_stream_leap;
 	};
 
-	struct DownloadCopy;
 	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
 	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
-	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
-		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
-	}
-	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
 	void TouchBuffer(const Buffer& buffer);
 	[[nodiscard]] OverlapResult ResolveOverlaps(uint64_t vaddr, uint64_t size);
@@ -106,8 +105,8 @@ private:
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
-	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
+	// Queues backing publication; callers wait before clearing dirty pages or reusing their data.
+	[[nodiscard]] bool DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/cache/faultManager.cpp

@@ -44,12 +44,7 @@ FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler
 	                                                &m_fault_process_desc_layout),
 	    "create fault-buffer descriptor layout");
 
-	vk::ShaderModuleCreateInfo module_info {};
-	module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
-	module_info.pCode    = FAULT_BUFFER_PROCESS_SPV;
-	vk::ShaderModule module = nullptr;
-	RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-	                     "create fault-buffer shader module");
+	const auto module = CompileSPV(FAULT_BUFFER_PROCESS_SPV, m_graphics.device);
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.setLayoutCount = 1;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -1,96 +0,0 @@
-#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
-
-#include "common/assert.h"
-#include "graphics/guest_gpu/graphicsRun.h"
-#include "graphics/host_gpu/renderer/commandScheduler.h"
-namespace Libs::Graphics {
-
-GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
-    : m_scheduler(scheduler), m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache),
-      m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache) {}
-
-GpuResourceManager::~GpuResourceManager() = default;
-
-bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	// The host reports the faulting byte, not the instruction's access width. Both caches
-	// resolve its page; guessing a width can cross the end of a valid guest mapping.
-	constexpr uint64_t fault_size = 1;
-	if (!IsMapped(fault_vaddr, fault_size)) {
-		return false;
-	}
-	if (access == PageFaultAccess::Write) {
-		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
-		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
-	} else {
-		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
-	}
-	return true;
-}
-
-bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
-	if (!IsMapped(vaddr, size)) {
-		return false;
-	}
-	m_buffer_cache.InvalidateMemory(vaddr, size);
-	m_texture_cache.InvalidateMemory(vaddr, size);
-	return true;
-}
-
-bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (!GuestRange {vaddr, size}.Valid()) {
-		return false;
-	}
-	std::shared_lock lock(m_mapped_ranges_mutex);
-	return m_mapped_ranges.Contains(vaddr, size);
-}
-
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
-	{
-		std::lock_guard lock(m_mapped_ranges_mutex);
-		m_mapped_ranges.Add(vaddr, size);
-	}
-}
-
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	if (CommandScheduler::InDeferredOperation()) {
-		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
-		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	const auto unmap = [this, vaddr, size] {
-		if (m_scheduler.Active()) {
-			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.Finish();
-			m_scheduler.WaitPriorityOperations(tick);
-		}
-		m_buffer_cache.InvalidateMemory(vaddr, size);
-		m_texture_cache.UnmapMemory(vaddr, size);
-		std::lock_guard lock(m_mapped_ranges_mutex);
-		m_mapped_ranges.Subtract(vaddr, size);
-	};
-	if (m_gpu == nullptr) {
-		unmap();
-		return;
-	}
-	m_gpu->SendCommandSync(unmap);
-}
-
-void GpuResourceManager::PrepareBda() {
-	std::shared_lock lock(m_mapped_ranges_mutex);
-	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
-		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
-	});
-	m_fault_process_pending = true;
-}
-
-void GpuResourceManager::RunGarbageCollector() {
-	if (m_fault_process_pending) {
-		m_fault_process_pending = false;
-		m_buffer_cache.ProcessFaultBuffer();
-	}
-	m_texture_cache.ProcessDownloadImages();
-	m_texture_cache.RunGarbageCollector();
-	m_buffer_cache.RunGarbageCollector();
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -1,49 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
-
-#include "common/abi.h"
-#include "common/common.h"
-#include "graphics/host_gpu/pageManager.h"
-#include "graphics/host_gpu/renderer/cache/bufferCache.h"
-#include "graphics/host_gpu/renderer/cache/textureCache.h"
-
-#include <cstdint>
-#include <shared_mutex>
-
-namespace Libs::Graphics {
-
-class CommandScheduler;
-class GuestGpu;
-
-class GpuResourceManager {
-public:
-	GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler);
-	~GpuResourceManager();
-	KYTY_CLASS_NO_COPY(GpuResourceManager);
-
-	[[nodiscard]] BufferCache&  GetBufferCache() { return m_buffer_cache; }
-	[[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; }
-	void                        SetGpu(GuestGpu* gpu) noexcept { m_gpu = gpu; }
-
-	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
-	void               PrepareBda();
-	void               RunGarbageCollector();
-
-private:
-	PageManager               m_page_manager;
-	CommandScheduler&         m_scheduler;
-	BufferCache               m_buffer_cache;
-	TextureCache              m_texture_cache;
-	mutable std::shared_mutex m_mapped_ranges_mutex;
-	RangeSet                  m_mapped_ranges;
-	GuestGpu*                 m_gpu = nullptr;
-	bool                      m_fault_process_pending = false;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -1,12 +1,12 @@
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/profiler.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 
 #include <cstring>
-#include <limits>
 #include <numeric>
 #include <vk_mem_alloc.h>
 
@@ -45,16 +45,11 @@ constexpr size_t WATCHES_RESERVE_CHUNK   = 0x1000;
 		result = value;
 		return true;
 	}
-	const auto remainder = value % alignment;
-	if (remainder == 0) {
-		result = value;
-		return true;
-	}
-	const auto increment = alignment - remainder;
-	if (value > std::numeric_limits<uint64_t>::max() - increment) {
+	const auto aligned = Common::AlignUp(value, alignment);
+	if (aligned < value) {
 		return false;
 	}
-	result = value + increment;
+	result = aligned;
 	return true;
 }
 
@@ -124,12 +119,6 @@ bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
 	return address >= m_cpu_address && size <= Size() && address - m_cpu_address <= Size() - size;
 }
 
-void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
-	EXIT_IF(source == nullptr || m_mapped.empty() || offset > Size() || size > Size() - offset);
-	std::memcpy(m_mapped.data() + offset, source, static_cast<size_t>(size));
-	Flush(offset, size);
-}
-
 void Buffer::Flush(uint64_t offset, uint64_t size) {
 	EXIT_IF(m_mapped.empty() || offset > Size() || size > Size() - offset);
 	if (!IsCoherent() && size != 0) {

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -55,7 +55,6 @@ public:
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
 	void               IncreaseStreamScore(int score) noexcept { stream_score += score; }
 	[[nodiscard]] int  StreamScore() const noexcept { return stream_score; }
-	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
 	void               Invalidate(uint64_t offset, uint64_t size);
 	void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
@@ -33,10 +34,14 @@ namespace {
 constexpr uint64_t NumFramesBeforeRemoval = 32;
 
 [[nodiscard]] bool DecodeDccClear(const TextureCache::ImageDesc& desc, vk::Format format,
-                                  uint32_t fill, vk::ClearColorValue& clear) {
-	const auto code = static_cast<uint8_t>(fill);
-	if (fill != static_cast<uint32_t>(code) * 0x01010101u) {
-		return false;
+                                  uint8_t code, vk::ClearColorValue& clear) {
+	switch (code) {
+		case 0x00:
+		case 0x20:
+		case 0x40:
+		case 0x80:
+		case 0xc0: break;
+		default: return false;
 	}
 	const auto& metadata = desc.info.metadata;
 	if (code == 0x20) {
@@ -45,9 +50,6 @@ constexpr uint64_t NumFramesBeforeRemoval = 32;
 		       metadata.dcc_clear_register_valid &&
 		       DecodePackedColorClear(format, metadata.dcc_clear_word, clear);
 	}
-	if (code != 0x00 && code != 0x40 && code != 0x80 && code != 0xc0) {
-		return false;
-	}
 	clear = {};
 	if (code == 0x00) {
 		return true;
@@ -187,15 +189,11 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
 }
 
 TextureCache::~TextureCache() {
-	std::vector<ImageId> registered;
 	m_slot_images.ForEach([&](ImageId id, const Image& image) {
 		if (image.registered) {
-			registered.push_back(id);
+			UnregisterImage(id);
 		}
 	});
-	for (const auto id: registered) {
-		UnregisterImage(id);
-	}
 }
 
 bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested,
@@ -270,9 +268,9 @@ void TextureCache::RegisterImage(ImageId id) {
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
 		EXIT("TextureCache: image registration is outside the guest address space\n");
 	}
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(image.info.data.address, image.info.data.size, [this, id](uint64_t page) {
 		m_image_page_table[page].push_back(id);
-	}
+	});
 	image.registered = true;
 	image.lru_id     = m_lru_cache.Insert(id, m_gc_tick);
 	m_total_used_memory += image.AccountedSize();
@@ -288,12 +286,12 @@ void TextureCache::UnregisterImage(ImageId id) {
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
 		EXIT("TextureCache: registered image is outside the guest address space\n");
 	}
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(image.info.data.address, image.info.data.size, [this, id](uint64_t page) {
 		auto* owners = m_image_page_table.Find(page);
 		if (owners == nullptr || !owners->Erase(id)) {
 			EXIT("TextureCache: image missing from page owner index\n");
 		}
-	}
+	});
 	m_lru_cache.Free(image.lru_id);
 	const auto accounted = image.AccountedSize();
 	if (accounted > m_total_used_memory) {
@@ -316,11 +314,7 @@ void TextureCache::DeleteImage(ImageId id) {
 			}
 		});
 		for (const auto association: associations) {
-			auto& associated = m_slot_images[association];
-			if (associated.IsGpuModified()) {
-				associated.ClearGpuModified();
-			}
-			DeleteImage(association);
+			FreeImage(association);
 		}
 	}
 	if (image->IsGpuModified()) {
@@ -330,10 +324,8 @@ void TextureCache::DeleteImage(ImageId id) {
 	if (image->info.HasMetadata()) {
 		const auto metadata = m_surface_metas.find(image->info.metadata.range.address);
 		if (metadata != m_surface_metas.end() &&
-		    ((image->info.metadata.kind == ImageMetadataKind::Dcc &&
-		      metadata->second.type == MetaDataInfo::Type::Dcc) ||
-		     (image->info.metadata.kind == ImageMetadataKind::Htile &&
-		      metadata->second.type == MetaDataInfo::Type::HTile))) {
+		    image->info.metadata.kind == ImageMetadataKind::Htile &&
+		    metadata->second.type == MetaDataInfo::Type::HTile) {
 			// A later binding may have reused this address for another metadata type.
 			m_surface_metas.erase(metadata);
 		}
@@ -447,7 +439,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
 	if (!image.IsTracked() || begin < image.track_addr) {
 		return;
 	}
-	const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE);
 	const auto size    = address - begin;
 	image.track_addr   = address;
 	if (image.track_addr == image.track_addr_end) {
@@ -464,7 +456,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
 	if (!image.IsTracked() || image.track_addr_end < end) {
 		return;
 	}
-	const auto address   = end & ~(TRACKER_PAGE_SIZE - 1);
+	const auto address   = Common::AlignDown(end, TRACKER_PAGE_SIZE);
 	const auto size      = end - address;
 	image.track_addr_end = address;
 	if (image.track_addr == image.track_addr_end) {
@@ -498,10 +490,10 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 	}
 
 	ImageIds result;
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(address, size, [&](uint64_t page) {
 		const auto* owners = m_image_page_table.Find(page);
 		if (owners == nullptr) {
-			continue;
+			return;
 		}
 		owners->ForEach([&](ImageId id) {
 			auto* image = m_slot_images.try_get(id);
@@ -516,7 +508,7 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 				result.push_back(id);
 			}
 		});
-	}
+	});
 	return result;
 }
 
@@ -817,17 +809,10 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 	const bool safe_to_delete =
 	    current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval;
 
-	if (requested.data.address == cached.info.data.address) {
-		const uint32_t requested_block = requested.bytes_per_block * requested.samples;
-		const uint32_t cached_block    = cached.info.bytes_per_block * cached.info.samples;
-		if (requested.BlockExtent() != cached.info.BlockExtent() ||
-		    requested_block != cached_block) {
-			if (safe_to_delete) {
-				FreeImage(cached_id);
-			}
-			return {merged_id};
-		}
-
+	const uint32_t requested_block = requested.bytes_per_block * requested.samples;
+	const uint32_t cached_block    = cached.info.bytes_per_block * cached.info.samples;
+	if (requested.data.address == cached.info.data.address &&
+	    requested.BlockExtent() == cached.info.BlockExtent() && requested_block == cached_block) {
 		if (const auto depth_id = ResolveDepthOverlap(requested, binding, cached_id)) {
 			return {depth_id};
 		}
@@ -880,38 +865,27 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		     static_cast<uint32_t>(cached.info.tile_mode));
 	}
 
-	if (requested.data.address > cached.info.data.address) {
-		const int32_t mip = requested.MipOf(cached.info);
-		if (mip >= 0) {
-			const int32_t layer = requested.SliceOf(cached.info, mip);
-			if (layer >= 0) {
-				return {cached_id, mip, layer};
-			}
-		}
-		if (safe_to_delete) {
-			FreeImage(cached_id);
-		}
-		return {};
+	const int32_t requested_mip = requested.MipOf(cached.info);
+	if (requested_mip >= 0) {
+		const int32_t layer = requested.SliceOf(cached.info, requested_mip);
+		return {cached_id, requested_mip, layer};
 	}
 
 	const int32_t mip = cached.info.MipOf(requested);
 	if (mip >= 0) {
 		const int32_t layer = cached.info.SliceOf(requested, mip);
-		if (layer >= 0) {
-			if (cached.binding.is_target) {
-				cached.binding.needs_rebind = true;
-				if (merged_id) {
-					m_slot_images[merged_id].binding.is_target = true;
-				}
-				FreeImage(cached_id);
-				return {merged_id};
-			}
-			if (merged_id) {
-				CopyImageMip(merged_id, cached_id, static_cast<uint32_t>(mip),
-				             static_cast<uint32_t>(layer));
-				FreeImage(cached_id);
-			}
+		if (!merged_id) {
+			return {ExpandImage(requested, cached_id)};
 		}
+		cached.binding.needs_rebind |= cached.binding.is_bound || cached.binding.is_target;
+		m_slot_images[merged_id].binding.is_target |= cached.binding.is_target;
+		CopyImageMip(merged_id, cached_id, static_cast<uint32_t>(mip),
+		             static_cast<uint32_t>(layer));
+		FreeImage(cached_id);
+		return {merged_id};
+	}
+	if (requested.data.address >= cached.info.data.address && safe_to_delete) {
+		FreeImage(cached_id);
 	}
 	return {merged_id};
 }
@@ -926,12 +900,19 @@ ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) {
 		source.binding.needs_rebind = true;
 	}
 	InitializeImage(expanded_id);
-	CopyImage(expanded_id, source_id);
+	const int32_t mip = source.info.MipOf(info);
+	const int32_t layer = source.info.SliceOf(info, mip);
+	if (layer >= 0) {
+		CopyImageMip(expanded_id, source_id, static_cast<uint32_t>(mip),
+		             static_cast<uint32_t>(layer));
+	} else {
+		CopyImage(expanded_id, source_id);
+	}
 	FreeImage(source_id);
 	return expanded_id;
 }
 
-struct TextureCache::TextureTransferPlan {
+struct TextureCache::TextureTransfer {
 	TextureUploadLayout              layout;
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
@@ -947,13 +928,13 @@ struct TextureCache::TextureTransferPlan {
 	}
 };
 
-struct TextureCache::DownloadPlan {
-	TextureTransferPlan texture;
+struct TextureCache::ImageDownload {
+	TextureTransfer texture;
 	bool                depth_target = false;
 	bool                valid        = false;
 };
 
-TextureCache::TextureTransferPlan
+TextureCache::TextureTransfer
 TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
                                     TransferDirection direction) const {
 	const auto& info             = image.info;
@@ -966,8 +947,8 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 	bool        allow_depth_tile = upload;
 	const char* owner            = "TextureCache readback";
 
-	TextureTransferPlan plan;
-	plan.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
+	TextureTransfer transfer;
+	transfer.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
 	if (render_target) {
 		format = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
 	}
@@ -997,44 +978,44 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 		}
 	}
 
-	plan.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
+	transfer.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
 	                                       info.resources.levels, layers, info.tile_mode,
 	                                       info.data.size, allow_depth_tile, volume, owner);
-	plan.regions = TextureBuildImageCopies(plan.layout);
+	transfer.regions = TextureBuildImageCopies(transfer.layout);
 	if (info.IsDepth()) {
-		for (auto& region: plan.regions) {
+		for (auto& region: transfer.regions) {
 			region.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth;
 		}
 	}
-	if (plan.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
-		if (!TextureBuildGpuTileInfos(info.data.size, plan.regions, plan.layout,
-		                              info.resources.levels, plan.tiles)) {
-			return plan;
+	if (transfer.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
+		if (!TextureBuildGpuTileInfos(info.data.size, transfer.regions, transfer.layout,
+		                              info.resources.levels, transfer.tiles)) {
+			return transfer;
 		}
 	}
-	plan.valid = true;
-	return plan;
+	transfer.valid = true;
+	return transfer;
 }
 
-TextureCache::DownloadPlan TextureCache::BuildDownload(const Image& image) const {
+TextureCache::ImageDownload TextureCache::BuildDownload(const Image& image) const {
 	const auto&  info    = image.info;
 	const auto   binding = UploadBinding(image);
-	DownloadPlan plan {.depth_target = binding == BindingType::DepthTarget};
+	ImageDownload transfer {.depth_target = binding == BindingType::DepthTarget};
 	if (info.samples != 1 || image.backing.samples != 1) {
-		return plan;
+		return transfer;
 	}
-	if (plan.depth_target) {
-		plan.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 &&
+	if (transfer.depth_target) {
+		transfer.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 &&
 		             info.data.size % info.resources.layers == 0 &&
 		             Prospero::NumBytesPerElement(info.guest_format) == info.bytes_per_block;
-		return plan;
+		return transfer;
 	}
 	if (info.metadata.compression != VideoOutCompression::Uncompressed) {
-		return plan;
+		return transfer;
 	}
-	plan.texture = BuildTextureTransfer(image, binding, TransferDirection::Download);
-	plan.valid   = plan.texture.valid;
-	return plan;
+	transfer.texture = BuildTextureTransfer(image, binding, TransferDirection::Download);
+	transfer.valid   = transfer.texture.valid;
+	return transfer;
 }
 
 void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_offset) {
@@ -1048,26 +1029,26 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	};
 
 	if (binding != BindingType::DepthTarget) {
-		auto plan = BuildTextureTransfer(image, binding, TransferDirection::Upload);
-		if (!plan.valid) {
+		auto transfer = BuildTextureTransfer(image, binding, TransferDirection::Upload);
+		if (!transfer.valid) {
 			EXIT("TextureCache: invalid texture upload: binding=%u addr=0x%016" PRIx64
 			     " size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u "
 			     "pitch=%u levels=%u layers=%u samples=%u\n",
 			     static_cast<uint32_t>(binding), info.data.address, info.data.size,
 			     static_cast<uint32_t>(info.guest_format), static_cast<uint32_t>(info.tile_mode),
-			     static_cast<uint32_t>(plan.layout.surface.texture.block.family), info.extent.width,
+			     static_cast<uint32_t>(transfer.layout.surface.texture.block.family), info.extent.width,
 			     info.extent.height, info.extent.depth, info.pitch, info.resources.levels,
 			     info.resources.layers, info.samples);
 		}
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
-		if (!plan.tiles.empty()) {
+		if (!transfer.tiles.empty()) {
 			linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size,
-			                        plan.LinearSize(), plan.tiles);
+			                        transfer.LinearSize(), transfer.tiles);
 		}
-		if (plan.swap_bgra16) {
+		if (transfer.swap_bgra16) {
 			linear = m_tiler.SwapBgra16(linear);
 		}
-		upload(plan.regions, linear);
+		upload(transfer.regions, linear);
 		return;
 	}
 
@@ -1142,53 +1123,71 @@ void TextureCache::InitializeImage(ImageId id) {
 	}
 }
 
-void TextureCache::PrepareDccClear(ImageId id, const ImageDesc& desc) {
+void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
+                                       uint32_t metadata_base_layer) {
 	if (desc.info.metadata.kind != ImageMetadataKind::Dcc) {
 		return;
 	}
-	auto& image            = m_slot_images[id];
-	image.info.metadata    = desc.info.metadata;
-	auto [entry, inserted] = m_surface_metas.try_emplace(
-	    desc.info.metadata.range.address, MetaDataInfo {.type = MetaDataInfo::Type::Dcc});
-	auto& metadata = entry->second;
-	if (!inserted && metadata.type == MetaDataInfo::Type::PendingDcc) {
-		metadata.type = MetaDataInfo::Type::Dcc;
-	} else if (metadata.type != MetaDataInfo::Type::Dcc) {
-		EXIT("TextureCache: image reuses non-DCC metadata\n");
+	const auto range = desc.info.metadata.range;
+	vk::Format format;
+	{
+		std::scoped_lock lock {m_lock};
+		auto& image         = m_slot_images[id];
+		image.info.metadata = desc.info.metadata;
+		// A native DCC allocation must not retain a reused HTile/CMask/FMask interpretation.
+		m_surface_metas.erase(range.address);
+		if (range.size == 0 || desc.info.resources.levels != 1 || image.info.resources.levels != 1) {
+			return;
+		}
+		format = image.backing.format;
 	}
-	if (metadata.clear_mask == 0 || image.info.resources.levels != 1 ||
-	    desc.info.metadata.range.size == 0 || metadata.fill_size < desc.info.metadata.range.size) {
-		return;
-	}
-	vk::ClearValue clear {};
-	if (!DecodeDccClear(desc, image.backing.format, metadata.fill_value, clear.color)) {
-		return;
+	const auto layers = desc.info.TransferLayers();
+	// These one-mip surfaces use complete 4 KiB DCC metadata blocks.
+	constexpr uint64_t MetadataBlockSize = 0x1000;
+	if (!range.Valid() || range.address % MetadataBlockSize != 0 || layers == 0 ||
+	    range.size % layers != 0 || (range.size / layers) % MetadataBlockSize != 0) {
+		EXIT("TextureCache: DCC slices must contain aligned 4 KiB blocks\n");
 	}
 	const auto& view           = desc.view_info;
-	const bool  volume_texture = image.info.IsVolume() && view.type == vk::ImageViewType::e3D;
-	const auto  first          = volume_texture ? 0u : view.base_layer;
-	const auto  count = volume_texture ? std::max(image.info.extent.depth >> view.base_level, 1u)
-	                                   : view.layer_count;
-	if (first >= 32 || count > 32 - first) {
-		return;
+	const bool  volume_texture = desc.info.IsVolume() && view.type == vk::ImageViewType::e3D;
+	const auto  first          = volume_texture ? 0u : metadata_base_layer;
+	const auto  image_first    = volume_texture ? 0u : view.base_layer;
+	const auto  count          = volume_texture ? desc.info.extent.depth : view.layer_count;
+	if (first >= layers || count > layers - first) {
+		EXIT("TextureCache: DCC view exceeds its native metadata slices\n");
 	}
-	// The metadata fill covers the complete allocation. Consume each layer only after its
-	// native image contents exist; already materialized layers may have been rendered since.
-	for (uint32_t layer = first; layer < first + count;) {
-		if ((metadata.clear_mask & (1u << layer)) == 0) {
-			layer++;
+	// Finish native metadata writes before reading backing bytes. This can submit the scheduler,
+	// so discovery runs before final draw uploads and never holds the texture lock across it.
+	if (m_buffer_cache.IsRegionGpuModified(range.address, range.size)) {
+		m_buffer_cache.ReadMemory(range.address, range.size, false);
+	}
+	const auto slice_size = range.size / layers;
+	for (uint32_t slice = 0; slice < count; slice++) {
+		const auto address = range.address + slice_size * (first + slice);
+		uint8_t code = 0;
+		if (!LibKernel::Memory::TryReadBacking(address, &code, sizeof(code))) {
+			EXIT("TextureCache: failed to read DCC metadata backing\n");
+		}
+		vk::ClearValue clear {};
+		if (!DecodeDccClear(desc, format, code, clear.color)) {
 			continue;
 		}
-		const auto start = layer;
-		uint32_t   mask  = 0;
-		do {
-			mask |= 1u << layer++;
-		} while (layer < first + count && (metadata.clear_mask & (1u << layer)) != 0);
-		ClearImage(m_scheduler.Current(), id,
-		           {vk::ImageAspectFlagBits::eColor, view.base_level, view.level_count, start,
-		            layer - start},
-		           clear);
-		metadata.clear_mask &= ~mask;
+		std::vector<uint8_t> bytes(slice_size);
+		if (!LibKernel::Memory::TryReadBacking(address, bytes.data(), bytes.size())) {
+			EXIT("TextureCache: failed to read DCC metadata slice\n");
+		}
+		if (!std::all_of(bytes.begin(), bytes.end(), [code](uint8_t byte) { return byte == code; })) {
+			continue;
+		}
+		{
+			std::scoped_lock lock {m_lock};
+			ClearImage(m_scheduler.Current(), id,
+			           {vk::ImageAspectFlagBits::eColor, view.base_level, view.level_count,
+			            image_first + slice, 1}, clear);
+		}
+		// Native expanded keys own consumption. Existing buffer tracking publishes this CPU
+		// write to future GPU readers; FillBuffer can fault and must run outside the texture lock.
+		m_buffer_cache.FillBuffer(address, slice_size, UINT32_MAX, false);
 	}
 }
 
@@ -1253,6 +1252,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		std::scoped_lock lock {m_lock};
 		return GetNullImage(desc);
 	}
+	const auto metadata_base_layer = desc.view_info.base_layer;
 
 	ImageId result {};
 	{
@@ -1320,6 +1320,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		image.tick_accessed_last = m_scheduler.CurrentTick();
 		TouchImage(image);
 	}
+	MaterializeDccClear(result, desc, metadata_base_layer);
 	return result;
 }
 
@@ -1334,8 +1335,8 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e
 	if (!GuestRange {address, size}.Valid()) {
 		return {};
 	}
-	std::scoped_lock     lock {m_lock};
-	std::vector<ImageId> matches;
+	std::scoped_lock lock {m_lock};
+	ImageIds         matches;
 	for (const auto id: FindImagesInRegion(address, size, false)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr || owner->info.data.address != address) {
@@ -1383,16 +1384,12 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
 		image.MarkGpuModified();
 	}
 	if (!image.info.data.Empty()) {
-		PrepareDccClear(id, desc);
 		RefreshImage(id);
 	}
 	switch (desc.type) {
 		case BindingType::Texture: break;
 		case BindingType::Storage:
 			if (!image.info.data.Empty()) {
-				if (!image.registered || image.depth_id) {
-					EXIT("TextureCache: cannot acquire an unavailable storage image\n");
-				}
 				CommitGpuWrite(image);
 			}
 			TrackImageDownload(id, image);
@@ -1416,7 +1413,6 @@ vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc)
 	TouchImage(image);
 	image.MarkGpuModified();
 	image.usage.render_target = true;
-	PrepareDccClear(id, desc);
 	RefreshImage(id);
 	CommitGpuWrite(image);
 	TrackImageDownload(id, image);
@@ -1440,16 +1436,9 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	RefreshImage(id);
 	if (desc.info.HasMetadata()) {
 		image.info.metadata = desc.info.metadata;
-		auto [metadata, inserted] =
-		    m_surface_metas.try_emplace(desc.info.metadata.range.address,
-		                                MetaDataInfo {.type       = MetaDataInfo::Type::HTile,
-		                                              .clear_mask = image.info.htile_clear_mask});
-		if (!inserted && metadata->second.type != MetaDataInfo::Type::HTile) {
-			// PS5 allocations can reuse DCC storage as HTile while the old color image is cached.
-			// The depth binding defines the new type; incompatible fill state cannot carry over.
-			metadata->second = {.type       = MetaDataInfo::Type::HTile,
-			                    .clear_mask = image.info.htile_clear_mask};
-		}
+		m_surface_metas.emplace(desc.info.metadata.range.address,
+		                        MetaDataInfo {.type       = MetaDataInfo::Type::HTile,
+		                                      .clear_mask = image.info.htile_clear_mask});
 	}
 	CommitGpuWrite(image);
 	if (desc.info.HasStencil()) {
@@ -1679,12 +1668,12 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	             destination_offset, info.data.size, tiles);
 }
 
-void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-                                     uint64_t destination_size, DownloadPlan plan) {
-	if (!plan.valid) {
-		EXIT("TextureCache: invalid image download plan\n");
+void TextureCache::DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset,
+                                     uint64_t destination_size, ImageDownload transfer) {
+	if (!transfer.valid) {
+		EXIT("TextureCache: invalid image download transfer\n");
 	}
-	if (plan.depth_target) {
+	if (transfer.depth_target) {
 		if (destination_size != image.info.data.size) {
 			EXIT("TextureCache: partial depth image download is unsupported\n");
 		}
@@ -1692,7 +1681,7 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t
 		return;
 	}
 
-	auto&      texture   = plan.texture;
+	auto&      texture   = transfer.texture;
 	const auto transform = texture.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16
 	                                           : TileManager::ColorTransform::None;
 	if (texture.tiles.empty()) {
@@ -1753,15 +1742,15 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 	if (copy_size == 0) {
 		return false;
 	}
-	auto plan = m_texture_cache.BuildDownload(image);
-	if (!plan.valid) {
+	auto transfer = m_texture_cache.BuildDownload(image);
+	if (!transfer.valid) {
 		return false;
 	}
-	if (plan.depth_target && copy_size != image.info.data.size) {
+	if (transfer.depth_target && copy_size != image.info.data.size) {
 		return false;
 	}
-	if (!plan.depth_target && levels < image.info.resources.levels) {
-		auto& texture = plan.texture;
+	if (!transfer.depth_target && levels < image.info.resources.levels) {
+		auto& texture = transfer.texture;
 		std::erase_if(texture.regions, [levels](const vk::BufferImageCopy& region) {
 			return region.imageSubresource.mipLevel >= levels;
 		});
@@ -1776,17 +1765,17 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 			}
 		}
 	}
-	m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan));
+	m_texture_cache.DownloadImage(image, buffer, buf_offset, copy_size, std::move(transfer));
 	return true;
 }
 
-bool TextureCache::TryDownloadImage(ImageId id) {
+bool TextureCache::DownloadImageMemory(ImageId id) {
 	auto& image = m_slot_images[id];
 	if (image.depth_id) {
 		return false;
 	}
-	auto plan = BuildDownload(image);
-	if (!plan.valid || !SafeToDownload(image)) {
+	auto transfer = BuildDownload(image);
+	if (!transfer.valid || !SafeToDownload(image)) {
 		return false;
 	}
 	const auto range    = image.info.data;
@@ -1802,7 +1791,7 @@ bool TextureCache::TryDownloadImage(ImageId id) {
 	}
 	download.Flush(offset, range.size);
 
-	DownloadImageData(image, download, offset, range.size, std::move(plan));
+	DownloadImage(image, download, offset, range.size, std::move(transfer));
 	vk::BufferMemoryBarrier barrier {};
 	barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eTransferWrite |
 	                        vk::AccessFlagBits::eShaderWrite;
@@ -1855,8 +1844,8 @@ bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
 }
 
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
-	const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
-	const auto page_end   = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
+	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {
@@ -1882,72 +1871,32 @@ void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 bool TextureCache::IsMeta(uint64_t address) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	return found != m_surface_metas.end() && found->second.type != MetaDataInfo::Type::PendingDcc;
+	return found != m_surface_metas.end();
 }
 
-bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice, uint32_t* fill_value) {
+bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    slice >= 32) {
+	if (found == m_surface_metas.end() || slice >= 32) {
 		return false;
 	}
-	if (fill_value != nullptr) {
-		*fill_value = found->second.fill_value;
-	}
 	return (found->second.clear_mask & (1u << slice)) != 0;
 }
 
 bool TextureCache::ClearMeta(uint64_t address) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    found->second.type == MetaDataInfo::Type::Dcc) {
-		// Preserve the broad metadata-clear operation for CMask/FMask/HTile. DCC requires a
-		// validated fill value, so an arbitrary compute write must not clear it.
+	if (found == m_surface_metas.end()) {
 		return false;
 	}
 	found->second.clear_mask = UINT32_MAX;
 	return true;
 }
 
-void TextureCache::TrackDccFill(uint64_t address, uint64_t size, uint32_t fill_value) {
-	if (!GuestRange {address, size}.Valid()) {
-		EXIT("TextureCache: invalid DCC fill range\n");
-	}
-	// DCC fills use a repeated byte code. Require all four bytes of the detected dword to agree,
-	// and mark only recognized deferred-clear encodings as logically clear.
-	const auto dcc_clear_mask = [fill_value] {
-		const auto code = static_cast<uint8_t>(fill_value);
-		if (fill_value != static_cast<uint32_t>(code) * 0x01010101u) {
-			return 0u;
-		}
-		switch (code) {
-			case 0x00:
-			case 0x20:
-			case 0x40:
-			case 0x80:
-			case 0xc0: return UINT32_MAX;
-			default: return 0u;
-		}
-	}();
-	std::scoped_lock lock {m_lock};
-	// The guest dispatch still writes metadata. An unknown address remains PendingDcc until an
-	// image descriptor confirms its role; never reinterpret CMask/FMask/HTile as DCC.
-	const auto found = m_surface_metas.try_emplace(address).first;
-	if (found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    found->second.type == MetaDataInfo::Type::Dcc) {
-		found->second.clear_mask = dcc_clear_mask;
-		found->second.fill_value = fill_value;
-		found->second.fill_size  = size;
-	}
-}
-
 bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    slice >= 32) {
+	if (found == m_surface_metas.end() || slice >= 32) {
 		return false;
 	}
 	if (is_clear) {
@@ -1963,10 +1912,13 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 		EXIT("TextureCache: invalid unmap range\n");
 	}
 	std::scoped_lock lock {m_lock};
-	const auto       end = address + size;
-	for (auto metadata = m_surface_metas.lower_bound(address);
-	     metadata != m_surface_metas.end() && metadata->first < end;) {
-		metadata = m_surface_metas.erase(metadata);
+	for (auto metadata = m_surface_metas.begin(); metadata != m_surface_metas.end();) {
+		const auto base = metadata->first;
+		if (base >= address && base < address + size) {
+			metadata = m_surface_metas.erase(metadata);
+		} else {
+			++metadata;
+		}
 	}
 	auto images = FindImagesInRegion(address, size, false);
 	for (const auto id: images) {
@@ -1974,10 +1926,7 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 		if (owner == nullptr) {
 			continue;
 		}
-		if (owner->IsGpuModified()) {
-			owner->ClearGpuModified();
-		}
-		DeleteImage(id);
+		FreeImage(id);
 	}
 }
 
@@ -2020,12 +1969,11 @@ void TextureCache::RunGarbageCollector() {
 				if (safe && !pressured) {
 					continue;
 				}
-				if (safe && !TryDownloadImage(id)) {
+				if (safe && !DownloadImageMemory(id)) {
 					continue;
 				}
-				owner->ClearGpuModified();
 			}
-			DeleteImage(id);
+			FreeImage(id);
 			if (m_total_used_memory < m_critical_gc_memory && aggressive) {
 				deletions >>= 2;
 				aggressive = false;
@@ -2047,7 +1995,7 @@ void TextureCache::ProcessDownloadImages() {
 	for (const auto id: m_download_images) {
 		const auto owner = m_slot_images.try_get(id);
 		if (owner != nullptr && owner->registered && owner->IsGpuModified()) {
-			(void)TryDownloadImage(id);
+			(void)DownloadImageMemory(id);
 		}
 	}
 	m_download_images.clear();

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -13,6 +13,7 @@
 #include "graphics/host_gpu/renderer/image/tiler.h"
 
 #include <map>
+#include <type_traits>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -63,11 +64,8 @@ public:
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
-	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice,
-	                                 uint32_t* fill_value = nullptr);
+	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);
 	[[nodiscard]] bool ClearMeta(uint64_t address);
-	// Record deferred DCC state while the original guest dispatch writes the metadata.
-	void               TrackDccFill(uint64_t address, uint64_t size, uint32_t fill_value);
 	[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
 
 	void UnmapMemory(uint64_t address, uint64_t size);
@@ -76,21 +74,14 @@ public:
 
 private:
 	enum class TransferDirection { Upload, Download };
-	struct TextureTransferPlan;
-	struct DownloadPlan;
+	struct TextureTransfer;
+	struct ImageDownload;
 
 	struct MetaDataInfo {
-		// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
-		// PendingDcc retains that exact fill until an image binding classifies the address,
-		// without exposing an unconfirmed buffer address to the normal metadata heuristics.
-		// Keep all surface metadata in one entry so CMask/FMask can be
-		// registered beside HTile and DCC without introducing parallel tracking paths.
-		enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc };
+		enum class Type : uint8_t { CMask, FMask, HTile };
 
-		Type     type       = Type::PendingDcc;
-		uint32_t clear_mask = 0;
-		uint32_t fill_value = 0xffffffffu;
-		uint64_t fill_size  = 0;
+		Type     type;
+		uint32_t clear_mask = UINT32_MAX;
 	};
 
 	struct OverlapResult {
@@ -102,6 +93,23 @@ private:
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
 	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
+	// Callers have validated the nonempty 40-bit range with TryGetPageRange.
+	template <typename Func>
+	static void ForEachPage(uint64_t address, size_t size, Func&& func) {
+		using FuncReturn = typename std::invoke_result<Func, uint64_t>::type;
+		static constexpr bool RETURNS_BOOL = std::is_same_v<FuncReturn, bool>;
+		const uint64_t page_end = (address + size - 1) >> ImagePageTable::kPageBits;
+		for (uint64_t page = address >> ImagePageTable::kPageBits; page <= page_end; ++page) {
+			if constexpr (RETURNS_BOOL) {
+				if (func(page)) {
+					break;
+				}
+			} else {
+				func(page);
+			}
+		}
+	}
+
 	[[nodiscard]] ImageId     InsertImage(const ImageInfo& info);
 	[[nodiscard]] ImageId     GetNullImage(const ImageDesc& desc);
 	void                      RegisterImage(ImageId id);
@@ -131,14 +139,15 @@ private:
 	                                                ImageId cached);
 	[[nodiscard]] ImageId       ExpandImage(const ImageInfo& info, ImageId source);
 	void                        RefreshImage(ImageId id);
-	void                        PrepareDccClear(ImageId id, const ImageDesc& desc);
+	void                        MaterializeDccClear(ImageId id, const ImageDesc& desc,
+	                                                uint32_t metadata_base_layer);
 	void                        InitializeImage(ImageId id);
-	[[nodiscard]] TextureTransferPlan
+	[[nodiscard]] TextureTransfer
 	BuildTextureTransfer(const Image& image, BindingType binding, TransferDirection direction) const;
-	[[nodiscard]] DownloadPlan BuildDownload(const Image& image) const;
+	[[nodiscard]] ImageDownload BuildDownload(const Image& image) const;
 	void UploadImage(Image& image, Buffer& source, uint64_t source_offset);
-	void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-	                       uint64_t destination_size, DownloadPlan plan);
+	void DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset,
+	                       uint64_t destination_size, ImageDownload transfer);
 	void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
 	void CommitGpuWrite(Image& image);
 	// Caller holds m_lock. Volume layer ranges select depth slices.
@@ -153,7 +162,7 @@ private:
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
 	void               InvalidateCpuAliases(uint64_t address, uint64_t size);
-	[[nodiscard]] bool TryDownloadImage(ImageId id);
+	[[nodiscard]] bool DownloadImageMemory(ImageId id);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -41,16 +41,13 @@ static bool DccAlphaOnMsb(const HW::ColorInfo& info) {
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer& buffer,
-                                              RenderColorInfo& r,
+void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& r,
                                               uint32_t         render_target_slice_offset,
-                                              uint32_t render_target_slot, bool ignore_target_mask,
+                                              uint32_t rt_slot, bool ignore_target_mask,
                                               bool exact_format) {
 	KYTY_PROFILER_FUNCTION();
 	const auto& hw = buffer.GetRegisters();
 
-	const auto  rt_slot = (render_target_slot == UINT32_MAX ? render_target_first_bound_slot(buffer)
-	                                                        : render_target_slot);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
 	auto        mask    = render_target_mask_slot(hw.GetRenderTargetMask(), rt_slot);
 	if (ignore_target_mask && rt.base.addr != 0 && mask == 0) {
@@ -206,17 +203,14 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer&
 		     " layer=%u/%u\n",
 		     rt.attrib3.dimension, rt.attrib3.depth, view.base_layer, view.image_layers);
 	}
-	if (tile) {
-		if (volume || texture_tile) {
-			pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
-		} else {
-			pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
-		}
-		if (pitch == 0) {
-			EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element);
-		}
+	// PPSA28068: the linear EULA target and its sampled view must share the padded pitch.
+	if (!tile || volume || texture_tile) {
+		pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
 	} else {
-		pitch = width;
+		pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
+	}
+	if (pitch == 0) {
+		EXIT("unsupported render-target pitch: width=%u bytes=%u\n", width, bytes_per_element);
 	}
 
 	TileSizeOffset    mip_sizes[16] {};

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -208,13 +208,7 @@ void CommandScheduler::Wait(uint64_t tick) {
 }
 
 void CommandScheduler::PopPendingOperations() {
-	PopPendingOperations(true);
-}
-
-void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) {
-	if (refresh_gpu_tick) {
-		m_master.Refresh();
-	}
+	m_master.Refresh();
 	for (;;) {
 		PendingOperation operation;
 		{

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -81,7 +81,6 @@ private:
 	};
 
 	void BeginNext();
-	void PopPendingOperations(bool refresh_gpu_tick);
 	void PriorityOperationsThread(std::stop_token stop);
 	void RunOperation(Common::UniqueFunction<void>&& operation);
 

src/graphics/host_gpu/renderer/context.cpp

@@ -61,11 +61,11 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0,
 }
 
 void CommandBuffer::BeginRendering(const RenderState& state) const {
-	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
-	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);
 	if (m_rendering && m_render_state == state) {
 		return;
 	}
+	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
+	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);
 	EndRendering();
 
 	std::array<vk::RenderingAttachmentInfo, RENDER_COLOR_ATTACHMENTS_MAX> colors {};

src/graphics/host_gpu/renderer/debug.cpp

@@ -4,7 +4,6 @@
 #include "common/common.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
-#include "common/stringUtils.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/renderer/render.h"
@@ -57,7 +56,7 @@ uint32_t render_target_first_bound_slot(const CommandBuffer& buffer) {
 
 bool graphics_debug_dump_enabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
-	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
+	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
 }
 
 void uc_print(const char* func, const HW::UserConfig& uc) {
@@ -89,91 +88,75 @@ void uc_check(const HW::UserConfig& uc) {
 	EXIT_NOT_IMPLEMENTED(user_en.vgpr3 != false);
 }
 
-void sh_print(const char* func, const HW::Shader& /*uc*/) {
-	LOGF("%s\n", func);
-}
+std::string rt_print(const char* func, const HW::RenderTarget& rt) {
+	std::string dst;
+	dst.reserve(4096);
 
-std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt) {
-	std::vector<std::string> dst;
-	dst.reserve(40);
+	dst += fmt::format("{}\n", func);
 
-	dst.push_back(fmt::format("{}\n", func));
+	dst += fmt::format("\t base.addr                       = 0x{:016x}\n", rt.base.addr);
+	dst += fmt::format("\t view.base_array_slice_index     = 0x{:08x}\n",
+	                  rt.view.base_array_slice_index);
+	dst += fmt::format("\t view.last_array_slice_index     = 0x{:08x}\n",
+	                  rt.view.last_array_slice_index);
+	dst += fmt::format("\t view.current_mip_level          = 0x{:08x}\n", rt.view.current_mip_level);
+	dst += fmt::format("\t info.fmask_compression_enable   = {}\n",
+	                  rt.info.fmask_compression_enable ? "true" : "false");
 
-	dst.push_back(fmt::format("\t base.addr                       = 0x{:016x}\n", rt.base.addr));
-	dst.push_back(fmt::format("\t view.base_array_slice_index     = 0x{:08x}\n",
-	                          rt.view.base_array_slice_index));
-	dst.push_back(fmt::format("\t view.last_array_slice_index     = 0x{:08x}\n",
-	                          rt.view.last_array_slice_index));
-	dst.push_back(
-	    fmt::format("\t view.current_mip_level          = 0x{:08x}\n", rt.view.current_mip_level));
-	dst.push_back(fmt::format("\t info.fmask_compression_enable   = {}\n",
-	                          rt.info.fmask_compression_enable ? "true" : "false"));
+	dst += fmt::format("\t info.fmask_data_compression_disable = {}\n",
+	                  rt.info.fmask_data_compression_disable ? "true" : "false");
+	dst += fmt::format("\t info.fmask_one_frag_mode        = {}\n",
+	                  rt.info.fmask_one_frag_mode ? "true" : "false");
 
-	// dst.push_back(fmt::format("\t info.fmask_compression_mode     = 0x{:08x}\n", //
-	// rt.info.fmask_compression_mode));
-	dst.push_back(fmt::format("\t info.fmask_data_compression_disable = {}\n",
-	                          rt.info.fmask_data_compression_disable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.fmask_one_frag_mode        = {}\n",
-	                          rt.info.fmask_one_frag_mode ? "true" : "false"));
-
-	dst.push_back(fmt::format("\t info.cmask_fast_clear_enable    = {}\n",
-	                          rt.info.cmask_fast_clear_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.dcc_compression_enable     = {}\n",
-	                          rt.info.dcc_compression_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.format                     = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.format)));
-	dst.push_back(fmt::format("\t info.channel_type               = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.channel_type)));
-	dst.push_back(fmt::format("\t info.channel_order              = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.channel_order)));
-	dst.push_back(fmt::format("\t info.blend_bypa                 = {}\n",
-	                          rt.info.blend_bypass ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.blend_clamp                = {}\n",
-	                          rt.info.blend_clamp ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.round_mode                 = {}\n",
-	                          rt.info.round_mode ? "true" : "false"));
-	dst.push_back(fmt::format("\t attrib.force_dest_alpha_to_one  = {}\n",
-	                          rt.attrib.force_dest_alpha_to_one ? "true" : "false"));
-	dst.push_back(
-	    fmt::format("\t attrib.num_samples              = 0x{:08x}\n", rt.attrib.num_samples));
-	dst.push_back(
-	    fmt::format("\t attrib.num_fragments            = 0x{:08x}\n", rt.attrib.num_fragments));
-	dst.push_back(fmt::format("\t attrib2.width                   = 0x{:08x}\n", rt.attrib2.width));
-	dst.push_back(
-	    fmt::format("\t attrib2.height                  = 0x{:08x}\n", rt.attrib2.height));
-	dst.push_back(
-	    fmt::format("\t attrib2.num_mip_levels          = 0x{:08x}\n", rt.attrib2.num_mip_levels));
-	dst.push_back(fmt::format("\t attrib3.depth                   = 0x{:08x}\n", rt.attrib3.depth));
-	dst.push_back(fmt::format("\t attrib3.tile_mode               = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.attrib3.tile_mode)));
-	dst.push_back(
-	    fmt::format("\t attrib3.dimension               = 0x{:08x}\n", rt.attrib3.dimension));
-	dst.push_back(fmt::format("\t attrib3.metadata_pipe_aligned   = {}\n",
-	                          rt.attrib3.metadata_pipe_aligned ? "true" : "false"));
-	dst.push_back(fmt::format("\t attrib3.write_vrs_rate_hint_to_cmask = {}\n",
-	                          rt.attrib3.write_vrs_rate_hint_to_cmask ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n",
-	                          rt.dcc.max_uncompressed_block_size));
-	dst.push_back(fmt::format("\t dcc.max_compressed_block_size   = 0x{:08x}\n",
-	                          rt.dcc.max_compressed_block_size));
-	dst.push_back(
-	    fmt::format("\t dcc.color_transform             = 0x{:08x}\n", rt.dcc.color_transform));
-	dst.push_back(fmt::format("\t dcc.overwrite_combiner_disable  = {}\n",
-	                          rt.dcc.overwrite_combiner_disable ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.independent_block_size      = 0x{:02x}\n",
-	                          static_cast<uint8_t>(rt.dcc.independent_block_size)));
-	dst.push_back(fmt::format("\t data_write_on_dcc_clear_to_reg  = {}\n",
-	                          rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.dcc_clear_key_enable        = {}\n",
-	                          rt.dcc.dcc_clear_key_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t cmask.addr                      = 0x{:016x}\n", rt.cmask.addr));
-	dst.push_back(fmt::format("\t fmask.addr                      = 0x{:016x}\n", rt.fmask.addr));
-	dst.push_back(
-	    fmt::format("\t clear_word0.word0               = 0x{:08x}\n", rt.clear_word0.word0));
-	dst.push_back(
-	    fmt::format("\t clear_word1.word1               = 0x{:08x}\n", rt.clear_word1.word1));
-	dst.push_back(
-	    fmt::format("\t dcc_addr.addr                   = 0x{:016x}\n", rt.dcc_addr.addr));
+	dst += fmt::format("\t info.cmask_fast_clear_enable    = {}\n",
+	                  rt.info.cmask_fast_clear_enable ? "true" : "false");
+	dst += fmt::format("\t info.dcc_compression_enable     = {}\n",
+	                  rt.info.dcc_compression_enable ? "true" : "false");
+	dst += fmt::format("\t info.format                     = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.format));
+	dst += fmt::format("\t info.channel_type               = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.channel_type));
+	dst += fmt::format("\t info.channel_order              = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.channel_order));
+	dst += fmt::format("\t info.blend_bypa                 = {}\n",
+	                  rt.info.blend_bypass ? "true" : "false");
+	dst += fmt::format("\t info.blend_clamp                = {}\n",
+	                  rt.info.blend_clamp ? "true" : "false");
+	dst += fmt::format("\t info.round_mode                 = {}\n",
+	                  rt.info.round_mode ? "true" : "false");
+	dst += fmt::format("\t attrib.force_dest_alpha_to_one  = {}\n",
+	                  rt.attrib.force_dest_alpha_to_one ? "true" : "false");
+	dst += fmt::format("\t attrib.num_samples              = 0x{:08x}\n", rt.attrib.num_samples);
+	dst += fmt::format("\t attrib.num_fragments            = 0x{:08x}\n", rt.attrib.num_fragments);
+	dst += fmt::format("\t attrib2.width                   = 0x{:08x}\n", rt.attrib2.width);
+	dst += fmt::format("\t attrib2.height                  = 0x{:08x}\n", rt.attrib2.height);
+	dst += fmt::format("\t attrib2.num_mip_levels          = 0x{:08x}\n", rt.attrib2.num_mip_levels);
+	dst += fmt::format("\t attrib3.depth                   = 0x{:08x}\n", rt.attrib3.depth);
+	dst += fmt::format("\t attrib3.tile_mode               = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.attrib3.tile_mode));
+	dst += fmt::format("\t attrib3.dimension               = 0x{:08x}\n", rt.attrib3.dimension);
+	dst += fmt::format("\t attrib3.metadata_pipe_aligned   = {}\n",
+	                  rt.attrib3.metadata_pipe_aligned ? "true" : "false");
+	dst += fmt::format("\t attrib3.write_vrs_rate_hint_to_cmask = {}\n",
+	                  rt.attrib3.write_vrs_rate_hint_to_cmask ? "true" : "false");
+	dst += fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n",
+	                  rt.dcc.max_uncompressed_block_size);
+	dst += fmt::format("\t dcc.max_compressed_block_size   = 0x{:08x}\n",
+	                  rt.dcc.max_compressed_block_size);
+	dst += fmt::format("\t dcc.color_transform             = 0x{:08x}\n", rt.dcc.color_transform);
+	dst += fmt::format("\t dcc.overwrite_combiner_disable  = {}\n",
+	                  rt.dcc.overwrite_combiner_disable ? "true" : "false");
+	dst += fmt::format("\t dcc.independent_block_size      = 0x{:02x}\n",
+	                  static_cast<uint8_t>(rt.dcc.independent_block_size));
+	dst += fmt::format("\t data_write_on_dcc_clear_to_reg  = {}\n",
+	                  rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false");
+	dst += fmt::format("\t dcc.dcc_clear_key_enable        = {}\n",
+	                  rt.dcc.dcc_clear_key_enable ? "true" : "false");
+	dst += fmt::format("\t cmask.addr                      = 0x{:016x}\n", rt.cmask.addr);
+	dst += fmt::format("\t fmask.addr                      = 0x{:016x}\n", rt.fmask.addr);
+	dst += fmt::format("\t clear_word0.word0               = 0x{:08x}\n", rt.clear_word0.word0);
+	dst += fmt::format("\t clear_word1.word1               = 0x{:08x}\n", rt.clear_word1.word1);
+	dst += fmt::format("\t dcc_addr.addr                   = 0x{:016x}\n", rt.dcc_addr.addr);
 
 	return dst;
 }
@@ -478,7 +461,6 @@ static void McCheck(const HW::ModeControl& c) {
 			logged = true;
 		}
 	}
-	EXIT_NOT_IMPLEMENTED(c.provoking_vtx_last != false);
 	EXIT_NOT_IMPLEMENTED(c.persp_corr_dis != false);
 }
 
@@ -505,16 +487,7 @@ static void BcPrint(const char* func, const HW::BlendControl& c, const HW::Blend
 	     color.red, color.green, color.blue, color.alpha, cc.mode, cc.op);
 }
 
-static void BcCheck(const HW::BlendControl& /*c*/, const HW::BlendColor& color,
-                    const HW::ColorControl& cc) {
-	// EXIT_NOT_IMPLEMENTED(c.color_srcblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.color_comb_fcn != 0);
-	// EXIT_NOT_IMPLEMENTED(c.color_destblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_srcblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_comb_fcn != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_destblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.separate_alpha_blend != false);
-	// EXIT_NOT_IMPLEMENTED(c.enable != false);
+static void BcCheck(const HW::BlendColor& color, const HW::ColorControl& cc) {
 	if (color.red != 0.0f || color.green != 0.0f || color.blue != 0.0f || color.alpha != 0.0f) {
 		static bool logged = false;
 		if (!logged) {
@@ -577,29 +550,6 @@ static void DPrint(const char* func, const HW::DepthControl& c, const HW::Stenci
 	     sm.stencil_opval_bf);
 }
 
-static void DCheck(const HW::DepthControl& c, const HW::StencilControl& s,
-                   const HW::StencilMask& /*sm*/) {
-	// EXIT_NOT_IMPLEMENTED(c.stencil_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.z_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.z_write_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.zfunc != 0);
-	// Back-face stencil state is handled separately when enabled.
-	// EXIT_NOT_IMPLEMENTED(c.stencilfunc != 0);
-	// EXIT_NOT_IMPLEMENTED(c.stencilfunc_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_fail != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_zpass != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_zfail != 0);
-	// Back-face stencil ops may legitimately differ from the front-face ops.
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_testval != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_mask != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_writemask != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_opval != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_testval_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_mask_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_writemask_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_opval_bf != 0);
-}
-
 static void EqaaPrint(const char* func, const HW::EqaaControl& c) {
 	LOGF("%s\n", func);
 
@@ -905,14 +855,10 @@ void hw_check(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
-	const auto& bc      = hw.GetBlendControl(rt_slot);
 	const auto& bclr    = hw.GetBlendColor();
 	const auto& vp      = hw.GetScreenViewport();
 	const auto& c       = hw.GetClipControl();
 	const auto& rc      = hw.GetRenderControl();
-	const auto& d       = hw.GetDepthControl();
-	const auto& s       = hw.GetStencilControl();
-	const auto& sm      = hw.GetStencilMask();
 	const auto& mc      = hw.GetModeControl();
 	const auto& eqaa    = hw.GetEqaaControl();
 	const auto& cc      = hw.GetColorControl();
@@ -938,11 +884,10 @@ void hw_check(const CommandBuffer& buffer) {
 	log_phase("rc");
 	RcCheck(rc);
 	log_phase("depth");
-	DCheck(d, s, sm);
 	log_phase("mode");
 	McCheck(mc);
 	log_phase("blend");
-	BcCheck(bc, bclr, cc);
+	BcCheck(bclr, cc);
 	log_phase("eqaa");
 	EqaaCheck(eqaa, ac);
 	log_phase("aa");
@@ -1001,7 +946,7 @@ void hw_print(const CommandBuffer& buffer) {
 		     hw.GetRenderTargetMask(), hw.GetDepthClearValue(), hw.GetStencilClearValue(),
 		     hw.GetLineWidth(), hw.GetPrimitiveResetIndex());
 
-		LOGF("%s", Common::Concat(rt_print("RenderTraget:", rt), "").c_str());
+		LOGF("%s", rt_print("RenderTraget:", rt).c_str());
 
 		ZPrint("DepthRenderTraget:", z);
 		VpPrint("ScreenViewport:", vp, smc);

src/graphics/host_gpu/renderer/debug.h

@@ -5,7 +5,6 @@
 
 #include <cstdint>
 #include <string>
-#include <vector>
 
 namespace Libs::Graphics {
 
@@ -13,7 +12,6 @@ class CommandBuffer;
 
 namespace HW {
 class Context;
-class Shader;
 class UserConfig;
 struct RenderTarget;
 struct ScanModeControl;
@@ -32,8 +30,7 @@ uint32_t                 render_target_first_bound_slot(const CommandBuffer& buf
 bool                     graphics_debug_dump_enabled();
 void                     uc_print(const char* func, const HW::UserConfig& uc);
 void                     uc_check(const HW::UserConfig& uc);
-void                     sh_print(const char* func, const HW::Shader& uc);
-std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt);
+std::string              rt_print(const char* func, const HW::RenderTarget& rt);
 bool                     RenderIsColorTileModeLinear(Prospero::TileMode tile_mode);
 void                     hw_print(const CommandBuffer& buffer);
 void                     hw_check(const CommandBuffer& buffer);

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -10,6 +10,7 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
@@ -18,6 +19,7 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <algorithm>
+#include <array>
 #include <atomic>
 #include <cmath>
 #include <cstdarg>
@@ -44,8 +46,12 @@ static vk::StencilOp ConvertStencilOp(uint8_t value, uint8_t write_mask, uint8_t
 	switch (static_cast<Prospero::StencilOp>(value)) {
 		case Prospero::StencilOp::kKeep: return vk::StencilOp::eKeep;
 		case Prospero::StencilOp::kZero: return vk::StencilOp::eZero;
-		case Prospero::StencilOp::kReplaceTest:
-		case Prospero::StencilOp::kReplaceOp: return vk::StencilOp::eReplace;
+		case Prospero::StencilOp::kReplaceTest: return vk::StencilOp::eReplace;
+		case Prospero::StencilOp::kReplaceOp:
+			if ((op_value & write_mask) == 0) {
+				return vk::StencilOp::eZero;
+			}
+			return vk::StencilOp::eReplace;
 		case Prospero::StencilOp::kAddClamp: return vk::StencilOp::eIncrementAndClamp;
 		case Prospero::StencilOp::kSubClamp: return vk::StencilOp::eDecrementAndClamp;
 		case Prospero::StencilOp::kInvert: return vk::StencilOp::eInvert;
@@ -65,9 +71,38 @@ static vk::StencilOp ConvertStencilOp(uint8_t value, uint8_t write_mask, uint8_t
 	}
 }
 
-static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
-	constexpr auto replace_op = static_cast<uint8_t>(Prospero::StencilOp::kReplaceOp);
-	return fail == replace_op || pass == replace_op || depth_fail == replace_op;
+static vk::StencilOpState ConvertStencilState(
+    uint8_t compare, const std::array<uint8_t, 3>& operations, uint8_t op_value,
+    const vk::StencilOpState& state) {
+	const auto test_value = state.reference;
+	auto reference       = test_value;
+	auto required_bits   = state.compareMask;
+	if (compare == static_cast<uint8_t>(vk::CompareOp::eAlways) ||
+	    compare == static_cast<uint8_t>(vk::CompareOp::eNever)) {
+		required_bits = 0;
+	}
+	std::array<vk::StencilOp, 3> converted {};
+	for (size_t i = 0; i < operations.size(); i++) {
+		converted[i] = ConvertStencilOp(operations[i], state.writeMask, op_value);
+		if (converted[i] != vk::StencilOp::eReplace) {
+			continue;
+		}
+		auto replacement = test_value;
+		if (static_cast<Prospero::StencilOp>(operations[i]) == Prospero::StencilOp::kReplaceOp) {
+			replacement = op_value;
+		}
+		if (((reference ^ replacement) & required_bits & state.writeMask) != 0) {
+			DepthFatal("unsupported stencil replacement: compare=%u, compare mask=0x%02" PRIx32
+			           ", write mask=0x%02" PRIx32 ", operation value=0x%02" PRIx8
+			           ", test value=0x%02" PRIx32,
+			           compare, state.compareMask, state.writeMask, op_value, test_value);
+		}
+		// Vulkan shares one reference between comparison and every replacement on this face.
+		reference = (reference & ~state.writeMask) | (replacement & state.writeMask);
+		required_bits |= state.writeMask;
+	}
+	return {converted[0], converted[1], converted[2], static_cast<vk::CompareOp>(compare),
+	        state.compareMask, state.writeMask, reference};
 }
 
 [[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(const CommandBuffer&     buffer,
@@ -96,53 +131,17 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
 	return vk::Format::eUndefined;
 }
 
-// NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer& buffer,
-                                              RenderDepthInfo& r) {
-	KYTY_PROFILER_FUNCTION();
-	(void)submit_id;
-	const auto& hw          = buffer.GetRegisters();
-	const auto& z           = hw.GetDepthRenderTarget();
-	const auto& rc          = hw.GetRenderControl();
-	const auto& dc          = hw.GetDepthControl();
-	const auto& sc          = hw.GetStencilControl();
-	const auto& sm          = hw.GetStencilMask();
-	const bool  has_stencil = z.stencil_info.format != Prospero::StencilFormat::kInvalid;
-	const bool depth_active = dc.z_enable || dc.depth_bounds_enable ||
-	                          rc.depth_clear_enable || rc.copy_depth_to_color;
-	const bool stencil_active =
-	    has_stencil && (dc.stencil_enable || rc.stencil_clear_enable || rc.copy_stencil_to_color);
-	if (!depth_active && !stencil_active) {
-		return;
-	}
+static TextureCache::ImageDesc MakeDepthTargetDesc(const CommandBuffer& buffer,
+                                                  const HW::DepthRenderTarget& z,
+                                                  bool write_buffer = false) {
+	const bool has_stencil = z.stencil_info.format != Prospero::StencilFormat::kInvalid;
+	const auto depth_address = write_buffer ? z.z_write_base_addr : z.z_read_base_addr;
+	const auto stencil_address =
+	    write_buffer ? z.stencil_write_base_addr : z.stencil_read_base_addr;
 	if (!z.z_info.HasValidTextureCompatibility() ||
 	    !z.stencil_info.HasValidTextureCompatibility()) {
 		DepthFatal("invalid PS5 depth texture-compatibility encoding");
 	}
-	const bool attachment_unbound =
-	    z.z_info.format == Prospero::DepthFormat::kInvalid &&
-	    z.stencil_info.format == Prospero::StencilFormat::kInvalid && z.z_info.num_samples == 0 &&
-	    z.z_info.texture_compatibility == Prospero::TextureCompatiblePlaneCompression::kDisable &&
-	    !z.z_info.expclear_enabled && !z.z_info.partially_resident && z.z_info.max_mip_level == 0 &&
-	    z.stencil_info.texture_compatibility == Prospero::TextureCompatibleStencil::kDisable &&
-	    !z.stencil_info.expclear_enabled && !z.stencil_info.partially_resident &&
-	    z.depth_view.slice_start == 0 && z.depth_view.slice_max == 0 &&
-	    z.depth_view.current_mip_level == 0 && !z.depth_view.depth_write_disable &&
-	    !z.depth_view.stencil_write_disable && z.z_read_base_addr == 0 && z.z_write_base_addr == 0 &&
-	    z.stencil_read_base_addr == 0 && z.stencil_write_base_addr == 0 &&
-	    z.htile_data_base_addr == 0 &&
-	    // DB_DEPTH_SIZE_XY is independent state and may remain programmed after the attachment
-	    // formats and addresses are unbound. A zero encoding is the valid 1x1 value, so its
-	    // presence alone must not manufacture a depth attachment.
-	    !z.z_info.htile_acceleration && z.shading_rate_encoding == 0 && z.size.x_max == 0 &&
-	    z.size.y_max == 0;
-	if (attachment_unbound) {
-		static std::atomic_bool logged = false;
-		if (!logged.exchange(true, std::memory_order_relaxed)) {
-			LOGF("DepthTarget: ignoring enabled depth/stencil state without a bound attachment\n");
-		}
-		return;
-	}
 	const bool has_htile = z.z_info.htile_acceleration;
 	const bool unsupported_shading_rate_encoding =
 	    z.shading_rate_encoding > 1 || (z.shading_rate_encoding != 0 && !has_htile);
@@ -160,23 +159,16 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer&
 			DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
 			           z.depth_view.slice_max);
 	}
-	if (rc.copy_depth_to_color || rc.copy_stencil_to_color || rc.copy_centroid ||
-	    rc.copy_sample != 0 || z.z_info.expclear_enabled ||
-	    z.stencil_info.expclear_enabled || z.z_info.partially_resident ||
+	if (z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
+	    z.z_info.partially_resident ||
 	    z.stencil_info.partially_resident || z.z_info.max_mip_level != 0 ||
 	    z.depth_view.current_mip_level != 0 || unsupported_shading_rate_encoding ||
-	    z.z_read_base_addr == 0 ||
-	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
-	    (z.z_read_base_addr & 0xffffu) != 0 ||
-	    dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways)) {
+	    depth_address == 0 || (depth_address & 0xffffu) != 0) {
 		DepthFatal("unsupported depth register state");
 	}
 	if (has_stencil) {
 		if (z.stencil_info.format != Prospero::StencilFormat::k8UInt || !htile_stencil_compat ||
-		    z.stencil_read_base_addr == 0 ||
-		    (!z.depth_view.stencil_write_disable &&
-		     z.stencil_write_base_addr != z.stencil_read_base_addr) ||
-		    (z.stencil_read_base_addr & 0xffffu) != 0) {
+		    stencil_address == 0 || (stencil_address & 0xffffu) != 0) {
 			DepthFatal("unsupported stencil attachment state");
 		}
 	} else if (z.stencil_read_base_addr != 0 || z.stencil_write_base_addr != 0) {
@@ -229,67 +221,16 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer&
 	const auto depth_backing_size   = depth_size.size * view.image_layers;
 	const auto stencil_backing_size = stencil_size.size * view.image_layers;
 	const auto htile_backing_size   = htile_size.size * view.image_layers;
-	if (!GuestRange {z.z_read_base_addr, depth_backing_size}.Valid() ||
-	    (has_stencil && !GuestRange {z.stencil_read_base_addr, stencil_backing_size}.Valid()) ||
+	if (!GuestRange {depth_address, depth_backing_size}.Valid() ||
+	    (has_stencil && !GuestRange {stencil_address, stencil_backing_size}.Valid()) ||
 	    (has_htile && !GuestRange {z.htile_data_base_addr, htile_backing_size}.Valid())) {
 		DepthFatal("layered depth backing range is invalid");
 	}
-	r.depth_clear_enable      = rc.depth_clear_enable;
-	r.depth_meta_clear_enable = false;
-	r.depth_load_clear_enable = r.depth_clear_enable;
-	r.depth_clear_value       = hw.GetDepthClearValue();
-	r.depth_test_enable       = dc.z_enable;
-	r.depth_write_enable      = r.depth_test_enable && dc.z_write_enable &&
-	                            !z.depth_view.depth_write_disable && !r.depth_clear_enable;
-	r.depth_compare_op        = static_cast<vk::CompareOp>(dc.zfunc);
-
-	r.depth_bounds_test_enable = dc.depth_bounds_enable;
-	r.depth_min_bounds         = hw.GetDepthBoundsMin();
-	r.depth_max_bounds         = hw.GetDepthBoundsMax();
-
-	r.stencil_clear_enable =
-	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;
-	r.stencil_clear_value = hw.GetStencilClearValue();
-	r.stencil_test_enable = has_stencil && dc.stencil_enable;
-	if (r.stencil_test_enable) {
-		const bool stencil_ops_disabled =
-		    rc.stencil_clear_enable || z.depth_view.stencil_write_disable;
-		const uint8_t front_write_mask = stencil_ops_disabled ? 0 : sm.stencil_writemask;
-		const uint8_t back_write_mask  = stencil_ops_disabled ? 0 : sm.stencil_writemask_bf;
-		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
-		    (dc.backface_enable &&
-		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||
-		    (front_write_mask != 0 &&
-		     UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
-		     sm.stencil_opval != sm.stencil_testval) ||
-		    (dc.backface_enable && back_write_mask != 0 &&
-		     UsesStencilOpValue(sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf) &&
-		     sm.stencil_opval_bf != sm.stencil_testval_bf)) {
-			DepthFatal("unsupported stencil compare or replacement state");
-		}
-		r.stencil_static_front = {
-		    ConvertStencilOp(sc.stencil_fail, front_write_mask, sm.stencil_opval),
-		    ConvertStencilOp(sc.stencil_zpass, front_write_mask, sm.stencil_opval),
-		    ConvertStencilOp(sc.stencil_zfail, front_write_mask, sm.stencil_opval),
-		    static_cast<vk::CompareOp>(dc.stencilfunc)};
-		r.stencil_dynamic_front = {sm.stencil_mask, front_write_mask, sm.stencil_testval};
-		if (dc.backface_enable) {
-			r.stencil_static_back = {
-			    ConvertStencilOp(sc.stencil_fail_bf, back_write_mask, sm.stencil_opval_bf),
-			    ConvertStencilOp(sc.stencil_zpass_bf, back_write_mask, sm.stencil_opval_bf),
-			    ConvertStencilOp(sc.stencil_zfail_bf, back_write_mask, sm.stencil_opval_bf),
-			    static_cast<vk::CompareOp>(dc.stencilfunc_bf)};
-			r.stencil_dynamic_back = {sm.stencil_mask_bf, back_write_mask, sm.stencil_testval_bf};
-		} else {
-			r.stencil_static_back  = r.stencil_static_front;
-			r.stencil_dynamic_back = r.stencil_dynamic_front;
-		}
-	}
 	TextureCache::ImageDesc desc {};
 	desc.type                 = TextureCache::BindingType::DepthTarget;
-	desc.info.data            = {z.z_read_base_addr, depth_backing_size};
+	desc.info.data            = {depth_address, depth_backing_size};
 	desc.info.stencil =
-	    has_stencil ? GuestRange {z.stencil_read_base_addr, stencil_backing_size} : GuestRange {};
+	    has_stencil ? GuestRange {stencil_address, stencil_backing_size} : GuestRange {};
 	desc.info.pixel_format    = format;
 	desc.info.guest_format    = guest_format;
 	desc.info.type            = Prospero::ImageType::kColor2D;
@@ -314,12 +255,164 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer&
 	desc.view_info.base_layer  = view.base_layer;
 	desc.view_info.layer_count = view.layer_count;
 	desc.view_info.usage       = vk::ImageUsageFlagBits::eDepthStencilAttachment;
-	r.desc                     = std::move(desc);
-	auto& cache                = m_context.GetTextureCache();
-	r.image_id                 = cache.FindImage(r.desc);
+	return desc;
+}
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepthInfo& r) {
+	KYTY_PROFILER_FUNCTION();
+	const auto& hw          = buffer.GetRegisters();
+	const auto& z           = hw.GetDepthRenderTarget();
+	const auto& rc          = hw.GetRenderControl();
+	const auto& dc          = hw.GetDepthControl();
+	const auto& sc          = hw.GetStencilControl();
+	const auto& sm          = hw.GetStencilMask();
+	const bool  has_stencil = z.stencil_info.format != Prospero::StencilFormat::kInvalid;
+	const bool depth_active = dc.z_enable || dc.depth_bounds_enable ||
+	                          rc.depth_clear_enable || rc.copy_depth_to_color;
+	const bool stencil_active =
+	    has_stencil && (dc.stencil_enable || rc.stencil_clear_enable || rc.copy_stencil_to_color);
+	if (!depth_active && !stencil_active) {
+		return;
+	}
+	const bool attachment_unbound =
+	    z.z_info.format == Prospero::DepthFormat::kInvalid &&
+	    z.stencil_info.format == Prospero::StencilFormat::kInvalid && z.z_info.num_samples == 0 &&
+	    z.z_info.texture_compatibility == Prospero::TextureCompatiblePlaneCompression::kDisable &&
+	    !z.z_info.expclear_enabled && !z.z_info.partially_resident && z.z_info.max_mip_level == 0 &&
+	    z.stencil_info.texture_compatibility == Prospero::TextureCompatibleStencil::kDisable &&
+	    !z.stencil_info.expclear_enabled && !z.stencil_info.partially_resident &&
+	    z.depth_view.slice_start == 0 && z.depth_view.slice_max == 0 &&
+	    z.depth_view.current_mip_level == 0 && !z.depth_view.depth_write_disable &&
+	    !z.depth_view.stencil_write_disable && z.z_read_base_addr == 0 && z.z_write_base_addr == 0 &&
+	    z.stencil_read_base_addr == 0 && z.stencil_write_base_addr == 0 &&
+	    z.htile_data_base_addr == 0 &&
+	    // DB_DEPTH_SIZE_XY is independent state and may remain programmed after the attachment
+	    // formats and addresses are unbound. A zero encoding is the valid 1x1 value, so its
+	    // presence alone must not manufacture a depth attachment.
+	    !z.z_info.htile_acceleration && z.shading_rate_encoding == 0 && z.size.x_max == 0 &&
+	    z.size.y_max == 0;
+	if (attachment_unbound) {
+		static std::atomic_bool logged = false;
+		if (!logged.exchange(true, std::memory_order_relaxed)) {
+			LOGF("DepthTarget: ignoring enabled depth/stencil state without a bound attachment\n");
+		}
+		return;
+	}
+	if (rc.copy_depth_to_color || rc.copy_stencil_to_color || rc.copy_centroid ||
+	    rc.copy_sample != 0 || dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
+	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
+	    (has_stencil && !z.depth_view.stencil_write_disable &&
+	     z.stencil_write_base_addr != z.stencil_read_base_addr)) {
+		DepthFatal("unsupported depth register state");
+	}
+	r.desc = MakeDepthTargetDesc(buffer, z);
+	r.depth_clear_enable      = rc.depth_clear_enable;
+	r.depth_load_clear_enable = r.depth_clear_enable;
+	r.depth_clear_value       = hw.GetDepthClearValue();
+	r.depth_test_enable       = dc.z_enable;
+	r.depth_write_enable      = r.depth_test_enable && dc.z_write_enable &&
+	                            !z.depth_view.depth_write_disable && !r.depth_clear_enable;
+	r.depth_compare_op        = static_cast<vk::CompareOp>(dc.zfunc);
+
+	r.depth_bounds_test_enable = dc.depth_bounds_enable;
+	r.depth_min_bounds         = hw.GetDepthBoundsMin();
+	r.depth_max_bounds         = hw.GetDepthBoundsMax();
+
+	r.stencil_clear_enable =
+	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;
+	r.stencil_clear_value = hw.GetStencilClearValue();
+	r.stencil_test_enable = has_stencil && dc.stencil_enable;
+	if (r.stencil_test_enable) {
+		const bool stencil_ops_disabled =
+		    rc.stencil_clear_enable || z.depth_view.stencil_write_disable;
+		const uint8_t front_write_mask = stencil_ops_disabled ? 0 : sm.stencil_writemask;
+		const uint8_t back_write_mask  = stencil_ops_disabled ? 0 : sm.stencil_writemask_bf;
+		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
+		    (dc.backface_enable &&
+		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways))) {
+			DepthFatal("unsupported stencil compare state");
+		}
+		r.stencil_front = ConvertStencilState(
+		    dc.stencilfunc, {sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail},
+		    sm.stencil_opval, {.compareMask = sm.stencil_mask, .writeMask = front_write_mask,
+		                       .reference = sm.stencil_testval});
+		if (dc.backface_enable) {
+			r.stencil_back = ConvertStencilState(
+			    dc.stencilfunc_bf, {sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf},
+			    sm.stencil_opval_bf, {.compareMask = sm.stencil_mask_bf, .writeMask = back_write_mask,
+			                          .reference = sm.stencil_testval_bf});
+		} else {
+			r.stencil_back = r.stencil_front;
+		}
+	}
+	auto& cache = m_context.GetTextureCache();
+	r.image_id = cache.FindImage(r.desc);
 	BindRenderTarget(r.image_id);
 }
 
+bool RenderExecutor::DepthStencilCopy(CommandBuffer& buffer) {
+	const auto& hw       = buffer.GetRegisters();
+	const auto& z        = hw.GetDepthRenderTarget();
+	const auto& override = hw.GetDepthRenderOverride();
+	if (hw.GetColorControl().mode != 0) {
+		return false;
+	}
+	const bool depth_copy = override.force_z_dirty && override.force_z_valid &&
+	                        z.z_info.format != Prospero::DepthFormat::kInvalid &&
+	                        z.z_read_base_addr != 0 && z.z_write_base_addr != 0 &&
+	                        z.z_read_base_addr != z.z_write_base_addr;
+	const bool stencil_copy = override.force_stencil_dirty && override.force_stencil_valid &&
+	                          z.stencil_info.format != Prospero::StencilFormat::kInvalid &&
+	                          z.stencil_read_base_addr != 0 && z.stencil_write_base_addr != 0 &&
+	                          z.stencil_read_base_addr != z.stencil_write_base_addr;
+	if (!depth_copy && !stencil_copy) {
+		return false;
+	}
+
+	auto  read_desc  = MakeDepthTargetDesc(buffer, z);
+	auto  write_desc = MakeDepthTargetDesc(buffer, z, true);
+	auto& cache      = m_context.GetTextureCache();
+	const auto read_id = cache.FindImage(read_desc);
+	BindRenderTarget(read_id);
+	const auto write_id = cache.FindImage(write_desc);
+	BindRenderTarget(write_id);
+	cache.UpdateImage(read_id);
+	cache.UpdateImage(write_id);
+	cache.MarkGpuWritten(write_id);
+	auto& source      = cache.GetImage(read_id);
+	auto& destination = cache.GetImage(write_id);
+	EXIT_IF(read_id == write_id || source.backing.format != destination.backing.format);
+
+	auto& scheduler = m_context.GetCommandScheduler();
+	scheduler.EndRendering();
+	const auto command = scheduler.Current().Handle();
+	const ImageSubresourceRange range {read_desc.view_info.base_level, 1,
+	                                  read_desc.view_info.base_layer,
+	                                  read_desc.view_info.layer_count};
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
+	               range, command);
+	destination.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
+	                    range, command);
+	std::array<vk::ImageCopy, 2> regions {};
+	uint32_t                   count = 0;
+	for (const auto aspect: {vk::ImageAspectFlagBits::eDepth, vk::ImageAspectFlagBits::eStencil}) {
+		if ((aspect == vk::ImageAspectFlagBits::eDepth && !depth_copy) ||
+		    (aspect == vk::ImageAspectFlagBits::eStencil && !stencil_copy)) {
+			continue;
+		}
+		auto& region = regions[count++];
+		region.srcSubresource = {aspect, range.base_level, range.base_layer,
+		                         range.layer_count};
+		region.dstSubresource = region.srcSubresource;
+		region.extent = write_desc.info.extent;
+	}
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
+	                  destination.backing.image, vk::ImageLayout::eTransferDstOptimal,
+	                  count, regions.data());
+	return true;
+}
+
 vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const {
 	const auto format = desc.view_info.format;
 	if (format == vk::Format::eUndefined) {
@@ -336,14 +429,13 @@ vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const {
 		return writes;
 	}
 
-	const auto face_writes = [&](const PipelineStencilStaticState&  state,
-	                             const PipelineStencilDynamicState& dynamic) {
-		if (dynamic.writeMask == 0) {
+	const auto face_writes = [&](const vk::StencilOpState& state) {
+		if (state.writeMask == 0) {
 			return false;
 		}
 		bool can_pass = state.compareOp != vk::CompareOp::eNever;
 		bool can_fail = state.compareOp != vk::CompareOp::eAlways;
-		if (dynamic.compareMask == 0) {
+		if (state.compareMask == 0) {
 			switch (state.compareOp) {
 				case vk::CompareOp::eEqual:
 				case vk::CompareOp::eLessOrEqual:
@@ -369,8 +461,7 @@ vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const {
 		       (can_pass && depth_fail && state.depthFailOp != vk::StencilOp::eKeep);
 	};
 	if (stencil_clear_enable ||
-	    (stencil_test_enable && (face_writes(stencil_static_front, stencil_dynamic_front) ||
-	                             face_writes(stencil_static_back, stencil_dynamic_back)))) {
+	    (stencil_test_enable && (face_writes(stencil_front) || face_writes(stencil_back)))) {
 		writes |= vk::ImageAspectFlagBits::eStencil;
 	}
 	return writes;

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -23,7 +23,6 @@ struct RenderDepthInfo {
 	TextureCache::ImageDesc     desc;
 	bool                        depth_clear_enable       = false;
 	bool                        depth_load_clear_enable  = false;
-	bool                        depth_meta_clear_enable  = false;
 	float                       depth_clear_value        = 0.0f;
 	bool                        depth_test_enable        = false;
 	// Effective draw writes; discovery applies test, target-write and clear controls.
@@ -35,10 +34,8 @@ struct RenderDepthInfo {
 	bool                        stencil_clear_enable     = false;
 	uint8_t                     stencil_clear_value      = 0;
 	bool                        stencil_test_enable      = false;
-	PipelineStencilStaticState  stencil_static_front;
-	PipelineStencilStaticState  stencil_static_back;
-	PipelineStencilDynamicState stencil_dynamic_front;
-	PipelineStencilDynamicState stencil_dynamic_back;
+	vk::StencilOpState          stencil_front;
+	vk::StencilOpState          stencil_back;
 	ImageId                     image_id;
 
 	[[nodiscard]] vk::ImageAspectFlags AttachmentWriteAspects() const;

src/graphics/host_gpu/renderer/image/blitHelper.cpp

@@ -10,7 +10,6 @@
 
 #include <algorithm>
 #include <array>
-#include <iterator>
 
 namespace Libs::Graphics {
 
@@ -37,9 +36,8 @@ BlitHelper::BlitHelper(GraphicContext& graphics, CommandScheduler& scheduler)
 	    m_graphics.device.createPipelineLayout(&layout_info, nullptr, &m_pipeline_layout),
 	    "create BlitHelper pipeline layout");
 
-	m_vertex_shader = CreateShader(GPU_BLIT_FS_TRIANGLE_SPV, std::size(GPU_BLIT_FS_TRIANGLE_SPV));
-	m_fragment_shader =
-	    CreateShader(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV, std::size(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV));
+	m_vertex_shader   = CompileSPV(GPU_BLIT_FS_TRIANGLE_SPV, m_graphics.device);
+	m_fragment_shader = CompileSPV(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV, m_graphics.device);
 }
 
 BlitHelper::~BlitHelper() {
@@ -60,17 +58,6 @@ BlitHelper::~BlitHelper() {
 	}
 }
 
-vk::ShaderModule BlitHelper::CreateShader(const uint32_t* code, size_t words) const {
-	EXIT_IF(code == nullptr || words == 0);
-	vk::ShaderModuleCreateInfo create {};
-	create.codeSize         = words * sizeof(uint32_t);
-	create.pCode            = code;
-	vk::ShaderModule module = nullptr;
-	RequireVulkanSuccess(m_graphics.device.createShaderModule(&create, nullptr, &module),
-	                     "create BlitHelper shader module");
-	return module;
-}
-
 vk::Pipeline BlitHelper::GetPipeline(PipelineKey key) {
 	const auto cached = std::ranges::find(m_pipelines, key, &Pipeline::key);
 	if (cached != m_pipelines.end()) {

src/graphics/host_gpu/renderer/image/blitHelper.h

@@ -37,8 +37,7 @@ private:
 		vk::Pipeline handle = nullptr;
 	};
 
-	[[nodiscard]] vk::ShaderModule CreateShader(const uint32_t* code, size_t words) const;
-	[[nodiscard]] vk::Pipeline     GetPipeline(PipelineKey key);
+	[[nodiscard]] vk::Pipeline GetPipeline(PipelineKey key);
 
 	GraphicContext&         m_graphics;
 	CommandScheduler&       m_scheduler;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -661,26 +661,17 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 		return;
 	}
 
-	backing.format      = info.pixel_format;
-	backing.image_type  = HostImageType(info.type);
-	backing.extent      = info.extent;
-	backing.layers      = info.IsVolume() ? 1u : info.resources.layers;
-	backing.mip_levels  = info.resources.levels;
-	backing.samples     = info.samples;
-	backing.flags       = ImageCreateFlags(graphics, info);
-	backing.usage       = ImageUsageFlags(graphics, info);
-
 	vk::ImageCreateInfo create {};
-	create.flags         = backing.flags;
-	create.imageType     = backing.image_type;
-	create.extent        = backing.extent;
-	create.mipLevels     = backing.mip_levels;
-	create.arrayLayers   = backing.layers;
-	create.format        = backing.format;
+	create.flags         = ImageCreateFlags(graphics, info);
+	create.imageType     = HostImageType(info.type);
+	create.extent        = info.extent;
+	create.mipLevels     = info.resources.levels;
+	create.arrayLayers   = info.IsVolume() ? 1u : info.resources.layers;
+	create.format        = info.pixel_format;
 	create.tiling        = vk::ImageTiling::eOptimal;
-	create.initialLayout = backing.state.layout;
-	create.usage         = backing.usage;
-	create.samples       = vulkan_sample_count(backing.samples);
+	create.initialLayout = vk::ImageLayout::eUndefined;
+	create.usage         = ImageUsageFlags(graphics, info);
+	create.samples       = vulkan_sample_count(info.samples);
 
 	vk::ImageFormatProperties properties {};
 	if (graphics.GetImageFormatProperties(create.format, create.imageType, create.tiling,
@@ -691,10 +682,9 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 		     "flags=0x%x samples=%u\n",
 		     static_cast<int>(create.format), static_cast<int>(create.imageType),
 		     static_cast<vk::ImageUsageFlags::MaskType>(create.usage),
-		     static_cast<vk::ImageCreateFlags::MaskType>(create.flags), backing.samples);
+		     static_cast<vk::ImageCreateFlags::MaskType>(create.flags), info.samples);
 	}
 
-	backing.memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal;
 	if (!graphics.CreateImage(create, backing)) {
 		EXIT("failed to create image: extent=%ux%ux%u format=%d layers=%u levels=%u\n",
 		     create.extent.width, create.extent.height, create.extent.depth,
@@ -703,11 +693,12 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 }
 
 uint64_t Image::HashGuestEdges() const {
-	constexpr uint64_t                         page_mask = TRACKER_PAGE_SIZE - 1;
 	std::array<uint8_t, TRACKER_PAGE_SIZE * 2> bytes {};
 	const auto                                 range = info.data;
-	const uint64_t head_end     = std::min(range.End(), (range.address + page_mask) & ~page_mask);
-	const uint64_t tail_begin   = std::max(range.address, range.End() & ~page_mask);
+	const uint64_t head_end =
+	    std::min(range.End(), Common::AlignUp(range.address, TRACKER_PAGE_SIZE));
+	const uint64_t tail_begin =
+	    std::max(range.address, Common::AlignDown(range.End(), TRACKER_PAGE_SIZE));
 	const uint64_t head_size    = head_end - range.address;
 	const uint64_t tail_address = tail_begin < head_end ? head_end : tail_begin;
 	const uint64_t tail_size    = range.End() - tail_address;

src/graphics/host_gpu/renderer/image/image.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/slotVector.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -133,7 +134,7 @@ public:
 	}
 	[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
 	[[nodiscard]] uint64_t AccountedSize() const noexcept {
-		return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
+		return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024);
 	}
 	[[nodiscard]] uint64_t HashGuestEdges() const;
 

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -82,9 +82,6 @@ struct ImageInfo {
 	[[nodiscard]] constexpr bool IsVolume() const noexcept {
 		return type == Prospero::ImageType::kColor3D;
 	}
-	[[nodiscard]] constexpr bool IsLayered() const noexcept {
-		return !IsVolume() && resources.layers > 1;
-	}
 	[[nodiscard]] constexpr uint32_t TransferLayers() const noexcept {
 		return IsVolume() ? extent.depth : resources.layers;
 	}
@@ -97,9 +94,24 @@ struct ImageInfo {
 		       bytes_per_block == other.bytes_per_block;
 	}
 	[[nodiscard]] int32_t MipOf(const ImageInfo& container) const noexcept {
-		if (!IsCompatible(container) || tile_mode != container.tile_mode || resources.levels != 1 ||
-		    container.resources.layers == 0 ||
-		    container.resources.levels > container.mip_layout.size()) {
+		if (container.resources.levels > container.mip_layout.size()) {
+			return -1;
+		}
+		for (uint32_t level = 0; level < container.resources.levels; level++) {
+			if (SliceOf(container, static_cast<int32_t>(level)) >= 0) {
+				return static_cast<int32_t>(level);
+			}
+		}
+		return -1;
+	}
+	[[nodiscard]] int32_t SliceOf(const ImageInfo& container, int32_t mip) const noexcept {
+		if (!IsCompatible(container) || tile_mode != container.tile_mode || type != container.type ||
+		    IsVolume() || resources.levels != 1 || resources.layers == 0 ||
+		    container.resources.layers == 0 || mip < 0 ||
+		    static_cast<uint32_t>(mip) >= container.resources.levels ||
+		    container.resources.levels > container.mip_layout.size() ||
+		    !data.Valid() || !container.data.Valid() || data.address < container.data.address ||
+		    data.End() > container.data.End()) {
 			return -1;
 		}
 		if (HasStencil() != container.HasStencil() ||
@@ -108,75 +120,36 @@ struct ImageInfo {
 			return -1;
 		}
 
-		int32_t mip = -1;
-		for (uint32_t level = 0; level < container.resources.levels; level++) {
-			const auto& layout = container.mip_layout[level];
-			if (layout.size == 0 || layout.size % container.resources.layers != 0 ||
-			    container.data.address > UINT64_MAX - layout.offset) {
-				continue;
-			}
-			const auto mip_base   = container.data.address + layout.offset;
-			const auto slice_size = layout.size / container.resources.layers;
-			if (slice_size == 0 || mip_base > UINT64_MAX - layout.size) {
-				continue;
-			}
-			const auto mip_end = mip_base + layout.size;
-			if (data.address >= mip_base && data.address < mip_end &&
-			    (data.address - mip_base) % slice_size == 0) {
-				mip = static_cast<int32_t>(level);
-				break;
-			}
-		}
-		if (mip < 0) {
+		const auto  level  = static_cast<uint32_t>(mip);
+		const auto& layout = container.mip_layout[level];
+		if (extent.width != std::max(container.extent.width >> level, 1u) ||
+		    extent.height != std::max(container.extent.height >> level, 1u) ||
+		    extent.depth != container.extent.depth || mip_layout[0].pitch != layout.pitch ||
+		    mip_layout[0].height != layout.height || layout.size == 0 ||
+		    layout.size % container.resources.layers != 0 ||
+		    container.data.size % container.resources.layers != 0 ||
+		    data.size % resources.layers != 0) {
 			return -1;
 		}
 
-		const auto level = static_cast<uint32_t>(mip);
-		if (extent.width != std::max(container.extent.width >> level, 1u) ||
-		    extent.height != std::max(container.extent.height >> level, 1u)) {
+		// PS5 block slices contain the entire reversed mip chain.
+		const auto slice_stride = container.data.size / container.resources.layers;
+		const auto child_stride = data.size / resources.layers;
+		if (child_stride != layout.size / container.resources.layers ||
+		    layout.offset >= slice_stride || child_stride > slice_stride - layout.offset ||
+		    (resources.layers > 1 && child_stride != slice_stride)) {
 			return -1;
 		}
-		const auto mip_depth = std::max(container.extent.depth >> level, 1u);
-		if (container.type == Prospero::ImageType::kColor3D &&
-		    type == Prospero::ImageType::kColor2D) {
-			if (resources.layers != mip_depth) {
-				return -1;
-			}
-		} else if (type != container.type) {
+		const auto address_delta = data.address - container.data.address;
+		if (address_delta < layout.offset || (address_delta - layout.offset) % slice_stride != 0) {
 			return -1;
 		}
-		return mip;
-	}
-	[[nodiscard]] int32_t SliceOf(const ImageInfo& container, int32_t mip) const noexcept {
-		if (!IsCompatible(container) || type != container.type || mip < 0 ||
-		    static_cast<uint32_t>(mip) >= container.resources.levels ||
-		    container.resources.levels > container.mip_layout.size() ||
-		    container.resources.layers == 0 || data.size == 0) {
+		const auto layer = (address_delta - layout.offset) / slice_stride;
+		if (layer > INT32_MAX || layer >= container.resources.layers ||
+		    resources.layers > container.resources.layers - layer) {
 			return -1;
 		}
-		const auto level = static_cast<uint32_t>(mip);
-		if (extent.width != std::max(container.extent.width >> level, 1u) ||
-		    extent.height != std::max(container.extent.height >> level, 1u)) {
-			return -1;
-		}
-		const auto& layout = container.mip_layout[level];
-		if (layout.size == 0 || layout.size % container.resources.layers != 0 ||
-		    container.data.address > UINT64_MAX - layout.offset) {
-			return -1;
-		}
-		const auto slice_size = layout.size / container.resources.layers;
-		if (slice_size == 0 || data.size % slice_size != 0) {
-			return -1;
-		}
-		const auto mip_base = container.data.address + layout.offset;
-		if (data.address < mip_base) {
-			return -1;
-		}
-		const auto address_delta = data.address - mip_base;
-		if (address_delta % data.size != 0 || address_delta / data.size > INT32_MAX) {
-			return -1;
-		}
-		return static_cast<int32_t>(address_delta / data.size);
+		return static_cast<int32_t>(layer);
 	}
 };
 
@@ -342,14 +315,6 @@ ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t
 	}
 }
 
-[[nodiscard]] inline constexpr bool
-CanUseVideoOutNativeWithoutUpload(VideoOutCompression compression, bool render_target,
-                                  bool gpu_modified, bool guest_modified) noexcept {
-	return compression != VideoOutCompression::Uncompressed &&
-	       compression != VideoOutCompression::Unsupported && !guest_modified &&
-	       (render_target || gpu_modified);
-}
-
 struct VideoOutPixelFormatInfo {
 	vk::Format             format            = vk::Format::eUndefined;
 	Prospero::BufferFormat guest_format      = Prospero::BufferFormat::kInvalid;
@@ -400,32 +365,6 @@ inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {
 	return false;
 }
 
-[[nodiscard]] inline constexpr bool
-IsSupportedDisplayRenderTargetTileMode(Prospero::TileMode tile_mode) noexcept {
-	return tile_mode == Prospero::TileMode::kRenderTarget;
-}
-
-[[nodiscard]] inline constexpr bool IsSupportedStandard64RenderTarget(const ImageInfo& info) {
-	if (info.tile_mode != Prospero::TileMode::kStandard64KB || info.data.address == 0 ||
-	    (info.data.address & 0xffffu) != 0 || info.extent.width == 0 || info.extent.height == 0 ||
-	    info.bytes_per_block != 4 || info.resources.levels != 1 || info.resources.layers != 1 ||
-	    info.samples != 1) {
-		return false;
-	}
-	const auto expected_pitch =
-	    (static_cast<uint64_t>(info.extent.width) + 127u) & ~uint64_t {127u};
-	const auto padded_height =
-	    (static_cast<uint64_t>(info.extent.height) + 127u) & ~uint64_t {127u};
-	return expected_pitch <= UINT32_MAX && info.pitch == expected_pitch &&
-	       expected_pitch <= UINT64_MAX / padded_height / info.bytes_per_block &&
-	       info.data.size == expected_pitch * padded_height * info.bytes_per_block;
-}
-
-[[nodiscard]] inline constexpr bool IsTiledRenderTarget(const ImageInfo& info) noexcept {
-	return info.tile_mode == Prospero::TileMode::kRenderTarget ||
-	       IsSupportedStandard64RenderTarget(info);
-}
-
 [[nodiscard]] inline bool DecodePackedColorClear(vk::Format format, uint32_t packed,
                                                  vk::ClearColorValue& clear) {
 	vk::ClearColorValue next {};

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -322,6 +322,12 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		normalized.aspect = vk::ImageAspectFlagBits::eStencil;
 	}
 	normalized.usage = is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
+	for (const auto& cached: views) {
+		if (cached.info == normalized) {
+			return cached.view;
+		}
+	}
+
 	const bool format_compatible = normalized.format != vk::Format::eUndefined &&
 	                               ImageViewOps::FormatsCompatible(image.format, normalized.format);
 	const bool slice_view =
@@ -351,12 +357,6 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     image.layers);
 	}
 
-	for (const auto& cached: views) {
-		if (cached.info == normalized) {
-			return cached.view;
-		}
-	}
-
 	vk::ImageViewUsageCreateInfo usage {};
 	usage.usage = image.usage;
 	if (!is_storage) {

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -1,15 +1,16 @@
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/host_gpu/renderer/image/tiler.h"
 #include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/shader.h"
 
 #include <algorithm>
-#include <bit>
+#include <array>
 #include <cinttypes>
-#include <cstring>
 
 namespace Libs::Graphics {
 namespace {
@@ -27,13 +28,13 @@ constexpr Prospero::ColorComponentMapping kRenderTargetColorMappings[4][4] = {
      Prospero::ColorMappingArgb},
 };
 
-struct RenderTargetHostFormatInfo {
+struct HostFormatInfo {
 	vk::Format                      format = vk::Format::eUndefined;
 	Prospero::ColorComponentMapping host_to_storage;
 };
 
-RenderTargetHostFormatInfo ResolveRenderTargetHostFormat(Prospero::BufferFormat guest_format,
-                                                         Prospero::ChannelOrder order) {
+HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
+                                 Prospero::ChannelOrder order) {
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k8_8_8_8UNorm:
@@ -47,17 +48,77 @@ RenderTargetHostFormatInfo ResolveRenderTargetHostFormat(Prospero::BufferFormat
 			default: break;
 		}
 	}
+	const auto format = VulkanFormat(guest_format);
 	switch (guest_format) {
-		case Prospero::BufferFormat::k5_5_5_1UNorm:
-			return {vk::Format::eA1R5G5B5UnormPack16, Prospero::ColorMappingBgra};
+		case Prospero::BufferFormat::k5_5_5_1UNorm: return {format, Prospero::ColorMappingBgra};
 		case Prospero::BufferFormat::k1_5_5_5UNorm:
-			return {vk::Format::eR5G5B5A1UnormPack16, Prospero::ColorMappingAbgr};
-		case Prospero::BufferFormat::k4_4_4_4UNorm:
-			return {vk::Format::eR4G4B4A4UnormPack16, Prospero::ColorMappingAbgr};
-		default: return {VulkanFormat(guest_format), {}};
+		case Prospero::BufferFormat::k4_4_4_4UNorm: return {format, Prospero::ColorMappingAbgr};
+		default: return {format, {}};
 	}
 }
 
+uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) {
+	return volume_texture ? std::max(depth >> level, 1u) : depth;
+}
+
+size_t GetTextureRegionCount(uint32_t depth, uint32_t levels, bool volume_texture) {
+	size_t count = 0;
+	for (uint32_t level = 0; level < levels; level++) {
+		count += GetTextureLevelDepth(depth, level, volume_texture);
+	}
+	return count;
+}
+
+uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint32_t levels,
+                                uint64_t total_size, uint32_t depth) {
+	uint64_t stride = 0;
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		stride = std::max(stride, mips[mip_level].offset + mips[mip_level].size);
+	}
+
+	if (depth > 1 && total_size != 0 && total_size % depth == 0) {
+		stride = std::max(stride, total_size / depth);
+	}
+
+	return stride;
+}
+
+uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch,
+                                   uint32_t height) {
+	const uint32_t element_columns =
+	    std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u);
+	const uint32_t element_rows =
+	    std::max((height + element.texel_height - 1u) / element.texel_height, 1u);
+	return static_cast<uint64_t>(element_columns) * element_rows * element.bytes;
+}
+
+uint64_t SetLinearUploadLevels(TextureUploadMipLayout*         mips,
+                               const TileTextureElementLayout& element, uint32_t height,
+                               uint32_t levels, uint32_t base_pitch) {
+	uint64_t offset     = 0;
+	auto     mip_pitch  = base_pitch;
+	auto     mip_height = height;
+
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		const auto size = CalcLinearUploadLevelSize(element, mip_pitch, mip_height);
+		mips[mip_level] = {offset, size};
+
+		offset += size;
+		if (mip_pitch > 1) {
+			mip_pitch /= 2;
+		}
+		if (mip_height > 1) {
+			mip_height /= 2;
+		}
+	}
+
+	return offset;
+}
+
+bool FitsBufferRange(uint64_t offset, uint64_t size, uint64_t capacity) {
+	return offset <= capacity && size <= capacity - offset;
+}
+
 } // namespace
 
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
@@ -65,135 +126,61 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layo
                                                     Prospero::ChannelOrder  order) {
 	const auto encoding = Prospero::ResolveRenderTargetFormat(layout, type);
 	if (encoding.IsValid() && encoding.SupportsOrder(order)) {
-		const auto host  = ResolveRenderTargetHostFormat(encoding.buffer_format, order);
-		const auto bytes = Prospero::RenderTargetBytesPerElement(encoding.buffer_format);
-		if (host.format != vk::Format::eUndefined && bytes != 0) {
+		const auto host_format = ResolveHostFormat(encoding.buffer_format, order);
+		const auto bytes       = Prospero::RenderTargetBytesPerElement(encoding.buffer_format);
+		if (host_format.format != vk::Format::eUndefined && bytes != 0) {
 			const auto order_mapping =
 			    kRenderTargetColorMappings[static_cast<size_t>(order)][encoding.components - 1u];
-			return {host.format, bytes, host.host_to_storage.Then(order_mapping)};
+			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping)};
 		}
 	}
 	EXIT("unsupported render-target format combination: layout=%u type=%u order=%u\n",
 	     static_cast<uint32_t>(layout), static_cast<uint32_t>(type), static_cast<uint32_t>(order));
 }
 
-namespace {
-
-static uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) {
-	return volume_texture ? std::max(depth >> level, 1u) : depth;
-}
-
-static size_t GetTextureRegionCount(uint32_t depth, uint64_t levels, bool volume_texture) {
-	size_t count = 0;
-	for (uint32_t level = 0; level < levels; level++) {
-		count += GetTextureLevelDepth(depth, level, volume_texture);
+vk::ComponentMapping TextureGetComponentMapping(uint32_t                        swizzle,
+                                                Prospero::ColorComponentMapping host_to_storage) {
+	constexpr std::array host_components {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eG,
+	                                      vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eA};
+	std::array<vk::ComponentSwizzle, 4> storage_components {};
+	for (uint32_t host_component = 0; host_component < host_components.size(); ++host_component) {
+		storage_components[host_to_storage.Map(host_component)] = host_components[host_component];
 	}
-	return count;
-}
-
-vk::ComponentSwizzle TextureGetComponentSwizzle(uint8_t s) {
-	switch (static_cast<Prospero::CompSwizzle>(s)) {
-		case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero;
-		case Prospero::CompSwizzle::kOne: return vk::ComponentSwizzle::eOne;
-		case Prospero::CompSwizzle::kRed: return vk::ComponentSwizzle::eR;
-		case Prospero::CompSwizzle::kGreen: return vk::ComponentSwizzle::eG;
-		case Prospero::CompSwizzle::kBlue: return vk::ComponentSwizzle::eB;
-		case Prospero::CompSwizzle::kAlpha: return vk::ComponentSwizzle::eA;
-		default: EXIT("unknown swizzle: %d\n", static_cast<int>(s));
-	}
-	return vk::ComponentSwizzle::eIdentity;
-}
-
-static uint32_t TextureGetDstSel(uint32_t swizzle, uint32_t channel) {
-	return (swizzle >> (channel * 3u)) & 0x7u;
-}
-
-} // namespace
-
-vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) {
-	vk::ComponentMapping components {};
-	components.r = TextureGetComponentSwizzle(static_cast<uint8_t>(TextureGetDstSel(swizzle, 0)));
-	components.g = TextureGetComponentSwizzle(static_cast<uint8_t>(TextureGetDstSel(swizzle, 1)));
-	components.b = TextureGetComponentSwizzle(static_cast<uint8_t>(TextureGetDstSel(swizzle, 2)));
-	components.a = TextureGetComponentSwizzle(static_cast<uint8_t>(TextureGetDstSel(swizzle, 3)));
-	return components;
+	const auto resolve = [&](uint32_t component) {
+		const auto selector = static_cast<Prospero::CompSwizzle>(GetDstSel(swizzle, component));
+		switch (selector) {
+			case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero;
+			case Prospero::CompSwizzle::kOne: return vk::ComponentSwizzle::eOne;
+			case Prospero::CompSwizzle::kRed: return storage_components[0];
+			case Prospero::CompSwizzle::kGreen: return storage_components[1];
+			case Prospero::CompSwizzle::kBlue: return storage_components[2];
+			case Prospero::CompSwizzle::kAlpha: return storage_components[3];
+			default: EXIT("unknown swizzle: %u\n", static_cast<uint32_t>(selector));
+		}
+	};
+	return {resolve(0), resolve(1), resolve(2), resolve(3)};
 }
 
 SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
-	const auto backing_format    = Prospero::RemapTextureFormat(format);
-	const auto vk_format         = VulkanFormat(backing_format);
+	const auto backing_format = Prospero::RemapTextureFormat(format);
+	const auto host_format = ResolveHostFormat(backing_format, Prospero::ChannelOrder::kStandard);
 	const auto conversion_format =
 	    backing_format != format ? format : Prospero::BufferFormat::kInvalid;
-	if (vk_format != vk::Format::eUndefined) {
-		return SurfaceFormatInfo(vk_format, conversion_format);
+	if (host_format.format != vk::Format::eUndefined) {
+		return {host_format.format, conversion_format, host_format.host_to_storage};
 	}
 	EXIT("unknown format: fmt = %u\n", static_cast<uint32_t>(format));
-	return SurfaceFormatInfo(vk::Format::eUndefined, Prospero::BufferFormat::kInvalid);
 }
 
-namespace {
-
-uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint64_t levels,
-                                uint64_t total_size, uint32_t depth) {
-	uint64_t stride = 0;
-	for (uint32_t i = 0; i < levels; i++) {
-		stride = std::max(stride, mips[i].offset + mips[i].size);
-	}
-
-	if (depth > 1 && total_size != 0 && total_size % depth == 0) {
-		const auto guest_stride = total_size / depth;
-		if (guest_stride >= stride) {
-			stride = guest_stride;
-		}
-	}
-
-	return stride;
-}
-
-uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch,
-                                   uint32_t height) {
-	const uint32_t elements_w =
-	    std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u);
-	const uint32_t elements_h =
-	    std::max((height + element.texel_height - 1u) / element.texel_height, 1u);
-	return static_cast<uint64_t>(elements_w) * elements_h * element.bytes;
-}
-
-uint64_t SetLinearUploadLevels(TextureUploadMipLayout*         mips,
-                               const TileTextureElementLayout& element, uint64_t height,
-                               uint64_t levels, uint32_t base_pitch) {
-	uint64_t offset = 0;
-	auto     pitch  = base_pitch;
-	auto     h      = static_cast<uint32_t>(height);
-
-	for (uint32_t i = 0; i < levels; i++) {
-		const auto size = CalcLinearUploadLevelSize(element, pitch, h);
-		mips[i].size    = size;
-		mips[i].offset  = offset;
-
-		offset += size;
-		if (pitch > 1) {
-			pitch /= 2;
-		}
-		if (h > 1) {
-			h /= 2;
-		}
-	}
-
-	return offset;
-}
-
-} // namespace
-
 TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                             uint32_t height, uint32_t levels, uint32_t depth,
-                                            Prospero::TileMode tile, uint64_t upload_size,
+                                            Prospero::TileMode tile_mode, uint64_t upload_size,
                                             bool allow_depth_tile, bool volume_texture,
                                             const char* owner) {
 	TextureUploadLayout layout {};
 	layout.surface.description = {
 	    format,
-	    tile,
+	    tile_mode,
 	    volume_texture ? TileSurfaceDimension::Dim3D : TileSurfaceDimension::Dim2D,
 	    width,
 	    height,
@@ -202,7 +189,6 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 	    volume_texture ? 1u : depth,
 	};
 	const auto&              description = layout.surface.description;
-	const auto               tile_mode   = description.tile_mode;
 	TileTextureElementLayout element {};
 
 	if (format == Prospero::BufferFormat::kInvalid) {
@@ -212,34 +198,19 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 		     levels);
 	}
 
-	switch (tile_mode) {
-		case Prospero::TileMode::kLinear:
-			if (!TileGetTextureElementLayout(format, element)) {
-				EXIT("%s: unsupported linear texture format: fmt=%u\n", owner,
-				     static_cast<uint32_t>(format));
-			}
-			break;
-		case Prospero::TileMode::kDepth:
-			if (!allow_depth_tile || !TileGetTiledTextureLayout(description, layout.surface)) {
-				EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
-				     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
-				     owner, static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(description.tile_mode), upload_size, width, height,
-				     levels);
-			}
-			break;
-		default:
-			if (!TileGetTiledTextureLayout(description, layout.surface)) {
-				EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
-				     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
-				     owner, static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(description.tile_mode), upload_size, width, height,
-				     levels);
-			}
-			break;
-	}
-
-	if (tile_mode != Prospero::TileMode::kLinear) {
+	if (tile_mode == Prospero::TileMode::kLinear) {
+		if (!TileGetTextureElementLayout(format, element)) {
+			EXIT("%s: unsupported linear texture format: fmt=%u\n", owner,
+			     static_cast<uint32_t>(format));
+		}
+	} else {
+		if ((tile_mode == Prospero::TileMode::kDepth && !allow_depth_tile) ||
+		    !TileGetTiledTextureLayout(description, layout.surface)) {
+			EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
+			     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
+			     owner, static_cast<uint32_t>(format), static_cast<uint32_t>(tile_mode),
+			     upload_size, width, height, levels);
+		}
 		element = {layout.surface.texture.block.bytes_per_element,
 		           layout.surface.texture.texel_width, layout.surface.texture.texel_height};
 	}
@@ -272,40 +243,33 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout) {
 	const auto& description    = layout.surface.description;
 	const bool  volume_texture = description.dimension == TileSurfaceDimension::Dim3D;
+	const bool  linear_texture = description.tile_mode == Prospero::TileMode::kLinear;
 	const auto  depth          = volume_texture ? description.depth : description.layers;
 	uint32_t    mip_width      = description.width;
 	uint32_t    mip_height     = description.height;
-	uint32_t    mip_pitch = volume_texture && description.tile_mode != Prospero::TileMode::kLinear
-	                            ? description.width
-	                            : layout.pitch;
+	uint32_t    mip_pitch = volume_texture && !linear_texture ? description.width : layout.pitch;
 
 	std::vector<vk::BufferImageCopy> regions;
 	regions.reserve(GetTextureRegionCount(depth, description.levels, volume_texture));
-	for (uint32_t i = 0; i < description.levels; i++) {
-		EXIT_NOT_IMPLEMENTED(layout.mips[i].size == 0);
-
-		const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture);
-
-		for (uint32_t z = 0; z < mip_depth; z++) {
-			const auto          slice_offset = z * layout.slice_stride;
-			vk::BufferImageCopy region {};
-			region.bufferOffset     = layout.mips[i].offset + slice_offset;
-			region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i, volume_texture ? 0u : z,
-			                           1};
-			region.imageOffset.z    = volume_texture ? static_cast<int>(z) : 0;
-			region.imageExtent      = {mip_width, mip_height, 1};
-			const bool linear       = description.tile_mode == Prospero::TileMode::kLinear;
-			if (linear) {
-				region.bufferRowLength   = layout.mips[i].row_length;
-				region.bufferImageHeight = layout.mips[i].image_height;
-			} else {
-				const auto texel_width = layout.surface.texture.texel_width;
-				const auto align       = [](uint32_t value, uint32_t block) {
-					return ((value + block - 1u) / block) * block;
-				};
-				const auto pitch       = align(mip_pitch, texel_width);
-				region.bufferRowLength = pitch > align(mip_width, texel_width) ? pitch : 0;
-			}
+	for (uint32_t mip_level = 0; mip_level < description.levels; ++mip_level) {
+		EXIT_NOT_IMPLEMENTED(layout.mips[mip_level].size == 0);
+		const auto          mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture);
+		vk::BufferImageCopy region {};
+		region.imageSubresource = {vk::ImageAspectFlagBits::eColor, mip_level, 0, 1};
+		region.imageExtent      = {mip_width, mip_height, 1};
+		if (linear_texture) {
+			region.bufferRowLength   = layout.mips[mip_level].row_length;
+			region.bufferImageHeight = layout.mips[mip_level].image_height;
+		} else {
+			const auto texel_width   = layout.surface.texture.texel_width;
+			const auto aligned_pitch = Common::AlignUp(mip_pitch, texel_width);
+			region.bufferRowLength =
+			    aligned_pitch > Common::AlignUp(mip_width, texel_width) ? aligned_pitch : 0;
+		}
+		for (uint32_t slice_index = 0; slice_index < mip_depth; ++slice_index) {
+			region.bufferOffset = layout.mips[mip_level].offset + slice_index * layout.slice_stride;
+			region.imageSubresource.baseArrayLayer = volume_texture ? 0u : slice_index;
+			region.imageOffset.z = volume_texture ? static_cast<int>(slice_index) : 0;
 			regions.push_back(region);
 		}
 
@@ -323,17 +287,9 @@ std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayo
 	return regions;
 }
 
-static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity, uint64_t& size) {
-	if (offset > capacity || length > capacity - offset) {
-		return false;
-	}
-	size = length;
-	return true;
-}
-
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,
-                              std::vector<GpuTileInfo>& out_infos) {
+                              std::vector<GpuTileInfo>& out_tile_infos) {
 	const auto& description    = layout.surface.description;
 	const bool  volume_texture = description.dimension == TileSurfaceDimension::Dim3D;
 	const auto  depth          = volume_texture ? description.depth : description.layers;
@@ -351,111 +307,69 @@ bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferI
 		return false;
 	}
 
-	std::vector<GpuTileInfo> infos;
-	infos.reserve(regions.size());
-	if (volume_texture) {
-		size_t region_base = 0;
-		for (uint32_t level = 0; level < levels; ++level) {
-			const uint32_t mip_depth     = GetTextureLevelDepth(depth, level, true);
-			const bool     tail          = level >= surface.first_tail_level;
-			const uint64_t linear_stride = layout.slice_stride;
-			for (uint32_t z = 0; z < mip_depth; z += block.block_depth) {
-				const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z);
-				const auto&    region     = regions[region_base + z];
-				const auto     pitch =
-				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
-				const auto  logical_height = region.bufferImageHeight != 0
-				                                 ? region.bufferImageHeight
-				                                 : region.imageExtent.height;
-				GpuTileInfo info {};
-				info.family            = block.family;
-				info.bytes_per_element = block.bytes_per_element;
-				info.linear_offset     = region.bufferOffset;
-				info.tiled_offset =
-				    static_cast<uint64_t>(z / block.block_depth) * surface.block_slice_size +
-				    surface.mips[level].offset;
-				const uint64_t linear_span =
-				    static_cast<uint64_t>(copy_depth - 1u) * linear_stride +
-				    layout.mips[level].size;
-				if (!SetGpuTileSize(info.linear_offset, linear_span, UINT64_MAX,
-				                    info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, surface.mips[level].size, tiled_size,
-				                    info.tiled_size)) {
-					return false;
-				}
-				info.linear_slice_stride = linear_stride;
-				info.width  = std::max((region.imageExtent.width + texture.texel_width - 1u) /
-				                           texture.texel_width,
-				                       1u);
-				info.height = std::max(
-				    (logical_height + texture.texel_height - 1u) / texture.texel_height, 1u);
-				info.depth = copy_depth;
-				info.surface_z =
+	std::vector<GpuTileInfo> tile_infos;
+	tile_infos.reserve(regions.size());
+	const auto slices_per_copy = volume_texture ? block.block_depth : 1u;
+	size_t     region_base     = 0;
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		const auto& mip       = surface.mips[mip_level];
+		const auto  mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture);
+		for (uint32_t slice_index = 0; slice_index < mip_depth; slice_index += slices_per_copy) {
+			const auto& region = regions[region_base + slice_index];
+			GpuTileInfo tile_info {};
+			tile_info.family            = block.family;
+			tile_info.bytes_per_element = block.bytes_per_element;
+			tile_info.linear_offset     = region.bufferOffset;
+			tile_info.linear_size       = layout.mips[mip_level].size;
+			tile_info.tiled_size        = mip.size;
+			if (volume_texture) {
+				tile_info.depth               = std::min(slices_per_copy, mip_depth - slice_index);
+				tile_info.linear_slice_stride = layout.slice_stride;
+				tile_info.linear_size += (tile_info.depth - 1u) * layout.slice_stride;
+				tile_info.tiled_offset =
+				    mip.offset +
+				    static_cast<uint64_t>(slice_index / slices_per_copy) * surface.block_slice_size;
+				tile_info.surface_z =
 				    block.block_depth == 1 ? static_cast<uint32_t>(region.imageOffset.z) : 0;
-				info.pitch = std::max((pitch + texture.texel_width - 1u) / texture.texel_width, 1u);
-				info.tail_x       = tail ? surface.mips[level].tail_x : 0;
-				info.tail_y       = tail ? surface.mips[level].tail_y : 0;
-				info.tail         = tail;
-				info.tiled_width  = surface.mips[level].padded_width;
-				info.tiled_height = surface.mips[level].padded_height;
-				infos.push_back(info);
-			}
-			region_base += mip_depth;
-		}
-	} else {
-		size_t region_index = 0;
-		for (uint32_t level = 0; level < levels; level++) {
-			const auto& level_size  = layout.mips[level];
-			const auto& mip         = surface.mips[level];
-			const bool  tail        = level >= surface.first_tail_level;
-			const auto  level_depth = GetTextureLevelDepth(depth, level, false);
-			for (uint32_t z = 0; z < level_depth; z++) {
-				const auto& region = regions[region_index++];
-				const auto  pitch =
-				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
-				const auto  logical_height = region.bufferImageHeight != 0
-				                                 ? region.bufferImageHeight
-				                                 : region.imageExtent.height;
-				GpuTileInfo info {};
-				info.family              = block.family;
-				info.bytes_per_element   = block.bytes_per_element;
-				info.linear_offset       = region.bufferOffset;
+			} else {
 				const auto source_stride = layout.source_slice_stride != 0
 				                               ? layout.source_slice_stride
 				                               : surface.block_slice_size;
-				if (z > (UINT64_MAX - mip.offset) / source_stride) {
+				if (slice_index > (UINT64_MAX - mip.offset) / source_stride) {
 					return false;
 				}
-				info.tiled_offset = mip.offset + static_cast<uint64_t>(z) * source_stride;
-				if (!SetGpuTileSize(info.linear_offset, level_size.size, UINT64_MAX,
-				                    info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, mip.size, tiled_size, info.tiled_size)) {
-					return false;
-				}
-				info.width  = std::max((region.imageExtent.width + texture.texel_width - 1u) /
-				                           texture.texel_width,
-				                       1u);
-				info.height = std::max(
-				    (logical_height + texture.texel_height - 1u) / texture.texel_height, 1u);
-				info.surface_z = block.family == TileBlockFamily::RenderTarget64KB ||
-				                         block.family == TileBlockFamily::Depth64KB
-				                     ? region.imageSubresource.baseArrayLayer
-				                     : 0;
-				info.pitch = std::max((pitch + texture.texel_width - 1u) / texture.texel_width, 1u);
-				info.tail  = tail;
-				info.tail_x       = tail ? mip.tail_x : 0;
-				info.tail_y       = tail ? mip.tail_y : 0;
-				info.tiled_width  = mip.padded_width;
-				info.tiled_height = mip.padded_height;
-				infos.push_back(info);
+				tile_info.tiled_offset =
+				    mip.offset + static_cast<uint64_t>(slice_index) * source_stride;
+				tile_info.surface_z = block.family == TileBlockFamily::RenderTarget64KB ||
+				                              block.family == TileBlockFamily::Depth64KB
+				                          ? region.imageSubresource.baseArrayLayer
+				                          : 0;
 			}
+			if (!FitsBufferRange(tile_info.linear_offset, tile_info.linear_size, UINT64_MAX) ||
+			    !FitsBufferRange(tile_info.tiled_offset, tile_info.tiled_size, tiled_size)) {
+				return false;
+			}
+			const auto row_length =
+			    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
+			const auto image_height = region.bufferImageHeight != 0 ? region.bufferImageHeight
+			                                                        : region.imageExtent.height;
+			tile_info.width         = std::max(
+			    (region.imageExtent.width + texture.texel_width - 1u) / texture.texel_width, 1u);
+			tile_info.height =
+			    std::max((image_height + texture.texel_height - 1u) / texture.texel_height, 1u);
+			tile_info.pitch =
+			    std::max((row_length + texture.texel_width - 1u) / texture.texel_width, 1u);
+			tile_info.tail         = mip_level >= surface.first_tail_level;
+			tile_info.tail_x       = tile_info.tail ? mip.tail_x : 0;
+			tile_info.tail_y       = tile_info.tail ? mip.tail_y : 0;
+			tile_info.tiled_width  = mip.padded_width;
+			tile_info.tiled_height = mip.padded_height;
+			tile_infos.push_back(tile_info);
 		}
+		region_base += mip_depth;
 	}
 
-	if (infos.empty()) {
-		return false;
-	}
-	out_infos = std::move(infos);
+	out_tile_infos = std::move(tile_infos);
 	return true;
 }
 

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -1,9 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 
-#include "common/abi.h"
-#include "common/common.h"
-#include "graphics/guest_gpu/gpu_format.h"
+#include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
@@ -20,12 +18,9 @@ struct RenderTargetFormatInfo {
 };
 
 struct SurfaceFormatInfo {
-	explicit constexpr SurfaceFormatInfo(vk::Format format,
-	                                     Prospero::BufferFormat conversion_format)
-	    : vk_format(format), conversion_format(conversion_format) {}
-
-	vk::Format             vk_format;
-	Prospero::BufferFormat conversion_format;
+	vk::Format                      vk_format;
+	Prospero::BufferFormat          conversion_format;
+	Prospero::ColorComponentMapping host_to_storage;
 };
 
 struct TextureUploadMipLayout {
@@ -43,20 +38,21 @@ struct TextureUploadLayout {
 	TextureUploadMipLayout mips[16] = {};
 };
 
-vk::ComponentMapping   TextureGetComponentMapping(uint32_t swizzle);
+vk::ComponentMapping   TextureGetComponentMapping(uint32_t                        swizzle,
+                                                  Prospero::ColorComponentMapping host_to_storage);
 SurfaceFormatInfo      TextureGetSurfaceFormatInfo(Prospero::BufferFormat format);
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
                                                     Prospero::ChannelType   type,
                                                     Prospero::ChannelOrder  order);
 TextureUploadLayout    TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                                uint32_t height, uint32_t levels, uint32_t depth,
-                                               Prospero::TileMode tile, uint64_t upload_size,
+                                               Prospero::TileMode tile_mode, uint64_t upload_size,
                                                bool allow_depth_tile, bool volume_texture,
                                                const char* owner);
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout);
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,
-                              std::vector<GpuTileInfo>& infos);
+                              std::vector<GpuTileInfo>& out_tile_infos);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/image/tiler.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_depth_spv.h"
@@ -221,7 +222,7 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 
 	const uint64_t uniform_alignment =
 	    std::max<uint64_t>(limits.minUniformBufferOffsetAlignment, 1);
-	const uint64_t stride = (sizeof(Push) + uniform_alignment - 1) & ~(uniform_alignment - 1);
+	const uint64_t stride = Common::AlignUp<uint64_t>(sizeof(Push), uniform_alignment);
 	EXIT_NOT_IMPLEMENTED(dispatches.size() > UINT64_MAX / stride);
 	const uint64_t bytes  = dispatches.size() * stride;
 	auto [mapped, offset] = m_stream_buffer.Map(bytes, uniform_alignment);
@@ -259,12 +260,8 @@ vk::Pipeline TileManager::GetPipeline(uint32_t slot) {
 	const uint32_t                   values[] {1u << element_index, direction_index};
 	const vk::SpecializationMapEntry entries[] {{0, 0, 4}, {1, 4, 4}};
 	const vk::SpecializationInfo     specialization {2, entries, sizeof(values), values};
-	vk::ShaderModuleCreateInfo       module_info {};
-	module_info.codeSize    = shaders[family_index].words * sizeof(uint32_t);
-	module_info.pCode       = shaders[family_index].code;
-	vk::ShaderModule module = nullptr;
-	RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-	                     "create TileManager shader module");
+	const auto module =
+	    CompileSPV({shaders[family_index].code, shaders[family_index].words}, m_graphics.device);
 	vk::PipelineShaderStageCreateInfo stage {};
 	stage.stage               = vk::ShaderStageFlagBits::eCompute;
 	stage.module              = module;
@@ -287,12 +284,12 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset,
 	const auto&    limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	const uint64_t descriptor_alignment =
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
-	const uint64_t source_descriptor_offset = source_offset & ~(descriptor_alignment - 1);
-	const uint64_t target_descriptor_offset = target_offset & ~(descriptor_alignment - 1);
+	const uint64_t source_descriptor_offset = Common::AlignDown(source_offset, descriptor_alignment);
+	const uint64_t target_descriptor_offset = Common::AlignDown(target_offset, descriptor_alignment);
 	const uint64_t source_base              = source_offset - source_descriptor_offset;
 	const uint64_t target_base              = target_offset - target_descriptor_offset;
-	const uint64_t source_range             = (source_base + source_capacity + 3u) & ~uint64_t {3};
-	const uint64_t target_range             = (target_base + target_capacity + 3u) & ~uint64_t {3};
+	const uint64_t source_range             = Common::AlignUp(source_base + source_capacity, 4);
+	const uint64_t target_range             = Common::AlignUp(target_base + target_capacity, 4);
 	EXIT_NOT_IMPLEMENTED(source_range > limits.maxStorageBufferRange ||
 	                     target_range > limits.maxStorageBufferRange || target_offset % 4 != 0 ||
 	                     target_capacity % 4 != 0);
@@ -372,7 +369,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
 	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
-	auto scratch = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3});
+	auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4));
 	DeferDestroy(scratch);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
@@ -406,7 +403,7 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	// Reserve all stream parameters before creating a scheduler-lived scratch dependency:
 	// StreamBuffer::Map is allowed to submit the current tick when it wraps.
 	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
-	auto linear = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3});
+	auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4));
 	DeferDestroy(linear);
 	image.Download(regions, linear.buffer, 0, linear.size);
 	Result source {linear.buffer, 0, linear.size};
@@ -418,7 +415,7 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 }
 
 TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
-	auto scratch = AllocateScratch((size + 3u) & ~uint64_t {3});
+	auto scratch = AllocateScratch(Common::AlignUp(size, 4));
 	DeferDestroy(scratch);
 	return {scratch.buffer, 0, scratch.size};
 }
@@ -426,11 +423,11 @@ TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
 TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const {
 	const auto& limits            = m_graphics.GetPhysicalDeviceProperties().limits;
 	const auto  alignment         = std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
-	const auto  descriptor_offset = buffer.offset - buffer.offset % alignment;
+	const auto  descriptor_offset = Common::AlignDown(buffer.offset, alignment);
 	const auto  base              = buffer.offset - descriptor_offset;
 	EXIT_IF(buffer.buffer == nullptr || size == 0 || buffer.size < size || base > UINT32_MAX ||
 	        size > UINT64_MAX - base || base + size > UINT64_MAX - 3);
-	const auto range = (base + size + 3) & ~uint64_t {3};
+	const auto range = Common::AlignUp(base + size, 4);
 	EXIT_IF(range > limits.maxStorageBufferRange || range > UINT32_MAX);
 	return {{buffer.buffer, descriptor_offset, range}, static_cast<uint32_t>(base)};
 }
@@ -471,12 +468,7 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 			code  = GPU_TILER_DEMOTE_D16_SPV;
 			words = std::size(GPU_TILER_DEMOTE_D16_SPV);
 		}
-		vk::ShaderModuleCreateInfo module_info {};
-		module_info.codeSize    = words * sizeof(uint32_t);
-		module_info.pCode       = code;
-		vk::ShaderModule module = nullptr;
-		RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-		                     "create D16 conversion shader module");
+		const auto module = CompileSPV({code, words}, m_graphics.device);
 		vk::PipelineShaderStageCreateInfo stage {};
 		stage.stage               = vk::ShaderStageFlagBits::eCompute;
 		stage.module              = module;
@@ -512,8 +504,8 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	const auto target_required = required(layout.height, layout.layers, layout.target_row_stride,
 	                                      layout.target_slice_stride, target_active);
 	EXIT_IF(source_required > UINT64_MAX - 3 || target_required > UINT64_MAX - 3);
-	const auto source_barrier_size = (source_required + 3) & ~uint64_t {3};
-	const auto target_barrier_size = (target_required + 3) & ~uint64_t {3};
+	const auto source_barrier_size = Common::AlignUp(source_required, 4);
+	const auto target_barrier_size = Common::AlignUp(target_required, 4);
 	EXIT_IF(source.size < source_barrier_size || target.size < target_barrier_size);
 
 	m_scheduler.EndRendering();
@@ -610,12 +602,7 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 
 void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 	if (m_swap_bgra16 == nullptr) {
-		vk::ShaderModuleCreateInfo module_info {};
-		module_info.codeSize    = std::size(GPU_TILER_SWAP_BGRA16_SPV) * sizeof(uint32_t);
-		module_info.pCode       = GPU_TILER_SWAP_BGRA16_SPV;
-		vk::ShaderModule module = nullptr;
-		RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-		                     "create BGRA16 swap shader module");
+		const auto module = CompileSPV(GPU_TILER_SWAP_BGRA16_SPV, m_graphics.device);
 		vk::PipelineShaderStageCreateInfo stage {};
 		stage.stage  = vk::ShaderStageFlagBits::eCompute;
 		stage.module = module;

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp

@@ -20,7 +20,7 @@ constexpr std::array DescriptorPoolSizes = {
 
 DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore)
     : m_graphics(graphics), m_master_semaphore(master_semaphore) {
-	CreatePool();
+	CreateDescriptorPool();
 }
 
 DescriptorHeap::~DescriptorHeap() {
@@ -49,7 +49,7 @@ vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) {
 		m_pending_pools.pop_front();
 		EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess);
 	} else {
-		CreatePool();
+		CreateDescriptorPool();
 	}
 
 	m_sets.clear();
@@ -82,7 +82,7 @@ bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) {
 	}
 }
 
-void DescriptorHeap::CreatePool() {
+void DescriptorHeap::CreateDescriptorPool() {
 	vk::DescriptorPoolCreateInfo create {};
 	create.maxSets       = DescriptorHeapCount;
 	create.poolSizeCount = static_cast<uint32_t>(DescriptorPoolSizes.size());

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h

@@ -31,7 +31,7 @@ private:
 	};
 
 	[[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch);
-	void               CreatePool();
+	void               CreateDescriptorPool();
 
 	GraphicContext&                                     m_graphics;
 	MasterSemaphore&                                    m_master_semaphore;

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/file.h"
@@ -14,6 +15,7 @@
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/hostMemory.h"
+#include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
@@ -92,22 +94,15 @@ static const char* ShaderStageResourceName(ShaderType stage) {
 	switch (stage) {
 		case ShaderType::Vertex: return "Vertex";
 		case ShaderType::Mesh: return "Mesh";
+		case ShaderType::Local: return "Local";
+		case ShaderType::TessellationControl: return "Hull";
+		case ShaderType::TessellationEvaluation: return "Domain";
 		case ShaderType::Pixel: return "Pixel";
 		case ShaderType::Compute: return "Compute";
 		default: return "Unknown";
 	}
 }
 
-static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) {
-	switch (stage) {
-		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
-		case ShaderType::Mesh: return vk::ShaderStageFlagBits::eMeshEXT;
-		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
-		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
-		default: EXIT("unknown native shader stage\n");
-	}
-}
-
 static Prospero::ImageType TextureType(const ShaderTextureResource& descriptor) {
 	const auto type = descriptor.Type();
 	return type == Prospero::ImageType::kCube ? Prospero::ImageType::kColor2DArray : type;
@@ -145,7 +140,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	}
 	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
 	                                                              resource.formatted, id);
-	const auto aligned_offset = offset - offset % alignment;
+	const auto aligned_offset = Common::AlignDown(offset, alignment);
 	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
 	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
@@ -153,7 +148,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
 	const vk::DescriptorBufferInfo result {buffer->Handle(), aligned_offset, size + adjustment};
-	if (resource.formatted && resource.written) {
+	if (resource.written) {
 		context.GetTextureCache().InvalidateMemoryFromGPU(address, size);
 	}
 	const char* access = "Read";
@@ -169,34 +164,6 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	return result;
 }
 
-static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) {
-	bool supported_dimension = false;
-	switch (resource.dimension) {
-		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim1DArray:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
-			supported_dimension = true;
-			break;
-		default: break;
-	}
-	return resource.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled &&
-	       resource.numeric_class != Prospero::TextureNumericClass::Unsupported &&
-	       supported_dimension && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
-	       resource.read && !resource.written && !resource.atomic && !resource.depth_compare;
-}
-
-bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
-                                    const ShaderTextureResource& descriptor, const Image& image) {
-	return image.usage.video_out && image.info.resources.layers == 1 &&
-	       IsSupportedSampledColorResource(resource) &&
-	       resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	       descriptor.Type() == Prospero::ImageType::kColor2D && descriptor.Depth() == 0 &&
-	       descriptor.BaseArray5() == 0;
-}
-
 bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bool r128) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
@@ -404,30 +371,22 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	     descriptor.fields[5], descriptor.fields[6], descriptor.fields[7]);
 }
 
-struct NullImageSpec {
-	vk::Format             format;
-	Prospero::BufferFormat guest_format;
-};
-
-static NullImageSpec NullTextureSpec(const ShaderRecompiler::IR::ImageResource& resource) {
-	switch (resource.numeric_class) {
-		case Prospero::TextureNumericClass::Float:
-			return {vk::Format::eR32Sfloat, Prospero::BufferFormat::k32Float};
-		case Prospero::TextureNumericClass::Uint:
-			return {vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt};
-		case Prospero::TextureNumericClass::Sint:
-			return {vk::Format::eR32Sint, Prospero::BufferFormat::k32SInt};
-		case Prospero::TextureNumericClass::Unsupported: break;
-	}
-	EXIT("null image has unsupported numeric class\n");
-}
-
 static TextureCache::ImageDesc NullTextureDesc(const ShaderRecompiler::IR::ImageResource& resource,
                                                TextureCache::BindingType                  binding) {
-	const auto              spec = NullTextureSpec(resource);
 	TextureCache::ImageDesc desc {};
-	desc.info.pixel_format    = spec.format;
-	desc.info.guest_format    = spec.guest_format;
+	switch (resource.numeric_class) {
+		case Prospero::TextureNumericClass::Float:
+			desc.info.guest_format = Prospero::BufferFormat::k32Float;
+			break;
+		case Prospero::TextureNumericClass::Uint:
+			desc.info.guest_format = Prospero::BufferFormat::k32UInt;
+			break;
+		case Prospero::TextureNumericClass::Sint:
+			desc.info.guest_format = Prospero::BufferFormat::k32SInt;
+			break;
+		default: EXIT("null image has unsupported numeric class\n");
+	}
+	desc.info.pixel_format    = VulkanFormat(desc.info.guest_format);
 	desc.info.type            = Prospero::ImageType::kColor2D;
 	desc.info.extent          = {1, 1, 1};
 	desc.info.resources       = {1, 1};
@@ -491,17 +450,18 @@ static void PopulateTextureMipLayout(ImageInfo& info) {
 
 static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource,
                                      const ShaderTextureResource& descriptor, vk::Format format,
-                                     bool shader_conversion, bool storage, uint32_t view_levels,
-                                     uint32_t image_layers) {
+                                     const SurfaceFormatInfo& surface_format, bool storage,
+                                     uint32_t view_levels, uint32_t image_layers) {
 	ImageViewInfo view {};
 	view.format      = format;
 	view.aspect      = vk::ImageAspectFlagBits::eColor;
 	view.base_level  = descriptor.BaseLevel();
 	view.level_count = view_levels;
-	view.usage   = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
-	view.mapping = storage || shader_conversion
-	                   ? vk::ComponentMapping {}
-	                   : TextureGetComponentMapping(descriptor.DstSelXYZW());
+	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
+	view.mapping =
+	    storage || surface_format.conversion_format != Prospero::BufferFormat::kInvalid
+	        ? vk::ComponentMapping {}
+	        : TextureGetComponentMapping(descriptor.DstSelXYZW(), surface_format.host_to_storage);
 	switch (resource.dimension) {
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
 			view.type       = vk::ImageViewType::e1D;
@@ -680,7 +640,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	} else {
 		PopulateTextureMipLayout(desc.info);
 	}
-	desc.view_info = TextureViewInfo(resource, descriptor, view_format, shader_conversion, storage,
+	desc.view_info = TextureViewInfo(resource, descriptor, view_format, surface_format, storage,
 	                                 view_levels, desc.info.resources.layers);
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
 
@@ -728,13 +688,6 @@ static vk::DescriptorBufferInfo NativeUpload(RenderContext&            context,
 	return {buffer.Handle(), offset, data.size_bytes()};
 }
 
-void RenderExecutor::TrackImageBinding(ImageId id) {
-	EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr);
-	if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) {
-		m_bound_images.push_back(id);
-	}
-}
-
 void RenderExecutor::BindImage(ImageId id, bool storage) {
 	auto& image = m_context.GetTextureCache().GetImage(id);
 	if (image.info.data.Empty()) {
@@ -745,13 +698,13 @@ void RenderExecutor::BindImage(ImageId id, bool storage) {
 	}
 	image.binding.is_bound = true;
 	image.binding.shader_write |= storage;
-	TrackImageBinding(id);
+	m_bound_images.push_back(id);
 }
 
 void RenderExecutor::BindRenderTarget(ImageId id) {
 	auto& image             = m_context.GetTextureCache().GetImage(id);
 	image.binding.is_target = true;
-	TrackImageBinding(id);
+	m_bound_images.push_back(id);
 }
 
 void RenderExecutor::ResetBindings() {
@@ -769,8 +722,7 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti
 	const auto& program  = *runtime.program;
 	const auto& snapshot = runtime.resources;
 	PreparedBindings prepared;
-	prepared.program  = runtime.program;
-	prepared.snapshot = &runtime.resources;
+	prepared.runtime = &runtime;
 	prepared.images.reserve(program.info.images.size());
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]);
@@ -795,9 +747,9 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti
 
 void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
-	const auto& program  = *prepared.program;
-	const auto& snapshot = *prepared.snapshot;
+	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
+	const auto& program  = *prepared.runtime->program;
+	const auto& snapshot = prepared.runtime->resources;
 	auto&       cache    = m_context.GetBufferCache();
 
 	prepared.buffer_sources.clear();
@@ -805,10 +757,7 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[i]);
 		const auto address = descriptor.Base48();
-		const auto stride  = descriptor.Stride();
-		const auto records = descriptor.NumRecords();
-		// The descriptor has a 14-bit stride and 32-bit record count, so the product fits u64.
-		const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+		const auto requested_size = descriptor.GetSize();
 		if (address == 0 || requested_size == 0) {
 			prepared.buffer_sources.push_back({});
 			continue;
@@ -820,9 +769,9 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
 
 void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
-	const auto& program   = *prepared.program;
-	const auto& snapshot  = *prepared.snapshot;
+	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
+	const auto& program   = *prepared.runtime->program;
+	const auto& snapshot  = prepared.runtime->resources;
 	const auto& layout    = program.bindings;
 	EXIT_IF(prepared.buffer_sources.size() != program.info.buffers.size());
 
@@ -855,9 +804,9 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 
 void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
-	const auto& program  = *prepared.program;
-	const auto& snapshot = *prepared.snapshot;
+	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
+	const auto& program  = *prepared.runtime->program;
+	const auto& snapshot = prepared.runtime->resources;
 	auto&       images   = prepared.images;
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
@@ -902,32 +851,39 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	}
 }
 
-RenderExecutor::GraphicsBindings
-RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
-                                        const ShaderStageRuntime& pixel, bool pixel_active) {
-	GraphicsBindings bindings {
-	    .vertex = PrepareBindings(vertex),
-	};
-	if (pixel_active) {
-		bindings.pixel.emplace(PrepareBindings(pixel));
+void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
+                                             std::span<RenderColorInfo> colors) {
+	bool uses_dma = false;
+	for (auto* stage: stages) {
+		FindBuffers(*stage);
+		uses_dma |= stage->runtime->program->info.uses_dma;
 	}
-	FindBuffers(bindings.vertex);
-	if (bindings.pixel) {
-		FindBuffers(*bindings.pixel);
+	if (uses_dma) {
+		m_context.PrepareBda();
 	}
-	if (bindings.vertex.program->info.uses_dma ||
-	    (bindings.pixel && bindings.pixel->program->info.uses_dma)) {
-		m_context.GetGpuResources().PrepareBda();
+	for (auto* stage: stages) {
+		RebindImages(*stage);
 	}
-	RebindBuffers(bindings.vertex);
-	if (bindings.pixel) {
-		RebindBuffers(*bindings.pixel);
+	auto& cache = m_context.GetTextureCache();
+	for (auto& target: colors) {
+		EXIT_IF(!target.image_id);
+		const auto old_image = cache.m_slot_images.try_get(target.image_id);
+		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
+		    old_image->binding.needs_rebind) {
+			if (old_image != nullptr) {
+				old_image->binding = {};
+			}
+			target.desc.view_info.base_level = target.guest_mip_level;
+			target.desc.view_info.base_layer = target.guest_array_layer;
+			target.image_id = cache.FindImage(target.desc);
+			BindRenderTarget(target.image_id);
+		}
 	}
-	RebindImages(bindings.vertex);
-	if (bindings.pixel) {
-		RebindImages(*bindings.pixel);
+	// Discovery can read back PS5 metadata and submit the scheduler. Reserve draw buffers only
+	// after image identities are final; attachment layout transitions follow buffer alias copies.
+	for (auto* stage: stages) {
+		RebindBuffers(*stage);
 	}
-	return bindings;
 }
 
 void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
@@ -940,20 +896,21 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 	size_t write_count      = 0;
 	ShaderRecompiler::IR::PushData push_data;
 	bool                           has_push_data = false;
-	constexpr auto                 GraphicsStages = vk::ShaderStageFlagBits::eVertex |
-	                                                vk::ShaderStageFlagBits::eMeshEXT |
-	                                                vk::ShaderStageFlagBits::eFragment;
+	constexpr auto                 GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eMeshEXT |
+	    vk::ShaderStageFlagBits::eTessellationControl |
+	    vk::ShaderStageFlagBits::eTessellationEvaluation | vk::ShaderStageFlagBits::eFragment;
 	vk::ShaderStageFlags push_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics
 	                                       ? vk::ShaderStageFlagBits::eFragment
 	                                       : vk::ShaderStageFlags {};
 	for (const auto* prepared: prepared_bindings) {
-		EXIT_IF(prepared == nullptr || prepared->program == nullptr ||
-		        prepared->snapshot == nullptr);
-		write_count += prepared->program->bindings.descriptors.size();
-		for (const auto& binding: prepared->program->bindings.descriptors) {
+		EXIT_IF(prepared == nullptr || prepared->runtime == nullptr || !*prepared->runtime);
+		const auto& program = *prepared->runtime->program;
+		write_count += program.bindings.descriptors.size();
+		for (const auto& binding: program.bindings.descriptors) {
 			descriptor_count += NativeDescriptorCount(binding);
 		}
-		const auto shader_stage = NativeShaderStage(prepared->program->stage);
+		const auto shader_stage = NativeShaderStage(program.stage);
 		push_stages |= shader_stage;
 		EXIT_IF((pipeline_bind_point == vk::PipelineBindPoint::eGraphics &&
 		         (shader_stage & GraphicsStages) == vk::ShaderStageFlags {}) ||
@@ -968,7 +925,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 	m_descriptor_writes.reserve(write_count);
 
 	for (auto* prepared: prepared_bindings) {
-		const auto& program       = *prepared->program;
+		const auto& program       = *prepared->runtime->program;
 		auto&       descriptors   = *prepared;
 		const auto  shader_stage  = NativeShaderStage(program.stage);
 		const auto  shader_stages = ShaderPipelineStages(shader_stage);

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -16,9 +16,7 @@
 
 namespace Libs::Graphics {
 
-namespace ShaderRecompiler::IR {
-struct ResourceSnapshot;
-}
+struct ShaderStageRuntime;
 
 struct TextureBinding {
 	ImageId                    image_id;
@@ -35,8 +33,8 @@ struct PreparedBindings {
 		BufferId id;
 	};
 
-	const ShaderRecompiler::IR::CompiledShaderInfo* program  = nullptr;
-	const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
+	// The draw owns the immutable compiled-program/runtime-snapshot association through commit.
+	const ShaderStageRuntime* runtime = nullptr;
 	// Keep the resolved guest range through cache preparation; only the host buffer ID may
 	// become stale and need resolving again when bindings are rebound.
 	std::vector<BufferSource>             buffer_sources;
@@ -68,9 +66,6 @@ template <typename T>
 
 [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
                                                    bool r128 = false);
-[[nodiscard]] bool
-IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
-                               const ShaderTextureResource& descriptor, const Image& image);
 void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
                             const ShaderTextureResource& descriptor, uint64_t size);
 

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -236,6 +236,9 @@ struct PipelineCache::ProgramCache {
 		switch (options.stage) {
 			case ShaderType::Vertex: stage_name = "vs"; break;
 			case ShaderType::Mesh: stage_name = "ms"; break;
+			case ShaderType::Local: stage_name = "ls"; break;
+			case ShaderType::TessellationControl: stage_name = "hs"; break;
+			case ShaderType::TessellationEvaluation: stage_name = "ds"; break;
 			case ShaderType::Pixel: stage_name = "ps"; break;
 			case ShaderType::Compute: stage_name = "cs"; break;
 			default: EXIT("invalid pipeline shader stage\n");
@@ -250,18 +253,9 @@ struct PipelineCache::ProgramCache {
 		}
 		DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
 
-		vk::ShaderModuleCreateInfo create_info {};
-		create_info.codeSize    = result.spirv.size() * sizeof(uint32_t);
-		create_info.pCode       = result.spirv.data();
-		vk::ShaderModule module = nullptr;
-		RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module),
-		                     "create recompiled shader module");
+		const auto module = CompileSPV(result.spirv, device);
 		EXIT_IF(module == nullptr);
 		if (options.dump_ir) {
-			if (!options.early_dump) {
-				LOGF("%s decoded RDNA2:\n%s", options.dump_label, result.decoded_dump.c_str());
-				LOGF("%s IR:\n%s", options.dump_label, result.ir_dump.c_str());
-			}
 			LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", options.dump_label,
 			     static_cast<uint64_t>(result.spirv.size()), options.wave_size);
 		}
@@ -277,7 +271,7 @@ struct PipelineCache::ProgramCache {
 	                  uint32_t& push_data_cursor) {
 		ShaderType stage;
 		if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
-			stage = input_info.mesh.threads_num[0] != 0 ? ShaderType::Mesh : ShaderType::Vertex;
+			stage = input_info.logical_stage;
 		} else if constexpr (std::is_same_v<InputInfo, ShaderPixelInputInfo>) {
 			stage = ShaderType::Pixel;
 		} else {
@@ -329,6 +323,9 @@ struct PipelineCache::ProgramCache {
 		switch (stage) {
 			case ShaderType::Vertex: label = "ShaderRecompiler VS"; break;
 			case ShaderType::Mesh: label = "ShaderRecompiler MS"; break;
+			case ShaderType::Local: label = "ShaderRecompiler LS"; break;
+			case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break;
+			case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break;
 			case ShaderType::Pixel: label = "ShaderRecompiler PS"; break;
 			case ShaderType::Compute: label = "ShaderRecompiler CS"; break;
 			default: EXIT("invalid pipeline shader stage\n");
@@ -338,19 +335,18 @@ struct PipelineCache::ProgramCache {
 		options.shader_hash = params.hash;
 		options.user_data   = params.user_data;
 		options.back_code      = params.back_code;
-		options.dump_ir     = Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent;
+		options.dump_ir     = Config::GetShaderLogDirection() != Config::LogDirection::Silent;
 		options.early_dump  = options.dump_ir;
 		options.dump_label  = label;
 		options.input_info  = stage_input;
-		options.scratch_dwords = input_info.scratch_size_dwords;
+
 		if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
 			options.user_data_base = 8;
-			if (stage == ShaderType::Mesh) {
+			if (stage == ShaderType::Mesh || stage == ShaderType::TessellationControl) {
 				options.user_data_base = 0;
-				options.wave_size      = input_info.mesh.wave_size;
-				options.scratch_dwords = input_info.mesh.scratch_size_dwords;
+				options.wave_size = stage == ShaderType::Mesh ? input_info.mesh.wave_size : 64u;
 			}
-		} else if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
+		} else {
 			options.wave_size = input_info.wave_size;
 		}
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
@@ -366,16 +362,19 @@ struct PipelineCache::ProgramCache {
 		input_info.stage = {.program = &permutation.program, .resources = std::move(resources)};
 		permutation.program.bindings.AdvancePushData(push_data_cursor);
 
-		std::array<size_t, static_cast<size_t>(ShaderType::Mesh) + 1> counts {};
+		std::array<size_t, static_cast<size_t>(ShaderType::TessellationEvaluation) + 1> counts {};
 		for (const auto& [key, source]: programs) {
 			counts[static_cast<size_t>(key.stage)] += source.permutations.size();
 		}
 		// Guest geometry shaders are compiled through the host mesh stage.
-		std::printf("Shaders: VS %zu | PS %zu | CS %zu | GS %zu\n",
+		std::printf("Shaders: VS %zu | PS %zu | CS %zu | GS %zu | LS %zu | HS %zu | TES %zu\n",
 		            counts[static_cast<size_t>(ShaderType::Vertex)],
 		            counts[static_cast<size_t>(ShaderType::Pixel)],
 		            counts[static_cast<size_t>(ShaderType::Compute)],
-		            counts[static_cast<size_t>(ShaderType::Mesh)]);
+		            counts[static_cast<size_t>(ShaderType::Mesh)],
+		            counts[static_cast<size_t>(ShaderType::Local)],
+		            counts[static_cast<size_t>(ShaderType::TessellationControl)],
+		            counts[static_cast<size_t>(ShaderType::TessellationEvaluation)]);
 		return permutation.handle;
 	}
 
@@ -569,12 +568,18 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
     const HW::VertexShaderInfo& vertex_regs, const HW::PixelShaderInfo& pixel_regs,
     const HW::ShaderRegisters& sh, const HW::Context& context, const HW::UserConfig& user_config,
     std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping, bool pixel_active,
-    ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info) {
-	const auto vertex_params = PrepareProgram(vertex_regs, context, user_config, vertex_info);
-	const bool mesh_active   = vertex_info.mesh.threads_num[0] != 0;
+    std::array<ShaderVertexInputInfo, 3>& vertex_info, ShaderPixelInputInfo& pixel_info) {
+	const bool tess_active = user_config.GetPrimType() == Prospero::PrimitiveType::kPatch;
+	std::array<ShaderParams, 3> vertex_params;
+	if (tess_active) {
+		vertex_params = PrepareTessellationPrograms(vertex_regs, context, vertex_info);
+	} else {
+		vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]);
+	}
+	const bool mesh_active = vertex_info[0].logical_stage == ShaderType::Mesh;
 	if (mesh_active) {
 		EXIT_NOT_IMPLEMENTED(!m_graphics.mesh_shader_enabled);
-		auto& mesh              = vertex_info.mesh;
+		auto& mesh              = vertex_info[0].mesh;
 		mesh.host_subgroup_size = m_graphics.subgroup_size;
 		const auto& limits      = m_graphics.mesh_shader_properties;
 		const auto  logical_threads =
@@ -597,7 +602,7 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 	if (context.GetClipControl().clip_disable) {
 		const auto& viewport = context.GetScreenViewport().viewports[0];
 		const auto& limits   = m_graphics.GetPhysicalDeviceProperties().limits;
-		auto&       clip     = vertex_info.clip_space;
+		auto&       clip     = vertex_info[tess_active ? 2u : 0u].clip_space;
 		clip.scale[0]        = viewport.xscale;
 		clip.scale[1]        = viewport.yscale;
 		clip.offset[0]       = viewport.xoffset;
@@ -615,7 +620,9 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 	if (pixel_active) {
 		result.pixel = m_program_cache->Get(pixel_params, pixel_info, push_data_cursor);
 	}
-	result.vertex = m_program_cache->Get(vertex_params, vertex_info, push_data_cursor);
+	for (uint32_t i = 0; i < (tess_active ? 3u : 1u); i++) {
+		result.vertex[i] = m_program_cache->Get(vertex_params[i], vertex_info[i], push_data_cursor);
+	}
 	return result;
 }
 
@@ -633,12 +640,14 @@ bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other)
 	return std::memcmp(this, &other, sizeof(*this)) == 0;
 }
 
-PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
+PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
     std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
-    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+    std::span<const ShaderVertexInputInfo> vertex_info, CommandBuffer& command,
     const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
-    bool primitive_restart_enable, const ShaderProgram& vertex_program,
-    const ShaderProgram& pixel_program) {
+    bool primitive_restart_enable, const GraphicsPrograms& programs) {
+	const auto& vs_input_info  = vertex_info.front();
+	const auto& vertex_program = programs.vertex[0];
+	const auto& pixel_program  = programs.pixel;
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200);
 
 	EXIT_IF(colors.size() > RENDER_COLOR_ATTACHMENTS_MAX);
@@ -656,7 +665,9 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 	const auto ps_id = ps_active ? pixel_program.id : 0;
 
 	GraphicsPipelineKey key {};
-	key.vs_shader_id            = vs_id;
+	for (uint32_t i = 0; i < programs.vertex.size(); i++) {
+		key.vertex_shader_ids[i] = programs.vertex[i].id;
+	}
 	key.ps_shader_id            = ps_id;
 	auto& static_params         = key.static_params;
 	auto& rendering             = key.rendering;
@@ -673,6 +684,16 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 			EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
 			     attachment_samples, colors[i].desc.info.samples);
 		}
+		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
+		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
+		static_params.color_srcblend[i]       = bc.color_srcblend;
+		static_params.color_comb_fcn[i]       = bc.color_comb_fcn;
+		static_params.color_destblend[i]      = bc.color_destblend;
+		static_params.alpha_srcblend[i]       = bc.alpha_srcblend;
+		static_params.alpha_comb_fcn[i]       = bc.alpha_comb_fcn;
+		static_params.alpha_destblend[i]      = bc.alpha_destblend;
+		static_params.separate_alpha_blend[i] = bc.separate_alpha_blend;
+		static_params.blend_enable[i]         = bc.enable && !rt.info.blend_bypass;
 	}
 	const bool with_depth =
 	    depth.desc.view_info.format != vk::Format::eUndefined && static_cast<bool>(depth.image_id);
@@ -721,29 +742,15 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 	static_params.depth_bounds_test_enable = depth.depth_bounds_test_enable;
 	static_params.depth_min_bounds         = depth.depth_min_bounds;
 	static_params.depth_max_bounds         = depth.depth_max_bounds;
-	static_params.stencil_test_enable      = depth.stencil_test_enable;
-	static_params.stencil_front            = depth.stencil_static_front;
-	static_params.stencil_back             = depth.stencil_static_back;
-	const bool rect_list     = topology == vk::PrimitiveTopology::ePatchList;
+	const bool rect_list =
+	    command.GetUserConfig().GetPrimType() == Prospero::PrimitiveType::kRectList;
 	static_params.cull_back  = !rect_list && mc.cull_back;
 	static_params.cull_front = !rect_list && mc.cull_front;
 	static_params.face       = mc.face;
+	static_params.provoking_vtx_last = mc.provoking_vtx_last;
 	static_params.polygon_mode =
 	    ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back);
 
-	for (uint32_t i = 0; i < color_count; i++) {
-		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
-		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		static_params.color_srcblend[i]       = bc.color_srcblend;
-		static_params.color_comb_fcn[i]       = bc.color_comb_fcn;
-		static_params.color_destblend[i]      = bc.color_destblend;
-		static_params.alpha_srcblend[i]       = bc.alpha_srcblend;
-		static_params.alpha_comb_fcn[i]       = bc.alpha_comb_fcn;
-		static_params.alpha_destblend[i]      = bc.alpha_destblend;
-		static_params.separate_alpha_blend[i] = bc.separate_alpha_blend;
-		static_params.blend_enable[i]         = bc.enable;
-		static_params.blend_bypass[i]         = rt.info.blend_bypass;
-	}
 	if (vs_input_info.stage.program->stage != ShaderType::Mesh) {
 		EXIT_IF(vs_input_info.buffers_num < 0 ||
 		        vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX ||
@@ -787,9 +794,8 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 
 	auto cached = std::make_unique<Pipeline>();
 	LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id);
-	CreatePipelineInternal(m_graphics, *cached, rendering, key.vertex_input, vs_input_info,
-	                       vertex_program, ps_input_info, pixel_program, static_params,
-	                       m_driver_cache);
+	CreatePipelineInternal(m_graphics, *cached, rendering, key.vertex_input, vertex_info,
+	                       ps_input_info, programs, static_params, m_driver_cache);
 	LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
@@ -802,8 +808,8 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 }
 
 PipelineCache::Pipeline&
-PipelineCache::CreateComputePipeline(const ShaderComputeInputInfo& input_info,
-                                     const ShaderProgram&          compute_program) {
+PipelineCache::GetComputePipeline(const ShaderComputeInputInfo& input_info,
+                                  const ShaderProgram&          compute_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100);
 
 	EXIT_IF(!compute_program);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -42,13 +42,11 @@ struct PipelineStaticParameters {
 	bool                       depth_bounds_test_enable = false;
 	float                      depth_min_bounds         = 0.0f;
 	float                      depth_max_bounds         = 0.0f;
-	bool                       stencil_test_enable      = false;
-	PipelineStencilStaticState stencil_front;
-	PipelineStencilStaticState stencil_back;
 	uint32_t                   color_mask[RENDER_COLOR_ATTACHMENTS_MAX]           = {};
 	bool                       cull_front                                         = false;
 	bool                       cull_back                                          = false;
 	bool                       face                                               = false;
+	bool                       provoking_vtx_last                                 = false;
 	vk::PolygonMode            polygon_mode                                       = vk::PolygonMode::eFill;
 	uint8_t                    color_srcblend[RENDER_COLOR_ATTACHMENTS_MAX]       = {};
 	uint8_t                    color_comb_fcn[RENDER_COLOR_ATTACHMENTS_MAX]       = {};
@@ -58,7 +56,6 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
-	bool                       blend_bypass[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
 
 	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
@@ -68,7 +65,7 @@ struct PipelineStaticParameters {
 static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
 static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 165);
+static_assert(sizeof(PipelineStaticParameters) == 125);
 
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};
@@ -121,8 +118,10 @@ public:
 	};
 
 	struct GraphicsPrograms {
-		ShaderProgram vertex;
+		std::array<ShaderProgram, 3> vertex;
 		ShaderProgram pixel;
+
+		[[nodiscard]] uint32_t VertexStageCount() const { return vertex[1] ? 3u : 1u; }
 	};
 
 	GraphicsPrograms
@@ -130,33 +129,33 @@ public:
 	                    const HW::PixelShaderInfo& pixel_regs, const HW::ShaderRegisters& sh,
 	                    const HW::Context& context, const HW::UserConfig& user_config,
 	                    std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
-	                    bool pixel_active, ShaderVertexInputInfo& vertex_info,
+	                    bool pixel_active, std::array<ShaderVertexInputInfo, 3>& vertex_info,
 	                    ShaderPixelInputInfo& pixel_info);
 	ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs,
 	                                const HW::ShaderRegisters&   sh,
 	                                ShaderComputeInputInfo&      input_info);
 
-	Pipeline&
-	CreateGraphicsPipeline(std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
-	                       const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
-	                       const ShaderPixelInputInfo* ps_input_info,
-	                       vk::PrimitiveTopology topology, bool primitive_restart_enable,
-	                       const ShaderProgram& vertex_program, const ShaderProgram& pixel_program);
-	Pipeline& CreateComputePipeline(const ShaderComputeInputInfo& input_info,
-	                                const ShaderProgram&          compute_program);
+	Pipeline& GetGraphicsPipeline(std::span<const RenderColorInfo>       colors,
+	                              const RenderDepthInfo&                 depth,
+	                              std::span<const ShaderVertexInputInfo> vertex_info,
+	                              CommandBuffer& command, const ShaderPixelInputInfo* ps_input_info,
+	                              vk::PrimitiveTopology topology, bool primitive_restart_enable,
+	                              const GraphicsPrograms& programs);
+	Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info,
+	                             const ShaderProgram&          compute_program);
 
 private:
 	struct ProgramCache;
 
 	struct GraphicsPipelineKey {
 		PipelineRenderingState   rendering;
-		uint64_t                 vs_shader_id = 0;
+		std::array<uint64_t, 3>  vertex_shader_ids {};
 		uint64_t                 ps_shader_id = 0;
 		PipelineVertexInputState vertex_input;
 		PipelineStaticParameters static_params;
 
 		bool operator==(const GraphicsPipelineKey& other) const {
-			return rendering == other.rendering && vs_shader_id == other.vs_shader_id &&
+			return rendering == other.rendering && vertex_shader_ids == other.vertex_shader_ids &&
 			       ps_shader_id == other.ps_shader_id && vertex_input == other.vertex_input &&
 			       static_params == other.static_params;
 		}
@@ -189,7 +188,9 @@ private:
 		std::size_t operator()(const GraphicsPipelineKey& key) const {
 			std::size_t hash = 0;
 			PipelineKeyHash::MixRendering(hash, key.rendering);
-			PipelineKeyHash::Mix(hash, key.vs_shader_id);
+			for (const auto id: key.vertex_shader_ids) {
+				PipelineKeyHash::Mix(hash, id);
+			}
 			PipelineKeyHash::Mix(hash, key.ps_shader_id);
 			PipelineKeyHash::Mix(hash, key.vertex_input.binding_count);
 			for (uint32_t i = 0; i < key.vertex_input.binding_count; i++) {
@@ -219,12 +220,14 @@ private:
 };
 
 void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id);
-void CreatePipelineInternal(
-    GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-    const PipelineRenderingState& rendering, const PipelineVertexInputState& vertex_input,
-    const ShaderVertexInputInfo& vs_input_info, const ShaderProgram& vertex_program,
-    const ShaderPixelInputInfo* ps_input_info, const ShaderProgram& pixel_program,
-    const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                            const PipelineRenderingState&          rendering,
+                            const PipelineVertexInputState&        vertex_input,
+                            std::span<const ShaderVertexInputInfo> vertex_info,
+                            const ShaderPixelInputInfo*            ps_input_info,
+                            const PipelineCache::GraphicsPrograms& programs,
+                            const PipelineStaticParameters&        static_params,
+                            vk::PipelineCache                      driver_cache);
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
                             const ShaderComputeInputInfo& input_info,
                             vk::ShaderModule compute_module, vk::PipelineCache driver_cache);

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp

@@ -8,6 +8,20 @@
 
 namespace Libs::Graphics {
 
+vk::ShaderStageFlagBits NativeShaderStage(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Local:
+		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
+		case ShaderType::Mesh: return vk::ShaderStageFlagBits::eMeshEXT;
+		case ShaderType::TessellationControl: return vk::ShaderStageFlagBits::eTessellationControl;
+		case ShaderType::TessellationEvaluation:
+			return vk::ShaderStageFlagBits::eTessellationEvaluation;
+		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
+		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
+		default: EXIT("unknown native shader stage\n");
+	}
+}
+
 vk::PipelineStageFlags ShaderPipelineStages(vk::ShaderStageFlags stages) {
 	vk::PipelineStageFlags result = {};
 	if (stages & vk::ShaderStageFlagBits::eVertex) {
@@ -16,6 +30,12 @@ vk::PipelineStageFlags ShaderPipelineStages(vk::ShaderStageFlags stages) {
 	if (stages & vk::ShaderStageFlagBits::eMeshEXT) {
 		result |= vk::PipelineStageFlagBits::eMeshShaderEXT;
 	}
+	if (stages & vk::ShaderStageFlagBits::eTessellationControl) {
+		result |= vk::PipelineStageFlagBits::eTessellationControlShader;
+	}
+	if (stages & vk::ShaderStageFlagBits::eTessellationEvaluation) {
+		result |= vk::PipelineStageFlagBits::eTessellationEvaluationShader;
+	}
 	if (stages & vk::ShaderStageFlagBits::eFragment) {
 		result |= vk::PipelineStageFlagBits::eFragmentShader;
 	}

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h

@@ -8,6 +8,7 @@ namespace Libs::Graphics {
 
 struct ShaderStageRuntime;
 
+vk::ShaderStageFlagBits NativeShaderStage(ShaderType stage);
 vk::PipelineStageFlags  ShaderPipelineStages(vk::ShaderStageFlags stages);
 VulkanMemoryBarrier     MakeShaderAccessDependency();
 VulkanMemoryBarrier     MakeShaderWriteHazardDependency();

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -8,10 +8,12 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
@@ -115,227 +117,13 @@ static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format,
 		}
 		format = vk::Format::eR16G16Sfloat;
 		size   = 2;
-		if (NarrowInputFormat(format, size, used_components)) {
-			LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", raw_format,
-			     vk::to_string(format).c_str(), used_components);
-		}
 		return;
 	}
 
-	switch (fmt) {
-		case Prospero::BufferFormat::k32_32_32_32Float:
-			format = vk::Format::eR32G32B32A32Sfloat;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32_32SInt:
-			format = vk::Format::eR32G32B32A32Sint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32_32UInt:
-			format = vk::Format::eR32G32B32A32Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32Float:
-			format = vk::Format::eR32G32B32Sfloat;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k32_32_32SInt:
-			format = vk::Format::eR32G32B32Sint;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k32_32_32UInt:
-			format = vk::Format::eR32G32B32Uint;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16Float:
-			format = vk::Format::eR16G16B16A16Sfloat;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SInt:
-			format = vk::Format::eR16G16B16A16Sint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UInt:
-			format = vk::Format::eR16G16B16A16Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SScaled:
-			format = vk::Format::eR16G16B16A16Sscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UScaled:
-			format = vk::Format::eR16G16B16A16Uscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SNorm:
-			format = vk::Format::eR16G16B16A16Snorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UNorm:
-			format = vk::Format::eR16G16B16A16Unorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32Float:
-			format = vk::Format::eR32G32Sfloat;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32_32SInt:
-			format = vk::Format::eR32G32Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32_32UInt:
-			format = vk::Format::eR32G32Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UInt:
-			format = vk::Format::eR8G8B8A8Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8SScaled:
-			format = vk::Format::eR8G8B8A8Sscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UScaled:
-			format = vk::Format::eR8G8B8A8Uscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8SNorm:
-			format = vk::Format::eR8G8B8A8Snorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UNorm:
-			format = vk::Format::eR8G8B8A8Unorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k10_10_10_2UNorm:
-			format = vk::Format::eA2B10G10R10UnormPack32;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k10_10_10_2SNorm:
-			format = vk::Format::eA2B10G10R10SnormPack32;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16Float:
-			format = vk::Format::eR16G16Sfloat;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SInt:
-			format = vk::Format::eR16G16Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UInt:
-			format = vk::Format::eR16G16Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SScaled:
-			format = vk::Format::eR16G16Sscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UScaled:
-			format = vk::Format::eR16G16Uscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SNorm:
-			format = vk::Format::eR16G16Snorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UNorm:
-			format = vk::Format::eR16G16Unorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32Float:
-			format = vk::Format::eR32Sfloat;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k32SInt:
-			format = vk::Format::eR32Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k32UInt:
-			format = vk::Format::eR32Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8_8SInt:
-			format = vk::Format::eR8G8Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UInt:
-			format = vk::Format::eR8G8Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8SScaled:
-			format = vk::Format::eR8G8Sscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UScaled:
-			format = vk::Format::eR8G8Uscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8SNorm:
-			format = vk::Format::eR8G8Snorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UNorm:
-			format = vk::Format::eR8G8Unorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16Float:
-			format = vk::Format::eR16Sfloat;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SInt:
-			format = vk::Format::eR16Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UInt:
-			format = vk::Format::eR16Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SScaled:
-			format = vk::Format::eR16Sscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UScaled:
-			format = vk::Format::eR16Uscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SNorm:
-			format = vk::Format::eR16Snorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UNorm:
-			format = vk::Format::eR16Unorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SInt:
-			format = vk::Format::eR8Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UInt:
-			format = vk::Format::eR8Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SScaled:
-			format = vk::Format::eR8Sscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UScaled:
-			format = vk::Format::eR8Uscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SNorm:
-			format = vk::Format::eR8Snorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UNorm:
-			format = vk::Format::eR8Unorm;
-			size   = 1;
-			break;
-		default:
-			EXIT("unknown format: fmt = %u\n", raw_format);
-			format = vk::Format::eUndefined;
-			size   = 4;
-			break;
+	format = VulkanFormat(fmt);
+	size   = ShaderRecompiler::Format::GetFormatInfo(fmt).component_count;
+	if (format == vk::Format::eUndefined || size == 0) {
+		EXIT("unknown vertex format: fmt = %u\n", raw_format);
 	}
 
 	if (NarrowInputFormat(format, size, used_components)) {
@@ -418,12 +206,18 @@ static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipe
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void CreatePipelineInternal(
-    GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-    const PipelineRenderingState& rendering, const PipelineVertexInputState& vertex_input,
-    const ShaderVertexInputInfo& vs_input_info, const ShaderProgram& vertex_program,
-    const ShaderPixelInputInfo* ps_input_info, const ShaderProgram& pixel_program,
-    const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache) {
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                            const PipelineRenderingState&          rendering,
+                            const PipelineVertexInputState&        vertex_input,
+                            std::span<const ShaderVertexInputInfo> vertex_info,
+                            const ShaderPixelInputInfo*            ps_input_info,
+                            const PipelineCache::GraphicsPrograms& programs,
+                            const PipelineStaticParameters&        static_params,
+                            vk::PipelineCache                      driver_cache) {
+	const auto& vs_input_info  = vertex_info.front();
+	const auto& vertex_program = programs.vertex[0];
+	const auto& pixel_program  = programs.pixel;
+	const bool  tessellation   = vertex_info.size() == 3;
 	const bool ps_active = ps_input_info != nullptr;
 	EXIT_IF(!vertex_program || (ps_active && !pixel_program));
 	const bool with_depth = rendering.depth_format != vk::Format::eUndefined ||
@@ -431,72 +225,53 @@ void CreatePipelineInternal(
 	EXIT_IF(!vs_input_info.stage);
 	const bool mesh = vs_input_info.stage.program->stage == ShaderType::Mesh;
 	EXIT_NOT_IMPLEMENTED(mesh && !graphics.mesh_shader_enabled);
-	const auto vertex_stage =
-	    mesh ? vk::ShaderStageFlagBits::eMeshEXT : vk::ShaderStageFlagBits::eVertex;
-
-	const bool rect_list = !mesh && static_params.topology == vk::PrimitiveTopology::ePatchList;
+	const bool rect_list =
+	    !mesh && !tessellation && static_params.topology == vk::PrimitiveTopology::ePatchList;
 
 	vk::ShaderModule tess_control_shader_module = nullptr;
 	vk::ShaderModule tess_eval_shader_module    = nullptr;
 
-	vk::ShaderModuleCreateInfo create_info {};
-	vk::Result result {};
 	if (rect_list) {
 		const auto shaders =
 		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
-		create_info.codeSize = shaders.control.size() * 4;
-		create_info.pCode    = shaders.control.data();
-		result =
-		    graphics.device.createShaderModule(&create_info, nullptr, &tess_control_shader_module);
+		tess_control_shader_module = CompileSPV(shaders.control, graphics.device);
 		if (graphics_debug_dump_enabled()) {
-			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done result=%s module=%p\n",
-			     vk::to_string(result).c_str(), static_cast<void*>(tess_control_shader_module));
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done module=%p\n",
+			     static_cast<void*>(tess_control_shader_module));
 		}
-		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
-		create_info.codeSize = shaders.evaluation.size() * 4;
-		create_info.pCode    = shaders.evaluation.data();
-		result =
-		    graphics.device.createShaderModule(&create_info, nullptr, &tess_eval_shader_module);
+		tess_eval_shader_module = CompileSPV(shaders.evaluation, graphics.device);
 		if (graphics_debug_dump_enabled()) {
-			LOGF("PipelineTrace: vkCreateShaderModule RectList TES done result=%s module=%p\n",
-			     vk::to_string(result).c_str(), static_cast<void*>(tess_eval_shader_module));
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TES done module=%p\n",
+			     static_cast<void*>(tess_eval_shader_module));
 		}
-		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	}
 
 	EXIT_NOT_IMPLEMENTED(
 	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
 
-	vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
-	vert_shader_stage_info.stage  = vertex_stage;
-	vert_shader_stage_info.module = vertex_program.module;
-	vert_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
-	frag_shader_stage_info.stage  = vk::ShaderStageFlagBits::eFragment;
-	frag_shader_stage_info.module = pixel_program.module;
-	frag_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
-	tess_control_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationControl;
-	tess_control_shader_stage_info.module = tess_control_shader_module;
-	tess_control_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo tess_eval_shader_stage_info {};
-	tess_eval_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationEvaluation;
-	tess_eval_shader_stage_info.module = tess_eval_shader_module;
-	tess_eval_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo shader_stages[4]   = {};
+	vk::PipelineShaderStageCreateInfo shader_stages[4] {};
 	uint32_t                          shader_stage_count = 0;
-	shader_stages[shader_stage_count++]                  = vert_shader_stage_info;
+	for (uint32_t i = 0; i < vertex_info.size(); i++) {
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           NativeShaderStage(vertex_info[i].logical_stage),
+		                                       .module = programs.vertex[i].module,
+		                                       .pName  = "main"};
+	}
 	if (rect_list) {
-		shader_stages[shader_stage_count++] = tess_control_shader_stage_info;
-		shader_stages[shader_stage_count++] = tess_eval_shader_stage_info;
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           vk::ShaderStageFlagBits::eTessellationControl,
+		                                       .module = tess_control_shader_module,
+		                                       .pName  = "main"};
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           vk::ShaderStageFlagBits::eTessellationEvaluation,
+		                                       .module = tess_eval_shader_module,
+		                                       .pName  = "main"};
 	}
 	if (ps_active) {
-		shader_stages[shader_stage_count++] = frag_shader_stage_info;
+		shader_stages[shader_stage_count++] = {.stage  = vk::ShaderStageFlagBits::eFragment,
+		                                       .module = pixel_program.module,
+		                                       .pName  = "main"};
 	}
 
 	vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX] {};
@@ -595,6 +370,15 @@ void CreatePipelineInternal(
 #if !defined(__APPLE__)
 	rasterizer.pNext = &clip_ext;
 #endif
+	vk::PipelineRasterizationProvokingVertexStateCreateInfoEXT provoking_vertex {};
+	EXIT_NOT_IMPLEMENTED(static_params.provoking_vtx_last &&
+	                     !graphics.provoking_vertex_last_enabled);
+	if (graphics.provoking_vertex_last_enabled) {
+		provoking_vertex.provokingVertexMode = static_params.provoking_vtx_last
+		    ? vk::ProvokingVertexModeEXT::eLastVertex : vk::ProvokingVertexModeEXT::eFirstVertex;
+		provoking_vertex.pNext = rasterizer.pNext;
+		rasterizer.pNext = &provoking_vertex;
+	}
 	rasterizer.cullMode  = cull_mode;
 	rasterizer.frontFace = front_face;
 	rasterizer.polygonMode = static_params.polygon_mode;
@@ -610,8 +394,7 @@ void CreatePipelineInternal(
 		EXIT_NOT_IMPLEMENTED((static_params.color_mask[i] & ~0x0fu) != 0);
 		color_blend_attachment[i].colorWriteMask =
 		    vk::ColorComponentFlags {static_params.color_mask[i]};
-		color_blend_attachment[i].blendEnable =
-		    (static_params.blend_enable[i] && !static_params.blend_bypass[i]) ? VK_TRUE : VK_FALSE;
+		color_blend_attachment[i].blendEnable = static_params.blend_enable[i] ? VK_TRUE : VK_FALSE;
 		color_blend_attachment[i].srcColorBlendFactor =
 		    GetBlendFactor(static_params.color_srcblend[i]);
 		color_blend_attachment[i].dstColorBlendFactor =
@@ -649,15 +432,19 @@ void CreatePipelineInternal(
 	color_blending.pAttachments    = color_blend_attachment;
 
 	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
-	AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program, vertex_stage);
+	vk::ShaderStageFlags graphics_stages = vk::ShaderStageFlagBits::eFragment;
+	for (const auto& stage: vertex_info) {
+		const auto native_stage = NativeShaderStage(stage.logical_stage);
+		AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage);
+		graphics_stages |= native_stage;
+	}
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
 		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
 		                  vk::ShaderStageFlagBits::eFragment);
 	}
 	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
-	const auto                  GraphicsStages = vertex_stage | vk::ShaderStageFlagBits::eFragment;
-	const vk::PushConstantRange push_constants {GraphicsStages, 0,
+	const vk::PushConstantRange push_constants {graphics_stages, 0,
 	                                            ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
@@ -673,8 +460,8 @@ void CreatePipelineInternal(
 		     " set_layouts=1 push_constants=%" PRIu32 "\n",
 		     vertex_program.id, ps_active ? pixel_program.id : 0, 1u);
 	}
-	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
-	                                              &pipeline.pipeline_layout);
+	auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
+	                                                   &pipeline.pipeline_layout);
 	if (graphics_debug_dump_enabled()) {
 		LOGF("PipelineTrace: vkCreatePipelineLayout done result=%s layout=%p\n",
 		     vk::to_string(result).c_str(), static_cast<void*>(pipeline.pipeline_layout));
@@ -690,15 +477,6 @@ void CreatePipelineInternal(
 #else
 	    (static_params.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
 #endif
-	depth_stencil_info.stencilTestEnable = (static_params.stencil_test_enable ? VK_TRUE : VK_FALSE);
-	depth_stencil_info.front.failOp      = static_params.stencil_front.failOp;
-	depth_stencil_info.front.passOp      = static_params.stencil_front.passOp;
-	depth_stencil_info.front.depthFailOp = static_params.stencil_front.depthFailOp;
-	depth_stencil_info.front.compareOp   = static_params.stencil_front.compareOp;
-	depth_stencil_info.back.failOp       = static_params.stencil_back.failOp;
-	depth_stencil_info.back.passOp       = static_params.stencil_back.passOp;
-	depth_stencil_info.back.depthFailOp  = static_params.stencil_back.depthFailOp;
-	depth_stencil_info.back.compareOp    = static_params.stencil_back.compareOp;
 	depth_stencil_info.minDepthBounds    = static_params.depth_min_bounds;
 	depth_stencil_info.maxDepthBounds    = static_params.depth_max_bounds;
 
@@ -711,6 +489,8 @@ void CreatePipelineInternal(
 	    vk::DynamicState::eDepthCompareOp,
 	    vk::DynamicState::eDepthBiasEnable,
 	    vk::DynamicState::eDepthBias,
+	    vk::DynamicState::eStencilTestEnable,
+	    vk::DynamicState::eStencilOp,
 	    vk::DynamicState::eStencilCompareMask,
 	    vk::DynamicState::eStencilReference,
 	    vk::DynamicState::eStencilWriteMask,
@@ -741,8 +521,9 @@ void CreatePipelineInternal(
 	pipeline_info.pVertexInputState        = mesh ? nullptr : &vertex_input_info;
 	pipeline_info.pInputAssemblyState      = mesh ? nullptr : &input_assembly;
 	vk::PipelineTessellationStateCreateInfo tessellation_state {};
-	tessellation_state.patchControlPoints = 3;
-	pipeline_info.pTessellationState      = (rect_list ? &tessellation_state : nullptr);
+	tessellation_state.patchControlPoints =
+	    tessellation ? vs_input_info.tess.input_control_points : 3u;
+	pipeline_info.pTessellationState = (rect_list || tessellation) ? &tessellation_state : nullptr;
 	pipeline_info.pViewportState          = &viewport_state;
 	pipeline_info.pRasterizationState     = &rasterizer;
 	pipeline_info.pMultisampleState       = &multisampling;

src/graphics/host_gpu/renderer/render.h

@@ -170,38 +170,34 @@ private:
 	void DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const DrawAutoArgs& args);
 
 	struct GraphicsBindings {
-		PreparedBindings                vertex;
+		std::array<PreparedBindings, 3> vertex;
 		std::optional<PreparedBindings> pixel;
 	};
 
 	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	                                            const ShaderRecompiler::IR::DescriptorValue& value);
-	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
-	                                                       const ShaderStageRuntime& pixel,
-	                                                       bool                      pixel_active);
-	void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer& buffer,
-	                              RenderColorInfo& target, uint32_t render_target_slice_offset = 0,
-	                              uint32_t render_target_slot = UINT32_MAX,
+	void PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
+	                             std::span<RenderColorInfo> colors);
+	void ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& target,
+	                              uint32_t render_target_slice_offset, uint32_t render_target_slot,
 	                              bool ignore_target_mask = false, bool exact_format = false);
-	void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer& buffer,
-	                              RenderDepthInfo& target);
-	[[nodiscard]] bool PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& buffer,
+	void ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepthInfo& target);
+	[[nodiscard]] bool DepthStencilCopy(CommandBuffer& buffer);
+	[[nodiscard]] bool PrepareDrawRenderState(CommandBuffer& buffer,
 	                                          const DrawCallInfo& draw,
 	                                          uint32_t            render_target_slice_offset,
-	                                          bool log_setup_phases, DrawRenderState& state);
+	                                          DrawRenderState& state);
 	void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer, const DrawCallInfo& draw,
 	                         DrawRenderState& state, vk::PrimitiveTopology topology,
 	                         const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
-	                         bool primitive_restart_enable, bool log_pipeline_phase,
-	                         bool set_bind_debug, bool set_auto_debug);
+	                         bool primitive_restart_enable);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth,
 	                                               const std::optional<PreparedBindings>& pixel = std::nullopt);
-	[[nodiscard]] bool        ResolveColorTargets(uint64_t submit_id, CommandBuffer& buffer,
+	[[nodiscard]] bool        ResolveColorTargets(CommandBuffer& buffer,
 	                                              uint32_t render_target_slice_offset);
 	void                      BindImage(ImageId id, bool storage);
 	void                      BindRenderTarget(ImageId id);
-	void                      TrackImageBinding(ImageId id);
 	void                      ResetBindings();
 	[[nodiscard]] bool        TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
 	                                                     const CommandBuffer&          buffer);

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -36,18 +36,12 @@
 #include <vector>
 
 namespace Libs::Graphics {
-static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) {
-	const uint64_t records = descriptor.NumRecords();
-	const uint64_t stride  = descriptor.Stride();
-	return stride == 0 ? records : records * stride;
-}
-
 static bool FillSourcesDisjoint(std::span<const ShaderRecompiler::IR::DescriptorValue> sources,
                                  GuestRange destination, uint32_t output_buffer = UINT32_MAX) {
 	for (uint32_t i = 0; i < sources.size(); ++i) {
 		if (i == output_buffer) continue;
 		const auto source = DecodeNativeDescriptor<ShaderBufferResource>(sources[i]);
-		const auto bytes  = BufferDescriptorSize(source);
+		const auto bytes  = source.GetSize();
 		if (source.Base48() < destination.End() && destination.address < source.Base48() + bytes)
 			return false;
 	}
@@ -67,7 +61,7 @@ bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& in
 		const auto  descriptor = DecodeNativeDescriptor<ShaderBufferResource>(resources.buffers[i]);
 		// A metadata resource that is also read is not proven to be a full overwrite. Execute it
 		// conservatively instead of replacing the dispatch with a coarse full-surface clear.
-		if (cache.IsMeta(descriptor.Base48()) && (!resource.written || resource.read)) {
+		if ((!resource.written || resource.read) && cache.IsMeta(descriptor.Base48())) {
 			return false;
 		}
 	}
@@ -115,7 +109,7 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	const uint64_t invocations = input.dispatch_thread_dimensions
 	                                 ? group_x
 	                                 : static_cast<uint64_t>(group_x) * input.threads_num[0];
-	const auto     size        = BufferDescriptorSize(descriptor);
+	const auto     size        = descriptor.GetSize();
 	if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
 	    (input.dispatch_thread_dimensions &&
 	     (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
@@ -189,14 +183,6 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 		return false;
 	}
 	if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) {
-		// Track deferred DCC state while the original dispatch writes the metadata allocation.
-		cache.TrackDccFill(descriptor.Base48(), size, packed_clear);
-		static std::atomic<uint32_t> logged_metadata_clears {0};
-		if (logged_metadata_clears.fetch_add(1, std::memory_order_relaxed) < 32) {
-			LOGF("GraphicsRenderDispatchDirect: metadata fill shader=0x%016" PRIx64
-			     " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n",
-			     input.stage.program->shader_hash, descriptor.Base48(), size, packed_clear);
-		}
 		return false;
 	}
 	static std::atomic<uint32_t> logged_clears {0};
@@ -264,9 +250,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		input_info.dispatch_threads_num[2]    = thread_group_z;
 	}
 
-	const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
-	const bool     large_workgroup =
-	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const auto& program   = *input_info.stage.program;
 	const auto& resources = input_info.stage.resources;
 	if (TryConsumeComputeMetaClear(input_info, buffer)) {
@@ -278,14 +261,17 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		ResetBindings();
 		return;
 	}
-	const auto sampled_images = std::count_if(
-	    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
-		    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
-	    });
+	const bool large_workgroup =
+	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const bool                   has_sampler = !program.info.samplers.empty();
 	static std::atomic<uint32_t> dispatch_log_count {0};
 	if ((large_workgroup || has_sampler) &&
 	    dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) {
+		const auto sampled_images = std::count_if(
+		    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
+			    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
+		    });
+		const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
 		LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64
 		     " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u "
 		     "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n",
@@ -374,14 +360,14 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 
 	buffer.EndRendering();
 	auto& pipeline =
-	    m_context.GetPipelineCache().CreateComputePipeline(input_info, compute_program);
+	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto bindings = PrepareBindings(input_info.stage);
 	FindBuffers(bindings);
 	if (program.info.uses_dma) {
-		m_context.GetGpuResources().PrepareBda();
+		m_context.PrepareBda();
 	}
-	RebindBuffers(bindings);
 	RebindImages(bindings);
+	RebindBuffers(bindings);
 
 	auto              vk_buffer        = buffer.Handle();
 	PreparedBindings* descriptor_stage = &bindings;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -14,7 +14,8 @@ RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
       m_pipeline_cache(graphics), m_sampler_cache(graphics),
-      m_gpu_resources(graphics, m_command_scheduler) {
+      m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
+      m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -27,12 +28,10 @@ void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) {
 	EXIT_IF(m_gpu != nullptr);
 	m_video_out = video_out;
 	m_gpu       = std::make_unique<GuestGpu>(*this);
-	m_gpu_resources.SetGpu(m_gpu.get());
 }
 
 void RenderContext::ShutdownGpu() {
 	if (m_gpu != nullptr) {
-		m_gpu_resources.SetGpu(nullptr);
 		m_gpu->Shutdown();
 		m_gpu.reset();
 	}
@@ -55,6 +54,88 @@ VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const {
 	return *m_video_out;
 }
 
+bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
+	// The host reports the faulting byte, not the instruction's access width. Both caches
+	// resolve its page; guessing a width can cross the end of a valid guest mapping.
+	constexpr uint64_t fault_size = 1;
+	if (!IsMapped(fault_vaddr, fault_size)) {
+		return false;
+	}
+	if (access == PageFaultAccess::Write) {
+		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
+		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+	} else {
+		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
+	}
+	return true;
+}
+
+bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (!IsMapped(vaddr, size)) {
+		return false;
+	}
+	m_buffer_cache.InvalidateMemory(vaddr, size);
+	m_texture_cache.InvalidateMemory(vaddr, size);
+	return true;
+}
+
+bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
+	if (!GuestRange {vaddr, size}.Valid()) {
+		return false;
+	}
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	return m_mapped_ranges.Contains(vaddr, size);
+}
+
+void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) {
+	std::lock_guard lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.Add(vaddr, size);
+}
+
+void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	const auto unmap = [this, vaddr, size] {
+		if (m_command_scheduler.Active()) {
+			const auto tick = m_command_scheduler.CurrentTick();
+			m_command_scheduler.Finish();
+			m_command_scheduler.WaitPriorityOperations(tick);
+		}
+		m_buffer_cache.InvalidateMemory(vaddr, size);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Subtract(vaddr, size);
+	};
+	// Shutdown still owns the GPU while queued rendering drains, but its command lane no
+	// longer accepts external work. Use the guest GPU's state for the teardown route.
+	if (m_gpu == nullptr || m_gpu->IsStopping()) {
+		unmap();
+		return;
+	}
+	m_gpu->SendCommandSync(unmap);
+}
+
+void RenderContext::PrepareBda() {
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
+		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
+	});
+	m_fault_process_pending = true;
+}
+
+void RenderContext::RunGarbageCollector() {
+	if (m_fault_process_pending) {
+		m_fault_process_pending = false;
+		m_buffer_cache.ProcessFaultBuffer();
+	}
+	m_texture_cache.ProcessDownloadImages();
+	m_texture_cache.RunGarbageCollector();
+	m_buffer_cache.RunGarbageCollector();
+}
+
 void RenderContext::AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
 	Common::LockGuard lock(m_interrupt_mutex);
 

src/graphics/host_gpu/renderer/renderContext.h

@@ -5,8 +5,9 @@
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/threads.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/rangeSet.h"
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
-#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
@@ -15,6 +16,7 @@
 #include "kernel/eventQueue.h"
 
 #include <memory>
+#include <shared_mutex>
 #include <vector>
 
 namespace Libs::VideoOut {
@@ -42,11 +44,18 @@ public:
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
 	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
-	GpuResourceManager& GetGpuResources() { return m_gpu_resources; }
-	BufferCache&        GetBufferCache() { return m_gpu_resources.GetBufferCache(); }
-	TextureCache&       GetTextureCache() { return m_gpu_resources.GetTextureCache(); }
+	BufferCache&        GetBufferCache() { return m_buffer_cache; }
+	TextureCache&       GetTextureCache() { return m_texture_cache; }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
 
+	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
+	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
+	void               MapMemory(uint64_t vaddr, uint64_t size);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
+	void               PrepareBda();
+	void               RunGarbageCollector();
+
 	void AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
 	void DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
 	void TriggerInterrupt(int event_id, uint32_t context_id);
@@ -64,9 +73,14 @@ private:
 	DescriptorHeap            m_descriptor_heap;
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
-	GpuResourceManager        m_gpu_resources;
+	PageManager               m_page_manager;
+	BufferCache               m_buffer_cache;
+	TextureCache              m_texture_cache;
+	mutable std::shared_mutex m_mapped_ranges_mutex;
+	RangeSet                  m_mapped_ranges;
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
+	bool                      m_fault_process_pending = false;
 
 	Common::Mutex                        m_interrupt_mutex;
 	std::vector<InterruptEqRegistration> m_interrupt_eqs;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -46,42 +46,34 @@
 
 namespace Libs::Graphics {
 
-int32_t ResolveVertexOffset(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info) {
-	if (index_offset != 0 || !vs_input_info.fetch_embedded) {
-		return static_cast<int32_t>(index_offset);
+std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
+	                                           const ShaderVertexInputInfo& vs_input_info) {
+	auto     vertex_offset   = static_cast<int32_t>(index_offset);
+	uint32_t instance_offset = 0;
+	if (!vs_input_info.fetch_embedded) {
+		return {vertex_offset, instance_offset};
 	}
 
 	EXIT_IF(!vs_input_info.stage);
 	const auto& program   = *vs_input_info.stage.program;
 	const auto& resources = vs_input_info.stage.resources;
-	if (program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
+	if (index_offset == 0 &&
+	    program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
 		    static_cast<uint32_t>(program.info.vertex_offset_sgpr) - program.user_data_base;
 		if (index < resources.user_data.size()) {
-			return static_cast<int32_t>(resources.user_data[index]);
+			vertex_offset = static_cast<int32_t>(resources.user_data[index]);
 		}
 	}
-
-	return 0;
-}
-
-uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info) {
-	if (!vs_input_info.fetch_embedded) {
-		return 0;
-	}
-
-	EXIT_IF(!vs_input_info.stage);
-	const auto& program   = *vs_input_info.stage.program;
-	const auto& resources = vs_input_info.stage.resources;
 	if (program.info.instance_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
 		    static_cast<uint32_t>(program.info.instance_offset_sgpr) - program.user_data_base;
 		if (index < resources.user_data.size()) {
-			return resources.user_data[index];
+			instance_offset = resources.user_data[index];
 		}
 	}
 
-	return 0;
+	return {vertex_offset, instance_offset};
 }
 
 static std::atomic<uint32_t> g_draw_state_log_count   = 0;
@@ -411,19 +403,16 @@ static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuf
 		vk_buffer.setDepthBias(constant_factor, poly_offset.clamp, slope_factor);
 	}
 
+	vk_buffer.setStencilTestEnable(depth.stencil_test_enable ? VK_TRUE : VK_FALSE);
 	if (depth.stencil_test_enable) {
-		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eFront,
-		                                depth.stencil_dynamic_front.compareMask);
-		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eBack,
-		                                depth.stencil_dynamic_back.compareMask);
-		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eFront,
-		                              depth.stencil_dynamic_front.writeMask);
-		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eBack,
-		                              depth.stencil_dynamic_back.writeMask);
-		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eFront,
-		                              depth.stencil_dynamic_front.reference);
-		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eBack,
-		                              depth.stencil_dynamic_back.reference);
+		const auto set_stencil = [&](vk::StencilFaceFlagBits face, const vk::StencilOpState& state) {
+			vk_buffer.setStencilOp(face, state.failOp, state.passOp, state.depthFailOp, state.compareOp);
+			vk_buffer.setStencilCompareMask(face, state.compareMask);
+			vk_buffer.setStencilWriteMask(face, state.writeMask);
+			vk_buffer.setStencilReference(face, state.reference);
+		};
+		set_stencil(vk::StencilFaceFlagBits::eFront, depth.stencil_front);
+		set_stencil(vk::StencilFaceFlagBits::eBack, depth.stencil_back);
 	}
 
 #if defined(__APPLE__)
@@ -461,17 +450,19 @@ struct DrawRenderState {
 	RenderColorInfo       color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	uint32_t              color_count                              = 0;
 	bool                  ps_active                                = true;
-	ShaderVertexInputInfo vs_input_info;
+	std::array<ShaderVertexInputInfo, 3> vertex_info;
 	ShaderPixelInputInfo  ps_input_info;
 	PipelineCache::GraphicsPrograms programs;
 };
 
 struct DrawCallInfo {
-	const char*          name           = nullptr;
 	CommandBufferDebugOp debug_op       = CommandBufferDebugOp::DrawIndex;
 	uint32_t             index_count    = 0;
 	uint32_t             instance_count = 0;
 	uint32_t             first_instance = 0;
+
+	[[nodiscard]] bool IsIndexed() const { return debug_op == CommandBufferDebugOp::DrawIndex; }
+	[[nodiscard]] const char* Name() const { return IsIndexed() ? "DrawIndex" : "DrawIndexAuto"; }
 };
 
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
@@ -488,14 +479,10 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
-		const auto old_image = cache.m_slot_images.try_get(target.image_id);
-		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
-		    old_image->binding.needs_rebind) {
-			if (old_image != nullptr) {
-				old_image->binding = {};
-			}
-			target.image_id = cache.FindImage(target.desc);
-			BindRenderTarget(target.image_id);
+		const auto owner = cache.m_slot_images.try_get(target.image_id);
+		if (owner == nullptr || (!owner->registered && !owner->info.data.Empty()) ||
+		    owner->binding.needs_rebind) {
+			EXIT("color target changed after render-state discovery\n");
 		}
 		const auto image_view = cache.FindRenderTarget(target.image_id, target.desc);
 		auto&      image      = cache.GetImage(target.image_id);
@@ -544,11 +531,11 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		    !cache.ClearMeta(metadata.range.address)) {
 			EXIT("failed to acquire HTile metadata for a depth clear\n");
 		}
-		depth.depth_meta_clear_enable =
+		const bool meta_clear =
 		    metadata.kind == ImageMetadataKind::Htile &&
 		    cache.IsMetaCleared(metadata.range.address, depth.desc.view_info.base_layer);
-		depth.depth_load_clear_enable = depth.depth_clear_enable || depth.depth_meta_clear_enable;
-		if (depth.depth_meta_clear_enable &&
+		depth.depth_load_clear_enable = depth.depth_clear_enable || meta_clear;
+		if (meta_clear &&
 		    !cache.TouchMeta(metadata.range.address, depth.desc.view_info.base_layer, false)) {
 			EXIT("failed to consume HTile clear state\n");
 		}
@@ -662,7 +649,6 @@ static bool ConsumeMetadataColorOperation(const CommandBuffer& buffer) {
 }
 
 struct DrawEmitInfo {
-	bool     indexed       = false;
 	int32_t  vertex_offset = 0;
 	uint32_t first_vertex  = 0;
 	uint32_t first_instance = 0;
@@ -716,6 +702,7 @@ struct PreparedVertexBuffers {
 
 	std::array<vk::Buffer, MaxBuffers>     buffers {};
 	std::array<vk::DeviceSize, MaxBuffers> offsets {};
+	std::array<vk::DeviceSize, MaxBuffers> sizes {};
 	uint32_t                               count = 0;
 };
 
@@ -800,6 +787,7 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer&               b
 
 		prepared.buffers[i] = range->binding.first->Handle();
 		prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address;
+		prepared.sizes[i]   = std::min(size, range->acquired_end - vertex.addr);
 		SetVulkanObjectNameF(
 		    buffer.GetContext().GetGraphics().device, prepared.buffers[i],
 		    "Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i,
@@ -838,6 +826,7 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
 		case Prospero::PrimitiveType::kTriStrip:
 			topology = vk::PrimitiveTopology::eTriangleStrip;
 			break;
+		case Prospero::PrimitiveType::kPatch:
 		case Prospero::PrimitiveType::kRectList:
 			topology = vk::PrimitiveTopology::ePatchList;
 			break;
@@ -863,8 +852,8 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
 	return true;
 }
 
-static bool ResolvePrimitiveRestart(const CommandBuffer& buffer, vk::PrimitiveTopology topology,
-                                    uint32_t index_type_and_size) {
+static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
+                                    const DrawIndexBufferSource& source) {
 	const auto control = buffer.GetUserConfig().GetPrimitiveResetControl();
 	EXIT_NOT_IMPLEMENTED((control & ~0x3u) != 0);
 	if ((control & 0x1u) == 0) {
@@ -876,58 +865,61 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer, vk::PrimitiveTo
 		case Prospero::PrimitiveType::kTriStrip: break;
 		default: return false;
 	}
-	if (topology != vk::PrimitiveTopology::eLineStrip &&
-	    topology != vk::PrimitiveTopology::eTriangleStrip &&
-	    topology != vk::PrimitiveTopology::eTriangleFan) {
-		return false;
-	}
 
-	uint32_t index_mask = 0;
-	switch (static_cast<Prospero::IndexType>(index_type_and_size)) {
-		case Prospero::IndexType::kIndex8: index_mask = 0xffu; break;
-		case Prospero::IndexType::kIndex16: index_mask = 0xffffu; break;
-		case Prospero::IndexType::kIndex32: index_mask = 0xffffffffu; break;
-		default: EXIT("unknown index_type_and_size: %u\n", index_type_and_size);
-	}
-
-	const auto reset_index = buffer.GetRegisters().GetPrimitiveResetIndex();
+	const auto element_size = source.guest_element_size;
+	const auto index_mask   = UINT32_MAX >> ((4 - element_size) * 8);
+	const auto reset_index  = buffer.GetRegisters().GetPrimitiveResetIndex();
 	if ((control & 0x2u) != 0 && (reset_index & ~index_mask) != 0) {
 		return false;
 	}
-	EXIT_NOT_IMPLEMENTED((reset_index & index_mask) != index_mask);
-	return true;
+	const auto restart_index = reset_index & index_mask;
+	if (restart_index == index_mask) {
+		// Use native restart; the 8-bit path widens its marker to 0xffff.
+		return true;
+	}
+
+	// A game can set a custom reset value without using it in the index buffer.
+	// Keep restart off in that case; fail if we actually find the value.
+	// Scan before preparing draw resources: readback can restart the command buffer.
+	EXIT_NOT_IMPLEMENTED(source.address == 0);
+	const auto* indices = reinterpret_cast<const uint8_t*>(source.address);
+	for (uint64_t offset = 0; offset < source.size; offset += element_size) {
+		uint32_t index = 0;
+		std::memcpy(&index, indices + offset, element_size);
+		EXIT_NOT_IMPLEMENTED(index == restart_index);
+	}
+	return false;
 }
 
-bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& buffer,
-                                            const DrawCallInfo& draw,
+bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
-                                            bool log_setup_phases, DrawRenderState& state) {
+	                                        DrawRenderState& state) {
 	auto& ctx = buffer.GetRegisters();
 
-	if (ResolveColorTargets(submit_id, buffer, render_target_slice_offset)) {
+	if (ResolveColorTargets(buffer, render_target_slice_offset)) {
 		return false;
 	}
-	if (log_setup_phases) {
-		LogDrawPhase(draw.name, "ResolveRenderColorTarget");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}
 	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
 		if (slot == 0 || (render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0 &&
 		                  ctx.GetRenderTarget(slot).base.addr != 0)) {
-			ResolveRenderColorTarget(submit_id, buffer, state.color_info[state.color_count],
+			ResolveRenderColorTarget(buffer, state.color_info[state.color_count],
 			                         render_target_slice_offset, slot);
 			if (state.color_info[state.color_count].image_id) {
 				state.color_count++;
 			}
 		}
 	}
-	if (log_setup_phases) {
-		LogDrawPhase(draw.name, "ResolveRenderDepthTarget");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "ResolveRenderDepthTarget");
 	}
-	ResolveRenderDepthTarget(submit_id, buffer, state.depth_info);
+	ResolveRenderDepthTarget(buffer, state.depth_info);
 
 	state.ps_active       = DrawHasActivePixelShader(buffer);
 	if (state.color_count == 0 && !state.depth_info.image_id && !state.ps_active) {
-		LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer,
+		LogFramebufferSkip(draw.Name(), state.color_info[0], state.depth_info, buffer,
 		                   draw.index_count, 0);
 		return false;
 	}
@@ -935,7 +927,7 @@ bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& b
 	return true;
 }
 
-static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
+static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
                            DrawRenderState& state) {
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
@@ -952,12 +944,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool
 		target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
 	}
 	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
-	if (log_phases) {
-		LogDrawPhase(draw.name, "GetGraphicsPrograms");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
 	}
 	state.programs = pipeline_cache.GetGraphicsPrograms(
 	    vertex_shader_info, pixel_shader_info, shader_regs, ctx, buffer.GetUserConfig(),
-	    target_export_mapping, state.ps_active, state.vs_input_info, state.ps_input_info);
+	    target_export_mapping, state.ps_active, state.vertex_info, state.ps_input_info);
 }
 
 static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffer,
@@ -971,17 +963,14 @@ static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffe
 		auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
 		prepared.offset = stream.Copy(source.host_data, source.size, 16);
 		prepared.buffer = stream.Handle();
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
+		                     static_cast<uint32_t>(source.type));
 	} else {
 		auto [buffer_ptr, offset] =
 		    buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
 		prepared.buffer = buffer_ptr->Handle();
 		prepared.offset = offset;
-	}
-	if (source.host_data != nullptr) {
-		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
-		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
-		                     static_cast<uint32_t>(source.type));
-	} else {
 		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
 		                     "Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
 		                     source.address, source.size, static_cast<uint32_t>(source.type));
@@ -995,8 +984,9 @@ static void CommitVertexBuffers(vk::CommandBuffer            vk_buffer,
 		EXIT_IF(prepared.buffers[i] == nullptr);
 	}
 	if (prepared.count != 0) {
-		vk_buffer.bindVertexBuffers(0, prepared.count, prepared.buffers.data(),
-		                            prepared.offsets.data());
+		// Guest descriptor bounds must survive allocation merging in the cache.
+		vk_buffer.bindVertexBuffers2(0, prepared.count, prepared.buffers.data(),
+		                             prepared.offsets.data(), prepared.sizes.data(), nullptr);
 	}
 }
 
@@ -1008,22 +998,22 @@ static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBu
 }
 
 static void LogDrawStateIfNeeded(const CommandBuffer& buffer, const DrawCallInfo& draw,
-                                 const DrawRenderState& state, bool always_log,
-                                 bool force_legacy_rect_log, uint32_t index_type_and_size,
+	                             const DrawRenderState& state, uint32_t index_type_and_size,
                                  const void* index_addr) {
 	if (!graphics_debug_dump_enabled()) {
 		return;
 	}
 
-	if (!always_log && !force_legacy_rect_log) {
+	if (!draw.IsIndexed() &&
+	    buffer.GetUserConfig().GetPrimType() != Prospero::PrimitiveType::kRectListLegacy) {
 		return;
 	}
 
 	if (state.ps_active) {
-		LogDrawTargetState(draw.name, state.color_info[0], state.depth_info, buffer,
+		LogDrawTargetState(draw.Name(), state.color_info[0], state.depth_info, buffer,
 		                   state.ps_input_info, draw.index_count, 0);
 	}
-	LogDrawInputState(buffer, state.color_info[0], state.vs_input_info, index_type_and_size,
+	LogDrawInputState(buffer, state.color_info[0], state.vertex_info[0], index_type_and_size,
 	                  draw.index_count, index_addr);
 }
 
@@ -1038,7 +1028,8 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
 		case Prospero::PrimitiveType::kRectList:
-			if (emit.indexed) {
+		case Prospero::PrimitiveType::kPatch:
+			if (draw.IsIndexed()) {
 				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
 				                      emit.first_instance);
 			} else {
@@ -1047,7 +1038,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 			}
 			break;
 		case Prospero::PrimitiveType::kRectListLegacy:
-			if (emit.indexed) {
+			if (draw.IsIndexed()) {
 				EXIT("unknown primitive type: %u\n", static_cast<uint32_t>(ucfg.GetPrimType()));
 			}
 			// Sarah
@@ -1057,7 +1048,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kQuadListLegacy:
 			EXIT_NOT_IMPLEMENTED((draw.index_count & 0x3u) != 0);
 			for (uint32_t i = 0; i < draw.index_count; i += 4) {
-				if (emit.indexed) {
+				if (draw.IsIndexed()) {
 					vk_buffer.drawIndexed(4, draw.instance_count, i, emit.vertex_offset,
 					                      emit.first_instance);
 				} else {
@@ -1074,16 +1065,17 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
                                          const DrawCallInfo& draw, DrawRenderState& state,
                                          vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
                                          const DrawIndexBufferSource& index_source,
-                                         bool primitive_restart_enable, bool log_pipeline_phase,
-                                         bool set_bind_debug, bool set_auto_debug) {
+	                                     bool primitive_restart_enable) {
 	auto& ucfg = buffer.GetUserConfig();
-	const bool mesh_active = state.vs_input_info.stage.program->stage == ShaderType::Mesh;
+	const auto vertex_stages =
+	    std::span {state.vertex_info.data(), state.programs.VertexStageCount()};
+	const bool mesh_active = state.vertex_info[0].stage.program->stage == ShaderType::Mesh;
 	uint32_t   mesh_groups = 0;
 	if (mesh_active) {
-		const auto& mesh = state.vs_input_info.mesh;
+		const auto& mesh = state.vertex_info[0].mesh;
 		if (primitive_restart_enable || mesh.primitives_per_group == 0) {
 			EXIT("unsupported mesh draw: primitive=%u indexed=%u restart=%u\n",
-			     static_cast<uint32_t>(ucfg.GetPrimType()), emit.indexed, primitive_restart_enable);
+			     static_cast<uint32_t>(ucfg.GetPrimType()), draw.IsIndexed(), primitive_restart_enable);
 		}
 		const auto primitives = mesh.InputPrimitiveCount(draw.index_count);
 		if (primitives == 0 || draw.instance_count == 0) {
@@ -1100,64 +1092,62 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		}
 	}
 
-	if (mesh_active && emit.indexed) {
+	if (mesh_active && draw.IsIndexed()) {
 		// Register the original guest indices for shader reads; PrepareGraphicsBindings
 		// synchronizes registered BDA ranges before any draw commands are committed.
 		(void)m_context.GetBufferCache().FindBuffer(
 		    index_source.address, static_cast<uint64_t>(draw.index_count) *
 		                              index_source.guest_element_size);
 	}
-	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage,
-	                                        state.ps_active);
+	LogDrawPhase(draw.Name(), "PrepareBindings");
+	GraphicsBindings                 bindings;
+	std::array<PreparedBindings*, 4> descriptor_stages {};
+	uint32_t                         stage_count = 0;
+	for (uint32_t i = 0; i < vertex_stages.size(); i++) {
+		bindings.vertex[i]               = PrepareBindings(state.vertex_info[i].stage);
+		descriptor_stages[stage_count++] = &bindings.vertex[i];
+	}
+	if (state.ps_active) {
+		bindings.pixel.emplace(PrepareBindings(state.ps_input_info.stage));
+		descriptor_stages[stage_count++] = &*bindings.pixel;
+	}
+	const auto stages = std::span {descriptor_stages.data(), stage_count};
+	PrepareGraphicsBindings(stages, std::span {state.color_info, state.color_count});
 	PreparedVertexBuffers vertex_bindings;
 	PreparedIndexBuffer   index_binding;
 	if (!mesh_active) {
-		LogDrawPhase(draw.name, "PrepareVertexBuffers");
-		vertex_bindings = AcquireVertexBuffers(buffer, state.vs_input_info);
+		LogDrawPhase(draw.Name(), "PrepareVertexBuffers");
+		vertex_bindings = AcquireVertexBuffers(buffer, state.vertex_info[0]);
 		index_binding   = PrepareIndexBuffer(buffer, index_source);
 	}
 	const auto rendering =
 	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
 	                         bindings.pixel);
 
-	if (log_pipeline_phase) {
-		LogDrawPhase(draw.name, "CreatePipeline");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "CreatePipeline");
 	}
-	auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline(
-	    std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer,
+	auto& pipeline = m_context.GetPipelineCache().GetGraphicsPipeline(
+	    std::span {state.color_info, state.color_count}, state.depth_info, vertex_stages, buffer,
 	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
-	    state.programs.vertex, state.programs.pixel);
+	    state.programs);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest
 	// memory.
 	auto vk_buffer = buffer.Handle();
-	if (set_bind_debug) {
-		SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
-	}
-	if (set_auto_debug) {
-		SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
-	}
+	SetDrawDebugPhase(buffer, submit_id, draw, draw.IsIndexed() ? 0x100u : 0x200u);
 	if (!mesh_active) {
 		CommitVertexBuffers(vk_buffer, vertex_bindings);
 	}
-	if (bindings.pixel.has_value()) {
-		if (set_auto_debug) {
-			SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
-		}
+	if (bindings.pixel && !draw.IsIndexed()) {
+		SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 	}
-	std::array<PreparedBindings*, 2> descriptor_stages {&bindings.vertex, nullptr};
-	const size_t                     descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u;
-	if (bindings.pixel) {
-		descriptor_stages[1] = &*bindings.pixel;
-	}
-	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
-	               std::span {descriptor_stages.data(), descriptor_stage_count});
+	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline, stages);
 	if (mesh_active) {
 		const uint32_t draw_data[] {
 		    draw.index_count,
-		    emit.indexed ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
+		    draw.IsIndexed() ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
 		    emit.first_instance, index_source.guest_element_size,
 		    static_cast<uint32_t>(index_source.address),
 		    static_cast<uint32_t>(index_source.address >> 32u)};
@@ -1170,7 +1160,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		CommitIndexBuffer(vk_buffer, index_binding);
 	}
 
-	SetGraphicsDynamicParams(buffer, vk_buffer, state.vs_input_info, state.color_info,
+	SetGraphicsDynamicParams(buffer, vk_buffer, vertex_stages.back(), state.color_info,
 	                         state.color_count, state.depth_info);
 	if (m_context.GetGraphics().attachment_feedback_loop_enabled) {
 		vk_buffer.setAttachmentFeedbackLoopEnableEXT(
@@ -1180,28 +1170,29 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		        : vk::ImageAspectFlags {});
 	}
 
-	LogDrawPhase(draw.name, "BeginRendering");
-	if (set_auto_debug) {
+	LogDrawPhase(draw.Name(), "BeginRendering");
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
 	}
 	m_context.GetCommandScheduler().BeginRendering(rendering);
 	vk_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline.pipeline);
-	if (set_auto_debug) {
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
 	}
 	if (mesh_active) {
 		vk_buffer.drawMeshTasksEXT(mesh_groups, draw.instance_count, 1);
 	} else {
-		EmitDrawPrimitives(ucfg, vk_buffer, state.vs_input_info, draw, emit);
+		EmitDrawPrimitives(ucfg, vk_buffer, state.vertex_info[0], draw, emit);
 	}
 
-	if (set_auto_debug) {
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
 	}
 	vk::PipelineStageFlags shader_write_stages = {};
-	if (HasShaderBufferWrites(state.vs_input_info.stage)) {
-		shader_write_stages |= mesh_active ? vk::PipelineStageFlagBits::eMeshShaderEXT
-		                                   : vk::PipelineStageFlagBits::eVertexShader;
+	for (const auto& stage: vertex_stages) {
+		if (HasShaderBufferWrites(stage.stage)) {
+			shader_write_stages |= ShaderPipelineStages(NativeShaderStage(stage.logical_stage));
+		}
 	}
 	if (state.ps_active && HasShaderBufferWrites(state.ps_input_info.stage)) {
 		shader_write_stages |= vk::PipelineStageFlagBits::eFragmentShader;
@@ -1210,8 +1201,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		m_context.GetCommandScheduler().EndRendering();
 		ShaderWriteBarrier(vk_buffer, shader_write_stages);
 	}
-	LogDrawPhase(draw.name, "DrawComplete");
-	if (set_auto_debug) {
+	LogDrawPhase(draw.Name(), "DrawComplete");
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
 	}
 }
@@ -1235,7 +1226,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	if (ConsumeMetadataColorOperation(buffer)) {
+	if (ConsumeMetadataColorOperation(buffer) || DepthStencilCopy(buffer)) {
 		ResetBindings();
 		return;
 	}
@@ -1245,7 +1236,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	}
 
 	if (graphics_debug_dump_enabled()) {
-		sh_print("GraphicsRenderDrawIndex():Shader:", sh_ctx);
+		LOGF("GraphicsRenderDrawIndex():Shader:\n");
 		uc_print("GraphicsRenderDrawIndex():UserConfig:", ucfg);
 		hw_print(buffer);
 
@@ -1270,77 +1261,61 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	vk::IndexType index_type           = vk::IndexType::eUint16;
-	uint64_t      index_size           = 0;
-	bool          expand_index8_to_u16 = false;
-	const bool    primitive_restart =
-	    ResolvePrimitiveRestart(buffer, topology, args.index_type_and_size);
-
+	DrawIndexBufferSource index_source {};
+	index_source.address = reinterpret_cast<uint64_t>(args.index_addr);
 	switch (static_cast<Prospero::IndexType>(args.index_type_and_size)) {
 		case Prospero::IndexType::kIndex16:
-			index_type = vk::IndexType::eUint16;
-			index_size = 2 * static_cast<uint64_t>(args.index_count);
+			index_source.type               = vk::IndexType::eUint16;
+			index_source.guest_element_size = 2;
 			break;
 		case Prospero::IndexType::kIndex32:
-			index_type = vk::IndexType::eUint32;
-			index_size = 4 * static_cast<uint64_t>(args.index_count);
+			index_source.type               = vk::IndexType::eUint32;
+			index_source.guest_element_size = 4;
 			break;
-		// Some games use it - need vulkan extension
 		case Prospero::IndexType::kIndex8:
-			index_type           = vk::IndexType::eUint16;
-			index_size           = static_cast<uint64_t>(args.index_count);
-			expand_index8_to_u16 = true;
+			index_source.guest_element_size = 1;
 			break;
 		default: EXIT("unknown index_type_and_size: %u\n", args.index_type_and_size);
 	}
+	index_source.size = static_cast<uint64_t>(args.index_count) * index_source.guest_element_size;
+	const bool primitive_restart = ResolvePrimitiveRestart(buffer, index_source);
 
-	const DrawCallInfo    draw {"DrawIndex", CommandBufferDebugOp::DrawIndex, args.index_count,
-	                            args.instance_count, args.first_instance};
 	std::vector<uint16_t> expanded_indices;
-	if (expand_index8_to_u16) {
+	if (index_source.guest_element_size == 1) {
 		EXIT_NOT_IMPLEMENTED(args.index_addr == nullptr);
 		const auto* src = static_cast<const uint8_t*>(args.index_addr);
 		expanded_indices.resize(args.index_count);
 		for (uint32_t i = 0; i < args.index_count; i++) {
 			expanded_indices[i] = primitive_restart && src[i] == 0xffu ? 0xffffu : src[i];
 		}
+		index_source.host_data = expanded_indices.data();
+		index_source.size      = expanded_indices.size() * sizeof(uint16_t);
 	}
 
-	DrawIndexBufferSource index_source {};
-	index_source.address = reinterpret_cast<uint64_t>(args.index_addr);
-	index_source.host_data =
-	    expanded_indices.empty() ? nullptr : static_cast<const void*>(expanded_indices.data());
-	index_source.size =
-	    expanded_indices.empty() ? index_size : expanded_indices.size() * sizeof(uint16_t);
-	index_source.type = index_type;
-	index_source.guest_element_size = static_cast<uint32_t>(index_size / args.index_count);
-
+	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndex, args.index_count,
+	                        args.instance_count, args.first_instance};
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(submit_id, buffer, draw, args.render_target_slice_offset, true,
-	                            state)) {
+	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return;
 	}
 
-	RefreshShaders(buffer, draw, true, state);
+	RefreshShaders(buffer, draw, state);
 
-	LogDrawStateIfNeeded(buffer, draw, state, true, false, args.index_type_and_size,
+	LogDrawStateIfNeeded(buffer, draw, state, args.index_type_and_size,
 	                     args.index_addr);
 
 	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto vertex_offset =
-	    indirect
-	        ? args.base_vertex
-	        : ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) + args.base_vertex;
+	const auto [vertex_offset, instance_offset] =
+	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
+	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
 
 	DrawEmitInfo emit {};
-	emit.indexed       = true;
-	emit.vertex_offset = vertex_offset;
-	emit.first_instance =
-	    indirect ? args.first_instance : ResolveInstanceOffset(state.vs_input_info);
+	emit.vertex_offset  = vertex_offset + args.base_vertex;
+	emit.first_instance = instance_offset;
 
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
-	                    primitive_restart, true, true, false);
+	                    primitive_restart);
 	ResetBindings();
 }
 
@@ -1363,7 +1338,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		return;
 	}
 
-	if (ConsumeMetadataColorOperation(buffer)) {
+	if (ConsumeMetadataColorOperation(buffer) || DepthStencilCopy(buffer)) {
 		ResetBindings();
 		return;
 	}
@@ -1373,7 +1348,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	}
 
 	if (graphics_debug_dump_enabled()) {
-		sh_print("GraphicsRenderDrawIndexAuto():Shader:", sh_ctx);
+		LOGF("GraphicsRenderDrawIndexAuto():Shader:\n");
 		uc_print("GraphicsRenderDrawIndexAuto():UserConfig:", ucfg);
 		hw_print(buffer);
 
@@ -1389,12 +1364,11 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 
 	hw_check(buffer);
 
-	const DrawCallInfo draw {"DrawIndexAuto", CommandBufferDebugOp::DrawIndexAuto,
+	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndexAuto,
 	                         args.vertex_count, args.instance_count, args.first_instance};
 
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(submit_id, buffer, draw, args.render_target_slice_offset, false,
-	                            state)) {
+	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return;
 	}
@@ -1404,11 +1378,11 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		ResetBindings();
 		return;
 	}
-	RefreshShaders(buffer, draw, false, state);
+	RefreshShaders(buffer, draw, state);
 
-	const bool rect_list = topology == vk::PrimitiveTopology::ePatchList;
-	if (rect_list && state.vs_input_info.buffers_num == 0 &&
-	    state.vs_input_info.stage.program->param_export_mask == 0 &&
+	const bool rect_list = ucfg.GetPrimType() == Prospero::PrimitiveType::kRectList;
+	if (rect_list && state.vertex_info[0].buffers_num == 0 &&
+	    state.vertex_info[0].stage.program->param_export_mask == 0 &&
 	    state.ps_input_info.input_num != 0) {
 		if (graphics_debug_dump_enabled()) {
 			LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: "
@@ -1420,28 +1394,22 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		return;
 	}
 
-	LogDrawStateIfNeeded(buffer, draw, state, false,
-	                     ucfg.GetPrimType() == Prospero::PrimitiveType::kRectListLegacy, 0,
-	                     nullptr);
+	LogDrawStateIfNeeded(buffer, draw, state, 0, nullptr);
 
 	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto vertex_offset =
-	    indirect ? static_cast<int32_t>(args.first_vertex)
-	             : ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) +
-	                   static_cast<int32_t>(args.first_vertex);
+	const auto [vertex_offset, instance_offset] =
+	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
+	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
 	DrawEmitInfo emit {};
-	emit.first_vertex = static_cast<uint32_t>(vertex_offset);
-	emit.first_instance =
-	    indirect ? args.first_instance : ResolveInstanceOffset(state.vs_input_info);
+	emit.first_vertex = static_cast<uint32_t>(vertex_offset + static_cast<int32_t>(args.first_vertex));
+	emit.first_instance = instance_offset;
 
 	DrawIndexBufferSource index_source {};
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false,
-	                    false, true);
+	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false);
 	ResetBindings();
 }
 
-bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, CommandBuffer& buffer,
-                                         uint32_t render_target_slice_offset) {
+bool RenderExecutor::ResolveColorTargets(CommandBuffer& buffer, uint32_t render_target_slice_offset) {
 	const auto& hw = buffer.GetRegisters();
 	if (hw.GetColorControl().mode != 3) {
 		return false;
@@ -1455,8 +1423,8 @@ bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, CommandBuffer& buff
 
 	RenderColorInfo src {};
 	RenderColorInfo dst {};
-	ResolveRenderColorTarget(submit_id, buffer, src, render_target_slice_offset, 0, true, true);
-	ResolveRenderColorTarget(submit_id, buffer, dst, render_target_slice_offset, 1, true, true);
+	ResolveRenderColorTarget(buffer, src, render_target_slice_offset, 0, true, true);
+	ResolveRenderColorTarget(buffer, dst, render_target_slice_offset, 1, true, true);
 	if (!src.image_id || !dst.image_id) {
 		return false;
 	}

src/graphics/host_gpu/renderer/renderDraw.h

@@ -2,14 +2,14 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 
 #include <cstdint>
+#include <utility>
 
 namespace Libs::Graphics {
 
 struct ShaderVertexInputInfo;
 
-[[nodiscard]] int32_t  ResolveVertexOffset(uint32_t                     index_offset,
-                                           const ShaderVertexInputInfo& vs_input_info);
-[[nodiscard]] uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info);
+[[nodiscard]] std::pair<int32_t, uint32_t>
+ResolveDrawOffsets(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/renderTarget.h

@@ -5,7 +5,6 @@
 
 #include <array>
 #include <cstdint>
-#include <type_traits>
 
 namespace Libs::Graphics {
 
@@ -67,41 +66,6 @@ inline constexpr TargetViewInfo ResolveTargetViewInfo(uint32_t base_layer, uint3
 	        base_layer, last_layer - base_layer + 1u, last_layer + 1u};
 }
 
-#pragma pack(push, 1)
-
-struct PipelineStencilStaticState {
-	vk::StencilOp failOp      = vk::StencilOp::eKeep;
-	vk::StencilOp passOp      = vk::StencilOp::eKeep;
-	vk::StencilOp depthFailOp = vk::StencilOp::eKeep;
-	vk::CompareOp compareOp   = vk::CompareOp::eNever;
-};
-
-struct PipelineStencilDynamicState {
-	uint32_t compareMask = 0;
-	uint32_t writeMask   = 0;
-	uint32_t reference   = 0;
-};
-
-#pragma pack(pop)
-
-inline constexpr bool stencil_face_accesses_attachment(const PipelineStencilStaticState&  state,
-                                                       const PipelineStencilDynamicState& dynamic) {
-	return state.compareOp != vk::CompareOp::eAlways ||
-	       (dynamic.writeMask != 0 &&
-	        (state.failOp != vk::StencilOp::eKeep || state.passOp != vk::StencilOp::eKeep ||
-	         state.depthFailOp != vk::StencilOp::eKeep));
-}
-
-static_assert(std::is_trivially_copyable_v<PipelineStencilStaticState>);
-static_assert(std::is_standard_layout_v<PipelineStencilStaticState>);
-static_assert(alignof(PipelineStencilStaticState) == 1);
-static_assert(sizeof(PipelineStencilStaticState) ==
-              sizeof(vk::StencilOp) * 3 + sizeof(vk::CompareOp));
-static_assert(std::is_trivially_copyable_v<PipelineStencilDynamicState>);
-static_assert(std::is_standard_layout_v<PipelineStencilDynamicState>);
-static_assert(alignof(PipelineStencilDynamicState) == 1);
-static_assert(sizeof(PipelineStencilDynamicState) == sizeof(uint32_t) * 3);
-
 } // namespace Libs::Graphics
 
 #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERTARGET_H_

src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc

@@ -1,29 +1,17 @@
+#include "gpu_tiler_standard.inc"
 uint standard64_offset(uvec2 p) {
 	uint x = p.x, y = p.y;
+	uint offset = standard_offset(p);
 	switch (ELEMENT_BYTES) {
-	case 1: return (x & 0x0fu) ^ ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^
-	               ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
-	               ((y << 4) & 0x1f0u) ^ ((y << 5) & 0x400u) ^
-	               ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u);
-	case 2: return ((x << 1) & 0x00eu) ^ ((x << 4) & 0x080u) ^
-	               ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^
-	               ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
-	               ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^
-	               ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
-	case 4: return ((x << 2) & 0x00cu) ^ ((x << 5) & 0x080u) ^
-	               ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^
-	               ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
-	               ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^
-	               ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
-	case 8: return ((x << 3) & 0x008u) ^ ((x << 5) & 0x0c0u) ^
-	               ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^
-	               ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
-	               ((y << 4) & 0x030u) ^ ((y << 6) & 0x100u) ^
-	               ((y << 7) & 0x400u) ^ ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
-	default: return ((x << 6) & 0x0c0u) ^ ((x << 7) & 0x200u) ^
-	                ((x << 8) & 0x800u) ^ ((x << 9) & 0x2000u) ^
-	                ((x << 10) & 0x8000u) ^ ((y << 4) & 0x030u) ^
-	                ((y << 6) & 0x100u) ^ ((y << 7) & 0x400u) ^
-	                ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
+	case 1: return offset ^ ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
+	                       ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u);
+	case 2: return offset ^ ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
+	                       ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
+	case 4: return offset ^ ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
+	                       ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
+	case 8: return offset ^ ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
+	                       ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
+	default: return offset ^ ((x << 9) & 0x2000u) ^ ((x << 10) & 0x8000u) ^
+	                        ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
 	}
 }

src/graphics/host_gpu/vma.cpp

@@ -130,35 +130,42 @@ uint64_t GraphicContext::GetTotalMemoryBudget() const {
 
 bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanImage& image) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(allocator == nullptr || image.image != nullptr || image.memory.allocation != nullptr);
+	EXIT_IF(allocator == nullptr || image.image != nullptr || image.allocation != nullptr);
 
-	auto&                   memory = image.memory;
 	VmaAllocationCreateInfo alloc_info {};
-	alloc_info.requiredFlags = static_cast<vk::MemoryPropertyFlags::MaskType>(memory.property);
-	alloc_info.preferredFlags =
-	    static_cast<vk::MemoryPropertyFlags::MaskType>(memory.preferred_property);
+	alloc_info.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
 
 	vk::Image::CType native_image = VK_NULL_HANDLE;
 	const auto        result       = static_cast<vk::Result>(
 	    vmaCreateImage(allocator, static_cast<const vk::ImageCreateInfo::NativeType*>(image_info),
-	                   &alloc_info, &native_image, &memory.allocation, nullptr));
+	                   &alloc_info, &native_image, &image.allocation, nullptr));
 	image.image = native_image;
 	if (result != vk::Result::eSuccess) {
 		LogMemoryBudget();
 		return false;
 	}
 
+	image.format     = image_info.format;
+	image.image_type = image_info.imageType;
+	image.extent     = image_info.extent;
+	image.layers     = image_info.arrayLayers;
+	image.mip_levels = image_info.mipLevels;
+	image.samples    = static_cast<uint32_t>(image_info.samples);
+	image.usage      = image_info.usage;
+	image.flags      = image_info.flags;
+	image.state      = {.layout = image_info.initialLayout};
+	image.subresource_states.clear();
+
 	return true;
 }
 
 void GraphicContext::DeleteImage(VulkanImage& image) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(allocator == nullptr || image.image == nullptr || image.memory.allocation == nullptr);
+	EXIT_IF(allocator == nullptr || image.image == nullptr || image.allocation == nullptr);
 
-	auto& memory = image.memory;
-	vmaDestroyImage(allocator, image.image, memory.allocation);
-	image.image            = nullptr;
-	memory.allocation      = nullptr;
+	vmaDestroyImage(allocator, image.image, image.allocation);
+	image.image      = nullptr;
+	image.allocation = nullptr;
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/vulkanCommon.cpp

@@ -17,14 +17,22 @@ struct FormatMapping {
 
 constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k8UNorm, vk::Format::eR8Unorm},
+    {Prospero::BufferFormat::k8SNorm, vk::Format::eR8Snorm},
+    {Prospero::BufferFormat::k8UScaled, vk::Format::eR8Uscaled},
+    {Prospero::BufferFormat::k8SScaled, vk::Format::eR8Sscaled},
     {Prospero::BufferFormat::k8UInt, vk::Format::eR8Uint},
+    {Prospero::BufferFormat::k8SInt, vk::Format::eR8Sint},
     {Prospero::BufferFormat::k16UNorm, vk::Format::eR16Unorm},
     {Prospero::BufferFormat::k16SNorm, vk::Format::eR16Snorm},
+    {Prospero::BufferFormat::k16UScaled, vk::Format::eR16Uscaled},
+    {Prospero::BufferFormat::k16SScaled, vk::Format::eR16Sscaled},
     {Prospero::BufferFormat::k16UInt, vk::Format::eR16Uint},
     {Prospero::BufferFormat::k16SInt, vk::Format::eR16Sint},
     {Prospero::BufferFormat::k16Float, vk::Format::eR16Sfloat},
     {Prospero::BufferFormat::k8_8UNorm, vk::Format::eR8G8Unorm},
     {Prospero::BufferFormat::k8_8SNorm, vk::Format::eR8G8Snorm},
+    {Prospero::BufferFormat::k8_8UScaled, vk::Format::eR8G8Uscaled},
+    {Prospero::BufferFormat::k8_8SScaled, vk::Format::eR8G8Sscaled},
     {Prospero::BufferFormat::k8_8UInt, vk::Format::eR8G8Uint},
     {Prospero::BufferFormat::k8_8SInt, vk::Format::eR8G8Sint},
     {Prospero::BufferFormat::k32UInt, vk::Format::eR32Uint},
@@ -32,14 +40,19 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32Float, vk::Format::eR32Sfloat},
     {Prospero::BufferFormat::k16_16UNorm, vk::Format::eR16G16Unorm},
     {Prospero::BufferFormat::k16_16SNorm, vk::Format::eR16G16Snorm},
+    {Prospero::BufferFormat::k16_16UScaled, vk::Format::eR16G16Uscaled},
+    {Prospero::BufferFormat::k16_16SScaled, vk::Format::eR16G16Sscaled},
     {Prospero::BufferFormat::k16_16UInt, vk::Format::eR16G16Uint},
     {Prospero::BufferFormat::k16_16SInt, vk::Format::eR16G16Sint},
     {Prospero::BufferFormat::k16_16Float, vk::Format::eR16G16Sfloat},
     {Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
     {Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
+    {Prospero::BufferFormat::k10_10_10_2SNorm, vk::Format::eA2B10G10R10SnormPack32},
     {Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
     {Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
     {Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},
+    {Prospero::BufferFormat::k8_8_8_8UScaled, vk::Format::eR8G8B8A8Uscaled},
+    {Prospero::BufferFormat::k8_8_8_8SScaled, vk::Format::eR8G8B8A8Sscaled},
     {Prospero::BufferFormat::k8_8_8_8UInt, vk::Format::eR8G8B8A8Uint},
     {Prospero::BufferFormat::k8_8_8_8SInt, vk::Format::eR8G8B8A8Sint},
     {Prospero::BufferFormat::k32_32UInt, vk::Format::eR32G32Uint},
@@ -47,6 +60,8 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32_32Float, vk::Format::eR32G32Sfloat},
     {Prospero::BufferFormat::k16_16_16_16UNorm, vk::Format::eR16G16B16A16Unorm},
     {Prospero::BufferFormat::k16_16_16_16SNorm, vk::Format::eR16G16B16A16Snorm},
+    {Prospero::BufferFormat::k16_16_16_16UScaled, vk::Format::eR16G16B16A16Uscaled},
+    {Prospero::BufferFormat::k16_16_16_16SScaled, vk::Format::eR16G16B16A16Sscaled},
     {Prospero::BufferFormat::k16_16_16_16UInt, vk::Format::eR16G16B16A16Uint},
     {Prospero::BufferFormat::k16_16_16_16SInt, vk::Format::eR16G16B16A16Sint},
     {Prospero::BufferFormat::k16_16_16_16Float, vk::Format::eR16G16B16A16Sfloat},
@@ -63,7 +78,8 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k8_8_8_8Srgb, vk::Format::eR8G8B8A8Srgb},
     {Prospero::BufferFormat::k9_9_9_5Float, vk::Format::eE5B9G9R9UfloatPack32},
     {Prospero::BufferFormat::k5_6_5UNorm, vk::Format::eB5G6R5UnormPack16},
-    {Prospero::BufferFormat::k5_5_5_1UNorm, vk::Format::eR5G5B5A1UnormPack16},
+    {Prospero::BufferFormat::k5_5_5_1UNorm, vk::Format::eA1R5G5B5UnormPack16},
+    {Prospero::BufferFormat::k1_5_5_5UNorm, vk::Format::eR5G5B5A1UnormPack16},
     {Prospero::BufferFormat::k4_4_4_4UNorm, vk::Format::eR4G4B4A4UnormPack16},
     {Prospero::BufferFormat::kBc1UNorm, vk::Format::eBc1RgbaUnormBlock},
     {Prospero::BufferFormat::kBc1Srgb, vk::Format::eBc1RgbaSrgbBlock},
@@ -107,4 +123,14 @@ void RequireVulkanSuccess(vk::Result result, const char* operation) {
 	}
 }
 
+vk::ShaderModule CompileSPV(std::span<const uint32_t> code, vk::Device device) {
+	vk::ShaderModuleCreateInfo create_info {};
+	create_info.codeSize    = code.size_bytes();
+	create_info.pCode       = code.data();
+	vk::ShaderModule module = nullptr;
+	RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module),
+	                     "create SPIR-V shader module");
+	return module;
+}
+
 } // namespace Libs::Graphics

src/graphics/host_gpu/vulkanCommon.h

@@ -15,6 +15,7 @@
 
 #include <cstdint>
 #include <fmt/format.h>
+#include <span>
 #include <string>
 #include <utility>
 #include <vector>
@@ -27,6 +28,7 @@ using VulkanMemoryBarrier = vk::MemoryBarrier;
 
 vk::Format  VulkanFormat(Prospero::BufferFormat guest_format);
 void        RequireVulkanSuccess(vk::Result result, const char* operation);
+vk::ShaderModule CompileSPV(std::span<const uint32_t> code, vk::Device device);
 
 template <typename Handle, typename... Args>
 void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<Args...> format,
@@ -38,7 +40,6 @@ void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<A
 
 	const auto                      name = fmt::format(format, std::forward<Args>(args)...);
 	vk::DebugUtilsObjectNameInfoEXT info {};
-	info.sType        = vk::StructureType::eDebugUtilsObjectNameInfoEXT;
 	info.objectType   = Handle::objectType;
 	info.objectHandle = static_cast<uint64_t>(
 	    reinterpret_cast<uintptr_t>(static_cast<typename Handle::CType>(handle)));

src/graphics/presentation/videoOut.cpp

@@ -237,7 +237,9 @@ private:
 
 struct VideoOutDriver::Impl {
 public:
-	static constexpr int VIDEO_OUT_NUM_MAX = 4;
+	static constexpr std::array VIDEO_OUT_BUSES {
+	    VIDEO_OUT_BUS_TYPE_MAIN, VIDEO_OUT_BUS_TYPE_OVERLAY, VIDEO_OUT_BUS_TYPE_SUB};
+	static constexpr int VIDEO_OUT_NUM_MAX = VIDEO_OUT_BUSES.size() + 1;
 
 	Impl(uint32_t width, uint32_t height, Graphics::Presenter& presenter)
 	    : m_renderer(presenter.Renderer()), m_presenter(presenter), m_flip_queue(presenter) {
@@ -248,7 +250,7 @@ public:
 	~Impl();
 	KYTY_CLASS_NO_COPY(Impl);
 
-	int             Open(int bus_type);
+	int             Open(int bus_type, int index);
 	bool            Close(int handle);
 	VideoOutConfig* Get(int handle);
 	VideoOutConfig* Get(int handle, uint64_t& generation);
@@ -610,12 +612,17 @@ void VideoOutDriver::Impl::Init(uint32_t width, uint32_t height) {
 	}
 }
 
-int VideoOutDriver::Impl::Open(int bus_type) {
+int VideoOutDriver::Impl::Open(int bus_type, int index) {
+	const auto bus = std::find(VIDEO_OUT_BUSES.begin(), VIDEO_OUT_BUSES.end(), bus_type);
+	if (bus == VIDEO_OUT_BUSES.end()) {
+		return VIDEO_OUT_ERROR_INVALID_VALUE;
+	}
+	EXIT_NOT_IMPLEMENTED(index != 0);
 	Common::LockGuard lock(m_mutex);
 
-	const int handle = bus_type + 1;
+	const int handle = static_cast<int>(bus - VIDEO_OUT_BUSES.begin()) + 1;
 	if (m_video_out_ctx[handle].opened) {
-		return -1;
+		return VIDEO_OUT_ERROR_RESOURCE_BUSY;
 	}
 	auto&             config = m_video_out_ctx[handle];
 	Common::LockGuard config_lock(config.mutex);
@@ -1152,7 +1159,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 	Graphics::RenderDocOnGuestFlip(m_presenter.Renderer());
 
 	if (Config::GraphicsDebugDumpEnabled() &&
-	    Config::GetPrintfDirection() != Config::OutputDirection::Silent) {
+	    Config::GetPrintfDirection() != Config::LogDirection::Silent) {
 		LOGF("Flip done: %d\n", r.index);
 	}
 
@@ -1169,21 +1176,9 @@ KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void*
 	PRINT_NAME();
 
 	EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0);
-	if (bus_type != VIDEO_OUT_BUS_TYPE_MAIN && bus_type != VIDEO_OUT_BUS_TYPE_OVERLAY &&
-	    bus_type != VIDEO_OUT_BUS_TYPE_SUB) {
-		return VIDEO_OUT_ERROR_INVALID_VALUE;
-	}
-	EXIT_NOT_IMPLEMENTED(index != 0);
-
 	LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast<uint64_t>(param));
 
-	int handle = DriverState().Open(bus_type);
-
-	if (handle < 0) {
-		return VIDEO_OUT_ERROR_RESOURCE_BUSY;
-	}
-
-	return handle;
+	return DriverState().Open(bus_type, index);
 }
 
 KYTY_SYSV_ABI int VideoOutClose(int handle) {

src/graphics/presentation/window/swapchain.cpp

@@ -191,24 +191,15 @@ void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
 	}
 	if (dst.image != nullptr) {
 		graphics.DeleteImage(dst);
-		dst.memory = {};
 	}
 
-	dst.extent     = {extent.width, extent.height, 1};
-	dst.format     = format;
-	dst.layers     = 1;
-	dst.mip_levels = 1;
-	dst.state      = {};
-	dst.subresource_states.clear();
-	dst.memory.property = vk::MemoryPropertyFlagBits::eDeviceLocal;
-
 	vk::ImageCreateInfo create {};
 	create.sType         = vk::StructureType::eImageCreateInfo;
 	create.imageType     = vk::ImageType::e2D;
-	create.extent        = {dst.extent.width, dst.extent.height, 1};
+	create.extent        = {extent.width, extent.height, 1};
 	create.mipLevels     = 1;
 	create.arrayLayers   = 1;
-	create.format        = dst.format;
+	create.format        = format;
 	create.tiling        = vk::ImageTiling::eOptimal;
 	create.initialLayout = vk::ImageLayout::eUndefined;
 	create.usage = vk::ImageUsageFlagBits::eTransferSrc | vk::ImageUsageFlagBits::eTransferDst;
@@ -216,7 +207,7 @@ void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
 	create.samples     = vk::SampleCountFlagBits::e1;
 	if (!graphics.CreateImage(create, dst)) {
 		EXIT("failed to allocate prepared presentation image, extent=%ux%u format=%d\n",
-		     dst.extent.width, dst.extent.height, static_cast<int>(dst.format));
+		     extent.width, extent.height, static_cast<int>(format));
 	}
 }
 

src/graphics/presentation/window/vulkanWindow.cpp

@@ -64,7 +64,6 @@ struct VulkanExtensions {
 
 vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
 	vk::PhysicalDeviceVulkan12Features features {};
-	features.sType                     = vk::StructureType::ePhysicalDeviceVulkan12Features;
 	features.samplerMirrorClampToEdge  = VK_TRUE;
 	features.timelineSemaphore         = VK_TRUE;
 	features.shaderOutputLayer         = VK_TRUE;
@@ -76,7 +75,6 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 
 vk::PhysicalDeviceVulkan13Features WindowContext::RequiredVulkan13Features() noexcept {
 	vk::PhysicalDeviceVulkan13Features features {};
-	features.sType            = vk::StructureType::ePhysicalDeviceVulkan13Features;
 	features.dynamicRendering = VK_TRUE;
 	features.synchronization2 = VK_TRUE;
 	return features;
@@ -201,35 +199,24 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		vk::PhysicalDeviceFeatures2 device_features2 {};
 
 		vk::PhysicalDeviceVulkan13Features features13 {};
-		features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-		features13.pNext = nullptr;
 
 		vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {};
-		color_write_ext.sType = vk::StructureType::ePhysicalDeviceColorWriteEnableFeaturesEXT;
-		color_write_ext.pNext = nullptr;
 
 		vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip_enable {};
-		depth_clip_enable.sType = vk::StructureType::ePhysicalDeviceDepthClipEnableFeaturesEXT;
 		depth_clip_enable.pNext = &color_write_ext;
 
 		vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
-		depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
 		depth_clip_control.pNext = &depth_clip_enable;
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
-		features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
 #if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
 #else
 		vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
-		fragment_barycentric.sType =
-		    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
 		features13.pNext = &features12;
-
-		device_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
 		device_features2.pNext = &features13;
 
 		device.getFeatures2(&device_features2);
@@ -506,83 +493,31 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 	}
 }
 
-static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device,
-                                          GraphicContext&    graphics) {
-	EXIT_IF(physical_device == nullptr);
-
-	vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {};
-	subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlProperties;
-	subgroup_size_control.pNext = nullptr;
-
-	vk::PhysicalDeviceVulkan11Properties properties11 {};
-	properties11.sType = vk::StructureType::ePhysicalDeviceVulkan11Properties;
-	properties11.pNext = &subgroup_size_control;
-
-	vk::PhysicalDeviceProperties2 properties2 {};
-	properties2.sType = vk::StructureType::ePhysicalDeviceProperties2;
-	properties2.pNext = &properties11;
-
-	physical_device.getProperties2(&properties2);
-
-	vk::PhysicalDeviceVulkan13Features features13 {};
-	features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-	features13.pNext = nullptr;
-
-	vk::PhysicalDeviceFeatures2 features2 {};
-	features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
-	features2.pNext = &features13;
-
-	physical_device.getFeatures2(&features2);
-
-	graphics.subgroup_size                 = properties11.subgroupSize;
-	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
-	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
-	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
-	graphics.compute_subgroup_size_control_enabled =
-	    features13.subgroupSizeControl == VK_TRUE &&
-	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
-	    subgroup_size_control.minSubgroupSize <= 64 &&
-	    subgroup_size_control.maxSubgroupSize >= 64;
-
-	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
-	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
-	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
-	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
-	     graphics.SupportsComputeWave64() ? "true" : "false");
-}
-
-static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r,
-                                     uint32_t                        queue_family,
-                                     const std::vector<const char*>& device_extensions,
-                                     GraphicContext&                 graphics) {
+static vk::Device VulkanCreateDevice(GraphicContext& graphics,
+	                                 const std::vector<const char*>& device_extensions) {
+	const auto physical_device = graphics.physical_device;
+	const auto queue_family    = graphics.queue_family;
 	EXIT_IF(physical_device == nullptr);
 	EXIT_IF(queue_family == static_cast<uint32_t>(-1));
 
 	const float               queue_priority = 1.0f;
 	vk::DeviceQueueCreateInfo queue_create_info {};
-	queue_create_info.sType            = vk::StructureType::eDeviceQueueCreateInfo;
 	queue_create_info.queueFamilyIndex = queue_family;
 	queue_create_info.queueCount       = 1;
 	queue_create_info.pQueuePriorities = &queue_priority;
 
 	vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {};
-	color_write_ext.sType = vk::StructureType::ePhysicalDeviceColorWriteEnableFeaturesEXT;
-	color_write_ext.pNext = nullptr;
 	color_write_ext.colorWriteEnable = VK_TRUE;
 
 	vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip_enable {};
-	depth_clip_enable.sType = vk::StructureType::ePhysicalDeviceDepthClipEnableFeaturesEXT;
 	depth_clip_enable.pNext = &color_write_ext;
 	depth_clip_enable.depthClipEnable = VK_TRUE;
 
 	vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
-	depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
 	// MoltenVK lacks VK_EXT_depth_clip_enable and VK_EXT_color_write_enable, so drop those
 	// feature structs from the chain on macOS (the renderer falls back to default depth
 	// clipping and static color-write masks).
-#if defined(__APPLE__)
-	depth_clip_control.pNext = nullptr;
-#else
+#if !defined(__APPLE__)
 	depth_clip_control.pNext = &depth_clip_enable;
 #endif
 	depth_clip_control.depthClipControl = VK_TRUE;
@@ -591,40 +526,19 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	features12.pNext = &depth_clip_control;
 
 	vk::PhysicalDeviceVulkan13Features supported_features13 {};
-	supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-
-	vk::PhysicalDeviceVulkan12Features supported_features12 {};
-	supported_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
-	supported_features12.pNext = nullptr;
-	supported_features13.pNext = &supported_features12;
-
-#if !defined(__APPLE__)
-	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_fragment_barycentric {};
-	supported_fragment_barycentric.sType =
-	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
-	supported_fragment_barycentric.pNext = nullptr;
-	supported_features12.pNext           = &supported_fragment_barycentric;
-#endif
 
 	const auto robustness2_ext_enabled =
 	    HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT supported_robustness2 {};
-	supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
-	supported_robustness2.pNext = nullptr;
 	if (robustness2_ext_enabled) {
-#if defined(__APPLE__)
-		supported_features12.pNext = &supported_robustness2;
-#else
-		supported_fragment_barycentric.pNext = &supported_robustness2;
-#endif
+		supported_features13.pNext = &supported_robustness2;
 	}
 
 	const bool mesh_extension = HasExtension(device_extensions, VK_EXT_MESH_SHADER_EXTENSION_NAME);
 	vk::PhysicalDeviceMeshShaderFeaturesEXT supported_mesh {};
 	supported_mesh.pNext = &supported_features13;
 	vk::PhysicalDeviceFeatures2 supported_features2 {};
-	supported_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
 	supported_features2.pNext = mesh_extension ? static_cast<void*>(&supported_mesh)
 	                                           : static_cast<void*>(&supported_features13);
 	const bool feedback_extensions =
@@ -637,53 +551,61 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		feedback_layout.pNext  = &feedback_dynamic;
 		supported_features2.pNext = &feedback_layout;
 	}
+	const bool provoking_extension =
+	    HasExtension(device_extensions, VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME);
+	vk::PhysicalDeviceProvokingVertexFeaturesEXT provoking_vertex {};
+	if (provoking_extension) {
+		provoking_vertex.pNext = supported_features2.pNext;
+		supported_features2.pNext = &provoking_vertex;
+	}
 	physical_device.getFeatures2(&supported_features2);
+	graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader;
+
+	vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {};
+
+	vk::PhysicalDeviceVulkan11Properties properties11 {};
+	properties11.pNext = &subgroup_size_control;
+
+	vk::PhysicalDeviceProperties2 properties2 {};
+	properties2.pNext = &properties11;
+
+	if (graphics.mesh_shader_enabled) {
+		subgroup_size_control.pNext = &graphics.mesh_shader_properties;
+	}
+	physical_device.getProperties2(&properties2);
+
+	graphics.subgroup_size                 = properties11.subgroupSize;
+	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
+	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
+	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
+	graphics.compute_subgroup_size_control_enabled =
+	    supported_features13.subgroupSizeControl == VK_TRUE &&
+	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
+	    subgroup_size_control.minSubgroupSize <= 64 &&
+	    subgroup_size_control.maxSubgroupSize >= 64;
+
+	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
+	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
+	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
+	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
+	     graphics.SupportsComputeWave64() ? "true" : "false");
+	graphics.provoking_vertex_last_enabled = provoking_extension && provoking_vertex.provokingVertexLast;
 	graphics.attachment_feedback_loop_enabled =
 	    feedback_extensions && feedback_layout.attachmentFeedbackLoopLayout &&
 	    feedback_dynamic.attachmentFeedbackLoopDynamicState;
 	LOGF("Vulkan depth feedback support: %s\n",
 	     graphics.attachment_feedback_loop_enabled ? "true" : "false");
-	graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader;
 	if (graphics.mesh_shader_enabled) {
-		vk::PhysicalDeviceProperties2 properties {};
-		properties.pNext = &graphics.mesh_shader_properties;
-		physical_device.getProperties2(&properties);
 		LOGF("Vulkan MeshEXT: invocations=%u vertices=%u primitives=%u shared=%u\n",
 		     graphics.mesh_shader_properties.maxMeshWorkGroupInvocations,
 		     graphics.mesh_shader_properties.maxMeshOutputVertices,
 		     graphics.mesh_shader_properties.maxMeshOutputPrimitives,
 		     graphics.mesh_shader_properties.maxMeshSharedMemorySize);
 	}
-	const auto required_features12 = WindowContext::RequiredVulkan12Features();
-	const auto required_features13 = WindowContext::RequiredVulkan13Features();
-	EXIT_NOT_IMPLEMENTED(required_features12.samplerMirrorClampToEdge == VK_TRUE &&
-	                     supported_features12.samplerMirrorClampToEdge != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.timelineSemaphore == VK_TRUE &&
-	                     supported_features12.timelineSemaphore != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features13.dynamicRendering == VK_TRUE &&
-	                     supported_features13.dynamicRendering != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features13.synchronization2 == VK_TRUE &&
-	                     supported_features13.synchronization2 != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.multiViewport != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.fillModeNonSolid != VK_TRUE);
+	// VulkanFindPhysicalDevice already checked the required creation features. These two
+	// requirements are specific to this creation path and are not part of device selection.
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderOutputLayer == VK_TRUE &&
-	                     supported_features12.shaderOutputLayer != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderOutputViewportIndex == VK_TRUE &&
-	                     supported_features12.shaderOutputViewportIndex != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.bufferDeviceAddress == VK_TRUE &&
-	                     supported_features12.bufferDeviceAddress != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderBufferInt64Atomics == VK_TRUE &&
-	                     supported_features12.shaderBufferInt64Atomics != VK_TRUE);
-#if !defined(__APPLE__)
-	EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
-#endif
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
 	device_features.samplerAnisotropy        = VK_TRUE;
@@ -707,13 +629,10 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	device_features.shaderInt64 = VK_TRUE;
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
-	robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
 #if defined(__APPLE__)
 	robustness2.pNext = &features12;
 #else
 	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
-	fragment_barycentric.sType =
-	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
 	fragment_barycentric.pNext                     = &features12;
 	fragment_barycentric.fragmentShaderBarycentric = VK_TRUE;
 	robustness2.pNext                              = &fragment_barycentric;
@@ -724,11 +643,7 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		robustness2.nullDescriptor      = supported_robustness2.nullDescriptor;
 	}
 
-	const bool subgroup_size_control_enabled =
-	    graphics.compute_subgroup_size_control_enabled &&
-	    supported_features13.subgroupSizeControl == VK_TRUE;
-
-	auto features13 = required_features13;
+	auto features13 = WindowContext::RequiredVulkan13Features();
 #if defined(__APPLE__)
 	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
 	                                           : static_cast<void*>(&features12);
@@ -737,7 +652,8 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	                                           : static_cast<void*>(&fragment_barycentric);
 #endif
 	features13.robustImageAccess   = supported_features13.robustImageAccess;
-	features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
+	features13.subgroupSizeControl =
+	    graphics.compute_subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
 
 	LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
 	     features13.robustImageAccess == VK_TRUE ? "true" : "false",
@@ -747,13 +663,16 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	vk::PhysicalDeviceMeshShaderFeaturesEXT mesh_features {};
 	mesh_features.pNext                 = &features13;
 	mesh_features.meshShader            = graphics.mesh_shader_enabled;
-	create_info.sType                   = vk::StructureType::eDeviceCreateInfo;
 	feedback_dynamic.pNext =
 	    mesh_extension ? static_cast<void*>(&mesh_features) : static_cast<void*>(&features13);
 	create_info.pNext = graphics.attachment_feedback_loop_enabled
 	                        ? static_cast<void*>(&feedback_layout)
 	                        : feedback_dynamic.pNext;
-	create_info.flags                   = {};
+	if (graphics.provoking_vertex_last_enabled) {
+		provoking_vertex.pNext = const_cast<void*>(create_info.pNext);
+		provoking_vertex.transformFeedbackPreservesProvokingVertex = VK_FALSE;
+		create_info.pNext = &provoking_vertex;
+	}
 	create_info.pQueueCreateInfos       = &queue_create_info;
 	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
@@ -783,8 +702,6 @@ static void VulkanGetExtensions(SDL_Window* window, VulkanExtensions& r) {
 
 	r.required_extensions =
 	    std::vector<const char*>(required_extensions_count); // @suppress("Ambiguous problem")
-	std::memset(r.required_extensions.data(), 0,
-	            sizeof(const char*) * r.required_extensions.size());
 
 	sdl_result = SDL_Vulkan_GetInstanceExtensions(window, &required_extensions_count,
 	                                              r.required_extensions.data());
@@ -976,8 +893,6 @@ void WindowContext::CreateVulkan() {
 	VulkanCheckInstanceVersion();
 
 	vk::ApplicationInfo app_info {};
-	app_info.sType              = vk::StructureType::eApplicationInfo;
-	app_info.pNext              = nullptr;
 	app_info.pApplicationName   = "Kyty";
 	app_info.applicationVersion = 1;
 	app_info.pEngineName        = "Kyty";
@@ -1006,17 +921,11 @@ void WindowContext::CreateVulkan() {
 	}
 
 	vk::ValidationFeaturesEXT validation_features {};
-	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
-	validation_features.pNext                          = nullptr;
 	validation_features.enabledValidationFeatureCount  = enabled_features_count;
 	validation_features.pEnabledValidationFeatures     = enabled_features;
-	validation_features.disabledValidationFeatureCount = 0;
-	validation_features.pDisabledValidationFeatures    = nullptr;
 
 	vk::DebugUtilsMessengerCreateInfoEXT dbg_create_info {};
-	dbg_create_info.sType           = vk::StructureType::eDebugUtilsMessengerCreateInfoEXT;
 	dbg_create_info.pNext           = &validation_features;
-	dbg_create_info.flags           = {};
 	dbg_create_info.messageSeverity = vk::DebugUtilsMessageSeverityFlagBitsEXT::eVerbose |
 	                                  vk::DebugUtilsMessageSeverityFlagBitsEXT::eInfo |
 	                                  vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning |
@@ -1025,12 +934,9 @@ void WindowContext::CreateVulkan() {
 	                                  vk::DebugUtilsMessageTypeFlagBitsEXT::eValidation |
 	                                  vk::DebugUtilsMessageTypeFlagBitsEXT::ePerformance;
 	dbg_create_info.pfnUserCallback = VulkanDebugMessengerCallback;
-	dbg_create_info.pUserData       = nullptr;
 
 	vk::InstanceCreateInfo inst_info {};
-	inst_info.sType                   = vk::StructureType::eInstanceCreateInfo;
 	inst_info.pNext                   = (r.enable_validation_layers ? &dbg_create_info : nullptr);
-	inst_info.flags                   = {};
 #if defined(__APPLE__)
 	// MoltenVK requires VK_KHR_portability_enumeration + flag to surface
 	// portability devices. Without this, enumeratePhysicalDevices hides the
@@ -1097,10 +1003,8 @@ void WindowContext::CreateVulkan() {
 	}
 #endif
 
-	uint32_t queue_family = static_cast<uint32_t>(-1);
-
 	VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, surface_capabilities,
-	                         graphic_ctx.physical_device, queue_family);
+	                         graphic_ctx.physical_device, graphic_ctx.queue_family);
 
 	if (graphic_ctx.physical_device == nullptr) {
 		EXIT("Could not find suitable device");
@@ -1108,9 +1012,6 @@ void WindowContext::CreateVulkan() {
 
 	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
 	vk::PhysicalDeviceProperties2              physical_device_properties {};
-	push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties;
-	push_descriptor_properties.pNext = nullptr;
-	physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2;
 	physical_device_properties.pNext = &push_descriptor_properties;
 	graphic_ctx.physical_device.getProperties2(&physical_device_properties);
 	graphic_ctx.physical_device_properties = physical_device_properties.properties;
@@ -1145,6 +1046,7 @@ void WindowContext::CreateVulkan() {
 			graphic_ctx.memory_budget_ext_enabled = true;
 		}
 		for (const auto* extension: {VK_EXT_ROBUSTNESS_2_EXTENSION_NAME,
+		                             VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME,
 		                             VK_EXT_MESH_SHADER_EXTENSION_NAME,
 		                             VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME}) {
 			if (HasExtension(available_extensions, extension)) {
@@ -1158,16 +1060,12 @@ void WindowContext::CreateVulkan() {
 		}
 	}
 
-	VulkanInitSubgroupSizeControl(graphic_ctx.physical_device, graphic_ctx);
-
-	graphic_ctx.device = VulkanCreateDevice(graphic_ctx.physical_device, r, queue_family,
-	                                        device_extensions, graphic_ctx);
+	graphic_ctx.device = VulkanCreateDevice(graphic_ctx, device_extensions);
 	if (graphic_ctx.device == nullptr) {
 		EXIT("Could not create device");
 	}
 	VULKAN_HPP_DEFAULT_DISPATCHER.init(graphic_ctx.device);
-	graphic_ctx.queue_family = queue_family;
-	graphic_ctx.device.getQueue(queue_family, 0, &graphic_ctx.queue);
+	graphic_ctx.device.getQueue(graphic_ctx.queue_family, 0, &graphic_ctx.queue);
 	EXIT_IF(graphic_ctx.queue == nullptr);
 
 	if (!graphic_ctx.CreateAllocator()) {
@@ -1175,7 +1073,7 @@ void WindowContext::CreateVulkan() {
 	}
 
 	render_context = std::make_unique<RenderContext>(graphic_ctx);
-	LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
+	LibKernel::Memory::InstallGpuResources(render_context.get());
 	presenter = std::make_unique<Presenter>(*this);
 }
 

src/graphics/presentation/window/window.cpp

@@ -1016,7 +1016,7 @@ void WindowContext::UpdateTitle() {
 
 	const auto* device_name = graphic_ctx.GetPhysicalDeviceProperties().deviceName.data();
 	auto text = fmt::format(
-	    "[{} | {}] {}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:f}", KYTY_BUILD_LABEL, build_type,
+	    "[{} | {}] {}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:.0f}", KYTY_BUILD_LABEL, build_type,
 	    (has_title ? title : ""), (has_title ? ", " : ""), (has_title_id ? title_id : ""),
 	    (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""), (has_app_ver ? " " : ""),
 	    device_name, processor_name, frame_num, current_fps);

src/graphics/rt/hardware.cpp

@@ -1,77 +0,0 @@
-#include "graphics/rt/hardware.h"
-
-#include "common/assert.h"
-#include "common/logging/log.h"
-
-#include <algorithm>
-#include <cstring>
-
-namespace Libs::Graphics {
-namespace {
-
-bool ExtensionAvailable(const std::vector<vk::ExtensionProperties>& available, const char* name) {
-	return std::ranges::any_of(available, [name](const auto& extension) {
-		return std::strcmp(extension.extensionName, name) == 0;
-	});
-}
-
-void AppendExtensionIfMissing(std::vector<const char*>& extensions, const char* name) {
-	if (std::ranges::none_of(extensions, [name](const char* extension) {
-		    return std::strcmp(extension, name) == 0;
-	    })) {
-		extensions.push_back(name);
-	}
-}
-
-} // namespace
-
-void GraphicContext::AppendHardwareRayTracingDeviceExtensions(
-    const std::vector<vk::ExtensionProperties>& available_extensions,
-    std::vector<const char*>&                   device_extensions) {
-
-	const char* required[] = {
-	    VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME,
-	    VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME,
-	    VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME,
-	    VK_KHR_RAY_QUERY_EXTENSION_NAME,
-	};
-	for (const auto* extension: required) {
-		if (!ExtensionAvailable(available_extensions, extension)) {
-			LOGF("Vulkan RT: extension %s is unavailable; hardware ray tracing disabled\n",
-			     extension);
-			rt_extensions_enabled = false;
-			return;
-		}
-	}
-	for (const auto* extension: required) {
-		AppendExtensionIfMissing(device_extensions, extension);
-	}
-	if (ExtensionAvailable(available_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME)) {
-		AppendExtensionIfMissing(device_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME);
-	}
-	if (ExtensionAvailable(available_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME)) {
-		AppendExtensionIfMissing(device_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
-	}
-
-	rt_extensions_enabled = true;
-	LOGF("Vulkan RT: enabling hardware ray query extensions\n");
-}
-
-void GraphicContext::LoadHardwareRayTracingFunctions() const {
-	EXIT_IF(device == nullptr);
-	if (!rt_extensions_enabled) {
-		return;
-	}
-
-	const auto& dispatcher = VULKAN_HPP_DEFAULT_DISPATCHER;
-	if (dispatcher.vkGetBufferDeviceAddressKHR == nullptr ||
-	    dispatcher.vkCreateAccelerationStructureKHR == nullptr ||
-	    dispatcher.vkDestroyAccelerationStructureKHR == nullptr ||
-	    dispatcher.vkGetAccelerationStructureBuildSizesKHR == nullptr ||
-	    dispatcher.vkCmdBuildAccelerationStructuresKHR == nullptr ||
-	    dispatcher.vkGetAccelerationStructureDeviceAddressKHR == nullptr) {
-		EXIT("Vulkan RT: failed to load required device functions\n");
-	}
-}
-
-} // namespace Libs::Graphics

src/graphics/rt/hardware.h

@@ -1,10 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_
-
-#include "graphics/host_gpu/graphicContext.h"
-
-#include <vector>
-
-namespace Libs::Graphics {} // namespace Libs::Graphics
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_ */

src/graphics/shader/recompiler/BufferFormat.h

@@ -238,28 +238,13 @@ constexpr BufferFormatInfo GetFormatInfo(Prospero::BufferFormat format) {
 	}
 }
 
-constexpr bool IsKnownFormat(Prospero::BufferFormat format) {
-	return GetFormatInfo(format).type != ComponentType::Unknown;
-}
-
-constexpr uint32_t GetFormatComponentCount(Prospero::BufferFormat format) {
-	return GetFormatInfo(format).component_count;
-}
-
-constexpr uint32_t GetFormatComponentByteOffset(Prospero::BufferFormat format, uint32_t component) {
-	const auto info = GetFormatInfo(format);
+constexpr uint32_t GetFormatComponentByteOffset(const BufferFormatInfo& info, uint32_t component) {
 	if (component >= info.component_count) {
 		return 0;
 	}
 	return info.packed_bitfield ? 0u : info.component_bit_offset[component] / 8u;
 }
 
-constexpr bool CanUseTypedBufferLoad(Prospero::BufferFormat format) {
-	const auto info = GetFormatInfo(format);
-	return info.type != ComponentType::Unknown && !info.packed_bitfield &&
-	       info.component_bits[0] == 32u;
-}
-
 } // namespace Libs::Graphics::ShaderRecompiler::Format
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_ */

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -1,4 +1,5 @@
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/Tessellation.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
@@ -47,14 +48,16 @@ const char* StageName(ShaderType stage) {
 	switch (stage) {
 		case ShaderType::Compute: return "CS";
 		case ShaderType::Vertex: return "VS";
+		case ShaderType::Local: return "LS";
+		case ShaderType::TessellationControl: return "HS";
+		case ShaderType::TessellationEvaluation: return "TES";
 		case ShaderType::Mesh: return "MS";
 		case ShaderType::Pixel: return "PS";
 		default: return "unknown";
 	}
 }
 
-void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase,
-                           const std::string& reason) {
+void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase) {
 	const auto* block        = cfg.FindBlock(cfg.failure_block);
 	const auto  start        = block != nullptr ? block->start_pc : UINT32_MAX;
 	const auto  end          = block != nullptr ? block->end_pc : UINT32_MAX;
@@ -68,7 +71,7 @@ void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg,
 	     CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
 	     static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
-	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
+	     static_cast<uint64_t>(cfg.back_edges.size()), cfg.unsupported_reason.c_str());
 }
 
 enum class EmbeddedFetchValueType {
@@ -77,8 +80,7 @@ enum class EmbeddedFetchValueType {
 	AttribTable,
 	Attrib,
 	BufferTable,
-	Buffer,
-	Index
+	Buffer
 };
 
 struct EmbeddedFetchSgprInfo {
@@ -88,10 +90,6 @@ struct EmbeddedFetchSgprInfo {
 	std::vector<uint32_t>  prolog_loads;
 };
 
-struct EmbeddedFetchVgprInfo {
-	EmbeddedFetchValueType type = EmbeddedFetchValueType::Unknown;
-};
-
 using EmbeddedFetchVectorLanes = std::map<uint64_t, EmbeddedFetchSgprInfo>;
 
 uint64_t EmbeddedFetchVectorLaneKey(uint32_t reg, uint32_t lane) {
@@ -108,7 +106,6 @@ void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg
 	lanes->erase(first, last);
 }
 
-using EmbeddedFetchLoad = Frontend::EmbeddedFetchLoad;
 using EmbeddedFetchData = Frontend::EmbeddedFetchPlan;
 
 bool IsDecodedSgpr(const Decoder::Operand& op) {
@@ -136,20 +133,6 @@ uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) {
 	return inst.opcode == Decoder::Opcode::V_MAD_U64_U32 ? 2u : DecodedDstSize(inst);
 }
 
-bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::S_SETPC_B64:
-		case Decoder::Opcode::S_BRANCH:
-		case Decoder::Opcode::S_CBRANCH_SCC0:
-		case Decoder::Opcode::S_CBRANCH_SCC1:
-		case Decoder::Opcode::S_CBRANCH_VCCZ:
-		case Decoder::Opcode::S_CBRANCH_VCCNZ:
-		case Decoder::Opcode::S_CBRANCH_EXECZ:
-		case Decoder::Opcode::S_CBRANCH_EXECNZ: return true;
-		default: return false;
-	}
-}
-
 void ClearEmbeddedFetchSgprs(std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
                              const Decoder::Operand& dst, uint32_t size) {
 	if (!IsDecodedSgpr(dst)) {
@@ -232,6 +215,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
                                             const ShaderVertexInputInfo* input_info,
                                             uint32_t user_data_base, uint32_t user_data_count,
                                             uint32_t wave_size) {
+	const uint32_t    vertex_index_reg   = input_info->logical_stage == ShaderType::Local ? 2u : 5u;
+	const uint32_t    instance_index_reg = input_info->logical_stage == ShaderType::Local ? 5u : 8u;
 	EmbeddedFetchData data;
 	data.loads.reserve(input_info->resources_num);
 	int32_t vertex_offset_candidate   = -1;
@@ -244,11 +229,14 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 	const int buffer_reg = input_info->fetch_buffer_reg + shift_regs;
 
 	std::array<EmbeddedFetchSgprInfo, 108> sgprs {};
-	std::array<EmbeddedFetchVgprInfo, 256> vgprs {};
+	std::array<bool, 256>                 vgpr_is_index {};
 	EmbeddedFetchVectorLanes               vector_lanes;
 	const bool                             track_vector_lanes =
 	    std::none_of(decoded.instructions.begin(), decoded.instructions.end(),
-	                 [](const auto& inst) { return EmbeddedFetchHasBranch(inst.opcode); });
+	                 [](const auto& inst) {
+		                 return Decoder::IsDirectBranch(inst.opcode) ||
+		                        inst.opcode == Decoder::Opcode::S_SETPC_B64;
+	                 });
 
 	if (attrib_reg >= 0 && attrib_reg < static_cast<int>(sgprs.size())) {
 		sgprs[attrib_reg].type = EmbeddedFetchValueType::AttribTable;
@@ -269,16 +257,17 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 		// corresponding direct-draw offsets before fetching.
 		const bool vertex_index_accumulator =
 		    IsDecodedVgpr(inst.dst) &&
-		    (inst.dst.reg == 0 || (user_data_base == 8 && inst.dst.reg == 5));
+		    (inst.dst.reg == 0 || (user_data_base == 8 && inst.dst.reg == vertex_index_reg));
 		const bool instance_index_accumulator =
-		    IsDecodedVgpr(inst.dst) && (inst.dst.reg == (user_data_base == 8 ? 8u : 3u));
+		    IsDecodedVgpr(inst.dst) &&
+		    (inst.dst.reg == (user_data_base == 8 ? instance_index_reg : 3u));
 		uint32_t   sad_zero = 0;
 		const bool index_offset_add =
-		    (vertex_index_accumulator || instance_index_accumulator) &&
-		    IsDecodedSgpr(inst.src0) &&
+		    (vertex_index_accumulator || instance_index_accumulator) && IsDecodedSgpr(inst.src0) &&
 		    ((inst.opcode == Decoder::Opcode::V_ADD_I32 && IsDecodedVgpr(inst.src1) &&
 		      inst.src1.reg == inst.dst.reg) ||
-		     (user_data_base == 8 && (inst.dst.reg == 5 || inst.dst.reg == 8) &&
+		     (user_data_base == 8 &&
+		      (inst.dst.reg == vertex_index_reg || inst.dst.reg == instance_index_reg) &&
 		      inst.opcode == Decoder::Opcode::V_SAD_U32 && IsDecodedVgpr(inst.src2) &&
 		      inst.src2.reg == inst.dst.reg &&
 		      TryDecodedOperandConstant(sgprs, inst.src1, sad_zero) && sad_zero == 0));
@@ -299,8 +288,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 		switch (inst.opcode) {
 			case Decoder::Opcode::V_WRITELANE_B32: {
 				uint32_t lane = 0;
-				if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
-					vgprs[inst.dst.reg] = {};
+				if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgpr_is_index.size()) {
+					vgpr_is_index[inst.dst.reg] = false;
 				}
 				if (track_vector_lanes && IsDecodedVgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
 				    DecodedSgprReg(inst.src0) < sgprs.size() &&
@@ -405,13 +394,13 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 						ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
 					}
 				} else if (inst.opcode == Decoder::Opcode::V_CNDMASK_B32) {
-					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
+					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgpr_is_index.size()) {
 						ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
 					}
-					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size() &&
-					    IsDecodedVgpr(inst.src0) && inst.src0.reg == 8 &&
-					    IsDecodedVgpr(inst.src1) && inst.src1.reg == 5) {
-						vgprs[inst.dst.reg].type = EmbeddedFetchValueType::Index;
+					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgpr_is_index.size() &&
+					    IsDecodedVgpr(inst.src0) && inst.src0.reg == instance_index_reg &&
+					    IsDecodedVgpr(inst.src1) && inst.src1.reg == vertex_index_reg) {
+						vgpr_is_index[inst.dst.reg] = true;
 					}
 				} else if (IsEmbeddedFetchAttribPropagationAlu(inst)) {
 					if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
@@ -441,16 +430,11 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 						}
 					}
 				} else if (IsEmbeddedFetchBufferLoad(inst)) {
-					if (IsDecodedVgpr(inst.src0) && inst.src0.reg < vgprs.size() &&
-					    vgprs[inst.src0.reg].type == EmbeddedFetchValueType::Index &&
+					if (IsDecodedVgpr(inst.src0) && inst.src0.reg < vgpr_is_index.size() &&
+					    vgpr_is_index[inst.src0.reg] &&
 					    IsDecodedSgpr(inst.src1) && DecodedSgprReg(inst.src1) < sgprs.size() &&
 					    sgprs[DecodedSgprReg(inst.src1)].type == EmbeddedFetchValueType::Buffer) {
-						const auto&       buffer = sgprs[DecodedSgprReg(inst.src1)];
-						EmbeddedFetchLoad load;
-						load.pc           = inst.pc;
-						load.attrib_id    = buffer.attrib_id;
-						load.components   = DecodedDstSize(inst);
-						load.prolog_loads = buffer.prolog_loads;
+						const auto& buffer = sgprs[DecodedSgprReg(inst.src1)];
 						if (data.loads.empty()) {
 							if (!vertex_offset_conflict) {
 								data.vertex_offset_sgpr = vertex_offset_candidate;
@@ -459,7 +443,11 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 								data.instance_offset_sgpr = instance_offset_candidate;
 							}
 						}
-						data.loads.push_back(load);
+						auto& load        = data.loads.emplace_back();
+						load.pc           = inst.pc;
+						load.attrib_id    = buffer.attrib_id;
+						load.components   = DecodedDstSize(inst);
+						load.prolog_loads = buffer.prolog_loads;
 					}
 				}
 				break;
@@ -467,7 +455,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 		if (inst.opcode == Decoder::Opcode::V_MOVRELD_B32) {
 			vector_lanes.clear();
 		} else if (inst.opcode != Decoder::Opcode::V_WRITELANE_B32 && IsDecodedVgpr(inst.dst)) {
-			for (uint32_t i = 0; i < EmbeddedFetchDstSize(inst) && inst.dst.reg + i < vgprs.size();
+			for (uint32_t i = 0;
+			     i < EmbeddedFetchDstSize(inst) && inst.dst.reg + i < vgpr_is_index.size();
 			     i++) {
 				ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg + i);
 			}
@@ -480,22 +469,9 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<const uint32_t> back,
                                     std::vector<uint32_t>& joined_code) {
 	EXIT_IF(back.empty());
-	Decoder::Program result;
-	uint32_t         front_words = 0;
-	while (front_words < front.size()) {
-		auto& inst = result.instructions.emplace_back();
-		Decoder::DecodeInstruction(front, front_words, inst);
-		front_words += inst.word_count;
-		if (inst.opcode == Decoder::Opcode::S_SETPC_B64) {
-			EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr ||
-			                     inst.src0.reg != 6u);
-			break;
-		}
-		EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_ENDPGM);
-	}
-	EXIT_IF(result.instructions.empty() ||
-	        result.instructions.back().opcode != Decoder::Opcode::S_SETPC_B64);
-	joined_code.assign(front.begin(), front.begin() + front_words);
+	auto       result      = Decoder::DecodeFrontProgram(front);
+	const auto front_words = static_cast<uint32_t>(result.code.size());
+	joined_code.assign(result.code.begin(), result.code.end());
 	joined_code.insert(joined_code.end(), back.begin(), back.end());
 	// The merged-stage ABI passes the back shader in s[6:7]. Give that handoff an
 	// ordinary CFG edge, retaining both bodies in one register and LDS lifetime.
@@ -523,7 +499,9 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		EXIT("shader recompiler input is empty\n");
 	}
 	if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
-	    options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh) {
+	    options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh &&
+	    options.stage != ShaderType::Local && options.stage != ShaderType::TessellationControl &&
+	    options.stage != ShaderType::TessellationEvaluation) {
 		EXIT("shader recompiler received unsupported stage %u\n",
 		     static_cast<unsigned>(options.stage));
 	}
@@ -541,8 +519,14 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 
 	Decoder::Program decoded;
 	std::vector<uint32_t> joined_code;
-	if (options.stage == ShaderType::Mesh) {
+	if (!options.back_code.empty()) {
 		decoded = DecodeFusedProgram(code, options.back_code, joined_code);
+	} else if (options.stage == ShaderType::Local) {
+		decoded = Decoder::DecodeFrontProgram(code);
+		// The separately compiled hull half runs in the next Vulkan stage.
+		auto& handoff     = decoded.instructions.back();
+		handoff.opcode    = Decoder::Opcode::S_ENDPGM;
+		handoff.src_count = 0;
 	} else {
 		Decoder::DecodeProgram(code, decoded);
 	}
@@ -567,27 +551,13 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
 	     static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
-	bool        dispatcher_fallback = false;
-	std::string dispatcher_reason;
 	if (cfg.irreducible) {
-		dispatcher_fallback = true;
-		dispatcher_reason   = cfg.unsupported_reason;
-		LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
+		LogDispatcherFallback(options, cfg, "build");
 	} else {
-		const auto unstructured_cfg = cfg;
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
 		if (!CFG::Structurize(cfg)) {
-			dispatcher_fallback      = true;
-			dispatcher_reason        = cfg.unsupported_reason;
-			const auto failure_kind  = cfg.failure_kind;
-			const auto failure_block = cfg.failure_block;
-			LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
-			cfg                    = unstructured_cfg;
-			cfg.unsupported        = true;
-			cfg.failure_kind       = failure_kind;
-			cfg.failure_block      = failure_block;
-			cfg.unsupported_reason = dispatcher_reason;
+			LogDispatcherFallback(options, cfg, "structurize");
 		} else {
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(cfg.blocks.size()));
@@ -599,45 +569,25 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		     static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
 	}
 
-	const ShaderVertexInputInfo*  vertex  = nullptr;
-	const ShaderPixelInputInfo*   pixel   = nullptr;
-	const ShaderComputeInputInfo* compute = nullptr;
-	switch (options.stage) {
-		case ShaderType::Vertex:
-		case ShaderType::Mesh: vertex = options.input_info.vertex; break;
-		case ShaderType::Pixel:
-			pixel = options.input_info.pixel;
-			break;
-		case ShaderType::Compute:
-			compute = options.input_info.compute;
-			break;
-		default: break;
-	}
 	EmbeddedFetchData embedded_fetch;
-	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
-		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base,
-		                                           static_cast<uint32_t>(options.user_data.size()),
-		                                           options.wave_size);
+	if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
+	    options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
+		embedded_fetch = DetectEmbeddedVertexFetch(
+		    decoded, options.input_info.vertex, options.user_data_base,
+		    static_cast<uint32_t>(options.user_data.size()), options.wave_size);
 		if (!embedded_fetch.loads.empty()) {
 			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(embedded_fetch.loads.size()));
 		}
 	}
 	Frontend::TranslateOptions translate_options {
-	    .stage               = options.stage,
-	    .wave_size           = options.wave_size,
-	    .shader_hash         = options.shader_hash,
-	    .user_data_base      = options.user_data_base,
-	    .user_data_count     = static_cast<uint32_t>(options.user_data.size()),
-	    .scratch_dwords      = options.scratch_dwords,
-	    .dispatcher_fallback = dispatcher_fallback,
-	    .cfg_failure_kind    = cfg.failure_kind,
-	    .fallback_reason     = dispatcher_reason.empty() ? cfg.unsupported_reason
-	                                                    : dispatcher_reason,
-	    .vertex              = vertex,
-	    .pixel               = pixel,
-	    .compute             = compute,
-	    .embedded_fetch      = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
+	    .stage            = options.stage,
+	    .wave_size        = options.wave_size,
+	    .shader_hash      = options.shader_hash,
+	    .user_data_base   = options.user_data_base,
+	    .user_data_count  = static_cast<uint32_t>(options.user_data.size()),
+	    .input_info       = options.input_info,
+	    .embedded_fetch   = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
 	};
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
@@ -660,15 +610,11 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		IR::RemoveIdentities(ir.blocks);
 		IR::EliminateDeadCode(ir.blocks);
 	}
+	LowerTessellationMemory(ir, options);
 	IR::BuildSrtPlan(ir);
 	IR::EliminateDeadCode(ir.blocks);
 	IR::TrackResources(ir);
 	IR::EliminateDeadCode(ir.blocks);
-	if (options.stage == ShaderType::Vertex) {
-		ir.info.vertex_offset_sgpr   = embedded_fetch.vertex_offset_sgpr;
-		ir.info.instance_offset_sgpr = embedded_fetch.instance_offset_sgpr;
-	}
-
 	TranslateResult result;
 	result.program = std::move(ir);
 	if (options.dump_ir) {
@@ -687,24 +633,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
 
-	const ShaderVertexInputInfo*  vertex  = nullptr;
-	const ShaderPixelInputInfo*   pixel   = nullptr;
-	const ShaderComputeInputInfo* compute = nullptr;
-	switch (options.stage) {
-		case ShaderType::Vertex:
-		case ShaderType::Mesh: vertex = options.input_info.vertex; break;
-		case ShaderType::Pixel: pixel = options.input_info.pixel; break;
-		case ShaderType::Compute: compute = options.input_info.compute; break;
-		default: EXIT("invalid shader stage\n");
-	}
-
-	IR::ShaderInfoOptions info_options;
-	info_options.vertex  = vertex;
-	info_options.pixel   = pixel;
-	info_options.compute = compute;
-	IR::CollectShaderInfo(ir, info_options);
+	IR::CollectShaderInfo(ir, options.input_info);
 	IR::AllocateBindings(ir, push_data_start_dword);
-	Spirv::AnalyzeProgramRequirements(ir);
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(translated.cfg_dump, ir);
@@ -714,11 +644,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	}
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
 	auto spirv = Spirv::EmitProgram(ir, options.input_info);
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,
 	     static_cast<uint64_t>(spirv.size()),
 	     static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
 	                               std::chrono::steady_clock::now() - emit_begin)

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -15,7 +15,6 @@ struct CompileOptions {
 	ShaderType                  stage           = ShaderType::Compute;
 	uint32_t                    wave_size       = 64;
 	uint32_t                    user_data_base  = 0;
-	uint32_t                    scratch_dwords  = 0;
 	uint64_t                    shader_hash     = 0;
 	bool                        dump_ir                    = true;
 	bool                        early_dump                 = false;

src/graphics/shader/recompiler/Tessellation.cpp

@@ -0,0 +1,330 @@
+#include "graphics/shader/recompiler/Tessellation.h"
+
+#include "common/assert.h"
+#include "common/logging/log.h"
+#include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
+#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
+
+#include <algorithm>
+#include <array>
+#include <unordered_map>
+#include <utility>
+
+namespace Libs::Graphics::ShaderRecompiler {
+namespace {
+
+struct TessellationAddress {
+	enum class Kind { Unknown, Affine, PackedControlPoint };
+	Kind     kind        = Kind::Unknown;
+	uint32_t coefficient = 0;
+	uint32_t constant    = 0;
+};
+
+uint32_t ReflectTessellationStride(const Decoder::Program& program, bool local,
+                                   uint32_t control_points, uint32_t input_stride) {
+	using namespace Decoder;
+	using Address = TessellationAddress;
+	using Kind    = Address::Kind;
+	std::array<Address, IR::NumVectorRegs> registers {};
+	registers[local ? 3u : 1u] =
+	    local ? Address {Kind::Affine, 1u, 0u} : Address {Kind::PackedControlPoint};
+	const auto constant = [](uint32_t value) { return Address {Kind::Affine, 0u, value}; };
+	const auto read     = [&](const Operand& operand) -> Address {
+		if (operand.absolute || operand.negate || operand.sdwa_sext || operand.op_sel ||
+		    operand.op_sel_hi || operand.negate_hi || operand.dpp)
+			return {};
+		Address value;
+		if (operand.kind == OperandKind::IntegerInlineConstant ||
+		    operand.kind == OperandKind::LiteralConstant) {
+			value = constant(operand.value);
+		} else if (operand.kind == OperandKind::Vgpr) {
+			value = registers.at(operand.reg);
+		}
+		// The native packed HS ID's low byte is the patch ordinal. The logical
+		// interface addresses one patch, as in the frontend's v1 initialization.
+		if (value.kind == Kind::PackedControlPoint && operand.sdwa_sel == 0u) return constant(0u);
+		return operand.sdwa_sel == 6u ? value : Address {};
+	};
+	const auto affine = [](uint64_t coefficient, uint64_t offset) {
+		return coefficient <= UINT32_MAX && offset <= UINT32_MAX
+		           ? Address {Kind::Affine, static_cast<uint32_t>(coefficient),
+		                      static_cast<uint32_t>(offset)}
+		           : Address {};
+	};
+	const auto add = [&](Address lhs, Address rhs) {
+		return lhs.kind == Kind::Affine && rhs.kind == Kind::Affine
+		           ? affine(uint64_t {lhs.coefficient} + rhs.coefficient,
+		                    uint64_t {lhs.constant} + rhs.constant)
+		           : Address {};
+	};
+	const auto multiply = [&](Address lhs, Address rhs) {
+		if ((lhs.kind == Kind::Affine && lhs.coefficient == 0u && lhs.constant == 0u) ||
+		    (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant == 0u))
+			return constant(0u);
+		if (lhs.kind != Kind::Affine || rhs.kind != Kind::Affine) return Address {};
+		if (rhs.coefficient != 0u) std::swap(lhs, rhs);
+		return rhs.coefficient == 0u ? affine(uint64_t {lhs.coefficient} * rhs.constant,
+		                                      uint64_t {lhs.constant} * rhs.constant)
+		                             : Address {};
+	};
+	const auto low24 = [&](Address value) {
+		if (value.kind == Kind::Affine && value.coefficient == 0u)
+			return constant(value.constant & 0xffffffu);
+		return value.kind == Kind::Affine &&
+		               uint64_t {value.coefficient} * (control_points - 1u) + value.constant <=
+		                   0xffffffu
+		           ? value
+		           : Address {};
+	};
+	uint32_t stride = 0;
+	for (const auto& inst: program.instructions) {
+		// Stage exits preserve the active path's definitions. An internal join
+		// would require merging register definitions.
+		EXIT_NOT_IMPLEMENTED(IsDirectBranch(inst.opcode) &&
+		                     inst.branch_target != program.instructions.back().pc);
+		const bool local_store =
+		    inst.opcode == Opcode::DS_WRITE_B32 || inst.opcode == Opcode::DS_WRITE2_B32;
+		const bool buffer_store = inst.opcode == Opcode::BUFFER_STORE_DWORD ||
+		                          inst.opcode == Opcode::BUFFER_STORE_DWORDX2 ||
+		                          inst.opcode == Opcode::BUFFER_STORE_DWORDX3 ||
+		                          inst.opcode == Opcode::BUFFER_STORE_DWORDX4;
+		const bool control_store =
+		    !local && buffer_store && inst.src2.kind == OperandKind::Sgpr && inst.src2.reg == 2u;
+		const bool control_read =
+		    !local && (inst.opcode == Opcode::DS_READ_B32 || inst.opcode == Opcode::DS_READ2_B32);
+		if ((local && local_store) || control_store || control_read) {
+			const auto address = read(inst.src0);
+			if (address.kind != Kind::Affine) {
+				EXIT("%s tessellation address is not affine at pc 0x%08x\n", local ? "LS" : "HS",
+				     inst.pc);
+			}
+			if (address.coefficient != 0u) {
+				const auto expected = control_read ? input_stride : stride;
+				EXIT_NOT_IMPLEMENTED((address.coefficient & 3u) != 0u ||
+				                     (expected != 0u && expected != address.coefficient));
+				if (!control_read) stride = address.coefficient;
+			}
+		}
+		// Store vdata is a source, despite occupying the decoder's dst field.
+		if (local_store || buffer_store || inst.dst.kind != OperandKind::Vgpr) continue;
+		const auto lhs   = read(inst.src0);
+		const auto rhs   = read(inst.src1);
+		const auto third = read(inst.src2);
+		Address    value;
+		switch (inst.opcode) {
+			case Opcode::V_BFE_U32:
+				if (lhs.kind == Kind::PackedControlPoint && rhs.kind == Kind::Affine &&
+				    rhs.coefficient == 0u && third.kind == Kind::Affine &&
+				    third.coefficient == 0u) {
+					if (rhs.constant == 8u && third.constant == 5u) value = {Kind::Affine, 1u, 0u};
+					if (rhs.constant == 0u && third.constant == 8u) value = constant(0u);
+				}
+				break;
+			case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break;
+			case Opcode::V_MAD_U32_U24: value = add(multiply(low24(lhs), low24(rhs)), third); break;
+			case Opcode::V_LSHL_ADD_U32:
+				if (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant < 32u)
+					value = add(multiply(lhs, constant(1u << rhs.constant)), third);
+				break;
+			case Opcode::V_LSHLREV_B32:
+				if (lhs.kind == Kind::Affine && lhs.coefficient == 0u && lhs.constant < 32u)
+					value = multiply(rhs, constant(1u << lhs.constant));
+				break;
+			case Opcode::V_SUB_NC_U32:
+				if (lhs.kind == Kind::Affine && rhs.kind == Kind::Affine &&
+				    lhs.coefficient >= rhs.coefficient && lhs.constant >= rhs.constant)
+					value = affine(lhs.coefficient - rhs.coefficient, lhs.constant - rhs.constant);
+				break;
+			default: break;
+		}
+		if (inst.dst.sdwa_sel != 6u || inst.dst.op_sel || inst.dst.omod || inst.dst.clamp ||
+		    inst.dst.dpp)
+			value = {};
+		for (uint32_t index = 0;
+		     index < std::max(inst.data_dwords, 1u) && inst.dst.reg + index < registers.size(); index++) {
+			registers[inst.dst.reg + index] = {};
+		}
+		registers.at(inst.dst.reg) = value;
+	}
+	EXIT_NOT_IMPLEMENTED(stride == 0u);
+	return stride;
+}
+
+const IR::Inst* TessellationBufferBase(const IR::Inst& inst) {
+	if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::None || inst.NumArgs() <= 3u)
+		return nullptr;
+	const auto* base = inst.Arg(3).Resolve().TryInstruction();
+	return base != nullptr && base->GetOpcode() == IR::ValueOpcode::TessellationBase ? base
+	                                                                                 : nullptr;
+}
+
+IR::Value ActiveAddress(IR::Block& block, IR::Block::iterator before, IR::Value value,
+                        IR::Value predicate, std::unordered_map<IR::Inst*, IR::Value>& resolved) {
+	using namespace IR;
+	value        = value.Resolve();
+	auto* source = value.TryInstruction();
+	if (source == nullptr) return value;
+	if (const auto found = resolved.find(source); found != resolved.end()) return found->second;
+	if (source->GetOpcode() == ValueOpcode::SelectU32) {
+		return source->Arg(0).Resolve() == predicate.Resolve()
+		           ? ActiveAddress(block, before, source->Arg(1), predicate, resolved)
+		           : value;
+	}
+	if (source->GetOpcode() != ValueOpcode::IAdd32 && source->GetOpcode() != ValueOpcode::ISub32)
+		return value;
+	const auto lhs = ActiveAddress(block, before, source->Arg(0), predicate, resolved);
+	const auto rhs = ActiveAddress(block, before, source->Arg(1), predicate, resolved);
+	if (lhs != source->Arg(0).Resolve() || rhs != source->Arg(1).Resolve()) {
+		auto copy = block.PrependNewInst(before, source->GetOpcode(), {lhs, rhs},
+		                                 source->Flags<uint64_t>());
+		value     = Value(&*copy);
+	}
+	resolved.emplace(source, value);
+	return value;
+}
+
+} // namespace
+
+void AnalyzeTessellationPrograms(std::span<const uint32_t> local, std::span<const uint32_t> control,
+                                 ShaderTessellationInputInfo& info) {
+	const auto       local_program = Decoder::DecodeFrontProgram(local);
+	Decoder::Program control_program;
+	Decoder::DecodeProgram(control, control_program);
+	info.ls_stride = ReflectTessellationStride(local_program, true, info.input_control_points, 0u);
+	info.hs_stride = ReflectTessellationStride(control_program, false, info.output_control_points,
+	                                           info.ls_stride);
+	LOGF("Tessellation interface: input_cp=%u output_cp=%u ls_stride=%u hs_stride=%u\n",
+	     info.input_control_points, info.output_control_points, info.ls_stride, info.hs_stride);
+}
+
+void LowerTessellationMemory(IR::Program& program, const CompileOptions& options) {
+	using namespace IR;
+	if (options.stage != ShaderType::Local && options.stage != ShaderType::TessellationControl &&
+	    options.stage != ShaderType::TessellationEvaluation) {
+		return;
+	}
+	const auto& tess = options.input_info.vertex->tess;
+	// Ring addresses can reuse data VGPRs. Their inactive values are irrelevant to
+	// a store guarded by the same EXEC predicate, but must remain intact elsewhere.
+	for (auto* block: program.blocks) {
+		for (auto it = block->begin(); it != block->end(); ++it) {
+			if (TessellationBufferBase(*it) == nullptr) continue;
+			std::unordered_map<Inst*, Value> resolved;
+			it->SetArg(
+			    2, ActiveAddress(*block, it, it->Arg(2), it->Arg(it->NumArgs() - 1u), resolved));
+		}
+	}
+	ConstantPropagationPass(program.blocks);
+	uint32_t reads = 0, writes = 0, factors = 0;
+	for (auto* block: program.blocks) {
+		for (auto it = block->begin(); it != block->end(); ++it) {
+			auto&      inst   = *it;
+			const auto shared = SharedAccessOf(inst.GetOpcode());
+			const auto buffer = BufferAccessOf(inst.GetOpcode());
+			if (shared == SharedAccess::None && buffer == BufferAccess::None) {
+				continue;
+			}
+			const auto&           memory = program.memory_info.at(inst.Flags<MemoryFlags>().index);
+			bool                  write  = false;
+			uint32_t              components = 0;
+			TessellationAttribute kind;
+			Value                 address, predicate;
+			if (shared != SharedAccess::None && memory.kind == ResourceKind::Lds) {
+				write = shared == SharedAccess::Write;
+				EXIT_NOT_IMPLEMENTED(memory.data_bits != 32u ||
+				                     (options.stage == ShaderType::Local
+				                          ? !write
+				                          : options.stage != ShaderType::TessellationControl ||
+				                                shared != SharedAccess::Read));
+				kind       = write ? TessellationAttribute::LocalOutput
+				                   : TessellationAttribute::ControlInput;
+				components = SharedComponentCount(inst.GetOpcode());
+				address    = inst.Arg(0);
+				predicate  = inst.Arg(inst.NumArgs() - 1u);
+			} else if (buffer != BufferAccess::None && memory.kind == ResourceKind::Buffer) {
+				const auto* base = TessellationBufferBase(inst);
+				if (base == nullptr) {
+					continue;
+				}
+				EXIT_NOT_IMPLEMENTED(memory.data_bits != 32u || memory.formatted || memory.idxen ||
+				                     !memory.offen || buffer == BufferAccess::Atomic);
+				write      = buffer == BufferAccess::Write;
+				components = BufferComponentCount(inst.GetOpcode());
+				address    = inst.Arg(2).Resolve();
+				predicate  = inst.Arg(inst.NumArgs() - 1u);
+				if (base->Arg(0).U32() == 1u) {
+					EXIT_NOT_IMPLEMENTED(!write ||
+					                     options.stage != ShaderType::TessellationControl);
+					kind = TessellationAttribute::Factor;
+					factors += components;
+				} else {
+					kind = write ? TessellationAttribute::ControlOutput
+					             : TessellationAttribute::EvaluationInput;
+					EXIT_NOT_IMPLEMENTED(write
+					                         ? options.stage != ShaderType::TessellationControl
+					                         : options.stage != ShaderType::TessellationEvaluation);
+					if (address.IsImmediate() &&
+					    address.U32() >= tess.hs_stride * tess.output_control_points) {
+						EXIT_NOT_IMPLEMENTED(!write);
+						kind = TessellationAttribute::PatchOutput;
+					}
+				}
+			} else {
+				continue;
+			}
+			const auto emit = [&](ValueOpcode opcode, std::initializer_list<Value> args) {
+				return Value(&*block->PrependNewInst(it, opcode, args));
+			};
+			std::array<Value, 4> values;
+			for (uint32_t component = 0; component < components; component++) {
+				const auto offset = memory.offset + 4u * component;
+				const auto byte_address =
+				    offset == 0u ? address : emit(ValueOpcode::IAdd32, {address, Value(offset)});
+				if (write) {
+					Value data = shared != SharedAccess::None ? inst.Arg(component + 1u)
+					             : components == 1u
+					                 ? inst.Arg(4)
+					                 : emit(components == 2u   ? ValueOpcode::CompositeExtractU32x2
+					                        : components == 3u ? ValueOpcode::CompositeExtractU32x3
+					                                           : ValueOpcode::CompositeExtractU32x4,
+					                        {inst.Arg(4), Value(component)});
+					emit(ValueOpcode::SetTessellationAttribute,
+					     {Value(static_cast<uint32_t>(kind)), byte_address, data, predicate});
+					writes++;
+				} else {
+					values[component] =
+					    emit(ValueOpcode::GetTessellationAttribute,
+					         {Value(static_cast<uint32_t>(kind)), byte_address, predicate});
+					reads++;
+				}
+			}
+			if (!write) {
+				Value replacement = values[0];
+				if (components == 2u)
+					replacement =
+					    emit(ValueOpcode::CompositeConstructU32x2, {values[0], values[1]});
+				if (components == 3u)
+					replacement = emit(ValueOpcode::CompositeConstructU32x3,
+					                   {values[0], values[1], values[2]});
+				if (components == 4u)
+					replacement = emit(ValueOpcode::CompositeConstructU32x4,
+					                   {values[0], values[1], values[2], values[3]});
+				inst.ReplaceUsesWith(replacement);
+			}
+			inst.Invalidate();
+		}
+	}
+	ConstantPropagationPass(program.blocks);
+	RemoveIdentities(program.blocks);
+	EliminateDeadCode(program.blocks);
+	LOGF("%s tessellation lowering: reads=%u writes=%u factors=%u\n",
+	     options.stage == ShaderType::Local                 ? "LS"
+	     : options.stage == ShaderType::TessellationControl ? "HS"
+	                                                       : "TES",
+	     reads, writes, factors);
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler

src/graphics/shader/recompiler/Tessellation.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+#include <span>
+
+namespace Libs::Graphics {
+struct ShaderTessellationInputInfo;
+}
+
+namespace Libs::Graphics::ShaderRecompiler {
+
+struct CompileOptions;
+namespace IR {
+struct Program;
+}
+
+void AnalyzeTessellationPrograms(std::span<const uint32_t> local, std::span<const uint32_t> control,
+                                 ShaderTessellationInputInfo& info);
+void LowerTessellationMemory(IR::Program& program, const CompileOptions& options);
+
+} // namespace Libs::Graphics::ShaderRecompiler

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -1,6 +1,6 @@
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 
-#include "common/debug.h"
+#include "common/assert.h"
 
 #include <algorithm>
 #include <cstring>
@@ -15,9 +15,9 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
 	if (words_num == 0) {
 		return;
 	}
-	const auto opcode     = words[0];
-	const auto word_count = static_cast<uint32_t>(words_num);
-	section.push_back((word_count << 16u) | opcode);
+	const uint32_t opcode     = words[0];
+	const auto     word_count = static_cast<uint32_t>(words_num);
+	section.push_back((word_count << spv::WordCountShift) | opcode);
 	section.insert(section.end(), words + 1, words + words_num);
 }
 
@@ -36,15 +36,17 @@ void Builder::RequireVersion(uint32_t version) {
 	m_version = std::max(m_version, version);
 }
 
-void Builder::RequireCapability(uint32_t capability) {
+void Builder::RequireCapability(spv::Capability capability) {
 	if (m_required_capabilities.insert(capability).second) {
-		AddCapability({capability});
+		AppendInstruction(m_capabilities, spv::OpCapability, capability);
 	}
 }
 
 void Builder::RequireExtension(const char* name) {
 	if (m_required_extensions.emplace(name).second) {
-		AddExtension(name);
+		std::vector<uint32_t> operands;
+		AppendString(operands, name);
+		AppendInstruction(m_extensions, spv::OpExtension, operands);
 	}
 }
 
@@ -54,41 +56,31 @@ uint32_t Builder::Import(const char* name) {
 	}
 	const auto id = AllocateId();
 	m_import_ids.emplace(name, id);
-	AddExtInstImport(id, name);
+	std::vector<uint32_t> operands = {id};
+	AppendString(operands, name);
+	AppendInstruction(m_ext_inst_imports, spv::OpExtInstImport, operands);
 	return id;
 }
 
-uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) {
-	return Type(opcode, std::vector<uint32_t>(operands));
-}
-
-uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
-	std::vector<uint32_t> key;
-	key.reserve(operands.size() + 2u);
-	key.push_back(opcode);
-	key.push_back(static_cast<uint32_t>(operands.size()));
-	key.insert(key.end(), operands.begin(), operands.end());
+uint32_t Builder::DeclareType(spv::Op opcode, std::vector<uint32_t> key) {
 	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
 		return it->second;
 	}
 	const auto id = AllocateId();
+	AppendInstruction(m_declarations, opcode, id, std::span<const uint32_t>(key).subspan(2));
 	m_declaration_ids.emplace(std::move(key), id);
-	std::vector<uint32_t> words {opcode, id};
-	words.insert(words.end(), operands.begin(), operands.end());
-	AppendInstructionWords(m_declarations, words.data(), words.size());
 	return id;
 }
 
-uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
-                                std::initializer_list<TypeAnnotation> annotations) {
+uint32_t Builder::DeclareDecoratedType(spv::Op opcode, std::vector<uint32_t> key,
+                                       std::initializer_list<TypeAnnotation> annotations) {
 	if (annotations.size() == 0) {
-		return Type(opcode, operands);
+		return DeclareType(opcode, std::move(key));
 	}
-	std::vector<uint32_t> key {opcode, static_cast<uint32_t>(operands.size())};
-	key.insert(key.end(), operands.begin(), operands.end());
+	const auto operand_count = key[1];
 	key.push_back(static_cast<uint32_t>(annotations.size()));
 	for (const auto& annotation: annotations) {
-		key.push_back(annotation.opcode);
+		AppendOperand(key, annotation.opcode);
 		key.push_back(static_cast<uint32_t>(annotation.operands.size()));
 		key.insert(key.end(), annotation.operands.begin(), annotation.operands.end());
 	}
@@ -96,48 +88,35 @@ uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t>
 		return it->second;
 	}
 	const auto id = AllocateId();
+	AppendInstruction(m_declarations, opcode, id,
+	                  std::span<const uint32_t>(key).subspan(2, operand_count));
 	m_declaration_ids.emplace(std::move(key), id);
-	std::vector<uint32_t> words {opcode, id};
-	words.insert(words.end(), operands.begin(), operands.end());
-	AppendInstructionWords(m_declarations, words.data(), words.size());
 	for (const auto& annotation: annotations) {
-		words = {annotation.opcode, id};
-		words.insert(words.end(), annotation.operands.begin(), annotation.operands.end());
-		AppendInstructionWords(m_annotations, words.data(), words.size());
+		AppendInstruction(m_annotations, annotation.opcode, id, annotation.operands);
 	}
 	return id;
 }
 
-uint32_t Builder::Constant(uint32_t opcode, uint32_t type,
-                           std::initializer_list<uint32_t> operands) {
-	return Constant(opcode, type, std::vector<uint32_t>(operands));
-}
-
-uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) {
-	std::vector<uint32_t> key;
-	key.reserve(operands.size() + 2u);
-	key.push_back(opcode);
-	key.push_back(type);
-	key.insert(key.end(), operands.begin(), operands.end());
+uint32_t Builder::DeclareConstant(spv::Op opcode, std::vector<uint32_t> key) {
 	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
 		return it->second;
 	}
 	const auto id = AllocateId();
+	AppendInstruction(m_declarations, opcode, key[1], id,
+	                  std::span<const uint32_t>(key).subspan(2));
 	m_declaration_ids.emplace(std::move(key), id);
-	std::vector<uint32_t> words {opcode, type, id};
-	words.insert(words.end(), operands.begin(), operands.end());
-	AppendInstructionWords(m_declarations, words.data(), words.size());
 	return id;
 }
 
-uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) {
+uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class) {
 	const auto id = AllocateId();
 	DefineGlobalVariable(id, pointer_type, storage_class);
 	return id;
 }
 
-void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) {
-	AddType({59u, pointer_type, id, storage_class});
+void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type,
+                                   spv::StorageClass storage_class) {
+	AppendInstruction(m_declarations, spv::OpVariable, pointer_type, id, storage_class);
 }
 
 void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
@@ -156,49 +135,20 @@ void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
 	}
 }
 
-void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-                                const std::vector<uint32_t>& operands) {
-	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
-	section.push_back((word_count << 16u) | opcode);
-	section.insert(section.end(), operands.begin(), operands.end());
+void Builder::AddMemoryModel(spv::AddressingModel addressing_model, spv::MemoryModel memory_model) {
+	AppendInstruction(m_memory_model, spv::OpMemoryModel, addressing_model, memory_model);
 }
 
-void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-                                std::initializer_list<uint32_t> operands) {
-	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
-	section.push_back((word_count << 16u) | opcode);
-	section.insert(section.end(), operands.begin(), operands.end());
-}
-
-void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_capabilities, 17u, operands);
-}
-
-void Builder::AddExtension(const char* name) {
+void Builder::AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point,
+                            const char* name, const std::vector<uint32_t>& interfaces) {
 	std::vector<uint32_t> operands;
-	AppendString(operands, name);
-	AppendInstruction(m_extensions, 10u, operands);
-}
-
-void Builder::AddExtInstImport(uint32_t id, const char* name) {
-	std::vector<uint32_t> operands = {id};
-	AppendString(operands, name);
-	AppendInstruction(m_ext_inst_imports, 11u, operands);
-}
-
-void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_memory_model, 14u, operands);
-}
-
-void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-                            const std::vector<uint32_t>& interfaces) {
-	std::vector<uint32_t> operands = {execution_model, entry_point};
+	AppendOperands(operands, execution_model, entry_point);
 	AppendString(operands, name);
 	operands.insert(operands.end(), interfaces.begin(), interfaces.end());
 	if (m_version >= 0x00010400u) {
 		for (size_t offset = 0; offset < m_declarations.size();) {
-			const auto count = m_declarations[offset] >> 16u;
-			if ((m_declarations[offset] & 0xffffu) == 59u) {
+			const auto count = m_declarations[offset] >> spv::WordCountShift;
+			if ((m_declarations[offset] & spv::OpCodeMask) == spv::OpVariable) {
 				const auto id = m_declarations[offset + 2u];
 				if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) {
 					operands.push_back(id);
@@ -207,46 +157,32 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons
 			offset += count;
 		}
 	}
-	AppendInstruction(m_entry_points, 15u, operands);
-}
-
-void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_execution_modes, 16u, operands);
+	AppendInstruction(m_entry_points, spv::OpEntryPoint, operands);
 }
 
 void Builder::AddName(uint32_t target, const char* name) {
 	std::vector<uint32_t> operands = {target};
 	AppendString(operands, name);
-	AppendInstruction(m_debug, 5u, operands);
+	AppendInstruction(m_debug, spv::OpName, operands);
 }
 
-void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_annotations, words.begin(), words.size());
-}
-
-void Builder::AddType(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_declarations, words.begin(), words.size());
-}
-
-void Builder::AddFunction(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_functions, words.begin(), words.size());
-}
-
-void Builder::AddFunction(const std::vector<uint32_t>& words) {
+void Builder::AddFunction(std::span<const uint32_t> words) {
 	AppendInstructionWords(m_functions, words.data(), words.size());
 }
 
 DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
-	std::vector<uint32_t> words {245u, type, result};
+	std::vector<uint32_t> words {spv::OpPhi, type, result};
 	words.resize(words.size() + incoming_count * 2u);
-	const DeferredPhi phi {m_functions.size(), incoming_count};
+	const DeferredPhi phi {m_functions.size()};
 	AddFunction(words);
 	m_unpatched_phi_incomings += incoming_count;
 	return phi;
 }
 
 void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
-	EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0);
+	const auto incoming_count =
+	    ((m_functions.at(phi.word_offset) >> spv::WordCountShift) - 3u) / 2u;
+	EXIT_IF(incoming >= incoming_count || value == 0 || parent == 0);
 	const auto value_word  = phi.word_offset + 3u + incoming * 2u;
 	const auto parent_word = value_word + 1u;
 	EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
@@ -264,7 +200,7 @@ std::vector<uint32_t> Builder::Build() const {
 	               m_debug.size() + m_annotations.size() + m_declarations.size() +
 	               m_functions.size());
 
-	module.push_back(0x07230203u);
+	module.push_back(spv::MagicNumber);
 	module.push_back(m_version);
 	module.push_back(0u);
 	module.push_back(m_next_id);

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -4,21 +4,25 @@
 #include "common/common.h"
 
 #include <initializer_list>
+#include <iterator>
 #include <map>
 #include <set>
+#include <span>
+#include <spirv/unified1/spirv.hpp>
 #include <string>
+#include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
 struct TypeAnnotation {
-	uint32_t              opcode = 0;
-	std::vector<uint32_t> operands;
+	spv::Op               opcode = spv::OpNop;
+	std::vector<uint32_t> operands; // Already encoded operand words.
 };
 
 struct DeferredPhi {
 	size_t word_offset    = 0;
-	size_t incoming_count = 0;
 };
 
 class Builder {
@@ -29,42 +33,113 @@ public:
 
 	uint32_t AllocateId();
 	void     RequireVersion(uint32_t version);
-	void     RequireCapability(uint32_t capability);
+	void     RequireCapability(spv::Capability capability);
 	void     RequireExtension(const char* name);
 	uint32_t Import(const char* name);
-	uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {});
-	uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands);
-	uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
-	                       std::initializer_list<TypeAnnotation> annotations);
-	uint32_t Constant(uint32_t opcode, uint32_t type,
-	                  std::initializer_list<uint32_t> operands = {});
-	uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands);
-	uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class);
-	void     DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
+	template <typename... Args>
+	uint32_t Type(spv::Op opcode, const Args&... operands) {
+		return DeclareType(opcode, MakeTypeKey(opcode, operands...));
+	}
 
-	void        AddMemoryModel(std::initializer_list<uint32_t> operands);
-	void        AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-	                          const std::vector<uint32_t>& interfaces);
-	void        AddExecutionMode(std::initializer_list<uint32_t> operands);
-	void        AddName(uint32_t target, const char* name);
-	void        AddAnnotation(std::initializer_list<uint32_t> words);
-	void        AddFunction(std::initializer_list<uint32_t> words);
-	void        AddFunction(const std::vector<uint32_t>& words);
+	template <typename... Args>
+	uint32_t DecoratedType(spv::Op opcode, std::initializer_list<TypeAnnotation> annotations,
+	                       const Args&... operands) {
+		return DeclareDecoratedType(opcode, MakeTypeKey(opcode, operands...), annotations);
+	}
+
+	template <typename... Args>
+	uint32_t Constant(spv::Op opcode, uint32_t type, const Args&... operands) {
+		std::vector<uint32_t> key;
+		key.reserve(2u + (0u + ... + OperandWordCount(operands)));
+		AppendOperands(key, opcode, type, operands...);
+		return DeclareConstant(opcode, std::move(key));
+	}
+
+	uint32_t DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class);
+	void DefineGlobalVariable(uint32_t id, uint32_t pointer_type, spv::StorageClass storage_class);
+
+	void AddMemoryModel(spv::AddressingModel addressing_model, spv::MemoryModel memory_model);
+	void AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point, const char* name,
+	                   const std::vector<uint32_t>& interfaces);
+	template <typename... Args>
+	void AddExecutionMode(uint32_t entry_point, spv::ExecutionMode mode, const Args&... operands) {
+		AppendInstruction(m_execution_modes, spv::OpExecutionMode, entry_point, mode, operands...);
+	}
+
+	void AddName(uint32_t target, const char* name);
+	template <typename... Args>
+	void AddAnnotation(spv::Op opcode, const Args&... operands) {
+		AppendInstruction(m_annotations, opcode, operands...);
+	}
+
+	template <typename... Args>
+	void AddFunction(spv::Op opcode, const Args&... operands) {
+		AppendInstruction(m_functions, opcode, operands...);
+	}
+
+	void        AddFunction(std::span<const uint32_t> words);
 	DeferredPhi AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count);
 	void        PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent);
 
 	[[nodiscard]] std::vector<uint32_t> Build() const;
 
 private:
-	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-	                              const std::vector<uint32_t>& operands);
-	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-	                              std::initializer_list<uint32_t> operands);
+	static void AppendOperand(std::vector<uint32_t>& words, uint32_t value) {
+		words.push_back(value);
+	}
+
+	static void AppendOperand(std::vector<uint32_t>& words, int32_t value) {
+		words.push_back(static_cast<uint32_t>(value));
+	}
+
+	template <typename T>
+	requires std::is_enum_v<T>
+	static void AppendOperand(std::vector<uint32_t>& words, T value) {
+		static_assert(sizeof(T) == sizeof(uint32_t));
+		words.push_back(static_cast<uint32_t>(value));
+	}
+
+	static void AppendOperand(std::vector<uint32_t>& words, std::span<const uint32_t> values) {
+		words.insert(words.end(), values.begin(), values.end());
+	}
+
+	template <typename... Args>
+	static void AppendOperands(std::vector<uint32_t>& words, const Args&... operands) {
+		(AppendOperand(words, operands), ...);
+	}
+
+	template <typename T>
+	static size_t OperandWordCount(const T& operand) {
+		if constexpr (std::is_integral_v<T> || std::is_enum_v<T>) {
+			return 1;
+		} else {
+			return std::size(operand);
+		}
+	}
+
+	template <typename... Args>
+	static std::vector<uint32_t> MakeTypeKey(spv::Op opcode, const Args&... operands) {
+		const auto operand_count = static_cast<uint32_t>((0u + ... + OperandWordCount(operands)));
+		std::vector<uint32_t> key;
+		key.reserve(2u + operand_count);
+		AppendOperands(key, opcode, operand_count, operands...);
+		return key;
+	}
+
+	template <typename... Args>
+	static void AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
+	                              const Args&... operands) {
+		const auto offset = section.size();
+		AppendOperands(section, opcode, operands...);
+		const auto word_count = static_cast<uint32_t>(section.size() - offset);
+		section[offset] |= word_count << spv::WordCountShift;
+	}
+
+	uint32_t    DeclareType(spv::Op opcode, std::vector<uint32_t> key);
+	uint32_t    DeclareDecoratedType(spv::Op opcode, std::vector<uint32_t> key,
+	                                 std::initializer_list<TypeAnnotation> annotations);
+	uint32_t    DeclareConstant(spv::Op opcode, std::vector<uint32_t> key);
 	static void AppendString(std::vector<uint32_t>& words, const char* text);
-	void        AddCapability(std::initializer_list<uint32_t> operands);
-	void        AddExtension(const char* name);
-	void        AddExtInstImport(uint32_t id, const char* name);
-	void        AddType(std::initializer_list<uint32_t> words);
 
 	uint32_t                                  m_next_id = 1;
 	uint32_t                                  m_version = 0;
@@ -78,7 +153,7 @@ private:
 	std::vector<uint32_t>                     m_annotations;
 	std::vector<uint32_t>                     m_declarations;
 	std::vector<uint32_t>                     m_functions;
-	std::set<uint32_t>                        m_required_capabilities;
+	std::set<spv::Capability>                 m_required_capabilities;
 	std::set<std::string>                     m_required_extensions;
 	std::map<std::string, uint32_t>           m_import_ids;
 	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -183,10 +183,8 @@ void ValidateNativeProgram(const IR::Program& program) {
 
 } // namespace
 
-void AnalyzeProgramRequirements(IR::Program& program) {
-	program.spirv_requirements.reset();
-	IR::SpirvRequirements requirements {};
-	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
+Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) {
+	SpirvRequirements requirements {};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
@@ -242,17 +240,17 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 				}
 				if (shared_access == IR::SharedAccess::Append ||
 				    shared_access == IR::SharedAccess::Consume) {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 				}
 			}
 			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::Ballot: MarkBallot(); break;
+				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
 				case IR::ValueOpcode::ReadFirstLane:
 				case IR::ValueOpcode::ReadLane: {
-					MarkBallot();
+					requirements.subgroup_ballot  = true;
 					requirements.subgroup_shuffle = true;
 					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
 						requirements.subgroup_local_invocation_id = true;
@@ -261,25 +259,26 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 				}
 				case IR::ValueOpcode::DppUpdateU32:
 				case IR::ValueOpcode::WriteLane: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::Permlane16U32: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::SwizzleU32:
 				case IR::ValueOpcode::BpermuteU32: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::LaneId:
-					requirements.subgroup_local_invocation_id = true;
+					requirements.subgroup_local_invocation_id |=
+					    program.stage != ShaderType::TessellationControl;
 					break;
 				case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
 				case IR::ValueOpcode::ImageGatherRaw:
@@ -300,7 +299,7 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 			}
 		}
 	}
-	program.spirv_requirements.emplace(requirements);
+	return requirements;
 }
 
 std::vector<uint32_t> EmitProgram(const IR::Program& program,
@@ -308,40 +307,29 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
 	using namespace Emitter;
 
 	if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
-	    program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh) {
-		Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
+	    program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh &&
+	    program.stage != ShaderType::Local && program.stage != ShaderType::TessellationControl &&
+	    program.stage != ShaderType::TessellationEvaluation) {
+		Fail(program, "binary SPIR-V emitter received an unsupported shader stage");
 	}
 	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete ||
-	    !program.spirv_requirements.has_value()) {
+	    !program.shader_info_complete || !program.binding_layout_complete) {
 		Fail(program, "SPIR-V emitter requires a fully planned native shader program");
 	}
 	ValidateNativeProgram(program);
 	IR::ValidateProgram(program, true);
 	EmitterState state(program, input_info);
-	state.stage     = program.stage;
-	state.wave_size = program.wave_size;
 	const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
 	state.lane_count =
 	    workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u
 	        ? 2u
 	        : 1u;
-	state.inputs.reserve(program.info.inputs.size());
-	state.outputs.reserve(program.info.outputs.size());
-	state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
-	CopyProgramInputsAndOutputs(state, program);
-	AllocateInputVariables(state);
-	AllocateOutputVariables(state);
 	DefineModule(state);
-	EmitProgram(state, program);
-	state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
-	                            state.interface_variables);
+	EmitProgram(state);
+	state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func,
+	                            "main", state.interface_variables);
 
-	auto binary = state.builder.Build();
-	if (binary.empty()) {
-		Fail(program, "SPIR-V builder returned an empty module");
-	}
-	return binary;
+	return state.builder.Build();
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,8 +9,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-void AnalyzeProgramRequirements(IR::Program& program);
-
 std::vector<uint32_t> EmitProgram(const IR::Program& program,
                                   ShaderStageInputInfo input_info);
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -2,116 +2,37 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
-                            uint32_t false_value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value});
-	return ret;
-}
-
-uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
-
-void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
-                                   uint32_t& out_low, uint32_t& out_high) {
-	const auto amount           = EmitAndConstant(state, shift, 63u);
-	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
-	const auto less32           = state.builder.AllocateId();
-	const auto non_zero         = state.builder.AllocateId();
-	const auto carry_count_base = state.builder.AllocateId();
-	const auto carry_count      = state.builder.AllocateId();
-	const auto low_part         = state.builder.AllocateId();
-	const auto high_part        = state.builder.AllocateId();
-	const auto carry_part       = state.builder.AllocateId();
-	const auto carry            = state.builder.AllocateId();
-	const auto high_less        = state.builder.AllocateId();
-	const auto high_ge          = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
-	state.builder.AddFunction(
-	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
-	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
-	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
-}
-
-void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
-                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
-	const auto amount           = EmitAndConstant(state, shift, 63u);
-	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
-	const auto less32           = state.builder.AllocateId();
-	const auto non_zero         = state.builder.AllocateId();
-	const auto carry_count_base = state.builder.AllocateId();
-	const auto carry_count      = state.builder.AllocateId();
-	const auto low_part         = state.builder.AllocateId();
-	const auto high_part        = state.builder.AllocateId();
-	const auto carry_part       = state.builder.AllocateId();
-	const auto carry            = state.builder.AllocateId();
-	const auto low_less         = state.builder.AllocateId();
-	const auto low_ge           = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
-	state.builder.AddFunction(
-	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
-	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
-	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
-}
-
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), ret, value,
+	                          ConstantU32(state, mask));
 	return ret;
 }
 
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), ret, value,
+	                          ConstantU32(state, shift));
 	return ret;
 }
 
-uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+uint32_t EmitCompareU32Constant(EmitterState& state, spv::Op opcode, uint32_t value,
                                 uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction(opcode, TypeBool(state), ret, value, ConstantU32(state, constant));
 	return ret;
 }
 
 uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), ret, ConstantU32(state, constant),
+	                          value);
 	return ret;
 }
 
 uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, f32);
 
 	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
 	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
@@ -122,26 +43,26 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
 	const auto normal_payload = state.builder.AllocateId();
 	const auto normal         = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
-	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp,
+	                          ConstantU32(state, 10));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), normal_payload, normal_exp,
+	                          normal_mant);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), normal, sign, normal_payload);
 
 	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
 	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
-	const auto exp_lt_103       = EmitCompareU32Constant(state, OpULessThan, exp, 103);
-	const auto exp_gt_112       = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112);
+	const auto exp_lt_103       = EmitCompareU32Constant(state, spv::OpULessThan, exp, 103);
+	const auto exp_gt_112       = EmitCompareU32Constant(state, spv::OpUGreaterThan, exp, 112);
 	const auto sub_shift_low =
 	    EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
 	const auto sub_shift =
 	    EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low);
 	const auto sub_mant  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden,
+	                          sub_shift);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant);
 
 	const auto nan_payload =
 	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
@@ -149,12 +70,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	    EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
 	const auto inf        = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
 	const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
-	const auto mant_zero  = EmitCompareU32Constant(state, OpIEqual, mant, 0);
+	const auto mant_zero  = EmitCompareU32Constant(state, spv::OpIEqual, mant, 0);
 	const auto special    = EmitSelectValueU32(state, mant_zero, inf, nan);
 
-	const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112);
-	const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143);
-	const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255);
+	const auto exp_le_112 = EmitCompareU32Constant(state, spv::OpULessThanEqual, exp, 112);
+	const auto exp_ge_143 = EmitCompareU32Constant(state, spv::OpUGreaterThanEqual, exp, 143);
+	const auto exp_eq_255 = EmitCompareU32Constant(state, spv::OpIEqual, exp, 255);
 	const auto finite0    = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
 	const auto finite1    = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
 	const auto finite2    = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
@@ -164,74 +85,43 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
+	state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state),
+	                          cond, lhs, rhs);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs);
 	return ret;
 }
 
 uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
+	state.builder.AddFunction(max_value ? spv::OpSGreaterThan : spv::OpSLessThan, TypeBool(state),
+	                          cond, lhs, rhs);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs);
 	return ret;
 }
 
-uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value});
-	return ret;
-}
-
-F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
+F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
 	F32Class cls;
-	cls.bits                 = EmitBitcastF32ToU32(state, value);
+	cls.bits                 = bits;
 	const auto abs_bits      = EmitAndConstant(state, cls.bits, 0x7fffffffu);
 	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
-	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
-	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
+	const auto exponent_max =
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
+	const auto mantissa_nonzero = EmitCompareU32Constant(state, spv::OpINotEqual, mantissa_bits, 0);
 	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
-	cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	cls.zero                    = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 	return cls;
 }
 
+F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
+	return EmitClassifyF32Bits(state, EmitBitcastF32ToU32(state, value));
+}
+
 uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
                                uint32_t class_match) {
 	const auto selected =
-	    EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
+	    EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
 	return EmitLogicalAndBool(state, selected, class_match);
 }
 
@@ -243,15 +133,16 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
 	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
 
-	const auto sign             = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0);
+	const auto sign             = EmitCompareU32Constant(state, spv::OpINotEqual, sign_bits, 0);
 	const auto positive         = EmitLogicalNotBool(state, sign);
-	const auto exponent_zero    = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0);
+	const auto exponent_zero    = EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0);
 	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
-	const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto exponent_inf =
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
 	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
-	const auto mantissa_zero    = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0);
+	const auto mantissa_zero    = EmitCompareU32Constant(state, spv::OpIEqual, mantissa_bits, 0);
 	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
-	const auto quiet            = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0);
+	const auto quiet            = EmitCompareU32Constant(state, spv::OpINotEqual, quiet_bits, 0);
 
 	const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
 	const auto snan       = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
@@ -259,7 +150,7 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto inf        = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
 	const auto normal     = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
 	const auto denorm     = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
-	const auto zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	const auto zero       = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 
 	uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
 	match          = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
@@ -292,8 +183,8 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto rhs_class = EmitClassifyF32(state, rhs);
 
 	const auto numeric_cond = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state),
-	                           numeric_cond, lhs, rhs});
+	state.builder.AddFunction(max_value ? spv::OpFOrdGreaterThanEqual : spv::OpFOrdLessThan,
+	                          TypeBool(state), numeric_cond, lhs, rhs);
 	const auto ordered_bits =
 	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
 
@@ -308,13 +199,6 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	return EmitBitcastU32ToF32(state, result_bits);
 }
 
-uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value});
-	return ret;
-}
-
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto bits      = state.builder.AllocateId();
 	const auto abs_bits  = state.builder.AllocateId();
@@ -324,18 +208,18 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
-	state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
-	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits,
+	                          ConstantU32(state, 0x7fffffffu));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
+	                          ConstantU32(state, 0x80000000u));
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
+	                          ConstantU32(state, 0x00800000u));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits);
+	state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
 	return ret;
 }
 
@@ -347,46 +231,35 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
 	const auto finite       = state.builder.AllocateId();
 	const auto large_finite = state.builder.AllocateId();
 	const auto reduced      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src});
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)});
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
-	state.builder.AddFunction(
-	    {OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
+	state.builder.AddFunction(spv::OpExtInst, TypeF32(state), fract, GlslStd450(state),
+	                          GLSLstd450Fract, src);
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, src);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits,
+	                          ConstantU32(state, 0x7fffffffu));
+	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), large, abs_bits,
+	                          ConstantU32(state, 0x4b000000u));
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), finite, abs_bits,
+	                          ConstantU32(state, 0x7f800000u));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), large_finite, large, finite);
+	state.builder.AddFunction(spv::OpSelect, TypeF32(state), reduced, large_finite,
+	                          ConstantF32(state, 0), fract);
 	if (!preserve_signed_zero) {
 		return reduced;
 	}
 	const auto zero = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced});
-	return ret;
-}
-
-uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value});
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), zero, abs_bits,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpSelect, TypeF32(state), ret, zero, src, reduced);
 	return ret;
 }
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
-	                           GlslUnpackHalf2x16, bits});
-	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
+	state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
+	                          GLSLstd450UnpackHalf2x16, bits);
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), ret, unpacked, 0);
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -6,13 +6,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
-	if (state.stage != ShaderType::Pixel) {
-		return attr;
-	}
-	return ShaderPixelParameterMappedLocation(*state.input_info.pixel, attr);
-}
-
 uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
 	std::array<uint32_t, 32> active_inputs {};
 	uint32_t                 active_count = 0;
@@ -21,42 +14,22 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
 			active_inputs[active_count++] = input.location;
 		}
 	}
-	return state.stage == ShaderType::Pixel
+	return state.program.stage == ShaderType::Pixel
 	           ? ShaderPixelParameterLocation(*state.input_info.pixel,
 	                                          {active_inputs.data(), active_count}, attr)
 	           : attr;
 }
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
-	return state.stage == ShaderType::Pixel &&
+	return state.program.stage == ShaderType::Pixel &&
 	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
 }
 
 bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
-	return state.stage == ShaderType::Pixel &&
+	return state.program.stage == ShaderType::Pixel &&
 	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
 }
 
-bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
-               uint32_t index) {
-	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
-		return binding.kind == kind && binding.index == index;
-	});
-}
-
-void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
-	for (const auto& input: program.info.inputs) {
-		state.inputs.push_back({input.kind, input.location, input.component_count, 0,
-		                        input.debug_name, input.per_vertex});
-	}
-	for (const auto& output: program.info.outputs) {
-		if (HasOutput(state.outputs, output.kind, output.index)) {
-			continue;
-		}
-		state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name});
-	}
-}
-
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
 	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
 		return state.per_vertex_variable;
@@ -81,7 +54,7 @@ uint32_t          ConstantU32(EmitterState& state, uint32_t value);
                                                const char* reason) {
 	EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
 	     " stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
-	     state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
+	     state.program.shader_hash, static_cast<unsigned>(state.program.stage), resource,
 	     static_cast<unsigned>(kind), reason);
 	std::abort();
 }
@@ -109,8 +82,8 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
 		ExitDescriptorBindingFailure(state, kind, resource, variable_name);
 	}
 	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
+	state.builder.AddFunction(spv::OpAccessChain, result_ptr_type, pointer, variable_id,
+	                          ConstantU32(state, array_index));
 	return pointer;
 }
 
@@ -136,12 +109,12 @@ uint32_t ImageScalarType(EmitterState& state, Prospero::TextureNumericClass nume
 
 uint32_t ImageVectorType(EmitterState& state, Prospero::TextureNumericClass numeric_class,
                          uint32_t components) {
-	return state.builder.Type(OpTypeVector, {ImageScalarType(state, numeric_class), components});
+	return state.builder.Type(spv::OpTypeVector, ImageScalarType(state, numeric_class), components);
 }
 
 uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 	uint32_t sampled = 0;
-	uint32_t format  = ImageFormatUnknown;
+	uint32_t format  = spv::ImageFormatUnknown;
 	if (image.resource_class == IR::ImageResourceClass::Sampled) {
 		EXIT_IF(image.atomic);
 		sampled = 1;
@@ -151,16 +124,15 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 		sampled = 2;
 		if (image.atomic) {
 			EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint);
-			format = ImageFormatR32ui;
+			format = spv::ImageFormatR32ui;
 		}
 	} else {
 		EXIT("invalid image resource class");
 	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
-	return state.builder.Type(OpTypeImage,
-	                          {ImageScalarType(state, image.numeric_class), info.spirv_dimension,
-	                           image.depth_compare ? 1u : 0u,
-	                           info.arrayed, info.multisampled, sampled, format});
+	return state.builder.Type(spv::OpTypeImage, ImageScalarType(state, image.numeric_class),
+	                          info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
+	                          info.multisampled, sampled, format);
 }
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
@@ -180,26 +152,26 @@ uint32_t LoadSampledImageDescriptor(EmitterState& state, uint32_t resource) {
 	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
 	const auto variable     = state.image_variables[IR::ImageBindingIndex(*kind)];
 	const auto pointer_type = state.builder.Type(
-	    OpTypePointer, {StorageClassUniformConstant, ImageType(state, image_resource)});
+	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image_resource));
 	const auto pointer =
 	    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 	                             "sampled image descriptor array was not emitted");
 	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, ImageType(state, image_resource), image, pointer});
+	state.builder.AddFunction(spv::OpLoad, ImageType(state, image_resource), image, pointer);
 	return image;
 }
 
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
 	const auto array_index =
 	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
-	const auto sampler_type = state.builder.Type(OpTypeSampler);
+	const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
 	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type});
+	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type);
 	const auto pointer = DescriptorElementPointer(
 	    state, pointer_type, state.sampler_variable, array_index,
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer});
+	state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer);
 	return sampler_id;
 }
 
@@ -208,9 +180,9 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
 	const auto  image          = LoadSampledImageDescriptor(state, resource);
 	const auto  sampler_id     = LoadSamplerDescriptor(state, sampler);
 	const auto  sampled_image = state.builder.AllocateId();
-	const auto  sampled_type  =
-	    state.builder.Type(OpTypeSampledImage, {ImageType(state, image_resource)});
-	state.builder.AddFunction({OpSampledImage, sampled_type, sampled_image, image, sampler_id});
+	const auto  sampled_type =
+	    state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource));
+	state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
 	return sampled_image;
 }
 
@@ -219,9 +191,9 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource) {
 	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
-	const auto array_index = ResourceForDescriptor(state, *kind, resource);
-	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, ImageType(state, image)});
+	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
+	const auto pointer_type = state.builder.Type(
+	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 	                                "storage image descriptor array was not emitted");
@@ -232,25 +204,25 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
 	if (!image.atomic) {
-		state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat);
+		state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
 	}
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
 	const auto array_index = ResourceForDescriptor(state, *kind, resource);
 	const auto image_type  = ImageType(state, image);
 	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, image_type});
+	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, image_type);
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	const auto LoadAt   = [&](uint32_t array_index) {
 		const auto pointer =
 		    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 		                             "storage image descriptor array was not emitted");
 		const auto descriptor = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, image_type, descriptor, pointer});
+		state.builder.AddFunction(spv::OpLoad, image_type, descriptor, pointer);
 		return descriptor;
 	};
 	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
-		state.builder.AddFunction({OpImageWrite, LoadAt(array_index), coord, texel});
+		state.builder.AddFunction(spv::OpImageWrite, LoadAt(array_index), coord, texel);
 		return;
 	}
 	if (image.mip_count == 0u) {
@@ -260,28 +232,31 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 
 	const auto            merge_label = state.builder.AllocateId();
 	std::vector<uint32_t> labels(image.mip_count);
-	std::vector<uint32_t> words {OpSwitch, mip_lod, merge_label};
+	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
 		labels[mip] = state.builder.AllocateId();
 		words.push_back(mip);
 		words.push_back(labels[mip]);
 	}
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
 	state.builder.AddFunction(words);
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
 		EmitLabel(state, labels[mip]);
-		state.builder.AddFunction({OpImageWrite, LoadAt(array_index + mip), coord, texel});
-		state.builder.AddFunction({OpBranch, merge_label});
+		state.builder.AddFunction(spv::OpImageWrite, LoadAt(array_index + mip), coord, texel);
+		state.builder.AddFunction(spv::OpBranch, merge_label);
 	}
 	EmitLabel(state, merge_label);
 }
 
-uint32_t ExecutionModelForStage(ShaderType stage) {
+spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {
 	switch (stage) {
-		case ShaderType::Vertex: return ExecutionModelVertex;
-		case ShaderType::Mesh: return 5365u; // MeshEXT
-		case ShaderType::Pixel: return ExecutionModelFragment;
-		default: return ExecutionModelGLCompute;
+		case ShaderType::Local:
+		case ShaderType::Vertex: return spv::ExecutionModelVertex;
+		case ShaderType::TessellationControl: return spv::ExecutionModelTessellationControl;
+		case ShaderType::TessellationEvaluation: return spv::ExecutionModelTessellationEvaluation;
+		case ShaderType::Mesh: return spv::ExecutionModelMeshEXT; // MeshEXT
+		case ShaderType::Pixel: return spv::ExecutionModelFragment;
+		default: return spv::ExecutionModelGLCompute;
 	}
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include <algorithm>
 
@@ -16,48 +16,27 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
 	return true;
 }
 
-uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
-	const auto shifted_one = state.builder.AllocateId();
-	const auto merged_one  = state.builder.AllocateId();
-	const auto shifted_two = state.builder.AllocateId();
-	const auto merged_two  = state.builder.AllocateId();
-	const auto quad_bits   = state.builder.AllocateId();
-	const auto result      = state.builder.AllocateId();
-	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
-	                           ConstantU64(state, 0x0000000100000001ull)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
-	                           ConstantU64(state, 0x0000000200000002ull)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
-	                           ConstantU64(state, 0x1111111111111111ull)});
-	state.builder.AddFunction(
-	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
-	return result;
-}
-
-uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
-	auto& state = ctx.state;
+uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
 	if (kind == IR::StageInputKind::LocalInvocationIndex) {
 		return EmitLocalInvocationIndex(state);
 	}
 	if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId ||
 	                              kind == IR::StageInputKind::GlobalInvocationId)) {
-		const auto* cs      = ShaderWorkgroupInput(state.stage, state.input_info);
+		const auto* cs      = ShaderWorkgroupInput(state.program.stage, state.input_info);
 		uint32_t    divisor = 1;
 		for (uint32_t axis = 0; axis < component; axis++) {
 			divisor *= std::max(cs->threads_num[axis], 1u);
 		}
 		const auto size    = std::max(cs->threads_num[component], 1u);
-		const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state),
+		const auto divided = EmitBinaryU32(state, spv::OpUDiv, EmitLocalInvocationIndex(state),
 		                                   ConstantU32(state, divisor));
-		const auto local   = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size));
+		const auto local   = EmitBinaryU32(state, spv::OpUMod, divided, ConstantU32(state, size));
 		if (kind == IR::StageInputKind::LocalInvocationId) {
 			return local;
 		}
 		const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component);
 		return EmitAddU32(state, local,
-		                  EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size)));
+		                  EmitBinaryU32(state, spv::OpIMul, group, ConstantU32(state, size)));
 	}
 	const auto variable = InputVariableForKind(state, kind);
 	if (variable == 0) {
@@ -66,28 +45,29 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 	if (kind == IR::StageInputKind::FrontFacing) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, TypeBool(state), value, variable});
-		state.builder.AddFunction(
-		    {OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		state.builder.AddFunction(spv::OpLoad, TypeBool(state), value, variable);
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1),
+		                          ConstantU32(state, 0));
 		return bits;
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
+	    kind == IR::StageInputKind::InvocationId || kind == IR::StageInputKind::PrimitiveId ||
 	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, TypeI32(state), value, variable});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction(spv::OpLoad, TypeI32(state), value, variable);
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
-	if (kind == IR::StageInputKind::FragCoord) {
+	if (kind == IR::StageInputKind::FragCoord || kind == IR::StageInputKind::TessCoord) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain,
-		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		                           variable, ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction(spv::OpAccessChain,
+		                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                          pointer, variable, ConstantU32(state, component));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
 	if (kind == IR::StageInputKind::BaryCoordSmooth ||
@@ -95,11 +75,11 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain,
-		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		                           variable, ConstantU32(state, component + 1u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction(spv::OpAccessChain,
+		                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                          pointer, variable, ConstantU32(state, component + 1u));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
 	return EmitInputComponentU32(state, kind, component);
@@ -120,55 +100,55 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	const auto bank_ok    = state.builder.AllocateId();
 	const auto row_ok     = state.builder.AllocateId();
 	const auto masks_ok   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), bank_shift, lane,
+	                          ConstantU32(state, 2));
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), row_shift, lane,
+	                          ConstantU32(state, 4));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), bank, bank_shift,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, row_shift,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), bank_bit,
+	                          ConstantU32(state, 1), bank);
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), row_bit,
+	                          ConstantU32(state, 1), row);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), bank_hit,
+	                          ConstantU32(state, flags.bank_mask), bank_bit);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row_hit,
+	                          ConstantU32(state, flags.row_mask), row_bit);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), bank_ok, bank_hit,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), row_ok, row_hit,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok);
 	uint32_t writable = masks_ok;
 	if (!flags.bound_control) {
 		const auto target  = EmitDppTargetLane(state, flags.control);
 		const auto bounded = state.builder.AllocateId();
-		state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
+		state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), bounded, writable,
+		                          target.valid);
 		writable = bounded;
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable});
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), result, exec, writable);
 	return result;
 }
 
-uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
-	auto&       state = ctx.state;
+uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
 	const auto* input = InputBindingForParameter(state, attr);
 	if (input == nullptr || input->variable_id == 0) {
 		return ConstantU32(state, 0);
 	}
-	if (state.stage == ShaderType::Vertex) {
+	if (state.program.stage == ShaderType::Vertex || state.program.stage == ShaderType::Local) {
 		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
 	}
 	const auto load_per_vertex = [&](uint32_t vertex) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		    spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
+		    input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
 		return value;
 	};
 	if (input->per_vertex) {
@@ -181,30 +161,31 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
 			const auto pointer = state.builder.AllocateId();
 			const auto weight  = state.builder.AllocateId();
 			const auto product = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain,
-			                           TypePointer(state, StorageClassInput, TypeF32(state)),
-			                           pointer, barycentric, ConstantU32(state, vertex)});
-			state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer});
-			state.builder.AddFunction(
-			    {OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
+			state.builder.AddFunction(spv::OpAccessChain,
+			                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+			                          pointer, barycentric, ConstantU32(state, vertex));
+			state.builder.AddFunction(spv::OpLoad, TypeF32(state), weight, pointer);
+			state.builder.AddFunction(spv::OpFMul, TypeF32(state), product, load_per_vertex(vertex),
+			                          weight);
 			if (vertex == 0u) {
 				sum = product;
 			} else {
 				const auto next = state.builder.AllocateId();
-				state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product});
+				state.builder.AddFunction(spv::OpFAdd, TypeF32(state), next, sum, product);
 				sum = next;
 			}
 		}
 		const auto bits = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum});
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, sum);
 		return bits;
 	}
 	const auto vector    = state.builder.AllocateId();
 	const auto component = state.builder.AllocateId();
 	const auto bits      = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
-	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component});
+	state.builder.AddFunction(spv::OpLoad, TypeF32Vector(state, 4), vector, input->variable_id);
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), component, vector,
+	                          chan & 3u);
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, component);
 	return bits;
 }
 
@@ -213,15 +194,15 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	auto&       state = ctx.state;
 	const auto* input = InputBindingForParameter(state, attr);
 	if (!input->per_vertex) {
-		return EmitAttribute(ctx, attr, chan);
+		return EmitAttribute(ctx.state, attr, chan);
 	}
 	const auto load_vertex = [&](uint32_t vertex) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		    spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
+		    input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
 		return value;
 	};
 
@@ -229,16 +210,16 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	uint32_t   value           = load_vertex(selected_vertex);
 	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
 		const auto delta = state.builder.AllocateId();
-		state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)});
+		state.builder.AddFunction(spv::OpFSub, TypeF32(state), delta, value, load_vertex(0));
 		value = delta;
 	}
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 	return bits;
 }
 
 uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
-	if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
+	if (state.program.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
 	    exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
 		return 0;
 	}
@@ -247,8 +228,8 @@ uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
 
 uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
 	const auto value = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(
-	    {OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), value, vector,
+	                              component);
 	return value;
 }
 
@@ -257,7 +238,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	auto& state = ctx.state;
 	if (exp.compr && !uint_output) {
 		const auto unpack =
-		    MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
+		    MrtOutputMode(state, exp) == 5u ? GLSLstd450UnpackUnorm2x16 : GLSLstd450UnpackHalf2x16;
 		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
 		                   ConstantF32(state, 0x3f800000u)};
 		for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -266,21 +247,21 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 			}
 			const auto packed   = state.builder.AllocateId();
 			const auto unpacked = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
-			state.builder.AddFunction(
-			    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed});
+			state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), packed, data, pair);
+			state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 2), unpacked,
+			                          GlslStd450(state), unpack, packed);
 			for (uint32_t lane = 0; lane < 2u; lane++) {
 				const auto component = pair * 2u + lane;
 				if (((exp.en >> component) & 1u) != 0u) {
 					f32[component] = state.builder.AllocateId();
-					state.builder.AddFunction(
-					    {OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
+					state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state),
+					                          f32[component], unpacked, lane);
 				}
 			}
 		}
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
-		                           f32[1], f32[2], f32[3]});
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector,
+		                          f32[0], f32[1], f32[2], f32[3]);
 		return vector;
 	}
 	uint32_t raw[4] = {
@@ -301,9 +282,9 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 					continue;
 				}
 				raw[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component],
-				                           packed, ConstantU32(state, lane * 16u),
-				                           ConstantU32(state, 16)});
+				state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), raw[component],
+				                          packed, ConstantU32(state, lane * 16u),
+				                          ConstantU32(state, 16));
 			}
 		}
 	} else {
@@ -315,18 +296,18 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	}
 	if (uint_output) {
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0],
-		                           raw[1], raw[2], raw[3]});
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), vector,
+		                          raw[0], raw[1], raw[2], raw[3]);
 		return vector;
 	}
 	uint32_t f32[4] {};
 	for (uint32_t component = 0; component < 4u; component++) {
 		f32[component] = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]});
+		state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32[component], raw[component]);
 	}
 	const auto vector = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]});
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
+	                          f32[1], f32[2], f32[3]);
 	return vector;
 }
 
@@ -342,19 +323,19 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 			const auto raw = ExportRawComponent(ctx, data, component);
 			if (output.layer) {
 				const auto layer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
-				                           ConstantU32(state, 0x7ffu)});
-				const auto pointer = state.stage == ShaderType::Mesh
+				state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), layer, raw,
+				                          ConstantU32(state, 0x7ffu));
+				const auto pointer = state.program.stage == ShaderType::Mesh
 				                         ? MeshOutputPointer(state, IR::StageOutputKind::Layer)
 				                         : state.layer_variable;
-				state.builder.AddFunction({OpStore, pointer, layer});
+				state.builder.AddFunction(spv::OpStore, pointer, layer);
 			}
 			if (output.viewport) {
 				// GFX10 MISC.z packs the viewport index in bits 16..19 alongside the layer.
 				const auto viewport = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), viewport, raw,
-				                           ConstantU32(state, 16), ConstantU32(state, 4)});
-				state.builder.AddFunction({OpStore, state.viewport_index_variable, viewport});
+				state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), viewport, raw,
+				                          ConstantU32(state, 16), ConstantU32(state, 4));
+				state.builder.AddFunction(spv::OpStore, state.viewport_index_variable, viewport);
 			}
 			continue;
 		}
@@ -365,9 +346,9 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 
 		const auto raw = ExportRawComponent(ctx, data, component);
 		const auto f32 = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+		state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32, raw);
 		if (output.point_size) {
-			state.builder.AddFunction({OpStore, state.point_size_variable, f32});
+			state.builder.AddFunction(spv::OpStore, state.point_size_variable, f32);
 			continue;
 		}
 		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
@@ -375,10 +356,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 				return;
 			}
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer,
-			     variable, ConstantU32(state, index)});
-			state.builder.AddFunction({OpStore, pointer, f32});
+			state.builder.AddFunction(spv::OpAccessChain,
+			                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+			                          pointer, variable, ConstantU32(state, index));
+			state.builder.AddFunction(spv::OpStore, pointer, f32);
 		};
 		StoreDistance(state.clip_distance_variable, output.clip_distance);
 		StoreDistance(state.cull_distance_variable, output.cull_distance);
@@ -391,14 +372,14 @@ uint32_t ConvertClipCoordinate(EmitterState& state, uint32_t coordinate, float s
 	const auto biased  = state.builder.AllocateId();
 	const auto divided = state.builder.AllocateId();
 	const auto ndc     = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpFMul, TypeF32(state), window, coordinate, ConstantF32Value(state, scale)});
-	state.builder.AddFunction(
-	    {OpFAdd, TypeF32(state), biased, window, ConstantF32Value(state, offset)});
-	state.builder.AddFunction(
-	    {OpFDiv, TypeF32(state), divided, biased, ConstantF32Value(state, half_extent)});
-	state.builder.AddFunction(
-	    {OpFSub, TypeF32(state), ndc, divided, ConstantF32Value(state, 1.0f)});
+	state.builder.AddFunction(spv::OpFMul, TypeF32(state), window, coordinate,
+	                          ConstantF32Value(state, scale));
+	state.builder.AddFunction(spv::OpFAdd, TypeF32(state), biased, window,
+	                          ConstantF32Value(state, offset));
+	state.builder.AddFunction(spv::OpFDiv, TypeF32(state), divided, biased,
+	                          ConstantF32Value(state, half_extent));
+	state.builder.AddFunction(spv::OpFSub, TypeF32(state), ndc, divided,
+	                          ConstantF32Value(state, 1.0f));
 	return ndc;
 }
 
@@ -407,29 +388,51 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) {
 	uint32_t    components[4] {};
 	for (uint32_t i = 0; i < 4; i++) {
 		components[i] = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeExtract, TypeF32(state), components[i], position, i});
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), components[i], position,
+		                          i);
 	}
 	components[0] = ConvertClipCoordinate(state, components[0], transform.scale[0],
 	                                      transform.offset[0], transform.half_extent[0]);
 	components[1] = ConvertClipCoordinate(state, components[1], transform.scale[1],
 	                                      transform.offset[1], transform.half_extent[1]);
 	const auto converted = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), converted,
-	                           components[0], components[1], components[2], components[3]});
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), converted,
+	                          components[0], components[1], components[2], components[3]);
 	return converted;
 }
 
-void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
+} // namespace
+uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
+	                          ConstantU64(state, 0x0000000100000001ull));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one);
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                          ConstantU64(state, 0x0000000200000002ull));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one,
+	                          shifted_two);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                          ConstantU64(state, 0x1111111111111111ull));
+	state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits,
+	                          ConstantU64(state, 0x0000000f0000000full));
+	return result;
+}
+
+void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&       state = ctx.state;
 	const auto& exp   = ctx.Export(inst);
 	const auto  exec  = ctx.Arg(inst, 1);
-	if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
+	if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
 	    state.pixel_valid_mask_variable != 0) {
 		const auto value = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
-		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1),
+		                          ConstantU32(state, 0));
+		state.builder.AddFunction(spv::OpStore, state.pixel_valid_mask_variable, value);
 	}
 	if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
 		return;
@@ -437,9 +440,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	EmitIfCondition(state, exec, [&]() {
 		const auto data = ctx.Arg(inst, 0);
 		if (exp.kind == IR::ExportTargetKind::Primitive) {
-			if (state.stage == ShaderType::Mesh) {
-				state.builder.AddFunction(
-				    {OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
+			if (state.program.stage == ShaderType::Mesh) {
+				state.builder.AddFunction(spv::OpStore, MeshPrimitivePointer(state),
+				                          ExportRawComponent(ctx, data, 0));
 			}
 			return;
 		}
@@ -451,37 +454,37 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
 				const auto raw = ExportRawComponent(ctx, data, 0);
 				const auto f32 = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
-				state.builder.AddFunction({OpStore, state.depth_variable, f32});
+				state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32, raw);
+				state.builder.AddFunction(spv::OpStore, state.depth_variable, f32);
 			}
 			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
 				const auto raw     = ExportRawComponent(ctx, data, 2);
 				const auto value   = state.builder.AllocateId();
 				const auto pointer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw});
+				state.builder.AddFunction(spv::OpBitcast, TypeI32(state), value, raw);
 				state.builder.AddFunction(
-				    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer,
-				     state.sample_mask_variable, ConstantU32(state, 0)});
-				state.builder.AddFunction({OpStore, pointer, value});
+				    spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, TypeI32(state)),
+				    pointer, state.sample_mask_variable, ConstantU32(state, 0));
+				state.builder.AddFunction(spv::OpStore, pointer, value);
 			}
 			return;
 		}
 		const auto variable =
-		    state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
-		if (state.stage != ShaderType::Mesh && variable == 0) {
+		    state.program.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
+		if (state.program.stage != ShaderType::Mesh && variable == 0) {
 			return;
 		}
 		const bool uint_output = MrtOutputMode(state, exp) == 7u;
 		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
-		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
 		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
 			const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];
 			if (!mapping.IsIdentity()) {
 				const auto mapped = state.builder.AllocateId();
-				state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
-				                           mapping.Map(0), mapping.Map(1), mapping.Map(2),
-				                           mapping.Map(3)});
+				state.builder.AddFunction(spv::OpVectorShuffle, vector_type, mapped, value, value,
+				                          mapping.Map(0), mapping.Map(1), mapping.Map(2),
+				                          mapping.Map(3));
 				value = mapped;
 			}
 		}
@@ -489,192 +492,192 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		    state.input_info.vertex->clip_space.enabled) {
 			value = ConvertPositionToClipSpace(state, value);
 		}
-		if (state.stage == ShaderType::Mesh) {
+		if (state.program.stage == ShaderType::Mesh) {
 			const auto kind = exp.kind == IR::ExportTargetKind::Position
 			                      ? IR::StageOutputKind::Position
 			                      : IR::StageOutputKind::Parameter;
-			state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value});
+			state.builder.AddFunction(spv::OpStore, MeshOutputPointer(state, kind, exp.index),
+			                          value);
 		} else if (exp.kind == IR::ExportTargetKind::Position) {
 			const auto pointer = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
-			     pointer, variable, ConstantU32(state, 0)});
-			state.builder.AddFunction({OpStore, pointer, value});
+			    spv::OpAccessChain,
+			    TypePointer(state, spv::StorageClassOutput, TypeF32Vector(state, 4)), pointer,
+			    variable, ConstantU32(state, 0));
+			state.builder.AddFunction(spv::OpStore, pointer, value);
 		} else {
-			state.builder.AddFunction({OpStore, variable, value});
+			state.builder.AddFunction(spv::OpStore, variable, value);
 		}
 	});
 }
 
-} // namespace
+uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
+	return value;
+}
 
-bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
-	auto& state = ctx.state;
-	switch (inst.GetOpcode()) {
-		case IR::ValueOpcode::Identity: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
-		case IR::ValueOpcode::Void:
-		case IR::ValueOpcode::Reference:
-		case IR::ValueOpcode::ReferenceU32:
-		case IR::ValueOpcode::ControlNop:
-		case IR::ValueOpcode::Waitcnt:
-		case IR::ValueOpcode::Sendmsg:
-		case IR::ValueOpcode::TtraceData:
-		case IR::ValueOpcode::InstPrefetch: return true;
-		case IR::ValueOpcode::Barrier: {
-			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
-			state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
-			                           ConstantU32(state, ScopeWorkgroup),
-			                           ConstantU32(state, semantics)});
-			return true;
-		}
-		case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true;
-		case IR::ValueOpcode::MeshDrawParameter: {
-			const auto index = inst.Arg(0).U32();
-			if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
-				ctx.Fail(inst, "invalid mesh draw parameter");
-			}
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state),
-			                           pointer, state.push_constant_variable, ConstantU32(state, 0),
-			                           ConstantU32(state, index)});
-			state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer});
-			return true;
-		}
-		case IR::ValueOpcode::GetUserData: {
-			const auto reg   = inst.Arg(0).ScalarRegister();
-			uint32_t   dword = 0;
-			if (!UserDataDwordIndex(state, reg, dword)) {
-				ctx.Define(inst, ConstantU32(state, 0));
-			} else {
-				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
-			}
-			return true;
-		}
-		case IR::ValueOpcode::GetBuiltin:
-			ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
-			                                inst.Arg(1).U32()));
-			return true;
-		case IR::ValueOpcode::UndefU1:
-		case IR::ValueOpcode::UndefU8:
-		case IR::ValueOpcode::UndefU16:
-		case IR::ValueOpcode::UndefU32:
-		case IR::ValueOpcode::UndefU64:
-			state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
-			return true;
-		case IR::ValueOpcode::DppMoveU32: {
-			const auto flags    = inst.Flags<IR::DppMoveFlags>();
-			const auto target   = EmitDppTargetLane(state, flags.control);
-			const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
-			if (flags.fetch_inactive) {
-				ctx.Define(inst, shuffled);
-				return true;
-			}
-			const auto ballot        = ctx.Ballot(inst.Arg(1));
-			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
-			const auto can_fetch     = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
-			return true;
-		}
-		case IR::ValueOpcode::DppUpdateU32: {
-			const auto flags = inst.Flags<IR::DppMoveFlags>();
-			const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
-			return true;
-		}
-		case IR::ValueOpcode::WqmU64:
-			ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
-			return true;
-		case IR::ValueOpcode::LaneId:
-			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
-			return true;
-		case IR::ValueOpcode::Ballot: ctx.Define(inst, ctx.Ballot(inst.Arg(0))); return true;
-		case IR::ValueOpcode::ReadFirstLane: {
-			const auto ballot = ctx.Ballot(inst.Arg(1));
-			const auto lane   = ctx.FirstLane(ballot);
-			ctx.Define(inst, ctx.Shuffle(inst, 0, lane));
-			return true;
-		}
-		case IR::ValueOpcode::ReadLane:
-			ctx.Define(inst, ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::WriteLane: {
-			const auto hit = state.builder.AllocateId();
-			state.builder.AddFunction({OpIEqual, TypeBool(state), hit,
-			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
-			return true;
-		}
-		case IR::ValueOpcode::Permlane16U32: {
-			const auto flags     = inst.Flags<IR::PermlaneFlags>();
-			const auto subid     = EmitSubgroupLocalInvocationId(state);
-			const auto row       = state.builder.AllocateId();
-			const auto row_value = state.builder.AllocateId();
-			const auto lane      = state.builder.AllocateId();
-			const auto lane8     = state.builder.AllocateId();
-			const auto shift     = state.builder.AllocateId();
-			const auto upper     = state.builder.AllocateId();
-			const auto selected  = state.builder.AllocateId();
-			const auto shifted   = state.builder.AllocateId();
-			const auto index     = state.builder.AllocateId();
-			const auto target    = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-			if (flags.x16) {
-				state.builder.AddFunction(
-				    {OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
-			} else {
-				state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row});
-			}
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
-			state.builder.AddFunction(
-			    {OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
-			state.builder.AddFunction(
-			    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
-			state.builder.AddFunction(
-			    {OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
-			state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
-			const auto shuffled = ctx.Shuffle(inst, 0, target);
-			uint32_t result = shuffled;
-			if (!flags.fetch_inactive) {
-				const auto source_exec = ctx.Shuffle(inst, 3, target);
-				result = state.builder.AllocateId();
-				state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
-				                           ConstantU32(state, 0)});
-			}
-			ctx.Define(inst, result);
-			return true;
-		}
-		case IR::ValueOpcode::GetAttribute:
-			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
-			return true;
-		case IR::ValueOpcode::GetInterpolationParameter:
-			ctx.Define(inst, EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(),
-			                                            inst.Arg(2).U32()));
-			return true;
-		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
-		case IR::ValueOpcode::GetShaderBase:
-			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
-			// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
-			ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
-			return true;
-		case IR::ValueOpcode::GetSrtResource:
-		case IR::ValueOpcode::GetBufferResource:
-		case IR::ValueOpcode::GetAddressResource:
-		case IR::ValueOpcode::GetScratchResource:
-		case IR::ValueOpcode::GetImageResource:
-		case IR::ValueOpcode::GetSamplerResource:
-		case IR::ValueOpcode::MakeImageAddress: return true;
-		default: return false;
+void EmitVoid(ValueEmitContext&) {}
+
+void EmitBarrier(EmitterState& state) {
+	const auto tessellation = state.program.stage == ShaderType::TessellationControl;
+	const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
+	const auto semantics    = tessellation ? spv::MemorySemanticsMaskNone
+	                                       : spv::MemorySemanticsAcquireReleaseMask |
+	                                             spv::MemorySemanticsWorkgroupMemoryMask;
+	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
+	                          ConstantU32(state, memory_scope), ConstantU32(state, semantics));
+}
+
+uint32_t EmitLaneId(EmitterState& state) {
+	return state.program.stage == ShaderType::TessellationControl
+	           ? EmitBuiltinU32(state, IR::StageInputKind::InvocationId, 0)
+	           : EmitSubgroupLocalInvocationId(state);
+}
+
+uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state  = ctx.state;
+	const auto result = state.builder.AllocateId();
+	const auto index  = inst.Arg(0).U32();
+	if (state.program.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
+		ctx.Fail(inst, "invalid mesh draw parameter");
+	}
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer,
+	                          state.push_constant_variable, ConstantU32(state, 0),
+	                          ConstantU32(state, index));
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), result, pointer);
+	return result;
+}
+
+uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg) {
+
+	uint32_t dword = 0;
+	if (!UserDataDwordIndex(state, reg, dword)) {
+		return ConstantU32(state, 0);
+	} else {
+		return EmitShaderDataDwordLoad(state, dword);
 	}
 }
 
+uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) {
+	return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
+}
+
+uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpUndef, TypeId(state, inst.GetType()), result);
+	return result;
+}
+
+uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state    = ctx.state;
+	const auto flags    = inst.Flags<IR::DppMoveFlags>();
+	const auto target   = EmitDppTargetLane(state, flags.control);
+	const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
+	if (flags.fetch_inactive) {
+		return shuffled;
+	}
+	const auto ballot        = ctx.Ballot(inst.Arg(1));
+	const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
+	const auto can_fetch     = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid,
+	                          source_active);
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, can_fetch, shuffled,
+	                                                ConstantU32(state, 0));
+}
+
+uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto flags = inst.Flags<IR::DppMoveFlags>();
+	const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, write, ctx.Arg(inst, 0),
+	                                                ctx.Arg(inst, 1));
+}
+
+uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
+	return ctx.Ballot(predicate);
+}
+
+uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto ballot = ctx.Ballot(inst.Arg(1));
+	const auto lane   = ctx.FirstLane(ballot);
+	return ctx.Shuffle(inst, 0, lane);
+}
+
+uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1));
+}
+
+uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state = ctx.state;
+	const auto hit   = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), hit,
+	                          EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2));
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, hit, ctx.Arg(inst, 1),
+	                                                ctx.Arg(inst, 0));
+}
+
+uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state     = ctx.state;
+	const auto flags     = inst.Flags<IR::PermlaneFlags>();
+	const auto subid     = EmitSubgroupLocalInvocationId(state);
+	const auto row       = state.builder.AllocateId();
+	const auto row_value = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto lane8     = state.builder.AllocateId();
+	const auto shift     = state.builder.AllocateId();
+	const auto upper     = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto shifted   = state.builder.AllocateId();
+	const auto index     = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
+	                          ConstantU32(state, 0xfffffff0u));
+	if (flags.x16) {
+		state.builder.AddFunction(spv::OpBitwiseXor, TypeU32(state), row_value, row,
+		                          ConstantU32(state, 16));
+	} else {
+		state.builder.AddFunction(spv::OpCopyObject, TypeU32(state), row_value, row);
+	}
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 15));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane8, lane,
+	                          ConstantU32(state, 7));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane8,
+	                          ConstantU32(state, 2));
+	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), upper, lane,
+	                          ConstantU32(state, 8));
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2),
+	                          ctx.Arg(inst, 1));
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), shifted, selected, shift);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), index, shifted,
+	                          ConstantU32(state, 15));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row_value, index);
+	const auto shuffled = ctx.Shuffle(inst, 0, target);
+	uint32_t   result   = shuffled;
+	if (!flags.fetch_inactive) {
+		const auto source_exec = ctx.Shuffle(inst, 3, target);
+		result                 = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), result, source_exec, shuffled,
+		                          ConstantU32(state, 0));
+	}
+	return result;
+}
+
+uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32());
+}
+
+uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(), inst.Arg(2).U32());
+}
+
+uint32_t EmitGetShaderBase(ValueEmitContext& ctx) {
+	// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
+	// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
+	return ctx.Def(IR::Value(uint64_t {0}));
+}
+
+void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) {
+	ctx.Fail(inst, "must be lowered before SPIR-V emission");
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -2,10 +2,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-uint32_t EmitTrueBool(EmitterState& state) {
-	return ConstantBool(state, true);
-}
-
 DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
 	const auto quad_base = state.builder.AllocateId();
 	const auto lane      = state.builder.AllocateId();
@@ -13,17 +9,18 @@ DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto selected0 = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected});
-	return {target, EmitTrueBool(state)};
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), quad_base, subid,
+	                          ConstantU32(state, 0xfffffffcu));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane,
+	                          ConstantU32(state, 1));
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), selected0,
+	                          ConstantU32(state, control), shift);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), selected, selected0,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, quad_base, selected);
+	return {target, ConstantBool(state, true)};
 }
 
 DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
@@ -33,21 +30,22 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto lane_shifted = state.builder.AllocateId();
 	const auto target       = state.builder.AllocateId();
 	const auto valid        = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
+	                          ConstantU32(state, 0xfffffff0u));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 15));
 	if (left) {
-		state.builder.AddFunction(
-		    {OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
-		state.builder.AddFunction(
-		    {OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
+		state.builder.AddFunction(spv::OpIAdd, TypeU32(state), lane_shifted, lane,
+		                          ConstantU32(state, amount));
+		state.builder.AddFunction(spv::OpULessThan, TypeBool(state), valid, lane,
+		                          ConstantU32(state, 16u - amount));
 	} else {
-		state.builder.AddFunction(
-		    {OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
+		state.builder.AddFunction(spv::OpISub, TypeU32(state), lane_shifted, lane,
+		                          ConstantU32(state, amount));
+		state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), valid, lane,
+		                          ConstantU32(state, amount));
 	}
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row, lane_shifted);
 	return {target, valid};
 }
 
@@ -60,17 +58,18 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
 	const auto plus     = state.builder.AllocateId();
 	const auto selected = state.builder.AllocateId();
 	const auto target   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction(
-	    {OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
-	state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected});
-	return {target, EmitTrueBool(state)};
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
+	                          ConstantU32(state, 0xfffffff0u));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 15));
+	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane,
+	                          ConstantU32(state, amount));
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount));
+	state.builder.AddFunction(spv::OpIAdd, TypeU32(state), plus, lane,
+	                          ConstantU32(state, 16u - amount));
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, in_high, minus, plus);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row, selected);
+	return {target, ConstantBool(state, true)};
 }
 
 DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
@@ -80,14 +79,14 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
 	const auto lane      = state.builder.AllocateId();
 	const auto mirrored  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored});
-	return {target, EmitTrueBool(state)};
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), base, subid,
+	                          ConstantU32(state, base_mask));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, lane_mask));
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask),
+	                          lane);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, base, mirrored);
+	return {target, ConstantBool(state, true)};
 }
 
 DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
@@ -112,11 +111,11 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
 	}
 	if (control >= 0x160u && control <= 0x16fu) {
 		const auto target = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseXor, TypeU32(state), target, subid, ConstantU32(state, control & 0xfu)});
-		return {target, EmitTrueBool(state)};
+		state.builder.AddFunction(spv::OpBitwiseXor, TypeU32(state), target, subid,
+		                          ConstantU32(state, control & 0xfu));
+		return {target, ConstantBool(state, true)};
 	}
-	return {subid, EmitTrueBool(state)};
+	return {subid, ConstantBool(state, true)};
 }
 
 uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
@@ -124,8 +123,8 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
 		EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n");
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value,
+	                          state.subgroup_local_invocation_id_variable);
 	return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32));
 }
 
@@ -154,9 +153,10 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
 	}
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)),
-	                           pointer, variable, ConstantU32(state, component)});
-	state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+	state.builder.AddFunction(spv::OpAccessChain,
+	                          TypePointer(state, spv::StorageClassInput, TypeU32(state)), pointer,
+	                          variable, ConstantU32(state, component));
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
 	return value;
 }
 
@@ -166,9 +166,10 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
 		return ConstantU32(state, 0);
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, variable);
 	if (state.lane_count == 2) {
-		const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u));
+		const auto wave_base =
+		    EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, ~31u));
 		return EmitAddU32(state, EmitAddU32(state, value, wave_base),
 		                  ConstantU32(state, state.lane_half * 32));
 	}
@@ -182,13 +183,13 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	const auto scalar_type = VertexParameterScalarType(state, kind);
 	uint32_t   raw         = state.builder.AllocateId();
 	if (count == 1u) {
-		state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
+		state.builder.AddFunction(spv::OpLoad, scalar_type, raw, input.variable_id);
 	} else {
-		const auto pointer_type = VertexParameterScalarPointerType(state, kind);
+		const auto pointer_type = TypePointer(state, spv::StorageClassInput, scalar_type);
 		const auto pointer      = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
-		                           ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
+		state.builder.AddFunction(spv::OpAccessChain, pointer_type, pointer, input.variable_id,
+		                          ConstantU32(state, component));
+		state.builder.AddFunction(spv::OpLoad, scalar_type, raw, pointer);
 	}
 
 	if (kind == VertexInputScalarKind::Uint) {
@@ -196,47 +197,48 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	}
 
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, raw);
 	return bits;
 }
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
 	const auto active_ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
-	                           ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
+	state.builder.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
+	                          ConstantU32(state, spv::ScopeSubgroup), ConstantBool(state, true));
 	return EmitBallotLaneActiveBool(state, active_ballot, lane);
 }
 uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
 	const auto low = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, active_ballot, 0);
 	uint32_t mask = low;
-	if (state.wave_size == 64u) {
+	if (state.program.wave_size == 64u) {
 		const auto high     = state.builder.AllocateId();
 		const auto in_high  = state.builder.AllocateId();
 		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low});
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, active_ballot, 1);
+		state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane,
+		                          ConstantU32(state, 32));
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, in_high, high, low);
 		mask = selected;
 	}
 
 	const auto lane_low = state.builder.AllocateId();
 	const auto bit      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane_low, lane,
+	                          ConstantU32(state, 31));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1),
+	                          lane_low);
 
 	const auto hit      = state.builder.AllocateId();
 	const auto active   = state.builder.AllocateId();
 	const auto in_range = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.wave_size)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), hit, mask, bit);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), active, hit,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), in_range, lane,
+	                          ConstantU32(state, state.program.wave_size));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), ret, active, in_range);
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -1,5 +1,5 @@
 #include "graphics/guest_gpu/gpu_format.h"
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
 
 #include <algorithm>
@@ -32,18 +32,6 @@ uint32_t ImageGatherComponent(uint32_t dmask) {
 	}
 }
 
-uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, type, result, value});
-	return result;
-}
-
-uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, type, result, lhs, rhs});
-	return result;
-}
-
 bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {
 	return (mem.image_sample_flags & flag) != 0u;
 }
@@ -56,10 +44,10 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	auto value = ctx.Def(address.Arg(packed));
 	if (layout.bit_width == 16u) {
 		if ((layout.bit_offset & 31u) != 0u) {
-			value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value,
+			value = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), value,
 			               ConstantU32(ctx.state, 16));
 		}
-		value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value,
+		value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), value,
 		               ConstantU32(ctx.state, 0xffffu));
 	}
 	return value;
@@ -70,7 +58,7 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	const auto value = AddressU32(ctx, mem, address, component);
 	return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u
 	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value);
+	           : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
 }
 
 ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) {
@@ -97,19 +85,19 @@ uint32_t ZeroF32(EmitterState& state) {
 }
 
 uint32_t CubeAxis(EmitterState& state, uint32_t value) {
-	return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
+	return Binary(state, spv::OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
 }
 
 uint32_t CubeLayer(EmitterState& state, uint32_t value) {
 	const auto guest = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value});
-	const auto padding =
-	    Binary(state, OpShiftLeftLogical, TypeU32(state),
-	           Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
-	           ConstantU32(state, 1));
-	const auto host   = Binary(state, OpISub, TypeU32(state), guest, padding);
+	state.builder.AddFunction(spv::OpConvertFToU, TypeU32(state), guest, value);
+	const auto padding = Binary(
+	    state, spv::OpShiftLeftLogical, TypeU32(state),
+	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
+	    ConstantU32(state, 1));
+	const auto host   = Binary(state, spv::OpISub, TypeU32(state), guest, padding);
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host});
+	state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), result, host);
 	return result;
 }
 
@@ -130,11 +118,11 @@ uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		             ? AddressF32(ctx, mem, address, first + 2u)
 		             : ZeroF32(ctx.state);
 		if (cube) z = CubeLayer(ctx.state, z);
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 3),
+		                              result, x, y, z);
 	} else {
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 2),
+		                              result, x, y);
 	}
 	return result;
 }
@@ -150,11 +138,11 @@ uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 	if (components == 3u) {
 		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
 		                                                 : ConstantU32(ctx.state, 0);
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 3),
+		                              result, x, y, z);
 	} else {
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 2),
+		                              result, x, y);
 	}
 	return result;
 }
@@ -168,7 +156,7 @@ uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst
 }
 
 uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
-	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+	return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
 }
 
 uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
@@ -176,7 +164,7 @@ uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
 	if (numeric_class == Prospero::TextureNumericClass::Uint) {
 		return value;
 	}
-	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+	return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
 }
 
 uint32_t SampledComponentZero(EmitterState& state, Prospero::TextureNumericClass numeric_class) {
@@ -205,9 +193,9 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 			if (dref) return value;
 			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
 			const auto result    = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpCompositeExtract,
-			                               ImageScalarType(ctx.state, value_class), result, value,
-			                               component});
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract,
+			                              ImageScalarType(ctx.state, value_class), result, value,
+			                              component);
 			return result;
 		};
 		for (uint32_t word = 0; word < mem.data_dwords; word++) {
@@ -221,24 +209,25 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 				const auto low_bits  = SampledComponentBits(ctx, low, value_class);
 				const auto high_bits = SampledComponentBits(ctx, high, value_class);
 				const auto mask      = ConstantU32(ctx.state, 0xffffu);
-				packed[word]         = Binary(
-				    ctx.state, OpBitwiseOr, TypeU32(ctx.state),
-				    Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
-				    Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
-				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high_bits, mask),
-				           ConstantU32(ctx.state, 16u)));
+				packed[word] =
+				    Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state),
+				           Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
+				           Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
+				                  Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
+				                         high_bits, mask),
+				                  ConstantU32(ctx.state, 16u)));
 			} else {
 				const auto pair = ctx.state.builder.AllocateId();
-				ctx.state.builder.AddFunction(
-				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
+				                              TypeF32Vector(ctx.state, 2), pair, low, high);
 				packed[word] = ctx.state.builder.AllocateId();
-				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
-				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+				ctx.state.builder.AddFunction(spv::OpExtInst, TypeU32(ctx.state), packed[word],
+				                              GlslStd450(ctx.state), GLSLstd450PackHalf2x16, pair);
 			}
 		}
 		const auto result = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-		                               packed[0], packed[1], packed[2], packed[3]});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4),
+		                              result, packed[0], packed[1], packed[2], packed[3]);
 		return result;
 	}
 	uint32_t component[4] {};
@@ -249,12 +238,12 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 		}
 		const auto scalar = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, ImageScalarType(ctx.state, value_class), scalar, value, index});
+		    spv::OpCompositeExtract, ImageScalarType(ctx.state, value_class), scalar, value, index);
 		component[index] = SampledComponentBits(ctx, scalar, value_class);
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                               component[0], component[1], component[2], component[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              component[0], component[1], component[2], component[3]);
 	return result;
 }
 
@@ -265,11 +254,12 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 	const auto  image     = LoadSampledImageDescriptor(ctx.state, mem.resource);
 	const auto  size      = ctx.state.builder.AllocateId();
 	if (info.multisampled != 0u) {
-		ctx.state.builder.AddFunction(
-		    {OpImageQuerySize, ImageViewSizeType(ctx.state, dimension), size, image});
+		ctx.state.builder.AddFunction(spv::OpImageQuerySize,
+		                              ImageViewSizeType(ctx.state, dimension), size, image);
 	} else {
-		ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, dimension),
-		                               size, image, AddressU32(ctx, mem, address, 0)});
+		ctx.state.builder.AddFunction(spv::OpImageQuerySizeLod,
+		                              ImageViewSizeType(ctx.state, dimension), size, image,
+		                              AddressU32(ctx, mem, address, 0));
 	}
 	const auto components = info.coordinate_components;
 	uint32_t   result[4]  = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
@@ -279,17 +269,18 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 	} else {
 		for (uint32_t index = 0; index < components; index++) {
 			result[index] = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state),
+			                              result[index], size, index);
 		}
 	}
 	if (info.multisampled == 0u) {
 		result[3] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
+		ctx.state.builder.AddFunction(spv::OpImageQueryLevels, TypeU32(ctx.state), result[3],
+		                              image);
 	}
 	const auto vector = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
-	                               result[0], result[1], result[2], result[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
+	                              result[0], result[1], result[2], result[3]);
 	return vector;
 }
 
@@ -301,31 +292,31 @@ uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR
 		if (components == 1u) return zero;
 		const auto result = ctx.state.builder.AllocateId();
 		if (components == 3u) {
-			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
+			ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3),
+			                              result, zero, zero, zero);
 		} else {
-			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
+			ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2),
+			                              result, zero, zero);
 		}
 		return result;
 	}
-	const auto packed    = Unary(ctx.state, OpBitcast, TypeI32(ctx.state),
+	const auto packed    = Unary(ctx.state, spv::OpBitcast, TypeI32(ctx.state),
 	                             AddressU32(ctx, mem, address, layout.offset));
 	uint32_t   values[3] = {zero, zero, zero};
 	for (uint32_t index = 0; index < components; index++) {
 		values[index] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index],
-		                               packed, ConstantU32(ctx.state, index * 8u),
-		                               ConstantU32(ctx.state, 6)});
+		ctx.state.builder.AddFunction(spv::OpBitFieldSExtract, TypeI32(ctx.state), values[index],
+		                              packed, ConstantU32(ctx.state, index * 8u),
+		                              ConstantU32(ctx.state, 6));
 	}
 	if (components == 1u) return values[0];
 	const auto result = ctx.state.builder.AllocateId();
 	if (components == 3u) {
-		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result,
-		                               values[0], values[1], values[2]});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3),
+		                              result, values[0], values[1], values[2]);
 	} else {
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2),
+		                              result, values[0], values[1]);
 	}
 	return result;
 }
@@ -334,19 +325,19 @@ uint32_t HorizontalOffsets(EmitterState& state, ImageDimension dimension) {
 	const auto components   = ImageDimensionInfoFor(dimension).spatial_components;
 	const auto count        = ConstantU32(state, 4);
 	const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
-	const auto array_type   = state.builder.Type(OpTypeArray, {element_type, count});
+	const auto array_type   = state.builder.Type(spv::OpTypeArray, element_type, count);
 	uint32_t   offsets[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
 		if (components == 1u) {
 			offsets[index] = x;
 		} else {
-			offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2),
-			                                        {x, ConstantI32(state, 0)});
+			offsets[index] = state.builder.Constant(
+			    spv::OpConstantComposite, TypeI32Vector(state, 2), x, ConstantI32(state, 0));
 		}
 	}
-	return state.builder.Constant(OpConstantComposite, array_type,
-	                              {offsets[0], offsets[1], offsets[2], offsets[3]});
+	return state.builder.Constant(spv::OpConstantComposite, array_type, offsets[0], offsets[1],
+	                              offsets[2], offsets[3]);
 }
 
 uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
@@ -374,15 +365,15 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
 
 	const auto packed = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u);
 	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
 	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		components[component] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state),
-		                               components[component], packed,
-		                               ConstantU32(ctx.state, info.component_bit_offset[component]),
-		                               ConstantU32(ctx.state, info.component_bits[component])});
+		ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(ctx.state),
+		                              components[component], packed,
+		                              ConstantU32(ctx.state, info.component_bit_offset[component]),
+		                              ConstantU32(ctx.state, info.component_bits[component]));
 	}
 	for (uint32_t component = info.component_count; component < 4u; component++) {
 		components[component] = components[component % info.component_count];
@@ -401,8 +392,8 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 		}
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                               selected[0], selected[1], selected[2], selected[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              selected[0], selected[1], selected[2], selected[3]);
 	return result;
 }
 
@@ -415,42 +406,44 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
 	if (selector < 4u) {
 		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
-		return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4),
-		                                  {value, value, value, value});
+		return ctx.state.builder.Constant(spv::OpConstantComposite, TypeU32Vector(ctx.state, 4),
+		                                  value, value, value, value);
 	}
 
 	uint32_t values[4] {};
 	for (uint32_t lane = 0; lane < 4u; lane++) {
 		const auto packed = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), packed, gathered,
+		                              lane);
 		values[lane]        = ctx.state.builder.AllocateId();
 		const auto physical = (selector - 4u) % info.component_count;
-		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed,
-		                               ConstantU32(ctx.state, info.component_bit_offset[physical]),
-		                               ConstantU32(ctx.state, info.component_bits[physical])});
+		ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(ctx.state), values[lane],
+		                              packed,
+		                              ConstantU32(ctx.state, info.component_bit_offset[physical]),
+		                              ConstantU32(ctx.state, info.component_bits[physical]));
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                               values[0], values[1], values[2], values[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              values[0], values[1], values[2], values[3]);
 	return result;
 }
 
 uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                                     uint32_t coord, Prospero::TextureNumericClass numeric_class) {
 	auto& state = ctx.state;
-	state.builder.RequireCapability(CapabilityImageQuery);
+	state.builder.RequireCapability(spv::CapabilityImageQuery);
 	const auto image = LoadSampledImageDescriptor(state, mem.resource);
 	const auto width = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)});
+	state.builder.AddFunction(spv::OpImageQuerySizeLod, TypeU32(state), width, image,
+	                          ConstantU32(state, 0));
 	const auto width_f32 = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), width_f32, width});
+	state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), width_f32, width);
 	const auto left = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32(state), left, GlslStd450(state), GlslFloor,
-	                           Binary(state, OpFSub, TypeF32(state),
-	                                  Binary(state, OpFMul, TypeF32(state), coord, width_f32),
-	                                  ConstantF32(state, 0x3f000000u))});
+	state.builder.AddFunction(spv::OpExtInst, TypeF32(state), left, GlslStd450(state),
+	                          GLSLstd450Floor,
+	                          Binary(state, spv::OpFSub, TypeF32(state),
+	                                 Binary(state, spv::OpFMul, TypeF32(state), coord, width_f32),
+	                                 ConstantF32(state, 0x3f000000u)));
 
 	const auto sampled     = MakeSampledImage(state, mem.resource, mem.sampler);
 	const auto vector_type = ImageVectorType(state, numeric_class, 4);
@@ -462,20 +455,20 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 	uint32_t values[2] {};
 	for (uint32_t index = 0; index < 2u; index++) {
 		const auto sample_coord =
-		    Binary(state, OpFDiv, TypeF32(state),
-		           Binary(state, OpFAdd, TypeF32(state), left,
+		    Binary(state, spv::OpFDiv, TypeF32(state),
+		           Binary(state, spv::OpFAdd, TypeF32(state), left,
 		                  ConstantF32(state, index == 0u ? 0x3f000000u : 0x3fc00000u)),
 		           width_f32);
 		const auto texel = state.builder.AllocateId();
-		state.builder.AddFunction({OpImageSampleExplicitLod, vector_type, texel, sampled,
-		                           sample_coord, ImageOperandsLodMask, ZeroF32(state)});
+		state.builder.AddFunction(spv::OpImageSampleExplicitLod, vector_type, texel, sampled,
+		                          sample_coord, spv::ImageOperandsLodMask, ZeroF32(state));
 		values[index] = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeExtract, scalar_type, values[index], texel, component});
+		state.builder.AddFunction(spv::OpCompositeExtract, scalar_type, values[index], texel,
+		                          component);
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpCompositeConstruct, vector_type, result, values[0], values[1], values[1], values[0]});
+	state.builder.AddFunction(spv::OpCompositeConstruct, vector_type, result, values[0], values[1],
+	                          values[1], values[0]);
 	return result;
 }
 
@@ -486,27 +479,28 @@ uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32
 	auto packed = ConstantU32(ctx.state, 0u);
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		const auto value = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), value, texel,
+		                              component);
 		const auto maximum =
 		    ConstantU32(ctx.state, info.component_bits[component] == 32u
 		                               ? UINT32_MAX
 		                               : (1u << info.component_bits[component]) - 1u);
-		const auto within  = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum);
+		const auto within =
+		    Binary(ctx.state, spv::OpULessThan, TypeBool(ctx.state), value, maximum);
 		const auto clamped = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
+		ctx.state.builder.AddFunction(spv::OpSelect, TypeU32(ctx.state), clamped, within, value,
+		                              maximum);
 		const auto shifted =
 		    info.component_bit_offset[component] == 0u
 		        ? clamped
-		        : Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped,
+		        : Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), clamped,
 		                 ConstantU32(ctx.state, info.component_bit_offset[component]));
-		packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
+		packed = Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
 	}
 	const auto zero   = ConstantU32(ctx.state, 0u);
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(
-	    {OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              packed, zero, zero, zero);
 	return result;
 }
 
@@ -520,83 +514,80 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 		if (source < 4u && ((dmask >> source) & 1u) != 0u) {
 			const auto packed_index = DmaskComponentIndex(dmask, source);
 			raw                     = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
-			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			                              mem.data_bits == 16u ? packed_index / 2u : packed_index);
 			if (mem.data_bits == 16u) {
 				if ((packed_index & 1u) != 0u) {
-					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					raw = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), raw,
 					             ConstantU32(ctx.state, 16u));
 				}
-				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				raw = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), raw,
 				             ConstantU32(ctx.state, 0xffffu));
 			}
 		}
 		values[component] = integer ? raw
 		                    : mem.data_bits == 16u
 		                        ? EmitF16BitsToF32(ctx.state, raw)
-		                        : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
+		                        : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), raw);
 	}
 	const auto texel = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(
-	    {OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
-	     texel, values[0], values[1], values[2], values[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
+	                              integer ? TypeU32Vector(ctx.state, 4)
+	                                      : TypeF32Vector(ctx.state, 4),
+	                              texel, values[0], values[1], values[2], values[3]);
 	return PackImageTexel(ctx, mem, texel);
 }
 
-uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
+spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) {
 	switch (opcode) {
-		case IR::ValueOpcode::ImageAtomicSwap32: return OpAtomicExchange;
-		case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
-		case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
-		case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
-		case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
-		case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
-		case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
-		default: return 0;
+		case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
+		case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
+		case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
+		default: return spv::OpNop;
 	}
 }
 
 } // namespace
 
-bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
+void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto op         = inst.GetOpcode();
 	const auto image_info = IR::ImageOpcodeInfoOf(op);
-	if (image_info.access == IR::ImageAccess::None) {
-		return false;
-	}
 	auto&       state     = ctx.state;
 	const auto& mem       = ctx.Memory(inst);
 	const auto  image_arg = inst.Arg(0);
 	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
 	const auto& image   = state.program.info.images.at(mem.resource);
 	const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
-	if (address == nullptr) return true;
 	if (op == IR::ValueOpcode::ImageQueryDimensions) {
-		state.builder.RequireCapability(CapabilityImageQuery);
+		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		ctx.Define(inst, QueryDimensions(ctx, mem, *address));
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageQueryLod) {
-		state.builder.RequireCapability(CapabilityImageQuery);
+		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		const auto dimension = image.dimension;
 		const auto sampled   = MakeSampledImage(state, mem.resource, mem.sampler);
 		const auto lod       = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
-		     CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components)});
+		    spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
+		    CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components));
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
 			const auto component = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index});
+			state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), component, lod,
+			                          index);
 			values[index] = FloatBits(ctx, component);
 		}
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0],
-		                           values[1], values[2], values[3]});
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result,
+		                          values[0], values[1], values[2], values[3]);
 		ctx.Define(inst, result);
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageRead) {
 		const auto  dimension      = image.dimension;
@@ -613,19 +604,19 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			        const auto coord      = CoordU32(ctx, mem, *address, dimension);
 			        if (dimension_info.multisampled != 0u) {
 				        state.builder.AddFunction(
-				            {OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
-				             descriptor, coord, ImageOperandsSampleMask,
-				             AddressU32(ctx, mem, *address, dimension_info.coordinate_components)});
+				            spv::OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
+				            descriptor, coord, spv::ImageOperandsSampleMask,
+				            AddressU32(ctx, mem, *address, dimension_info.coordinate_components));
 			        } else {
-				        state.builder.AddFunction({OpImageFetch,
-				                                   ImageVectorType(state, numeric_class, 4), color,
-				                                   descriptor, coord, ImageOperandsLodMask,
-				                                   LodU32(ctx, mem, *address, dimension)});
+				        state.builder.AddFunction(spv::OpImageFetch,
+				                                  ImageVectorType(state, numeric_class, 4), color,
+				                                  descriptor, coord, spv::ImageOperandsLodMask,
+				                                  LodU32(ctx, mem, *address, dimension));
 			        }
 			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class,
 			                            false, mem);
 		        }));
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageWrite) {
 		const bool uint_image = image.numeric_class == Prospero::TextureNumericClass::Uint;
@@ -639,7 +630,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
 			EmitStorageImageWrite(state, mem.resource, mip_lod, coord, texel);
 		});
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
 		const auto  dimension      = image.dimension;
@@ -650,7 +641,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (dref && state.program.info.images[mem.resource].conversion_format !=
 		                Prospero::BufferFormat::kInvalid) {
 			ctx.Fail(inst, "uses depth comparison with a packed integer image");
-			return true;
+			return;
 		}
 		const auto coord =
 		    CoordF32(ctx, mem, *address, layout.coord, dimension_info.coordinate_components);
@@ -660,16 +651,16 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				    HasFlag(mem, Decoder::ImageSampleFlagOffset) ||
 				    HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
 					ctx.Fail(inst, "has an unsupported 1D gather variant");
-					return true;
+					return;
 				}
 				const auto sample = EmitOneDimensionalGatherLz(ctx, mem, coord, numeric_class);
 				ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 				                              numeric_class, false, mem, true));
-				return true;
+				return;
 			}
 			if (dimension == ImageDimension::Dim1DArray) {
 				ctx.Fail(inst, "has an unsupported 1D-array gather");
-				return true;
+				return;
 			}
 			const auto            sampled = MakeSampledImage(state, mem.resource, mem.sampler);
 			const auto            sample  = state.builder.AllocateId();
@@ -679,22 +670,29 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				if (layout.dref != NoImageComponent) {
 					dref_value = AddressF32(ctx, mem, *address, layout.dref);
 				}
-				words = {OpImageDrefGather, TypeF32Vector(state, 4), sample, sampled, coord,
+				words = {spv::OpImageDrefGather,
+				         TypeF32Vector(state, 4),
+				         sample,
+				         sampled,
+				         coord,
 				         dref_value};
 			} else {
 				uint32_t component = 0;
 				if (ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid) {
 					component = ImageGatherComponent(mem.dmask);
 				}
-				words = {OpImageGather, ImageVectorType(state, numeric_class, 4),
-				         sample,        sampled,
-				         coord,         ConstantU32(state, component)};
+				words = {spv::OpImageGather,
+				         ImageVectorType(state, numeric_class, 4),
+				         sample,
+				         sampled,
+				         coord,
+				         ConstantU32(state, component)};
 			}
 			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
-				words.push_back(ImageOperandsConstOffsetsMask);
+				words.push_back(spv::ImageOperandsConstOffsetsMask);
 				words.push_back(HorizontalOffsets(state, dimension));
 			} else if (layout.offset != NoImageComponent) {
-				words.push_back(ImageOperandsOffsetMask);
+				words.push_back(spv::ImageOperandsOffsetMask);
 				words.push_back(PackedOffset(ctx, mem, *address, layout, dimension));
 			}
 			state.builder.AddFunction(words);
@@ -704,17 +702,17 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			}
 			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 			                              result_numeric_class, false, mem, true));
-			return true;
+			return;
 		}
 		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
 		                          HasFlag(mem, Decoder::ImageSampleFlagLod) ||
 		                          HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
-		                          state.stage != ShaderType::Pixel;
-		uint32_t opcode = OpImageSampleImplicitLod;
+		                          state.program.stage != ShaderType::Pixel;
+		auto       opcode       = spv::OpImageSampleImplicitLod;
 		if (explicit_lod) {
-			opcode = dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
+			opcode = dref ? spv::OpImageSampleDrefExplicitLod : spv::OpImageSampleExplicitLod;
 		} else if (dref) {
-			opcode = OpImageSampleDrefImplicitLod;
+			opcode = spv::OpImageSampleDrefImplicitLod;
 		}
 		uint32_t result_type = ImageVectorType(state, numeric_class, 4);
 		uint32_t dref_value  = 0;
@@ -728,34 +726,34 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		uint32_t              operand_mask = 0;
 		std::vector<uint32_t> operands;
 		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
-			operand_mask |= ImageOperandsGradMask;
+			operand_mask |= spv::ImageOperandsGradMask;
 			operands.push_back(
 			    CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components));
 			operands.push_back(
 			    CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components));
 		} else if (explicit_lod) {
-			operand_mask |= ImageOperandsLodMask;
+			operand_mask |= spv::ImageOperandsLodMask;
 			auto lod = ZeroF32(state);
 			if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) {
 				lod = AddressF32(ctx, mem, *address, layout.lod);
 			}
 			operands.push_back(lod);
 		} else if (layout.bias != NoImageComponent) {
-			operand_mask |= ImageOperandsBiasMask;
+			operand_mask |= spv::ImageOperandsBiasMask;
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
 		const auto EmitSample = [&](uint32_t resource) {
 			const auto            sampled = MakeSampledImage(state, resource, mem.sampler);
 			const auto            sample  = state.builder.AllocateId();
-			std::vector<uint32_t> words {opcode, result_type, sample, sampled, coord};
+			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};
 			if (dref) {
-				words.push_back(dref_value);
+				sample_operands.push_back(dref_value);
 			}
 			if (operand_mask != 0u) {
-				words.push_back(operand_mask);
-				words.insert(words.end(), operands.begin(), operands.end());
+				sample_operands.push_back(operand_mask);
+				sample_operands.insert(sample_operands.end(), operands.begin(), operands.end());
 			}
-			state.builder.AddFunction(words);
+			state.builder.AddFunction(opcode, sample_operands);
 			return sample;
 		};
 		if (image.indirect_root != mem.resource) {
@@ -765,30 +763,30 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				result = UnpackImageTexel(ctx, mem, sample);
 			}
 			ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-			return true;
+			return;
 		}
 		const auto* handle = image_arg.ResolveInstruction();
-		const auto* source = image.source < ctx.program.descriptor_sources.size()
-		                         ? &ctx.program.descriptor_sources[image.source]
+		const auto* source = image.source < state.program.descriptor_sources.size()
+		                         ? &state.program.descriptor_sources[image.source]
 		                         : nullptr;
 		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
 		    source->indirect_image->key_arg >= handle->NumArgs()) {
 			ctx.Fail(inst, "has invalid indirect image key provenance");
-			return true;
+			return;
 		}
 		const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg));
 		if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u ||
 		    image.indirect_resources.size() < 2u) {
 			ctx.Fail(inst, "has no indirect image runtime mapping");
-			return true;
+			return;
 		}
 		const auto LoadMapping = [&](uint32_t index) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state),
-			                           pointer, state.flattened_srt_variable, ConstantU32(state, 0),
-			                           index});
+			state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+			                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+			                          index);
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+			state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
 			return value;
 		};
 		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
@@ -797,61 +795,63 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		auto       selected = ConstantU32(state, 0u);
 		for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
 		     iteration++) {
-			const auto active = Binary(state, OpULessThan, TypeBool(state), low, high);
-			const auto mid =
-			    Binary(state, OpShiftRightLogical, TypeU32(state),
-			           Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u));
-			const auto probe = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
-			const auto entry      = Binary(state, OpIAdd, TypeU32(state), mapping,
-			                               Binary(state, OpIAdd, TypeU32(state),
-			                                      Binary(state, OpShiftLeftLogical, TypeU32(state),
-			                                             probe, ConstantU32(state, 1u)),
-			                                      ConstantU32(state, 1u)));
+			const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
+			const auto mid    = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+			                           Binary(state, spv::OpIAdd, TypeU32(state), low, high),
+			                           ConstantU32(state, 1u));
+			const auto probe  = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid,
+			                          ConstantU32(state, 0u));
+			const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
+			                          Binary(state, spv::OpIAdd, TypeU32(state),
+			                                 Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
+			                                        probe, ConstantU32(state, 1u)),
+			                                 ConstantU32(state, 1u)));
 			const auto mapped_key = LoadMapping(entry);
-			const auto candidate =
-			    LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
-			const auto equal         = Binary(state, OpIEqual, TypeBool(state), mapped_key, key);
-			const auto match         = Binary(state, OpLogicalAnd, TypeBool(state), active, equal);
+			const auto candidate  = LoadMapping(
+			    Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
+			const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
+			const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
 			const auto next_selected = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), next_selected, match, candidate, selected});
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match,
+			                          candidate, selected);
 			selected              = next_selected;
-			const auto less       = Binary(state, OpULessThan, TypeBool(state), mapped_key, key);
-			const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less);
-			const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active,
-			                               Unary(state, OpLogicalNot, TypeBool(state), less));
+			const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
+			const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
+			const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
+			                               Unary(state, spv::OpLogicalNot, TypeBool(state), less));
 			const auto next_low   = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), next_low, take_upper,
-			     Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
+			    spv::OpSelect, TypeU32(state), next_low, take_upper,
+			    Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
 			low                  = next_low;
 			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high});
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid,
+			                          high);
 			high = next_high;
 		}
 		const auto            default_label = state.builder.AllocateId();
 		const auto            merge_label   = state.builder.AllocateId();
 		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
-		std::vector<uint32_t> switch_words {OpSwitch, selected, default_label};
+		std::vector<uint32_t> switch_words {spv::OpSwitch, selected, default_label};
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
 			labels[candidate - 1u] = state.builder.AllocateId();
 			switch_words.push_back(candidate);
 			switch_words.push_back(labels[candidate - 1u]);
 		}
-		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+		state.builder.AddFunction(spv::OpSelectionMerge, merge_label,
+		                          spv::SelectionControlMaskNone);
 		state.builder.AddFunction(switch_words);
-		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
+		std::vector<uint32_t> phi_words {spv::OpPhi, result_type, state.builder.AllocateId()};
 		EmitLabel(state, default_label);
 		phi_words.push_back(EmitSample(image.indirect_resources[0]));
 		phi_words.push_back(default_label);
-		state.builder.AddFunction({OpBranch, merge_label});
+		state.builder.AddFunction(spv::OpBranch, merge_label);
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
 			EmitLabel(state, labels[candidate - 1u]);
 			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
 			phi_words.push_back(labels[candidate - 1u]);
-			state.builder.AddFunction({OpBranch, merge_label});
+			state.builder.AddFunction(spv::OpBranch, merge_label);
 		}
 		EmitLabel(state, merge_label);
 		state.builder.AddFunction(phi_words);
@@ -860,30 +860,30 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			result = UnpackImageTexel(ctx, mem, result);
 		}
 		ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-		return true;
+		return;
 	}
 	const auto atomic_opcode = ImageAtomicOpcode(op);
-	if (atomic_opcode != 0u) {
+	if (atomic_opcode != spv::OpNop) {
 		const auto dimension = image.dimension;
 		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
-			           const auto pointer = state.builder.AllocateId();
-			           const auto pointer_type =
-			               state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)});
-			           state.builder.AddFunction(
-			               {OpImageTexelPointer, pointer_type, pointer,
-			                StorageImageDescriptorPointer(state, mem.resource),
-			                CoordU32(ctx, mem, *address, dimension), ConstantU32(state, 0)});
+			           const auto pointer      = state.builder.AllocateId();
+			           const auto pointer_type = state.builder.Type(
+			               spv::OpTypePointer, spv::StorageClassImage, TypeU32(state));
+			           state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
+			                                     StorageImageDescriptorPointer(state, mem.resource),
+			                                     CoordU32(ctx, mem, *address, dimension),
+			                                     ConstantU32(state, 0));
 			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
-			                                      ConstantU32(state, ScopeDevice),
-			                                      ConstantU32(state, MemorySemanticsNone),
-			                                      ctx.Arg(inst, 2)});
+			           state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer,
+			                                     ConstantU32(state, spv::ScopeDevice),
+			                                     ConstantU32(state, spv::MemorySemanticsMaskNone),
+			                                     ctx.Arg(inst, 2));
 			           EmitDeviceAtomicMemoryBarrier(state);
 			           return old;
 		           }));
-		return true;
+		return;
 	}
-	return false;
+	ctx.Fail(inst, "has no image SPIR-V emitter");
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter