Make AMD CPU patch optional

Keep the fan center across subgroups and reuse the existing index fetch.

.github/workflows/build.yml

@@ -6,7 +6,12 @@ on:
     branches: [ main, master ]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 env:
+  SCCACHE_GHA_ENABLED: "true"
   KYTY_BUILD_ORIGIN: ${{ github.repository != 'KytyPS5/KytyPS5' && 'Fork' || github.event_name == 'push' && 'Official' || 'Source' }}
   KYTY_BUILD_REPOSITORY: ${{ github.repository }}
 
@@ -78,6 +83,9 @@ jobs:
           arch: win64_msvc2022_64
           cache: true
 
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+
       - name: Verify toolchain
         shell: cmd
         run: |
@@ -100,6 +108,8 @@ jobs:
             -DKYTY_RELEASE_TAG=${{ needs.metadata.outputs.release_tag }} ^
             -DCMAKE_C_COMPILER=clang-cl ^
             -DCMAKE_CXX_COMPILER=clang-cl ^
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache ^
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ^
             -DCMAKE_PREFIX_PATH="%Qt6_DIR%"
 
       - name: Build
@@ -156,6 +166,9 @@ jobs:
           arch: clang_64
           cache: true
 
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+
       - name: Verify toolchain
         shell: bash
         run: |
@@ -180,6 +193,8 @@ jobs:
             -DCMAKE_OSX_ARCHITECTURES=x86_64 \
             -DCMAKE_C_COMPILER=clang \
             -DCMAKE_CXX_COMPILER=clang++ \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
             -DCMAKE_PREFIX_PATH="$QT_ROOT_DIR"
 
       - name: Build
@@ -289,9 +304,15 @@ jobs:
             clang \
             glslang-tools \
             libasound2-dev \
+            libavcodec-dev \
+            libavfilter-dev \
+            libavformat-dev \
+            libavutil-dev \
             libdbus-1-dev \
             libgl1-mesa-dev \
             libpulse-dev \
+            libswresample-dev \
+            libswscale-dev \
             libudev-dev \
             libwayland-dev \
             libx11-dev \
@@ -304,6 +325,7 @@ jobs:
             libxss-dev \
             lld \
             ninja-build \
+            pkg-config \
             wayland-protocols
 
       - name: Install Qt 6.10.3
@@ -315,6 +337,9 @@ jobs:
           arch: linux_gcc_64
           cache: true
 
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+
       - name: Verify toolchain
         shell: bash
         run: |
@@ -339,6 +364,8 @@ jobs:
             -DKYTY_RELEASE_TAG="${{ needs.metadata.outputs.release_tag }}" \
             -DCMAKE_C_COMPILER=clang \
             -DCMAKE_CXX_COMPILER=clang++ \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
             -DCMAKE_PREFIX_PATH="$Qt6_DIR" 2>&1 | tee _Build/configure.log
           exit "${PIPESTATUS[0]}"
 

3rdparty/CMakeLists.txt

@@ -88,6 +88,13 @@ set(SPDLOG_FMT_EXTERNAL ON CACHE BOOL "" FORCE)
 set(SPDLOG_NO_EXCEPTIONS ON CACHE BOOL "" FORCE)
 add_subdirectory(spdlog EXCLUDE_FROM_ALL)
 
+if (NOT TARGET FFmpeg::ffmpeg AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
+	find_package(PkgConfig REQUIRED)
+	pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET GLOBAL
+		libavformat libavcodec libswscale libavutil libavfilter libswresample)
+	add_library(FFmpeg::ffmpeg ALIAS PkgConfig::FFMPEG)
+endif()
+
 if (NOT TARGET FFmpeg::ffmpeg)
 	# On macOS the target arch is driven by CMAKE_OSX_ARCHITECTURES, not the host
 	# CMAKE_SYSTEM_PROCESSOR (which reports arm64 even for an x86_64/Rosetta build).

CMakeLists.txt

@@ -317,24 +317,10 @@ endif()
 
 set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL2-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak)
 
-# Linux system libraries required by the static FFmpeg archive.
+# Linux system libraries.
 if(LINUX)
 	find_package(Threads REQUIRED)
 	list(APPEND kyty_emulator_link_libraries m ${CMAKE_DL_LIBS} Threads::Threads)
-
-	# Optional FFmpeg dependencies.
-	find_package(ZLIB)
-	if(ZLIB_FOUND)
-		list(APPEND kyty_emulator_link_libraries ZLIB::ZLIB)
-	endif()
-	find_library(KYTY_BZ2_LIBRARY bz2)
-	if(KYTY_BZ2_LIBRARY)
-		list(APPEND kyty_emulator_link_libraries ${KYTY_BZ2_LIBRARY})
-	endif()
-	find_library(KYTY_LZMA_LIBRARY lzma)
-	if(KYTY_LZMA_LIBRARY)
-		list(APPEND kyty_emulator_link_libraries ${KYTY_LZMA_LIBRARY})
-	endif()
 endif()
 
 set(inc_headers
@@ -342,6 +328,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/SDL2/include
 	${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include
 	${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
+	${KYTY_THIRD_PARTY_DIR}/SPIRV-Headers/include
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/xxhash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
@@ -550,6 +537,10 @@ if(BUILD_TESTING)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME shader_recompiler_alignbyte
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
+	add_test(NAME graphics_pipeline_rasterization
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --polygon-mode-only)
+	add_test(NAME graphics_draw_offsets
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --draw-offset-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME guest_red_zone_patcher

README.md

@@ -90,8 +90,14 @@ starting a large change.
 
 Set up the clang-format hook after cloning:
 
-```powershell
-python -m pip install pre-commit
+Install `pre-commit` using the method appropriate for your platform:
+
+- **Arch Linux / CachyOS:** `sudo pacman -S pre-commit`
+- **Other Linux / macOS / Windows:** `python -m pip install pre-commit`
+
+Then install the Git hook:
+
+```bash
 python -m pre_commit install --install-hooks
 ```
 
@@ -159,13 +165,14 @@ The finished application and its runtime dependencies will be placed in
 
 ### Building on Linux
 
-Install the toolchain and the libraries the bundled SDL2 needs. Without the audio, Wayland and
+Install the toolchain, system FFmpeg, and the libraries the bundled SDL2 needs. Without the audio, Wayland and
 udev development packages SDL2 quietly configures itself without those backends, and the resulting
 build has no working sound and no gamepad hotplug:
 
 ```bash
 sudo apt-get install --no-install-recommends \
-  clang lld ninja-build cmake git glslang-tools \
+  clang lld ninja-build cmake git glslang-tools pkg-config \
+  libavformat-dev libavcodec-dev libswscale-dev libavutil-dev libavfilter-dev libswresample-dev \
   libgl1-mesa-dev libx11-dev libxcursor-dev libxext-dev libxfixes-dev \
   libxi-dev libxrandr-dev libxss-dev libxkbcommon-dev \
   libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
@@ -186,7 +193,8 @@ cmake --install _Build/linux --prefix _Build/linux/install
 ```
 
 The install step copies the Qt libraries and plugins next to the binaries, so
-`_Build/linux/install` runs without a matching system Qt.
+`_Build/linux/install` runs without a matching system Qt. The emulator still requires
+the system FFmpeg runtime libraries matching those used to build it.
 
 As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
 time.

src/common/alignment.h

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2014 Jannik Vogel <email@jannikvogel.de>
+// SPDX-License-Identifier: CC0-1.0
+
+#pragma once
+
+#include <cstddef>
+#include <type_traits>
+
+namespace Common {
+
+template <typename T>
+[[nodiscard]] constexpr T AlignUp(T value, std::size_t size) {
+	static_assert(std::is_unsigned_v<T>, "T must be an unsigned value.");
+	auto mod{static_cast<T>(value % size)};
+	value -= mod;
+	return static_cast<T>(mod == T{0} ? value : value + size);
+}
+
+template <typename T>
+[[nodiscard]] constexpr T AlignDown(T value, std::size_t size) {
+	static_assert(std::is_unsigned_v<T>, "T must be an unsigned value.");
+	return static_cast<T>(value - value % size);
+}
+
+} // namespace Common

src/common/emulatorConfig.cpp

@@ -43,6 +43,10 @@ int32_t GetUserId() {
 	return g_config->user_id;
 }
 
+const std::string& GetAudioInputDevice() {
+	return g_config->audio_input_device;
+}
+
 PresentMode GetPresentMode() {
 	return g_config->present_mode;
 }
@@ -55,6 +59,14 @@ bool FullscreenEnabled() {
 	return g_config->fullscreen_enabled;
 }
 
+bool VrEnabled() {
+	return g_config->vr_enabled;
+}
+
+bool AmdCpuEnabled() {
+	return g_config->amd_cpu_enabled;
+}
+
 uint32_t GetVblankFrequency() {
 	return std::clamp(g_config->vblank_frequency, 30u, 360u);
 }
@@ -75,7 +87,7 @@ ShaderOptimizationType GetShaderOptimizationType() {
 	return g_config->shader_optimization_type;
 }
 
-ShaderLogDirection GetShaderLogDirection() {
+LogDirection GetShaderLogDirection() {
 	return g_config->shader_log_direction;
 }
 
@@ -95,7 +107,7 @@ bool GraphicsDebugDumpEnabled() {
 	return g_config->graphics_debug_dump_enabled;
 }
 
-OutputDirection GetPrintfDirection() {
+LogDirection GetPrintfDirection() {
 	return g_config->printf_direction;
 }
 
@@ -103,8 +115,8 @@ std::filesystem::path GetPrintfOutputFile() {
 	return g_config->printf_output_file;
 }
 
-ProfilerDirection GetProfilerDirection() {
-	return g_config->profiler_direction;
+bool ProfilerEnabled() {
+	return g_config->profiler_enabled;
 }
 
 bool SpirvDebugPrintfEnabled() {

src/common/emulatorConfig.h

@@ -21,11 +21,7 @@ struct Lifecycle {
 
 enum class ShaderOptimizationType { None, Size, Performance };
 
-enum class ShaderLogDirection { Silent, Console, File };
-
-enum class ProfilerDirection { None, Network };
-
-enum class OutputDirection { Silent, Console, File };
+enum class LogDirection { Silent, Console, File };
 
 enum class PresentMode { Fifo, Mailbox, Immediate };
 
@@ -47,22 +43,25 @@ struct ConfigOptions {
 	uint32_t               screen_height               = 720;
 	std::string            user_name                   = "Kyty";
 	int32_t                user_id                     = DEFAULT_USER_ID;
-	PresentMode            present_mode                = PresentMode::Fifo;
+	std::string            audio_input_device;
+	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
+	bool                   vr_enabled                  = false;
+	bool                   amd_cpu_enabled             = false;
 	uint32_t               vblank_frequency            = 60;
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
 	bool                   shader_validation_enabled   = false;
 	ShaderOptimizationType shader_optimization_type    = ShaderOptimizationType::None;
-	ShaderLogDirection     shader_log_direction        = ShaderLogDirection::Silent;
+	LogDirection           shader_log_direction        = LogDirection::Silent;
 	std::filesystem::path  shader_log_folder           = "_Shaders";
 	bool                   command_buffer_dump_enabled = false;
 	std::filesystem::path  command_buffer_dump_folder  = "_Buffers";
 	bool                   graphics_debug_dump_enabled = false;
-	OutputDirection        printf_direction            = OutputDirection::Silent;
+	LogDirection           printf_direction            = LogDirection::Silent;
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
-	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
+	bool                   profiler_enabled            = false;
 	bool                   spirv_debug_printf_enabled  = false;
 	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
@@ -80,16 +79,19 @@ uint32_t GetScreenWidth();
 uint32_t GetScreenHeight();
 const std::string& GetUserName();
 int32_t  GetUserId();
+const std::string& GetAudioInputDevice();
 PresentMode GetPresentMode();
 int32_t GetGpuIndex();
 bool     FullscreenEnabled();
+bool     VrEnabled();
+bool     AmdCpuEnabled();
 uint32_t GetVblankFrequency();
 uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
 
 bool                   ShaderValidationEnabled();
 ShaderOptimizationType GetShaderOptimizationType();
-ShaderLogDirection     GetShaderLogDirection();
+LogDirection           GetShaderLogDirection();
 std::filesystem::path  GetShaderLogFolder();
 
 bool                  CommandBufferDumpEnabled();
@@ -97,10 +99,10 @@ std::filesystem::path GetCommandBufferDumpFolder();
 
 bool GraphicsDebugDumpEnabled();
 
-OutputDirection       GetPrintfDirection();
+LogDirection          GetPrintfDirection();
 std::filesystem::path GetPrintfOutputFile();
 
-ProfilerDirection GetProfilerDirection();
+bool ProfilerEnabled();
 
 bool SpirvDebugPrintfEnabled();
 

src/common/logging/log.cpp

@@ -129,6 +129,14 @@ static void WriteImpl(std::string_view text, fmt::text_style style = {}) {
 	}
 }
 
+void WriteToConsoleAndLog(std::string_view text) {
+	WriteImpl(text);
+	if (g_initialized && g_direction != Direction::Console) {
+		WriteStdout(text);
+	}
+	Flush();
+}
+
 void WriteFatal(std::string_view text) {
 	if (g_direction == Direction::Silent || !g_initialized) {
 		WriteStdout(text);
@@ -152,9 +160,9 @@ void WriteFatal(fmt::text_style style, std::string_view text) {
 void Initialize() {
 	g_initialized = true;
 	switch (Config::GetPrintfDirection()) {
-		case Config::OutputDirection::Silent: g_direction = Direction::Silent; break;
-		case Config::OutputDirection::Console: g_direction = Direction::Console; break;
-		case Config::OutputDirection::File: g_direction = Direction::File; break;
+		case Config::LogDirection::Silent: g_direction = Direction::Silent; break;
+		case Config::LogDirection::Console: g_direction = Direction::Console; break;
+		case Config::LogDirection::File: g_direction = Direction::File; break;
 	}
 	g_output_file =
 	    (g_direction == Direction::File ? Config::GetPrintfOutputFile() : std::filesystem::path {});

src/common/logging/log.h

@@ -25,6 +25,7 @@ Direction GetDirection();
 bool      IsSilent();
 void      Write(std::string_view text);
 void      Write(fmt::text_style style, std::string_view text);
+void      WriteToConsoleAndLog(std::string_view text);
 void      WriteFatal(std::string_view text);
 void      WriteFatal(fmt::text_style style, std::string_view text);
 void      Flush();

src/common/platform/sysLinuxVirtual.cpp

@@ -31,8 +31,7 @@
 namespace Common {
 
 static pthread_mutex_t              g_virtual_mutex {};
-static std::map<uintptr_t, size_t>* g_allocs   = nullptr;
-static std::map<uintptr_t, int>*    g_protects = nullptr;
+static std::map<uintptr_t, size_t>* g_allocs = nullptr;
 
 void SysVirtualInit() {
 	pthread_mutexattr_t attr {};
@@ -46,8 +45,7 @@ void SysVirtualInit() {
 	pthread_mutex_init(&g_virtual_mutex, &attr);
 	pthread_mutexattr_destroy(&attr);
 
-	g_allocs   = new std::map<uintptr_t, size_t>;
-	g_protects = new std::map<uintptr_t, int>;
+	g_allocs = new std::map<uintptr_t, size_t>;
 }
 
 static int get_protection_flag(VirtualMemory::Mode mode) {
@@ -68,21 +66,6 @@ static int get_protection_flag(VirtualMemory::Mode mode) {
 	return protect;
 }
 
-static VirtualMemory::Mode get_protection_flag(int mode) {
-	switch (mode) {
-		case PROT_NONE: return VirtualMemory::Mode::NoAccess;
-		case PROT_READ: return VirtualMemory::Mode::Read;
-		case PROT_WRITE: return VirtualMemory::Mode::Write;
-		case PROT_READ | PROT_WRITE: return VirtualMemory::Mode::ReadWrite; // NOLINT
-		case PROT_EXEC: return VirtualMemory::Mode::Execute;
-		case PROT_EXEC | PROT_WRITE: return VirtualMemory::Mode::ExecuteWrite; // NOLINT
-		case PROT_EXEC | PROT_READ: return VirtualMemory::Mode::ExecuteRead;   // NOLINT
-		case PROT_EXEC | PROT_WRITE | PROT_READ:
-			return VirtualMemory::Mode::ExecuteReadWrite; // NOLINT
-		default: return VirtualMemory::Mode::NoAccess;
-	}
-}
-
 // Keep automatic mappings inside the guest and GPU-addressable low window.
 #ifdef KYTY_FIXED_NOREPLACE
 static constexpr uintptr_t LOW_ARENA_LIMIT = 0x000000FC00000000ULL; // libc mspace window ceiling
@@ -161,11 +144,6 @@ uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mo
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start = ret_addr >> 12u;
-		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = protect;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 	}
 
@@ -251,11 +229,6 @@ uint64_t SysVirtualAllocAligned(uint64_t address, uint64_t size, VirtualMemory::
 
 	pthread_mutex_lock(&g_virtual_mutex);
 	record_alloc(ret_addr, size);
-	uintptr_t page_start = ret_addr >> 12u;
-	uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-	for (uintptr_t page = page_start; page <= page_end; page++) {
-		(*g_protects)[page] = protect;
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 
 	return ret_addr;
@@ -337,11 +310,6 @@ bool SysVirtualAllocFixed(uint64_t address, uint64_t size, VirtualMemory::Mode m
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start = ret_addr >> 12u;
-		uintptr_t page_end   = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = protect;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 
 		return true;
@@ -516,18 +484,7 @@ bool SysVirtualFree(uint64_t address) {
 		return false;
 	}
 
-	if (munmap(reinterpret_cast<void*>(addr), size) == 0) {
-		uintptr_t page_start = addr >> 12u;
-		uintptr_t page_end   = (addr + size - 1) >> 12u;
-		pthread_mutex_lock(&g_virtual_mutex);
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			g_protects->erase(page);
-		}
-		pthread_mutex_unlock(&g_virtual_mutex);
-		return true;
-	}
-
-	return false;
+	return munmap(reinterpret_cast<void*>(addr), size) == 0;
 }
 
 bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
@@ -582,40 +539,16 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 	if (end < alloc_end) {
 		(*g_allocs)[end] = alloc_end - end;
 	}
-	for (uintptr_t page = addr >> 12u; page <= (end - 1u) >> 12u; page++) {
-		g_protects->erase(page);
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 	return true;
 }
 
-bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                       VirtualMemory::Mode* old_mode) {
-	auto addr = static_cast<uintptr_t>(address);
-
-	pthread_mutex_lock(&g_virtual_mutex);
-	if (old_mode != nullptr) {
-		if (auto s = g_protects->find(addr >> 12u); s != g_protects->end()) {
-			*old_mode = get_protection_flag(s->second);
-		} else {
-			*old_mode = VirtualMemory::Mode::NoAccess;
-		}
-	}
-	pthread_mutex_unlock(&g_virtual_mutex);
-
-	uintptr_t page_start = addr >> 12u;
-	uintptr_t page_end   = (addr + size - 1) >> 12u;
-	if (mprotect(reinterpret_cast<void*>(page_start << 12u), (page_end - page_start + 1) << 12u,
-	             get_protection_flag(mode)) == 0) {
-		pthread_mutex_lock(&g_virtual_mutex);
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = get_protection_flag(mode);
-		}
-		pthread_mutex_unlock(&g_virtual_mutex);
-		return true;
-	}
-
-	return false;
+bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
+	const auto addr       = static_cast<uintptr_t>(address);
+	const auto page_start = addr >> 12u;
+	const auto page_end   = (addr + size - 1) >> 12u;
+	return mprotect(reinterpret_cast<void*>(page_start << 12u), (page_end - page_start + 1) << 12u,
+	                get_protection_flag(mode)) == 0;
 }
 
 bool SysVirtualFlushInstructionCache(uint64_t /*address*/, uint64_t /*size*/) {

src/common/platform/sysVirtual.h

@@ -19,8 +19,7 @@ bool     SysVirtualReserveFixed(uint64_t address, uint64_t size);
 bool     SysVirtualDecommit(uint64_t address, uint64_t size);
 bool     SysVirtualFree(uint64_t address);
 bool     SysVirtualFreeRange(uint64_t address, uint64_t size);
-bool     SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                           VirtualMemory::Mode* old_mode = nullptr);
+bool     SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode);
 bool     SysVirtualFlushInstructionCache(uint64_t address, uint64_t size);
 
 } // namespace Common

src/common/platform/sysWindowsVirtual.cpp

@@ -45,18 +45,6 @@ static DWORD GetProtectionFlag(VirtualMemory::Mode mode) {
 	return protect;
 }
 
-static VirtualMemory::Mode GetProtectionFlag(DWORD mode) {
-	switch (mode) {
-		case PAGE_NOACCESS: return VirtualMemory::Mode::NoAccess;
-		case PAGE_READONLY: return VirtualMemory::Mode::Read;
-		case PAGE_READWRITE: return VirtualMemory::Mode::ReadWrite;
-		case PAGE_EXECUTE: return VirtualMemory::Mode::Execute;
-		case PAGE_EXECUTE_READ: return VirtualMemory::Mode::ExecuteRead;
-		case PAGE_EXECUTE_READWRITE: return VirtualMemory::Mode::ExecuteReadWrite;
-		default: return VirtualMemory::Mode::NoAccess;
-	}
-}
-
 void SysVirtualInit() {}
 
 uint64_t SysVirtualAlloc(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
@@ -328,17 +316,13 @@ bool SysVirtualFreeRange(uint64_t address, uint64_t size) {
 	return current - address == size && SysVirtualFree(address);
 }
 
-bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode,
-                       VirtualMemory::Mode* old_mode) {
+bool SysVirtualProtect(uint64_t address, uint64_t size, VirtualMemory::Mode mode) {
 	DWORD old_protect = 0;
 	if (VirtualProtect(reinterpret_cast<LPVOID>(static_cast<uintptr_t>(address)), size,
 	                   GetProtectionFlag(mode), &old_protect) == 0) {
 		printf("VirtualProtect() failed: 0x%08" PRIx32 "\n", static_cast<uint32_t>(GetLastError()));
 		return false;
 	}
-	if (old_mode != nullptr) {
-		*old_mode = GetProtectionFlag(old_protect);
-	}
 	return true;
 }
 

src/common/profiler.cpp

@@ -55,19 +55,13 @@ void SetThreadName(const char* name) {
 }
 
 void Initialize() {
-	switch (Config::GetProfilerDirection()) {
-		case Config::ProfilerDirection::Network:
-			if (!tracy::ProfilerAvailable()) {
-				tracy::StartupProfiler();
-				TracySetProgramName("KytyPS5");
-				::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, "
-				         "broadcast %u, connect to 127.0.0.1:8086\n",
-				         tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch,
-				         tracy::ProtocolVersion, tracy::BroadcastVersion);
-			}
-			break;
-		case Config::ProfilerDirection::None:
-		default: break;
+	if (Config::ProfilerEnabled() && !tracy::ProfilerAvailable()) {
+		tracy::StartupProfiler();
+		TracySetProgramName("KytyPS5");
+		::printf("Tracy profiler enabled: client %d.%d.%d, protocol %u, "
+		         "broadcast %u, connect to 127.0.0.1:8086\n",
+		         tracy::Version::Major, tracy::Version::Minor, tracy::Version::Patch,
+		         tracy::ProtocolVersion, tracy::BroadcastVersion);
 	}
 }
 

src/common/threads.cpp

@@ -8,7 +8,6 @@
 #include <chrono>             // IWYU pragma: keep
 #include <condition_variable> // IWYU pragma: keep
 #include <mutex>
-#include <vector>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_COMPILER == KYTY_COMPILER_CLANG
 #define KYTY_WIN_CS
@@ -197,37 +196,7 @@ struct CondVarPrivate {
 #endif
 };
 
-static std::recursive_mutex                         g_cond_waiters_mutex;
-static std::vector<std::pair<int, CondVarPrivate*>> g_cond_waiters;
-static wait_poll_func_t                             g_cond_wait_poll_callback = nullptr;
-
-static void WakeCondVar(CondVarPrivate* cond_var) {
-#ifdef KYTY_WIN_CS
-	static auto func = ResolveWakeAllConditionVariable();
-	EXIT_NOT_IMPLEMENTED(func == nullptr);
-	func(&cond_var->m_cv);
-#else
-	cond_var->m_cv.notify_all();
-#endif
-}
-
-static void RegisterCondWaiter(CondVarPrivate* cond_var) {
-	std::lock_guard lock(g_cond_waiters_mutex);
-	g_cond_waiters.emplace_back(Thread::GetThreadIdUnique(), cond_var);
-}
-
-static void UnregisterCondWaiter(CondVarPrivate* cond_var) {
-	const auto      thread_id = Thread::GetThreadIdUnique();
-	std::lock_guard lock(g_cond_waiters_mutex);
-
-	const auto it = std::find_if(g_cond_waiters.begin(), g_cond_waiters.end(),
-	                             [thread_id, cond_var](const auto& waiter) {
-		                             return waiter.first == thread_id && waiter.second == cond_var;
-	                             });
-	if (it != g_cond_waiters.end()) {
-		g_cond_waiters.erase(it);
-	}
-}
+static wait_poll_func_t g_cond_wait_poll_callback = nullptr;
 
 struct ThreadPrivate {
 	ThreadPrivate(thread_func_t f, void* a): func(f), arg(a), m_thread(&Run, this) {}
@@ -365,7 +334,6 @@ CondVar::~CondVar() {
 }
 
 void CondVar::Wait(Mutex* mutex) {
-	RegisterCondWaiter(m_cond_var.get());
 #ifndef KYTY_WIN_CS
 	std::unique_lock<std::recursive_mutex> cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t());
 #endif
@@ -406,7 +374,6 @@ void CondVar::Wait(Mutex* mutex) {
 	}
 	cpp_lock.release();
 #endif
-	UnregisterCondWaiter(m_cond_var.get());
 }
 
 void CondVar::SetWaitPollCallback(wait_poll_func_t callback) {
@@ -415,7 +382,6 @@ void CondVar::SetWaitPollCallback(wait_poll_func_t callback) {
 
 bool CondVar::WaitFor(Mutex* mutex, uint32_t micros) {
 	bool ok = false;
-	RegisterCondWaiter(m_cond_var.get());
 #ifndef KYTY_WIN_CS
 	std::unique_lock<std::recursive_mutex> cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t());
 #endif
@@ -430,7 +396,6 @@ bool CondVar::WaitFor(Mutex* mutex, uint32_t micros) {
 	      std::cv_status::no_timeout);
 	cpp_lock.release();
 #endif
-	UnregisterCondWaiter(m_cond_var.get());
 	return ok;
 }
 
@@ -445,16 +410,13 @@ void CondVar::Signal() {
 }
 
 void CondVar::SignalAll() {
-	WakeCondVar(m_cond_var.get());
-}
-
-void CondVar::SignalThread(int thread_id) {
-	std::lock_guard lock(g_cond_waiters_mutex);
-	for (const auto& waiter: g_cond_waiters) {
-		if (waiter.first == thread_id) {
-			WakeCondVar(waiter.second);
-		}
-	}
+#ifdef KYTY_WIN_CS
+	static auto func = ResolveWakeAllConditionVariable();
+	EXIT_NOT_IMPLEMENTED(func == nullptr);
+	func(&m_cond_var->m_cv);
+#else
+	m_cond_var->m_cv.notify_all();
+#endif
 }
 
 int Thread::GetThreadIdUnique() {

src/common/threads.h

@@ -77,7 +77,6 @@ public:
 	void Signal();
 	void SignalAll();
 
-	static void SignalThread(int thread_id);
 	static void SetWaitPollCallback(wait_poll_func_t callback);
 
 	KYTY_CLASS_NO_COPY(CondVar);

src/common/virtualMemory.cpp

@@ -50,8 +50,8 @@ bool FreeRange(uint64_t address, uint64_t size) {
 	return SysVirtualFreeRange(address, size);
 }
 
-bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode) {
-	return SysVirtualProtect(address, size, mode, old_mode);
+bool Protect(uint64_t address, uint64_t size, Mode mode) {
+	return SysVirtualProtect(address, size, mode);
 }
 
 bool FlushInstructionCache(uint64_t address, uint64_t size) {

src/common/virtualMemory.h

@@ -35,7 +35,7 @@ bool     ReserveFixed(uint64_t address, uint64_t size);
 bool     Decommit(uint64_t address, uint64_t size);
 bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
-bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
+bool     Protect(uint64_t address, uint64_t size, Mode mode);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
 
 } // namespace VirtualMemory

src/emulator.cpp

@@ -196,6 +196,10 @@ void Run(const RunOptions& options) {
 	ClearDebugTextureFolder();
 
 	PrintSystemInfo();
+	std::string title_id;
+	if (Loader::SystemContentParamSfoGetString("TITLE_ID", &title_id) && !title_id.empty()) {
+		Log::WriteToConsoleAndLog(fmt::format("Title ID: {}\n", title_id));
+	}
 
 	int ok = atexit(KytyClose);
 	EXIT_NOT_IMPLEMENTED(ok != 0);

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -32,11 +32,12 @@ private:
 
 	struct BufferCursor {
 		std::span<const uint32_t> commands;
-		uint32_t                  offset_dw           = 0;
-		uint32_t                  deferred_advance_dw = 0;
+		uint32_t                  offset_dw = 0;
 	};
 
 	std::vector<BufferCursor> m_buffer_stack;
+	std::span<const uint32_t> m_next_buffer;
+	bool                      m_chain         = false;
 	bool                      m_suspended     = false;
 	bool                      m_made_progress = false;
 };
@@ -129,7 +130,7 @@ public:
 	[[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; }
 
 	Pm4ProcessResult Process(Pm4Execution& execution, std::span<const uint32_t> commands);
-	void             ProcessIndirectBuffer(std::span<const uint32_t> commands);
+	void             ProcessIndirectBuffer(std::span<const uint32_t> commands, bool chain);
 
 	void SetFlip(const FlipInfo& flip) { m_flip = flip; }
 
@@ -143,12 +144,11 @@ private:
 	                      uint32_t cache_action, uint32_t event_index, uint32_t event_write_source,
 	                      void* dst_gpu_addr, T value, uint32_t interrupt_selector,
 	                      uint32_t interrupt_context_id);
-	void ProcessPm4(Pm4Execution& execution, size_t stop_depth);
+	void ProcessPm4(Pm4Execution& execution);
 	void SuspendPm4();
 	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
 	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
 	void                CheckBuffer() const { GetScheduler().CheckActive(); }
-	GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); }
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -1495,10 +1495,10 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	     reinterpret_cast<uint64_t>(else_buffer), else_num_dw);
 
 	if (take_then) {
-		cp.ProcessIndirectBuffer({then_buffer, then_num_dw});
+		cp.ProcessIndirectBuffer({then_buffer, then_num_dw}, true);
 	} else if (mode == 2 && else_num_dw != 0) {
 		EXIT_NOT_IMPLEMENTED(else_buffer == nullptr);
-		cp.ProcessIndirectBuffer({else_buffer, else_num_dw});
+		cp.ProcessIndirectBuffer({else_buffer, else_num_dw}, true);
 	}
 
 	return payload_dw;
@@ -1959,7 +1959,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 
 	GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer);
 
-	cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw});
+	cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw}, (control & (1u << 20u)) != 0);
 
 	return 3;
 }
@@ -3635,6 +3635,11 @@ void GraphicsInitJmpTablesShIndirect() {
 	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_RSRC4_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
+	// Toolkit state restoration emits the compiler's GS-front allocation
+	// metadata here. Native GS resource registers already carry that allocation.
+	g_hw_sh_indirect_func[0x0ca] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+		HwShIgnoreShaderRegister(cmd_offset, value);
+	};
 	g_hw_sh_indirect_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
@@ -3654,12 +3659,13 @@ void GraphicsInitJmpTablesShIndirect() {
 	g_hw_sh_indirect_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
-	g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
-		HwShIgnoreShaderRegister(cmd_offset, value);
-	};
-	g_hw_sh_indirect_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
-		HwShIgnoreShaderRegister(cmd_offset, value);
-	};
+	for (uint32_t offset = Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS;
+	     offset <= Pm4::SPI_SHADER_USER_DATA_ADDR_HI_HS; offset++) {
+		g_hw_sh_indirect_func[offset] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+			cp.GetShCtx().SetHsUserDataAddress(cmd_offset - Pm4::SPI_SHADER_USER_DATA_ADDR_LO_HS,
+			                                   value);
+		};
+	}
 	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_HS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		auto base = cp.GetShCtx().GetVs().hs_regs.data_addr;
 		base &= 0xFFFFFF00000000FFull;
@@ -3818,6 +3824,25 @@ void GraphicsInitJmpTablesUcIndirect() {
 	for (auto& func: g_hw_uc_indirect_func) {
 		func = nullptr;
 	}
+	for (uint32_t i = 0; i < 4; ++i) {
+		g_hw_uc_indirect_func[Pm4::FSR_CONTROL_POINTS_LEFT_X + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrControlPoint(0, cmd_offset - Pm4::FSR_CONTROL_POINTS_LEFT_X, value);
+		};
+		g_hw_uc_indirect_func[Pm4::FSR_CONTROL_POINTS_LEFT_Y + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrControlPoint(1, cmd_offset - Pm4::FSR_CONTROL_POINTS_LEFT_Y, value);
+		};
+	}
+	for (uint32_t i = 0; i < 2; ++i) {
+		g_hw_uc_indirect_func[Pm4::FSR_ALPHA_LEFT_X + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrAlpha(0, cmd_offset - Pm4::FSR_ALPHA_LEFT_X, value);
+		};
+		g_hw_uc_indirect_func[Pm4::FSR_ALPHA_LEFT_Y + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrAlpha(1, cmd_offset - Pm4::FSR_ALPHA_LEFT_Y, value);
+		};
+		g_hw_uc_indirect_func[Pm4::FSR_WINDOW_LEFT + i] = [](KYTY_HW_UC_INDIRECT_ARGS) {
+			cp.GetUcfg().SetFsrWindow(cmd_offset - Pm4::FSR_WINDOW_LEFT, value);
+		};
+	}
 
 	g_hw_uc_indirect_func[Pm4::GE_CNTL] = [](KYTY_HW_UC_INDIRECT_ARGS) {
 		HW::GeControl r;

src/graphics/guest_gpu/graphicsRun.cpp

@@ -77,7 +77,7 @@ private:
 
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
-	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
+	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
 }
 
 GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
@@ -427,7 +427,7 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 	if (!decode_gds(dst_sel, dst_gds)) {
 		EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel);
 	}
-	auto& buffer_cache = GetGpuResources().GetBufferCache();
+	auto& buffer_cache = m_renderer.GetBufferCache();
 	if (src_sel == 2) {
 		buffer_cache.FillBuffer(
 		    dst_address_or_offset, num_bytes,
@@ -603,11 +603,11 @@ bool GuestGpu::Process(Submission& submission) {
 			}
 			if (progressed) {
 				if (complete) {
-					m_renderer.GetGpuResources().RunGarbageCollector();
+					m_renderer.RunGarbageCollector();
 				}
 				cp.BufferFlush();
 			} else if (complete) {
-				m_renderer.GetGpuResources().RunGarbageCollector();
+				m_renderer.RunGarbageCollector();
 			}
 			break;
 		}
@@ -629,16 +629,16 @@ bool GuestGpu::Process(Submission& submission) {
 			           Pm4ProcessResult::Complete;
 			if (submission.command_execution.MadeProgress()) {
 				if (complete) {
-					m_renderer.GetGpuResources().RunGarbageCollector();
+					m_renderer.RunGarbageCollector();
 				}
 				cp.BufferFlush();
 			} else if (complete) {
-				m_renderer.GetGpuResources().RunGarbageCollector();
+				m_renderer.RunGarbageCollector();
 			}
 			break;
 		}
 		case SubmissionType::FlipPreparation:
-			m_renderer.GetGpuResources().RunGarbageCollector();
+			m_renderer.RunGarbageCollector();
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
 	}
@@ -672,20 +672,16 @@ Pm4ProcessResult CommandProcessor::Process(Pm4Execution&             execution,
 		Pm4Execution*     previous_execution;
 	} execution_scope(*this, execution);
 
-	ProcessPm4(execution, 0);
+	ProcessPm4(execution);
 	return execution.m_buffer_stack.empty() ? Pm4ProcessResult::Complete
 	                                        : Pm4ProcessResult::Blocked;
 }
 
-void CommandProcessor::ProcessIndirectBuffer(std::span<const uint32_t> commands) {
+void CommandProcessor::ProcessIndirectBuffer(std::span<const uint32_t> commands, bool chain) {
 	EXIT_IF(g_current_execution == nullptr);
-	if (commands.empty()) {
-		return;
-	}
-	auto&      execution  = *g_current_execution;
-	const auto stop_depth = execution.m_buffer_stack.size();
-	execution.m_buffer_stack.push_back({commands});
-	ProcessPm4(execution, stop_depth);
+	EXIT_IF(!g_current_execution->m_next_buffer.empty());
+	g_current_execution->m_next_buffer = commands;
+	g_current_execution->m_chain       = chain;
 }
 
 void CommandProcessor::SuspendPm4() {
@@ -693,21 +689,13 @@ void CommandProcessor::SuspendPm4() {
 	g_current_execution->m_suspended = true;
 }
 
-void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
-	while (execution.m_buffer_stack.size() > stop_depth) {
+void CommandProcessor::ProcessPm4(Pm4Execution& execution) {
+	while (!execution.m_buffer_stack.empty()) {
 		if (g_gpu_state != nullptr) {
 			g_gpu_state->ProcessCommands();
 		}
-		const auto buffer_index = execution.m_buffer_stack.size() - 1;
-		auto&      cursor       = execution.m_buffer_stack[buffer_index];
+		auto& cursor = execution.m_buffer_stack.back();
 		EXIT_IF(cursor.offset_dw > cursor.commands.size());
-		if (cursor.deferred_advance_dw != 0) {
-			EXIT_IF(cursor.deferred_advance_dw > cursor.commands.size() - cursor.offset_dw);
-			cursor.offset_dw += cursor.deferred_advance_dw;
-			cursor.deferred_advance_dw = 0;
-			execution.m_made_progress  = true;
-			continue;
-		}
 		if (cursor.offset_dw == cursor.commands.size()) {
 			execution.m_buffer_stack.pop_back();
 			continue;
@@ -783,14 +771,19 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 		    handler(*this, packet_header & ~1u, packet + 1, remaining_dw, total_dw) + 1;
 		EXIT_IF(packet_dw > remaining_dw);
 		if (execution.m_suspended) {
-			if (execution.m_buffer_stack.size() > buffer_index + 1) {
-				execution.m_buffer_stack[buffer_index].deferred_advance_dw = packet_dw;
-			}
 			return;
 		}
-		EXIT_IF(execution.m_buffer_stack.size() != buffer_index + 1);
-		execution.m_buffer_stack[buffer_index].offset_dw += packet_dw;
+		cursor.offset_dw += packet_dw;
 		execution.m_made_progress = true;
+		if (!execution.m_next_buffer.empty()) {
+			// Chains and taken branches reuse the fetcher; only calls retain a return cursor.
+			if (execution.m_chain) {
+				cursor = {execution.m_next_buffer};
+			} else {
+				execution.m_buffer_stack.push_back({execution.m_next_buffer});
+			}
+			execution.m_next_buffer = {};
+		}
 	}
 }
 
@@ -825,36 +818,55 @@ void CommandProcessor::SetNumInstances(uint32_t num_instances) {
 
 void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t wait_op,
                                       const volatile void* address, uint32_t count_in_dwords) {
-	if (wait_op != 0) {
-		BufferFlushAndWait();
-	}
-
 	(void)count_in_dwords;
+	uint64_t value = 0;
 
 	switch (op) {
-		case 0x00: {
+		case 0x00:
 			m_predicate_skip = false;
-		} break;
-		case 0x03: {
+			return;
+		case 0x01: {
 			EXIT_NOT_IMPLEMENTED(address == nullptr);
-
-			auto value = *reinterpret_cast<const volatile uint64_t*>(address);
-
-			switch (condition) {
-				case 0x00: m_predicate_skip = (value != 0); break;
-				case 0x01: m_predicate_skip = (value == 0); break;
-				default: EXIT("unknown predication condition: 0x%08" PRIx32 "\n", condition);
-			}
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1) < 128) {
-				LOGF("\t bool predication: addr=0x%016" PRIx64 ", value=0x%016" PRIx64
-				     ", condition=%" PRIu32 ", skip=%u, wait_op=%" PRIu32 "\n",
-				     reinterpret_cast<uint64_t>(address), value, condition,
-				     m_predicate_skip ? 1u : 0u, wait_op);
+			// One begin/end pair per DB; bit 63 marks each counter ready.
+			constexpr uint64_t ready_bit = 1ull << 63u;
+			const auto* results = reinterpret_cast<const volatile uint64_t*>(address);
+			for (uint32_t db = 0; db < 16u; db++) {
+				const auto begin = results[db * 2u];
+				const auto end   = results[db * 2u + 1u];
+				if ((begin & end & ready_bit) == 0) {
+					if (wait_op == 0) {
+						SuspendPm4();
+					} else {
+						m_predicate_skip = false;
+					}
+					return;
+				}
+				value += end - begin;
 			}
 		} break;
+		case 0x03:
+			if (wait_op != 0) {
+				BufferFlushAndWait();
+			}
+			EXIT_NOT_IMPLEMENTED(address == nullptr);
+			value = *reinterpret_cast<const volatile uint64_t*>(address);
+			break;
 		default: EXIT("unknown predication op: 0x%08" PRIx32 "\n", op);
 	}
+	switch (condition) {
+		case 0x00: m_predicate_skip = (value != 0); break;
+		case 0x01: m_predicate_skip = (value == 0); break;
+		default: EXIT("unknown predication condition: 0x%08" PRIx32 "\n", condition);
+	}
+	if (op == 0x03) {
+		static std::atomic<uint32_t> log_count {0};
+		if (log_count.fetch_add(1) < 128) {
+			LOGF("\t bool predication: addr=0x%016" PRIx64 ", value=0x%016" PRIx64
+			     ", condition=%" PRIu32 ", skip=%u, wait_op=%" PRIu32 "\n",
+			     reinterpret_cast<uint64_t>(address), value, condition,
+			     m_predicate_skip ? 1u : 0u, wait_op);
+		}
+	}
 }
 
 void CommandProcessor::DrawIndex(DrawIndexArgs args) {
@@ -1090,17 +1102,6 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 		// local_x        = std::max(cs.num_thread_x, 1u);
 		// local_y        = std::max(cs.num_thread_y, 1u);
 		// local_z        = std::max(cs.num_thread_z, 1u);
-		if (cs.wave_size == 64u) {
-			static std::atomic_bool logged_wave64_shader {false};
-			if (!logged_wave64_shader.exchange(true, std::memory_order_relaxed)) {
-				LOGF("warning: executing wave64 compute shader cs=0x%016" PRIx64 "\n",
-				     cs.data_addr);
-				std::printf("warning: executing wave64 compute shader cs=0x%016" PRIx64 "\n",
-				            cs.data_addr);
-				std::fflush(stdout);
-			}
-		}
-
 		m_renderer.GetRenderExecutor().DispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x,
 		                                              thread_group_y, thread_group_z, mode);
 	}
@@ -1292,6 +1293,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 								break;
 							case 0x14:
 							case 0x28:
+							case 0x2f:
 								if (event_index == 0x00) {
 									write64(false);
 									return;
@@ -1299,7 +1301,6 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 								break;
 							case 0x2b:
 							case 0x2d:
-							case 0x2f:
 							case 0x30:
 								if (event_index == 0x00 && !with_interrupt) {
 									write64(false);

src/graphics/guest_gpu/hardwareContext.h

@@ -498,7 +498,8 @@ struct LsStageRegisters {
 };
 
 struct HsStageRegisters {
-	uint64_t          data_addr = 0;
+	uint64_t          data_addr      = 0;
+	uint64_t          user_data_addr = 0;
 	HsShaderResource1 rsrc1;
 	HsShaderResource2 rsrc2;
 };
@@ -935,6 +936,12 @@ private:
 	ShaderRegisters m_sh_regs;
 };
 
+struct FsrView {
+	uint32_t control_points[2][4] {};
+	uint32_t alphas[2][2] {};
+	uint32_t window[2] {};
+};
+
 class UserConfig {
 public:
 	UserConfig()  = default;
@@ -964,8 +971,17 @@ public:
 	void SetGdsOaCntl(uint32_t value) { m_gds_oa.cntl = value; }
 	void SetGdsOaCounter(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].counter = value; }
 	void SetGdsOaAddress(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].address = value; }
+	void SetFsrControlPoint(uint32_t axis, uint32_t index, uint32_t value) {
+		m_fsr_view.control_points[axis][index] = value;
+	}
+	void SetFsrAlpha(uint32_t axis, uint32_t index, uint32_t value) {
+		m_fsr_view.alphas[axis][index] = value;
+	}
+	void SetFsrWindow(uint32_t index, uint32_t value) { m_fsr_view.window[index] = value; }
+	[[nodiscard]] const FsrView& GetFsrView() const { return m_fsr_view; }
 
 private:
+	FsrView m_fsr_view;
 	Prospero::PrimitiveType m_prim_type               = Prospero::PrimitiveType::kNone;
 	uint32_t                m_index_offset            = 0;
 	uint32_t                m_object_id               = 0;
@@ -988,14 +1004,14 @@ public:
 	void SetEsShaderBase(uint64_t addr) { m_vs.es_regs.data_addr = addr; }
 	void SetLsShaderBase(uint64_t addr) { m_vs.ls_regs.data_addr = addr; }
 	void SetHsShaderBase(uint64_t addr) { m_vs.hs_regs.data_addr = addr; }
+	void SetHsUserDataAddress(uint32_t word, uint32_t value) {
+		SetUserDataAddressWord(m_vs.hs_regs.user_data_addr, word, value);
+	}
 	void SetHsShaderResource1(const HsShaderResource1& rsrc1) { m_vs.hs_regs.rsrc1 = rsrc1; }
 	void SetHsShaderResource2(const HsShaderResource2& rsrc2) { m_vs.hs_regs.rsrc2 = rsrc2; }
 	void SetGsShaderBase(uint64_t addr) { m_vs.gs_regs.data_addr = addr; }
 	void SetGsUserDataAddress(uint32_t word, uint32_t value) {
-		const auto shift   = word * 32u;
-		auto&      address = m_vs.gs_regs.user_data_addr;
-		address = (address & ~(uint64_t {0xffffffffu} << shift)) |
-		          (static_cast<uint64_t>(value) << shift);
+		SetUserDataAddressWord(m_vs.gs_regs.user_data_addr, word, value);
 	}
 	void SetGsShaderResource1(const GsShaderResource1& rsrc1) { m_vs.gs_regs.rsrc1 = rsrc1; }
 	void SetGsShaderResource2(const GsShaderResource2& rsrc2) { m_vs.gs_regs.rsrc2 = rsrc2; }
@@ -1037,6 +1053,12 @@ public:
 	[[nodiscard]] const ComputeShaderInfo& GetCs() const { return m_cs; }
 
 private:
+	static void SetUserDataAddressWord(uint64_t& address, uint32_t word, uint32_t value) {
+		const auto shift = word * 32u;
+		address = (address & ~(uint64_t {0xffffffffu} << shift)) |
+		          (static_cast<uint64_t>(value) << shift);
+	}
+
 	VertexShaderInfo  m_vs;
 	PixelShaderInfo   m_ps;
 	ComputeShaderInfo m_cs;

src/graphics/guest_gpu/pm4.h

@@ -987,6 +987,12 @@ constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR2_MASK  = 0x1;
 constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR3_SHIFT = 2;
 constexpr uint32_t GE_USER_VGPR_EN_EN_USER_VGPR3_MASK  = 0x1;
 
+constexpr uint32_t FSR_WINDOW_LEFT           = 0x2BC;
+constexpr uint32_t FSR_CONTROL_POINTS_LEFT_X = 0x2C4;
+constexpr uint32_t FSR_CONTROL_POINTS_LEFT_Y = 0x2CC;
+constexpr uint32_t FSR_ALPHA_LEFT_X          = 0x2E4;
+constexpr uint32_t FSR_ALPHA_LEFT_Y          = 0x2E8;
+
 constexpr uint32_t TA_CS_BC_BASE_ADDR       = 0x380;
 constexpr uint32_t TA_CS_BC_BASE_ADDR_HI    = 0x381;
 constexpr uint32_t TEXTURE_GRADIENT_FACTORS = 0x383;

src/graphics/guest_gpu/tile.cpp

@@ -1,5 +1,6 @@
 #include "graphics/guest_gpu/tile.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -15,10 +16,6 @@
 
 namespace Libs::Graphics {
 
-static uint32_t AlignUp(uint32_t value, uint32_t alignment) {
-	return (value + alignment - 1u) & ~(alignment - 1u);
-}
-
 static uint32_t ShiftCeil(uint32_t value, uint32_t shift) {
 	return static_cast<uint32_t>((static_cast<uint64_t>(value) + (1ull << shift) - 1ull) >> shift);
 }
@@ -33,7 +30,7 @@ static uint32_t CalcLinearAlignedLevelPitch(uint32_t base_width, uint32_t base_h
 	const uint32_t level_width  = ShiftCeil(base_width, level);
 	const uint32_t level_height = ShiftCeil(base_height, level);
 	const uint32_t padded_width =
-	    AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element));
+	    Common::AlignUp(std::max(level_width, 1u), CalcLinearBlockWidth(bytes_per_element));
 	const uint64_t size =
 	    static_cast<uint64_t>(padded_width) * std::max(level_height, 1u) * bytes_per_element;
 	EXIT_NOT_IMPLEMENTED(size > 0xffffffffull);
@@ -61,7 +58,7 @@ static uint32_t SetLinearMipChainLayout(uint32_t levels, const uint32_t* mip_pit
 		offset += mip_size[level];
 	}
 
-	return AlignUp(offset, 256u);
+	return Common::AlignUp(offset, 256u);
 }
 
 struct Log2BlockDimensions {
@@ -366,8 +363,8 @@ bool TileGetTiledTextureLayout(const TileSurfaceDescription& description, TileSu
 		    ((description.width >> level) + texture.texel_width - 1u) / texture.texel_width, 1u);
 		mip.height = std::max(
 		    ((description.height >> level) + texture.texel_height - 1u) / texture.texel_height, 1u);
-		mip.padded_width  = AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width);
-		mip.padded_height = AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height);
+		mip.padded_width  = Common::AlignUp(std::max(ShiftCeil(width0, level), 1u), block.block_width);
+		mip.padded_height = Common::AlignUp(std::max(ShiftCeil(height0, level), 1u), block.block_height);
 		mip.size = static_cast<uint64_t>(block.block_depth) * mip.padded_width * mip.padded_height *
 		           block.bytes_per_element;
 		result.block_slice_size += mip.size;
@@ -1121,8 +1118,8 @@ static bool TileGetHtileSize(uint32_t width, uint32_t height, TileSizeAlign& hti
 	}
 	// Prospero HTile stores one DWORD per depth tile. Its 32 KiB allocation blocks cover
 	// 1024x512 pixels, independently of the attachment's fragment count.
-	const uint64_t size = static_cast<uint64_t>(AlignUp(width, 1024u) / 1024u) *
-	                      (AlignUp(height, 512u) / 512u) * 32768u;
+	const uint64_t size = static_cast<uint64_t>(Common::AlignUp(width, 1024u) / 1024u) *
+	                      (Common::AlignUp(height, 512u) / 512u) * 32768u;
 	if (size == 0 || size > UINT32_MAX) {
 		return false;
 	}
@@ -1154,13 +1151,13 @@ bool TileGetDepthSize(uint32_t width, uint32_t height, uint32_t pitch,
 		                                           &stencil_block_height));
 		const uint32_t fragments = 1u << num_fragments_log2;
 		const uint64_t depth_bytes_total =
-		    valid_blocks ? static_cast<uint64_t>(AlignUp(width, depth_block_width)) *
-		                       AlignUp(height, depth_block_height) * depth_bytes * fragments
+		    valid_blocks ? static_cast<uint64_t>(Common::AlignUp(width, depth_block_width)) *
+		                       Common::AlignUp(height, depth_block_height) * depth_bytes * fragments
 		                 : 0;
 		const uint64_t stencil_bytes_total =
 		    stencil_format == Prospero::StencilFormat::k8UInt && valid_blocks
-		        ? static_cast<uint64_t>(AlignUp(width, stencil_block_width)) *
-		              AlignUp(height, stencil_block_height) * fragments
+		        ? static_cast<uint64_t>(Common::AlignUp(width, stencil_block_width)) *
+		              Common::AlignUp(height, stencil_block_height) * fragments
 		        : 0;
 		TileSizeAlign calculated_htile {};
 		const bool    htile_valid = !htile || TileGetHtileSize(width, height, calculated_htile);
@@ -1187,8 +1184,7 @@ uint32_t TileGetRenderTargetPitch(uint32_t width, uint32_t bytes_per_element,
 	                                                         &block_width, &block_height)) {
 		return 0;
 	}
-	const uint64_t pitch = (static_cast<uint64_t>(width) + block_width - 1u) &
-	                       ~static_cast<uint64_t>(block_width - 1u);
+	const uint64_t pitch = Common::AlignUp(static_cast<uint64_t>(width), block_width);
 	return pitch <= UINT32_MAX ? static_cast<uint32_t>(pitch) : 0;
 }
 
@@ -1209,8 +1205,7 @@ bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
 	    pitch != TileGetRenderTargetPitch(width, bytes_per_element, num_fragments_log2)) {
 		return false;
 	}
-	const uint64_t padded_height = (static_cast<uint64_t>(height) + block_height - 1u) &
-	                               ~static_cast<uint64_t>(block_height - 1u);
+	const uint64_t padded_height = Common::AlignUp(static_cast<uint64_t>(height), block_height);
 	const uint64_t size = static_cast<uint64_t>(pitch) * padded_height * bytes_per_element *
 	                      (1u << num_fragments_log2);
 	if (size == 0 || size > UINT32_MAX) {
@@ -1372,7 +1367,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 			TileTextureElementLayout element {};
 			if (TileGetTextureElementLayout(format, element) && element.texel_width == 1 &&
 			    element.texel_height == 1) {
-				pitch = AlignUp(pitch, CalcLinearBlockWidth(element.bytes));
+				pitch = Common::AlignUp(pitch, CalcLinearBlockWidth(element.bytes));
 			}
 			break;
 		}
@@ -1383,7 +1378,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 		case Prospero::TileMode::kRenderTarget: {
 			TileTextureBlockLayout layout {};
 			if (TileGetTextureBlockLayout(format, tile, false, layout)) {
-				pitch = AlignUp(pitch, layout.block.block_width * layout.texel_width);
+				pitch = Common::AlignUp(pitch, layout.block.block_width * layout.texel_width);
 			}
 			break;
 		}

src/graphics/host_gpu/graphicContext.h

@@ -27,7 +27,6 @@ struct GraphicContext {
 	vk::Device                         device                                = nullptr;
 	VmaAllocator                       allocator                             = nullptr;
 	bool                               memory_budget_ext_enabled             = false;
-	bool                               rt_extensions_enabled                 = false;
 	bool                               compute_subgroup_size_control_enabled = false;
 	bool                               sample_rate_shading_enabled           = false;
 	bool                               attachment_feedback_loop_enabled      = false;
@@ -99,10 +98,6 @@ struct GraphicContext {
 	[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
 	[[nodiscard]] bool     CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
 	void                   DeleteImage(VulkanImage& image);
-	void                   AppendHardwareRayTracingDeviceExtensions(
-	    const std::vector<vk::ExtensionProperties>& available_extensions,
-	    std::vector<const char*>&                   device_extensions);
-	void LoadHardwareRayTracingFunctions() const;
 
 	uint32_t screen_width  = 0;
 	uint32_t screen_height = 0;

src/graphics/host_gpu/memoryTracker.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/memoryTracker.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 
 namespace Libs::Graphics {
@@ -8,9 +9,6 @@ static_assert(std::atomic<void*>::is_always_lock_free);
 
 MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) {
 	m_regions = std::make_unique<std::atomic<RegionManager*>[]>(REGION_COUNT);
-	for (size_t i = 0; i < REGION_COUNT; i++) {
-		m_regions[i].store(nullptr, std::memory_order_relaxed);
-	}
 }
 
 MemoryTracker::~MemoryTracker() = default;
@@ -34,8 +32,8 @@ void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr,
 void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
                                               const char* operation) {
 	ValidateRange(vaddr, size);
-	const auto begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-	const auto end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
+	const auto end   = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
 	for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
 		const bool has_dirty_bytes = dirty.Intersects(page, TRACKER_PAGE_SIZE);
 		if (IsRegionGpuModified(page, TRACKER_PAGE_SIZE) != has_dirty_bytes) {

src/graphics/host_gpu/memoryTracker.h

@@ -62,43 +62,18 @@ public:
 	void ValidateGpuDirtyOwnership(const RangeSet&, uint64_t, uint64_t, const char*) {}
 #endif
 
-	template <bool clear, typename Preflight, typename Func>
-	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Preflight&& preflight, Func&& func) {
-		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
-		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
-		CheckNotInUploadCallback();
-		std::vector<RegionManager*> managers;
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
-			managers.push_back(manager);
-		});
-		std::vector<std::unique_lock<TrackingSpinLock>> locks;
-		locks.reserve(managers.size());
-		for (auto* manager: managers) {
-			locks.emplace_back(manager->lock);
-		}
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			const auto address = manager->GetCpuAddr() + offset;
-			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes,
-			                                                                preflight);
-		});
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(
-			    manager->GetCpuAddr() + offset, bytes, func);
-		});
-		if constexpr (clear) {
-			Iterate<false>(vaddr, size,
-			               [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
-				               const auto address = manager->GetCpuAddr() + offset;
-				               manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
-				                   address, bytes, [](uint64_t, uint64_t) noexcept {});
-			               });
-		}
-	}
-
 	template <bool clear, typename Func>
 	void ForEachDownloadRange(uint64_t vaddr, uint64_t size, Func&& func) {
-		ForEachDownloadRange<clear>(
-		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
+		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
+		CheckNotInUploadCallback();
+		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+			std::scoped_lock lock(manager->lock);
+			const auto       address = manager->GetCpuAddr() + offset;
+			manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes, func);
+			if constexpr (clear) {
+				manager->template ChangeState<DirtySource::Gpu, false>(address, bytes);
+			}
+		});
 	}
 
 	template <typename RangeFunc, typename UploadFunc>

src/graphics/host_gpu/pageManager.cpp

@@ -1,5 +1,7 @@
 #include "graphics/host_gpu/pageManager.h"
 
+#include "common/alignment.h"
+#include "common/virtualMemory.h"
 #include "graphics/host_gpu/regionDefinitions.h"
 #include "kernel/memory.h"
 
@@ -32,19 +34,8 @@ constexpr uint64_t REGION_SIZE  = TRACKER_REGION_SIZE;
 constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
 constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
 
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-// The tracker reuses Win32 memory-protection tags as internal page-state values.
-// Mirror their canonical numeric values so the shared state-machine logic is identical.
-constexpr uint32_t PAGE_NOACCESS  = 0x01;
-constexpr uint32_t PAGE_READONLY  = 0x02;
-constexpr uint32_t PAGE_READWRITE = 0x04;
-#endif
 constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 
-constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
-constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
-constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-
 [[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
 	std::fputs("PageManager fail-fast: ", stderr);
 	std::fputs(reason != nullptr ? reason : "invalid page state", stderr);
@@ -67,15 +58,6 @@ constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
 	std::_Exit(322);
 }
 
-Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
-	switch (protection) {
-		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
-		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
-		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
-		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
-	}
-}
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -96,15 +78,6 @@ void ValidateRange(uint64_t vaddr, uint64_t size) {
 	}
 }
 
-uint64_t PageStart(uint64_t vaddr) {
-	return vaddr & ~(PAGE_SIZE - 1);
-}
-
-uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
-	ValidateRange(vaddr, size);
-	return PageStart(vaddr + size - 1) + PAGE_SIZE;
-}
-
 } // namespace
 
 struct PageManager::Impl {
@@ -112,14 +85,14 @@ struct PageManager::Impl {
 		uint8_t write_watchers  : 7 = 0;
 		uint8_t access_watchers : 1 = 0;
 
-		[[nodiscard]] uint32_t Perms() const noexcept {
+		[[nodiscard]] Common::VirtualMemory::Mode Perms() const noexcept {
 			if (access_watchers != 0) {
-				return NO_ACCESS_PROTECTION;
+				return Common::VirtualMemory::Mode::NoAccess;
 			}
 			if (write_watchers != 0) {
-				return READ_ONLY_PROTECTION;
+				return Common::VirtualMemory::Mode::Read;
 			}
-			return READ_WRITE_PROTECTION;
+			return Common::VirtualMemory::Mode::ReadWrite;
 		}
 
 		template <int delta, bool is_read>
@@ -183,9 +156,6 @@ struct PageManager::Impl {
 		}
 #endif
 		regions = std::make_unique<std::atomic<Region*>[]>(REGION_COUNT);
-		for (uint64_t i = 0; i < REGION_COUNT; i++) {
-			regions[i].store(nullptr, std::memory_order_relaxed);
-		}
 	}
 
 	~Impl() {
@@ -220,11 +190,10 @@ struct PageManager::Impl {
 		return ptr;
 	}
 
-	void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept {
-		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
-		                                                     ToMemoryMode(protection))) {
-			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
-			      protection);
+	void Protect(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) noexcept {
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, mode)) {
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", mode=0x%08" PRIx32, vaddr,
+			      static_cast<uint32_t>(mode));
 		}
 	}
 
@@ -281,11 +250,12 @@ struct PageManager::Impl {
 
 	template <bool track, bool is_read>
 	void UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
-		const auto begin = PageStart(vaddr);
-		const auto end   = PageEnd(vaddr, size);
+		ValidateRange(vaddr, size);
+		const auto begin = Common::AlignDown(vaddr, PAGE_SIZE);
+		const auto end   = Common::AlignUp(vaddr + size, PAGE_SIZE);
 		for (auto chunk_begin = begin; chunk_begin < end;) {
-			const auto chunk_end   = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
-			const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE;
+			const auto chunk_end = std::min(end, Common::AlignUp(chunk_begin + 1, REGION_SIZE));
+			const auto region_base = Common::AlignDown(chunk_begin, REGION_SIZE);
 			auto*      region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin);
 			if (region == nullptr) {
 				Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);

src/graphics/host_gpu/rangeSet.h

@@ -6,17 +6,11 @@
 #include <algorithm>
 #include <cstdint>
 #include <map>
-#include <vector>
 
 namespace Libs::Graphics {
 
 class RangeSet final {
 public:
-	struct Range {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-
 	void Add(uint64_t address, uint64_t size) {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.lower_bound(address);
@@ -62,12 +56,6 @@ public:
 		}
 	}
 
-	[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
-		std::vector<Range> result;
-		ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
-		return result;
-	}
-
 	[[nodiscard]] bool Intersects(uint64_t address, uint64_t size) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.lower_bound(address);
@@ -88,7 +76,7 @@ public:
 	}
 
 	template <typename Func>
-	void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
+	void ForEachInRange(uint64_t address, uint64_t size, Func&& func) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.upper_bound(address);
 		if (it != m_ranges.begin()) {
@@ -98,7 +86,7 @@ public:
 			const auto begin = std::max(address, it->first);
 			const auto last  = std::min(end, it->second);
 			if (begin < last) {
-				func(Range {begin, last - begin});
+				func(begin, last);
 			}
 		}
 	}

src/graphics/host_gpu/regionManager.h

@@ -111,56 +111,43 @@ public:
 			bits.UnsetRange(start, end);
 		}
 		if constexpr (source == DirtySource::Cpu) {
-			UpdateCpuProtection<!enable>();
+			UpdateProtection<!enable, false>();
 		} else {
-			UpdateGpuProtection<enable>();
+			UpdateProtection<enable, true>();
 		}
 	}
 
 	template <DirtySource source, bool clear, typename Func>
 	void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
 		const auto [start, end] = GetPageRange(vaddr, size);
-		RegionBits mask(GetBits<source>(), start, end);
+		auto&      bits         = GetBits<source>();
+		RegionBits mask(bits, start, end);
 		if constexpr (clear) {
-			GetBits<source>().UnsetRange(start, end);
+			bits.UnsetRange(start, end);
+			if constexpr (source == DirtySource::Cpu) {
+				UpdateProtection<true, false>();
+			} else {
+				UpdateProtection<false, true>();
+			}
 		}
-		if constexpr (source == DirtySource::Cpu && clear) {
-			UpdateCpuProtection<true>();
-			ForEachRange(mask, std::forward<Func>(func));
-			return;
+		for (const auto [first, last]: mask) {
+			func(m_cpu_addr + first * TRACKER_PAGE_SIZE, (last - first) * TRACKER_PAGE_SIZE);
 		}
-		if constexpr (source == DirtySource::Gpu && clear) {
-			UpdateGpuProtection<false>();
-		}
-		ForEachRange(mask, std::forward<Func>(func));
 	}
 
 	TrackingSpinLock lock;
 
 private:
-	template <bool track>
-	void UpdateCpuProtection() {
-		auto mask  = m_cpu_dirty ^ m_writable;
-		m_writable = m_cpu_dirty;
+	template <bool track, bool is_read>
+	void UpdateProtection() {
+		const auto protection = is_read ? ~m_gpu_dirty : m_cpu_dirty;
+		auto&      previous   = is_read ? m_readable : m_writable;
+		auto       mask       = protection ^ previous;
 		if (mask.None()) {
 			return;
 		}
-		m_page_manager.UpdatePageWatchersForRegion<track>(m_cpu_addr, mask);
-	}
-
-	template <bool track>
-	void UpdateGpuProtection() {
-		auto readable = ~m_gpu_dirty;
-		auto mask     = readable ^ m_readable;
-		m_readable    = readable;
-		if (mask.None()) {
-			return;
-		}
-		if constexpr (track) {
-			m_page_manager.UpdatePageWatchersForRegion<true, true>(m_cpu_addr, mask);
-		} else {
-			m_page_manager.UpdatePageWatchersForRegion<false, true>(m_cpu_addr, mask);
-		}
+		previous = protection;
+		m_page_manager.UpdatePageWatchersForRegion<track, is_read>(m_cpu_addr, mask);
 	}
 
 	template <DirtySource source>
@@ -191,13 +178,6 @@ private:
 		        static_cast<size_t>((offset + size + TRACKER_PAGE_SIZE - 1) / TRACKER_PAGE_SIZE)};
 	}
 
-	template <typename Func>
-	void ForEachRange(const RegionBits& bits, Func&& func) const {
-		for (const auto [start, end]: bits) {
-			func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE);
-		}
-	}
-
 	PageManager& m_page_manager;
 	uint64_t     m_cpu_addr = 0;
 	RegionBits   m_cpu_dirty;

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -42,13 +43,6 @@ void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void*
 	}
 }
 
-struct BufferCache::DownloadCopy {
-	Buffer*  buffer        = nullptr;
-	uint64_t source_offset = 0;
-	uint64_t address       = 0;
-	uint64_t size          = 0;
-};
-
 void BufferCache::Register(BufferId id) {
 	ChangeRegister<true>(id);
 }
@@ -103,8 +97,7 @@ void BufferCache::TouchBuffer(const Buffer& buffer) {
 }
 
 void BufferCache::DeleteBuffer(BufferId id) {
-	auto* buffer = m_slot_buffers.try_get(id);
-	if (buffer == nullptr || buffer->is_deleted) {
+	if (IsBufferInvalid(id)) {
 		return;
 	}
 	Unregister(id);
@@ -115,79 +108,70 @@ void BufferCache::DeleteBuffer(BufferId id) {
 	}
 }
 
-std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
-	if (copy.buffer == nullptr || copy.size == 0 || copy.source_offset > copy.buffer->Size() ||
-	    copy.size > copy.buffer->Size() - copy.source_offset) {
-		EXIT("BufferCache: invalid download copy\n");
+bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size) {
+	std::vector<vk::BufferCopy> copies;
+	uint64_t                    total_size     = 0;
+	const auto                  buffer_address = buffer.CpuAddress();
+	m_memory_tracker.ForEachDownloadRange<false>(
+	    vaddr, size, [&](uint64_t address, uint64_t bytes) noexcept {
+		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
+		                                           "buffer download");
+		    m_gpu_modified_ranges.ForEachInRange(address, bytes, [&](uint64_t start, uint64_t end) {
+			    copies.emplace_back(start - buffer_address, total_size, end - start);
+			    // Keep packed ranges on separate cache lines, as in shadPS4.
+			    total_size += Common::AlignUp(end - start, 64);
+		    });
+		    m_gpu_modified_ranges.Subtract(address, bytes);
+	    });
+	if (copies.empty()) {
+		return false;
 	}
-	const auto begin = copy.source_offset & ~uint64_t {3};
-	if (copy.source_offset > UINT64_MAX - copy.size ||
-	    copy.source_offset + copy.size > UINT64_MAX - 3) {
-		EXIT("BufferCache: download copy alignment overflow\n");
-	}
-	const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3};
-	if (end > copy.buffer->Size()) {
-		EXIT("BufferCache: aligned download copy exceeds its owner\n");
-	}
-	return {begin, end - begin};
-}
 
-void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
-	std::vector<DownloadCopy> batch;
-	batch.reserve(copies.size());
-	uint64_t                  packed_size = 0;
-	auto&                     download    = m_download_buffer;
-	const auto flush = [&] {
-		const auto [mapped, base_offset] = download.Map(packed_size, DOWNLOAD_ALIGNMENT);
-		EXIT_IF(mapped == nullptr);
-		uint64_t cursor = 0;
-		for (const auto& copy: batch) {
-			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-			download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
-			                  envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
-			                  vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
-			                  vk::AccessFlagBits::eHostRead);
-			cursor += AlignDownload(envelope_size);
-		}
-		download.Commit();
-		const auto completion_tick = m_scheduler.CurrentTick();
-		m_scheduler.Finish();
-		m_scheduler.WaitPriorityOperations(completion_tick);
-		cursor = 0;
-		for (const auto& copy: batch) {
-			const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
-			const auto offset = cursor + copy.source_offset - source_begin;
-			download.Invalidate(base_offset + offset, copy.size);
-			Libs::LibKernel::Memory::WriteBacking(copy.address, mapped + offset, copy.size);
-			cursor += AlignDownload(envelope_size);
-		}
-		batch.clear();
-		packed_size = 0;
-	};
-	for (auto copy: copies) {
-		while (copy.size != 0) {
-			const auto available = download.Size() - packed_size;
-			const auto prefix    = copy.source_offset & 3u;
-			const auto bytes     = std::min(copy.size, available - prefix);
-			DownloadCopy part {copy.buffer, copy.source_offset, copy.address, bytes};
-			const auto [source_begin, envelope_size] = DownloadEnvelope(part);
-			(void)source_begin;
-			packed_size += AlignDownload(envelope_size);
-			batch.push_back(part);
-			copy.source_offset += bytes;
-			copy.address += bytes;
-			copy.size -= bytes;
-			if (packed_size == download.Size()) {
-				flush();
-			}
-		}
+	const auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
+	if (mapped == nullptr) {
+		EXIT("BufferCache: download exceeds 32 MiB staging buffer capacity\n");
 	}
-	if (!batch.empty()) {
-		flush();
-	}
-	for (const auto& copy: copies) {
-		m_gpu_modified_ranges.Subtract(copy.address, copy.size);
+	m_download_buffer.Commit();
+	for (auto& copy: copies) {
+		copy.dstOffset += offset;
 	}
+
+	auto& command = m_scheduler.Current();
+	command.EndRendering();
+	const auto              native = command.Handle();
+	vk::BufferMemoryBarrier before {};
+	before.srcAccessMask       = vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite;
+	before.dstAccessMask       = vk::AccessFlagBits::eTransferRead;
+	before.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+	before.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+	before.buffer              = buffer.Handle();
+	before.offset              = 0;
+	before.size                = buffer.Size();
+	native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
+	                       vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &before, 0,
+	                       nullptr);
+	native.copyBuffer(buffer.Handle(), m_download_buffer.Handle(),
+	                  static_cast<uint32_t>(copies.size()), copies.data());
+
+	auto after          = before;
+	after.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+	after.dstAccessMask = vk::AccessFlagBits::eHostRead;
+	after.buffer        = m_download_buffer.Handle();
+	after.offset        = offset;
+	after.size          = total_size;
+	native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+	                       vk::PipelineStageFlagBits::eAllCommands |
+	                           vk::PipelineStageFlagBits::eHost,
+	                       {}, 0, nullptr, 1, &after, 0, nullptr);
+	m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address,
+	                                    copies = std::move(copies)] {
+		m_download_buffer.Invalidate(offset, total_size);
+		for (const auto& copy: copies) {
+			Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset,
+			                                      mapped + (copy.dstOffset - offset), copy.size);
+		}
+	});
+	return true;
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -252,44 +236,29 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 		     vaddr, size);
 	}
-	m_scheduler.Context().GetGpu().SendCommandSync(
-	    [this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); });
-}
+	m_scheduler.Context().GetGpu().SendCommandSync([this, vaddr, size, is_write] {
+		if (is_write && !IsRegionRegistered(vaddr, size)) {
+			return;
+		}
+		auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
 
-void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) {
-	if (is_write && !IsRegionRegistered(vaddr, size)) {
-		return;
-	}
-	auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
+		// Widen nearby CPU reads so they share one GPU drain.
+		constexpr uint64_t WindowSize   = 512 * 1024;
+		const auto         buffer_begin = buffer.CpuAddress();
+		const auto         buffer_end   = buffer_begin + buffer.Size();
+		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
+		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
 
-	// Widen nearby CPU reads so they share one GPU drain.
-	constexpr uint64_t WindowSize   = 512 * 1024;
-	const auto         buffer_begin = buffer.CpuAddress();
-	const auto         buffer_end   = buffer_begin + buffer.Size();
-	const auto         window_begin = std::max(vaddr & ~(WindowSize - 1), buffer_begin);
-	const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
-
-	std::vector<DownloadCopy> copies;
-	m_memory_tracker.ForEachDownloadRange<false>(
-	    window_begin, window_end - window_begin,
-	    [&](uint64_t address, uint64_t bytes) noexcept {
-		    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
-		                                           "memory invalidation");
-	    },
-	    [&](uint64_t address, uint64_t bytes) noexcept {
-		    m_gpu_modified_ranges.ForEachIntersection(address, bytes, [&](RangeSet::Range range) {
-			    copies.push_back(
-			        {&buffer, buffer.Offset(range.address), range.address, range.size});
-		    });
-	    });
-	if (!copies.empty()) {
-		DownloadBufferMemory(copies);
-		// The enumeration covered whole dirty pages and every exact interval on them.
-		m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
-	}
-	if (is_write) {
-		m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
-	}
+		if (DownloadBufferMemory(buffer, window_begin, window_end - window_begin)) {
+			const auto tick = m_scheduler.CurrentTick();
+			m_scheduler.Wait(tick);
+			m_scheduler.WaitPriorityOperations(tick);
+			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
+		}
+		if (is_write) {
+			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
+		}
+	});
 }
 
 BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
@@ -340,7 +309,7 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t
 		end                       = std::max(end, buffer_end);
 		if (!has_stream_leap && (stream_score += buffer.StreamScore()) > StreamLeapThreshold) {
 			has_stream_leap = true;
-			// Fix the shadPS4 bug that reserves space opposite to the incoming stream's growth.
+			// Reserve space in the incoming stream's direction of growth.
 			// The old buffer extending left of the request predicts growth to the right, and vice versa.
 			if (expands_left) {
 				end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
@@ -371,8 +340,8 @@ void BufferCache::JoinOverlap(BufferId new_id, BufferId overlap_id, bool accumul
 
 BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
 	EXIT_IF(m_scheduler.Current().IsInvalid());
-	const auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
-	vaddr &= ~(CACHING_PAGESIZE - 1);
+	const auto end = Common::AlignUp(vaddr + size, CACHING_PAGESIZE);
+	vaddr = Common::AlignDown(vaddr, CACHING_PAGESIZE);
 	size               = end - vaddr;
 	const auto overlap = ResolveOverlaps(vaddr, size);
 
@@ -485,17 +454,16 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
 		}
 	}
 
-	auto* buffer = m_slot_buffers.try_get(id);
-	if (buffer == nullptr || buffer->is_deleted || !buffer->IsInBounds(vaddr, size)) {
-		id     = FindBuffer(vaddr, size);
-		buffer = &m_slot_buffers[id];
+	if (IsBufferInvalid(id) || !m_slot_buffers[id].IsInBounds(vaddr, size)) {
+		id = FindBuffer(vaddr, size);
 	}
-	TouchBuffer(*buffer);
-	(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
+	auto& buffer = m_slot_buffers[id];
+	TouchBuffer(buffer);
+	(void)SynchronizeBuffer(buffer, vaddr, size, is_written, is_texel_buffer);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
-	return {buffer, buffer->Offset(vaddr)};
+	return {&buffer, buffer.Offset(vaddr)};
 }
 
 std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
@@ -547,9 +515,7 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	}
 
 	m_texture_cache.InvalidateMemoryFromGPU(vaddr, size);
-	const auto id          = FindBuffer(vaddr, size);
-	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true, id);
-	EXIT_IF(dst == nullptr);
+	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true);
 	dst->Fill(dst_offset, size, value);
 }
 
@@ -583,10 +549,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	                                    : std::pair {&m_gds_buffer, src_vaddr};
 	auto [dst, dst_offset] = dst_memory ? ObtainBuffer(dst_vaddr, size, true, true, dst_id)
 	                                    : std::pair {&m_gds_buffer, dst_vaddr};
-	EXIT_IF(src == nullptr || dst == nullptr);
-	if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) {
-		EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
-	}
 	dst->CopyFrom(command, *src, src_offset, dst_offset, size);
 }
 
@@ -630,8 +592,7 @@ void BufferCache::RunGarbageCollector() {
 	const size_t   limit      = aggressive ? 64 : 32;
 
 	std::vector<BufferId> dirty_buffers;
-	std::vector<DownloadCopy> copies;
-	size_t                    retire_count = 0;
+	size_t                retire_count = 0;
 	m_lru_cache.ForEachItemBelow(tick - age, [&](BufferId id) {
 		auto& buffer = m_slot_buffers[id];
 		EXIT_IF(buffer.is_deleted);
@@ -642,19 +603,7 @@ void BufferCache::RunGarbageCollector() {
 			return false;
 		}
 		if (dirty) {
-			m_memory_tracker.ForEachDownloadRange<false>(
-			    buffer.CpuAddress(), buffer.Size(),
-			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address,
-				                                           dirty_size, "garbage collection");
-			    },
-			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
-				    m_gpu_modified_ranges.ForEachIntersection(
-				        dirty_address, dirty_size, [&](RangeSet::Range range) {
-					    copies.push_back({&buffer, range.address - buffer.CpuAddress(),
-					                      range.address, range.size});
-				        });
-				});
+			EXIT_IF(!DownloadBufferMemory(buffer, buffer.CpuAddress(), buffer.Size()));
 			dirty_buffers.push_back(id);
 		} else {
 			m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());
@@ -666,8 +615,10 @@ void BufferCache::RunGarbageCollector() {
 		return;
 	}
 
-	EXIT_IF(copies.empty());
-	DownloadBufferMemory(copies);
+	// Publish all queued downloads before releasing their tracked pages and owners.
+	const auto completion_tick = m_scheduler.CurrentTick();
+	m_scheduler.Wait(completion_tick);
+	m_scheduler.WaitPriorityOperations(completion_tick);
 	for (const auto id: dirty_buffers) {
 		auto& buffer = m_slot_buffers[id];
 		m_memory_tracker.UnmarkRegionAsGpuModified(buffer.CpuAddress(), buffer.Size());

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -74,6 +74,11 @@ public:
 private:
 	friend struct BufferCacheTestAccess;
 
+	bool IsBufferInvalid(BufferId id) const {
+		const auto* buffer = m_slot_buffers.try_get(id);
+		return buffer == nullptr || buffer->is_deleted;
+	}
+
 	using BufferMap = std::map<uint64_t, BufferId>;
 	struct OverlapResult {
 		BufferMap::iterator first;
@@ -83,14 +88,8 @@ private:
 		bool                has_stream_leap;
 	};
 
-	struct DownloadCopy;
 	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
 	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
-	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
-		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
-	}
-	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
 	void TouchBuffer(const Buffer& buffer);
 	[[nodiscard]] OverlapResult ResolveOverlaps(uint64_t vaddr, uint64_t size);
@@ -106,8 +105,8 @@ private:
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
-	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
+	// Queues backing publication; callers wait before clearing dirty pages or reusing their data.
+	[[nodiscard]] bool DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/cache/faultManager.cpp

@@ -44,12 +44,7 @@ FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler
 	                                                &m_fault_process_desc_layout),
 	    "create fault-buffer descriptor layout");
 
-	vk::ShaderModuleCreateInfo module_info {};
-	module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
-	module_info.pCode    = FAULT_BUFFER_PROCESS_SPV;
-	vk::ShaderModule module = nullptr;
-	RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-	                     "create fault-buffer shader module");
+	const auto module = CompileSPV(FAULT_BUFFER_PROCESS_SPV, m_graphics.device);
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.setLayoutCount = 1;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -1,96 +0,0 @@
-#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
-
-#include "common/assert.h"
-#include "graphics/guest_gpu/graphicsRun.h"
-#include "graphics/host_gpu/renderer/commandScheduler.h"
-namespace Libs::Graphics {
-
-GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
-    : m_scheduler(scheduler), m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache),
-      m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache) {}
-
-GpuResourceManager::~GpuResourceManager() = default;
-
-bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	// The host reports the faulting byte, not the instruction's access width. Both caches
-	// resolve its page; guessing a width can cross the end of a valid guest mapping.
-	constexpr uint64_t fault_size = 1;
-	if (!IsMapped(fault_vaddr, fault_size)) {
-		return false;
-	}
-	if (access == PageFaultAccess::Write) {
-		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
-		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
-	} else {
-		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
-	}
-	return true;
-}
-
-bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
-	if (!IsMapped(vaddr, size)) {
-		return false;
-	}
-	m_buffer_cache.InvalidateMemory(vaddr, size);
-	m_texture_cache.InvalidateMemory(vaddr, size);
-	return true;
-}
-
-bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (!GuestRange {vaddr, size}.Valid()) {
-		return false;
-	}
-	std::shared_lock lock(m_mapped_ranges_mutex);
-	return m_mapped_ranges.Contains(vaddr, size);
-}
-
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
-	{
-		std::lock_guard lock(m_mapped_ranges_mutex);
-		m_mapped_ranges.Add(vaddr, size);
-	}
-}
-
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	if (CommandScheduler::InDeferredOperation()) {
-		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
-		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-		     vaddr, size);
-	}
-	const auto unmap = [this, vaddr, size] {
-		if (m_scheduler.Active()) {
-			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.Finish();
-			m_scheduler.WaitPriorityOperations(tick);
-		}
-		m_buffer_cache.InvalidateMemory(vaddr, size);
-		m_texture_cache.UnmapMemory(vaddr, size);
-		std::lock_guard lock(m_mapped_ranges_mutex);
-		m_mapped_ranges.Subtract(vaddr, size);
-	};
-	if (m_gpu == nullptr) {
-		unmap();
-		return;
-	}
-	m_gpu->SendCommandSync(unmap);
-}
-
-void GpuResourceManager::PrepareBda() {
-	std::shared_lock lock(m_mapped_ranges_mutex);
-	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
-		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
-	});
-	m_fault_process_pending = true;
-}
-
-void GpuResourceManager::RunGarbageCollector() {
-	if (m_fault_process_pending) {
-		m_fault_process_pending = false;
-		m_buffer_cache.ProcessFaultBuffer();
-	}
-	m_texture_cache.ProcessDownloadImages();
-	m_texture_cache.RunGarbageCollector();
-	m_buffer_cache.RunGarbageCollector();
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -1,49 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
-
-#include "common/abi.h"
-#include "common/common.h"
-#include "graphics/host_gpu/pageManager.h"
-#include "graphics/host_gpu/renderer/cache/bufferCache.h"
-#include "graphics/host_gpu/renderer/cache/textureCache.h"
-
-#include <cstdint>
-#include <shared_mutex>
-
-namespace Libs::Graphics {
-
-class CommandScheduler;
-class GuestGpu;
-
-class GpuResourceManager {
-public:
-	GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler);
-	~GpuResourceManager();
-	KYTY_CLASS_NO_COPY(GpuResourceManager);
-
-	[[nodiscard]] BufferCache&  GetBufferCache() { return m_buffer_cache; }
-	[[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; }
-	void                        SetGpu(GuestGpu* gpu) noexcept { m_gpu = gpu; }
-
-	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
-	void               PrepareBda();
-	void               RunGarbageCollector();
-
-private:
-	PageManager               m_page_manager;
-	CommandScheduler&         m_scheduler;
-	BufferCache               m_buffer_cache;
-	TextureCache              m_texture_cache;
-	mutable std::shared_mutex m_mapped_ranges_mutex;
-	RangeSet                  m_mapped_ranges;
-	GuestGpu*                 m_gpu = nullptr;
-	bool                      m_fault_process_pending = false;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -1,12 +1,12 @@
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/profiler.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 
 #include <cstring>
-#include <limits>
 #include <numeric>
 #include <vk_mem_alloc.h>
 
@@ -45,16 +45,11 @@ constexpr size_t WATCHES_RESERVE_CHUNK   = 0x1000;
 		result = value;
 		return true;
 	}
-	const auto remainder = value % alignment;
-	if (remainder == 0) {
-		result = value;
-		return true;
-	}
-	const auto increment = alignment - remainder;
-	if (value > std::numeric_limits<uint64_t>::max() - increment) {
+	const auto aligned = Common::AlignUp(value, alignment);
+	if (aligned < value) {
 		return false;
 	}
-	result = value + increment;
+	result = aligned;
 	return true;
 }
 
@@ -124,12 +119,6 @@ bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
 	return address >= m_cpu_address && size <= Size() && address - m_cpu_address <= Size() - size;
 }
 
-void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
-	EXIT_IF(source == nullptr || m_mapped.empty() || offset > Size() || size > Size() - offset);
-	std::memcpy(m_mapped.data() + offset, source, static_cast<size_t>(size));
-	Flush(offset, size);
-}
-
 void Buffer::Flush(uint64_t offset, uint64_t size) {
 	EXIT_IF(m_mapped.empty() || offset > Size() || size > Size() - offset);
 	if (!IsCoherent() && size != 0) {

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -55,7 +55,6 @@ public:
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
 	void               IncreaseStreamScore(int score) noexcept { stream_score += score; }
 	[[nodiscard]] int  StreamScore() const noexcept { return stream_score; }
-	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
 	void               Invalidate(uint64_t offset, uint64_t size);
 	void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
@@ -32,22 +33,24 @@ namespace {
 
 constexpr uint64_t NumFramesBeforeRemoval = 32;
 
-[[nodiscard]] bool DecodeDccClear(const TextureCache::ImageDesc& desc, vk::Format format,
-                                  uint32_t fill, vk::ClearColorValue& clear) {
-	const auto code = static_cast<uint8_t>(fill);
-	if (fill != static_cast<uint32_t>(code) * 0x01010101u) {
-		return false;
+[[nodiscard]] bool DecodeDccClear(const TextureCache::ImageDesc& desc, uint8_t code,
+                                  vk::ClearColorValue& clear) {
+	switch (code) {
+		case 0x00:
+		case 0x20:
+		case 0x40:
+		case 0x80:
+		case 0xc0: break;
+		default: return false;
 	}
 	const auto& metadata = desc.info.metadata;
+	const auto  format   = desc.view_info.format;
 	if (code == 0x20) {
 		// Clear-to-register is a color-buffer operation; the texture pipe cannot decode it.
 		return desc.type == TextureCache::BindingType::RenderTarget &&
 		       metadata.dcc_clear_register_valid &&
 		       DecodePackedColorClear(format, metadata.dcc_clear_word, clear);
 	}
-	if (code != 0x00 && code != 0x40 && code != 0x80 && code != 0xc0) {
-		return false;
-	}
 	clear = {};
 	if (code == 0x00) {
 		return true;
@@ -266,9 +269,9 @@ void TextureCache::RegisterImage(ImageId id) {
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
 		EXIT("TextureCache: image registration is outside the guest address space\n");
 	}
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(image.info.data.address, image.info.data.size, [this, id](uint64_t page) {
 		m_image_page_table[page].push_back(id);
-	}
+	});
 	image.registered = true;
 	image.lru_id     = m_lru_cache.Insert(id, m_gc_tick);
 	m_total_used_memory += image.AccountedSize();
@@ -284,12 +287,12 @@ void TextureCache::UnregisterImage(ImageId id) {
 	if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
 		EXIT("TextureCache: registered image is outside the guest address space\n");
 	}
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(image.info.data.address, image.info.data.size, [this, id](uint64_t page) {
 		auto* owners = m_image_page_table.Find(page);
 		if (owners == nullptr || !owners->Erase(id)) {
 			EXIT("TextureCache: image missing from page owner index\n");
 		}
-	}
+	});
 	m_lru_cache.Free(image.lru_id);
 	const auto accounted = image.AccountedSize();
 	if (accounted > m_total_used_memory) {
@@ -312,11 +315,7 @@ void TextureCache::DeleteImage(ImageId id) {
 			}
 		});
 		for (const auto association: associations) {
-			auto& associated = m_slot_images[association];
-			if (associated.IsGpuModified()) {
-				associated.ClearGpuModified();
-			}
-			DeleteImage(association);
+			FreeImage(association);
 		}
 	}
 	if (image->IsGpuModified()) {
@@ -326,10 +325,8 @@ void TextureCache::DeleteImage(ImageId id) {
 	if (image->info.HasMetadata()) {
 		const auto metadata = m_surface_metas.find(image->info.metadata.range.address);
 		if (metadata != m_surface_metas.end() &&
-		    ((image->info.metadata.kind == ImageMetadataKind::Dcc &&
-		      metadata->second.type == MetaDataInfo::Type::Dcc) ||
-		     (image->info.metadata.kind == ImageMetadataKind::Htile &&
-		      metadata->second.type == MetaDataInfo::Type::HTile))) {
+		    image->info.metadata.kind == ImageMetadataKind::Htile &&
+		    metadata->second.type == MetaDataInfo::Type::HTile) {
 			// A later binding may have reused this address for another metadata type.
 			m_surface_metas.erase(metadata);
 		}
@@ -443,7 +440,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
 	if (!image.IsTracked() || begin < image.track_addr) {
 		return;
 	}
-	const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE);
 	const auto size    = address - begin;
 	image.track_addr   = address;
 	if (image.track_addr == image.track_addr_end) {
@@ -460,7 +457,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
 	if (!image.IsTracked() || image.track_addr_end < end) {
 		return;
 	}
-	const auto address   = end & ~(TRACKER_PAGE_SIZE - 1);
+	const auto address   = Common::AlignDown(end, TRACKER_PAGE_SIZE);
 	const auto size      = end - address;
 	image.track_addr_end = address;
 	if (image.track_addr == image.track_addr_end) {
@@ -494,10 +491,10 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 	}
 
 	ImageIds result;
-	for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
+	ForEachPage(address, size, [&](uint64_t page) {
 		const auto* owners = m_image_page_table.Find(page);
 		if (owners == nullptr) {
-			continue;
+			return;
 		}
 		owners->ForEach([&](ImageId id) {
 			auto* image = m_slot_images.try_get(id);
@@ -512,7 +509,7 @@ TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64
 				result.push_back(id);
 			}
 		});
-	}
+	});
 	return result;
 }
 
@@ -813,17 +810,10 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 	const bool safe_to_delete =
 	    current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval;
 
-	if (requested.data.address == cached.info.data.address) {
-		const uint32_t requested_block = requested.bytes_per_block * requested.samples;
-		const uint32_t cached_block    = cached.info.bytes_per_block * cached.info.samples;
-		if (requested.BlockExtent() != cached.info.BlockExtent() ||
-		    requested_block != cached_block) {
-			if (safe_to_delete) {
-				FreeImage(cached_id);
-			}
-			return {merged_id};
-		}
-
+	const uint32_t requested_block = requested.bytes_per_block * requested.samples;
+	const uint32_t cached_block    = cached.info.bytes_per_block * cached.info.samples;
+	if (requested.data.address == cached.info.data.address &&
+	    requested.BlockExtent() == cached.info.BlockExtent() && requested_block == cached_block) {
 		if (const auto depth_id = ResolveDepthOverlap(requested, binding, cached_id)) {
 			return {depth_id};
 		}
@@ -876,38 +866,27 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		     static_cast<uint32_t>(cached.info.tile_mode));
 	}
 
-	if (requested.data.address > cached.info.data.address) {
-		const int32_t mip = requested.MipOf(cached.info);
-		if (mip >= 0) {
-			const int32_t layer = requested.SliceOf(cached.info, mip);
-			if (layer >= 0) {
-				return {cached_id, mip, layer};
-			}
-		}
-		if (safe_to_delete) {
-			FreeImage(cached_id);
-		}
-		return {};
+	const int32_t requested_mip = requested.MipOf(cached.info);
+	if (requested_mip >= 0) {
+		const int32_t layer = requested.SliceOf(cached.info, requested_mip);
+		return {cached_id, requested_mip, layer};
 	}
 
 	const int32_t mip = cached.info.MipOf(requested);
 	if (mip >= 0) {
 		const int32_t layer = cached.info.SliceOf(requested, mip);
-		if (layer >= 0) {
-			if (cached.binding.is_target) {
-				cached.binding.needs_rebind = true;
-				if (merged_id) {
-					m_slot_images[merged_id].binding.is_target = true;
-				}
-				FreeImage(cached_id);
-				return {merged_id};
-			}
-			if (merged_id) {
-				CopyImageMip(merged_id, cached_id, static_cast<uint32_t>(mip),
-				             static_cast<uint32_t>(layer));
-				FreeImage(cached_id);
-			}
+		if (!merged_id) {
+			return {ExpandImage(requested, cached_id)};
 		}
+		cached.binding.needs_rebind |= cached.binding.is_bound || cached.binding.is_target;
+		m_slot_images[merged_id].binding.is_target |= cached.binding.is_target;
+		CopyImageMip(merged_id, cached_id, static_cast<uint32_t>(mip),
+		             static_cast<uint32_t>(layer));
+		FreeImage(cached_id);
+		return {merged_id};
+	}
+	if (requested.data.address >= cached.info.data.address && safe_to_delete) {
+		FreeImage(cached_id);
 	}
 	return {merged_id};
 }
@@ -922,12 +901,19 @@ ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) {
 		source.binding.needs_rebind = true;
 	}
 	InitializeImage(expanded_id);
-	CopyImage(expanded_id, source_id);
+	const int32_t mip = source.info.MipOf(info);
+	const int32_t layer = source.info.SliceOf(info, mip);
+	if (layer >= 0) {
+		CopyImageMip(expanded_id, source_id, static_cast<uint32_t>(mip),
+		             static_cast<uint32_t>(layer));
+	} else {
+		CopyImage(expanded_id, source_id);
+	}
 	FreeImage(source_id);
 	return expanded_id;
 }
 
-struct TextureCache::TextureTransferPlan {
+struct TextureCache::TextureTransfer {
 	TextureUploadLayout              layout;
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
@@ -943,13 +929,13 @@ struct TextureCache::TextureTransferPlan {
 	}
 };
 
-struct TextureCache::DownloadPlan {
-	TextureTransferPlan texture;
+struct TextureCache::ImageDownload {
+	TextureTransfer texture;
 	bool                depth_target = false;
 	bool                valid        = false;
 };
 
-TextureCache::TextureTransferPlan
+TextureCache::TextureTransfer
 TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
                                     TransferDirection direction) const {
 	const auto& info             = image.info;
@@ -962,8 +948,8 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 	bool        allow_depth_tile = upload;
 	const char* owner            = "TextureCache readback";
 
-	TextureTransferPlan plan;
-	plan.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
+	TextureTransfer transfer;
+	transfer.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
 	if (render_target) {
 		format = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
 	}
@@ -993,44 +979,44 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 		}
 	}
 
-	plan.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
+	transfer.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
 	                                       info.resources.levels, layers, info.tile_mode,
 	                                       info.data.size, allow_depth_tile, volume, owner);
-	plan.regions = TextureBuildImageCopies(plan.layout);
+	transfer.regions = TextureBuildImageCopies(transfer.layout);
 	if (info.IsDepth()) {
-		for (auto& region: plan.regions) {
+		for (auto& region: transfer.regions) {
 			region.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth;
 		}
 	}
-	if (plan.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
-		if (!TextureBuildGpuTileInfos(info.data.size, plan.regions, plan.layout,
-		                              info.resources.levels, plan.tiles)) {
-			return plan;
+	if (transfer.layout.surface.description.tile_mode != Prospero::TileMode::kLinear) {
+		if (!TextureBuildGpuTileInfos(info.data.size, transfer.regions, transfer.layout,
+		                              info.resources.levels, transfer.tiles)) {
+			return transfer;
 		}
 	}
-	plan.valid = true;
-	return plan;
+	transfer.valid = true;
+	return transfer;
 }
 
-TextureCache::DownloadPlan TextureCache::BuildDownload(const Image& image) const {
+TextureCache::ImageDownload TextureCache::BuildDownload(const Image& image) const {
 	const auto&  info    = image.info;
 	const auto   binding = UploadBinding(image);
-	DownloadPlan plan {.depth_target = binding == BindingType::DepthTarget};
+	ImageDownload transfer {.depth_target = binding == BindingType::DepthTarget};
 	if (info.samples != 1 || image.backing.samples != 1) {
-		return plan;
+		return transfer;
 	}
-	if (plan.depth_target) {
-		plan.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 &&
+	if (transfer.depth_target) {
+		transfer.valid = IsSupportedDepthPlaneReadback(info) && info.resources.layers != 0 &&
 		             info.data.size % info.resources.layers == 0 &&
 		             Prospero::NumBytesPerElement(info.guest_format) == info.bytes_per_block;
-		return plan;
+		return transfer;
 	}
 	if (info.metadata.compression != VideoOutCompression::Uncompressed) {
-		return plan;
+		return transfer;
 	}
-	plan.texture = BuildTextureTransfer(image, binding, TransferDirection::Download);
-	plan.valid   = plan.texture.valid;
-	return plan;
+	transfer.texture = BuildTextureTransfer(image, binding, TransferDirection::Download);
+	transfer.valid   = transfer.texture.valid;
+	return transfer;
 }
 
 void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_offset) {
@@ -1044,26 +1030,26 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	};
 
 	if (binding != BindingType::DepthTarget) {
-		auto plan = BuildTextureTransfer(image, binding, TransferDirection::Upload);
-		if (!plan.valid) {
+		auto transfer = BuildTextureTransfer(image, binding, TransferDirection::Upload);
+		if (!transfer.valid) {
 			EXIT("TextureCache: invalid texture upload: binding=%u addr=0x%016" PRIx64
 			     " size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u "
 			     "pitch=%u levels=%u layers=%u samples=%u\n",
 			     static_cast<uint32_t>(binding), info.data.address, info.data.size,
 			     static_cast<uint32_t>(info.guest_format), static_cast<uint32_t>(info.tile_mode),
-			     static_cast<uint32_t>(plan.layout.surface.texture.block.family), info.extent.width,
+			     static_cast<uint32_t>(transfer.layout.surface.texture.block.family), info.extent.width,
 			     info.extent.height, info.extent.depth, info.pitch, info.resources.levels,
 			     info.resources.layers, info.samples);
 		}
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
-		if (!plan.tiles.empty()) {
+		if (!transfer.tiles.empty()) {
 			linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size,
-			                        plan.LinearSize(), plan.tiles);
+			                        transfer.LinearSize(), transfer.tiles);
 		}
-		if (plan.swap_bgra16) {
+		if (transfer.swap_bgra16) {
 			linear = m_tiler.SwapBgra16(linear);
 		}
-		upload(plan.regions, linear);
+		upload(transfer.regions, linear);
 		return;
 	}
 
@@ -1138,53 +1124,69 @@ void TextureCache::InitializeImage(ImageId id) {
 	}
 }
 
-void TextureCache::PrepareDccClear(ImageId id, const ImageDesc& desc) {
+void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
+                                       uint32_t metadata_base_layer) {
 	if (desc.info.metadata.kind != ImageMetadataKind::Dcc) {
 		return;
 	}
-	auto& image            = m_slot_images[id];
-	image.info.metadata    = desc.info.metadata;
-	auto [entry, inserted] = m_surface_metas.try_emplace(
-	    desc.info.metadata.range.address, MetaDataInfo {.type = MetaDataInfo::Type::Dcc});
-	auto& metadata = entry->second;
-	if (!inserted && metadata.type == MetaDataInfo::Type::PendingDcc) {
-		metadata.type = MetaDataInfo::Type::Dcc;
-	} else if (metadata.type != MetaDataInfo::Type::Dcc) {
-		EXIT("TextureCache: image reuses non-DCC metadata\n");
+	const auto range = desc.info.metadata.range;
+	{
+		std::scoped_lock lock {m_lock};
+		auto& image         = m_slot_images[id];
+		image.info.metadata = desc.info.metadata;
+		// A native DCC allocation must not retain a reused HTile/CMask/FMask interpretation.
+		m_surface_metas.erase(range.address);
+		if (range.size == 0 || desc.info.resources.levels != 1 || image.info.resources.levels != 1) {
+			return;
+		}
 	}
-	if (metadata.clear_mask == 0 || image.info.resources.levels != 1 ||
-	    desc.info.metadata.range.size == 0 || metadata.fill_size < desc.info.metadata.range.size) {
-		return;
-	}
-	vk::ClearValue clear {};
-	if (!DecodeDccClear(desc, image.backing.format, metadata.fill_value, clear.color)) {
-		return;
+	const auto layers = desc.info.TransferLayers();
+	// These one-mip surfaces use complete 4 KiB DCC metadata blocks.
+	constexpr uint64_t MetadataBlockSize = 0x1000;
+	if (!range.Valid() || range.address % MetadataBlockSize != 0 || layers == 0 ||
+	    range.size % layers != 0 || (range.size / layers) % MetadataBlockSize != 0) {
+		EXIT("TextureCache: DCC slices must contain aligned 4 KiB blocks\n");
 	}
 	const auto& view           = desc.view_info;
-	const bool  volume_texture = image.info.IsVolume() && view.type == vk::ImageViewType::e3D;
-	const auto  first          = volume_texture ? 0u : view.base_layer;
-	const auto  count = volume_texture ? std::max(image.info.extent.depth >> view.base_level, 1u)
-	                                   : view.layer_count;
-	if (first >= 32 || count > 32 - first) {
-		return;
+	const bool  volume_texture = desc.info.IsVolume() && view.type == vk::ImageViewType::e3D;
+	const auto  first          = volume_texture ? 0u : metadata_base_layer;
+	const auto  image_first    = volume_texture ? 0u : view.base_layer;
+	const auto  count          = volume_texture ? desc.info.extent.depth : view.layer_count;
+	if (first >= layers || count > layers - first) {
+		EXIT("TextureCache: DCC view exceeds its native metadata slices\n");
 	}
-	// The metadata fill covers the complete allocation. Consume each layer only after its
-	// native image contents exist; already materialized layers may have been rendered since.
-	for (uint32_t layer = first; layer < first + count;) {
-		if ((metadata.clear_mask & (1u << layer)) == 0) {
-			layer++;
+	// Finish native metadata writes before reading backing bytes. This can submit the scheduler,
+	// so discovery runs before final draw uploads and never holds the texture lock across it.
+	if (m_buffer_cache.IsRegionGpuModified(range.address, range.size)) {
+		m_buffer_cache.ReadMemory(range.address, range.size, false);
+	}
+	const auto slice_size = range.size / layers;
+	for (uint32_t slice = 0; slice < count; slice++) {
+		const auto address = range.address + slice_size * (first + slice);
+		uint8_t code = 0;
+		if (!LibKernel::Memory::TryReadBacking(address, &code, sizeof(code))) {
+			EXIT("TextureCache: failed to read DCC metadata backing\n");
+		}
+		vk::ClearValue clear {};
+		if (!DecodeDccClear(desc, code, clear.color)) {
 			continue;
 		}
-		const auto start = layer;
-		uint32_t   mask  = 0;
-		do {
-			mask |= 1u << layer++;
-		} while (layer < first + count && (metadata.clear_mask & (1u << layer)) != 0);
-		ClearImage(m_scheduler.Current(), id,
-		           {vk::ImageAspectFlagBits::eColor, view.base_level, view.level_count, start,
-		            layer - start},
-		           clear);
-		metadata.clear_mask &= ~mask;
+		std::vector<uint8_t> bytes(slice_size);
+		if (!LibKernel::Memory::TryReadBacking(address, bytes.data(), bytes.size())) {
+			EXIT("TextureCache: failed to read DCC metadata slice\n");
+		}
+		if (!std::all_of(bytes.begin(), bytes.end(), [code](uint8_t byte) { return byte == code; })) {
+			continue;
+		}
+		{
+			std::scoped_lock lock {m_lock};
+			ClearImage(m_scheduler.Current(), id, view.format,
+			           {vk::ImageAspectFlagBits::eColor, view.base_level, view.level_count,
+			            image_first + slice, 1}, clear);
+		}
+		// Native expanded keys own consumption. Existing buffer tracking publishes this CPU
+		// write to future GPU readers; FillBuffer can fault and must run outside the texture lock.
+		m_buffer_cache.FillBuffer(address, slice_size, UINT32_MAX, false);
 	}
 }
 
@@ -1249,6 +1251,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		std::scoped_lock lock {m_lock};
 		return GetNullImage(desc);
 	}
+	const auto metadata_base_layer = desc.view_info.base_layer;
 
 	ImageId result {};
 	{
@@ -1316,6 +1319,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		image.tick_accessed_last = m_scheduler.CurrentTick();
 		TouchImage(image);
 	}
+	MaterializeDccClear(result, desc, metadata_base_layer);
 	return result;
 }
 
@@ -1379,16 +1383,12 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
 		image.MarkGpuModified();
 	}
 	if (!image.info.data.Empty()) {
-		PrepareDccClear(id, desc);
 		RefreshImage(id);
 	}
 	switch (desc.type) {
 		case BindingType::Texture: break;
 		case BindingType::Storage:
 			if (!image.info.data.Empty()) {
-				if (!image.registered || image.depth_id) {
-					EXIT("TextureCache: cannot acquire an unavailable storage image\n");
-				}
 				CommitGpuWrite(image);
 			}
 			TrackImageDownload(id, image);
@@ -1412,7 +1412,6 @@ vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc)
 	TouchImage(image);
 	image.MarkGpuModified();
 	image.usage.render_target = true;
-	PrepareDccClear(id, desc);
 	RefreshImage(id);
 	CommitGpuWrite(image);
 	TrackImageDownload(id, image);
@@ -1436,16 +1435,9 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	RefreshImage(id);
 	if (desc.info.HasMetadata()) {
 		image.info.metadata = desc.info.metadata;
-		auto [metadata, inserted] =
-		    m_surface_metas.try_emplace(desc.info.metadata.range.address,
-		                                MetaDataInfo {.type       = MetaDataInfo::Type::HTile,
-		                                              .clear_mask = image.info.htile_clear_mask});
-		if (!inserted && metadata->second.type != MetaDataInfo::Type::HTile) {
-			// PS5 allocations can reuse DCC storage as HTile while the old color image is cached.
-			// The depth binding defines the new type; incompatible fill state cannot carry over.
-			metadata->second = {.type       = MetaDataInfo::Type::HTile,
-			                    .clear_mask = image.info.htile_clear_mask};
-		}
+		m_surface_metas.emplace(desc.info.metadata.range.address,
+		                        MetaDataInfo {.type       = MetaDataInfo::Type::HTile,
+		                                      .clear_mask = image.info.htile_clear_mask});
 	}
 	CommitGpuWrite(image);
 	if (desc.info.HasStencil()) {
@@ -1533,12 +1525,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		}
 		clear.depthStencil.stencil = stencil_clear;
 	}
-	ClearImage(command, selected,
+	ClearImage(command, selected, image.backing.format,
 	           {aspect, 0, image.info.resources.levels, 0, image.info.TransferLayers()}, clear);
 	return true;
 }
 
-void TextureCache::ClearImage(CommandBuffer& command, ImageId id,
+void TextureCache::ClearImage(CommandBuffer& command, ImageId id, vk::Format format,
                               const vk::ImageSubresourceRange& range, const vk::ClearValue& clear) {
 	auto& image = m_slot_images[id];
 	const auto aspects = image.info.IsDepth() ? ImageViewOps::DepthAspectMask(image.backing.format)
@@ -1563,11 +1555,12 @@ void TextureCache::ClearImage(CommandBuffer& command, ImageId id,
 		}
 	}
 	command.EndRendering();
-	if (image.info.IsVolume() && !full_image) {
+	// Transfer clears use the backing format; aliased clears must encode through their view.
+	if (format != image.backing.format || (image.info.IsVolume() && !full_image)) {
 		EXIT_NOT_IMPLEMENTED(range.aspectMask != vk::ImageAspectFlagBits::eColor ||
 		                     range.levelCount != 1);
 		ImageViewInfo view {};
-		view.format = image.backing.format;
+		view.format = format;
 		view.type   = range.layerCount == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
 		view.base_level  = range.baseMipLevel;
 		view.base_layer  = range.baseArrayLayer;
@@ -1675,12 +1668,12 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	             destination_offset, info.data.size, tiles);
 }
 
-void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-                                     uint64_t destination_size, DownloadPlan plan) {
-	if (!plan.valid) {
-		EXIT("TextureCache: invalid image download plan\n");
+void TextureCache::DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset,
+                                     uint64_t destination_size, ImageDownload transfer) {
+	if (!transfer.valid) {
+		EXIT("TextureCache: invalid image download transfer\n");
 	}
-	if (plan.depth_target) {
+	if (transfer.depth_target) {
 		if (destination_size != image.info.data.size) {
 			EXIT("TextureCache: partial depth image download is unsupported\n");
 		}
@@ -1688,7 +1681,7 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t
 		return;
 	}
 
-	auto&      texture   = plan.texture;
+	auto&      texture   = transfer.texture;
 	const auto transform = texture.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16
 	                                           : TileManager::ColorTransform::None;
 	if (texture.tiles.empty()) {
@@ -1749,15 +1742,15 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 	if (copy_size == 0) {
 		return false;
 	}
-	auto plan = m_texture_cache.BuildDownload(image);
-	if (!plan.valid) {
+	auto transfer = m_texture_cache.BuildDownload(image);
+	if (!transfer.valid) {
 		return false;
 	}
-	if (plan.depth_target && copy_size != image.info.data.size) {
+	if (transfer.depth_target && copy_size != image.info.data.size) {
 		return false;
 	}
-	if (!plan.depth_target && levels < image.info.resources.levels) {
-		auto& texture = plan.texture;
+	if (!transfer.depth_target && levels < image.info.resources.levels) {
+		auto& texture = transfer.texture;
 		std::erase_if(texture.regions, [levels](const vk::BufferImageCopy& region) {
 			return region.imageSubresource.mipLevel >= levels;
 		});
@@ -1772,17 +1765,17 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 			}
 		}
 	}
-	m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan));
+	m_texture_cache.DownloadImage(image, buffer, buf_offset, copy_size, std::move(transfer));
 	return true;
 }
 
-bool TextureCache::TryDownloadImage(ImageId id) {
+bool TextureCache::DownloadImageMemory(ImageId id) {
 	auto& image = m_slot_images[id];
 	if (image.depth_id) {
 		return false;
 	}
-	auto plan = BuildDownload(image);
-	if (!plan.valid || !SafeToDownload(image)) {
+	auto transfer = BuildDownload(image);
+	if (!transfer.valid || !SafeToDownload(image)) {
 		return false;
 	}
 	const auto range    = image.info.data;
@@ -1798,7 +1791,7 @@ bool TextureCache::TryDownloadImage(ImageId id) {
 	}
 	download.Flush(offset, range.size);
 
-	DownloadImageData(image, download, offset, range.size, std::move(plan));
+	DownloadImage(image, download, offset, range.size, std::move(transfer));
 	vk::BufferMemoryBarrier barrier {};
 	barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite | vk::AccessFlagBits::eTransferWrite |
 	                        vk::AccessFlagBits::eShaderWrite;
@@ -1851,8 +1844,8 @@ bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
 }
 
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
-	const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
-	const auto page_end   = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
+	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {
@@ -1878,72 +1871,32 @@ void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 bool TextureCache::IsMeta(uint64_t address) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	return found != m_surface_metas.end() && found->second.type != MetaDataInfo::Type::PendingDcc;
+	return found != m_surface_metas.end();
 }
 
-bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice, uint32_t* fill_value) {
+bool TextureCache::IsMetaCleared(uint64_t address, uint32_t slice) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    slice >= 32) {
+	if (found == m_surface_metas.end() || slice >= 32) {
 		return false;
 	}
-	if (fill_value != nullptr) {
-		*fill_value = found->second.fill_value;
-	}
 	return (found->second.clear_mask & (1u << slice)) != 0;
 }
 
 bool TextureCache::ClearMeta(uint64_t address) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    found->second.type == MetaDataInfo::Type::Dcc) {
-		// Preserve the broad metadata-clear operation for CMask/FMask/HTile. DCC requires a
-		// validated fill value, so an arbitrary compute write must not clear it.
+	if (found == m_surface_metas.end()) {
 		return false;
 	}
 	found->second.clear_mask = UINT32_MAX;
 	return true;
 }
 
-void TextureCache::TrackDccFill(uint64_t address, uint64_t size, uint32_t fill_value) {
-	if (!GuestRange {address, size}.Valid()) {
-		EXIT("TextureCache: invalid DCC fill range\n");
-	}
-	// DCC fills use a repeated byte code. Require all four bytes of the detected dword to agree,
-	// and mark only recognized deferred-clear encodings as logically clear.
-	const auto dcc_clear_mask = [fill_value] {
-		const auto code = static_cast<uint8_t>(fill_value);
-		if (fill_value != static_cast<uint32_t>(code) * 0x01010101u) {
-			return 0u;
-		}
-		switch (code) {
-			case 0x00:
-			case 0x20:
-			case 0x40:
-			case 0x80:
-			case 0xc0: return UINT32_MAX;
-			default: return 0u;
-		}
-	}();
-	std::scoped_lock lock {m_lock};
-	// The guest dispatch still writes metadata. An unknown address remains PendingDcc until an
-	// image descriptor confirms its role; never reinterpret CMask/FMask/HTile as DCC.
-	const auto found = m_surface_metas.try_emplace(address).first;
-	if (found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    found->second.type == MetaDataInfo::Type::Dcc) {
-		found->second.clear_mask = dcc_clear_mask;
-		found->second.fill_value = fill_value;
-		found->second.fill_size  = size;
-	}
-}
-
 bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) {
 	std::scoped_lock lock {m_lock};
 	const auto       found = m_surface_metas.find(address);
-	if (found == m_surface_metas.end() || found->second.type == MetaDataInfo::Type::PendingDcc ||
-	    slice >= 32) {
+	if (found == m_surface_metas.end() || slice >= 32) {
 		return false;
 	}
 	if (is_clear) {
@@ -1959,10 +1912,13 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 		EXIT("TextureCache: invalid unmap range\n");
 	}
 	std::scoped_lock lock {m_lock};
-	const auto       end = address + size;
-	for (auto metadata = m_surface_metas.lower_bound(address);
-	     metadata != m_surface_metas.end() && metadata->first < end;) {
-		metadata = m_surface_metas.erase(metadata);
+	for (auto metadata = m_surface_metas.begin(); metadata != m_surface_metas.end();) {
+		const auto base = metadata->first;
+		if (base >= address && base < address + size) {
+			metadata = m_surface_metas.erase(metadata);
+		} else {
+			++metadata;
+		}
 	}
 	auto images = FindImagesInRegion(address, size, false);
 	for (const auto id: images) {
@@ -1970,10 +1926,7 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 		if (owner == nullptr) {
 			continue;
 		}
-		if (owner->IsGpuModified()) {
-			owner->ClearGpuModified();
-		}
-		DeleteImage(id);
+		FreeImage(id);
 	}
 }
 
@@ -2016,12 +1969,11 @@ void TextureCache::RunGarbageCollector() {
 				if (safe && !pressured) {
 					continue;
 				}
-				if (safe && !TryDownloadImage(id)) {
+				if (safe && !DownloadImageMemory(id)) {
 					continue;
 				}
-				owner->ClearGpuModified();
 			}
-			DeleteImage(id);
+			FreeImage(id);
 			if (m_total_used_memory < m_critical_gc_memory && aggressive) {
 				deletions >>= 2;
 				aggressive = false;
@@ -2043,7 +1995,7 @@ void TextureCache::ProcessDownloadImages() {
 	for (const auto id: m_download_images) {
 		const auto owner = m_slot_images.try_get(id);
 		if (owner != nullptr && owner->registered && owner->IsGpuModified()) {
-			(void)TryDownloadImage(id);
+			(void)DownloadImageMemory(id);
 		}
 	}
 	m_download_images.clear();

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -13,6 +13,7 @@
 #include "graphics/host_gpu/renderer/image/tiler.h"
 
 #include <map>
+#include <type_traits>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -63,11 +64,8 @@ public:
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
-	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice,
-	                                 uint32_t* fill_value = nullptr);
+	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);
 	[[nodiscard]] bool ClearMeta(uint64_t address);
-	// Record deferred DCC state while the original guest dispatch writes the metadata.
-	void               TrackDccFill(uint64_t address, uint64_t size, uint32_t fill_value);
 	[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
 
 	void UnmapMemory(uint64_t address, uint64_t size);
@@ -76,21 +74,14 @@ public:
 
 private:
 	enum class TransferDirection { Upload, Download };
-	struct TextureTransferPlan;
-	struct DownloadPlan;
+	struct TextureTransfer;
+	struct ImageDownload;
 
 	struct MetaDataInfo {
-		// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
-		// PendingDcc retains that exact fill until an image binding classifies the address,
-		// without exposing an unconfirmed buffer address to the normal metadata heuristics.
-		// Keep all surface metadata in one entry so CMask/FMask can be
-		// registered beside HTile and DCC without introducing parallel tracking paths.
-		enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc };
+		enum class Type : uint8_t { CMask, FMask, HTile };
 
-		Type     type       = Type::PendingDcc;
-		uint32_t clear_mask = 0;
-		uint32_t fill_value = 0xffffffffu;
-		uint64_t fill_size  = 0;
+		Type     type;
+		uint32_t clear_mask = UINT32_MAX;
 	};
 
 	struct OverlapResult {
@@ -102,6 +93,23 @@ private:
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
 	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
+	// Callers have validated the nonempty 40-bit range with TryGetPageRange.
+	template <typename Func>
+	static void ForEachPage(uint64_t address, size_t size, Func&& func) {
+		using FuncReturn = typename std::invoke_result<Func, uint64_t>::type;
+		static constexpr bool RETURNS_BOOL = std::is_same_v<FuncReturn, bool>;
+		const uint64_t page_end = (address + size - 1) >> ImagePageTable::kPageBits;
+		for (uint64_t page = address >> ImagePageTable::kPageBits; page <= page_end; ++page) {
+			if constexpr (RETURNS_BOOL) {
+				if (func(page)) {
+					break;
+				}
+			} else {
+				func(page);
+			}
+		}
+	}
+
 	[[nodiscard]] ImageId     InsertImage(const ImageInfo& info);
 	[[nodiscard]] ImageId     GetNullImage(const ImageDesc& desc);
 	void                      RegisterImage(ImageId id);
@@ -131,19 +139,20 @@ private:
 	                                                ImageId cached);
 	[[nodiscard]] ImageId       ExpandImage(const ImageInfo& info, ImageId source);
 	void                        RefreshImage(ImageId id);
-	void                        PrepareDccClear(ImageId id, const ImageDesc& desc);
+	void                        MaterializeDccClear(ImageId id, const ImageDesc& desc,
+	                                                uint32_t metadata_base_layer);
 	void                        InitializeImage(ImageId id);
-	[[nodiscard]] TextureTransferPlan
+	[[nodiscard]] TextureTransfer
 	BuildTextureTransfer(const Image& image, BindingType binding, TransferDirection direction) const;
-	[[nodiscard]] DownloadPlan BuildDownload(const Image& image) const;
+	[[nodiscard]] ImageDownload BuildDownload(const Image& image) const;
 	void UploadImage(Image& image, Buffer& source, uint64_t source_offset);
-	void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
-	                       uint64_t destination_size, DownloadPlan plan);
+	void DownloadImage(Image& image, Buffer& destination, uint64_t destination_offset,
+	                       uint64_t destination_size, ImageDownload transfer);
 	void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
 	void CommitGpuWrite(Image& image);
 	// Caller holds m_lock. Volume layer ranges select depth slices.
-	void ClearImage(CommandBuffer& command, ImageId id, const vk::ImageSubresourceRange& range,
-	                const vk::ClearValue& clear);
+	void ClearImage(CommandBuffer& command, ImageId id, vk::Format format,
+	                const vk::ImageSubresourceRange& range, const vk::ClearValue& clear);
 	void PrepareImageCopy(Image& image);
 	void RefreshCopySource(ImageId id);
 	[[nodiscard]] bool CopyD16(Image& destination, Image& source);
@@ -153,7 +162,7 @@ private:
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
 	void               InvalidateCpuAliases(uint64_t address, uint64_t size);
-	[[nodiscard]] bool TryDownloadImage(ImageId id);
+	[[nodiscard]] bool DownloadImageMemory(ImageId id);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;

src/graphics/host_gpu/renderer/context.cpp

@@ -61,11 +61,11 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0,
 }
 
 void CommandBuffer::BeginRendering(const RenderState& state) const {
-	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
-	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);
 	if (m_rendering && m_render_state == state) {
 		return;
 	}
+	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
+	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);
 	EndRendering();
 
 	std::array<vk::RenderingAttachmentInfo, RENDER_COLOR_ATTACHMENTS_MAX> colors {};

src/graphics/host_gpu/renderer/debug.cpp

@@ -4,7 +4,6 @@
 #include "common/common.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
-#include "common/stringUtils.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/renderer/render.h"
@@ -57,7 +56,7 @@ uint32_t render_target_first_bound_slot(const CommandBuffer& buffer) {
 
 bool graphics_debug_dump_enabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
-	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
+	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
 }
 
 void uc_print(const char* func, const HW::UserConfig& uc) {
@@ -89,91 +88,75 @@ void uc_check(const HW::UserConfig& uc) {
 	EXIT_NOT_IMPLEMENTED(user_en.vgpr3 != false);
 }
 
-void sh_print(const char* func, const HW::Shader& /*uc*/) {
-	LOGF("%s\n", func);
-}
+std::string rt_print(const char* func, const HW::RenderTarget& rt) {
+	std::string dst;
+	dst.reserve(4096);
 
-std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt) {
-	std::vector<std::string> dst;
-	dst.reserve(40);
+	dst += fmt::format("{}\n", func);
 
-	dst.push_back(fmt::format("{}\n", func));
+	dst += fmt::format("\t base.addr                       = 0x{:016x}\n", rt.base.addr);
+	dst += fmt::format("\t view.base_array_slice_index     = 0x{:08x}\n",
+	                  rt.view.base_array_slice_index);
+	dst += fmt::format("\t view.last_array_slice_index     = 0x{:08x}\n",
+	                  rt.view.last_array_slice_index);
+	dst += fmt::format("\t view.current_mip_level          = 0x{:08x}\n", rt.view.current_mip_level);
+	dst += fmt::format("\t info.fmask_compression_enable   = {}\n",
+	                  rt.info.fmask_compression_enable ? "true" : "false");
 
-	dst.push_back(fmt::format("\t base.addr                       = 0x{:016x}\n", rt.base.addr));
-	dst.push_back(fmt::format("\t view.base_array_slice_index     = 0x{:08x}\n",
-	                          rt.view.base_array_slice_index));
-	dst.push_back(fmt::format("\t view.last_array_slice_index     = 0x{:08x}\n",
-	                          rt.view.last_array_slice_index));
-	dst.push_back(
-	    fmt::format("\t view.current_mip_level          = 0x{:08x}\n", rt.view.current_mip_level));
-	dst.push_back(fmt::format("\t info.fmask_compression_enable   = {}\n",
-	                          rt.info.fmask_compression_enable ? "true" : "false"));
+	dst += fmt::format("\t info.fmask_data_compression_disable = {}\n",
+	                  rt.info.fmask_data_compression_disable ? "true" : "false");
+	dst += fmt::format("\t info.fmask_one_frag_mode        = {}\n",
+	                  rt.info.fmask_one_frag_mode ? "true" : "false");
 
-	// dst.push_back(fmt::format("\t info.fmask_compression_mode     = 0x{:08x}\n", //
-	// rt.info.fmask_compression_mode));
-	dst.push_back(fmt::format("\t info.fmask_data_compression_disable = {}\n",
-	                          rt.info.fmask_data_compression_disable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.fmask_one_frag_mode        = {}\n",
-	                          rt.info.fmask_one_frag_mode ? "true" : "false"));
-
-	dst.push_back(fmt::format("\t info.cmask_fast_clear_enable    = {}\n",
-	                          rt.info.cmask_fast_clear_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.dcc_compression_enable     = {}\n",
-	                          rt.info.dcc_compression_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.format                     = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.format)));
-	dst.push_back(fmt::format("\t info.channel_type               = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.channel_type)));
-	dst.push_back(fmt::format("\t info.channel_order              = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.info.channel_order)));
-	dst.push_back(fmt::format("\t info.blend_bypa                 = {}\n",
-	                          rt.info.blend_bypass ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.blend_clamp                = {}\n",
-	                          rt.info.blend_clamp ? "true" : "false"));
-	dst.push_back(fmt::format("\t info.round_mode                 = {}\n",
-	                          rt.info.round_mode ? "true" : "false"));
-	dst.push_back(fmt::format("\t attrib.force_dest_alpha_to_one  = {}\n",
-	                          rt.attrib.force_dest_alpha_to_one ? "true" : "false"));
-	dst.push_back(
-	    fmt::format("\t attrib.num_samples              = 0x{:08x}\n", rt.attrib.num_samples));
-	dst.push_back(
-	    fmt::format("\t attrib.num_fragments            = 0x{:08x}\n", rt.attrib.num_fragments));
-	dst.push_back(fmt::format("\t attrib2.width                   = 0x{:08x}\n", rt.attrib2.width));
-	dst.push_back(
-	    fmt::format("\t attrib2.height                  = 0x{:08x}\n", rt.attrib2.height));
-	dst.push_back(
-	    fmt::format("\t attrib2.num_mip_levels          = 0x{:08x}\n", rt.attrib2.num_mip_levels));
-	dst.push_back(fmt::format("\t attrib3.depth                   = 0x{:08x}\n", rt.attrib3.depth));
-	dst.push_back(fmt::format("\t attrib3.tile_mode               = 0x{:08x}\n",
-	                          static_cast<uint32_t>(rt.attrib3.tile_mode)));
-	dst.push_back(
-	    fmt::format("\t attrib3.dimension               = 0x{:08x}\n", rt.attrib3.dimension));
-	dst.push_back(fmt::format("\t attrib3.metadata_pipe_aligned   = {}\n",
-	                          rt.attrib3.metadata_pipe_aligned ? "true" : "false"));
-	dst.push_back(fmt::format("\t attrib3.write_vrs_rate_hint_to_cmask = {}\n",
-	                          rt.attrib3.write_vrs_rate_hint_to_cmask ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n",
-	                          rt.dcc.max_uncompressed_block_size));
-	dst.push_back(fmt::format("\t dcc.max_compressed_block_size   = 0x{:08x}\n",
-	                          rt.dcc.max_compressed_block_size));
-	dst.push_back(
-	    fmt::format("\t dcc.color_transform             = 0x{:08x}\n", rt.dcc.color_transform));
-	dst.push_back(fmt::format("\t dcc.overwrite_combiner_disable  = {}\n",
-	                          rt.dcc.overwrite_combiner_disable ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.independent_block_size      = 0x{:02x}\n",
-	                          static_cast<uint8_t>(rt.dcc.independent_block_size)));
-	dst.push_back(fmt::format("\t data_write_on_dcc_clear_to_reg  = {}\n",
-	                          rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false"));
-	dst.push_back(fmt::format("\t dcc.dcc_clear_key_enable        = {}\n",
-	                          rt.dcc.dcc_clear_key_enable ? "true" : "false"));
-	dst.push_back(fmt::format("\t cmask.addr                      = 0x{:016x}\n", rt.cmask.addr));
-	dst.push_back(fmt::format("\t fmask.addr                      = 0x{:016x}\n", rt.fmask.addr));
-	dst.push_back(
-	    fmt::format("\t clear_word0.word0               = 0x{:08x}\n", rt.clear_word0.word0));
-	dst.push_back(
-	    fmt::format("\t clear_word1.word1               = 0x{:08x}\n", rt.clear_word1.word1));
-	dst.push_back(
-	    fmt::format("\t dcc_addr.addr                   = 0x{:016x}\n", rt.dcc_addr.addr));
+	dst += fmt::format("\t info.cmask_fast_clear_enable    = {}\n",
+	                  rt.info.cmask_fast_clear_enable ? "true" : "false");
+	dst += fmt::format("\t info.dcc_compression_enable     = {}\n",
+	                  rt.info.dcc_compression_enable ? "true" : "false");
+	dst += fmt::format("\t info.format                     = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.format));
+	dst += fmt::format("\t info.channel_type               = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.channel_type));
+	dst += fmt::format("\t info.channel_order              = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.info.channel_order));
+	dst += fmt::format("\t info.blend_bypa                 = {}\n",
+	                  rt.info.blend_bypass ? "true" : "false");
+	dst += fmt::format("\t info.blend_clamp                = {}\n",
+	                  rt.info.blend_clamp ? "true" : "false");
+	dst += fmt::format("\t info.round_mode                 = {}\n",
+	                  rt.info.round_mode ? "true" : "false");
+	dst += fmt::format("\t attrib.force_dest_alpha_to_one  = {}\n",
+	                  rt.attrib.force_dest_alpha_to_one ? "true" : "false");
+	dst += fmt::format("\t attrib.num_samples              = 0x{:08x}\n", rt.attrib.num_samples);
+	dst += fmt::format("\t attrib.num_fragments            = 0x{:08x}\n", rt.attrib.num_fragments);
+	dst += fmt::format("\t attrib2.width                   = 0x{:08x}\n", rt.attrib2.width);
+	dst += fmt::format("\t attrib2.height                  = 0x{:08x}\n", rt.attrib2.height);
+	dst += fmt::format("\t attrib2.num_mip_levels          = 0x{:08x}\n", rt.attrib2.num_mip_levels);
+	dst += fmt::format("\t attrib3.depth                   = 0x{:08x}\n", rt.attrib3.depth);
+	dst += fmt::format("\t attrib3.tile_mode               = 0x{:08x}\n",
+	                  static_cast<uint32_t>(rt.attrib3.tile_mode));
+	dst += fmt::format("\t attrib3.dimension               = 0x{:08x}\n", rt.attrib3.dimension);
+	dst += fmt::format("\t attrib3.metadata_pipe_aligned   = {}\n",
+	                  rt.attrib3.metadata_pipe_aligned ? "true" : "false");
+	dst += fmt::format("\t attrib3.write_vrs_rate_hint_to_cmask = {}\n",
+	                  rt.attrib3.write_vrs_rate_hint_to_cmask ? "true" : "false");
+	dst += fmt::format("\t dcc.max_uncompressed_block_size = 0x{:08x}\n",
+	                  rt.dcc.max_uncompressed_block_size);
+	dst += fmt::format("\t dcc.max_compressed_block_size   = 0x{:08x}\n",
+	                  rt.dcc.max_compressed_block_size);
+	dst += fmt::format("\t dcc.color_transform             = 0x{:08x}\n", rt.dcc.color_transform);
+	dst += fmt::format("\t dcc.overwrite_combiner_disable  = {}\n",
+	                  rt.dcc.overwrite_combiner_disable ? "true" : "false");
+	dst += fmt::format("\t dcc.independent_block_size      = 0x{:02x}\n",
+	                  static_cast<uint8_t>(rt.dcc.independent_block_size));
+	dst += fmt::format("\t data_write_on_dcc_clear_to_reg  = {}\n",
+	                  rt.dcc.data_write_on_dcc_clear_to_reg ? "true" : "false");
+	dst += fmt::format("\t dcc.dcc_clear_key_enable        = {}\n",
+	                  rt.dcc.dcc_clear_key_enable ? "true" : "false");
+	dst += fmt::format("\t cmask.addr                      = 0x{:016x}\n", rt.cmask.addr);
+	dst += fmt::format("\t fmask.addr                      = 0x{:016x}\n", rt.fmask.addr);
+	dst += fmt::format("\t clear_word0.word0               = 0x{:08x}\n", rt.clear_word0.word0);
+	dst += fmt::format("\t clear_word1.word1               = 0x{:08x}\n", rt.clear_word1.word1);
+	dst += fmt::format("\t dcc_addr.addr                   = 0x{:016x}\n", rt.dcc_addr.addr);
 
 	return dst;
 }
@@ -504,16 +487,7 @@ static void BcPrint(const char* func, const HW::BlendControl& c, const HW::Blend
 	     color.red, color.green, color.blue, color.alpha, cc.mode, cc.op);
 }
 
-static void BcCheck(const HW::BlendControl& /*c*/, const HW::BlendColor& color,
-                    const HW::ColorControl& cc) {
-	// EXIT_NOT_IMPLEMENTED(c.color_srcblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.color_comb_fcn != 0);
-	// EXIT_NOT_IMPLEMENTED(c.color_destblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_srcblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_comb_fcn != 0);
-	// EXIT_NOT_IMPLEMENTED(c.alpha_destblend != 0);
-	// EXIT_NOT_IMPLEMENTED(c.separate_alpha_blend != false);
-	// EXIT_NOT_IMPLEMENTED(c.enable != false);
+static void BcCheck(const HW::BlendColor& color, const HW::ColorControl& cc) {
 	if (color.red != 0.0f || color.green != 0.0f || color.blue != 0.0f || color.alpha != 0.0f) {
 		static bool logged = false;
 		if (!logged) {
@@ -576,29 +550,6 @@ static void DPrint(const char* func, const HW::DepthControl& c, const HW::Stenci
 	     sm.stencil_opval_bf);
 }
 
-static void DCheck(const HW::DepthControl& c, const HW::StencilControl& s,
-                   const HW::StencilMask& /*sm*/) {
-	// EXIT_NOT_IMPLEMENTED(c.stencil_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.z_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.z_write_enable != false);
-	// EXIT_NOT_IMPLEMENTED(c.zfunc != 0);
-	// Back-face stencil state is handled separately when enabled.
-	// EXIT_NOT_IMPLEMENTED(c.stencilfunc != 0);
-	// EXIT_NOT_IMPLEMENTED(c.stencilfunc_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_fail != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_zpass != 0);
-	// EXIT_NOT_IMPLEMENTED(s.stencil_zfail != 0);
-	// Back-face stencil ops may legitimately differ from the front-face ops.
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_testval != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_mask != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_writemask != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_opval != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_testval_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_mask_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_writemask_bf != 0);
-	// EXIT_NOT_IMPLEMENTED(sm.stencil_opval_bf != 0);
-}
-
 static void EqaaPrint(const char* func, const HW::EqaaControl& c) {
 	LOGF("%s\n", func);
 
@@ -904,14 +855,10 @@ void hw_check(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
-	const auto& bc      = hw.GetBlendControl(rt_slot);
 	const auto& bclr    = hw.GetBlendColor();
 	const auto& vp      = hw.GetScreenViewport();
 	const auto& c       = hw.GetClipControl();
 	const auto& rc      = hw.GetRenderControl();
-	const auto& d       = hw.GetDepthControl();
-	const auto& s       = hw.GetStencilControl();
-	const auto& sm      = hw.GetStencilMask();
 	const auto& mc      = hw.GetModeControl();
 	const auto& eqaa    = hw.GetEqaaControl();
 	const auto& cc      = hw.GetColorControl();
@@ -937,11 +884,10 @@ void hw_check(const CommandBuffer& buffer) {
 	log_phase("rc");
 	RcCheck(rc);
 	log_phase("depth");
-	DCheck(d, s, sm);
 	log_phase("mode");
 	McCheck(mc);
 	log_phase("blend");
-	BcCheck(bc, bclr, cc);
+	BcCheck(bclr, cc);
 	log_phase("eqaa");
 	EqaaCheck(eqaa, ac);
 	log_phase("aa");
@@ -1000,7 +946,7 @@ void hw_print(const CommandBuffer& buffer) {
 		     hw.GetRenderTargetMask(), hw.GetDepthClearValue(), hw.GetStencilClearValue(),
 		     hw.GetLineWidth(), hw.GetPrimitiveResetIndex());
 
-		LOGF("%s", Common::Concat(rt_print("RenderTraget:", rt), "").c_str());
+		LOGF("%s", rt_print("RenderTraget:", rt).c_str());
 
 		ZPrint("DepthRenderTraget:", z);
 		VpPrint("ScreenViewport:", vp, smc);

src/graphics/host_gpu/renderer/debug.h

@@ -5,7 +5,6 @@
 
 #include <cstdint>
 #include <string>
-#include <vector>
 
 namespace Libs::Graphics {
 
@@ -13,7 +12,6 @@ class CommandBuffer;
 
 namespace HW {
 class Context;
-class Shader;
 class UserConfig;
 struct RenderTarget;
 struct ScanModeControl;
@@ -32,8 +30,7 @@ uint32_t                 render_target_first_bound_slot(const CommandBuffer& buf
 bool                     graphics_debug_dump_enabled();
 void                     uc_print(const char* func, const HW::UserConfig& uc);
 void                     uc_check(const HW::UserConfig& uc);
-void                     sh_print(const char* func, const HW::Shader& uc);
-std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt);
+std::string              rt_print(const char* func, const HW::RenderTarget& rt);
 bool                     RenderIsColorTileModeLinear(Prospero::TileMode tile_mode);
 void                     hw_print(const CommandBuffer& buffer);
 void                     hw_check(const CommandBuffer& buffer);

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -19,6 +19,7 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <algorithm>
+#include <array>
 #include <atomic>
 #include <cmath>
 #include <cstdarg>
@@ -45,8 +46,12 @@ static vk::StencilOp ConvertStencilOp(uint8_t value, uint8_t write_mask, uint8_t
 	switch (static_cast<Prospero::StencilOp>(value)) {
 		case Prospero::StencilOp::kKeep: return vk::StencilOp::eKeep;
 		case Prospero::StencilOp::kZero: return vk::StencilOp::eZero;
-		case Prospero::StencilOp::kReplaceTest:
-		case Prospero::StencilOp::kReplaceOp: return vk::StencilOp::eReplace;
+		case Prospero::StencilOp::kReplaceTest: return vk::StencilOp::eReplace;
+		case Prospero::StencilOp::kReplaceOp:
+			if ((op_value & write_mask) == 0) {
+				return vk::StencilOp::eZero;
+			}
+			return vk::StencilOp::eReplace;
 		case Prospero::StencilOp::kAddClamp: return vk::StencilOp::eIncrementAndClamp;
 		case Prospero::StencilOp::kSubClamp: return vk::StencilOp::eDecrementAndClamp;
 		case Prospero::StencilOp::kInvert: return vk::StencilOp::eInvert;
@@ -66,9 +71,38 @@ static vk::StencilOp ConvertStencilOp(uint8_t value, uint8_t write_mask, uint8_t
 	}
 }
 
-static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
-	constexpr auto replace_op = static_cast<uint8_t>(Prospero::StencilOp::kReplaceOp);
-	return fail == replace_op || pass == replace_op || depth_fail == replace_op;
+static vk::StencilOpState ConvertStencilState(
+    uint8_t compare, const std::array<uint8_t, 3>& operations, uint8_t op_value,
+    const vk::StencilOpState& state) {
+	const auto test_value = state.reference;
+	auto reference       = test_value;
+	auto required_bits   = state.compareMask;
+	if (compare == static_cast<uint8_t>(vk::CompareOp::eAlways) ||
+	    compare == static_cast<uint8_t>(vk::CompareOp::eNever)) {
+		required_bits = 0;
+	}
+	std::array<vk::StencilOp, 3> converted {};
+	for (size_t i = 0; i < operations.size(); i++) {
+		converted[i] = ConvertStencilOp(operations[i], state.writeMask, op_value);
+		if (converted[i] != vk::StencilOp::eReplace) {
+			continue;
+		}
+		auto replacement = test_value;
+		if (static_cast<Prospero::StencilOp>(operations[i]) == Prospero::StencilOp::kReplaceOp) {
+			replacement = op_value;
+		}
+		if (((reference ^ replacement) & required_bits & state.writeMask) != 0) {
+			DepthFatal("unsupported stencil replacement: compare=%u, compare mask=0x%02" PRIx32
+			           ", write mask=0x%02" PRIx32 ", operation value=0x%02" PRIx8
+			           ", test value=0x%02" PRIx32,
+			           compare, state.compareMask, state.writeMask, op_value, test_value);
+		}
+		// Vulkan shares one reference between comparison and every replacement on this face.
+		reference = (reference & ~state.writeMask) | (replacement & state.writeMask);
+		required_bits |= state.writeMask;
+	}
+	return {converted[0], converted[1], converted[2], static_cast<vk::CompareOp>(compare),
+	        state.compareMask, state.writeMask, reference};
 }
 
 [[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(const CommandBuffer&     buffer,
@@ -296,31 +330,20 @@ void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepth
 		const uint8_t back_write_mask  = stencil_ops_disabled ? 0 : sm.stencil_writemask_bf;
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
-		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||
-		    (front_write_mask != 0 &&
-		     UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
-		     sm.stencil_opval != sm.stencil_testval) ||
-		    (dc.backface_enable && back_write_mask != 0 &&
-		     UsesStencilOpValue(sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf) &&
-		     sm.stencil_opval_bf != sm.stencil_testval_bf)) {
-			DepthFatal("unsupported stencil compare or replacement state");
+		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways))) {
+			DepthFatal("unsupported stencil compare state");
 		}
-		r.stencil_static_front = {
-		    ConvertStencilOp(sc.stencil_fail, front_write_mask, sm.stencil_opval),
-		    ConvertStencilOp(sc.stencil_zpass, front_write_mask, sm.stencil_opval),
-		    ConvertStencilOp(sc.stencil_zfail, front_write_mask, sm.stencil_opval),
-		    static_cast<vk::CompareOp>(dc.stencilfunc)};
-		r.stencil_dynamic_front = {sm.stencil_mask, front_write_mask, sm.stencil_testval};
+		r.stencil_front = ConvertStencilState(
+		    dc.stencilfunc, {sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail},
+		    sm.stencil_opval, {.compareMask = sm.stencil_mask, .writeMask = front_write_mask,
+		                       .reference = sm.stencil_testval});
 		if (dc.backface_enable) {
-			r.stencil_static_back = {
-			    ConvertStencilOp(sc.stencil_fail_bf, back_write_mask, sm.stencil_opval_bf),
-			    ConvertStencilOp(sc.stencil_zpass_bf, back_write_mask, sm.stencil_opval_bf),
-			    ConvertStencilOp(sc.stencil_zfail_bf, back_write_mask, sm.stencil_opval_bf),
-			    static_cast<vk::CompareOp>(dc.stencilfunc_bf)};
-			r.stencil_dynamic_back = {sm.stencil_mask_bf, back_write_mask, sm.stencil_testval_bf};
+			r.stencil_back = ConvertStencilState(
+			    dc.stencilfunc_bf, {sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf},
+			    sm.stencil_opval_bf, {.compareMask = sm.stencil_mask_bf, .writeMask = back_write_mask,
+			                          .reference = sm.stencil_testval_bf});
 		} else {
-			r.stencil_static_back  = r.stencil_static_front;
-			r.stencil_dynamic_back = r.stencil_dynamic_front;
+			r.stencil_back = r.stencil_front;
 		}
 	}
 	auto& cache = m_context.GetTextureCache();
@@ -406,14 +429,13 @@ vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const {
 		return writes;
 	}
 
-	const auto face_writes = [&](const PipelineStencilStaticState&  state,
-	                             const PipelineStencilDynamicState& dynamic) {
-		if (dynamic.writeMask == 0) {
+	const auto face_writes = [&](const vk::StencilOpState& state) {
+		if (state.writeMask == 0) {
 			return false;
 		}
 		bool can_pass = state.compareOp != vk::CompareOp::eNever;
 		bool can_fail = state.compareOp != vk::CompareOp::eAlways;
-		if (dynamic.compareMask == 0) {
+		if (state.compareMask == 0) {
 			switch (state.compareOp) {
 				case vk::CompareOp::eEqual:
 				case vk::CompareOp::eLessOrEqual:
@@ -439,8 +461,7 @@ vk::ImageAspectFlags RenderDepthInfo::AttachmentWriteAspects() const {
 		       (can_pass && depth_fail && state.depthFailOp != vk::StencilOp::eKeep);
 	};
 	if (stencil_clear_enable ||
-	    (stencil_test_enable && (face_writes(stencil_static_front, stencil_dynamic_front) ||
-	                             face_writes(stencil_static_back, stencil_dynamic_back)))) {
+	    (stencil_test_enable && (face_writes(stencil_front) || face_writes(stencil_back)))) {
 		writes |= vk::ImageAspectFlagBits::eStencil;
 	}
 	return writes;

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -34,10 +34,8 @@ struct RenderDepthInfo {
 	bool                        stencil_clear_enable     = false;
 	uint8_t                     stencil_clear_value      = 0;
 	bool                        stencil_test_enable      = false;
-	PipelineStencilStaticState  stencil_static_front;
-	PipelineStencilStaticState  stencil_static_back;
-	PipelineStencilDynamicState stencil_dynamic_front;
-	PipelineStencilDynamicState stencil_dynamic_back;
+	vk::StencilOpState          stencil_front;
+	vk::StencilOpState          stencil_back;
 	ImageId                     image_id;
 
 	[[nodiscard]] vk::ImageAspectFlags AttachmentWriteAspects() const;

src/graphics/host_gpu/renderer/image/blitHelper.cpp

@@ -10,7 +10,6 @@
 
 #include <algorithm>
 #include <array>
-#include <iterator>
 
 namespace Libs::Graphics {
 
@@ -37,9 +36,8 @@ BlitHelper::BlitHelper(GraphicContext& graphics, CommandScheduler& scheduler)
 	    m_graphics.device.createPipelineLayout(&layout_info, nullptr, &m_pipeline_layout),
 	    "create BlitHelper pipeline layout");
 
-	m_vertex_shader = CreateShader(GPU_BLIT_FS_TRIANGLE_SPV, std::size(GPU_BLIT_FS_TRIANGLE_SPV));
-	m_fragment_shader =
-	    CreateShader(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV, std::size(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV));
+	m_vertex_shader   = CompileSPV(GPU_BLIT_FS_TRIANGLE_SPV, m_graphics.device);
+	m_fragment_shader = CompileSPV(GPU_BLIT_COLOR_TO_MS_DEPTH_SPV, m_graphics.device);
 }
 
 BlitHelper::~BlitHelper() {
@@ -60,17 +58,6 @@ BlitHelper::~BlitHelper() {
 	}
 }
 
-vk::ShaderModule BlitHelper::CreateShader(const uint32_t* code, size_t words) const {
-	EXIT_IF(code == nullptr || words == 0);
-	vk::ShaderModuleCreateInfo create {};
-	create.codeSize         = words * sizeof(uint32_t);
-	create.pCode            = code;
-	vk::ShaderModule module = nullptr;
-	RequireVulkanSuccess(m_graphics.device.createShaderModule(&create, nullptr, &module),
-	                     "create BlitHelper shader module");
-	return module;
-}
-
 vk::Pipeline BlitHelper::GetPipeline(PipelineKey key) {
 	const auto cached = std::ranges::find(m_pipelines, key, &Pipeline::key);
 	if (cached != m_pipelines.end()) {

src/graphics/host_gpu/renderer/image/blitHelper.h

@@ -37,8 +37,7 @@ private:
 		vk::Pipeline handle = nullptr;
 	};
 
-	[[nodiscard]] vk::ShaderModule CreateShader(const uint32_t* code, size_t words) const;
-	[[nodiscard]] vk::Pipeline     GetPipeline(PipelineKey key);
+	[[nodiscard]] vk::Pipeline GetPipeline(PipelineKey key);
 
 	GraphicContext&         m_graphics;
 	CommandScheduler&       m_scheduler;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -693,11 +693,12 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 }
 
 uint64_t Image::HashGuestEdges() const {
-	constexpr uint64_t                         page_mask = TRACKER_PAGE_SIZE - 1;
 	std::array<uint8_t, TRACKER_PAGE_SIZE * 2> bytes {};
 	const auto                                 range = info.data;
-	const uint64_t head_end     = std::min(range.End(), (range.address + page_mask) & ~page_mask);
-	const uint64_t tail_begin   = std::max(range.address, range.End() & ~page_mask);
+	const uint64_t head_end =
+	    std::min(range.End(), Common::AlignUp(range.address, TRACKER_PAGE_SIZE));
+	const uint64_t tail_begin =
+	    std::max(range.address, Common::AlignDown(range.End(), TRACKER_PAGE_SIZE));
 	const uint64_t head_size    = head_end - range.address;
 	const uint64_t tail_address = tail_begin < head_end ? head_end : tail_begin;
 	const uint64_t tail_size    = range.End() - tail_address;

src/graphics/host_gpu/renderer/image/image.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/slotVector.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -133,7 +134,7 @@ public:
 	}
 	[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
 	[[nodiscard]] uint64_t AccountedSize() const noexcept {
-		return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
+		return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024);
 	}
 	[[nodiscard]] uint64_t HashGuestEdges() const;
 

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -82,9 +82,6 @@ struct ImageInfo {
 	[[nodiscard]] constexpr bool IsVolume() const noexcept {
 		return type == Prospero::ImageType::kColor3D;
 	}
-	[[nodiscard]] constexpr bool IsLayered() const noexcept {
-		return !IsVolume() && resources.layers > 1;
-	}
 	[[nodiscard]] constexpr uint32_t TransferLayers() const noexcept {
 		return IsVolume() ? extent.depth : resources.layers;
 	}
@@ -97,9 +94,24 @@ struct ImageInfo {
 		       bytes_per_block == other.bytes_per_block;
 	}
 	[[nodiscard]] int32_t MipOf(const ImageInfo& container) const noexcept {
-		if (!IsCompatible(container) || tile_mode != container.tile_mode || resources.levels != 1 ||
-		    container.resources.layers == 0 ||
-		    container.resources.levels > container.mip_layout.size()) {
+		if (container.resources.levels > container.mip_layout.size()) {
+			return -1;
+		}
+		for (uint32_t level = 0; level < container.resources.levels; level++) {
+			if (SliceOf(container, static_cast<int32_t>(level)) >= 0) {
+				return static_cast<int32_t>(level);
+			}
+		}
+		return -1;
+	}
+	[[nodiscard]] int32_t SliceOf(const ImageInfo& container, int32_t mip) const noexcept {
+		if (!IsCompatible(container) || tile_mode != container.tile_mode || type != container.type ||
+		    IsVolume() || resources.levels != 1 || resources.layers == 0 ||
+		    container.resources.layers == 0 || mip < 0 ||
+		    static_cast<uint32_t>(mip) >= container.resources.levels ||
+		    container.resources.levels > container.mip_layout.size() ||
+		    !data.Valid() || !container.data.Valid() || data.address < container.data.address ||
+		    data.End() > container.data.End()) {
 			return -1;
 		}
 		if (HasStencil() != container.HasStencil() ||
@@ -108,75 +120,36 @@ struct ImageInfo {
 			return -1;
 		}
 
-		int32_t mip = -1;
-		for (uint32_t level = 0; level < container.resources.levels; level++) {
-			const auto& layout = container.mip_layout[level];
-			if (layout.size == 0 || layout.size % container.resources.layers != 0 ||
-			    container.data.address > UINT64_MAX - layout.offset) {
-				continue;
-			}
-			const auto mip_base   = container.data.address + layout.offset;
-			const auto slice_size = layout.size / container.resources.layers;
-			if (slice_size == 0 || mip_base > UINT64_MAX - layout.size) {
-				continue;
-			}
-			const auto mip_end = mip_base + layout.size;
-			if (data.address >= mip_base && data.address < mip_end &&
-			    (data.address - mip_base) % slice_size == 0) {
-				mip = static_cast<int32_t>(level);
-				break;
-			}
-		}
-		if (mip < 0) {
+		const auto  level  = static_cast<uint32_t>(mip);
+		const auto& layout = container.mip_layout[level];
+		if (extent.width != std::max(container.extent.width >> level, 1u) ||
+		    extent.height != std::max(container.extent.height >> level, 1u) ||
+		    extent.depth != container.extent.depth || mip_layout[0].pitch != layout.pitch ||
+		    mip_layout[0].height != layout.height || layout.size == 0 ||
+		    layout.size % container.resources.layers != 0 ||
+		    container.data.size % container.resources.layers != 0 ||
+		    data.size % resources.layers != 0) {
 			return -1;
 		}
 
-		const auto level = static_cast<uint32_t>(mip);
-		if (extent.width != std::max(container.extent.width >> level, 1u) ||
-		    extent.height != std::max(container.extent.height >> level, 1u)) {
+		// PS5 block slices contain the entire reversed mip chain.
+		const auto slice_stride = container.data.size / container.resources.layers;
+		const auto child_stride = data.size / resources.layers;
+		if (child_stride != layout.size / container.resources.layers ||
+		    layout.offset >= slice_stride || child_stride > slice_stride - layout.offset ||
+		    (resources.layers > 1 && child_stride != slice_stride)) {
 			return -1;
 		}
-		const auto mip_depth = std::max(container.extent.depth >> level, 1u);
-		if (container.type == Prospero::ImageType::kColor3D &&
-		    type == Prospero::ImageType::kColor2D) {
-			if (resources.layers != mip_depth) {
-				return -1;
-			}
-		} else if (type != container.type) {
+		const auto address_delta = data.address - container.data.address;
+		if (address_delta < layout.offset || (address_delta - layout.offset) % slice_stride != 0) {
 			return -1;
 		}
-		return mip;
-	}
-	[[nodiscard]] int32_t SliceOf(const ImageInfo& container, int32_t mip) const noexcept {
-		if (!IsCompatible(container) || type != container.type || mip < 0 ||
-		    static_cast<uint32_t>(mip) >= container.resources.levels ||
-		    container.resources.levels > container.mip_layout.size() ||
-		    container.resources.layers == 0 || data.size == 0) {
+		const auto layer = (address_delta - layout.offset) / slice_stride;
+		if (layer > INT32_MAX || layer >= container.resources.layers ||
+		    resources.layers > container.resources.layers - layer) {
 			return -1;
 		}
-		const auto level = static_cast<uint32_t>(mip);
-		if (extent.width != std::max(container.extent.width >> level, 1u) ||
-		    extent.height != std::max(container.extent.height >> level, 1u)) {
-			return -1;
-		}
-		const auto& layout = container.mip_layout[level];
-		if (layout.size == 0 || layout.size % container.resources.layers != 0 ||
-		    container.data.address > UINT64_MAX - layout.offset) {
-			return -1;
-		}
-		const auto slice_size = layout.size / container.resources.layers;
-		if (slice_size == 0 || data.size % slice_size != 0) {
-			return -1;
-		}
-		const auto mip_base = container.data.address + layout.offset;
-		if (data.address < mip_base) {
-			return -1;
-		}
-		const auto address_delta = data.address - mip_base;
-		if (address_delta % data.size != 0 || address_delta / data.size > INT32_MAX) {
-			return -1;
-		}
-		return static_cast<int32_t>(address_delta / data.size);
+		return static_cast<int32_t>(layer);
 	}
 };
 
@@ -342,14 +315,6 @@ ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t
 	}
 }
 
-[[nodiscard]] inline constexpr bool
-CanUseVideoOutNativeWithoutUpload(VideoOutCompression compression, bool render_target,
-                                  bool gpu_modified, bool guest_modified) noexcept {
-	return compression != VideoOutCompression::Uncompressed &&
-	       compression != VideoOutCompression::Unsupported && !guest_modified &&
-	       (render_target || gpu_modified);
-}
-
 struct VideoOutPixelFormatInfo {
 	vk::Format             format            = vk::Format::eUndefined;
 	Prospero::BufferFormat guest_format      = Prospero::BufferFormat::kInvalid;
@@ -400,32 +365,6 @@ inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {
 	return false;
 }
 
-[[nodiscard]] inline constexpr bool
-IsSupportedDisplayRenderTargetTileMode(Prospero::TileMode tile_mode) noexcept {
-	return tile_mode == Prospero::TileMode::kRenderTarget;
-}
-
-[[nodiscard]] inline constexpr bool IsSupportedStandard64RenderTarget(const ImageInfo& info) {
-	if (info.tile_mode != Prospero::TileMode::kStandard64KB || info.data.address == 0 ||
-	    (info.data.address & 0xffffu) != 0 || info.extent.width == 0 || info.extent.height == 0 ||
-	    info.bytes_per_block != 4 || info.resources.levels != 1 || info.resources.layers != 1 ||
-	    info.samples != 1) {
-		return false;
-	}
-	const auto expected_pitch =
-	    (static_cast<uint64_t>(info.extent.width) + 127u) & ~uint64_t {127u};
-	const auto padded_height =
-	    (static_cast<uint64_t>(info.extent.height) + 127u) & ~uint64_t {127u};
-	return expected_pitch <= UINT32_MAX && info.pitch == expected_pitch &&
-	       expected_pitch <= UINT64_MAX / padded_height / info.bytes_per_block &&
-	       info.data.size == expected_pitch * padded_height * info.bytes_per_block;
-}
-
-[[nodiscard]] inline constexpr bool IsTiledRenderTarget(const ImageInfo& info) noexcept {
-	return info.tile_mode == Prospero::TileMode::kRenderTarget ||
-	       IsSupportedStandard64RenderTarget(info);
-}
-
 [[nodiscard]] inline bool DecodePackedColorClear(vk::Format format, uint32_t packed,
                                                  vk::ClearColorValue& clear) {
 	vk::ClearColorValue next {};

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -322,6 +322,12 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		normalized.aspect = vk::ImageAspectFlagBits::eStencil;
 	}
 	normalized.usage = is_storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlags {};
+	for (const auto& cached: views) {
+		if (cached.info == normalized) {
+			return cached.view;
+		}
+	}
+
 	const bool format_compatible = normalized.format != vk::Format::eUndefined &&
 	                               ImageViewOps::FormatsCompatible(image.format, normalized.format);
 	const bool slice_view =
@@ -351,12 +357,6 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     image.layers);
 	}
 
-	for (const auto& cached: views) {
-		if (cached.info == normalized) {
-			return cached.view;
-		}
-	}
-
 	vk::ImageViewUsageCreateInfo usage {};
 	usage.usage = image.usage;
 	if (!is_storage) {

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/gpu_format.h"
@@ -8,9 +9,8 @@
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
-#include <bit>
+#include <array>
 #include <cinttypes>
-#include <cstring>
 
 namespace Libs::Graphics {
 namespace {
@@ -28,13 +28,13 @@ constexpr Prospero::ColorComponentMapping kRenderTargetColorMappings[4][4] = {
      Prospero::ColorMappingArgb},
 };
 
-struct RenderTargetHostFormatInfo {
+struct HostFormatInfo {
 	vk::Format                      format = vk::Format::eUndefined;
 	Prospero::ColorComponentMapping host_to_storage;
 };
 
-RenderTargetHostFormatInfo ResolveRenderTargetHostFormat(Prospero::BufferFormat guest_format,
-                                                         Prospero::ChannelOrder order) {
+HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
+                                 Prospero::ChannelOrder order) {
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k8_8_8_8UNorm:
@@ -48,17 +48,77 @@ RenderTargetHostFormatInfo ResolveRenderTargetHostFormat(Prospero::BufferFormat
 			default: break;
 		}
 	}
+	const auto format = VulkanFormat(guest_format);
 	switch (guest_format) {
-		case Prospero::BufferFormat::k5_5_5_1UNorm:
-			return {vk::Format::eA1R5G5B5UnormPack16, Prospero::ColorMappingBgra};
+		case Prospero::BufferFormat::k5_5_5_1UNorm: return {format, Prospero::ColorMappingBgra};
 		case Prospero::BufferFormat::k1_5_5_5UNorm:
-			return {vk::Format::eR5G5B5A1UnormPack16, Prospero::ColorMappingAbgr};
-		case Prospero::BufferFormat::k4_4_4_4UNorm:
-			return {vk::Format::eR4G4B4A4UnormPack16, Prospero::ColorMappingAbgr};
-		default: return {VulkanFormat(guest_format), {}};
+		case Prospero::BufferFormat::k4_4_4_4UNorm: return {format, Prospero::ColorMappingAbgr};
+		default: return {format, {}};
 	}
 }
 
+uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) {
+	return volume_texture ? std::max(depth >> level, 1u) : depth;
+}
+
+size_t GetTextureRegionCount(uint32_t depth, uint32_t levels, bool volume_texture) {
+	size_t count = 0;
+	for (uint32_t level = 0; level < levels; level++) {
+		count += GetTextureLevelDepth(depth, level, volume_texture);
+	}
+	return count;
+}
+
+uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint32_t levels,
+                                uint64_t total_size, uint32_t depth) {
+	uint64_t stride = 0;
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		stride = std::max(stride, mips[mip_level].offset + mips[mip_level].size);
+	}
+
+	if (depth > 1 && total_size != 0 && total_size % depth == 0) {
+		stride = std::max(stride, total_size / depth);
+	}
+
+	return stride;
+}
+
+uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch,
+                                   uint32_t height) {
+	const uint32_t element_columns =
+	    std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u);
+	const uint32_t element_rows =
+	    std::max((height + element.texel_height - 1u) / element.texel_height, 1u);
+	return static_cast<uint64_t>(element_columns) * element_rows * element.bytes;
+}
+
+uint64_t SetLinearUploadLevels(TextureUploadMipLayout*         mips,
+                               const TileTextureElementLayout& element, uint32_t height,
+                               uint32_t levels, uint32_t base_pitch) {
+	uint64_t offset     = 0;
+	auto     mip_pitch  = base_pitch;
+	auto     mip_height = height;
+
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		const auto size = CalcLinearUploadLevelSize(element, mip_pitch, mip_height);
+		mips[mip_level] = {offset, size};
+
+		offset += size;
+		if (mip_pitch > 1) {
+			mip_pitch /= 2;
+		}
+		if (mip_height > 1) {
+			mip_height /= 2;
+		}
+	}
+
+	return offset;
+}
+
+bool FitsBufferRange(uint64_t offset, uint64_t size, uint64_t capacity) {
+	return offset <= capacity && size <= capacity - offset;
+}
+
 } // namespace
 
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
@@ -66,131 +126,61 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layo
                                                     Prospero::ChannelOrder  order) {
 	const auto encoding = Prospero::ResolveRenderTargetFormat(layout, type);
 	if (encoding.IsValid() && encoding.SupportsOrder(order)) {
-		const auto host  = ResolveRenderTargetHostFormat(encoding.buffer_format, order);
-		const auto bytes = Prospero::RenderTargetBytesPerElement(encoding.buffer_format);
-		if (host.format != vk::Format::eUndefined && bytes != 0) {
+		const auto host_format = ResolveHostFormat(encoding.buffer_format, order);
+		const auto bytes       = Prospero::RenderTargetBytesPerElement(encoding.buffer_format);
+		if (host_format.format != vk::Format::eUndefined && bytes != 0) {
 			const auto order_mapping =
 			    kRenderTargetColorMappings[static_cast<size_t>(order)][encoding.components - 1u];
-			return {host.format, bytes, host.host_to_storage.Then(order_mapping)};
+			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping)};
 		}
 	}
 	EXIT("unsupported render-target format combination: layout=%u type=%u order=%u\n",
 	     static_cast<uint32_t>(layout), static_cast<uint32_t>(type), static_cast<uint32_t>(order));
 }
 
-namespace {
-
-static uint32_t GetTextureLevelDepth(uint32_t depth, uint32_t level, bool volume_texture) {
-	return volume_texture ? std::max(depth >> level, 1u) : depth;
-}
-
-static size_t GetTextureRegionCount(uint32_t depth, uint64_t levels, bool volume_texture) {
-	size_t count = 0;
-	for (uint32_t level = 0; level < levels; level++) {
-		count += GetTextureLevelDepth(depth, level, volume_texture);
+vk::ComponentMapping TextureGetComponentMapping(uint32_t                        swizzle,
+                                                Prospero::ColorComponentMapping host_to_storage) {
+	constexpr std::array host_components {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eG,
+	                                      vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eA};
+	std::array<vk::ComponentSwizzle, 4> storage_components {};
+	for (uint32_t host_component = 0; host_component < host_components.size(); ++host_component) {
+		storage_components[host_to_storage.Map(host_component)] = host_components[host_component];
 	}
-	return count;
-}
-
-vk::ComponentSwizzle TextureGetComponentSwizzle(uint8_t s) {
-	switch (static_cast<Prospero::CompSwizzle>(s)) {
-		case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero;
-		case Prospero::CompSwizzle::kOne: return vk::ComponentSwizzle::eOne;
-		case Prospero::CompSwizzle::kRed: return vk::ComponentSwizzle::eR;
-		case Prospero::CompSwizzle::kGreen: return vk::ComponentSwizzle::eG;
-		case Prospero::CompSwizzle::kBlue: return vk::ComponentSwizzle::eB;
-		case Prospero::CompSwizzle::kAlpha: return vk::ComponentSwizzle::eA;
-		default: EXIT("unknown swizzle: %d\n", static_cast<int>(s));
-	}
-	return vk::ComponentSwizzle::eIdentity;
-}
-
-} // namespace
-
-vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) {
-	vk::ComponentMapping components {};
-	components.r = TextureGetComponentSwizzle(GetDstSel(swizzle, 0));
-	components.g = TextureGetComponentSwizzle(GetDstSel(swizzle, 1));
-	components.b = TextureGetComponentSwizzle(GetDstSel(swizzle, 2));
-	components.a = TextureGetComponentSwizzle(GetDstSel(swizzle, 3));
-	return components;
+	const auto resolve = [&](uint32_t component) {
+		const auto selector = static_cast<Prospero::CompSwizzle>(GetDstSel(swizzle, component));
+		switch (selector) {
+			case Prospero::CompSwizzle::kZero: return vk::ComponentSwizzle::eZero;
+			case Prospero::CompSwizzle::kOne: return vk::ComponentSwizzle::eOne;
+			case Prospero::CompSwizzle::kRed: return storage_components[0];
+			case Prospero::CompSwizzle::kGreen: return storage_components[1];
+			case Prospero::CompSwizzle::kBlue: return storage_components[2];
+			case Prospero::CompSwizzle::kAlpha: return storage_components[3];
+			default: EXIT("unknown swizzle: %u\n", static_cast<uint32_t>(selector));
+		}
+	};
+	return {resolve(0), resolve(1), resolve(2), resolve(3)};
 }
 
 SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
-	const auto backing_format    = Prospero::RemapTextureFormat(format);
-	const auto vk_format         = VulkanFormat(backing_format);
+	const auto backing_format = Prospero::RemapTextureFormat(format);
+	const auto host_format = ResolveHostFormat(backing_format, Prospero::ChannelOrder::kStandard);
 	const auto conversion_format =
 	    backing_format != format ? format : Prospero::BufferFormat::kInvalid;
-	if (vk_format != vk::Format::eUndefined) {
-		return SurfaceFormatInfo(vk_format, conversion_format);
+	if (host_format.format != vk::Format::eUndefined) {
+		return {host_format.format, conversion_format, host_format.host_to_storage};
 	}
 	EXIT("unknown format: fmt = %u\n", static_cast<uint32_t>(format));
-	return SurfaceFormatInfo(vk::Format::eUndefined, Prospero::BufferFormat::kInvalid);
 }
 
-namespace {
-
-uint64_t CalcTextureSliceStride(const TextureUploadMipLayout* mips, uint64_t levels,
-                                uint64_t total_size, uint32_t depth) {
-	uint64_t stride = 0;
-	for (uint32_t i = 0; i < levels; i++) {
-		stride = std::max(stride, mips[i].offset + mips[i].size);
-	}
-
-	if (depth > 1 && total_size != 0 && total_size % depth == 0) {
-		const auto guest_stride = total_size / depth;
-		if (guest_stride >= stride) {
-			stride = guest_stride;
-		}
-	}
-
-	return stride;
-}
-
-uint64_t CalcLinearUploadLevelSize(const TileTextureElementLayout& element, uint32_t pitch,
-                                   uint32_t height) {
-	const uint32_t elements_w =
-	    std::max((pitch + element.texel_width - 1u) / element.texel_width, 1u);
-	const uint32_t elements_h =
-	    std::max((height + element.texel_height - 1u) / element.texel_height, 1u);
-	return static_cast<uint64_t>(elements_w) * elements_h * element.bytes;
-}
-
-uint64_t SetLinearUploadLevels(TextureUploadMipLayout*         mips,
-                               const TileTextureElementLayout& element, uint64_t height,
-                               uint64_t levels, uint32_t base_pitch) {
-	uint64_t offset = 0;
-	auto     pitch  = base_pitch;
-	auto     h      = static_cast<uint32_t>(height);
-
-	for (uint32_t i = 0; i < levels; i++) {
-		const auto size = CalcLinearUploadLevelSize(element, pitch, h);
-		mips[i].size    = size;
-		mips[i].offset  = offset;
-
-		offset += size;
-		if (pitch > 1) {
-			pitch /= 2;
-		}
-		if (h > 1) {
-			h /= 2;
-		}
-	}
-
-	return offset;
-}
-
-} // namespace
-
 TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                             uint32_t height, uint32_t levels, uint32_t depth,
-                                            Prospero::TileMode tile, uint64_t upload_size,
+                                            Prospero::TileMode tile_mode, uint64_t upload_size,
                                             bool allow_depth_tile, bool volume_texture,
                                             const char* owner) {
 	TextureUploadLayout layout {};
 	layout.surface.description = {
 	    format,
-	    tile,
+	    tile_mode,
 	    volume_texture ? TileSurfaceDimension::Dim3D : TileSurfaceDimension::Dim2D,
 	    width,
 	    height,
@@ -199,7 +189,6 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 	    volume_texture ? 1u : depth,
 	};
 	const auto&              description = layout.surface.description;
-	const auto               tile_mode   = description.tile_mode;
 	TileTextureElementLayout element {};
 
 	if (format == Prospero::BufferFormat::kInvalid) {
@@ -209,34 +198,19 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 		     levels);
 	}
 
-	switch (tile_mode) {
-		case Prospero::TileMode::kLinear:
-			if (!TileGetTextureElementLayout(format, element)) {
-				EXIT("%s: unsupported linear texture format: fmt=%u\n", owner,
-				     static_cast<uint32_t>(format));
-			}
-			break;
-		case Prospero::TileMode::kDepth:
-			if (!allow_depth_tile || !TileGetTiledTextureLayout(description, layout.surface)) {
-				EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
-				     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
-				     owner, static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(description.tile_mode), upload_size, width, height,
-				     levels);
-			}
-			break;
-		default:
-			if (!TileGetTiledTextureLayout(description, layout.surface)) {
-				EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
-				     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
-				     owner, static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(description.tile_mode), upload_size, width, height,
-				     levels);
-			}
-			break;
-	}
-
-	if (tile_mode != Prospero::TileMode::kLinear) {
+	if (tile_mode == Prospero::TileMode::kLinear) {
+		if (!TileGetTextureElementLayout(format, element)) {
+			EXIT("%s: unsupported linear texture format: fmt=%u\n", owner,
+			     static_cast<uint32_t>(format));
+		}
+	} else {
+		if ((tile_mode == Prospero::TileMode::kDepth && !allow_depth_tile) ||
+		    !TileGetTiledTextureLayout(description, layout.surface)) {
+			EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
+			     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
+			     owner, static_cast<uint32_t>(format), static_cast<uint32_t>(tile_mode),
+			     upload_size, width, height, levels);
+		}
 		element = {layout.surface.texture.block.bytes_per_element,
 		           layout.surface.texture.texel_width, layout.surface.texture.texel_height};
 	}
@@ -269,40 +243,33 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout) {
 	const auto& description    = layout.surface.description;
 	const bool  volume_texture = description.dimension == TileSurfaceDimension::Dim3D;
+	const bool  linear_texture = description.tile_mode == Prospero::TileMode::kLinear;
 	const auto  depth          = volume_texture ? description.depth : description.layers;
 	uint32_t    mip_width      = description.width;
 	uint32_t    mip_height     = description.height;
-	uint32_t    mip_pitch = volume_texture && description.tile_mode != Prospero::TileMode::kLinear
-	                            ? description.width
-	                            : layout.pitch;
+	uint32_t    mip_pitch = volume_texture && !linear_texture ? description.width : layout.pitch;
 
 	std::vector<vk::BufferImageCopy> regions;
 	regions.reserve(GetTextureRegionCount(depth, description.levels, volume_texture));
-	for (uint32_t i = 0; i < description.levels; i++) {
-		EXIT_NOT_IMPLEMENTED(layout.mips[i].size == 0);
-
-		const auto mip_depth = GetTextureLevelDepth(depth, i, volume_texture);
-
-		for (uint32_t z = 0; z < mip_depth; z++) {
-			const auto          slice_offset = z * layout.slice_stride;
-			vk::BufferImageCopy region {};
-			region.bufferOffset     = layout.mips[i].offset + slice_offset;
-			region.imageSubresource = {vk::ImageAspectFlagBits::eColor, i, volume_texture ? 0u : z,
-			                           1};
-			region.imageOffset.z    = volume_texture ? static_cast<int>(z) : 0;
-			region.imageExtent      = {mip_width, mip_height, 1};
-			const bool linear       = description.tile_mode == Prospero::TileMode::kLinear;
-			if (linear) {
-				region.bufferRowLength   = layout.mips[i].row_length;
-				region.bufferImageHeight = layout.mips[i].image_height;
-			} else {
-				const auto texel_width = layout.surface.texture.texel_width;
-				const auto align       = [](uint32_t value, uint32_t block) {
-					return ((value + block - 1u) / block) * block;
-				};
-				const auto pitch       = align(mip_pitch, texel_width);
-				region.bufferRowLength = pitch > align(mip_width, texel_width) ? pitch : 0;
-			}
+	for (uint32_t mip_level = 0; mip_level < description.levels; ++mip_level) {
+		EXIT_NOT_IMPLEMENTED(layout.mips[mip_level].size == 0);
+		const auto          mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture);
+		vk::BufferImageCopy region {};
+		region.imageSubresource = {vk::ImageAspectFlagBits::eColor, mip_level, 0, 1};
+		region.imageExtent      = {mip_width, mip_height, 1};
+		if (linear_texture) {
+			region.bufferRowLength   = layout.mips[mip_level].row_length;
+			region.bufferImageHeight = layout.mips[mip_level].image_height;
+		} else {
+			const auto texel_width   = layout.surface.texture.texel_width;
+			const auto aligned_pitch = Common::AlignUp(mip_pitch, texel_width);
+			region.bufferRowLength =
+			    aligned_pitch > Common::AlignUp(mip_width, texel_width) ? aligned_pitch : 0;
+		}
+		for (uint32_t slice_index = 0; slice_index < mip_depth; ++slice_index) {
+			region.bufferOffset = layout.mips[mip_level].offset + slice_index * layout.slice_stride;
+			region.imageSubresource.baseArrayLayer = volume_texture ? 0u : slice_index;
+			region.imageOffset.z = volume_texture ? static_cast<int>(slice_index) : 0;
 			regions.push_back(region);
 		}
 
@@ -320,17 +287,9 @@ std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayo
 	return regions;
 }
 
-static bool SetGpuTileSize(uint64_t offset, uint64_t length, uint64_t capacity, uint64_t& size) {
-	if (offset > capacity || length > capacity - offset) {
-		return false;
-	}
-	size = length;
-	return true;
-}
-
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,
-                              std::vector<GpuTileInfo>& out_infos) {
+                              std::vector<GpuTileInfo>& out_tile_infos) {
 	const auto& description    = layout.surface.description;
 	const bool  volume_texture = description.dimension == TileSurfaceDimension::Dim3D;
 	const auto  depth          = volume_texture ? description.depth : description.layers;
@@ -348,111 +307,69 @@ bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferI
 		return false;
 	}
 
-	std::vector<GpuTileInfo> infos;
-	infos.reserve(regions.size());
-	if (volume_texture) {
-		size_t region_base = 0;
-		for (uint32_t level = 0; level < levels; ++level) {
-			const uint32_t mip_depth     = GetTextureLevelDepth(depth, level, true);
-			const bool     tail          = level >= surface.first_tail_level;
-			const uint64_t linear_stride = layout.slice_stride;
-			for (uint32_t z = 0; z < mip_depth; z += block.block_depth) {
-				const uint32_t copy_depth = std::min(block.block_depth, mip_depth - z);
-				const auto&    region     = regions[region_base + z];
-				const auto     pitch =
-				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
-				const auto  logical_height = region.bufferImageHeight != 0
-				                                 ? region.bufferImageHeight
-				                                 : region.imageExtent.height;
-				GpuTileInfo info {};
-				info.family            = block.family;
-				info.bytes_per_element = block.bytes_per_element;
-				info.linear_offset     = region.bufferOffset;
-				info.tiled_offset =
-				    static_cast<uint64_t>(z / block.block_depth) * surface.block_slice_size +
-				    surface.mips[level].offset;
-				const uint64_t linear_span =
-				    static_cast<uint64_t>(copy_depth - 1u) * linear_stride +
-				    layout.mips[level].size;
-				if (!SetGpuTileSize(info.linear_offset, linear_span, UINT64_MAX,
-				                    info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, surface.mips[level].size, tiled_size,
-				                    info.tiled_size)) {
-					return false;
-				}
-				info.linear_slice_stride = linear_stride;
-				info.width  = std::max((region.imageExtent.width + texture.texel_width - 1u) /
-				                           texture.texel_width,
-				                       1u);
-				info.height = std::max(
-				    (logical_height + texture.texel_height - 1u) / texture.texel_height, 1u);
-				info.depth = copy_depth;
-				info.surface_z =
+	std::vector<GpuTileInfo> tile_infos;
+	tile_infos.reserve(regions.size());
+	const auto slices_per_copy = volume_texture ? block.block_depth : 1u;
+	size_t     region_base     = 0;
+	for (uint32_t mip_level = 0; mip_level < levels; ++mip_level) {
+		const auto& mip       = surface.mips[mip_level];
+		const auto  mip_depth = GetTextureLevelDepth(depth, mip_level, volume_texture);
+		for (uint32_t slice_index = 0; slice_index < mip_depth; slice_index += slices_per_copy) {
+			const auto& region = regions[region_base + slice_index];
+			GpuTileInfo tile_info {};
+			tile_info.family            = block.family;
+			tile_info.bytes_per_element = block.bytes_per_element;
+			tile_info.linear_offset     = region.bufferOffset;
+			tile_info.linear_size       = layout.mips[mip_level].size;
+			tile_info.tiled_size        = mip.size;
+			if (volume_texture) {
+				tile_info.depth               = std::min(slices_per_copy, mip_depth - slice_index);
+				tile_info.linear_slice_stride = layout.slice_stride;
+				tile_info.linear_size += (tile_info.depth - 1u) * layout.slice_stride;
+				tile_info.tiled_offset =
+				    mip.offset +
+				    static_cast<uint64_t>(slice_index / slices_per_copy) * surface.block_slice_size;
+				tile_info.surface_z =
 				    block.block_depth == 1 ? static_cast<uint32_t>(region.imageOffset.z) : 0;
-				info.pitch = std::max((pitch + texture.texel_width - 1u) / texture.texel_width, 1u);
-				info.tail_x       = tail ? surface.mips[level].tail_x : 0;
-				info.tail_y       = tail ? surface.mips[level].tail_y : 0;
-				info.tail         = tail;
-				info.tiled_width  = surface.mips[level].padded_width;
-				info.tiled_height = surface.mips[level].padded_height;
-				infos.push_back(info);
-			}
-			region_base += mip_depth;
-		}
-	} else {
-		size_t region_index = 0;
-		for (uint32_t level = 0; level < levels; level++) {
-			const auto& level_size  = layout.mips[level];
-			const auto& mip         = surface.mips[level];
-			const bool  tail        = level >= surface.first_tail_level;
-			const auto  level_depth = GetTextureLevelDepth(depth, level, false);
-			for (uint32_t z = 0; z < level_depth; z++) {
-				const auto& region = regions[region_index++];
-				const auto  pitch =
-				    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
-				const auto  logical_height = region.bufferImageHeight != 0
-				                                 ? region.bufferImageHeight
-				                                 : region.imageExtent.height;
-				GpuTileInfo info {};
-				info.family              = block.family;
-				info.bytes_per_element   = block.bytes_per_element;
-				info.linear_offset       = region.bufferOffset;
+			} else {
 				const auto source_stride = layout.source_slice_stride != 0
 				                               ? layout.source_slice_stride
 				                               : surface.block_slice_size;
-				if (z > (UINT64_MAX - mip.offset) / source_stride) {
+				if (slice_index > (UINT64_MAX - mip.offset) / source_stride) {
 					return false;
 				}
-				info.tiled_offset = mip.offset + static_cast<uint64_t>(z) * source_stride;
-				if (!SetGpuTileSize(info.linear_offset, level_size.size, UINT64_MAX,
-				                    info.linear_size) ||
-				    !SetGpuTileSize(info.tiled_offset, mip.size, tiled_size, info.tiled_size)) {
-					return false;
-				}
-				info.width  = std::max((region.imageExtent.width + texture.texel_width - 1u) /
-				                           texture.texel_width,
-				                       1u);
-				info.height = std::max(
-				    (logical_height + texture.texel_height - 1u) / texture.texel_height, 1u);
-				info.surface_z = block.family == TileBlockFamily::RenderTarget64KB ||
-				                         block.family == TileBlockFamily::Depth64KB
-				                     ? region.imageSubresource.baseArrayLayer
-				                     : 0;
-				info.pitch = std::max((pitch + texture.texel_width - 1u) / texture.texel_width, 1u);
-				info.tail  = tail;
-				info.tail_x       = tail ? mip.tail_x : 0;
-				info.tail_y       = tail ? mip.tail_y : 0;
-				info.tiled_width  = mip.padded_width;
-				info.tiled_height = mip.padded_height;
-				infos.push_back(info);
+				tile_info.tiled_offset =
+				    mip.offset + static_cast<uint64_t>(slice_index) * source_stride;
+				tile_info.surface_z = block.family == TileBlockFamily::RenderTarget64KB ||
+				                              block.family == TileBlockFamily::Depth64KB
+				                          ? region.imageSubresource.baseArrayLayer
+				                          : 0;
 			}
+			if (!FitsBufferRange(tile_info.linear_offset, tile_info.linear_size, UINT64_MAX) ||
+			    !FitsBufferRange(tile_info.tiled_offset, tile_info.tiled_size, tiled_size)) {
+				return false;
+			}
+			const auto row_length =
+			    region.bufferRowLength != 0 ? region.bufferRowLength : region.imageExtent.width;
+			const auto image_height = region.bufferImageHeight != 0 ? region.bufferImageHeight
+			                                                        : region.imageExtent.height;
+			tile_info.width         = std::max(
+			    (region.imageExtent.width + texture.texel_width - 1u) / texture.texel_width, 1u);
+			tile_info.height =
+			    std::max((image_height + texture.texel_height - 1u) / texture.texel_height, 1u);
+			tile_info.pitch =
+			    std::max((row_length + texture.texel_width - 1u) / texture.texel_width, 1u);
+			tile_info.tail         = mip_level >= surface.first_tail_level;
+			tile_info.tail_x       = tile_info.tail ? mip.tail_x : 0;
+			tile_info.tail_y       = tile_info.tail ? mip.tail_y : 0;
+			tile_info.tiled_width  = mip.padded_width;
+			tile_info.tiled_height = mip.padded_height;
+			tile_infos.push_back(tile_info);
 		}
+		region_base += mip_depth;
 	}
 
-	if (infos.empty()) {
-		return false;
-	}
-	out_infos = std::move(infos);
+	out_tile_infos = std::move(tile_infos);
 	return true;
 }
 

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -1,9 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_HOST_GPU_RENDERER_IMAGE_TEXTURECOMMON_H_
 
-#include "common/abi.h"
-#include "common/common.h"
-#include "graphics/guest_gpu/gpu_format.h"
+#include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
@@ -20,12 +18,9 @@ struct RenderTargetFormatInfo {
 };
 
 struct SurfaceFormatInfo {
-	explicit constexpr SurfaceFormatInfo(vk::Format format,
-	                                     Prospero::BufferFormat conversion_format)
-	    : vk_format(format), conversion_format(conversion_format) {}
-
-	vk::Format             vk_format;
-	Prospero::BufferFormat conversion_format;
+	vk::Format                      vk_format;
+	Prospero::BufferFormat          conversion_format;
+	Prospero::ColorComponentMapping host_to_storage;
 };
 
 struct TextureUploadMipLayout {
@@ -43,20 +38,21 @@ struct TextureUploadLayout {
 	TextureUploadMipLayout mips[16] = {};
 };
 
-vk::ComponentMapping   TextureGetComponentMapping(uint32_t swizzle);
+vk::ComponentMapping   TextureGetComponentMapping(uint32_t                        swizzle,
+                                                  Prospero::ColorComponentMapping host_to_storage);
 SurfaceFormatInfo      TextureGetSurfaceFormatInfo(Prospero::BufferFormat format);
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
                                                     Prospero::ChannelType   type,
                                                     Prospero::ChannelOrder  order);
 TextureUploadLayout    TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                                uint32_t height, uint32_t levels, uint32_t depth,
-                                               Prospero::TileMode tile, uint64_t upload_size,
+                                               Prospero::TileMode tile_mode, uint64_t upload_size,
                                                bool allow_depth_tile, bool volume_texture,
                                                const char* owner);
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout);
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,
-                              std::vector<GpuTileInfo>& infos);
+                              std::vector<GpuTileInfo>& out_tile_infos);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/image/tiler.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "gpu_tiler_shaders/gpu_tiler_demote_d16_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_depth_spv.h"
@@ -221,7 +222,7 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 
 	const uint64_t uniform_alignment =
 	    std::max<uint64_t>(limits.minUniformBufferOffsetAlignment, 1);
-	const uint64_t stride = (sizeof(Push) + uniform_alignment - 1) & ~(uniform_alignment - 1);
+	const uint64_t stride = Common::AlignUp<uint64_t>(sizeof(Push), uniform_alignment);
 	EXIT_NOT_IMPLEMENTED(dispatches.size() > UINT64_MAX / stride);
 	const uint64_t bytes  = dispatches.size() * stride;
 	auto [mapped, offset] = m_stream_buffer.Map(bytes, uniform_alignment);
@@ -259,12 +260,8 @@ vk::Pipeline TileManager::GetPipeline(uint32_t slot) {
 	const uint32_t                   values[] {1u << element_index, direction_index};
 	const vk::SpecializationMapEntry entries[] {{0, 0, 4}, {1, 4, 4}};
 	const vk::SpecializationInfo     specialization {2, entries, sizeof(values), values};
-	vk::ShaderModuleCreateInfo       module_info {};
-	module_info.codeSize    = shaders[family_index].words * sizeof(uint32_t);
-	module_info.pCode       = shaders[family_index].code;
-	vk::ShaderModule module = nullptr;
-	RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-	                     "create TileManager shader module");
+	const auto module =
+	    CompileSPV({shaders[family_index].code, shaders[family_index].words}, m_graphics.device);
 	vk::PipelineShaderStageCreateInfo stage {};
 	stage.stage               = vk::ShaderStageFlagBits::eCompute;
 	stage.module              = module;
@@ -287,12 +284,12 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset,
 	const auto&    limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	const uint64_t descriptor_alignment =
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
-	const uint64_t source_descriptor_offset = source_offset & ~(descriptor_alignment - 1);
-	const uint64_t target_descriptor_offset = target_offset & ~(descriptor_alignment - 1);
+	const uint64_t source_descriptor_offset = Common::AlignDown(source_offset, descriptor_alignment);
+	const uint64_t target_descriptor_offset = Common::AlignDown(target_offset, descriptor_alignment);
 	const uint64_t source_base              = source_offset - source_descriptor_offset;
 	const uint64_t target_base              = target_offset - target_descriptor_offset;
-	const uint64_t source_range             = (source_base + source_capacity + 3u) & ~uint64_t {3};
-	const uint64_t target_range             = (target_base + target_capacity + 3u) & ~uint64_t {3};
+	const uint64_t source_range             = Common::AlignUp(source_base + source_capacity, 4);
+	const uint64_t target_range             = Common::AlignUp(target_base + target_capacity, 4);
 	EXIT_NOT_IMPLEMENTED(source_range > limits.maxStorageBufferRange ||
 	                     target_range > limits.maxStorageBufferRange || target_offset % 4 != 0 ||
 	                     target_capacity % 4 != 0);
@@ -372,7 +369,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
 	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
-	auto scratch = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3});
+	auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4));
 	DeferDestroy(scratch);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
@@ -406,7 +403,7 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	// Reserve all stream parameters before creating a scheduler-lived scratch dependency:
 	// StreamBuffer::Map is allowed to submit the current tick when it wraps.
 	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
-	auto linear = AllocateScratch((linear_capacity + 3u) & ~uint64_t {3});
+	auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4));
 	DeferDestroy(linear);
 	image.Download(regions, linear.buffer, 0, linear.size);
 	Result source {linear.buffer, 0, linear.size};
@@ -418,7 +415,7 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 }
 
 TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
-	auto scratch = AllocateScratch((size + 3u) & ~uint64_t {3});
+	auto scratch = AllocateScratch(Common::AlignUp(size, 4));
 	DeferDestroy(scratch);
 	return {scratch.buffer, 0, scratch.size};
 }
@@ -426,11 +423,11 @@ TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
 TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const {
 	const auto& limits            = m_graphics.GetPhysicalDeviceProperties().limits;
 	const auto  alignment         = std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
-	const auto  descriptor_offset = buffer.offset - buffer.offset % alignment;
+	const auto  descriptor_offset = Common::AlignDown(buffer.offset, alignment);
 	const auto  base              = buffer.offset - descriptor_offset;
 	EXIT_IF(buffer.buffer == nullptr || size == 0 || buffer.size < size || base > UINT32_MAX ||
 	        size > UINT64_MAX - base || base + size > UINT64_MAX - 3);
-	const auto range = (base + size + 3) & ~uint64_t {3};
+	const auto range = Common::AlignUp(base + size, 4);
 	EXIT_IF(range > limits.maxStorageBufferRange || range > UINT32_MAX);
 	return {{buffer.buffer, descriptor_offset, range}, static_cast<uint32_t>(base)};
 }
@@ -471,12 +468,7 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 			code  = GPU_TILER_DEMOTE_D16_SPV;
 			words = std::size(GPU_TILER_DEMOTE_D16_SPV);
 		}
-		vk::ShaderModuleCreateInfo module_info {};
-		module_info.codeSize    = words * sizeof(uint32_t);
-		module_info.pCode       = code;
-		vk::ShaderModule module = nullptr;
-		RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-		                     "create D16 conversion shader module");
+		const auto module = CompileSPV({code, words}, m_graphics.device);
 		vk::PipelineShaderStageCreateInfo stage {};
 		stage.stage               = vk::ShaderStageFlagBits::eCompute;
 		stage.module              = module;
@@ -512,8 +504,8 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	const auto target_required = required(layout.height, layout.layers, layout.target_row_stride,
 	                                      layout.target_slice_stride, target_active);
 	EXIT_IF(source_required > UINT64_MAX - 3 || target_required > UINT64_MAX - 3);
-	const auto source_barrier_size = (source_required + 3) & ~uint64_t {3};
-	const auto target_barrier_size = (target_required + 3) & ~uint64_t {3};
+	const auto source_barrier_size = Common::AlignUp(source_required, 4);
+	const auto target_barrier_size = Common::AlignUp(target_required, 4);
 	EXIT_IF(source.size < source_barrier_size || target.size < target_barrier_size);
 
 	m_scheduler.EndRendering();
@@ -610,12 +602,7 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 
 void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 	if (m_swap_bgra16 == nullptr) {
-		vk::ShaderModuleCreateInfo module_info {};
-		module_info.codeSize    = std::size(GPU_TILER_SWAP_BGRA16_SPV) * sizeof(uint32_t);
-		module_info.pCode       = GPU_TILER_SWAP_BGRA16_SPV;
-		vk::ShaderModule module = nullptr;
-		RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
-		                     "create BGRA16 swap shader module");
+		const auto module = CompileSPV(GPU_TILER_SWAP_BGRA16_SPV, m_graphics.device);
 		vk::PipelineShaderStageCreateInfo stage {};
 		stage.stage  = vk::ShaderStageFlagBits::eCompute;
 		stage.module = module;

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp

@@ -20,7 +20,7 @@ constexpr std::array DescriptorPoolSizes = {
 
 DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore)
     : m_graphics(graphics), m_master_semaphore(master_semaphore) {
-	CreatePool();
+	CreateDescriptorPool();
 }
 
 DescriptorHeap::~DescriptorHeap() {
@@ -49,7 +49,7 @@ vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) {
 		m_pending_pools.pop_front();
 		EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess);
 	} else {
-		CreatePool();
+		CreateDescriptorPool();
 	}
 
 	m_sets.clear();
@@ -82,7 +82,7 @@ bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) {
 	}
 }
 
-void DescriptorHeap::CreatePool() {
+void DescriptorHeap::CreateDescriptorPool() {
 	vk::DescriptorPoolCreateInfo create {};
 	create.maxSets       = DescriptorHeapCount;
 	create.poolSizeCount = static_cast<uint32_t>(DescriptorPoolSizes.size());

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h

@@ -31,7 +31,7 @@ private:
 	};
 
 	[[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch);
-	void               CreatePool();
+	void               CreateDescriptorPool();
 
 	GraphicContext&                                     m_graphics;
 	MasterSemaphore&                                    m_master_semaphore;

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -1,5 +1,6 @@
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/file.h"
@@ -14,6 +15,7 @@
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/hostMemory.h"
+#include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
@@ -92,22 +94,15 @@ static const char* ShaderStageResourceName(ShaderType stage) {
 	switch (stage) {
 		case ShaderType::Vertex: return "Vertex";
 		case ShaderType::Mesh: return "Mesh";
+		case ShaderType::Local: return "Local";
+		case ShaderType::TessellationControl: return "Hull";
+		case ShaderType::TessellationEvaluation: return "Domain";
 		case ShaderType::Pixel: return "Pixel";
 		case ShaderType::Compute: return "Compute";
 		default: return "Unknown";
 	}
 }
 
-static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) {
-	switch (stage) {
-		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
-		case ShaderType::Mesh: return vk::ShaderStageFlagBits::eMeshEXT;
-		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
-		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
-		default: EXIT("unknown native shader stage\n");
-	}
-}
-
 static Prospero::ImageType TextureType(const ShaderTextureResource& descriptor) {
 	const auto type = descriptor.Type();
 	return type == Prospero::ImageType::kCube ? Prospero::ImageType::kColor2DArray : type;
@@ -145,7 +140,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	}
 	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
 	                                                              resource.formatted, id);
-	const auto aligned_offset = offset - offset % alignment;
+	const auto aligned_offset = Common::AlignDown(offset, alignment);
 	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
 	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
@@ -153,7 +148,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
 	const vk::DescriptorBufferInfo result {buffer->Handle(), aligned_offset, size + adjustment};
-	if (resource.formatted && resource.written) {
+	if (resource.written) {
 		context.GetTextureCache().InvalidateMemoryFromGPU(address, size);
 	}
 	const char* access = "Read";
@@ -169,34 +164,6 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	return result;
 }
 
-static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) {
-	bool supported_dimension = false;
-	switch (resource.dimension) {
-		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim1DArray:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2D:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DArray:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa:
-		case ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray:
-			supported_dimension = true;
-			break;
-		default: break;
-	}
-	return resource.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled &&
-	       resource.numeric_class != Prospero::TextureNumericClass::Unsupported &&
-	       supported_dimension && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
-	       resource.read && !resource.written && !resource.atomic && !resource.depth_compare;
-}
-
-bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
-                                    const ShaderTextureResource& descriptor, const Image& image) {
-	return image.usage.video_out && image.info.resources.layers == 1 &&
-	       IsSupportedSampledColorResource(resource) &&
-	       resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	       descriptor.Type() == Prospero::ImageType::kColor2D && descriptor.Depth() == 0 &&
-	       descriptor.BaseArray5() == 0;
-}
-
 bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bool r128) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
@@ -483,17 +450,18 @@ static void PopulateTextureMipLayout(ImageInfo& info) {
 
 static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource,
                                      const ShaderTextureResource& descriptor, vk::Format format,
-                                     bool shader_conversion, bool storage, uint32_t view_levels,
-                                     uint32_t image_layers) {
+                                     const SurfaceFormatInfo& surface_format, bool storage,
+                                     uint32_t view_levels, uint32_t image_layers) {
 	ImageViewInfo view {};
 	view.format      = format;
 	view.aspect      = vk::ImageAspectFlagBits::eColor;
 	view.base_level  = descriptor.BaseLevel();
 	view.level_count = view_levels;
-	view.usage   = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
-	view.mapping = storage || shader_conversion
-	                   ? vk::ComponentMapping {}
-	                   : TextureGetComponentMapping(descriptor.DstSelXYZW());
+	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
+	view.mapping =
+	    storage || surface_format.conversion_format != Prospero::BufferFormat::kInvalid
+	        ? vk::ComponentMapping {}
+	        : TextureGetComponentMapping(descriptor.DstSelXYZW(), surface_format.host_to_storage);
 	switch (resource.dimension) {
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
 			view.type       = vk::ImageViewType::e1D;
@@ -672,7 +640,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	} else {
 		PopulateTextureMipLayout(desc.info);
 	}
-	desc.view_info = TextureViewInfo(resource, descriptor, view_format, shader_conversion, storage,
+	desc.view_info = TextureViewInfo(resource, descriptor, view_format, surface_format, storage,
 	                                 view_levels, desc.info.resources.layers);
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
 
@@ -720,13 +688,6 @@ static vk::DescriptorBufferInfo NativeUpload(RenderContext&            context,
 	return {buffer.Handle(), offset, data.size_bytes()};
 }
 
-void RenderExecutor::TrackImageBinding(ImageId id) {
-	EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr);
-	if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) {
-		m_bound_images.push_back(id);
-	}
-}
-
 void RenderExecutor::BindImage(ImageId id, bool storage) {
 	auto& image = m_context.GetTextureCache().GetImage(id);
 	if (image.info.data.Empty()) {
@@ -737,13 +698,13 @@ void RenderExecutor::BindImage(ImageId id, bool storage) {
 	}
 	image.binding.is_bound = true;
 	image.binding.shader_write |= storage;
-	TrackImageBinding(id);
+	m_bound_images.push_back(id);
 }
 
 void RenderExecutor::BindRenderTarget(ImageId id) {
 	auto& image             = m_context.GetTextureCache().GetImage(id);
 	image.binding.is_target = true;
-	TrackImageBinding(id);
+	m_bound_images.push_back(id);
 }
 
 void RenderExecutor::ResetBindings() {
@@ -796,10 +757,7 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[i]);
 		const auto address = descriptor.Base48();
-		const auto stride  = descriptor.Stride();
-		const auto records = descriptor.NumRecords();
-		// The descriptor has a 14-bit stride and 32-bit record count, so the product fits u64.
-		const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+		const auto requested_size = descriptor.GetSize();
 		if (address == 0 || requested_size == 0) {
 			prepared.buffer_sources.push_back({});
 			continue;
@@ -893,32 +851,39 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	}
 }
 
-RenderExecutor::GraphicsBindings
-RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
-                                        const ShaderStageRuntime& pixel, bool pixel_active) {
-	GraphicsBindings bindings {
-	    .vertex = PrepareBindings(vertex),
-	};
-	if (pixel_active) {
-		bindings.pixel.emplace(PrepareBindings(pixel));
+void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
+                                             std::span<RenderColorInfo> colors) {
+	bool uses_dma = false;
+	for (auto* stage: stages) {
+		FindBuffers(*stage);
+		uses_dma |= stage->runtime->program->info.uses_dma;
 	}
-	FindBuffers(bindings.vertex);
-	if (bindings.pixel) {
-		FindBuffers(*bindings.pixel);
+	if (uses_dma) {
+		m_context.PrepareBda();
 	}
-	if (bindings.vertex.runtime->program->info.uses_dma ||
-	    (bindings.pixel && bindings.pixel->runtime->program->info.uses_dma)) {
-		m_context.GetGpuResources().PrepareBda();
+	for (auto* stage: stages) {
+		RebindImages(*stage);
 	}
-	RebindBuffers(bindings.vertex);
-	if (bindings.pixel) {
-		RebindBuffers(*bindings.pixel);
+	auto& cache = m_context.GetTextureCache();
+	for (auto& target: colors) {
+		EXIT_IF(!target.image_id);
+		const auto old_image = cache.m_slot_images.try_get(target.image_id);
+		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
+		    old_image->binding.needs_rebind) {
+			if (old_image != nullptr) {
+				old_image->binding = {};
+			}
+			target.desc.view_info.base_level = target.guest_mip_level;
+			target.desc.view_info.base_layer = target.guest_array_layer;
+			target.image_id = cache.FindImage(target.desc);
+			BindRenderTarget(target.image_id);
+		}
 	}
-	RebindImages(bindings.vertex);
-	if (bindings.pixel) {
-		RebindImages(*bindings.pixel);
+	// Discovery can read back PS5 metadata and submit the scheduler. Reserve draw buffers only
+	// after image identities are final; attachment layout transitions follow buffer alias copies.
+	for (auto* stage: stages) {
+		RebindBuffers(*stage);
 	}
-	return bindings;
 }
 
 void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
@@ -931,9 +896,10 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 	size_t write_count      = 0;
 	ShaderRecompiler::IR::PushData push_data;
 	bool                           has_push_data = false;
-	constexpr auto                 GraphicsStages = vk::ShaderStageFlagBits::eVertex |
-	                                                vk::ShaderStageFlagBits::eMeshEXT |
-	                                                vk::ShaderStageFlagBits::eFragment;
+	constexpr auto                 GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eMeshEXT |
+	    vk::ShaderStageFlagBits::eTessellationControl |
+	    vk::ShaderStageFlagBits::eTessellationEvaluation | vk::ShaderStageFlagBits::eFragment;
 	vk::ShaderStageFlags push_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics
 	                                       ? vk::ShaderStageFlagBits::eFragment
 	                                       : vk::ShaderStageFlags {};

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -66,9 +66,6 @@ template <typename T>
 
 [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
                                                    bool r128 = false);
-[[nodiscard]] bool
-IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
-                               const ShaderTextureResource& descriptor, const Image& image);
 void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
                             const ShaderTextureResource& descriptor, uint64_t size);
 

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -89,12 +89,7 @@ template <typename... Args>
 void PipelineCacheLog(fmt::format_string<Args...> format, Args&&... args) {
 	auto message = fmt::format(format, std::forward<Args>(args)...);
 	message += '\n';
-	if (Log::GetDirection() != Log::Direction::Console) {
-		std::fwrite(message.data(), 1, message.size(), stdout);
-		std::fflush(stdout);
-	}
-	Log::Write(message);
-	Log::Flush();
+	Log::WriteToConsoleAndLog(message);
 }
 
 bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) {
@@ -236,6 +231,9 @@ struct PipelineCache::ProgramCache {
 		switch (options.stage) {
 			case ShaderType::Vertex: stage_name = "vs"; break;
 			case ShaderType::Mesh: stage_name = "ms"; break;
+			case ShaderType::Local: stage_name = "ls"; break;
+			case ShaderType::TessellationControl: stage_name = "hs"; break;
+			case ShaderType::TessellationEvaluation: stage_name = "ds"; break;
 			case ShaderType::Pixel: stage_name = "ps"; break;
 			case ShaderType::Compute: stage_name = "cs"; break;
 			default: EXIT("invalid pipeline shader stage\n");
@@ -250,18 +248,9 @@ struct PipelineCache::ProgramCache {
 		}
 		DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
 
-		vk::ShaderModuleCreateInfo create_info {};
-		create_info.codeSize    = result.spirv.size() * sizeof(uint32_t);
-		create_info.pCode       = result.spirv.data();
-		vk::ShaderModule module = nullptr;
-		RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module),
-		                     "create recompiled shader module");
+		const auto module = CompileSPV(result.spirv, device);
 		EXIT_IF(module == nullptr);
 		if (options.dump_ir) {
-			if (!options.early_dump) {
-				LOGF("%s decoded RDNA2:\n%s", options.dump_label, result.decoded_dump.c_str());
-				LOGF("%s IR:\n%s", options.dump_label, result.ir_dump.c_str());
-			}
 			LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", options.dump_label,
 			     static_cast<uint64_t>(result.spirv.size()), options.wave_size);
 		}
@@ -277,7 +266,7 @@ struct PipelineCache::ProgramCache {
 	                  uint32_t& push_data_cursor) {
 		ShaderType stage;
 		if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
-			stage = input_info.mesh.threads_num[0] != 0 ? ShaderType::Mesh : ShaderType::Vertex;
+			stage = input_info.logical_stage;
 		} else if constexpr (std::is_same_v<InputInfo, ShaderPixelInputInfo>) {
 			stage = ShaderType::Pixel;
 		} else {
@@ -329,6 +318,9 @@ struct PipelineCache::ProgramCache {
 		switch (stage) {
 			case ShaderType::Vertex: label = "ShaderRecompiler VS"; break;
 			case ShaderType::Mesh: label = "ShaderRecompiler MS"; break;
+			case ShaderType::Local: label = "ShaderRecompiler LS"; break;
+			case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break;
+			case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break;
 			case ShaderType::Pixel: label = "ShaderRecompiler PS"; break;
 			case ShaderType::Compute: label = "ShaderRecompiler CS"; break;
 			default: EXIT("invalid pipeline shader stage\n");
@@ -338,19 +330,18 @@ struct PipelineCache::ProgramCache {
 		options.shader_hash = params.hash;
 		options.user_data   = params.user_data;
 		options.back_code      = params.back_code;
-		options.dump_ir     = Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent;
+		options.dump_ir     = Config::GetShaderLogDirection() != Config::LogDirection::Silent;
 		options.early_dump  = options.dump_ir;
 		options.dump_label  = label;
 		options.input_info  = stage_input;
-		options.scratch_dwords = input_info.scratch_size_dwords;
+
 		if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
 			options.user_data_base = 8;
-			if (stage == ShaderType::Mesh) {
+			if (stage == ShaderType::Mesh || stage == ShaderType::TessellationControl) {
 				options.user_data_base = 0;
-				options.wave_size      = input_info.mesh.wave_size;
-				options.scratch_dwords = input_info.mesh.scratch_size_dwords;
+				options.wave_size = stage == ShaderType::Mesh ? input_info.mesh.wave_size : 64u;
 			}
-		} else if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
+		} else {
 			options.wave_size = input_info.wave_size;
 		}
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
@@ -366,16 +357,19 @@ struct PipelineCache::ProgramCache {
 		input_info.stage = {.program = &permutation.program, .resources = std::move(resources)};
 		permutation.program.bindings.AdvancePushData(push_data_cursor);
 
-		std::array<size_t, static_cast<size_t>(ShaderType::Mesh) + 1> counts {};
+		std::array<size_t, static_cast<size_t>(ShaderType::TessellationEvaluation) + 1> counts {};
 		for (const auto& [key, source]: programs) {
 			counts[static_cast<size_t>(key.stage)] += source.permutations.size();
 		}
 		// Guest geometry shaders are compiled through the host mesh stage.
-		std::printf("Shaders: VS %zu | PS %zu | CS %zu | GS %zu\n",
+		std::printf("Shaders: VS %zu | PS %zu | CS %zu | GS %zu | LS %zu | HS %zu | TES %zu\n",
 		            counts[static_cast<size_t>(ShaderType::Vertex)],
 		            counts[static_cast<size_t>(ShaderType::Pixel)],
 		            counts[static_cast<size_t>(ShaderType::Compute)],
-		            counts[static_cast<size_t>(ShaderType::Mesh)]);
+		            counts[static_cast<size_t>(ShaderType::Mesh)],
+		            counts[static_cast<size_t>(ShaderType::Local)],
+		            counts[static_cast<size_t>(ShaderType::TessellationControl)],
+		            counts[static_cast<size_t>(ShaderType::TessellationEvaluation)]);
 		return permutation.handle;
 	}
 
@@ -569,12 +563,18 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
     const HW::VertexShaderInfo& vertex_regs, const HW::PixelShaderInfo& pixel_regs,
     const HW::ShaderRegisters& sh, const HW::Context& context, const HW::UserConfig& user_config,
     std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping, bool pixel_active,
-    ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info) {
-	const auto vertex_params = PrepareProgram(vertex_regs, context, user_config, vertex_info);
-	const bool mesh_active   = vertex_info.mesh.threads_num[0] != 0;
+    std::array<ShaderVertexInputInfo, 3>& vertex_info, ShaderPixelInputInfo& pixel_info) {
+	const bool tess_active = user_config.GetPrimType() == Prospero::PrimitiveType::kPatch;
+	std::array<ShaderParams, 3> vertex_params;
+	if (tess_active) {
+		vertex_params = PrepareTessellationPrograms(vertex_regs, context, vertex_info);
+	} else {
+		vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]);
+	}
+	const bool mesh_active = vertex_info[0].logical_stage == ShaderType::Mesh;
 	if (mesh_active) {
 		EXIT_NOT_IMPLEMENTED(!m_graphics.mesh_shader_enabled);
-		auto& mesh              = vertex_info.mesh;
+		auto& mesh              = vertex_info[0].mesh;
 		mesh.host_subgroup_size = m_graphics.subgroup_size;
 		const auto& limits      = m_graphics.mesh_shader_properties;
 		const auto  logical_threads =
@@ -597,7 +597,7 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 	if (context.GetClipControl().clip_disable) {
 		const auto& viewport = context.GetScreenViewport().viewports[0];
 		const auto& limits   = m_graphics.GetPhysicalDeviceProperties().limits;
-		auto&       clip     = vertex_info.clip_space;
+		auto&       clip     = vertex_info[tess_active ? 2u : 0u].clip_space;
 		clip.scale[0]        = viewport.xscale;
 		clip.scale[1]        = viewport.yscale;
 		clip.offset[0]       = viewport.xoffset;
@@ -615,7 +615,9 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 	if (pixel_active) {
 		result.pixel = m_program_cache->Get(pixel_params, pixel_info, push_data_cursor);
 	}
-	result.vertex = m_program_cache->Get(vertex_params, vertex_info, push_data_cursor);
+	for (uint32_t i = 0; i < (tess_active ? 3u : 1u); i++) {
+		result.vertex[i] = m_program_cache->Get(vertex_params[i], vertex_info[i], push_data_cursor);
+	}
 	return result;
 }
 
@@ -633,12 +635,14 @@ bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other)
 	return std::memcmp(this, &other, sizeof(*this)) == 0;
 }
 
-PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
+PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
     std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
-    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+    std::span<const ShaderVertexInputInfo> vertex_info, CommandBuffer& command,
     const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
-    bool primitive_restart_enable, const ShaderProgram& vertex_program,
-    const ShaderProgram& pixel_program) {
+    bool primitive_restart_enable, const GraphicsPrograms& programs) {
+	const auto& vs_input_info  = vertex_info.front();
+	const auto& vertex_program = programs.vertex[0];
+	const auto& pixel_program  = programs.pixel;
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200);
 
 	EXIT_IF(colors.size() > RENDER_COLOR_ATTACHMENTS_MAX);
@@ -656,7 +660,9 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 	const auto ps_id = ps_active ? pixel_program.id : 0;
 
 	GraphicsPipelineKey key {};
-	key.vs_shader_id            = vs_id;
+	for (uint32_t i = 0; i < programs.vertex.size(); i++) {
+		key.vertex_shader_ids[i] = programs.vertex[i].id;
+	}
 	key.ps_shader_id            = ps_id;
 	auto& static_params         = key.static_params;
 	auto& rendering             = key.rendering;
@@ -673,6 +679,16 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 			EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
 			     attachment_samples, colors[i].desc.info.samples);
 		}
+		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
+		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
+		static_params.color_srcblend[i]       = bc.color_srcblend;
+		static_params.color_comb_fcn[i]       = bc.color_comb_fcn;
+		static_params.color_destblend[i]      = bc.color_destblend;
+		static_params.alpha_srcblend[i]       = bc.alpha_srcblend;
+		static_params.alpha_comb_fcn[i]       = bc.alpha_comb_fcn;
+		static_params.alpha_destblend[i]      = bc.alpha_destblend;
+		static_params.separate_alpha_blend[i] = bc.separate_alpha_blend;
+		static_params.blend_enable[i]         = bc.enable && !rt.info.blend_bypass;
 	}
 	const bool with_depth =
 	    depth.desc.view_info.format != vk::Format::eUndefined && static_cast<bool>(depth.image_id);
@@ -721,10 +737,8 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 	static_params.depth_bounds_test_enable = depth.depth_bounds_test_enable;
 	static_params.depth_min_bounds         = depth.depth_min_bounds;
 	static_params.depth_max_bounds         = depth.depth_max_bounds;
-	static_params.stencil_test_enable      = depth.stencil_test_enable;
-	static_params.stencil_front            = depth.stencil_static_front;
-	static_params.stencil_back             = depth.stencil_static_back;
-	const bool rect_list     = topology == vk::PrimitiveTopology::ePatchList;
+	const bool rect_list =
+	    command.GetUserConfig().GetPrimType() == Prospero::PrimitiveType::kRectList;
 	static_params.cull_back  = !rect_list && mc.cull_back;
 	static_params.cull_front = !rect_list && mc.cull_front;
 	static_params.face       = mc.face;
@@ -732,19 +746,6 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 	static_params.polygon_mode =
 	    ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back);
 
-	for (uint32_t i = 0; i < color_count; i++) {
-		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
-		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		static_params.color_srcblend[i]       = bc.color_srcblend;
-		static_params.color_comb_fcn[i]       = bc.color_comb_fcn;
-		static_params.color_destblend[i]      = bc.color_destblend;
-		static_params.alpha_srcblend[i]       = bc.alpha_srcblend;
-		static_params.alpha_comb_fcn[i]       = bc.alpha_comb_fcn;
-		static_params.alpha_destblend[i]      = bc.alpha_destblend;
-		static_params.separate_alpha_blend[i] = bc.separate_alpha_blend;
-		static_params.blend_enable[i]         = bc.enable;
-		static_params.blend_bypass[i]         = rt.info.blend_bypass;
-	}
 	if (vs_input_info.stage.program->stage != ShaderType::Mesh) {
 		EXIT_IF(vs_input_info.buffers_num < 0 ||
 		        vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX ||
@@ -788,9 +789,8 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 
 	auto cached = std::make_unique<Pipeline>();
 	LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id);
-	CreatePipelineInternal(m_graphics, *cached, rendering, key.vertex_input, vs_input_info,
-	                       vertex_program, ps_input_info, pixel_program, static_params,
-	                       m_driver_cache);
+	CreatePipelineInternal(m_graphics, *cached, rendering, key.vertex_input, vertex_info,
+	                       ps_input_info, programs, static_params, m_driver_cache);
 	LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
@@ -803,8 +803,8 @@ PipelineCache::Pipeline& PipelineCache::CreateGraphicsPipeline(
 }
 
 PipelineCache::Pipeline&
-PipelineCache::CreateComputePipeline(const ShaderComputeInputInfo& input_info,
-                                     const ShaderProgram&          compute_program) {
+PipelineCache::GetComputePipeline(const ShaderComputeInputInfo& input_info,
+                                  const ShaderProgram&          compute_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100);
 
 	EXIT_IF(!compute_program);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -42,9 +42,6 @@ struct PipelineStaticParameters {
 	bool                       depth_bounds_test_enable = false;
 	float                      depth_min_bounds         = 0.0f;
 	float                      depth_max_bounds         = 0.0f;
-	bool                       stencil_test_enable      = false;
-	PipelineStencilStaticState stencil_front;
-	PipelineStencilStaticState stencil_back;
 	uint32_t                   color_mask[RENDER_COLOR_ATTACHMENTS_MAX]           = {};
 	bool                       cull_front                                         = false;
 	bool                       cull_back                                          = false;
@@ -59,7 +56,6 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
-	bool                       blend_bypass[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
 
 	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
@@ -69,7 +65,7 @@ struct PipelineStaticParameters {
 static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
 static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 166);
+static_assert(sizeof(PipelineStaticParameters) == 125);
 
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};
@@ -122,8 +118,10 @@ public:
 	};
 
 	struct GraphicsPrograms {
-		ShaderProgram vertex;
+		std::array<ShaderProgram, 3> vertex;
 		ShaderProgram pixel;
+
+		[[nodiscard]] uint32_t VertexStageCount() const { return vertex[1] ? 3u : 1u; }
 	};
 
 	GraphicsPrograms
@@ -131,33 +129,33 @@ public:
 	                    const HW::PixelShaderInfo& pixel_regs, const HW::ShaderRegisters& sh,
 	                    const HW::Context& context, const HW::UserConfig& user_config,
 	                    std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
-	                    bool pixel_active, ShaderVertexInputInfo& vertex_info,
+	                    bool pixel_active, std::array<ShaderVertexInputInfo, 3>& vertex_info,
 	                    ShaderPixelInputInfo& pixel_info);
 	ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs,
 	                                const HW::ShaderRegisters&   sh,
 	                                ShaderComputeInputInfo&      input_info);
 
-	Pipeline&
-	CreateGraphicsPipeline(std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
-	                       const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
-	                       const ShaderPixelInputInfo* ps_input_info,
-	                       vk::PrimitiveTopology topology, bool primitive_restart_enable,
-	                       const ShaderProgram& vertex_program, const ShaderProgram& pixel_program);
-	Pipeline& CreateComputePipeline(const ShaderComputeInputInfo& input_info,
-	                                const ShaderProgram&          compute_program);
+	Pipeline& GetGraphicsPipeline(std::span<const RenderColorInfo>       colors,
+	                              const RenderDepthInfo&                 depth,
+	                              std::span<const ShaderVertexInputInfo> vertex_info,
+	                              CommandBuffer& command, const ShaderPixelInputInfo* ps_input_info,
+	                              vk::PrimitiveTopology topology, bool primitive_restart_enable,
+	                              const GraphicsPrograms& programs);
+	Pipeline& GetComputePipeline(const ShaderComputeInputInfo& input_info,
+	                             const ShaderProgram&          compute_program);
 
 private:
 	struct ProgramCache;
 
 	struct GraphicsPipelineKey {
 		PipelineRenderingState   rendering;
-		uint64_t                 vs_shader_id = 0;
+		std::array<uint64_t, 3>  vertex_shader_ids {};
 		uint64_t                 ps_shader_id = 0;
 		PipelineVertexInputState vertex_input;
 		PipelineStaticParameters static_params;
 
 		bool operator==(const GraphicsPipelineKey& other) const {
-			return rendering == other.rendering && vs_shader_id == other.vs_shader_id &&
+			return rendering == other.rendering && vertex_shader_ids == other.vertex_shader_ids &&
 			       ps_shader_id == other.ps_shader_id && vertex_input == other.vertex_input &&
 			       static_params == other.static_params;
 		}
@@ -190,7 +188,9 @@ private:
 		std::size_t operator()(const GraphicsPipelineKey& key) const {
 			std::size_t hash = 0;
 			PipelineKeyHash::MixRendering(hash, key.rendering);
-			PipelineKeyHash::Mix(hash, key.vs_shader_id);
+			for (const auto id: key.vertex_shader_ids) {
+				PipelineKeyHash::Mix(hash, id);
+			}
 			PipelineKeyHash::Mix(hash, key.ps_shader_id);
 			PipelineKeyHash::Mix(hash, key.vertex_input.binding_count);
 			for (uint32_t i = 0; i < key.vertex_input.binding_count; i++) {
@@ -220,12 +220,14 @@ private:
 };
 
 void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id);
-void CreatePipelineInternal(
-    GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-    const PipelineRenderingState& rendering, const PipelineVertexInputState& vertex_input,
-    const ShaderVertexInputInfo& vs_input_info, const ShaderProgram& vertex_program,
-    const ShaderPixelInputInfo* ps_input_info, const ShaderProgram& pixel_program,
-    const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                            const PipelineRenderingState&          rendering,
+                            const PipelineVertexInputState&        vertex_input,
+                            std::span<const ShaderVertexInputInfo> vertex_info,
+                            const ShaderPixelInputInfo*            ps_input_info,
+                            const PipelineCache::GraphicsPrograms& programs,
+                            const PipelineStaticParameters&        static_params,
+                            vk::PipelineCache                      driver_cache);
 void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
                             const ShaderComputeInputInfo& input_info,
                             vk::ShaderModule compute_module, vk::PipelineCache driver_cache);

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp

@@ -8,6 +8,20 @@
 
 namespace Libs::Graphics {
 
+vk::ShaderStageFlagBits NativeShaderStage(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Local:
+		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
+		case ShaderType::Mesh: return vk::ShaderStageFlagBits::eMeshEXT;
+		case ShaderType::TessellationControl: return vk::ShaderStageFlagBits::eTessellationControl;
+		case ShaderType::TessellationEvaluation:
+			return vk::ShaderStageFlagBits::eTessellationEvaluation;
+		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
+		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
+		default: EXIT("unknown native shader stage\n");
+	}
+}
+
 vk::PipelineStageFlags ShaderPipelineStages(vk::ShaderStageFlags stages) {
 	vk::PipelineStageFlags result = {};
 	if (stages & vk::ShaderStageFlagBits::eVertex) {
@@ -16,6 +30,12 @@ vk::PipelineStageFlags ShaderPipelineStages(vk::ShaderStageFlags stages) {
 	if (stages & vk::ShaderStageFlagBits::eMeshEXT) {
 		result |= vk::PipelineStageFlagBits::eMeshShaderEXT;
 	}
+	if (stages & vk::ShaderStageFlagBits::eTessellationControl) {
+		result |= vk::PipelineStageFlagBits::eTessellationControlShader;
+	}
+	if (stages & vk::ShaderStageFlagBits::eTessellationEvaluation) {
+		result |= vk::PipelineStageFlagBits::eTessellationEvaluationShader;
+	}
 	if (stages & vk::ShaderStageFlagBits::eFragment) {
 		result |= vk::PipelineStageFlagBits::eFragmentShader;
 	}

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h

@@ -8,6 +8,7 @@ namespace Libs::Graphics {
 
 struct ShaderStageRuntime;
 
+vk::ShaderStageFlagBits NativeShaderStage(ShaderType stage);
 vk::PipelineStageFlags  ShaderPipelineStages(vk::ShaderStageFlags stages);
 VulkanMemoryBarrier     MakeShaderAccessDependency();
 VulkanMemoryBarrier     MakeShaderWriteHazardDependency();

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -8,10 +8,12 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
+#include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/rectListShader.h"
 #include "graphics/shader/shader.h"
@@ -115,227 +117,13 @@ static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format,
 		}
 		format = vk::Format::eR16G16Sfloat;
 		size   = 2;
-		if (NarrowInputFormat(format, size, used_components)) {
-			LOGF("InputFormat: narrowing fmt=%u to %s for used_components=%u\n", raw_format,
-			     vk::to_string(format).c_str(), used_components);
-		}
 		return;
 	}
 
-	switch (fmt) {
-		case Prospero::BufferFormat::k32_32_32_32Float:
-			format = vk::Format::eR32G32B32A32Sfloat;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32_32SInt:
-			format = vk::Format::eR32G32B32A32Sint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32_32UInt:
-			format = vk::Format::eR32G32B32A32Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32_32Float:
-			format = vk::Format::eR32G32B32Sfloat;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k32_32_32SInt:
-			format = vk::Format::eR32G32B32Sint;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k32_32_32UInt:
-			format = vk::Format::eR32G32B32Uint;
-			size   = 3;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16Float:
-			format = vk::Format::eR16G16B16A16Sfloat;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SInt:
-			format = vk::Format::eR16G16B16A16Sint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UInt:
-			format = vk::Format::eR16G16B16A16Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SScaled:
-			format = vk::Format::eR16G16B16A16Sscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UScaled:
-			format = vk::Format::eR16G16B16A16Uscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16SNorm:
-			format = vk::Format::eR16G16B16A16Snorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16_16_16UNorm:
-			format = vk::Format::eR16G16B16A16Unorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k32_32Float:
-			format = vk::Format::eR32G32Sfloat;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32_32SInt:
-			format = vk::Format::eR32G32Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32_32UInt:
-			format = vk::Format::eR32G32Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UInt:
-			format = vk::Format::eR8G8B8A8Uint;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8SScaled:
-			format = vk::Format::eR8G8B8A8Sscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UScaled:
-			format = vk::Format::eR8G8B8A8Uscaled;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8SNorm:
-			format = vk::Format::eR8G8B8A8Snorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k8_8_8_8UNorm:
-			format = vk::Format::eR8G8B8A8Unorm;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k10_10_10_2UNorm:
-			format = vk::Format::eA2B10G10R10UnormPack32;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k10_10_10_2SNorm:
-			format = vk::Format::eA2B10G10R10SnormPack32;
-			size   = 4;
-			break;
-		case Prospero::BufferFormat::k16_16Float:
-			format = vk::Format::eR16G16Sfloat;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SInt:
-			format = vk::Format::eR16G16Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UInt:
-			format = vk::Format::eR16G16Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SScaled:
-			format = vk::Format::eR16G16Sscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UScaled:
-			format = vk::Format::eR16G16Uscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16SNorm:
-			format = vk::Format::eR16G16Snorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16_16UNorm:
-			format = vk::Format::eR16G16Unorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k32Float:
-			format = vk::Format::eR32Sfloat;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k32SInt:
-			format = vk::Format::eR32Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k32UInt:
-			format = vk::Format::eR32Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8_8SInt:
-			format = vk::Format::eR8G8Sint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UInt:
-			format = vk::Format::eR8G8Uint;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8SScaled:
-			format = vk::Format::eR8G8Sscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UScaled:
-			format = vk::Format::eR8G8Uscaled;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8SNorm:
-			format = vk::Format::eR8G8Snorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k8_8UNorm:
-			format = vk::Format::eR8G8Unorm;
-			size   = 2;
-			break;
-		case Prospero::BufferFormat::k16Float:
-			format = vk::Format::eR16Sfloat;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SInt:
-			format = vk::Format::eR16Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UInt:
-			format = vk::Format::eR16Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SScaled:
-			format = vk::Format::eR16Sscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UScaled:
-			format = vk::Format::eR16Uscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16SNorm:
-			format = vk::Format::eR16Snorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k16UNorm:
-			format = vk::Format::eR16Unorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SInt:
-			format = vk::Format::eR8Sint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UInt:
-			format = vk::Format::eR8Uint;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SScaled:
-			format = vk::Format::eR8Sscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UScaled:
-			format = vk::Format::eR8Uscaled;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8SNorm:
-			format = vk::Format::eR8Snorm;
-			size   = 1;
-			break;
-		case Prospero::BufferFormat::k8UNorm:
-			format = vk::Format::eR8Unorm;
-			size   = 1;
-			break;
-		default:
-			EXIT("unknown format: fmt = %u\n", raw_format);
-			format = vk::Format::eUndefined;
-			size   = 4;
-			break;
+	format = VulkanFormat(fmt);
+	size   = ShaderRecompiler::Format::GetFormatInfo(fmt).component_count;
+	if (format == vk::Format::eUndefined || size == 0) {
+		EXIT("unknown vertex format: fmt = %u\n", raw_format);
 	}
 
 	if (NarrowInputFormat(format, size, used_components)) {
@@ -418,12 +206,18 @@ static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipe
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void CreatePipelineInternal(
-    GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
-    const PipelineRenderingState& rendering, const PipelineVertexInputState& vertex_input,
-    const ShaderVertexInputInfo& vs_input_info, const ShaderProgram& vertex_program,
-    const ShaderPixelInputInfo* ps_input_info, const ShaderProgram& pixel_program,
-    const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache) {
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                            const PipelineRenderingState&          rendering,
+                            const PipelineVertexInputState&        vertex_input,
+                            std::span<const ShaderVertexInputInfo> vertex_info,
+                            const ShaderPixelInputInfo*            ps_input_info,
+                            const PipelineCache::GraphicsPrograms& programs,
+                            const PipelineStaticParameters&        static_params,
+                            vk::PipelineCache                      driver_cache) {
+	const auto& vs_input_info  = vertex_info.front();
+	const auto& vertex_program = programs.vertex[0];
+	const auto& pixel_program  = programs.pixel;
+	const bool  tessellation   = vertex_info.size() == 3;
 	const bool ps_active = ps_input_info != nullptr;
 	EXIT_IF(!vertex_program || (ps_active && !pixel_program));
 	const bool with_depth = rendering.depth_format != vk::Format::eUndefined ||
@@ -431,72 +225,53 @@ void CreatePipelineInternal(
 	EXIT_IF(!vs_input_info.stage);
 	const bool mesh = vs_input_info.stage.program->stage == ShaderType::Mesh;
 	EXIT_NOT_IMPLEMENTED(mesh && !graphics.mesh_shader_enabled);
-	const auto vertex_stage =
-	    mesh ? vk::ShaderStageFlagBits::eMeshEXT : vk::ShaderStageFlagBits::eVertex;
-
-	const bool rect_list = !mesh && static_params.topology == vk::PrimitiveTopology::ePatchList;
+	const bool rect_list =
+	    !mesh && !tessellation && static_params.topology == vk::PrimitiveTopology::ePatchList;
 
 	vk::ShaderModule tess_control_shader_module = nullptr;
 	vk::ShaderModule tess_eval_shader_module    = nullptr;
 
-	vk::ShaderModuleCreateInfo create_info {};
-	vk::Result result {};
 	if (rect_list) {
 		const auto shaders =
 		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
-		create_info.codeSize = shaders.control.size() * 4;
-		create_info.pCode    = shaders.control.data();
-		result =
-		    graphics.device.createShaderModule(&create_info, nullptr, &tess_control_shader_module);
+		tess_control_shader_module = CompileSPV(shaders.control, graphics.device);
 		if (graphics_debug_dump_enabled()) {
-			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done result=%s module=%p\n",
-			     vk::to_string(result).c_str(), static_cast<void*>(tess_control_shader_module));
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TCS done module=%p\n",
+			     static_cast<void*>(tess_control_shader_module));
 		}
-		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
-		create_info.codeSize = shaders.evaluation.size() * 4;
-		create_info.pCode    = shaders.evaluation.data();
-		result =
-		    graphics.device.createShaderModule(&create_info, nullptr, &tess_eval_shader_module);
+		tess_eval_shader_module = CompileSPV(shaders.evaluation, graphics.device);
 		if (graphics_debug_dump_enabled()) {
-			LOGF("PipelineTrace: vkCreateShaderModule RectList TES done result=%s module=%p\n",
-			     vk::to_string(result).c_str(), static_cast<void*>(tess_eval_shader_module));
+			LOGF("PipelineTrace: vkCreateShaderModule RectList TES done module=%p\n",
+			     static_cast<void*>(tess_eval_shader_module));
 		}
-		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	}
 
 	EXIT_NOT_IMPLEMENTED(
 	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
 
-	vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
-	vert_shader_stage_info.stage  = vertex_stage;
-	vert_shader_stage_info.module = vertex_program.module;
-	vert_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
-	frag_shader_stage_info.stage  = vk::ShaderStageFlagBits::eFragment;
-	frag_shader_stage_info.module = pixel_program.module;
-	frag_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
-	tess_control_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationControl;
-	tess_control_shader_stage_info.module = tess_control_shader_module;
-	tess_control_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo tess_eval_shader_stage_info {};
-	tess_eval_shader_stage_info.stage  = vk::ShaderStageFlagBits::eTessellationEvaluation;
-	tess_eval_shader_stage_info.module = tess_eval_shader_module;
-	tess_eval_shader_stage_info.pName  = "main";
-
-	vk::PipelineShaderStageCreateInfo shader_stages[4]   = {};
+	vk::PipelineShaderStageCreateInfo shader_stages[4] {};
 	uint32_t                          shader_stage_count = 0;
-	shader_stages[shader_stage_count++]                  = vert_shader_stage_info;
+	for (uint32_t i = 0; i < vertex_info.size(); i++) {
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           NativeShaderStage(vertex_info[i].logical_stage),
+		                                       .module = programs.vertex[i].module,
+		                                       .pName  = "main"};
+	}
 	if (rect_list) {
-		shader_stages[shader_stage_count++] = tess_control_shader_stage_info;
-		shader_stages[shader_stage_count++] = tess_eval_shader_stage_info;
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           vk::ShaderStageFlagBits::eTessellationControl,
+		                                       .module = tess_control_shader_module,
+		                                       .pName  = "main"};
+		shader_stages[shader_stage_count++] = {.stage =
+		                                           vk::ShaderStageFlagBits::eTessellationEvaluation,
+		                                       .module = tess_eval_shader_module,
+		                                       .pName  = "main"};
 	}
 	if (ps_active) {
-		shader_stages[shader_stage_count++] = frag_shader_stage_info;
+		shader_stages[shader_stage_count++] = {.stage  = vk::ShaderStageFlagBits::eFragment,
+		                                       .module = pixel_program.module,
+		                                       .pName  = "main"};
 	}
 
 	vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX] {};
@@ -619,8 +394,7 @@ void CreatePipelineInternal(
 		EXIT_NOT_IMPLEMENTED((static_params.color_mask[i] & ~0x0fu) != 0);
 		color_blend_attachment[i].colorWriteMask =
 		    vk::ColorComponentFlags {static_params.color_mask[i]};
-		color_blend_attachment[i].blendEnable =
-		    (static_params.blend_enable[i] && !static_params.blend_bypass[i]) ? VK_TRUE : VK_FALSE;
+		color_blend_attachment[i].blendEnable = static_params.blend_enable[i] ? VK_TRUE : VK_FALSE;
 		color_blend_attachment[i].srcColorBlendFactor =
 		    GetBlendFactor(static_params.color_srcblend[i]);
 		color_blend_attachment[i].dstColorBlendFactor =
@@ -658,15 +432,19 @@ void CreatePipelineInternal(
 	color_blending.pAttachments    = color_blend_attachment;
 
 	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
-	AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program, vertex_stage);
+	vk::ShaderStageFlags graphics_stages = vk::ShaderStageFlagBits::eFragment;
+	for (const auto& stage: vertex_info) {
+		const auto native_stage = NativeShaderStage(stage.logical_stage);
+		AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage);
+		graphics_stages |= native_stage;
+	}
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
 		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
 		                  vk::ShaderStageFlagBits::eFragment);
 	}
 	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
-	const auto                  GraphicsStages = vertex_stage | vk::ShaderStageFlagBits::eFragment;
-	const vk::PushConstantRange push_constants {GraphicsStages, 0,
+	const vk::PushConstantRange push_constants {graphics_stages, 0,
 	                                            ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
@@ -682,8 +460,8 @@ void CreatePipelineInternal(
 		     " set_layouts=1 push_constants=%" PRIu32 "\n",
 		     vertex_program.id, ps_active ? pixel_program.id : 0, 1u);
 	}
-	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
-	                                              &pipeline.pipeline_layout);
+	auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
+	                                                   &pipeline.pipeline_layout);
 	if (graphics_debug_dump_enabled()) {
 		LOGF("PipelineTrace: vkCreatePipelineLayout done result=%s layout=%p\n",
 		     vk::to_string(result).c_str(), static_cast<void*>(pipeline.pipeline_layout));
@@ -699,15 +477,6 @@ void CreatePipelineInternal(
 #else
 	    (static_params.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
 #endif
-	depth_stencil_info.stencilTestEnable = (static_params.stencil_test_enable ? VK_TRUE : VK_FALSE);
-	depth_stencil_info.front.failOp      = static_params.stencil_front.failOp;
-	depth_stencil_info.front.passOp      = static_params.stencil_front.passOp;
-	depth_stencil_info.front.depthFailOp = static_params.stencil_front.depthFailOp;
-	depth_stencil_info.front.compareOp   = static_params.stencil_front.compareOp;
-	depth_stencil_info.back.failOp       = static_params.stencil_back.failOp;
-	depth_stencil_info.back.passOp       = static_params.stencil_back.passOp;
-	depth_stencil_info.back.depthFailOp  = static_params.stencil_back.depthFailOp;
-	depth_stencil_info.back.compareOp    = static_params.stencil_back.compareOp;
 	depth_stencil_info.minDepthBounds    = static_params.depth_min_bounds;
 	depth_stencil_info.maxDepthBounds    = static_params.depth_max_bounds;
 
@@ -720,6 +489,8 @@ void CreatePipelineInternal(
 	    vk::DynamicState::eDepthCompareOp,
 	    vk::DynamicState::eDepthBiasEnable,
 	    vk::DynamicState::eDepthBias,
+	    vk::DynamicState::eStencilTestEnable,
+	    vk::DynamicState::eStencilOp,
 	    vk::DynamicState::eStencilCompareMask,
 	    vk::DynamicState::eStencilReference,
 	    vk::DynamicState::eStencilWriteMask,
@@ -750,8 +521,9 @@ void CreatePipelineInternal(
 	pipeline_info.pVertexInputState        = mesh ? nullptr : &vertex_input_info;
 	pipeline_info.pInputAssemblyState      = mesh ? nullptr : &input_assembly;
 	vk::PipelineTessellationStateCreateInfo tessellation_state {};
-	tessellation_state.patchControlPoints = 3;
-	pipeline_info.pTessellationState      = (rect_list ? &tessellation_state : nullptr);
+	tessellation_state.patchControlPoints =
+	    tessellation ? vs_input_info.tess.input_control_points : 3u;
+	pipeline_info.pTessellationState = (rect_list || tessellation) ? &tessellation_state : nullptr;
 	pipeline_info.pViewportState          = &viewport_state;
 	pipeline_info.pRasterizationState     = &rasterizer;
 	pipeline_info.pMultisampleState       = &multisampling;

src/graphics/host_gpu/renderer/render.h

@@ -170,15 +170,14 @@ private:
 	void DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const DrawAutoArgs& args);
 
 	struct GraphicsBindings {
-		PreparedBindings                vertex;
+		std::array<PreparedBindings, 3> vertex;
 		std::optional<PreparedBindings> pixel;
 	};
 
 	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	                                            const ShaderRecompiler::IR::DescriptorValue& value);
-	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
-	                                                       const ShaderStageRuntime& pixel,
-	                                                       bool                      pixel_active);
+	void PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
+	                             std::span<RenderColorInfo> colors);
 	void ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& target,
 	                              uint32_t render_target_slice_offset, uint32_t render_target_slot,
 	                              bool ignore_target_mask = false, bool exact_format = false);
@@ -187,12 +186,11 @@ private:
 	[[nodiscard]] bool PrepareDrawRenderState(CommandBuffer& buffer,
 	                                          const DrawCallInfo& draw,
 	                                          uint32_t            render_target_slice_offset,
-	                                          bool log_setup_phases, DrawRenderState& state);
+	                                          DrawRenderState& state);
 	void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer, const DrawCallInfo& draw,
 	                         DrawRenderState& state, vk::PrimitiveTopology topology,
 	                         const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
-	                         bool primitive_restart_enable, bool log_pipeline_phase,
-	                         bool set_bind_debug, bool set_auto_debug);
+	                         bool primitive_restart_enable);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth,
 	                                               const std::optional<PreparedBindings>& pixel = std::nullopt);
@@ -200,7 +198,6 @@ private:
 	                                              uint32_t render_target_slice_offset);
 	void                      BindImage(ImageId id, bool storage);
 	void                      BindRenderTarget(ImageId id);
-	void                      TrackImageBinding(ImageId id);
 	void                      ResetBindings();
 	[[nodiscard]] bool        TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
 	                                                     const CommandBuffer&          buffer);

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -36,18 +36,12 @@
 #include <vector>
 
 namespace Libs::Graphics {
-static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) {
-	const uint64_t records = descriptor.NumRecords();
-	const uint64_t stride  = descriptor.Stride();
-	return stride == 0 ? records : records * stride;
-}
-
 static bool FillSourcesDisjoint(std::span<const ShaderRecompiler::IR::DescriptorValue> sources,
                                  GuestRange destination, uint32_t output_buffer = UINT32_MAX) {
 	for (uint32_t i = 0; i < sources.size(); ++i) {
 		if (i == output_buffer) continue;
 		const auto source = DecodeNativeDescriptor<ShaderBufferResource>(sources[i]);
-		const auto bytes  = BufferDescriptorSize(source);
+		const auto bytes  = source.GetSize();
 		if (source.Base48() < destination.End() && destination.address < source.Base48() + bytes)
 			return false;
 	}
@@ -67,7 +61,7 @@ bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& in
 		const auto  descriptor = DecodeNativeDescriptor<ShaderBufferResource>(resources.buffers[i]);
 		// A metadata resource that is also read is not proven to be a full overwrite. Execute it
 		// conservatively instead of replacing the dispatch with a coarse full-surface clear.
-		if (cache.IsMeta(descriptor.Base48()) && (!resource.written || resource.read)) {
+		if ((!resource.written || resource.read) && cache.IsMeta(descriptor.Base48())) {
 			return false;
 		}
 	}
@@ -115,7 +109,7 @@ bool ResolveComputeBufferFill(const ShaderComputeInputInfo& input, uint32_t grou
 	const uint64_t invocations = input.dispatch_thread_dimensions
 	                                 ? group_x
 	                                 : static_cast<uint64_t>(group_x) * input.threads_num[0];
-	const auto     size        = BufferDescriptorSize(descriptor);
+	const auto     size        = descriptor.GetSize();
 	if (invocations != descriptor.NumRecords() || size == 0 || size > UINT32_MAX ||
 	    (input.dispatch_thread_dimensions &&
 	     (group_x % input.threads_num[0] != 0 || input.dispatch_threads_num[0] != group_x ||
@@ -178,7 +172,7 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 		                                       1, view.base_layer, view.layer_count};
 		vk::ClearValue clear {};
 		clear.depthStencil = vk::ClearDepthStencilValue {0.0f, fill.value};
-		cache.ClearImage(command, binding.image_id, range, clear);
+		cache.ClearImage(command, binding.image_id, image.backing.format, range, clear);
 		return true;
 	}
 	ShaderBufferResource descriptor;
@@ -189,14 +183,6 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 		return false;
 	}
 	if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) {
-		// Track deferred DCC state while the original dispatch writes the metadata allocation.
-		cache.TrackDccFill(descriptor.Base48(), size, packed_clear);
-		static std::atomic<uint32_t> logged_metadata_clears {0};
-		if (logged_metadata_clears.fetch_add(1, std::memory_order_relaxed) < 32) {
-			LOGF("GraphicsRenderDispatchDirect: metadata fill shader=0x%016" PRIx64
-			     " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n",
-			     input.stage.program->shader_hash, descriptor.Base48(), size, packed_clear);
-		}
 		return false;
 	}
 	static std::atomic<uint32_t> logged_clears {0};
@@ -264,9 +250,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		input_info.dispatch_threads_num[2]    = thread_group_z;
 	}
 
-	const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
-	const bool     large_workgroup =
-	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const auto& program   = *input_info.stage.program;
 	const auto& resources = input_info.stage.resources;
 	if (TryConsumeComputeMetaClear(input_info, buffer)) {
@@ -278,14 +261,17 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		ResetBindings();
 		return;
 	}
-	const auto sampled_images = std::count_if(
-	    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
-		    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
-	    });
+	const bool large_workgroup =
+	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const bool                   has_sampler = !program.info.samplers.empty();
 	static std::atomic<uint32_t> dispatch_log_count {0};
 	if ((large_workgroup || has_sampler) &&
 	    dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) {
+		const auto sampled_images = std::count_if(
+		    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
+			    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
+		    });
+		const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
 		LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64
 		     " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u "
 		     "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n",
@@ -374,14 +360,14 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 
 	buffer.EndRendering();
 	auto& pipeline =
-	    m_context.GetPipelineCache().CreateComputePipeline(input_info, compute_program);
+	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto bindings = PrepareBindings(input_info.stage);
 	FindBuffers(bindings);
 	if (program.info.uses_dma) {
-		m_context.GetGpuResources().PrepareBda();
+		m_context.PrepareBda();
 	}
-	RebindBuffers(bindings);
 	RebindImages(bindings);
+	RebindBuffers(bindings);
 
 	auto              vk_buffer        = buffer.Handle();
 	PreparedBindings* descriptor_stage = &bindings;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -14,7 +14,8 @@ RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
       m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
       m_pipeline_cache(graphics), m_sampler_cache(graphics),
-      m_gpu_resources(graphics, m_command_scheduler) {
+      m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
+      m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -27,12 +28,10 @@ void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) {
 	EXIT_IF(m_gpu != nullptr);
 	m_video_out = video_out;
 	m_gpu       = std::make_unique<GuestGpu>(*this);
-	m_gpu_resources.SetGpu(m_gpu.get());
 }
 
 void RenderContext::ShutdownGpu() {
 	if (m_gpu != nullptr) {
-		m_gpu_resources.SetGpu(nullptr);
 		m_gpu->Shutdown();
 		m_gpu.reset();
 	}
@@ -55,6 +54,88 @@ VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const {
 	return *m_video_out;
 }
 
+bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
+	// The host reports the faulting byte, not the instruction's access width. Both caches
+	// resolve its page; guessing a width can cross the end of a valid guest mapping.
+	constexpr uint64_t fault_size = 1;
+	if (!IsMapped(fault_vaddr, fault_size)) {
+		return false;
+	}
+	if (access == PageFaultAccess::Write) {
+		m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
+		m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+	} else {
+		m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
+	}
+	return true;
+}
+
+bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (!IsMapped(vaddr, size)) {
+		return false;
+	}
+	m_buffer_cache.InvalidateMemory(vaddr, size);
+	m_texture_cache.InvalidateMemory(vaddr, size);
+	return true;
+}
+
+bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
+	if (!GuestRange {vaddr, size}.Valid()) {
+		return false;
+	}
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	return m_mapped_ranges.Contains(vaddr, size);
+}
+
+void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) {
+	std::lock_guard lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.Add(vaddr, size);
+}
+
+void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	if (CommandScheduler::InDeferredOperation()) {
+		EXIT("unsupported memory unmap from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+		     vaddr, size);
+	}
+	const auto unmap = [this, vaddr, size] {
+		if (m_command_scheduler.Active()) {
+			const auto tick = m_command_scheduler.CurrentTick();
+			m_command_scheduler.Finish();
+			m_command_scheduler.WaitPriorityOperations(tick);
+		}
+		m_buffer_cache.InvalidateMemory(vaddr, size);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Subtract(vaddr, size);
+	};
+	// Shutdown still owns the GPU while queued rendering drains, but its command lane no
+	// longer accepts external work. Use the guest GPU's state for the teardown route.
+	if (m_gpu == nullptr || m_gpu->IsStopping()) {
+		unmap();
+		return;
+	}
+	m_gpu->SendCommandSync(unmap);
+}
+
+void RenderContext::PrepareBda() {
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
+		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
+	});
+	m_fault_process_pending = true;
+}
+
+void RenderContext::RunGarbageCollector() {
+	if (m_fault_process_pending) {
+		m_fault_process_pending = false;
+		m_buffer_cache.ProcessFaultBuffer();
+	}
+	m_texture_cache.ProcessDownloadImages();
+	m_texture_cache.RunGarbageCollector();
+	m_buffer_cache.RunGarbageCollector();
+}
+
 void RenderContext::AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
 	Common::LockGuard lock(m_interrupt_mutex);
 

src/graphics/host_gpu/renderer/renderContext.h

@@ -5,8 +5,9 @@
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/threads.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/rangeSet.h"
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
-#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
@@ -15,6 +16,7 @@
 #include "kernel/eventQueue.h"
 
 #include <memory>
+#include <shared_mutex>
 #include <vector>
 
 namespace Libs::VideoOut {
@@ -42,11 +44,18 @@ public:
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
 	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
-	GpuResourceManager& GetGpuResources() { return m_gpu_resources; }
-	BufferCache&        GetBufferCache() { return m_gpu_resources.GetBufferCache(); }
-	TextureCache&       GetTextureCache() { return m_gpu_resources.GetTextureCache(); }
+	BufferCache&        GetBufferCache() { return m_buffer_cache; }
+	TextureCache&       GetTextureCache() { return m_texture_cache; }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
 
+	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
+	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
+	void               MapMemory(uint64_t vaddr, uint64_t size);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
+	void               PrepareBda();
+	void               RunGarbageCollector();
+
 	void AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
 	void DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
 	void TriggerInterrupt(int event_id, uint32_t context_id);
@@ -64,9 +73,14 @@ private:
 	DescriptorHeap            m_descriptor_heap;
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
-	GpuResourceManager        m_gpu_resources;
+	PageManager               m_page_manager;
+	BufferCache               m_buffer_cache;
+	TextureCache              m_texture_cache;
+	mutable std::shared_mutex m_mapped_ranges_mutex;
+	RangeSet                  m_mapped_ranges;
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
+	bool                      m_fault_process_pending = false;
 
 	Common::Mutex                        m_interrupt_mutex;
 	std::vector<InterruptEqRegistration> m_interrupt_eqs;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -46,42 +46,34 @@
 
 namespace Libs::Graphics {
 
-int32_t ResolveVertexOffset(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info) {
-	if (index_offset != 0 || !vs_input_info.fetch_embedded) {
-		return static_cast<int32_t>(index_offset);
+std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
+	                                           const ShaderVertexInputInfo& vs_input_info) {
+	auto     vertex_offset   = static_cast<int32_t>(index_offset);
+	uint32_t instance_offset = 0;
+	if (!vs_input_info.fetch_embedded) {
+		return {vertex_offset, instance_offset};
 	}
 
 	EXIT_IF(!vs_input_info.stage);
 	const auto& program   = *vs_input_info.stage.program;
 	const auto& resources = vs_input_info.stage.resources;
-	if (program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
+	if (index_offset == 0 &&
+	    program.info.vertex_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
 		    static_cast<uint32_t>(program.info.vertex_offset_sgpr) - program.user_data_base;
 		if (index < resources.user_data.size()) {
-			return static_cast<int32_t>(resources.user_data[index]);
+			vertex_offset = static_cast<int32_t>(resources.user_data[index]);
 		}
 	}
-
-	return 0;
-}
-
-uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info) {
-	if (!vs_input_info.fetch_embedded) {
-		return 0;
-	}
-
-	EXIT_IF(!vs_input_info.stage);
-	const auto& program   = *vs_input_info.stage.program;
-	const auto& resources = vs_input_info.stage.resources;
 	if (program.info.instance_offset_sgpr >= static_cast<int32_t>(program.user_data_base)) {
 		const auto index =
 		    static_cast<uint32_t>(program.info.instance_offset_sgpr) - program.user_data_base;
 		if (index < resources.user_data.size()) {
-			return resources.user_data[index];
+			instance_offset = resources.user_data[index];
 		}
 	}
 
-	return 0;
+	return {vertex_offset, instance_offset};
 }
 
 static std::atomic<uint32_t> g_draw_state_log_count   = 0;
@@ -411,19 +403,16 @@ static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuf
 		vk_buffer.setDepthBias(constant_factor, poly_offset.clamp, slope_factor);
 	}
 
+	vk_buffer.setStencilTestEnable(depth.stencil_test_enable ? VK_TRUE : VK_FALSE);
 	if (depth.stencil_test_enable) {
-		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eFront,
-		                                depth.stencil_dynamic_front.compareMask);
-		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eBack,
-		                                depth.stencil_dynamic_back.compareMask);
-		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eFront,
-		                              depth.stencil_dynamic_front.writeMask);
-		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eBack,
-		                              depth.stencil_dynamic_back.writeMask);
-		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eFront,
-		                              depth.stencil_dynamic_front.reference);
-		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eBack,
-		                              depth.stencil_dynamic_back.reference);
+		const auto set_stencil = [&](vk::StencilFaceFlagBits face, const vk::StencilOpState& state) {
+			vk_buffer.setStencilOp(face, state.failOp, state.passOp, state.depthFailOp, state.compareOp);
+			vk_buffer.setStencilCompareMask(face, state.compareMask);
+			vk_buffer.setStencilWriteMask(face, state.writeMask);
+			vk_buffer.setStencilReference(face, state.reference);
+		};
+		set_stencil(vk::StencilFaceFlagBits::eFront, depth.stencil_front);
+		set_stencil(vk::StencilFaceFlagBits::eBack, depth.stencil_back);
 	}
 
 #if defined(__APPLE__)
@@ -461,17 +450,19 @@ struct DrawRenderState {
 	RenderColorInfo       color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	uint32_t              color_count                              = 0;
 	bool                  ps_active                                = true;
-	ShaderVertexInputInfo vs_input_info;
+	std::array<ShaderVertexInputInfo, 3> vertex_info;
 	ShaderPixelInputInfo  ps_input_info;
 	PipelineCache::GraphicsPrograms programs;
 };
 
 struct DrawCallInfo {
-	const char*          name           = nullptr;
 	CommandBufferDebugOp debug_op       = CommandBufferDebugOp::DrawIndex;
 	uint32_t             index_count    = 0;
 	uint32_t             instance_count = 0;
 	uint32_t             first_instance = 0;
+
+	[[nodiscard]] bool IsIndexed() const { return debug_op == CommandBufferDebugOp::DrawIndex; }
+	[[nodiscard]] const char* Name() const { return IsIndexed() ? "DrawIndex" : "DrawIndexAuto"; }
 };
 
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
@@ -488,14 +479,10 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
-		const auto old_image = cache.m_slot_images.try_get(target.image_id);
-		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
-		    old_image->binding.needs_rebind) {
-			if (old_image != nullptr) {
-				old_image->binding = {};
-			}
-			target.image_id = cache.FindImage(target.desc);
-			BindRenderTarget(target.image_id);
+		const auto owner = cache.m_slot_images.try_get(target.image_id);
+		if (owner == nullptr || (!owner->registered && !owner->info.data.Empty()) ||
+		    owner->binding.needs_rebind) {
+			EXIT("color target changed after render-state discovery\n");
 		}
 		const auto image_view = cache.FindRenderTarget(target.image_id, target.desc);
 		auto&      image      = cache.GetImage(target.image_id);
@@ -662,7 +649,6 @@ static bool ConsumeMetadataColorOperation(const CommandBuffer& buffer) {
 }
 
 struct DrawEmitInfo {
-	bool     indexed       = false;
 	int32_t  vertex_offset = 0;
 	uint32_t first_vertex  = 0;
 	uint32_t first_instance = 0;
@@ -716,6 +702,7 @@ struct PreparedVertexBuffers {
 
 	std::array<vk::Buffer, MaxBuffers>     buffers {};
 	std::array<vk::DeviceSize, MaxBuffers> offsets {};
+	std::array<vk::DeviceSize, MaxBuffers> sizes {};
 	uint32_t                               count = 0;
 };
 
@@ -800,6 +787,7 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer&               b
 
 		prepared.buffers[i] = range->binding.first->Handle();
 		prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address;
+		prepared.sizes[i]   = std::min(size, range->acquired_end - vertex.addr);
 		SetVulkanObjectNameF(
 		    buffer.GetContext().GetGraphics().device, prepared.buffers[i],
 		    "Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i,
@@ -838,6 +826,7 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
 		case Prospero::PrimitiveType::kTriStrip:
 			topology = vk::PrimitiveTopology::eTriangleStrip;
 			break;
+		case Prospero::PrimitiveType::kPatch:
 		case Prospero::PrimitiveType::kRectList:
 			topology = vk::PrimitiveTopology::ePatchList;
 			break;
@@ -904,14 +893,14 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
 
 bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
-                                            bool log_setup_phases, DrawRenderState& state) {
+	                                        DrawRenderState& state) {
 	auto& ctx = buffer.GetRegisters();
 
 	if (ResolveColorTargets(buffer, render_target_slice_offset)) {
 		return false;
 	}
-	if (log_setup_phases) {
-		LogDrawPhase(draw.name, "ResolveRenderColorTarget");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}
 	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
 		if (slot == 0 || (render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0 &&
@@ -923,14 +912,14 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 			}
 		}
 	}
-	if (log_setup_phases) {
-		LogDrawPhase(draw.name, "ResolveRenderDepthTarget");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "ResolveRenderDepthTarget");
 	}
 	ResolveRenderDepthTarget(buffer, state.depth_info);
 
 	state.ps_active       = DrawHasActivePixelShader(buffer);
 	if (state.color_count == 0 && !state.depth_info.image_id && !state.ps_active) {
-		LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer,
+		LogFramebufferSkip(draw.Name(), state.color_info[0], state.depth_info, buffer,
 		                   draw.index_count, 0);
 		return false;
 	}
@@ -938,7 +927,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 	return true;
 }
 
-static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
+static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
                            DrawRenderState& state) {
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
@@ -955,12 +944,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool
 		target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
 	}
 	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
-	if (log_phases) {
-		LogDrawPhase(draw.name, "GetGraphicsPrograms");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
 	}
 	state.programs = pipeline_cache.GetGraphicsPrograms(
 	    vertex_shader_info, pixel_shader_info, shader_regs, ctx, buffer.GetUserConfig(),
-	    target_export_mapping, state.ps_active, state.vs_input_info, state.ps_input_info);
+	    target_export_mapping, state.ps_active, state.vertex_info, state.ps_input_info);
 }
 
 static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffer,
@@ -974,17 +963,14 @@ static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffe
 		auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
 		prepared.offset = stream.Copy(source.host_data, source.size, 16);
 		prepared.buffer = stream.Handle();
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
+		                     static_cast<uint32_t>(source.type));
 	} else {
 		auto [buffer_ptr, offset] =
 		    buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
 		prepared.buffer = buffer_ptr->Handle();
 		prepared.offset = offset;
-	}
-	if (source.host_data != nullptr) {
-		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
-		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
-		                     static_cast<uint32_t>(source.type));
-	} else {
 		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
 		                     "Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
 		                     source.address, source.size, static_cast<uint32_t>(source.type));
@@ -998,8 +984,9 @@ static void CommitVertexBuffers(vk::CommandBuffer            vk_buffer,
 		EXIT_IF(prepared.buffers[i] == nullptr);
 	}
 	if (prepared.count != 0) {
-		vk_buffer.bindVertexBuffers(0, prepared.count, prepared.buffers.data(),
-		                            prepared.offsets.data());
+		// Guest descriptor bounds must survive allocation merging in the cache.
+		vk_buffer.bindVertexBuffers2(0, prepared.count, prepared.buffers.data(),
+		                             prepared.offsets.data(), prepared.sizes.data(), nullptr);
 	}
 }
 
@@ -1011,22 +998,22 @@ static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBu
 }
 
 static void LogDrawStateIfNeeded(const CommandBuffer& buffer, const DrawCallInfo& draw,
-                                 const DrawRenderState& state, bool always_log,
-                                 bool force_legacy_rect_log, uint32_t index_type_and_size,
+	                             const DrawRenderState& state, uint32_t index_type_and_size,
                                  const void* index_addr) {
 	if (!graphics_debug_dump_enabled()) {
 		return;
 	}
 
-	if (!always_log && !force_legacy_rect_log) {
+	if (!draw.IsIndexed() &&
+	    buffer.GetUserConfig().GetPrimType() != Prospero::PrimitiveType::kRectListLegacy) {
 		return;
 	}
 
 	if (state.ps_active) {
-		LogDrawTargetState(draw.name, state.color_info[0], state.depth_info, buffer,
+		LogDrawTargetState(draw.Name(), state.color_info[0], state.depth_info, buffer,
 		                   state.ps_input_info, draw.index_count, 0);
 	}
-	LogDrawInputState(buffer, state.color_info[0], state.vs_input_info, index_type_and_size,
+	LogDrawInputState(buffer, state.color_info[0], state.vertex_info[0], index_type_and_size,
 	                  draw.index_count, index_addr);
 }
 
@@ -1041,7 +1028,8 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
 		case Prospero::PrimitiveType::kRectList:
-			if (emit.indexed) {
+		case Prospero::PrimitiveType::kPatch:
+			if (draw.IsIndexed()) {
 				vk_buffer.drawIndexed(draw.index_count, draw.instance_count, 0, emit.vertex_offset,
 				                      emit.first_instance);
 			} else {
@@ -1050,7 +1038,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 			}
 			break;
 		case Prospero::PrimitiveType::kRectListLegacy:
-			if (emit.indexed) {
+			if (draw.IsIndexed()) {
 				EXIT("unknown primitive type: %u\n", static_cast<uint32_t>(ucfg.GetPrimType()));
 			}
 			// Sarah
@@ -1060,7 +1048,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kQuadListLegacy:
 			EXIT_NOT_IMPLEMENTED((draw.index_count & 0x3u) != 0);
 			for (uint32_t i = 0; i < draw.index_count; i += 4) {
-				if (emit.indexed) {
+				if (draw.IsIndexed()) {
 					vk_buffer.drawIndexed(4, draw.instance_count, i, emit.vertex_offset,
 					                      emit.first_instance);
 				} else {
@@ -1077,16 +1065,17 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
                                          const DrawCallInfo& draw, DrawRenderState& state,
                                          vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
                                          const DrawIndexBufferSource& index_source,
-                                         bool primitive_restart_enable, bool log_pipeline_phase,
-                                         bool set_bind_debug, bool set_auto_debug) {
+	                                     bool primitive_restart_enable) {
 	auto& ucfg = buffer.GetUserConfig();
-	const bool mesh_active = state.vs_input_info.stage.program->stage == ShaderType::Mesh;
+	const auto vertex_stages =
+	    std::span {state.vertex_info.data(), state.programs.VertexStageCount()};
+	const bool mesh_active = state.vertex_info[0].stage.program->stage == ShaderType::Mesh;
 	uint32_t   mesh_groups = 0;
 	if (mesh_active) {
-		const auto& mesh = state.vs_input_info.mesh;
+		const auto& mesh = state.vertex_info[0].mesh;
 		if (primitive_restart_enable || mesh.primitives_per_group == 0) {
 			EXIT("unsupported mesh draw: primitive=%u indexed=%u restart=%u\n",
-			     static_cast<uint32_t>(ucfg.GetPrimType()), emit.indexed, primitive_restart_enable);
+			     static_cast<uint32_t>(ucfg.GetPrimType()), draw.IsIndexed(), primitive_restart_enable);
 		}
 		const auto primitives = mesh.InputPrimitiveCount(draw.index_count);
 		if (primitives == 0 || draw.instance_count == 0) {
@@ -1103,64 +1092,62 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		}
 	}
 
-	if (mesh_active && emit.indexed) {
+	if (mesh_active && draw.IsIndexed()) {
 		// Register the original guest indices for shader reads; PrepareGraphicsBindings
 		// synchronizes registered BDA ranges before any draw commands are committed.
 		(void)m_context.GetBufferCache().FindBuffer(
 		    index_source.address, static_cast<uint64_t>(draw.index_count) *
 		                              index_source.guest_element_size);
 	}
-	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage,
-	                                        state.ps_active);
+	LogDrawPhase(draw.Name(), "PrepareBindings");
+	GraphicsBindings                 bindings;
+	std::array<PreparedBindings*, 4> descriptor_stages {};
+	uint32_t                         stage_count = 0;
+	for (uint32_t i = 0; i < vertex_stages.size(); i++) {
+		bindings.vertex[i]               = PrepareBindings(state.vertex_info[i].stage);
+		descriptor_stages[stage_count++] = &bindings.vertex[i];
+	}
+	if (state.ps_active) {
+		bindings.pixel.emplace(PrepareBindings(state.ps_input_info.stage));
+		descriptor_stages[stage_count++] = &*bindings.pixel;
+	}
+	const auto stages = std::span {descriptor_stages.data(), stage_count};
+	PrepareGraphicsBindings(stages, std::span {state.color_info, state.color_count});
 	PreparedVertexBuffers vertex_bindings;
 	PreparedIndexBuffer   index_binding;
 	if (!mesh_active) {
-		LogDrawPhase(draw.name, "PrepareVertexBuffers");
-		vertex_bindings = AcquireVertexBuffers(buffer, state.vs_input_info);
+		LogDrawPhase(draw.Name(), "PrepareVertexBuffers");
+		vertex_bindings = AcquireVertexBuffers(buffer, state.vertex_info[0]);
 		index_binding   = PrepareIndexBuffer(buffer, index_source);
 	}
 	const auto rendering =
 	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
 	                         bindings.pixel);
 
-	if (log_pipeline_phase) {
-		LogDrawPhase(draw.name, "CreatePipeline");
+	if (draw.IsIndexed()) {
+		LogDrawPhase(draw.Name(), "CreatePipeline");
 	}
-	auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline(
-	    std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer,
+	auto& pipeline = m_context.GetPipelineCache().GetGraphicsPipeline(
+	    std::span {state.color_info, state.color_count}, state.depth_info, vertex_stages, buffer,
 	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
-	    state.programs.vertex, state.programs.pixel);
+	    state.programs);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest
 	// memory.
 	auto vk_buffer = buffer.Handle();
-	if (set_bind_debug) {
-		SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
-	}
-	if (set_auto_debug) {
-		SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
-	}
+	SetDrawDebugPhase(buffer, submit_id, draw, draw.IsIndexed() ? 0x100u : 0x200u);
 	if (!mesh_active) {
 		CommitVertexBuffers(vk_buffer, vertex_bindings);
 	}
-	if (bindings.pixel.has_value()) {
-		if (set_auto_debug) {
-			SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
-		}
+	if (bindings.pixel && !draw.IsIndexed()) {
+		SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 	}
-	std::array<PreparedBindings*, 2> descriptor_stages {&bindings.vertex, nullptr};
-	const size_t                     descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u;
-	if (bindings.pixel) {
-		descriptor_stages[1] = &*bindings.pixel;
-	}
-	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
-	               std::span {descriptor_stages.data(), descriptor_stage_count});
+	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline, stages);
 	if (mesh_active) {
 		const uint32_t draw_data[] {
 		    draw.index_count,
-		    emit.indexed ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
+		    draw.IsIndexed() ? static_cast<uint32_t>(emit.vertex_offset) : emit.first_vertex,
 		    emit.first_instance, index_source.guest_element_size,
 		    static_cast<uint32_t>(index_source.address),
 		    static_cast<uint32_t>(index_source.address >> 32u)};
@@ -1173,7 +1160,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		CommitIndexBuffer(vk_buffer, index_binding);
 	}
 
-	SetGraphicsDynamicParams(buffer, vk_buffer, state.vs_input_info, state.color_info,
+	SetGraphicsDynamicParams(buffer, vk_buffer, vertex_stages.back(), state.color_info,
 	                         state.color_count, state.depth_info);
 	if (m_context.GetGraphics().attachment_feedback_loop_enabled) {
 		vk_buffer.setAttachmentFeedbackLoopEnableEXT(
@@ -1183,28 +1170,29 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		        : vk::ImageAspectFlags {});
 	}
 
-	LogDrawPhase(draw.name, "BeginRendering");
-	if (set_auto_debug) {
+	LogDrawPhase(draw.Name(), "BeginRendering");
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
 	}
 	m_context.GetCommandScheduler().BeginRendering(rendering);
 	vk_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline.pipeline);
-	if (set_auto_debug) {
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
 	}
 	if (mesh_active) {
 		vk_buffer.drawMeshTasksEXT(mesh_groups, draw.instance_count, 1);
 	} else {
-		EmitDrawPrimitives(ucfg, vk_buffer, state.vs_input_info, draw, emit);
+		EmitDrawPrimitives(ucfg, vk_buffer, state.vertex_info[0], draw, emit);
 	}
 
-	if (set_auto_debug) {
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
 	}
 	vk::PipelineStageFlags shader_write_stages = {};
-	if (HasShaderBufferWrites(state.vs_input_info.stage)) {
-		shader_write_stages |= mesh_active ? vk::PipelineStageFlagBits::eMeshShaderEXT
-		                                   : vk::PipelineStageFlagBits::eVertexShader;
+	for (const auto& stage: vertex_stages) {
+		if (HasShaderBufferWrites(stage.stage)) {
+			shader_write_stages |= ShaderPipelineStages(NativeShaderStage(stage.logical_stage));
+		}
 	}
 	if (state.ps_active && HasShaderBufferWrites(state.ps_input_info.stage)) {
 		shader_write_stages |= vk::PipelineStageFlagBits::eFragmentShader;
@@ -1213,8 +1201,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		m_context.GetCommandScheduler().EndRendering();
 		ShaderWriteBarrier(vk_buffer, shader_write_stages);
 	}
-	LogDrawPhase(draw.name, "DrawComplete");
-	if (set_auto_debug) {
+	LogDrawPhase(draw.Name(), "DrawComplete");
+	if (!draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
 	}
 }
@@ -1248,7 +1236,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	}
 
 	if (graphics_debug_dump_enabled()) {
-		sh_print("GraphicsRenderDrawIndex():Shader:", sh_ctx);
+		LOGF("GraphicsRenderDrawIndex():Shader:\n");
 		uc_print("GraphicsRenderDrawIndex():UserConfig:", ucfg);
 		hw_print(buffer);
 
@@ -1304,34 +1292,30 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 		index_source.size      = expanded_indices.size() * sizeof(uint16_t);
 	}
 
-	const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex, args.index_count,
+	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndex, args.index_count,
 	                        args.instance_count, args.first_instance};
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, true,
-	                            state)) {
+	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return;
 	}
 
-	RefreshShaders(buffer, draw, true, state);
+	RefreshShaders(buffer, draw, state);
 
-	LogDrawStateIfNeeded(buffer, draw, state, true, false, args.index_type_and_size,
+	LogDrawStateIfNeeded(buffer, draw, state, args.index_type_and_size,
 	                     args.index_addr);
 
 	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto vertex_offset =
-	    indirect
-	        ? args.base_vertex
-	        : ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) + args.base_vertex;
+	const auto [vertex_offset, instance_offset] =
+	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
+	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
 
 	DrawEmitInfo emit {};
-	emit.indexed       = true;
-	emit.vertex_offset = vertex_offset;
-	emit.first_instance =
-	    indirect ? args.first_instance : ResolveInstanceOffset(state.vs_input_info);
+	emit.vertex_offset  = vertex_offset + args.base_vertex;
+	emit.first_instance = instance_offset;
 
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
-	                    primitive_restart, true, true, false);
+	                    primitive_restart);
 	ResetBindings();
 }
 
@@ -1364,7 +1348,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	}
 
 	if (graphics_debug_dump_enabled()) {
-		sh_print("GraphicsRenderDrawIndexAuto():Shader:", sh_ctx);
+		LOGF("GraphicsRenderDrawIndexAuto():Shader:\n");
 		uc_print("GraphicsRenderDrawIndexAuto():UserConfig:", ucfg);
 		hw_print(buffer);
 
@@ -1380,12 +1364,11 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 
 	hw_check(buffer);
 
-	const DrawCallInfo draw {"DrawIndexAuto", CommandBufferDebugOp::DrawIndexAuto,
+	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndexAuto,
 	                         args.vertex_count, args.instance_count, args.first_instance};
 
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, false,
-	                            state)) {
+	if (!PrepareDrawRenderState(buffer, draw, args.render_target_slice_offset, state)) {
 		ResetBindings();
 		return;
 	}
@@ -1395,11 +1378,11 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		ResetBindings();
 		return;
 	}
-	RefreshShaders(buffer, draw, false, state);
+	RefreshShaders(buffer, draw, state);
 
-	const bool rect_list = topology == vk::PrimitiveTopology::ePatchList;
-	if (rect_list && state.vs_input_info.buffers_num == 0 &&
-	    state.vs_input_info.stage.program->param_export_mask == 0 &&
+	const bool rect_list = ucfg.GetPrimType() == Prospero::PrimitiveType::kRectList;
+	if (rect_list && state.vertex_info[0].buffers_num == 0 &&
+	    state.vertex_info[0].stage.program->param_export_mask == 0 &&
 	    state.ps_input_info.input_num != 0) {
 		if (graphics_debug_dump_enabled()) {
 			LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: "
@@ -1411,23 +1394,18 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 		return;
 	}
 
-	LogDrawStateIfNeeded(buffer, draw, state, false,
-	                     ucfg.GetPrimType() == Prospero::PrimitiveType::kRectListLegacy, 0,
-	                     nullptr);
+	LogDrawStateIfNeeded(buffer, draw, state, 0, nullptr);
 
 	const bool indirect = args.offset_source == DrawOffsetSource::IndirectArgs;
-	const auto vertex_offset =
-	    indirect ? static_cast<int32_t>(args.first_vertex)
-	             : ResolveVertexOffset(ucfg.GetIndexOffset(), state.vs_input_info) +
-	                   static_cast<int32_t>(args.first_vertex);
+	const auto [vertex_offset, instance_offset] =
+	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
+	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
 	DrawEmitInfo emit {};
-	emit.first_vertex = static_cast<uint32_t>(vertex_offset);
-	emit.first_instance =
-	    indirect ? args.first_instance : ResolveInstanceOffset(state.vs_input_info);
+	emit.first_vertex = static_cast<uint32_t>(vertex_offset + static_cast<int32_t>(args.first_vertex));
+	emit.first_instance = instance_offset;
 
 	DrawIndexBufferSource index_source {};
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false,
-	                    false, true);
+	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false);
 	ResetBindings();
 }
 

src/graphics/host_gpu/renderer/renderDraw.h

@@ -2,14 +2,14 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 
 #include <cstdint>
+#include <utility>
 
 namespace Libs::Graphics {
 
 struct ShaderVertexInputInfo;
 
-[[nodiscard]] int32_t  ResolveVertexOffset(uint32_t                     index_offset,
-                                           const ShaderVertexInputInfo& vs_input_info);
-[[nodiscard]] uint32_t ResolveInstanceOffset(const ShaderVertexInputInfo& vs_input_info);
+[[nodiscard]] std::pair<int32_t, uint32_t>
+ResolveDrawOffsets(uint32_t index_offset, const ShaderVertexInputInfo& vs_input_info);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/renderTarget.h

@@ -5,7 +5,6 @@
 
 #include <array>
 #include <cstdint>
-#include <type_traits>
 
 namespace Libs::Graphics {
 
@@ -67,41 +66,6 @@ inline constexpr TargetViewInfo ResolveTargetViewInfo(uint32_t base_layer, uint3
 	        base_layer, last_layer - base_layer + 1u, last_layer + 1u};
 }
 
-#pragma pack(push, 1)
-
-struct PipelineStencilStaticState {
-	vk::StencilOp failOp      = vk::StencilOp::eKeep;
-	vk::StencilOp passOp      = vk::StencilOp::eKeep;
-	vk::StencilOp depthFailOp = vk::StencilOp::eKeep;
-	vk::CompareOp compareOp   = vk::CompareOp::eNever;
-};
-
-struct PipelineStencilDynamicState {
-	uint32_t compareMask = 0;
-	uint32_t writeMask   = 0;
-	uint32_t reference   = 0;
-};
-
-#pragma pack(pop)
-
-inline constexpr bool stencil_face_accesses_attachment(const PipelineStencilStaticState&  state,
-                                                       const PipelineStencilDynamicState& dynamic) {
-	return state.compareOp != vk::CompareOp::eAlways ||
-	       (dynamic.writeMask != 0 &&
-	        (state.failOp != vk::StencilOp::eKeep || state.passOp != vk::StencilOp::eKeep ||
-	         state.depthFailOp != vk::StencilOp::eKeep));
-}
-
-static_assert(std::is_trivially_copyable_v<PipelineStencilStaticState>);
-static_assert(std::is_standard_layout_v<PipelineStencilStaticState>);
-static_assert(alignof(PipelineStencilStaticState) == 1);
-static_assert(sizeof(PipelineStencilStaticState) ==
-              sizeof(vk::StencilOp) * 3 + sizeof(vk::CompareOp));
-static_assert(std::is_trivially_copyable_v<PipelineStencilDynamicState>);
-static_assert(std::is_standard_layout_v<PipelineStencilDynamicState>);
-static_assert(alignof(PipelineStencilDynamicState) == 1);
-static_assert(sizeof(PipelineStencilDynamicState) == sizeof(uint32_t) * 3);
-
 } // namespace Libs::Graphics
 
 #endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERTARGET_H_

src/graphics/host_gpu/shaders/gpu_tiler_standard64.inc

@@ -1,29 +1,17 @@
+#include "gpu_tiler_standard.inc"
 uint standard64_offset(uvec2 p) {
 	uint x = p.x, y = p.y;
+	uint offset = standard_offset(p);
 	switch (ELEMENT_BYTES) {
-	case 1: return (x & 0x0fu) ^ ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^
-	               ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
-	               ((y << 4) & 0x1f0u) ^ ((y << 5) & 0x400u) ^
-	               ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u);
-	case 2: return ((x << 1) & 0x00eu) ^ ((x << 4) & 0x080u) ^
-	               ((x << 5) & 0x200u) ^ ((x << 6) & 0x800u) ^
-	               ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
-	               ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^
-	               ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
-	case 4: return ((x << 2) & 0x00cu) ^ ((x << 5) & 0x080u) ^
-	               ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^
-	               ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
-	               ((y << 4) & 0x070u) ^ ((y << 5) & 0x100u) ^
-	               ((y << 6) & 0x400u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
-	case 8: return ((x << 3) & 0x008u) ^ ((x << 5) & 0x0c0u) ^
-	               ((x << 6) & 0x200u) ^ ((x << 7) & 0x800u) ^
-	               ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
-	               ((y << 4) & 0x030u) ^ ((y << 6) & 0x100u) ^
-	               ((y << 7) & 0x400u) ^ ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
-	default: return ((x << 6) & 0x0c0u) ^ ((x << 7) & 0x200u) ^
-	                ((x << 8) & 0x800u) ^ ((x << 9) & 0x2000u) ^
-	                ((x << 10) & 0x8000u) ^ ((y << 4) & 0x030u) ^
-	                ((y << 6) & 0x100u) ^ ((y << 7) & 0x400u) ^
-	                ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
+	case 1: return offset ^ ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
+	                       ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u);
+	case 2: return offset ^ ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^
+	                       ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
+	case 4: return offset ^ ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
+	                       ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u);
+	case 8: return offset ^ ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^
+	                       ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
+	default: return offset ^ ((x << 9) & 0x2000u) ^ ((x << 10) & 0x8000u) ^
+	                        ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u);
 	}
 }

src/graphics/host_gpu/vulkanCommon.cpp

@@ -17,14 +17,22 @@ struct FormatMapping {
 
 constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k8UNorm, vk::Format::eR8Unorm},
+    {Prospero::BufferFormat::k8SNorm, vk::Format::eR8Snorm},
+    {Prospero::BufferFormat::k8UScaled, vk::Format::eR8Uscaled},
+    {Prospero::BufferFormat::k8SScaled, vk::Format::eR8Sscaled},
     {Prospero::BufferFormat::k8UInt, vk::Format::eR8Uint},
+    {Prospero::BufferFormat::k8SInt, vk::Format::eR8Sint},
     {Prospero::BufferFormat::k16UNorm, vk::Format::eR16Unorm},
     {Prospero::BufferFormat::k16SNorm, vk::Format::eR16Snorm},
+    {Prospero::BufferFormat::k16UScaled, vk::Format::eR16Uscaled},
+    {Prospero::BufferFormat::k16SScaled, vk::Format::eR16Sscaled},
     {Prospero::BufferFormat::k16UInt, vk::Format::eR16Uint},
     {Prospero::BufferFormat::k16SInt, vk::Format::eR16Sint},
     {Prospero::BufferFormat::k16Float, vk::Format::eR16Sfloat},
     {Prospero::BufferFormat::k8_8UNorm, vk::Format::eR8G8Unorm},
     {Prospero::BufferFormat::k8_8SNorm, vk::Format::eR8G8Snorm},
+    {Prospero::BufferFormat::k8_8UScaled, vk::Format::eR8G8Uscaled},
+    {Prospero::BufferFormat::k8_8SScaled, vk::Format::eR8G8Sscaled},
     {Prospero::BufferFormat::k8_8UInt, vk::Format::eR8G8Uint},
     {Prospero::BufferFormat::k8_8SInt, vk::Format::eR8G8Sint},
     {Prospero::BufferFormat::k32UInt, vk::Format::eR32Uint},
@@ -32,14 +40,19 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32Float, vk::Format::eR32Sfloat},
     {Prospero::BufferFormat::k16_16UNorm, vk::Format::eR16G16Unorm},
     {Prospero::BufferFormat::k16_16SNorm, vk::Format::eR16G16Snorm},
+    {Prospero::BufferFormat::k16_16UScaled, vk::Format::eR16G16Uscaled},
+    {Prospero::BufferFormat::k16_16SScaled, vk::Format::eR16G16Sscaled},
     {Prospero::BufferFormat::k16_16UInt, vk::Format::eR16G16Uint},
     {Prospero::BufferFormat::k16_16SInt, vk::Format::eR16G16Sint},
     {Prospero::BufferFormat::k16_16Float, vk::Format::eR16G16Sfloat},
     {Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
     {Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
+    {Prospero::BufferFormat::k10_10_10_2SNorm, vk::Format::eA2B10G10R10SnormPack32},
     {Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
     {Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
     {Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},
+    {Prospero::BufferFormat::k8_8_8_8UScaled, vk::Format::eR8G8B8A8Uscaled},
+    {Prospero::BufferFormat::k8_8_8_8SScaled, vk::Format::eR8G8B8A8Sscaled},
     {Prospero::BufferFormat::k8_8_8_8UInt, vk::Format::eR8G8B8A8Uint},
     {Prospero::BufferFormat::k8_8_8_8SInt, vk::Format::eR8G8B8A8Sint},
     {Prospero::BufferFormat::k32_32UInt, vk::Format::eR32G32Uint},
@@ -47,6 +60,8 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k32_32Float, vk::Format::eR32G32Sfloat},
     {Prospero::BufferFormat::k16_16_16_16UNorm, vk::Format::eR16G16B16A16Unorm},
     {Prospero::BufferFormat::k16_16_16_16SNorm, vk::Format::eR16G16B16A16Snorm},
+    {Prospero::BufferFormat::k16_16_16_16UScaled, vk::Format::eR16G16B16A16Uscaled},
+    {Prospero::BufferFormat::k16_16_16_16SScaled, vk::Format::eR16G16B16A16Sscaled},
     {Prospero::BufferFormat::k16_16_16_16UInt, vk::Format::eR16G16B16A16Uint},
     {Prospero::BufferFormat::k16_16_16_16SInt, vk::Format::eR16G16B16A16Sint},
     {Prospero::BufferFormat::k16_16_16_16Float, vk::Format::eR16G16B16A16Sfloat},
@@ -63,7 +78,8 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k8_8_8_8Srgb, vk::Format::eR8G8B8A8Srgb},
     {Prospero::BufferFormat::k9_9_9_5Float, vk::Format::eE5B9G9R9UfloatPack32},
     {Prospero::BufferFormat::k5_6_5UNorm, vk::Format::eB5G6R5UnormPack16},
-    {Prospero::BufferFormat::k5_5_5_1UNorm, vk::Format::eR5G5B5A1UnormPack16},
+    {Prospero::BufferFormat::k5_5_5_1UNorm, vk::Format::eA1R5G5B5UnormPack16},
+    {Prospero::BufferFormat::k1_5_5_5UNorm, vk::Format::eR5G5B5A1UnormPack16},
     {Prospero::BufferFormat::k4_4_4_4UNorm, vk::Format::eR4G4B4A4UnormPack16},
     {Prospero::BufferFormat::kBc1UNorm, vk::Format::eBc1RgbaUnormBlock},
     {Prospero::BufferFormat::kBc1Srgb, vk::Format::eBc1RgbaSrgbBlock},
@@ -107,4 +123,14 @@ void RequireVulkanSuccess(vk::Result result, const char* operation) {
 	}
 }
 
+vk::ShaderModule CompileSPV(std::span<const uint32_t> code, vk::Device device) {
+	vk::ShaderModuleCreateInfo create_info {};
+	create_info.codeSize    = code.size_bytes();
+	create_info.pCode       = code.data();
+	vk::ShaderModule module = nullptr;
+	RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module),
+	                     "create SPIR-V shader module");
+	return module;
+}
+
 } // namespace Libs::Graphics

src/graphics/host_gpu/vulkanCommon.h

@@ -15,6 +15,7 @@
 
 #include <cstdint>
 #include <fmt/format.h>
+#include <span>
 #include <string>
 #include <utility>
 #include <vector>
@@ -27,6 +28,7 @@ using VulkanMemoryBarrier = vk::MemoryBarrier;
 
 vk::Format  VulkanFormat(Prospero::BufferFormat guest_format);
 void        RequireVulkanSuccess(vk::Result result, const char* operation);
+vk::ShaderModule CompileSPV(std::span<const uint32_t> code, vk::Device device);
 
 template <typename Handle, typename... Args>
 void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<Args...> format,
@@ -38,7 +40,6 @@ void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<A
 
 	const auto                      name = fmt::format(format, std::forward<Args>(args)...);
 	vk::DebugUtilsObjectNameInfoEXT info {};
-	info.sType        = vk::StructureType::eDebugUtilsObjectNameInfoEXT;
 	info.objectType   = Handle::objectType;
 	info.objectHandle = static_cast<uint64_t>(
 	    reinterpret_cast<uintptr_t>(static_cast<typename Handle::CType>(handle)));

src/graphics/presentation/videoOut.cpp

@@ -237,7 +237,9 @@ private:
 
 struct VideoOutDriver::Impl {
 public:
-	static constexpr int VIDEO_OUT_NUM_MAX = 4;
+	static constexpr std::array VIDEO_OUT_BUSES {
+	    VIDEO_OUT_BUS_TYPE_MAIN, VIDEO_OUT_BUS_TYPE_OVERLAY, VIDEO_OUT_BUS_TYPE_SUB};
+	static constexpr int VIDEO_OUT_NUM_MAX = VIDEO_OUT_BUSES.size() + 1;
 
 	Impl(uint32_t width, uint32_t height, Graphics::Presenter& presenter)
 	    : m_renderer(presenter.Renderer()), m_presenter(presenter), m_flip_queue(presenter) {
@@ -248,7 +250,7 @@ public:
 	~Impl();
 	KYTY_CLASS_NO_COPY(Impl);
 
-	int             Open(int bus_type);
+	int             Open(int bus_type, int index);
 	bool            Close(int handle);
 	VideoOutConfig* Get(int handle);
 	VideoOutConfig* Get(int handle, uint64_t& generation);
@@ -610,12 +612,17 @@ void VideoOutDriver::Impl::Init(uint32_t width, uint32_t height) {
 	}
 }
 
-int VideoOutDriver::Impl::Open(int bus_type) {
+int VideoOutDriver::Impl::Open(int bus_type, int index) {
+	const auto bus = std::find(VIDEO_OUT_BUSES.begin(), VIDEO_OUT_BUSES.end(), bus_type);
+	if (bus == VIDEO_OUT_BUSES.end()) {
+		return VIDEO_OUT_ERROR_INVALID_VALUE;
+	}
+	EXIT_NOT_IMPLEMENTED(index != 0);
 	Common::LockGuard lock(m_mutex);
 
-	const int handle = bus_type + 1;
+	const int handle = static_cast<int>(bus - VIDEO_OUT_BUSES.begin()) + 1;
 	if (m_video_out_ctx[handle].opened) {
-		return -1;
+		return VIDEO_OUT_ERROR_RESOURCE_BUSY;
 	}
 	auto&             config = m_video_out_ctx[handle];
 	Common::LockGuard config_lock(config.mutex);
@@ -1152,7 +1159,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 	Graphics::RenderDocOnGuestFlip(m_presenter.Renderer());
 
 	if (Config::GraphicsDebugDumpEnabled() &&
-	    Config::GetPrintfDirection() != Config::OutputDirection::Silent) {
+	    Config::GetPrintfDirection() != Config::LogDirection::Silent) {
 		LOGF("Flip done: %d\n", r.index);
 	}
 
@@ -1169,21 +1176,9 @@ KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void*
 	PRINT_NAME();
 
 	EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0);
-	if (bus_type != VIDEO_OUT_BUS_TYPE_MAIN && bus_type != VIDEO_OUT_BUS_TYPE_OVERLAY &&
-	    bus_type != VIDEO_OUT_BUS_TYPE_SUB) {
-		return VIDEO_OUT_ERROR_INVALID_VALUE;
-	}
-	EXIT_NOT_IMPLEMENTED(index != 0);
-
 	LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast<uint64_t>(param));
 
-	int handle = DriverState().Open(bus_type);
-
-	if (handle < 0) {
-		return VIDEO_OUT_ERROR_RESOURCE_BUSY;
-	}
-
-	return handle;
+	return DriverState().Open(bus_type, index);
 }
 
 KYTY_SYSV_ABI int VideoOutClose(int handle) {

src/graphics/presentation/window/hostInput.cpp

@@ -220,6 +220,8 @@ uint32_t DefaultKeyboardButton(int key_code) {
 		case SDLK_l: return Controller::PAD_BUTTON_CIRCLE;
 		case SDLK_q: return Controller::PAD_BUTTON_L1;
 		case SDLK_e: return Controller::PAD_BUTTON_R1;
+		case SDLK_z: return Controller::PAD_BUTTON_L2;
+		case SDLK_c: return Controller::PAD_BUTTON_R2;
 		case SDLK_LSHIFT: return Controller::PAD_BUTTON_L3;
 		case SDLK_LCTRL: return Controller::PAD_BUTTON_R3;
 		case SDLK_RETURN:

src/graphics/presentation/window/vulkanWindow.cpp

@@ -64,7 +64,6 @@ struct VulkanExtensions {
 
 vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
 	vk::PhysicalDeviceVulkan12Features features {};
-	features.sType                     = vk::StructureType::ePhysicalDeviceVulkan12Features;
 	features.samplerMirrorClampToEdge  = VK_TRUE;
 	features.timelineSemaphore         = VK_TRUE;
 	features.shaderOutputLayer         = VK_TRUE;
@@ -76,7 +75,6 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 
 vk::PhysicalDeviceVulkan13Features WindowContext::RequiredVulkan13Features() noexcept {
 	vk::PhysicalDeviceVulkan13Features features {};
-	features.sType            = vk::StructureType::ePhysicalDeviceVulkan13Features;
 	features.dynamicRendering = VK_TRUE;
 	features.synchronization2 = VK_TRUE;
 	return features;
@@ -201,35 +199,24 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		vk::PhysicalDeviceFeatures2 device_features2 {};
 
 		vk::PhysicalDeviceVulkan13Features features13 {};
-		features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-		features13.pNext = nullptr;
 
 		vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {};
-		color_write_ext.sType = vk::StructureType::ePhysicalDeviceColorWriteEnableFeaturesEXT;
-		color_write_ext.pNext = nullptr;
 
 		vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip_enable {};
-		depth_clip_enable.sType = vk::StructureType::ePhysicalDeviceDepthClipEnableFeaturesEXT;
 		depth_clip_enable.pNext = &color_write_ext;
 
 		vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
-		depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
 		depth_clip_control.pNext = &depth_clip_enable;
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
-		features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
 #if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
 #else
 		vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
-		fragment_barycentric.sType =
-		    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
 		features13.pNext = &features12;
-
-		device_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
 		device_features2.pNext = &features13;
 
 		device.getFeatures2(&device_features2);
@@ -506,83 +493,31 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 	}
 }
 
-static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device,
-                                          GraphicContext&    graphics) {
-	EXIT_IF(physical_device == nullptr);
-
-	vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {};
-	subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlProperties;
-	subgroup_size_control.pNext = nullptr;
-
-	vk::PhysicalDeviceVulkan11Properties properties11 {};
-	properties11.sType = vk::StructureType::ePhysicalDeviceVulkan11Properties;
-	properties11.pNext = &subgroup_size_control;
-
-	vk::PhysicalDeviceProperties2 properties2 {};
-	properties2.sType = vk::StructureType::ePhysicalDeviceProperties2;
-	properties2.pNext = &properties11;
-
-	physical_device.getProperties2(&properties2);
-
-	vk::PhysicalDeviceVulkan13Features features13 {};
-	features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-	features13.pNext = nullptr;
-
-	vk::PhysicalDeviceFeatures2 features2 {};
-	features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
-	features2.pNext = &features13;
-
-	physical_device.getFeatures2(&features2);
-
-	graphics.subgroup_size                 = properties11.subgroupSize;
-	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
-	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
-	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
-	graphics.compute_subgroup_size_control_enabled =
-	    features13.subgroupSizeControl == VK_TRUE &&
-	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
-	    subgroup_size_control.minSubgroupSize <= 64 &&
-	    subgroup_size_control.maxSubgroupSize >= 64;
-
-	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
-	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
-	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
-	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
-	     graphics.SupportsComputeWave64() ? "true" : "false");
-}
-
-static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r,
-                                     uint32_t                        queue_family,
-                                     const std::vector<const char*>& device_extensions,
-                                     GraphicContext&                 graphics) {
+static vk::Device VulkanCreateDevice(GraphicContext& graphics,
+	                                 const std::vector<const char*>& device_extensions) {
+	const auto physical_device = graphics.physical_device;
+	const auto queue_family    = graphics.queue_family;
 	EXIT_IF(physical_device == nullptr);
 	EXIT_IF(queue_family == static_cast<uint32_t>(-1));
 
 	const float               queue_priority = 1.0f;
 	vk::DeviceQueueCreateInfo queue_create_info {};
-	queue_create_info.sType            = vk::StructureType::eDeviceQueueCreateInfo;
 	queue_create_info.queueFamilyIndex = queue_family;
 	queue_create_info.queueCount       = 1;
 	queue_create_info.pQueuePriorities = &queue_priority;
 
 	vk::PhysicalDeviceColorWriteEnableFeaturesEXT color_write_ext {};
-	color_write_ext.sType = vk::StructureType::ePhysicalDeviceColorWriteEnableFeaturesEXT;
-	color_write_ext.pNext = nullptr;
 	color_write_ext.colorWriteEnable = VK_TRUE;
 
 	vk::PhysicalDeviceDepthClipEnableFeaturesEXT depth_clip_enable {};
-	depth_clip_enable.sType = vk::StructureType::ePhysicalDeviceDepthClipEnableFeaturesEXT;
 	depth_clip_enable.pNext = &color_write_ext;
 	depth_clip_enable.depthClipEnable = VK_TRUE;
 
 	vk::PhysicalDeviceDepthClipControlFeaturesEXT depth_clip_control {};
-	depth_clip_control.sType = vk::StructureType::ePhysicalDeviceDepthClipControlFeaturesEXT;
 	// MoltenVK lacks VK_EXT_depth_clip_enable and VK_EXT_color_write_enable, so drop those
 	// feature structs from the chain on macOS (the renderer falls back to default depth
 	// clipping and static color-write masks).
-#if defined(__APPLE__)
-	depth_clip_control.pNext = nullptr;
-#else
+#if !defined(__APPLE__)
 	depth_clip_control.pNext = &depth_clip_enable;
 #endif
 	depth_clip_control.depthClipControl = VK_TRUE;
@@ -591,40 +526,19 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	features12.pNext = &depth_clip_control;
 
 	vk::PhysicalDeviceVulkan13Features supported_features13 {};
-	supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
-
-	vk::PhysicalDeviceVulkan12Features supported_features12 {};
-	supported_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
-	supported_features12.pNext = nullptr;
-	supported_features13.pNext = &supported_features12;
-
-#if !defined(__APPLE__)
-	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_fragment_barycentric {};
-	supported_fragment_barycentric.sType =
-	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
-	supported_fragment_barycentric.pNext = nullptr;
-	supported_features12.pNext           = &supported_fragment_barycentric;
-#endif
 
 	const auto robustness2_ext_enabled =
 	    HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT supported_robustness2 {};
-	supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
-	supported_robustness2.pNext = nullptr;
 	if (robustness2_ext_enabled) {
-#if defined(__APPLE__)
-		supported_features12.pNext = &supported_robustness2;
-#else
-		supported_fragment_barycentric.pNext = &supported_robustness2;
-#endif
+		supported_features13.pNext = &supported_robustness2;
 	}
 
 	const bool mesh_extension = HasExtension(device_extensions, VK_EXT_MESH_SHADER_EXTENSION_NAME);
 	vk::PhysicalDeviceMeshShaderFeaturesEXT supported_mesh {};
 	supported_mesh.pNext = &supported_features13;
 	vk::PhysicalDeviceFeatures2 supported_features2 {};
-	supported_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
 	supported_features2.pNext = mesh_extension ? static_cast<void*>(&supported_mesh)
 	                                           : static_cast<void*>(&supported_features13);
 	const bool feedback_extensions =
@@ -645,53 +559,53 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		supported_features2.pNext = &provoking_vertex;
 	}
 	physical_device.getFeatures2(&supported_features2);
+	graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader;
+
+	vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {};
+
+	vk::PhysicalDeviceVulkan11Properties properties11 {};
+	properties11.pNext = &subgroup_size_control;
+
+	vk::PhysicalDeviceProperties2 properties2 {};
+	properties2.pNext = &properties11;
+
+	if (graphics.mesh_shader_enabled) {
+		subgroup_size_control.pNext = &graphics.mesh_shader_properties;
+	}
+	physical_device.getProperties2(&properties2);
+
+	graphics.subgroup_size                 = properties11.subgroupSize;
+	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
+	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
+	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
+	graphics.compute_subgroup_size_control_enabled =
+	    supported_features13.subgroupSizeControl == VK_TRUE &&
+	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
+	    subgroup_size_control.minSubgroupSize <= 64 &&
+	    subgroup_size_control.maxSubgroupSize >= 64;
+
+	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
+	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
+	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
+	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
+	     graphics.SupportsComputeWave64() ? "true" : "false");
 	graphics.provoking_vertex_last_enabled = provoking_extension && provoking_vertex.provokingVertexLast;
 	graphics.attachment_feedback_loop_enabled =
 	    feedback_extensions && feedback_layout.attachmentFeedbackLoopLayout &&
 	    feedback_dynamic.attachmentFeedbackLoopDynamicState;
 	LOGF("Vulkan depth feedback support: %s\n",
 	     graphics.attachment_feedback_loop_enabled ? "true" : "false");
-	graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader;
 	if (graphics.mesh_shader_enabled) {
-		vk::PhysicalDeviceProperties2 properties {};
-		properties.pNext = &graphics.mesh_shader_properties;
-		physical_device.getProperties2(&properties);
 		LOGF("Vulkan MeshEXT: invocations=%u vertices=%u primitives=%u shared=%u\n",
 		     graphics.mesh_shader_properties.maxMeshWorkGroupInvocations,
 		     graphics.mesh_shader_properties.maxMeshOutputVertices,
 		     graphics.mesh_shader_properties.maxMeshOutputPrimitives,
 		     graphics.mesh_shader_properties.maxMeshSharedMemorySize);
 	}
-	const auto required_features12 = WindowContext::RequiredVulkan12Features();
-	const auto required_features13 = WindowContext::RequiredVulkan13Features();
-	EXIT_NOT_IMPLEMENTED(required_features12.samplerMirrorClampToEdge == VK_TRUE &&
-	                     supported_features12.samplerMirrorClampToEdge != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.timelineSemaphore == VK_TRUE &&
-	                     supported_features12.timelineSemaphore != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features13.dynamicRendering == VK_TRUE &&
-	                     supported_features13.dynamicRendering != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features13.synchronization2 == VK_TRUE &&
-	                     supported_features13.synchronization2 != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.multiViewport != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(supported_features2.features.fillModeNonSolid != VK_TRUE);
+	// VulkanFindPhysicalDevice already checked the required creation features. These two
+	// requirements are specific to this creation path and are not part of device selection.
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderOutputLayer == VK_TRUE &&
-	                     supported_features12.shaderOutputLayer != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderOutputViewportIndex == VK_TRUE &&
-	                     supported_features12.shaderOutputViewportIndex != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.bufferDeviceAddress == VK_TRUE &&
-	                     supported_features12.bufferDeviceAddress != VK_TRUE);
-	EXIT_NOT_IMPLEMENTED(required_features12.shaderBufferInt64Atomics == VK_TRUE &&
-	                     supported_features12.shaderBufferInt64Atomics != VK_TRUE);
-#if !defined(__APPLE__)
-	EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
-#endif
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
 	device_features.samplerAnisotropy        = VK_TRUE;
@@ -715,13 +629,10 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	device_features.shaderInt64 = VK_TRUE;
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
-	robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
 #if defined(__APPLE__)
 	robustness2.pNext = &features12;
 #else
 	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
-	fragment_barycentric.sType =
-	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
 	fragment_barycentric.pNext                     = &features12;
 	fragment_barycentric.fragmentShaderBarycentric = VK_TRUE;
 	robustness2.pNext                              = &fragment_barycentric;
@@ -732,11 +643,7 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		robustness2.nullDescriptor      = supported_robustness2.nullDescriptor;
 	}
 
-	const bool subgroup_size_control_enabled =
-	    graphics.compute_subgroup_size_control_enabled &&
-	    supported_features13.subgroupSizeControl == VK_TRUE;
-
-	auto features13 = required_features13;
+	auto features13 = WindowContext::RequiredVulkan13Features();
 #if defined(__APPLE__)
 	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
 	                                           : static_cast<void*>(&features12);
@@ -745,7 +652,8 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	                                           : static_cast<void*>(&fragment_barycentric);
 #endif
 	features13.robustImageAccess   = supported_features13.robustImageAccess;
-	features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
+	features13.subgroupSizeControl =
+	    graphics.compute_subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
 
 	LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
 	     features13.robustImageAccess == VK_TRUE ? "true" : "false",
@@ -755,7 +663,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	vk::PhysicalDeviceMeshShaderFeaturesEXT mesh_features {};
 	mesh_features.pNext                 = &features13;
 	mesh_features.meshShader            = graphics.mesh_shader_enabled;
-	create_info.sType                   = vk::StructureType::eDeviceCreateInfo;
 	feedback_dynamic.pNext =
 	    mesh_extension ? static_cast<void*>(&mesh_features) : static_cast<void*>(&features13);
 	create_info.pNext = graphics.attachment_feedback_loop_enabled
@@ -766,7 +673,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 		provoking_vertex.transformFeedbackPreservesProvokingVertex = VK_FALSE;
 		create_info.pNext = &provoking_vertex;
 	}
-	create_info.flags                   = {};
 	create_info.pQueueCreateInfos       = &queue_create_info;
 	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
@@ -796,8 +702,6 @@ static void VulkanGetExtensions(SDL_Window* window, VulkanExtensions& r) {
 
 	r.required_extensions =
 	    std::vector<const char*>(required_extensions_count); // @suppress("Ambiguous problem")
-	std::memset(r.required_extensions.data(), 0,
-	            sizeof(const char*) * r.required_extensions.size());
 
 	sdl_result = SDL_Vulkan_GetInstanceExtensions(window, &required_extensions_count,
 	                                              r.required_extensions.data());
@@ -989,8 +893,6 @@ void WindowContext::CreateVulkan() {
 	VulkanCheckInstanceVersion();
 
 	vk::ApplicationInfo app_info {};
-	app_info.sType              = vk::StructureType::eApplicationInfo;
-	app_info.pNext              = nullptr;
 	app_info.pApplicationName   = "Kyty";
 	app_info.applicationVersion = 1;
 	app_info.pEngineName        = "Kyty";
@@ -1019,17 +921,11 @@ void WindowContext::CreateVulkan() {
 	}
 
 	vk::ValidationFeaturesEXT validation_features {};
-	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
-	validation_features.pNext                          = nullptr;
 	validation_features.enabledValidationFeatureCount  = enabled_features_count;
 	validation_features.pEnabledValidationFeatures     = enabled_features;
-	validation_features.disabledValidationFeatureCount = 0;
-	validation_features.pDisabledValidationFeatures    = nullptr;
 
 	vk::DebugUtilsMessengerCreateInfoEXT dbg_create_info {};
-	dbg_create_info.sType           = vk::StructureType::eDebugUtilsMessengerCreateInfoEXT;
 	dbg_create_info.pNext           = &validation_features;
-	dbg_create_info.flags           = {};
 	dbg_create_info.messageSeverity = vk::DebugUtilsMessageSeverityFlagBitsEXT::eVerbose |
 	                                  vk::DebugUtilsMessageSeverityFlagBitsEXT::eInfo |
 	                                  vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning |
@@ -1038,12 +934,9 @@ void WindowContext::CreateVulkan() {
 	                                  vk::DebugUtilsMessageTypeFlagBitsEXT::eValidation |
 	                                  vk::DebugUtilsMessageTypeFlagBitsEXT::ePerformance;
 	dbg_create_info.pfnUserCallback = VulkanDebugMessengerCallback;
-	dbg_create_info.pUserData       = nullptr;
 
 	vk::InstanceCreateInfo inst_info {};
-	inst_info.sType                   = vk::StructureType::eInstanceCreateInfo;
 	inst_info.pNext                   = (r.enable_validation_layers ? &dbg_create_info : nullptr);
-	inst_info.flags                   = {};
 #if defined(__APPLE__)
 	// MoltenVK requires VK_KHR_portability_enumeration + flag to surface
 	// portability devices. Without this, enumeratePhysicalDevices hides the
@@ -1110,10 +1003,8 @@ void WindowContext::CreateVulkan() {
 	}
 #endif
 
-	uint32_t queue_family = static_cast<uint32_t>(-1);
-
 	VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, surface_capabilities,
-	                         graphic_ctx.physical_device, queue_family);
+	                         graphic_ctx.physical_device, graphic_ctx.queue_family);
 
 	if (graphic_ctx.physical_device == nullptr) {
 		EXIT("Could not find suitable device");
@@ -1121,9 +1012,6 @@ void WindowContext::CreateVulkan() {
 
 	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
 	vk::PhysicalDeviceProperties2              physical_device_properties {};
-	push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties;
-	push_descriptor_properties.pNext = nullptr;
-	physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2;
 	physical_device_properties.pNext = &push_descriptor_properties;
 	graphic_ctx.physical_device.getProperties2(&physical_device_properties);
 	graphic_ctx.physical_device_properties = physical_device_properties.properties;
@@ -1172,16 +1060,12 @@ void WindowContext::CreateVulkan() {
 		}
 	}
 
-	VulkanInitSubgroupSizeControl(graphic_ctx.physical_device, graphic_ctx);
-
-	graphic_ctx.device = VulkanCreateDevice(graphic_ctx.physical_device, r, queue_family,
-	                                        device_extensions, graphic_ctx);
+	graphic_ctx.device = VulkanCreateDevice(graphic_ctx, device_extensions);
 	if (graphic_ctx.device == nullptr) {
 		EXIT("Could not create device");
 	}
 	VULKAN_HPP_DEFAULT_DISPATCHER.init(graphic_ctx.device);
-	graphic_ctx.queue_family = queue_family;
-	graphic_ctx.device.getQueue(queue_family, 0, &graphic_ctx.queue);
+	graphic_ctx.device.getQueue(graphic_ctx.queue_family, 0, &graphic_ctx.queue);
 	EXIT_IF(graphic_ctx.queue == nullptr);
 
 	if (!graphic_ctx.CreateAllocator()) {
@@ -1189,7 +1073,7 @@ void WindowContext::CreateVulkan() {
 	}
 
 	render_context = std::make_unique<RenderContext>(graphic_ctx);
-	LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
+	LibKernel::Memory::InstallGpuResources(render_context.get());
 	presenter = std::make_unique<Presenter>(*this);
 }
 

src/graphics/rt/hardware.cpp

@@ -1,77 +0,0 @@
-#include "graphics/rt/hardware.h"
-
-#include "common/assert.h"
-#include "common/logging/log.h"
-
-#include <algorithm>
-#include <cstring>
-
-namespace Libs::Graphics {
-namespace {
-
-bool ExtensionAvailable(const std::vector<vk::ExtensionProperties>& available, const char* name) {
-	return std::ranges::any_of(available, [name](const auto& extension) {
-		return std::strcmp(extension.extensionName, name) == 0;
-	});
-}
-
-void AppendExtensionIfMissing(std::vector<const char*>& extensions, const char* name) {
-	if (std::ranges::none_of(extensions, [name](const char* extension) {
-		    return std::strcmp(extension, name) == 0;
-	    })) {
-		extensions.push_back(name);
-	}
-}
-
-} // namespace
-
-void GraphicContext::AppendHardwareRayTracingDeviceExtensions(
-    const std::vector<vk::ExtensionProperties>& available_extensions,
-    std::vector<const char*>&                   device_extensions) {
-
-	const char* required[] = {
-	    VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME,
-	    VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME,
-	    VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME,
-	    VK_KHR_RAY_QUERY_EXTENSION_NAME,
-	};
-	for (const auto* extension: required) {
-		if (!ExtensionAvailable(available_extensions, extension)) {
-			LOGF("Vulkan RT: extension %s is unavailable; hardware ray tracing disabled\n",
-			     extension);
-			rt_extensions_enabled = false;
-			return;
-		}
-	}
-	for (const auto* extension: required) {
-		AppendExtensionIfMissing(device_extensions, extension);
-	}
-	if (ExtensionAvailable(available_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME)) {
-		AppendExtensionIfMissing(device_extensions, VK_KHR_SPIRV_1_4_EXTENSION_NAME);
-	}
-	if (ExtensionAvailable(available_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME)) {
-		AppendExtensionIfMissing(device_extensions, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
-	}
-
-	rt_extensions_enabled = true;
-	LOGF("Vulkan RT: enabling hardware ray query extensions\n");
-}
-
-void GraphicContext::LoadHardwareRayTracingFunctions() const {
-	EXIT_IF(device == nullptr);
-	if (!rt_extensions_enabled) {
-		return;
-	}
-
-	const auto& dispatcher = VULKAN_HPP_DEFAULT_DISPATCHER;
-	if (dispatcher.vkGetBufferDeviceAddressKHR == nullptr ||
-	    dispatcher.vkCreateAccelerationStructureKHR == nullptr ||
-	    dispatcher.vkDestroyAccelerationStructureKHR == nullptr ||
-	    dispatcher.vkGetAccelerationStructureBuildSizesKHR == nullptr ||
-	    dispatcher.vkCmdBuildAccelerationStructuresKHR == nullptr ||
-	    dispatcher.vkGetAccelerationStructureDeviceAddressKHR == nullptr) {
-		EXIT("Vulkan RT: failed to load required device functions\n");
-	}
-}
-
-} // namespace Libs::Graphics

src/graphics/rt/hardware.h

@@ -1,10 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_
-
-#include "graphics/host_gpu/graphicContext.h"
-
-#include <vector>
-
-namespace Libs::Graphics {} // namespace Libs::Graphics
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RT_HARDWARE_H_ */

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -1,4 +1,5 @@
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/Tessellation.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
@@ -47,14 +48,16 @@ const char* StageName(ShaderType stage) {
 	switch (stage) {
 		case ShaderType::Compute: return "CS";
 		case ShaderType::Vertex: return "VS";
+		case ShaderType::Local: return "LS";
+		case ShaderType::TessellationControl: return "HS";
+		case ShaderType::TessellationEvaluation: return "TES";
 		case ShaderType::Mesh: return "MS";
 		case ShaderType::Pixel: return "PS";
 		default: return "unknown";
 	}
 }
 
-void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase,
-                           const std::string& reason) {
+void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase) {
 	const auto* block        = cfg.FindBlock(cfg.failure_block);
 	const auto  start        = block != nullptr ? block->start_pc : UINT32_MAX;
 	const auto  end          = block != nullptr ? block->end_pc : UINT32_MAX;
@@ -68,7 +71,7 @@ void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg,
 	     CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
 	     static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
-	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
+	     static_cast<uint64_t>(cfg.back_edges.size()), cfg.unsupported_reason.c_str());
 }
 
 enum class EmbeddedFetchValueType {
@@ -84,7 +87,6 @@ struct EmbeddedFetchSgprInfo {
 	EmbeddedFetchValueType type      = EmbeddedFetchValueType::Unknown;
 	int                    attrib_id = 0;
 	uint32_t               value     = 0;
-	std::vector<uint32_t>  prolog_loads;
 };
 
 using EmbeddedFetchVectorLanes = std::map<uint64_t, EmbeddedFetchSgprInfo>;
@@ -130,20 +132,6 @@ uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) {
 	return inst.opcode == Decoder::Opcode::V_MAD_U64_U32 ? 2u : DecodedDstSize(inst);
 }
 
-bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) {
-	switch (opcode) {
-		case Decoder::Opcode::S_SETPC_B64:
-		case Decoder::Opcode::S_BRANCH:
-		case Decoder::Opcode::S_CBRANCH_SCC0:
-		case Decoder::Opcode::S_CBRANCH_SCC1:
-		case Decoder::Opcode::S_CBRANCH_VCCZ:
-		case Decoder::Opcode::S_CBRANCH_VCCNZ:
-		case Decoder::Opcode::S_CBRANCH_EXECZ:
-		case Decoder::Opcode::S_CBRANCH_EXECNZ: return true;
-		default: return false;
-	}
-}
-
 void ClearEmbeddedFetchSgprs(std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
                              const Decoder::Operand& dst, uint32_t size) {
 	if (!IsDecodedSgpr(dst)) {
@@ -226,6 +214,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
                                             const ShaderVertexInputInfo* input_info,
                                             uint32_t user_data_base, uint32_t user_data_count,
                                             uint32_t wave_size) {
+	const uint32_t    vertex_index_reg   = input_info->logical_stage == ShaderType::Local ? 2u : 5u;
+	const uint32_t    instance_index_reg = input_info->logical_stage == ShaderType::Local ? 5u : 8u;
 	EmbeddedFetchData data;
 	data.loads.reserve(input_info->resources_num);
 	int32_t vertex_offset_candidate   = -1;
@@ -242,7 +232,10 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 	EmbeddedFetchVectorLanes               vector_lanes;
 	const bool                             track_vector_lanes =
 	    std::none_of(decoded.instructions.begin(), decoded.instructions.end(),
-	                 [](const auto& inst) { return EmbeddedFetchHasBranch(inst.opcode); });
+	                 [](const auto& inst) {
+		                 return Decoder::IsDirectBranch(inst.opcode) ||
+		                        inst.opcode == Decoder::Opcode::S_SETPC_B64;
+	                 });
 
 	if (attrib_reg >= 0 && attrib_reg < static_cast<int>(sgprs.size())) {
 		sgprs[attrib_reg].type = EmbeddedFetchValueType::AttribTable;
@@ -263,16 +256,17 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 		// corresponding direct-draw offsets before fetching.
 		const bool vertex_index_accumulator =
 		    IsDecodedVgpr(inst.dst) &&
-		    (inst.dst.reg == 0 || (user_data_base == 8 && inst.dst.reg == 5));
+		    (inst.dst.reg == 0 || (user_data_base == 8 && inst.dst.reg == vertex_index_reg));
 		const bool instance_index_accumulator =
-		    IsDecodedVgpr(inst.dst) && (inst.dst.reg == (user_data_base == 8 ? 8u : 3u));
+		    IsDecodedVgpr(inst.dst) &&
+		    (inst.dst.reg == (user_data_base == 8 ? instance_index_reg : 3u));
 		uint32_t   sad_zero = 0;
 		const bool index_offset_add =
-		    (vertex_index_accumulator || instance_index_accumulator) &&
-		    IsDecodedSgpr(inst.src0) &&
+		    (vertex_index_accumulator || instance_index_accumulator) && IsDecodedSgpr(inst.src0) &&
 		    ((inst.opcode == Decoder::Opcode::V_ADD_I32 && IsDecodedVgpr(inst.src1) &&
 		      inst.src1.reg == inst.dst.reg) ||
-		     (user_data_base == 8 && (inst.dst.reg == 5 || inst.dst.reg == 8) &&
+		     (user_data_base == 8 &&
+		      (inst.dst.reg == vertex_index_reg || inst.dst.reg == instance_index_reg) &&
 		      inst.opcode == Decoder::Opcode::V_SAD_U32 && IsDecodedVgpr(inst.src2) &&
 		      inst.src2.reg == inst.dst.reg &&
 		      TryDecodedOperandConstant(sgprs, inst.src1, sad_zero) && sad_zero == 0));
@@ -331,7 +325,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 						auto& dst = sgprs[DecodedSgprReg(inst.dst)];
 						dst.type  = EmbeddedFetchValueType::Constant;
 						dst.value = value;
-						dst.prolog_loads.clear();
 					} else {
 						ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1);
 					}
@@ -342,7 +335,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 					auto& dst = sgprs[DecodedSgprReg(inst.dst)];
 					dst.type  = EmbeddedFetchValueType::Constant;
 					dst.value = inst.src0.value;
-					dst.prolog_loads.clear();
 				}
 				break;
 			default:
@@ -359,7 +351,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 								auto& dst        = sgprs[register_id + i];
 								dst.type         = EmbeddedFetchValueType::Attrib;
 								dst.attrib_id    = index + static_cast<int>(i);
-								dst.prolog_loads = {inst.pc};
 							}
 						} else {
 							ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
@@ -377,7 +368,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 								dst.type  = EmbeddedFetchValueType::Buffer;
 								dst.attrib_id =
 								    BufferTableAttribFromOffset(raw_offset, static_cast<int>(i));
-								dst.prolog_loads = {inst.pc};
 							}
 						} else if (IsDecodedSgpr(inst.src1) &&
 						           DecodedSgprReg(inst.src1) < sgprs.size() &&
@@ -389,8 +379,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 								auto& dst        = sgprs[register_id + i];
 								dst.type         = EmbeddedFetchValueType::Buffer;
 								dst.attrib_id    = sgprs[DecodedSgprReg(inst.src1)].attrib_id;
-								dst.prolog_loads = sgprs[DecodedSgprReg(inst.src1)].prolog_loads;
-								dst.prolog_loads.push_back(inst.pc);
 							}
 						} else {
 							ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
@@ -403,8 +391,8 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 						ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
 					}
 					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgpr_is_index.size() &&
-					    IsDecodedVgpr(inst.src0) && inst.src0.reg == 8 &&
-					    IsDecodedVgpr(inst.src1) && inst.src1.reg == 5) {
+					    IsDecodedVgpr(inst.src0) && inst.src0.reg == instance_index_reg &&
+					    IsDecodedVgpr(inst.src1) && inst.src1.reg == vertex_index_reg) {
 						vgpr_is_index[inst.dst.reg] = true;
 					}
 				} else if (IsEmbeddedFetchAttribPropagationAlu(inst)) {
@@ -429,7 +417,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 									break;
 								default: dst.value = src0 + src1; break;
 							}
-							dst.prolog_loads.clear();
 						} else {
 							ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1);
 						}
@@ -452,7 +439,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 						load.pc           = inst.pc;
 						load.attrib_id    = buffer.attrib_id;
 						load.components   = DecodedDstSize(inst);
-						load.prolog_loads = buffer.prolog_loads;
 					}
 				}
 				break;
@@ -474,22 +460,9 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<const uint32_t> back,
                                     std::vector<uint32_t>& joined_code) {
 	EXIT_IF(back.empty());
-	Decoder::Program result;
-	uint32_t         front_words = 0;
-	while (front_words < front.size()) {
-		auto& inst = result.instructions.emplace_back();
-		Decoder::DecodeInstruction(front, front_words, inst);
-		front_words += inst.word_count;
-		if (inst.opcode == Decoder::Opcode::S_SETPC_B64) {
-			EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr ||
-			                     inst.src0.reg != 6u);
-			break;
-		}
-		EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_ENDPGM);
-	}
-	EXIT_IF(result.instructions.empty() ||
-	        result.instructions.back().opcode != Decoder::Opcode::S_SETPC_B64);
-	joined_code.assign(front.begin(), front.begin() + front_words);
+	auto       result      = Decoder::DecodeFrontProgram(front);
+	const auto front_words = static_cast<uint32_t>(result.code.size());
+	joined_code.assign(result.code.begin(), result.code.end());
 	joined_code.insert(joined_code.end(), back.begin(), back.end());
 	// The merged-stage ABI passes the back shader in s[6:7]. Give that handoff an
 	// ordinary CFG edge, retaining both bodies in one register and LDS lifetime.
@@ -517,7 +490,9 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		EXIT("shader recompiler input is empty\n");
 	}
 	if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
-	    options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh) {
+	    options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh &&
+	    options.stage != ShaderType::Local && options.stage != ShaderType::TessellationControl &&
+	    options.stage != ShaderType::TessellationEvaluation) {
 		EXIT("shader recompiler received unsupported stage %u\n",
 		     static_cast<unsigned>(options.stage));
 	}
@@ -537,6 +512,12 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	std::vector<uint32_t> joined_code;
 	if (!options.back_code.empty()) {
 		decoded = DecodeFusedProgram(code, options.back_code, joined_code);
+	} else if (options.stage == ShaderType::Local) {
+		decoded = Decoder::DecodeFrontProgram(code);
+		// The separately compiled hull half runs in the next Vulkan stage.
+		auto& handoff     = decoded.instructions.back();
+		handoff.opcode    = Decoder::Opcode::S_ENDPGM;
+		handoff.src_count = 0;
 	} else {
 		Decoder::DecodeProgram(code, decoded);
 	}
@@ -561,27 +542,13 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
 	     static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
-	bool        dispatcher_fallback = false;
-	std::string dispatcher_reason;
 	if (cfg.irreducible) {
-		dispatcher_fallback = true;
-		dispatcher_reason   = cfg.unsupported_reason;
-		LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
+		LogDispatcherFallback(options, cfg, "build");
 	} else {
-		const auto unstructured_cfg = cfg;
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
 		if (!CFG::Structurize(cfg)) {
-			dispatcher_fallback      = true;
-			dispatcher_reason        = cfg.unsupported_reason;
-			const auto failure_kind  = cfg.failure_kind;
-			const auto failure_block = cfg.failure_block;
-			LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
-			cfg                    = unstructured_cfg;
-			cfg.unsupported        = true;
-			cfg.failure_kind       = failure_kind;
-			cfg.failure_block      = failure_block;
-			cfg.unsupported_reason = dispatcher_reason;
+			LogDispatcherFallback(options, cfg, "structurize");
 		} else {
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(cfg.blocks.size()));
@@ -593,45 +560,25 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		     static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
 	}
 
-	const ShaderVertexInputInfo*  vertex  = nullptr;
-	const ShaderPixelInputInfo*   pixel   = nullptr;
-	const ShaderComputeInputInfo* compute = nullptr;
-	switch (options.stage) {
-		case ShaderType::Vertex:
-		case ShaderType::Mesh: vertex = options.input_info.vertex; break;
-		case ShaderType::Pixel:
-			pixel = options.input_info.pixel;
-			break;
-		case ShaderType::Compute:
-			compute = options.input_info.compute;
-			break;
-		default: break;
-	}
 	EmbeddedFetchData embedded_fetch;
-	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
-		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base,
-		                                           static_cast<uint32_t>(options.user_data.size()),
-		                                           options.wave_size);
+	if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
+	    options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
+		embedded_fetch = DetectEmbeddedVertexFetch(
+		    decoded, options.input_info.vertex, options.user_data_base,
+		    static_cast<uint32_t>(options.user_data.size()), options.wave_size);
 		if (!embedded_fetch.loads.empty()) {
 			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(embedded_fetch.loads.size()));
 		}
 	}
 	Frontend::TranslateOptions translate_options {
-	    .stage               = options.stage,
-	    .wave_size           = options.wave_size,
-	    .shader_hash         = options.shader_hash,
-	    .user_data_base      = options.user_data_base,
-	    .user_data_count     = static_cast<uint32_t>(options.user_data.size()),
-	    .scratch_dwords      = options.scratch_dwords,
-	    .dispatcher_fallback = dispatcher_fallback,
-	    .cfg_failure_kind    = cfg.failure_kind,
-	    .fallback_reason     = dispatcher_reason.empty() ? cfg.unsupported_reason
-	                                                    : dispatcher_reason,
-	    .vertex              = vertex,
-	    .pixel               = pixel,
-	    .compute             = compute,
-	    .embedded_fetch      = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
+	    .stage            = options.stage,
+	    .wave_size        = options.wave_size,
+	    .shader_hash      = options.shader_hash,
+	    .user_data_base   = options.user_data_base,
+	    .user_data_count  = static_cast<uint32_t>(options.user_data.size()),
+	    .input_info       = options.input_info,
+	    .embedded_fetch   = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
 	};
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
@@ -654,6 +601,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		IR::RemoveIdentities(ir.blocks);
 		IR::EliminateDeadCode(ir.blocks);
 	}
+	LowerTessellationMemory(ir, options);
 	IR::BuildSrtPlan(ir);
 	IR::EliminateDeadCode(ir.blocks);
 	IR::TrackResources(ir);
@@ -676,24 +624,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
 
-	const ShaderVertexInputInfo*  vertex  = nullptr;
-	const ShaderPixelInputInfo*   pixel   = nullptr;
-	const ShaderComputeInputInfo* compute = nullptr;
-	switch (options.stage) {
-		case ShaderType::Vertex:
-		case ShaderType::Mesh: vertex = options.input_info.vertex; break;
-		case ShaderType::Pixel: pixel = options.input_info.pixel; break;
-		case ShaderType::Compute: compute = options.input_info.compute; break;
-		default: EXIT("invalid shader stage\n");
-	}
-
-	IR::ShaderInfoOptions info_options;
-	info_options.vertex  = vertex;
-	info_options.pixel   = pixel;
-	info_options.compute = compute;
-	IR::CollectShaderInfo(ir, info_options);
+	IR::CollectShaderInfo(ir, options.input_info);
 	IR::AllocateBindings(ir, push_data_start_dword);
-	Spirv::AnalyzeProgramRequirements(ir);
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(translated.cfg_dump, ir);
@@ -703,11 +635,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	}
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
 	auto spirv = Spirv::EmitProgram(ir, options.input_info);
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,
 	     static_cast<uint64_t>(spirv.size()),
 	     static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
 	                               std::chrono::steady_clock::now() - emit_begin)

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -15,7 +15,6 @@ struct CompileOptions {
 	ShaderType                  stage           = ShaderType::Compute;
 	uint32_t                    wave_size       = 64;
 	uint32_t                    user_data_base  = 0;
-	uint32_t                    scratch_dwords  = 0;
 	uint64_t                    shader_hash     = 0;
 	bool                        dump_ir                    = true;
 	bool                        early_dump                 = false;

src/graphics/shader/recompiler/Tessellation.cpp

@@ -0,0 +1,330 @@
+#include "graphics/shader/recompiler/Tessellation.h"
+
+#include "common/assert.h"
+#include "common/logging/log.h"
+#include "graphics/shader/recompiler/ShaderRecompiler.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
+#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
+
+#include <algorithm>
+#include <array>
+#include <unordered_map>
+#include <utility>
+
+namespace Libs::Graphics::ShaderRecompiler {
+namespace {
+
+struct TessellationAddress {
+	enum class Kind { Unknown, Affine, PackedControlPoint };
+	Kind     kind        = Kind::Unknown;
+	uint32_t coefficient = 0;
+	uint32_t constant    = 0;
+};
+
+uint32_t ReflectTessellationStride(const Decoder::Program& program, bool local,
+                                   uint32_t control_points, uint32_t input_stride) {
+	using namespace Decoder;
+	using Address = TessellationAddress;
+	using Kind    = Address::Kind;
+	std::array<Address, IR::NumVectorRegs> registers {};
+	registers[local ? 3u : 1u] =
+	    local ? Address {Kind::Affine, 1u, 0u} : Address {Kind::PackedControlPoint};
+	const auto constant = [](uint32_t value) { return Address {Kind::Affine, 0u, value}; };
+	const auto read     = [&](const Operand& operand) -> Address {
+		if (operand.absolute || operand.negate || operand.sdwa_sext || operand.op_sel ||
+		    operand.op_sel_hi || operand.negate_hi || operand.dpp)
+			return {};
+		Address value;
+		if (operand.kind == OperandKind::IntegerInlineConstant ||
+		    operand.kind == OperandKind::LiteralConstant) {
+			value = constant(operand.value);
+		} else if (operand.kind == OperandKind::Vgpr) {
+			value = registers.at(operand.reg);
+		}
+		// The native packed HS ID's low byte is the patch ordinal. The logical
+		// interface addresses one patch, as in the frontend's v1 initialization.
+		if (value.kind == Kind::PackedControlPoint && operand.sdwa_sel == 0u) return constant(0u);
+		return operand.sdwa_sel == 6u ? value : Address {};
+	};
+	const auto affine = [](uint64_t coefficient, uint64_t offset) {
+		return coefficient <= UINT32_MAX && offset <= UINT32_MAX
+		           ? Address {Kind::Affine, static_cast<uint32_t>(coefficient),
+		                      static_cast<uint32_t>(offset)}
+		           : Address {};
+	};
+	const auto add = [&](Address lhs, Address rhs) {
+		return lhs.kind == Kind::Affine && rhs.kind == Kind::Affine
+		           ? affine(uint64_t {lhs.coefficient} + rhs.coefficient,
+		                    uint64_t {lhs.constant} + rhs.constant)
+		           : Address {};
+	};
+	const auto multiply = [&](Address lhs, Address rhs) {
+		if ((lhs.kind == Kind::Affine && lhs.coefficient == 0u && lhs.constant == 0u) ||
+		    (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant == 0u))
+			return constant(0u);
+		if (lhs.kind != Kind::Affine || rhs.kind != Kind::Affine) return Address {};
+		if (rhs.coefficient != 0u) std::swap(lhs, rhs);
+		return rhs.coefficient == 0u ? affine(uint64_t {lhs.coefficient} * rhs.constant,
+		                                      uint64_t {lhs.constant} * rhs.constant)
+		                             : Address {};
+	};
+	const auto low24 = [&](Address value) {
+		if (value.kind == Kind::Affine && value.coefficient == 0u)
+			return constant(value.constant & 0xffffffu);
+		return value.kind == Kind::Affine &&
+		               uint64_t {value.coefficient} * (control_points - 1u) + value.constant <=
+		                   0xffffffu
+		           ? value
+		           : Address {};
+	};
+	uint32_t stride = 0;
+	for (const auto& inst: program.instructions) {
+		// Stage exits preserve the active path's definitions. An internal join
+		// would require merging register definitions.
+		EXIT_NOT_IMPLEMENTED(IsDirectBranch(inst.opcode) &&
+		                     inst.branch_target != program.instructions.back().pc);
+		const bool local_store =
+		    inst.opcode == Opcode::DS_WRITE_B32 || inst.opcode == Opcode::DS_WRITE2_B32;
+		const bool buffer_store = inst.opcode == Opcode::BUFFER_STORE_DWORD ||
+		                          inst.opcode == Opcode::BUFFER_STORE_DWORDX2 ||
+		                          inst.opcode == Opcode::BUFFER_STORE_DWORDX3 ||
+		                          inst.opcode == Opcode::BUFFER_STORE_DWORDX4;
+		const bool control_store =
+		    !local && buffer_store && inst.src2.kind == OperandKind::Sgpr && inst.src2.reg == 2u;
+		const bool control_read =
+		    !local && (inst.opcode == Opcode::DS_READ_B32 || inst.opcode == Opcode::DS_READ2_B32);
+		if ((local && local_store) || control_store || control_read) {
+			const auto address = read(inst.src0);
+			if (address.kind != Kind::Affine) {
+				EXIT("%s tessellation address is not affine at pc 0x%08x\n", local ? "LS" : "HS",
+				     inst.pc);
+			}
+			if (address.coefficient != 0u) {
+				const auto expected = control_read ? input_stride : stride;
+				EXIT_NOT_IMPLEMENTED((address.coefficient & 3u) != 0u ||
+				                     (expected != 0u && expected != address.coefficient));
+				if (!control_read) stride = address.coefficient;
+			}
+		}
+		// Store vdata is a source, despite occupying the decoder's dst field.
+		if (local_store || buffer_store || inst.dst.kind != OperandKind::Vgpr) continue;
+		const auto lhs   = read(inst.src0);
+		const auto rhs   = read(inst.src1);
+		const auto third = read(inst.src2);
+		Address    value;
+		switch (inst.opcode) {
+			case Opcode::V_BFE_U32:
+				if (lhs.kind == Kind::PackedControlPoint && rhs.kind == Kind::Affine &&
+				    rhs.coefficient == 0u && third.kind == Kind::Affine &&
+				    third.coefficient == 0u) {
+					if (rhs.constant == 8u && third.constant == 5u) value = {Kind::Affine, 1u, 0u};
+					if (rhs.constant == 0u && third.constant == 8u) value = constant(0u);
+				}
+				break;
+			case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break;
+			case Opcode::V_MAD_U32_U24: value = add(multiply(low24(lhs), low24(rhs)), third); break;
+			case Opcode::V_LSHL_ADD_U32:
+				if (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant < 32u)
+					value = add(multiply(lhs, constant(1u << rhs.constant)), third);
+				break;
+			case Opcode::V_LSHLREV_B32:
+				if (lhs.kind == Kind::Affine && lhs.coefficient == 0u && lhs.constant < 32u)
+					value = multiply(rhs, constant(1u << lhs.constant));
+				break;
+			case Opcode::V_SUB_NC_U32:
+				if (lhs.kind == Kind::Affine && rhs.kind == Kind::Affine &&
+				    lhs.coefficient >= rhs.coefficient && lhs.constant >= rhs.constant)
+					value = affine(lhs.coefficient - rhs.coefficient, lhs.constant - rhs.constant);
+				break;
+			default: break;
+		}
+		if (inst.dst.sdwa_sel != 6u || inst.dst.op_sel || inst.dst.omod || inst.dst.clamp ||
+		    inst.dst.dpp)
+			value = {};
+		for (uint32_t index = 0;
+		     index < std::max(inst.data_dwords, 1u) && inst.dst.reg + index < registers.size(); index++) {
+			registers[inst.dst.reg + index] = {};
+		}
+		registers.at(inst.dst.reg) = value;
+	}
+	EXIT_NOT_IMPLEMENTED(stride == 0u);
+	return stride;
+}
+
+const IR::Inst* TessellationBufferBase(const IR::Inst& inst) {
+	if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::None || inst.NumArgs() <= 3u)
+		return nullptr;
+	const auto* base = inst.Arg(3).Resolve().TryInstruction();
+	return base != nullptr && base->GetOpcode() == IR::ValueOpcode::TessellationBase ? base
+	                                                                                 : nullptr;
+}
+
+IR::Value ActiveAddress(IR::Block& block, IR::Block::iterator before, IR::Value value,
+                        IR::Value predicate, std::unordered_map<IR::Inst*, IR::Value>& resolved) {
+	using namespace IR;
+	value        = value.Resolve();
+	auto* source = value.TryInstruction();
+	if (source == nullptr) return value;
+	if (const auto found = resolved.find(source); found != resolved.end()) return found->second;
+	if (source->GetOpcode() == ValueOpcode::SelectU32) {
+		return source->Arg(0).Resolve() == predicate.Resolve()
+		           ? ActiveAddress(block, before, source->Arg(1), predicate, resolved)
+		           : value;
+	}
+	if (source->GetOpcode() != ValueOpcode::IAdd32 && source->GetOpcode() != ValueOpcode::ISub32)
+		return value;
+	const auto lhs = ActiveAddress(block, before, source->Arg(0), predicate, resolved);
+	const auto rhs = ActiveAddress(block, before, source->Arg(1), predicate, resolved);
+	if (lhs != source->Arg(0).Resolve() || rhs != source->Arg(1).Resolve()) {
+		auto copy = block.PrependNewInst(before, source->GetOpcode(), {lhs, rhs},
+		                                 source->Flags<uint64_t>());
+		value     = Value(&*copy);
+	}
+	resolved.emplace(source, value);
+	return value;
+}
+
+} // namespace
+
+void AnalyzeTessellationPrograms(std::span<const uint32_t> local, std::span<const uint32_t> control,
+                                 ShaderTessellationInputInfo& info) {
+	const auto       local_program = Decoder::DecodeFrontProgram(local);
+	Decoder::Program control_program;
+	Decoder::DecodeProgram(control, control_program);
+	info.ls_stride = ReflectTessellationStride(local_program, true, info.input_control_points, 0u);
+	info.hs_stride = ReflectTessellationStride(control_program, false, info.output_control_points,
+	                                           info.ls_stride);
+	LOGF("Tessellation interface: input_cp=%u output_cp=%u ls_stride=%u hs_stride=%u\n",
+	     info.input_control_points, info.output_control_points, info.ls_stride, info.hs_stride);
+}
+
+void LowerTessellationMemory(IR::Program& program, const CompileOptions& options) {
+	using namespace IR;
+	if (options.stage != ShaderType::Local && options.stage != ShaderType::TessellationControl &&
+	    options.stage != ShaderType::TessellationEvaluation) {
+		return;
+	}
+	const auto& tess = options.input_info.vertex->tess;
+	// Ring addresses can reuse data VGPRs. Their inactive values are irrelevant to
+	// a store guarded by the same EXEC predicate, but must remain intact elsewhere.
+	for (auto* block: program.blocks) {
+		for (auto it = block->begin(); it != block->end(); ++it) {
+			if (TessellationBufferBase(*it) == nullptr) continue;
+			std::unordered_map<Inst*, Value> resolved;
+			it->SetArg(
+			    2, ActiveAddress(*block, it, it->Arg(2), it->Arg(it->NumArgs() - 1u), resolved));
+		}
+	}
+	ConstantPropagationPass(program.blocks);
+	uint32_t reads = 0, writes = 0, factors = 0;
+	for (auto* block: program.blocks) {
+		for (auto it = block->begin(); it != block->end(); ++it) {
+			auto&      inst   = *it;
+			const auto shared = SharedAccessOf(inst.GetOpcode());
+			const auto buffer = BufferAccessOf(inst.GetOpcode());
+			if (shared == SharedAccess::None && buffer == BufferAccess::None) {
+				continue;
+			}
+			const auto&           memory = program.memory_info.at(inst.Flags<MemoryFlags>().index);
+			bool                  write  = false;
+			uint32_t              components = 0;
+			TessellationAttribute kind;
+			Value                 address, predicate;
+			if (shared != SharedAccess::None && memory.kind == ResourceKind::Lds) {
+				write = shared == SharedAccess::Write;
+				EXIT_NOT_IMPLEMENTED(memory.data_bits != 32u ||
+				                     (options.stage == ShaderType::Local
+				                          ? !write
+				                          : options.stage != ShaderType::TessellationControl ||
+				                                shared != SharedAccess::Read));
+				kind       = write ? TessellationAttribute::LocalOutput
+				                   : TessellationAttribute::ControlInput;
+				components = SharedComponentCount(inst.GetOpcode());
+				address    = inst.Arg(0);
+				predicate  = inst.Arg(inst.NumArgs() - 1u);
+			} else if (buffer != BufferAccess::None && memory.kind == ResourceKind::Buffer) {
+				const auto* base = TessellationBufferBase(inst);
+				if (base == nullptr) {
+					continue;
+				}
+				EXIT_NOT_IMPLEMENTED(memory.data_bits != 32u || memory.formatted || memory.idxen ||
+				                     !memory.offen || buffer == BufferAccess::Atomic);
+				write      = buffer == BufferAccess::Write;
+				components = BufferComponentCount(inst.GetOpcode());
+				address    = inst.Arg(2).Resolve();
+				predicate  = inst.Arg(inst.NumArgs() - 1u);
+				if (base->Arg(0).U32() == 1u) {
+					EXIT_NOT_IMPLEMENTED(!write ||
+					                     options.stage != ShaderType::TessellationControl);
+					kind = TessellationAttribute::Factor;
+					factors += components;
+				} else {
+					kind = write ? TessellationAttribute::ControlOutput
+					             : TessellationAttribute::EvaluationInput;
+					EXIT_NOT_IMPLEMENTED(write
+					                         ? options.stage != ShaderType::TessellationControl
+					                         : options.stage != ShaderType::TessellationEvaluation);
+					if (address.IsImmediate() &&
+					    address.U32() >= tess.hs_stride * tess.output_control_points) {
+						EXIT_NOT_IMPLEMENTED(!write);
+						kind = TessellationAttribute::PatchOutput;
+					}
+				}
+			} else {
+				continue;
+			}
+			const auto emit = [&](ValueOpcode opcode, std::initializer_list<Value> args) {
+				return Value(&*block->PrependNewInst(it, opcode, args));
+			};
+			std::array<Value, 4> values;
+			for (uint32_t component = 0; component < components; component++) {
+				const auto offset = memory.offset + 4u * component;
+				const auto byte_address =
+				    offset == 0u ? address : emit(ValueOpcode::IAdd32, {address, Value(offset)});
+				if (write) {
+					Value data = shared != SharedAccess::None ? inst.Arg(component + 1u)
+					             : components == 1u
+					                 ? inst.Arg(4)
+					                 : emit(components == 2u   ? ValueOpcode::CompositeExtractU32x2
+					                        : components == 3u ? ValueOpcode::CompositeExtractU32x3
+					                                           : ValueOpcode::CompositeExtractU32x4,
+					                        {inst.Arg(4), Value(component)});
+					emit(ValueOpcode::SetTessellationAttribute,
+					     {Value(static_cast<uint32_t>(kind)), byte_address, data, predicate});
+					writes++;
+				} else {
+					values[component] =
+					    emit(ValueOpcode::GetTessellationAttribute,
+					         {Value(static_cast<uint32_t>(kind)), byte_address, predicate});
+					reads++;
+				}
+			}
+			if (!write) {
+				Value replacement = values[0];
+				if (components == 2u)
+					replacement =
+					    emit(ValueOpcode::CompositeConstructU32x2, {values[0], values[1]});
+				if (components == 3u)
+					replacement = emit(ValueOpcode::CompositeConstructU32x3,
+					                   {values[0], values[1], values[2]});
+				if (components == 4u)
+					replacement = emit(ValueOpcode::CompositeConstructU32x4,
+					                   {values[0], values[1], values[2], values[3]});
+				inst.ReplaceUsesWith(replacement);
+			}
+			inst.Invalidate();
+		}
+	}
+	ConstantPropagationPass(program.blocks);
+	RemoveIdentities(program.blocks);
+	EliminateDeadCode(program.blocks);
+	LOGF("%s tessellation lowering: reads=%u writes=%u factors=%u\n",
+	     options.stage == ShaderType::Local                 ? "LS"
+	     : options.stage == ShaderType::TessellationControl ? "HS"
+	                                                       : "TES",
+	     reads, writes, factors);
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler

src/graphics/shader/recompiler/Tessellation.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+#include <span>
+
+namespace Libs::Graphics {
+struct ShaderTessellationInputInfo;
+}
+
+namespace Libs::Graphics::ShaderRecompiler {
+
+struct CompileOptions;
+namespace IR {
+struct Program;
+}
+
+void AnalyzeTessellationPrograms(std::span<const uint32_t> local, std::span<const uint32_t> control,
+                                 ShaderTessellationInputInfo& info);
+void LowerTessellationMemory(IR::Program& program, const CompileOptions& options);
+
+} // namespace Libs::Graphics::ShaderRecompiler

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -15,9 +15,9 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
 	if (words_num == 0) {
 		return;
 	}
-	const auto opcode     = words[0];
-	const auto word_count = static_cast<uint32_t>(words_num);
-	section.push_back((word_count << 16u) | opcode);
+	const uint32_t opcode     = words[0];
+	const auto     word_count = static_cast<uint32_t>(words_num);
+	section.push_back((word_count << spv::WordCountShift) | opcode);
 	section.insert(section.end(), words + 1, words + words_num);
 }
 
@@ -36,15 +36,17 @@ void Builder::RequireVersion(uint32_t version) {
 	m_version = std::max(m_version, version);
 }
 
-void Builder::RequireCapability(uint32_t capability) {
+void Builder::RequireCapability(spv::Capability capability) {
 	if (m_required_capabilities.insert(capability).second) {
-		AddCapability({capability});
+		AppendInstruction(m_capabilities, spv::OpCapability, capability);
 	}
 }
 
 void Builder::RequireExtension(const char* name) {
 	if (m_required_extensions.emplace(name).second) {
-		AddExtension(name);
+		std::vector<uint32_t> operands;
+		AppendString(operands, name);
+		AppendInstruction(m_extensions, spv::OpExtension, operands);
 	}
 }
 
@@ -54,41 +56,31 @@ uint32_t Builder::Import(const char* name) {
 	}
 	const auto id = AllocateId();
 	m_import_ids.emplace(name, id);
-	AddExtInstImport(id, name);
+	std::vector<uint32_t> operands = {id};
+	AppendString(operands, name);
+	AppendInstruction(m_ext_inst_imports, spv::OpExtInstImport, operands);
 	return id;
 }
 
-uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) {
-	return Type(opcode, std::vector<uint32_t>(operands));
-}
-
-uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
-	std::vector<uint32_t> key;
-	key.reserve(operands.size() + 2u);
-	key.push_back(opcode);
-	key.push_back(static_cast<uint32_t>(operands.size()));
-	key.insert(key.end(), operands.begin(), operands.end());
+uint32_t Builder::DeclareType(spv::Op opcode, std::vector<uint32_t> key) {
 	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
 		return it->second;
 	}
 	const auto id = AllocateId();
+	AppendInstruction(m_declarations, opcode, id, std::span<const uint32_t>(key).subspan(2));
 	m_declaration_ids.emplace(std::move(key), id);
-	std::vector<uint32_t> words {opcode, id};
-	words.insert(words.end(), operands.begin(), operands.end());
-	AppendInstructionWords(m_declarations, words.data(), words.size());
 	return id;
 }
 
-uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
-                                std::initializer_list<TypeAnnotation> annotations) {
+uint32_t Builder::DeclareDecoratedType(spv::Op opcode, std::vector<uint32_t> key,
+                                       std::initializer_list<TypeAnnotation> annotations) {
 	if (annotations.size() == 0) {
-		return Type(opcode, operands);
+		return DeclareType(opcode, std::move(key));
 	}
-	std::vector<uint32_t> key {opcode, static_cast<uint32_t>(operands.size())};
-	key.insert(key.end(), operands.begin(), operands.end());
+	const auto operand_count = key[1];
 	key.push_back(static_cast<uint32_t>(annotations.size()));
 	for (const auto& annotation: annotations) {
-		key.push_back(annotation.opcode);
+		AppendOperand(key, annotation.opcode);
 		key.push_back(static_cast<uint32_t>(annotation.operands.size()));
 		key.insert(key.end(), annotation.operands.begin(), annotation.operands.end());
 	}
@@ -96,48 +88,35 @@ uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t>
 		return it->second;
 	}
 	const auto id = AllocateId();
+	AppendInstruction(m_declarations, opcode, id,
+	                  std::span<const uint32_t>(key).subspan(2, operand_count));
 	m_declaration_ids.emplace(std::move(key), id);
-	std::vector<uint32_t> words {opcode, id};
-	words.insert(words.end(), operands.begin(), operands.end());
-	AppendInstructionWords(m_declarations, words.data(), words.size());
 	for (const auto& annotation: annotations) {
-		words = {annotation.opcode, id};
-		words.insert(words.end(), annotation.operands.begin(), annotation.operands.end());
-		AppendInstructionWords(m_annotations, words.data(), words.size());
+		AppendInstruction(m_annotations, annotation.opcode, id, annotation.operands);
 	}
 	return id;
 }
 
-uint32_t Builder::Constant(uint32_t opcode, uint32_t type,
-                           std::initializer_list<uint32_t> operands) {
-	return Constant(opcode, type, std::vector<uint32_t>(operands));
-}
-
-uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) {
-	std::vector<uint32_t> key;
-	key.reserve(operands.size() + 2u);
-	key.push_back(opcode);
-	key.push_back(type);
-	key.insert(key.end(), operands.begin(), operands.end());
+uint32_t Builder::DeclareConstant(spv::Op opcode, std::vector<uint32_t> key) {
 	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
 		return it->second;
 	}
 	const auto id = AllocateId();
+	AppendInstruction(m_declarations, opcode, key[1], id,
+	                  std::span<const uint32_t>(key).subspan(2));
 	m_declaration_ids.emplace(std::move(key), id);
-	std::vector<uint32_t> words {opcode, type, id};
-	words.insert(words.end(), operands.begin(), operands.end());
-	AppendInstructionWords(m_declarations, words.data(), words.size());
 	return id;
 }
 
-uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) {
+uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class) {
 	const auto id = AllocateId();
 	DefineGlobalVariable(id, pointer_type, storage_class);
 	return id;
 }
 
-void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) {
-	AddType({59u, pointer_type, id, storage_class});
+void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type,
+                                   spv::StorageClass storage_class) {
+	AppendInstruction(m_declarations, spv::OpVariable, pointer_type, id, storage_class);
 }
 
 void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
@@ -156,49 +135,20 @@ void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
 	}
 }
 
-void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-                                const std::vector<uint32_t>& operands) {
-	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
-	section.push_back((word_count << 16u) | opcode);
-	section.insert(section.end(), operands.begin(), operands.end());
+void Builder::AddMemoryModel(spv::AddressingModel addressing_model, spv::MemoryModel memory_model) {
+	AppendInstruction(m_memory_model, spv::OpMemoryModel, addressing_model, memory_model);
 }
 
-void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-                                std::initializer_list<uint32_t> operands) {
-	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
-	section.push_back((word_count << 16u) | opcode);
-	section.insert(section.end(), operands.begin(), operands.end());
-}
-
-void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_capabilities, 17u, operands);
-}
-
-void Builder::AddExtension(const char* name) {
+void Builder::AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point,
+                            const char* name, const std::vector<uint32_t>& interfaces) {
 	std::vector<uint32_t> operands;
-	AppendString(operands, name);
-	AppendInstruction(m_extensions, 10u, operands);
-}
-
-void Builder::AddExtInstImport(uint32_t id, const char* name) {
-	std::vector<uint32_t> operands = {id};
-	AppendString(operands, name);
-	AppendInstruction(m_ext_inst_imports, 11u, operands);
-}
-
-void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_memory_model, 14u, operands);
-}
-
-void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-                            const std::vector<uint32_t>& interfaces) {
-	std::vector<uint32_t> operands = {execution_model, entry_point};
+	AppendOperands(operands, execution_model, entry_point);
 	AppendString(operands, name);
 	operands.insert(operands.end(), interfaces.begin(), interfaces.end());
 	if (m_version >= 0x00010400u) {
 		for (size_t offset = 0; offset < m_declarations.size();) {
-			const auto count = m_declarations[offset] >> 16u;
-			if ((m_declarations[offset] & 0xffffu) == 59u) {
+			const auto count = m_declarations[offset] >> spv::WordCountShift;
+			if ((m_declarations[offset] & spv::OpCodeMask) == spv::OpVariable) {
 				const auto id = m_declarations[offset + 2u];
 				if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) {
 					operands.push_back(id);
@@ -207,46 +157,32 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons
 			offset += count;
 		}
 	}
-	AppendInstruction(m_entry_points, 15u, operands);
-}
-
-void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
-	AppendInstruction(m_execution_modes, 16u, operands);
+	AppendInstruction(m_entry_points, spv::OpEntryPoint, operands);
 }
 
 void Builder::AddName(uint32_t target, const char* name) {
 	std::vector<uint32_t> operands = {target};
 	AppendString(operands, name);
-	AppendInstruction(m_debug, 5u, operands);
+	AppendInstruction(m_debug, spv::OpName, operands);
 }
 
-void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_annotations, words.begin(), words.size());
-}
-
-void Builder::AddType(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_declarations, words.begin(), words.size());
-}
-
-void Builder::AddFunction(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_functions, words.begin(), words.size());
-}
-
-void Builder::AddFunction(const std::vector<uint32_t>& words) {
+void Builder::AddFunction(std::span<const uint32_t> words) {
 	AppendInstructionWords(m_functions, words.data(), words.size());
 }
 
 DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
-	std::vector<uint32_t> words {245u, type, result};
+	std::vector<uint32_t> words {spv::OpPhi, type, result};
 	words.resize(words.size() + incoming_count * 2u);
-	const DeferredPhi phi {m_functions.size(), incoming_count};
+	const DeferredPhi phi {m_functions.size()};
 	AddFunction(words);
 	m_unpatched_phi_incomings += incoming_count;
 	return phi;
 }
 
 void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
-	EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0);
+	const auto incoming_count =
+	    ((m_functions.at(phi.word_offset) >> spv::WordCountShift) - 3u) / 2u;
+	EXIT_IF(incoming >= incoming_count || value == 0 || parent == 0);
 	const auto value_word  = phi.word_offset + 3u + incoming * 2u;
 	const auto parent_word = value_word + 1u;
 	EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
@@ -264,7 +200,7 @@ std::vector<uint32_t> Builder::Build() const {
 	               m_debug.size() + m_annotations.size() + m_declarations.size() +
 	               m_functions.size());
 
-	module.push_back(0x07230203u);
+	module.push_back(spv::MagicNumber);
 	module.push_back(m_version);
 	module.push_back(0u);
 	module.push_back(m_next_id);

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -4,21 +4,25 @@
 #include "common/common.h"
 
 #include <initializer_list>
+#include <iterator>
 #include <map>
 #include <set>
+#include <span>
+#include <spirv/unified1/spirv.hpp>
 #include <string>
+#include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
 struct TypeAnnotation {
-	uint32_t              opcode = 0;
-	std::vector<uint32_t> operands;
+	spv::Op               opcode = spv::OpNop;
+	std::vector<uint32_t> operands; // Already encoded operand words.
 };
 
 struct DeferredPhi {
 	size_t word_offset    = 0;
-	size_t incoming_count = 0;
 };
 
 class Builder {
@@ -29,42 +33,113 @@ public:
 
 	uint32_t AllocateId();
 	void     RequireVersion(uint32_t version);
-	void     RequireCapability(uint32_t capability);
+	void     RequireCapability(spv::Capability capability);
 	void     RequireExtension(const char* name);
 	uint32_t Import(const char* name);
-	uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {});
-	uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands);
-	uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
-	                       std::initializer_list<TypeAnnotation> annotations);
-	uint32_t Constant(uint32_t opcode, uint32_t type,
-	                  std::initializer_list<uint32_t> operands = {});
-	uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands);
-	uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class);
-	void     DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
+	template <typename... Args>
+	uint32_t Type(spv::Op opcode, const Args&... operands) {
+		return DeclareType(opcode, MakeTypeKey(opcode, operands...));
+	}
 
-	void        AddMemoryModel(std::initializer_list<uint32_t> operands);
-	void        AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-	                          const std::vector<uint32_t>& interfaces);
-	void        AddExecutionMode(std::initializer_list<uint32_t> operands);
-	void        AddName(uint32_t target, const char* name);
-	void        AddAnnotation(std::initializer_list<uint32_t> words);
-	void        AddFunction(std::initializer_list<uint32_t> words);
-	void        AddFunction(const std::vector<uint32_t>& words);
+	template <typename... Args>
+	uint32_t DecoratedType(spv::Op opcode, std::initializer_list<TypeAnnotation> annotations,
+	                       const Args&... operands) {
+		return DeclareDecoratedType(opcode, MakeTypeKey(opcode, operands...), annotations);
+	}
+
+	template <typename... Args>
+	uint32_t Constant(spv::Op opcode, uint32_t type, const Args&... operands) {
+		std::vector<uint32_t> key;
+		key.reserve(2u + (0u + ... + OperandWordCount(operands)));
+		AppendOperands(key, opcode, type, operands...);
+		return DeclareConstant(opcode, std::move(key));
+	}
+
+	uint32_t DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class);
+	void DefineGlobalVariable(uint32_t id, uint32_t pointer_type, spv::StorageClass storage_class);
+
+	void AddMemoryModel(spv::AddressingModel addressing_model, spv::MemoryModel memory_model);
+	void AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point, const char* name,
+	                   const std::vector<uint32_t>& interfaces);
+	template <typename... Args>
+	void AddExecutionMode(uint32_t entry_point, spv::ExecutionMode mode, const Args&... operands) {
+		AppendInstruction(m_execution_modes, spv::OpExecutionMode, entry_point, mode, operands...);
+	}
+
+	void AddName(uint32_t target, const char* name);
+	template <typename... Args>
+	void AddAnnotation(spv::Op opcode, const Args&... operands) {
+		AppendInstruction(m_annotations, opcode, operands...);
+	}
+
+	template <typename... Args>
+	void AddFunction(spv::Op opcode, const Args&... operands) {
+		AppendInstruction(m_functions, opcode, operands...);
+	}
+
+	void        AddFunction(std::span<const uint32_t> words);
 	DeferredPhi AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count);
 	void        PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent);
 
 	[[nodiscard]] std::vector<uint32_t> Build() const;
 
 private:
-	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-	                              const std::vector<uint32_t>& operands);
-	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
-	                              std::initializer_list<uint32_t> operands);
+	static void AppendOperand(std::vector<uint32_t>& words, uint32_t value) {
+		words.push_back(value);
+	}
+
+	static void AppendOperand(std::vector<uint32_t>& words, int32_t value) {
+		words.push_back(static_cast<uint32_t>(value));
+	}
+
+	template <typename T>
+	requires std::is_enum_v<T>
+	static void AppendOperand(std::vector<uint32_t>& words, T value) {
+		static_assert(sizeof(T) == sizeof(uint32_t));
+		words.push_back(static_cast<uint32_t>(value));
+	}
+
+	static void AppendOperand(std::vector<uint32_t>& words, std::span<const uint32_t> values) {
+		words.insert(words.end(), values.begin(), values.end());
+	}
+
+	template <typename... Args>
+	static void AppendOperands(std::vector<uint32_t>& words, const Args&... operands) {
+		(AppendOperand(words, operands), ...);
+	}
+
+	template <typename T>
+	static size_t OperandWordCount(const T& operand) {
+		if constexpr (std::is_integral_v<T> || std::is_enum_v<T>) {
+			return 1;
+		} else {
+			return std::size(operand);
+		}
+	}
+
+	template <typename... Args>
+	static std::vector<uint32_t> MakeTypeKey(spv::Op opcode, const Args&... operands) {
+		const auto operand_count = static_cast<uint32_t>((0u + ... + OperandWordCount(operands)));
+		std::vector<uint32_t> key;
+		key.reserve(2u + operand_count);
+		AppendOperands(key, opcode, operand_count, operands...);
+		return key;
+	}
+
+	template <typename... Args>
+	static void AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
+	                              const Args&... operands) {
+		const auto offset = section.size();
+		AppendOperands(section, opcode, operands...);
+		const auto word_count = static_cast<uint32_t>(section.size() - offset);
+		section[offset] |= word_count << spv::WordCountShift;
+	}
+
+	uint32_t    DeclareType(spv::Op opcode, std::vector<uint32_t> key);
+	uint32_t    DeclareDecoratedType(spv::Op opcode, std::vector<uint32_t> key,
+	                                 std::initializer_list<TypeAnnotation> annotations);
+	uint32_t    DeclareConstant(spv::Op opcode, std::vector<uint32_t> key);
 	static void AppendString(std::vector<uint32_t>& words, const char* text);
-	void        AddCapability(std::initializer_list<uint32_t> operands);
-	void        AddExtension(const char* name);
-	void        AddExtInstImport(uint32_t id, const char* name);
-	void        AddType(std::initializer_list<uint32_t> words);
 
 	uint32_t                                  m_next_id = 1;
 	uint32_t                                  m_version = 0;
@@ -78,7 +153,7 @@ private:
 	std::vector<uint32_t>                     m_annotations;
 	std::vector<uint32_t>                     m_declarations;
 	std::vector<uint32_t>                     m_functions;
-	std::set<uint32_t>                        m_required_capabilities;
+	std::set<spv::Capability>                 m_required_capabilities;
 	std::set<std::string>                     m_required_extensions;
 	std::map<std::string, uint32_t>           m_import_ids;
 	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -183,10 +183,8 @@ void ValidateNativeProgram(const IR::Program& program) {
 
 } // namespace
 
-void AnalyzeProgramRequirements(IR::Program& program) {
-	program.spirv_requirements.reset();
-	IR::SpirvRequirements requirements {};
-	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
+Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) {
+	SpirvRequirements requirements {};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
@@ -242,17 +240,17 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 				}
 				if (shared_access == IR::SharedAccess::Append ||
 				    shared_access == IR::SharedAccess::Consume) {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 				}
 			}
 			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::Ballot: MarkBallot(); break;
+				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
 				case IR::ValueOpcode::ReadFirstLane:
 				case IR::ValueOpcode::ReadLane: {
-					MarkBallot();
+					requirements.subgroup_ballot  = true;
 					requirements.subgroup_shuffle = true;
 					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
 						requirements.subgroup_local_invocation_id = true;
@@ -261,25 +259,26 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 				}
 				case IR::ValueOpcode::DppUpdateU32:
 				case IR::ValueOpcode::WriteLane: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::Permlane16U32: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::SwizzleU32:
 				case IR::ValueOpcode::BpermuteU32: {
-					MarkBallot();
+					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
 					requirements.subgroup_local_invocation_id = true;
 					break;
 				}
 				case IR::ValueOpcode::LaneId:
-					requirements.subgroup_local_invocation_id = true;
+					requirements.subgroup_local_invocation_id |=
+					    program.stage != ShaderType::TessellationControl;
 					break;
 				case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
 				case IR::ValueOpcode::ImageGatherRaw:
@@ -300,7 +299,7 @@ void AnalyzeProgramRequirements(IR::Program& program) {
 			}
 		}
 	}
-	program.spirv_requirements.emplace(requirements);
+	return requirements;
 }
 
 std::vector<uint32_t> EmitProgram(const IR::Program& program,
@@ -308,39 +307,29 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
 	using namespace Emitter;
 
 	if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
-	    program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh) {
-		Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
+	    program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh &&
+	    program.stage != ShaderType::Local && program.stage != ShaderType::TessellationControl &&
+	    program.stage != ShaderType::TessellationEvaluation) {
+		Fail(program, "binary SPIR-V emitter received an unsupported shader stage");
 	}
 	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete ||
-	    !program.spirv_requirements.has_value()) {
+	    !program.shader_info_complete || !program.binding_layout_complete) {
 		Fail(program, "SPIR-V emitter requires a fully planned native shader program");
 	}
 	ValidateNativeProgram(program);
 	IR::ValidateProgram(program, true);
 	EmitterState state(program, input_info);
-	state.stage = program.stage;
 	const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
 	state.lane_count =
 	    workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u
 	        ? 2u
 	        : 1u;
-	state.inputs.reserve(program.info.inputs.size());
-	state.outputs.reserve(program.info.outputs.size());
-	state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
-	CopyProgramInputsAndOutputs(state, program);
-	AllocateInputVariables(state);
-	AllocateOutputVariables(state);
 	DefineModule(state);
 	EmitProgram(state);
-	state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
-	                            state.interface_variables);
+	state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func,
+	                            "main", state.interface_variables);
 
-	auto binary = state.builder.Build();
-	if (binary.empty()) {
-		Fail(program, "SPIR-V builder returned an empty module");
-	}
-	return binary;
+	return state.builder.Build();
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,8 +9,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-void AnalyzeProgramRequirements(IR::Program& program);
-
 std::vector<uint32_t> EmitProgram(const IR::Program& program,
                                   ShaderStageInputInfo input_info);
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -2,116 +2,37 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
-                            uint32_t false_value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value});
-	return ret;
-}
-
-uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
-
-void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
-                                   uint32_t& out_low, uint32_t& out_high) {
-	const auto amount           = EmitAndConstant(state, shift, 63u);
-	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
-	const auto less32           = state.builder.AllocateId();
-	const auto non_zero         = state.builder.AllocateId();
-	const auto carry_count_base = state.builder.AllocateId();
-	const auto carry_count      = state.builder.AllocateId();
-	const auto low_part         = state.builder.AllocateId();
-	const auto high_part        = state.builder.AllocateId();
-	const auto carry_part       = state.builder.AllocateId();
-	const auto carry            = state.builder.AllocateId();
-	const auto high_less        = state.builder.AllocateId();
-	const auto high_ge          = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
-	state.builder.AddFunction(
-	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
-	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
-	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
-}
-
-void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
-                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
-	const auto amount           = EmitAndConstant(state, shift, 63u);
-	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
-	const auto less32           = state.builder.AllocateId();
-	const auto non_zero         = state.builder.AllocateId();
-	const auto carry_count_base = state.builder.AllocateId();
-	const auto carry_count      = state.builder.AllocateId();
-	const auto low_part         = state.builder.AllocateId();
-	const auto high_part        = state.builder.AllocateId();
-	const auto carry_part       = state.builder.AllocateId();
-	const auto carry            = state.builder.AllocateId();
-	const auto low_less         = state.builder.AllocateId();
-	const auto low_ge           = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
-	state.builder.AddFunction(
-	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
-	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
-	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
-}
-
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), ret, value,
+	                          ConstantU32(state, mask));
 	return ret;
 }
 
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), ret, value,
+	                          ConstantU32(state, shift));
 	return ret;
 }
 
-uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+uint32_t EmitCompareU32Constant(EmitterState& state, spv::Op opcode, uint32_t value,
                                 uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction(opcode, TypeBool(state), ret, value, ConstantU32(state, constant));
 	return ret;
 }
 
 uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), ret, ConstantU32(state, constant),
+	                          value);
 	return ret;
 }
 
 uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, f32);
 
 	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
 	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
@@ -122,26 +43,26 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
 	const auto normal_payload = state.builder.AllocateId();
 	const auto normal         = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
-	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp,
+	                          ConstantU32(state, 10));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), normal_payload, normal_exp,
+	                          normal_mant);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), normal, sign, normal_payload);
 
 	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
 	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
-	const auto exp_lt_103       = EmitCompareU32Constant(state, OpULessThan, exp, 103);
-	const auto exp_gt_112       = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112);
+	const auto exp_lt_103       = EmitCompareU32Constant(state, spv::OpULessThan, exp, 103);
+	const auto exp_gt_112       = EmitCompareU32Constant(state, spv::OpUGreaterThan, exp, 112);
 	const auto sub_shift_low =
 	    EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
 	const auto sub_shift =
 	    EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low);
 	const auto sub_mant  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden,
+	                          sub_shift);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant);
 
 	const auto nan_payload =
 	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
@@ -149,12 +70,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	    EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
 	const auto inf        = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
 	const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
-	const auto mant_zero  = EmitCompareU32Constant(state, OpIEqual, mant, 0);
+	const auto mant_zero  = EmitCompareU32Constant(state, spv::OpIEqual, mant, 0);
 	const auto special    = EmitSelectValueU32(state, mant_zero, inf, nan);
 
-	const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112);
-	const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143);
-	const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255);
+	const auto exp_le_112 = EmitCompareU32Constant(state, spv::OpULessThanEqual, exp, 112);
+	const auto exp_ge_143 = EmitCompareU32Constant(state, spv::OpUGreaterThanEqual, exp, 143);
+	const auto exp_eq_255 = EmitCompareU32Constant(state, spv::OpIEqual, exp, 255);
 	const auto finite0    = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
 	const auto finite1    = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
 	const auto finite2    = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
@@ -164,54 +85,18 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
+	state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state),
+	                          cond, lhs, rhs);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs);
 	return ret;
 }
 
 uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value});
+	state.builder.AddFunction(max_value ? spv::OpSGreaterThan : spv::OpSLessThan, TypeBool(state),
+	                          cond, lhs, rhs);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs);
 	return ret;
 }
 
@@ -221,10 +106,11 @@ F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
 	const auto abs_bits      = EmitAndConstant(state, cls.bits, 0x7fffffffu);
 	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
-	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
-	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
+	const auto exponent_max =
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
+	const auto mantissa_nonzero = EmitCompareU32Constant(state, spv::OpINotEqual, mantissa_bits, 0);
 	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
-	cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	cls.zero                    = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 	return cls;
 }
 
@@ -235,7 +121,7 @@ F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
 uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
                                uint32_t class_match) {
 	const auto selected =
-	    EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
+	    EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
 	return EmitLogicalAndBool(state, selected, class_match);
 }
 
@@ -247,15 +133,16 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
 	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
 
-	const auto sign             = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0);
+	const auto sign             = EmitCompareU32Constant(state, spv::OpINotEqual, sign_bits, 0);
 	const auto positive         = EmitLogicalNotBool(state, sign);
-	const auto exponent_zero    = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0);
+	const auto exponent_zero    = EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0);
 	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
-	const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto exponent_inf =
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
 	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
-	const auto mantissa_zero    = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0);
+	const auto mantissa_zero    = EmitCompareU32Constant(state, spv::OpIEqual, mantissa_bits, 0);
 	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
-	const auto quiet            = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0);
+	const auto quiet            = EmitCompareU32Constant(state, spv::OpINotEqual, quiet_bits, 0);
 
 	const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
 	const auto snan       = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
@@ -263,7 +150,7 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto inf        = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
 	const auto normal     = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
 	const auto denorm     = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
-	const auto zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	const auto zero       = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 
 	uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
 	match          = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
@@ -296,8 +183,8 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto rhs_class = EmitClassifyF32(state, rhs);
 
 	const auto numeric_cond = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state),
-	                           numeric_cond, lhs, rhs});
+	state.builder.AddFunction(max_value ? spv::OpFOrdGreaterThanEqual : spv::OpFOrdLessThan,
+	                          TypeBool(state), numeric_cond, lhs, rhs);
 	const auto ordered_bits =
 	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
 
@@ -312,13 +199,6 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	return EmitBitcastU32ToF32(state, result_bits);
 }
 
-uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value});
-	return ret;
-}
-
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto bits      = state.builder.AllocateId();
 	const auto abs_bits  = state.builder.AllocateId();
@@ -328,18 +208,18 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
-	state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
-	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits,
+	                          ConstantU32(state, 0x7fffffffu));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
+	                          ConstantU32(state, 0x80000000u));
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
+	                          ConstantU32(state, 0x00800000u));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal);
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits);
+	state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
 	return ret;
 }
 
@@ -351,46 +231,35 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
 	const auto finite       = state.builder.AllocateId();
 	const auto large_finite = state.builder.AllocateId();
 	const auto reduced      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src});
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)});
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
-	state.builder.AddFunction(
-	    {OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
+	state.builder.AddFunction(spv::OpExtInst, TypeF32(state), fract, GlslStd450(state),
+	                          GLSLstd450Fract, src);
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, src);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits,
+	                          ConstantU32(state, 0x7fffffffu));
+	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), large, abs_bits,
+	                          ConstantU32(state, 0x4b000000u));
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), finite, abs_bits,
+	                          ConstantU32(state, 0x7f800000u));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), large_finite, large, finite);
+	state.builder.AddFunction(spv::OpSelect, TypeF32(state), reduced, large_finite,
+	                          ConstantF32(state, 0), fract);
 	if (!preserve_signed_zero) {
 		return reduced;
 	}
 	const auto zero = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced});
-	return ret;
-}
-
-uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value});
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), zero, abs_bits,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpSelect, TypeF32(state), ret, zero, src, reduced);
 	return ret;
 }
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
-	                           GlslUnpackHalf2x16, bits});
-	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
+	state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
+	                          GLSLstd450UnpackHalf2x16, bits);
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), ret, unpacked, 0);
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -14,42 +14,22 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
 			active_inputs[active_count++] = input.location;
 		}
 	}
-	return state.stage == ShaderType::Pixel
+	return state.program.stage == ShaderType::Pixel
 	           ? ShaderPixelParameterLocation(*state.input_info.pixel,
 	                                          {active_inputs.data(), active_count}, attr)
 	           : attr;
 }
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
-	return state.stage == ShaderType::Pixel &&
+	return state.program.stage == ShaderType::Pixel &&
 	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
 }
 
 bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
-	return state.stage == ShaderType::Pixel &&
+	return state.program.stage == ShaderType::Pixel &&
 	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
 }
 
-bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
-               uint32_t index) {
-	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
-		return binding.kind == kind && binding.index == index;
-	});
-}
-
-void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
-	for (const auto& input: program.info.inputs) {
-		state.inputs.push_back({input.kind, input.location, input.component_count, 0,
-		                        input.debug_name, input.per_vertex});
-	}
-	for (const auto& output: program.info.outputs) {
-		if (HasOutput(state.outputs, output.kind, output.index)) {
-			continue;
-		}
-		state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name});
-	}
-}
-
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
 	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
 		return state.per_vertex_variable;
@@ -74,7 +54,7 @@ uint32_t          ConstantU32(EmitterState& state, uint32_t value);
                                                const char* reason) {
 	EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
 	     " stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
-	     state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
+	     state.program.shader_hash, static_cast<unsigned>(state.program.stage), resource,
 	     static_cast<unsigned>(kind), reason);
 	std::abort();
 }
@@ -102,8 +82,8 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
 		ExitDescriptorBindingFailure(state, kind, resource, variable_name);
 	}
 	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
+	state.builder.AddFunction(spv::OpAccessChain, result_ptr_type, pointer, variable_id,
+	                          ConstantU32(state, array_index));
 	return pointer;
 }
 
@@ -129,12 +109,12 @@ uint32_t ImageScalarType(EmitterState& state, Prospero::TextureNumericClass nume
 
 uint32_t ImageVectorType(EmitterState& state, Prospero::TextureNumericClass numeric_class,
                          uint32_t components) {
-	return state.builder.Type(OpTypeVector, {ImageScalarType(state, numeric_class), components});
+	return state.builder.Type(spv::OpTypeVector, ImageScalarType(state, numeric_class), components);
 }
 
 uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 	uint32_t sampled = 0;
-	uint32_t format  = ImageFormatUnknown;
+	uint32_t format  = spv::ImageFormatUnknown;
 	if (image.resource_class == IR::ImageResourceClass::Sampled) {
 		EXIT_IF(image.atomic);
 		sampled = 1;
@@ -144,16 +124,15 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 		sampled = 2;
 		if (image.atomic) {
 			EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint);
-			format = ImageFormatR32ui;
+			format = spv::ImageFormatR32ui;
 		}
 	} else {
 		EXIT("invalid image resource class");
 	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
-	return state.builder.Type(OpTypeImage,
-	                          {ImageScalarType(state, image.numeric_class), info.spirv_dimension,
-	                           image.depth_compare ? 1u : 0u,
-	                           info.arrayed, info.multisampled, sampled, format});
+	return state.builder.Type(spv::OpTypeImage, ImageScalarType(state, image.numeric_class),
+	                          info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
+	                          info.multisampled, sampled, format);
 }
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
@@ -173,26 +152,26 @@ uint32_t LoadSampledImageDescriptor(EmitterState& state, uint32_t resource) {
 	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
 	const auto variable     = state.image_variables[IR::ImageBindingIndex(*kind)];
 	const auto pointer_type = state.builder.Type(
-	    OpTypePointer, {StorageClassUniformConstant, ImageType(state, image_resource)});
+	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image_resource));
 	const auto pointer =
 	    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 	                             "sampled image descriptor array was not emitted");
 	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, ImageType(state, image_resource), image, pointer});
+	state.builder.AddFunction(spv::OpLoad, ImageType(state, image_resource), image, pointer);
 	return image;
 }
 
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
 	const auto array_index =
 	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
-	const auto sampler_type = state.builder.Type(OpTypeSampler);
+	const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
 	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type});
+	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type);
 	const auto pointer = DescriptorElementPointer(
 	    state, pointer_type, state.sampler_variable, array_index,
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer});
+	state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer);
 	return sampler_id;
 }
 
@@ -201,9 +180,9 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
 	const auto  image          = LoadSampledImageDescriptor(state, resource);
 	const auto  sampler_id     = LoadSamplerDescriptor(state, sampler);
 	const auto  sampled_image = state.builder.AllocateId();
-	const auto  sampled_type  =
-	    state.builder.Type(OpTypeSampledImage, {ImageType(state, image_resource)});
-	state.builder.AddFunction({OpSampledImage, sampled_type, sampled_image, image, sampler_id});
+	const auto  sampled_type =
+	    state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource));
+	state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
 	return sampled_image;
 }
 
@@ -212,9 +191,9 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource) {
 	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
-	const auto array_index = ResourceForDescriptor(state, *kind, resource);
-	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, ImageType(state, image)});
+	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
+	const auto pointer_type = state.builder.Type(
+	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 	                                "storage image descriptor array was not emitted");
@@ -225,25 +204,25 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
 	if (!image.atomic) {
-		state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat);
+		state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
 	}
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
 	const auto array_index = ResourceForDescriptor(state, *kind, resource);
 	const auto image_type  = ImageType(state, image);
 	const auto pointer_type =
-	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, image_type});
+	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, image_type);
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	const auto LoadAt   = [&](uint32_t array_index) {
 		const auto pointer =
 		    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
 		                             "storage image descriptor array was not emitted");
 		const auto descriptor = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, image_type, descriptor, pointer});
+		state.builder.AddFunction(spv::OpLoad, image_type, descriptor, pointer);
 		return descriptor;
 	};
 	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
-		state.builder.AddFunction({OpImageWrite, LoadAt(array_index), coord, texel});
+		state.builder.AddFunction(spv::OpImageWrite, LoadAt(array_index), coord, texel);
 		return;
 	}
 	if (image.mip_count == 0u) {
@@ -253,28 +232,31 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 
 	const auto            merge_label = state.builder.AllocateId();
 	std::vector<uint32_t> labels(image.mip_count);
-	std::vector<uint32_t> words {OpSwitch, mip_lod, merge_label};
+	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
 		labels[mip] = state.builder.AllocateId();
 		words.push_back(mip);
 		words.push_back(labels[mip]);
 	}
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
 	state.builder.AddFunction(words);
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
 		EmitLabel(state, labels[mip]);
-		state.builder.AddFunction({OpImageWrite, LoadAt(array_index + mip), coord, texel});
-		state.builder.AddFunction({OpBranch, merge_label});
+		state.builder.AddFunction(spv::OpImageWrite, LoadAt(array_index + mip), coord, texel);
+		state.builder.AddFunction(spv::OpBranch, merge_label);
 	}
 	EmitLabel(state, merge_label);
 }
 
-uint32_t ExecutionModelForStage(ShaderType stage) {
+spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {
 	switch (stage) {
-		case ShaderType::Vertex: return ExecutionModelVertex;
-		case ShaderType::Mesh: return 5365u; // MeshEXT
-		case ShaderType::Pixel: return ExecutionModelFragment;
-		default: return ExecutionModelGLCompute;
+		case ShaderType::Local:
+		case ShaderType::Vertex: return spv::ExecutionModelVertex;
+		case ShaderType::TessellationControl: return spv::ExecutionModelTessellationControl;
+		case ShaderType::TessellationEvaluation: return spv::ExecutionModelTessellationEvaluation;
+		case ShaderType::Mesh: return spv::ExecutionModelMeshEXT; // MeshEXT
+		case ShaderType::Pixel: return spv::ExecutionModelFragment;
+		default: return spv::ExecutionModelGLCompute;
 	}
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -1,6 +1,9 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+#include "common/logging/log.h"
 
 #include <algorithm>
+#include <atomic>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -16,90 +19,82 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
 	return true;
 }
 
-uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
-	const auto shifted_one = state.builder.AllocateId();
-	const auto merged_one  = state.builder.AllocateId();
-	const auto shifted_two = state.builder.AllocateId();
-	const auto merged_two  = state.builder.AllocateId();
-	const auto quad_bits   = state.builder.AllocateId();
-	const auto result      = state.builder.AllocateId();
-	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
-	                           ConstantU64(state, 0x0000000100000001ull)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
-	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
-	                           ConstantU64(state, 0x0000000200000002ull)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
-	                           ConstantU64(state, 0x1111111111111111ull)});
-	state.builder.AddFunction(
-	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
-	return result;
-}
-
-uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
-	auto& state = ctx.state;
+uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
 	if (kind == IR::StageInputKind::LocalInvocationIndex) {
 		return EmitLocalInvocationIndex(state);
 	}
 	if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId ||
 	                              kind == IR::StageInputKind::GlobalInvocationId)) {
-		const auto* cs      = ShaderWorkgroupInput(state.stage, state.input_info);
+		const auto* cs      = ShaderWorkgroupInput(state.program.stage, state.input_info);
 		uint32_t    divisor = 1;
 		for (uint32_t axis = 0; axis < component; axis++) {
 			divisor *= std::max(cs->threads_num[axis], 1u);
 		}
 		const auto size    = std::max(cs->threads_num[component], 1u);
-		const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state),
+		const auto divided = EmitBinaryU32(state, spv::OpUDiv, EmitLocalInvocationIndex(state),
 		                                   ConstantU32(state, divisor));
-		const auto local   = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size));
+		const auto local   = EmitBinaryU32(state, spv::OpUMod, divided, ConstantU32(state, size));
 		if (kind == IR::StageInputKind::LocalInvocationId) {
 			return local;
 		}
 		const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component);
 		return EmitAddU32(state, local,
-		                  EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size)));
+		                  EmitBinaryU32(state, spv::OpIMul, group, ConstantU32(state, size)));
 	}
-	const auto variable = InputVariableForKind(state, kind);
+	const bool centroid = kind == IR::StageInputKind::BaryCoordSmoothCentroid;
+	const auto variable = InputVariableForKind(
+	    state, centroid ? IR::StageInputKind::BaryCoordSmooth : kind);
 	if (variable == 0) {
 		return ConstantU32(state, 0);
 	}
 	if (kind == IR::StageInputKind::FrontFacing) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, TypeBool(state), value, variable});
-		state.builder.AddFunction(
-		    {OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		state.builder.AddFunction(spv::OpLoad, TypeBool(state), value, variable);
+		// PS5 initializes v_front_face with float +1.0/-1.0 bits.
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), bits, value,
+		                          ConstantU32(state, 0x3f800000u), ConstantU32(state, 0xbf800000u));
 		return bits;
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
+	    kind == IR::StageInputKind::InvocationId || kind == IR::StageInputKind::PrimitiveId ||
 	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, TypeI32(state), value, variable});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction(spv::OpLoad, TypeI32(state), value, variable);
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
-	if (kind == IR::StageInputKind::FragCoord) {
+	if (kind == IR::StageInputKind::FragCoord || kind == IR::StageInputKind::TessCoord) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain,
-		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		                           variable, ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		state.builder.AddFunction(spv::OpAccessChain,
+		                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+		                          pointer, variable, ConstantU32(state, component));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
-	if (kind == IR::StageInputKind::BaryCoordSmooth ||
+	if (centroid || kind == IR::StageInputKind::BaryCoordSmooth ||
 	    kind == IR::StageInputKind::BaryCoordNoPerspective) {
-		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain,
-		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		                           variable, ConstantU32(state, component + 1u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		if (centroid) {
+			const auto coordinates = state.builder.AllocateId();
+			state.builder.RequireCapability(spv::CapabilityInterpolationFunction);
+			state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 3), coordinates,
+			                          GlslStd450(state), GLSLstd450InterpolateAtCentroid, variable);
+			state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), value,
+			                          coordinates, component + 1u);
+		} else {
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpAccessChain,
+			                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+			                          pointer, variable, ConstantU32(state, component + 1u));
+			state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
+		}
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
 	return EmitInputComponentU32(state, kind, component);
@@ -120,55 +115,55 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	const auto bank_ok    = state.builder.AllocateId();
 	const auto row_ok     = state.builder.AllocateId();
 	const auto masks_ok   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), bank_shift, lane,
+	                          ConstantU32(state, 2));
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), row_shift, lane,
+	                          ConstantU32(state, 4));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), bank, bank_shift,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, row_shift,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), bank_bit,
+	                          ConstantU32(state, 1), bank);
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), row_bit,
+	                          ConstantU32(state, 1), row);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), bank_hit,
+	                          ConstantU32(state, flags.bank_mask), bank_bit);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row_hit,
+	                          ConstantU32(state, flags.row_mask), row_bit);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), bank_ok, bank_hit,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), row_ok, row_hit,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok);
 	uint32_t writable = masks_ok;
 	if (!flags.bound_control) {
 		const auto target  = EmitDppTargetLane(state, flags.control);
 		const auto bounded = state.builder.AllocateId();
-		state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
+		state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), bounded, writable,
+		                          target.valid);
 		writable = bounded;
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable});
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), result, exec, writable);
 	return result;
 }
 
-uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
-	auto&       state = ctx.state;
+uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
 	const auto* input = InputBindingForParameter(state, attr);
 	if (input == nullptr || input->variable_id == 0) {
 		return ConstantU32(state, 0);
 	}
-	if (state.stage == ShaderType::Vertex) {
+	if (state.program.stage == ShaderType::Vertex || state.program.stage == ShaderType::Local) {
 		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
 	}
 	const auto load_per_vertex = [&](uint32_t vertex) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		    spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
+		    input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
 		return value;
 	};
 	if (input->per_vertex) {
@@ -181,30 +176,31 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
 			const auto pointer = state.builder.AllocateId();
 			const auto weight  = state.builder.AllocateId();
 			const auto product = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain,
-			                           TypePointer(state, StorageClassInput, TypeF32(state)),
-			                           pointer, barycentric, ConstantU32(state, vertex)});
-			state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer});
-			state.builder.AddFunction(
-			    {OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
+			state.builder.AddFunction(spv::OpAccessChain,
+			                          TypePointer(state, spv::StorageClassInput, TypeF32(state)),
+			                          pointer, barycentric, ConstantU32(state, vertex));
+			state.builder.AddFunction(spv::OpLoad, TypeF32(state), weight, pointer);
+			state.builder.AddFunction(spv::OpFMul, TypeF32(state), product, load_per_vertex(vertex),
+			                          weight);
 			if (vertex == 0u) {
 				sum = product;
 			} else {
 				const auto next = state.builder.AllocateId();
-				state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product});
+				state.builder.AddFunction(spv::OpFAdd, TypeF32(state), next, sum, product);
 				sum = next;
 			}
 		}
 		const auto bits = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum});
+		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, sum);
 		return bits;
 	}
 	const auto vector    = state.builder.AllocateId();
 	const auto component = state.builder.AllocateId();
 	const auto bits      = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
-	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component});
+	state.builder.AddFunction(spv::OpLoad, TypeF32Vector(state, 4), vector, input->variable_id);
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), component, vector,
+	                          chan & 3u);
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, component);
 	return bits;
 }
 
@@ -213,15 +209,15 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	auto&       state = ctx.state;
 	const auto* input = InputBindingForParameter(state, attr);
 	if (!input->per_vertex) {
-		return EmitAttribute(ctx, attr, chan);
+		return EmitAttribute(ctx.state, attr, chan);
 	}
 	const auto load_vertex = [&](uint32_t vertex) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
-		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
-		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		    spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer,
+		    input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u));
+		state.builder.AddFunction(spv::OpLoad, TypeF32(state), value, pointer);
 		return value;
 	};
 
@@ -229,16 +225,16 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 	uint32_t   value           = load_vertex(selected_vertex);
 	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
 		const auto delta = state.builder.AllocateId();
-		state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)});
+		state.builder.AddFunction(spv::OpFSub, TypeF32(state), delta, value, load_vertex(0));
 		value = delta;
 	}
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 	return bits;
 }
 
 uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
-	if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
+	if (state.program.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
 	    exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
 		return 0;
 	}
@@ -247,8 +243,8 @@ uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
 
 uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
 	const auto value = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(
-	    {OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), value, vector,
+	                              component);
 	return value;
 }
 
@@ -257,7 +253,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	auto& state = ctx.state;
 	if (exp.compr && !uint_output) {
 		const auto unpack =
-		    MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
+		    MrtOutputMode(state, exp) == 5u ? GLSLstd450UnpackUnorm2x16 : GLSLstd450UnpackHalf2x16;
 		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
 		                   ConstantF32(state, 0x3f800000u)};
 		for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -266,21 +262,21 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 			}
 			const auto packed   = state.builder.AllocateId();
 			const auto unpacked = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
-			state.builder.AddFunction(
-			    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed});
+			state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), packed, data, pair);
+			state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 2), unpacked,
+			                          GlslStd450(state), unpack, packed);
 			for (uint32_t lane = 0; lane < 2u; lane++) {
 				const auto component = pair * 2u + lane;
 				if (((exp.en >> component) & 1u) != 0u) {
 					f32[component] = state.builder.AllocateId();
-					state.builder.AddFunction(
-					    {OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
+					state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state),
+					                          f32[component], unpacked, lane);
 				}
 			}
 		}
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
-		                           f32[1], f32[2], f32[3]});
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector,
+		                          f32[0], f32[1], f32[2], f32[3]);
 		return vector;
 	}
 	uint32_t raw[4] = {
@@ -301,9 +297,9 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 					continue;
 				}
 				raw[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component],
-				                           packed, ConstantU32(state, lane * 16u),
-				                           ConstantU32(state, 16)});
+				state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), raw[component],
+				                          packed, ConstantU32(state, lane * 16u),
+				                          ConstantU32(state, 16));
 			}
 		}
 	} else {
@@ -315,18 +311,18 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	}
 	if (uint_output) {
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0],
-		                           raw[1], raw[2], raw[3]});
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), vector,
+		                          raw[0], raw[1], raw[2], raw[3]);
 		return vector;
 	}
 	uint32_t f32[4] {};
 	for (uint32_t component = 0; component < 4u; component++) {
 		f32[component] = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]});
+		state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32[component], raw[component]);
 	}
 	const auto vector = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]});
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
+	                          f32[1], f32[2], f32[3]);
 	return vector;
 }
 
@@ -342,19 +338,19 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 			const auto raw = ExportRawComponent(ctx, data, component);
 			if (output.layer) {
 				const auto layer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
-				                           ConstantU32(state, 0x7ffu)});
-				const auto pointer = state.stage == ShaderType::Mesh
+				state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), layer, raw,
+				                          ConstantU32(state, 0x7ffu));
+				const auto pointer = state.program.stage == ShaderType::Mesh
 				                         ? MeshOutputPointer(state, IR::StageOutputKind::Layer)
 				                         : state.layer_variable;
-				state.builder.AddFunction({OpStore, pointer, layer});
+				state.builder.AddFunction(spv::OpStore, pointer, layer);
 			}
 			if (output.viewport) {
 				// GFX10 MISC.z packs the viewport index in bits 16..19 alongside the layer.
 				const auto viewport = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), viewport, raw,
-				                           ConstantU32(state, 16), ConstantU32(state, 4)});
-				state.builder.AddFunction({OpStore, state.viewport_index_variable, viewport});
+				state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), viewport, raw,
+				                          ConstantU32(state, 16), ConstantU32(state, 4));
+				state.builder.AddFunction(spv::OpStore, state.viewport_index_variable, viewport);
 			}
 			continue;
 		}
@@ -365,9 +361,9 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 
 		const auto raw = ExportRawComponent(ctx, data, component);
 		const auto f32 = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+		state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32, raw);
 		if (output.point_size) {
-			state.builder.AddFunction({OpStore, state.point_size_variable, f32});
+			state.builder.AddFunction(spv::OpStore, state.point_size_variable, f32);
 			continue;
 		}
 		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
@@ -375,10 +371,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 				return;
 			}
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer,
-			     variable, ConstantU32(state, index)});
-			state.builder.AddFunction({OpStore, pointer, f32});
+			state.builder.AddFunction(spv::OpAccessChain,
+			                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+			                          pointer, variable, ConstantU32(state, index));
+			state.builder.AddFunction(spv::OpStore, pointer, f32);
 		};
 		StoreDistance(state.clip_distance_variable, output.clip_distance);
 		StoreDistance(state.cull_distance_variable, output.cull_distance);
@@ -391,14 +387,14 @@ uint32_t ConvertClipCoordinate(EmitterState& state, uint32_t coordinate, float s
 	const auto biased  = state.builder.AllocateId();
 	const auto divided = state.builder.AllocateId();
 	const auto ndc     = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpFMul, TypeF32(state), window, coordinate, ConstantF32Value(state, scale)});
-	state.builder.AddFunction(
-	    {OpFAdd, TypeF32(state), biased, window, ConstantF32Value(state, offset)});
-	state.builder.AddFunction(
-	    {OpFDiv, TypeF32(state), divided, biased, ConstantF32Value(state, half_extent)});
-	state.builder.AddFunction(
-	    {OpFSub, TypeF32(state), ndc, divided, ConstantF32Value(state, 1.0f)});
+	state.builder.AddFunction(spv::OpFMul, TypeF32(state), window, coordinate,
+	                          ConstantF32Value(state, scale));
+	state.builder.AddFunction(spv::OpFAdd, TypeF32(state), biased, window,
+	                          ConstantF32Value(state, offset));
+	state.builder.AddFunction(spv::OpFDiv, TypeF32(state), divided, biased,
+	                          ConstantF32Value(state, half_extent));
+	state.builder.AddFunction(spv::OpFSub, TypeF32(state), ndc, divided,
+	                          ConstantF32Value(state, 1.0f));
 	return ndc;
 }
 
@@ -407,29 +403,51 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) {
 	uint32_t    components[4] {};
 	for (uint32_t i = 0; i < 4; i++) {
 		components[i] = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeExtract, TypeF32(state), components[i], position, i});
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), components[i], position,
+		                          i);
 	}
 	components[0] = ConvertClipCoordinate(state, components[0], transform.scale[0],
 	                                      transform.offset[0], transform.half_extent[0]);
 	components[1] = ConvertClipCoordinate(state, components[1], transform.scale[1],
 	                                      transform.offset[1], transform.half_extent[1]);
 	const auto converted = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), converted,
-	                           components[0], components[1], components[2], components[3]});
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), converted,
+	                          components[0], components[1], components[2], components[3]);
 	return converted;
 }
 
-void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
+} // namespace
+uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
+	                          ConstantU64(state, 0x0000000100000001ull));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one);
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                          ConstantU64(state, 0x0000000200000002ull));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one,
+	                          shifted_two);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                          ConstantU64(state, 0x1111111111111111ull));
+	state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits,
+	                          ConstantU64(state, 0x0000000f0000000full));
+	return result;
+}
+
+void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&       state = ctx.state;
 	const auto& exp   = ctx.Export(inst);
 	const auto  exec  = ctx.Arg(inst, 1);
-	if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
+	if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
 	    state.pixel_valid_mask_variable != 0) {
 		const auto value = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
-		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1),
+		                          ConstantU32(state, 0));
+		state.builder.AddFunction(spv::OpStore, state.pixel_valid_mask_variable, value);
 	}
 	if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
 		return;
@@ -437,9 +455,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	EmitIfCondition(state, exec, [&]() {
 		const auto data = ctx.Arg(inst, 0);
 		if (exp.kind == IR::ExportTargetKind::Primitive) {
-			if (state.stage == ShaderType::Mesh) {
-				state.builder.AddFunction(
-				    {OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
+			if (state.program.stage == ShaderType::Mesh) {
+				state.builder.AddFunction(spv::OpStore, MeshPrimitivePointer(state),
+				                          ExportRawComponent(ctx, data, 0));
 			}
 			return;
 		}
@@ -451,37 +469,37 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
 				const auto raw = ExportRawComponent(ctx, data, 0);
 				const auto f32 = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
-				state.builder.AddFunction({OpStore, state.depth_variable, f32});
+				state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32, raw);
+				state.builder.AddFunction(spv::OpStore, state.depth_variable, f32);
 			}
 			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
 				const auto raw     = ExportRawComponent(ctx, data, 2);
 				const auto value   = state.builder.AllocateId();
 				const auto pointer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw});
+				state.builder.AddFunction(spv::OpBitcast, TypeI32(state), value, raw);
 				state.builder.AddFunction(
-				    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer,
-				     state.sample_mask_variable, ConstantU32(state, 0)});
-				state.builder.AddFunction({OpStore, pointer, value});
+				    spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, TypeI32(state)),
+				    pointer, state.sample_mask_variable, ConstantU32(state, 0));
+				state.builder.AddFunction(spv::OpStore, pointer, value);
 			}
 			return;
 		}
 		const auto variable =
-		    state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
-		if (state.stage != ShaderType::Mesh && variable == 0) {
+		    state.program.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
+		if (state.program.stage != ShaderType::Mesh && variable == 0) {
 			return;
 		}
 		const bool uint_output = MrtOutputMode(state, exp) == 7u;
 		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
-		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
 		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
 			const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];
 			if (!mapping.IsIdentity()) {
 				const auto mapped = state.builder.AllocateId();
-				state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
-				                           mapping.Map(0), mapping.Map(1), mapping.Map(2),
-				                           mapping.Map(3)});
+				state.builder.AddFunction(spv::OpVectorShuffle, vector_type, mapped, value, value,
+				                          mapping.Map(0), mapping.Map(1), mapping.Map(2),
+				                          mapping.Map(3));
 				value = mapped;
 			}
 		}
@@ -489,192 +507,217 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		    state.input_info.vertex->clip_space.enabled) {
 			value = ConvertPositionToClipSpace(state, value);
 		}
-		if (state.stage == ShaderType::Mesh) {
+		if (state.program.stage == ShaderType::Mesh) {
 			const auto kind = exp.kind == IR::ExportTargetKind::Position
 			                      ? IR::StageOutputKind::Position
 			                      : IR::StageOutputKind::Parameter;
-			state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value});
+			state.builder.AddFunction(spv::OpStore, MeshOutputPointer(state, kind, exp.index),
+			                          value);
 		} else if (exp.kind == IR::ExportTargetKind::Position) {
+			if (state.invalid_position_clip_distance != UINT32_MAX) {
+				const auto zero = state.builder.Constant(spv::OpConstantNull, TypeF32Vector(state, 4));
+				const auto equal = state.builder.AllocateId();
+				const auto invalid = state.builder.AllocateId();
+				const auto distance = state.builder.AllocateId();
+				const auto distance_pointer = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpFOrdEqual, TypeBoolVector(state, 4), equal,
+				                          value, zero);
+				state.builder.AddFunction(spv::OpAll, TypeBool(state), invalid, equal);
+				// Zero at valid vertices makes a primitive containing an invalid position
+				// collapse to its remaining edge, before the undefined 0/0 perspective divide.
+				state.builder.AddFunction(spv::OpSelect, TypeF32(state), distance, invalid,
+				                          ConstantF32Value(state, -1.0f),
+				                          ConstantF32Value(state, 0.0f));
+				state.builder.AddFunction(
+				    spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+				    distance_pointer, state.clip_distance_variable,
+				    ConstantU32(state, state.invalid_position_clip_distance));
+				state.builder.AddFunction(spv::OpStore, distance_pointer, distance);
+				static std::atomic_bool logged = false;
+				if (!logged.exchange(true, std::memory_order_relaxed)) {
+					Log::WriteToConsoleAndLog(
+					    "Shader: emitted zero-position clip guard\n");
+				}
+			}
 			const auto pointer = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
-			     pointer, variable, ConstantU32(state, 0)});
-			state.builder.AddFunction({OpStore, pointer, value});
+			    spv::OpAccessChain,
+			    TypePointer(state, spv::StorageClassOutput, TypeF32Vector(state, 4)), pointer,
+			    variable, ConstantU32(state, 0));
+			state.builder.AddFunction(spv::OpStore, pointer, value);
 		} else {
-			state.builder.AddFunction({OpStore, variable, value});
+			state.builder.AddFunction(spv::OpStore, variable, value);
 		}
 	});
 }
 
-} // namespace
+uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
+	return value;
+}
 
-bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
-	auto& state = ctx.state;
-	switch (inst.GetOpcode()) {
-		case IR::ValueOpcode::Identity: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
-		case IR::ValueOpcode::Void:
-		case IR::ValueOpcode::Reference:
-		case IR::ValueOpcode::ReferenceU32:
-		case IR::ValueOpcode::ControlNop:
-		case IR::ValueOpcode::Waitcnt:
-		case IR::ValueOpcode::Sendmsg:
-		case IR::ValueOpcode::TtraceData:
-		case IR::ValueOpcode::InstPrefetch: return true;
-		case IR::ValueOpcode::Barrier: {
-			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
-			state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
-			                           ConstantU32(state, ScopeWorkgroup),
-			                           ConstantU32(state, semantics)});
-			return true;
-		}
-		case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true;
-		case IR::ValueOpcode::MeshDrawParameter: {
-			const auto index = inst.Arg(0).U32();
-			if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
-				ctx.Fail(inst, "invalid mesh draw parameter");
-			}
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state),
-			                           pointer, state.push_constant_variable, ConstantU32(state, 0),
-			                           ConstantU32(state, index)});
-			state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer});
-			return true;
-		}
-		case IR::ValueOpcode::GetUserData: {
-			const auto reg   = inst.Arg(0).ScalarRegister();
-			uint32_t   dword = 0;
-			if (!UserDataDwordIndex(state, reg, dword)) {
-				ctx.Define(inst, ConstantU32(state, 0));
-			} else {
-				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
-			}
-			return true;
-		}
-		case IR::ValueOpcode::GetBuiltin:
-			ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
-			                                inst.Arg(1).U32()));
-			return true;
-		case IR::ValueOpcode::UndefU1:
-		case IR::ValueOpcode::UndefU8:
-		case IR::ValueOpcode::UndefU16:
-		case IR::ValueOpcode::UndefU32:
-		case IR::ValueOpcode::UndefU64:
-			state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
-			return true;
-		case IR::ValueOpcode::DppMoveU32: {
-			const auto flags    = inst.Flags<IR::DppMoveFlags>();
-			const auto target   = EmitDppTargetLane(state, flags.control);
-			const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
-			if (flags.fetch_inactive) {
-				ctx.Define(inst, shuffled);
-				return true;
-			}
-			const auto ballot        = ctx.Ballot(inst.Arg(1));
-			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
-			const auto can_fetch     = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
-			return true;
-		}
-		case IR::ValueOpcode::DppUpdateU32: {
-			const auto flags = inst.Flags<IR::DppMoveFlags>();
-			const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
-			return true;
-		}
-		case IR::ValueOpcode::WqmU64:
-			ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
-			return true;
-		case IR::ValueOpcode::LaneId:
-			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
-			return true;
-		case IR::ValueOpcode::Ballot: ctx.Define(inst, ctx.Ballot(inst.Arg(0))); return true;
-		case IR::ValueOpcode::ReadFirstLane: {
-			const auto ballot = ctx.Ballot(inst.Arg(1));
-			const auto lane   = ctx.FirstLane(ballot);
-			ctx.Define(inst, ctx.Shuffle(inst, 0, lane));
-			return true;
-		}
-		case IR::ValueOpcode::ReadLane:
-			ctx.Define(inst, ctx.Shuffle(inst, 0, ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::WriteLane: {
-			const auto hit = state.builder.AllocateId();
-			state.builder.AddFunction({OpIEqual, TypeBool(state), hit,
-			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
-			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
-			return true;
-		}
-		case IR::ValueOpcode::Permlane16U32: {
-			const auto flags     = inst.Flags<IR::PermlaneFlags>();
-			const auto subid     = EmitSubgroupLocalInvocationId(state);
-			const auto row       = state.builder.AllocateId();
-			const auto row_value = state.builder.AllocateId();
-			const auto lane      = state.builder.AllocateId();
-			const auto lane8     = state.builder.AllocateId();
-			const auto shift     = state.builder.AllocateId();
-			const auto upper     = state.builder.AllocateId();
-			const auto selected  = state.builder.AllocateId();
-			const auto shifted   = state.builder.AllocateId();
-			const auto index     = state.builder.AllocateId();
-			const auto target    = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-			if (flags.x16) {
-				state.builder.AddFunction(
-				    {OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
-			} else {
-				state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row});
-			}
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
-			state.builder.AddFunction(
-			    {OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
-			state.builder.AddFunction(
-			    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
-			state.builder.AddFunction(
-			    {OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
-			state.builder.AddFunction(
-			    {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
-			state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
-			const auto shuffled = ctx.Shuffle(inst, 0, target);
-			uint32_t result = shuffled;
-			if (!flags.fetch_inactive) {
-				const auto source_exec = ctx.Shuffle(inst, 3, target);
-				result = state.builder.AllocateId();
-				state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
-				                           ConstantU32(state, 0)});
-			}
-			ctx.Define(inst, result);
-			return true;
-		}
-		case IR::ValueOpcode::GetAttribute:
-			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
-			return true;
-		case IR::ValueOpcode::GetInterpolationParameter:
-			ctx.Define(inst, EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(),
-			                                            inst.Arg(2).U32()));
-			return true;
-		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
-		case IR::ValueOpcode::GetShaderBase:
-			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
-			// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
-			ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
-			return true;
-		case IR::ValueOpcode::GetSrtResource:
-		case IR::ValueOpcode::GetBufferResource:
-		case IR::ValueOpcode::GetAddressResource:
-		case IR::ValueOpcode::GetScratchResource:
-		case IR::ValueOpcode::GetImageResource:
-		case IR::ValueOpcode::GetSamplerResource:
-		case IR::ValueOpcode::MakeImageAddress: return true;
-		default: return false;
+void EmitVoid(ValueEmitContext&) {}
+
+void EmitBarrier(EmitterState& state) {
+	const auto tessellation = state.program.stage == ShaderType::TessellationControl;
+	const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
+	const auto semantics    = tessellation ? spv::MemorySemanticsMaskNone
+	                                       : spv::MemorySemanticsAcquireReleaseMask |
+	                                             spv::MemorySemanticsWorkgroupMemoryMask;
+	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
+	                          ConstantU32(state, memory_scope), ConstantU32(state, semantics));
+}
+
+uint32_t EmitLaneId(EmitterState& state) {
+	return state.program.stage == ShaderType::TessellationControl
+	           ? EmitBuiltinU32(state, IR::StageInputKind::InvocationId, 0)
+	           : EmitSubgroupLocalInvocationId(state);
+}
+
+uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state  = ctx.state;
+	const auto result = state.builder.AllocateId();
+	const auto index  = inst.Arg(0).U32();
+	if (state.program.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) {
+		ctx.Fail(inst, "invalid mesh draw parameter");
+	}
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer,
+	                          state.push_constant_variable, ConstantU32(state, 0),
+	                          ConstantU32(state, index));
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), result, pointer);
+	return result;
+}
+
+uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg) {
+
+	uint32_t dword = 0;
+	if (!UserDataDwordIndex(state, reg, dword)) {
+		return ConstantU32(state, 0);
+	} else {
+		return EmitShaderDataDwordLoad(state, dword);
 	}
 }
 
+uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) {
+	return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
+}
+
+uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpUndef, TypeId(state, inst.GetType()), result);
+	return result;
+}
+
+uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state    = ctx.state;
+	const auto flags    = inst.Flags<IR::DppMoveFlags>();
+	const auto target   = EmitDppTargetLane(state, flags.control);
+	const auto shuffled = ctx.Shuffle(inst, 0, target.lane);
+	if (flags.fetch_inactive) {
+		return shuffled;
+	}
+	const auto ballot        = ctx.Ballot(inst.Arg(1));
+	const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
+	const auto can_fetch     = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid,
+	                          source_active);
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, can_fetch, shuffled,
+	                                                ConstantU32(state, 0));
+}
+
+uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto flags = inst.Flags<IR::DppMoveFlags>();
+	const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, write, ctx.Arg(inst, 0),
+	                                                ctx.Arg(inst, 1));
+}
+
+uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
+	return ctx.Ballot(predicate);
+}
+
+uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto ballot = ctx.Ballot(inst.Arg(1));
+	const auto lane   = ctx.FirstLane(ballot);
+	return ctx.Shuffle(inst, 0, lane);
+}
+
+uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1));
+}
+
+uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state = ctx.state;
+	const auto hit   = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), hit,
+	                          EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2));
+	return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, hit, ctx.Arg(inst, 1),
+	                                                ctx.Arg(inst, 0));
+}
+
+uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state     = ctx.state;
+	const auto flags     = inst.Flags<IR::PermlaneFlags>();
+	const auto subid     = EmitSubgroupLocalInvocationId(state);
+	const auto row       = state.builder.AllocateId();
+	const auto row_value = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto lane8     = state.builder.AllocateId();
+	const auto shift     = state.builder.AllocateId();
+	const auto upper     = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto shifted   = state.builder.AllocateId();
+	const auto index     = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
+	                          ConstantU32(state, 0xfffffff0u));
+	if (flags.x16) {
+		state.builder.AddFunction(spv::OpBitwiseXor, TypeU32(state), row_value, row,
+		                          ConstantU32(state, 16));
+	} else {
+		state.builder.AddFunction(spv::OpCopyObject, TypeU32(state), row_value, row);
+	}
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 15));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane8, lane,
+	                          ConstantU32(state, 7));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane8,
+	                          ConstantU32(state, 2));
+	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), upper, lane,
+	                          ConstantU32(state, 8));
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2),
+	                          ctx.Arg(inst, 1));
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), shifted, selected, shift);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), index, shifted,
+	                          ConstantU32(state, 15));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row_value, index);
+	const auto shuffled = ctx.Shuffle(inst, 0, target);
+	uint32_t   result   = shuffled;
+	if (!flags.fetch_inactive) {
+		const auto source_exec = ctx.Shuffle(inst, 3, target);
+		result                 = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), result, source_exec, shuffled,
+		                          ConstantU32(state, 0));
+	}
+	return result;
+}
+
+uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32());
+}
+
+uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(), inst.Arg(2).U32());
+}
+
+uint32_t EmitGetShaderBase(ValueEmitContext& ctx) {
+	// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
+	// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
+	return ctx.Def(IR::Value(uint64_t {0}));
+}
+
+void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) {
+	ctx.Fail(inst, "must be lowered before SPIR-V emission");
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -9,16 +9,17 @@ DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto selected0 = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), quad_base, subid,
+	                          ConstantU32(state, 0xfffffffcu));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane,
+	                          ConstantU32(state, 1));
+	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), selected0,
+	                          ConstantU32(state, control), shift);
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), selected, selected0,
+	                          ConstantU32(state, 3));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, quad_base, selected);
 	return {target, ConstantBool(state, true)};
 }
 
@@ -29,21 +30,22 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto lane_shifted = state.builder.AllocateId();
 	const auto target       = state.builder.AllocateId();
 	const auto valid        = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
+	                          ConstantU32(state, 0xfffffff0u));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 15));
 	if (left) {
-		state.builder.AddFunction(
-		    {OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
-		state.builder.AddFunction(
-		    {OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
+		state.builder.AddFunction(spv::OpIAdd, TypeU32(state), lane_shifted, lane,
+		                          ConstantU32(state, amount));
+		state.builder.AddFunction(spv::OpULessThan, TypeBool(state), valid, lane,
+		                          ConstantU32(state, 16u - amount));
 	} else {
-		state.builder.AddFunction(
-		    {OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
+		state.builder.AddFunction(spv::OpISub, TypeU32(state), lane_shifted, lane,
+		                          ConstantU32(state, amount));
+		state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), valid, lane,
+		                          ConstantU32(state, amount));
 	}
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row, lane_shifted);
 	return {target, valid};
 }
 
@@ -56,16 +58,17 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
 	const auto plus     = state.builder.AllocateId();
 	const auto selected = state.builder.AllocateId();
 	const auto target   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction(
-	    {OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
-	state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), row, subid,
+	                          ConstantU32(state, 0xfffffff0u));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 15));
+	state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane,
+	                          ConstantU32(state, amount));
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount));
+	state.builder.AddFunction(spv::OpIAdd, TypeU32(state), plus, lane,
+	                          ConstantU32(state, 16u - amount));
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, in_high, minus, plus);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, row, selected);
 	return {target, ConstantBool(state, true)};
 }
 
@@ -76,13 +79,13 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
 	const auto lane      = state.builder.AllocateId();
 	const auto mirrored  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
-	state.builder.AddFunction(
-	    {OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), base, subid,
+	                          ConstantU32(state, base_mask));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, lane_mask));
+	state.builder.AddFunction(spv::OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask),
+	                          lane);
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, base, mirrored);
 	return {target, ConstantBool(state, true)};
 }
 
@@ -108,8 +111,8 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
 	}
 	if (control >= 0x160u && control <= 0x16fu) {
 		const auto target = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseXor, TypeU32(state), target, subid, ConstantU32(state, control & 0xfu)});
+		state.builder.AddFunction(spv::OpBitwiseXor, TypeU32(state), target, subid,
+		                          ConstantU32(state, control & 0xfu));
 		return {target, ConstantBool(state, true)};
 	}
 	return {subid, ConstantBool(state, true)};
@@ -120,8 +123,8 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
 		EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n");
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value,
+	                          state.subgroup_local_invocation_id_variable);
 	return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32));
 }
 
@@ -150,9 +153,10 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
 	}
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)),
-	                           pointer, variable, ConstantU32(state, component)});
-	state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+	state.builder.AddFunction(spv::OpAccessChain,
+	                          TypePointer(state, spv::StorageClassInput, TypeU32(state)), pointer,
+	                          variable, ConstantU32(state, component));
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
 	return value;
 }
 
@@ -162,9 +166,10 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
 		return ConstantU32(state, 0);
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, variable);
 	if (state.lane_count == 2) {
-		const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u));
+		const auto wave_base =
+		    EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, ~31u));
 		return EmitAddU32(state, EmitAddU32(state, value, wave_base),
 		                  ConstantU32(state, state.lane_half * 32));
 	}
@@ -178,13 +183,13 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	const auto scalar_type = VertexParameterScalarType(state, kind);
 	uint32_t   raw         = state.builder.AllocateId();
 	if (count == 1u) {
-		state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
+		state.builder.AddFunction(spv::OpLoad, scalar_type, raw, input.variable_id);
 	} else {
-		const auto pointer_type = TypePointer(state, StorageClassInput, scalar_type);
+		const auto pointer_type = TypePointer(state, spv::StorageClassInput, scalar_type);
 		const auto pointer      = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
-		                           ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
+		state.builder.AddFunction(spv::OpAccessChain, pointer_type, pointer, input.variable_id,
+		                          ConstantU32(state, component));
+		state.builder.AddFunction(spv::OpLoad, scalar_type, raw, pointer);
 	}
 
 	if (kind == VertexInputScalarKind::Uint) {
@@ -192,47 +197,48 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	}
 
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw});
+	state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, raw);
 	return bits;
 }
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
 	const auto active_ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
-	                           ConstantU32(state, ScopeSubgroup), ConstantBool(state, true)});
+	state.builder.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
+	                          ConstantU32(state, spv::ScopeSubgroup), ConstantBool(state, true));
 	return EmitBallotLaneActiveBool(state, active_ballot, lane);
 }
 uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
 	const auto low = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, active_ballot, 0);
 	uint32_t mask = low;
 	if (state.program.wave_size == 64u) {
 		const auto high     = state.builder.AllocateId();
 		const auto in_high  = state.builder.AllocateId();
 		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low});
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, active_ballot, 1);
+		state.builder.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane,
+		                          ConstantU32(state, 32));
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, in_high, high, low);
 		mask = selected;
 	}
 
 	const auto lane_low = state.builder.AllocateId();
 	const auto bit      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane_low, lane,
+	                          ConstantU32(state, 31));
+	state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1),
+	                          lane_low);
 
 	const auto hit      = state.builder.AllocateId();
 	const auto active   = state.builder.AllocateId();
 	const auto in_range = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.program.wave_size)});
-	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), hit, mask, bit);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), active, hit,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), in_range, lane,
+	                          ConstantU32(state, state.program.wave_size));
+	state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), ret, active, in_range);
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -1,5 +1,5 @@
 #include "graphics/guest_gpu/gpu_format.h"
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
 
 #include <algorithm>
@@ -44,10 +44,10 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	auto value = ctx.Def(address.Arg(packed));
 	if (layout.bit_width == 16u) {
 		if ((layout.bit_offset & 31u) != 0u) {
-			value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value,
+			value = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), value,
 			               ConstantU32(ctx.state, 16));
 		}
-		value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value,
+		value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), value,
 		               ConstantU32(ctx.state, 0xffffu));
 	}
 	return value;
@@ -58,7 +58,7 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	const auto value = AddressU32(ctx, mem, address, component);
 	return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u
 	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value);
+	           : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
 }
 
 ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) {
@@ -85,19 +85,19 @@ uint32_t ZeroF32(EmitterState& state) {
 }
 
 uint32_t CubeAxis(EmitterState& state, uint32_t value) {
-	return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
+	return Binary(state, spv::OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
 }
 
 uint32_t CubeLayer(EmitterState& state, uint32_t value) {
 	const auto guest = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value});
-	const auto padding =
-	    Binary(state, OpShiftLeftLogical, TypeU32(state),
-	           Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
-	           ConstantU32(state, 1));
-	const auto host   = Binary(state, OpISub, TypeU32(state), guest, padding);
+	state.builder.AddFunction(spv::OpConvertFToU, TypeU32(state), guest, value);
+	const auto padding = Binary(
+	    state, spv::OpShiftLeftLogical, TypeU32(state),
+	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
+	    ConstantU32(state, 1));
+	const auto host   = Binary(state, spv::OpISub, TypeU32(state), guest, padding);
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host});
+	state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), result, host);
 	return result;
 }
 
@@ -118,11 +118,11 @@ uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		             ? AddressF32(ctx, mem, address, first + 2u)
 		             : ZeroF32(ctx.state);
 		if (cube) z = CubeLayer(ctx.state, z);
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 3),
+		                              result, x, y, z);
 	} else {
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 2),
+		                              result, x, y);
 	}
 	return result;
 }
@@ -138,11 +138,11 @@ uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 	if (components == 3u) {
 		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
 		                                                 : ConstantU32(ctx.state, 0);
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 3),
+		                              result, x, y, z);
 	} else {
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 2),
+		                              result, x, y);
 	}
 	return result;
 }
@@ -156,7 +156,7 @@ uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst
 }
 
 uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
-	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+	return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
 }
 
 uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
@@ -164,7 +164,7 @@ uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
 	if (numeric_class == Prospero::TextureNumericClass::Uint) {
 		return value;
 	}
-	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+	return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
 }
 
 uint32_t SampledComponentZero(EmitterState& state, Prospero::TextureNumericClass numeric_class) {
@@ -193,9 +193,9 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 			if (dref) return value;
 			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
 			const auto result    = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpCompositeExtract,
-			                               ImageScalarType(ctx.state, value_class), result, value,
-			                               component});
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract,
+			                              ImageScalarType(ctx.state, value_class), result, value,
+			                              component);
 			return result;
 		};
 		for (uint32_t word = 0; word < mem.data_dwords; word++) {
@@ -209,24 +209,25 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 				const auto low_bits  = SampledComponentBits(ctx, low, value_class);
 				const auto high_bits = SampledComponentBits(ctx, high, value_class);
 				const auto mask      = ConstantU32(ctx.state, 0xffffu);
-				packed[word]         = Binary(
-				    ctx.state, OpBitwiseOr, TypeU32(ctx.state),
-				    Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
-				    Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
-				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high_bits, mask),
-				           ConstantU32(ctx.state, 16u)));
+				packed[word] =
+				    Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state),
+				           Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
+				           Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
+				                  Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
+				                         high_bits, mask),
+				                  ConstantU32(ctx.state, 16u)));
 			} else {
 				const auto pair = ctx.state.builder.AllocateId();
-				ctx.state.builder.AddFunction(
-				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
+				                              TypeF32Vector(ctx.state, 2), pair, low, high);
 				packed[word] = ctx.state.builder.AllocateId();
-				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
-				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+				ctx.state.builder.AddFunction(spv::OpExtInst, TypeU32(ctx.state), packed[word],
+				                              GlslStd450(ctx.state), GLSLstd450PackHalf2x16, pair);
 			}
 		}
 		const auto result = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-		                               packed[0], packed[1], packed[2], packed[3]});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4),
+		                              result, packed[0], packed[1], packed[2], packed[3]);
 		return result;
 	}
 	uint32_t component[4] {};
@@ -237,12 +238,12 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
 		}
 		const auto scalar = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, ImageScalarType(ctx.state, value_class), scalar, value, index});
+		    spv::OpCompositeExtract, ImageScalarType(ctx.state, value_class), scalar, value, index);
 		component[index] = SampledComponentBits(ctx, scalar, value_class);
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                               component[0], component[1], component[2], component[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              component[0], component[1], component[2], component[3]);
 	return result;
 }
 
@@ -253,11 +254,12 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 	const auto  image     = LoadSampledImageDescriptor(ctx.state, mem.resource);
 	const auto  size      = ctx.state.builder.AllocateId();
 	if (info.multisampled != 0u) {
-		ctx.state.builder.AddFunction(
-		    {OpImageQuerySize, ImageViewSizeType(ctx.state, dimension), size, image});
+		ctx.state.builder.AddFunction(spv::OpImageQuerySize,
+		                              ImageViewSizeType(ctx.state, dimension), size, image);
 	} else {
-		ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, dimension),
-		                               size, image, AddressU32(ctx, mem, address, 0)});
+		ctx.state.builder.AddFunction(spv::OpImageQuerySizeLod,
+		                              ImageViewSizeType(ctx.state, dimension), size, image,
+		                              AddressU32(ctx, mem, address, 0));
 	}
 	const auto components = info.coordinate_components;
 	uint32_t   result[4]  = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
@@ -267,17 +269,18 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 	} else {
 		for (uint32_t index = 0; index < components; index++) {
 			result[index] = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state),
+			                              result[index], size, index);
 		}
 	}
 	if (info.multisampled == 0u) {
 		result[3] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
+		ctx.state.builder.AddFunction(spv::OpImageQueryLevels, TypeU32(ctx.state), result[3],
+		                              image);
 	}
 	const auto vector = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
-	                               result[0], result[1], result[2], result[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
+	                              result[0], result[1], result[2], result[3]);
 	return vector;
 }
 
@@ -289,31 +292,31 @@ uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR
 		if (components == 1u) return zero;
 		const auto result = ctx.state.builder.AllocateId();
 		if (components == 3u) {
-			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
+			ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3),
+			                              result, zero, zero, zero);
 		} else {
-			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
+			ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2),
+			                              result, zero, zero);
 		}
 		return result;
 	}
-	const auto packed    = Unary(ctx.state, OpBitcast, TypeI32(ctx.state),
+	const auto packed    = Unary(ctx.state, spv::OpBitcast, TypeI32(ctx.state),
 	                             AddressU32(ctx, mem, address, layout.offset));
 	uint32_t   values[3] = {zero, zero, zero};
 	for (uint32_t index = 0; index < components; index++) {
 		values[index] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index],
-		                               packed, ConstantU32(ctx.state, index * 8u),
-		                               ConstantU32(ctx.state, 6)});
+		ctx.state.builder.AddFunction(spv::OpBitFieldSExtract, TypeI32(ctx.state), values[index],
+		                              packed, ConstantU32(ctx.state, index * 8u),
+		                              ConstantU32(ctx.state, 6));
 	}
 	if (components == 1u) return values[0];
 	const auto result = ctx.state.builder.AllocateId();
 	if (components == 3u) {
-		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result,
-		                               values[0], values[1], values[2]});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3),
+		                              result, values[0], values[1], values[2]);
 	} else {
-		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
+		ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2),
+		                              result, values[0], values[1]);
 	}
 	return result;
 }
@@ -322,19 +325,19 @@ uint32_t HorizontalOffsets(EmitterState& state, ImageDimension dimension) {
 	const auto components   = ImageDimensionInfoFor(dimension).spatial_components;
 	const auto count        = ConstantU32(state, 4);
 	const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
-	const auto array_type   = state.builder.Type(OpTypeArray, {element_type, count});
+	const auto array_type   = state.builder.Type(spv::OpTypeArray, element_type, count);
 	uint32_t   offsets[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
 		if (components == 1u) {
 			offsets[index] = x;
 		} else {
-			offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2),
-			                                        {x, ConstantI32(state, 0)});
+			offsets[index] = state.builder.Constant(
+			    spv::OpConstantComposite, TypeI32Vector(state, 2), x, ConstantI32(state, 0));
 		}
 	}
-	return state.builder.Constant(OpConstantComposite, array_type,
-	                              {offsets[0], offsets[1], offsets[2], offsets[3]});
+	return state.builder.Constant(spv::OpConstantComposite, array_type, offsets[0], offsets[1],
+	                              offsets[2], offsets[3]);
 }
 
 uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
@@ -362,15 +365,15 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
 
 	const auto packed = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u);
 	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
 	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		components[component] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state),
-		                               components[component], packed,
-		                               ConstantU32(ctx.state, info.component_bit_offset[component]),
-		                               ConstantU32(ctx.state, info.component_bits[component])});
+		ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(ctx.state),
+		                              components[component], packed,
+		                              ConstantU32(ctx.state, info.component_bit_offset[component]),
+		                              ConstantU32(ctx.state, info.component_bits[component]));
 	}
 	for (uint32_t component = info.component_count; component < 4u; component++) {
 		components[component] = components[component % info.component_count];
@@ -389,8 +392,8 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 		}
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                               selected[0], selected[1], selected[2], selected[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              selected[0], selected[1], selected[2], selected[3]);
 	return result;
 }
 
@@ -403,42 +406,44 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
 	if (selector < 4u) {
 		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
-		return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4),
-		                                  {value, value, value, value});
+		return ctx.state.builder.Constant(spv::OpConstantComposite, TypeU32Vector(ctx.state, 4),
+		                                  value, value, value, value);
 	}
 
 	uint32_t values[4] {};
 	for (uint32_t lane = 0; lane < 4u; lane++) {
 		const auto packed = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), packed, gathered,
+		                              lane);
 		values[lane]        = ctx.state.builder.AllocateId();
 		const auto physical = (selector - 4u) % info.component_count;
-		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed,
-		                               ConstantU32(ctx.state, info.component_bit_offset[physical]),
-		                               ConstantU32(ctx.state, info.component_bits[physical])});
+		ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(ctx.state), values[lane],
+		                              packed,
+		                              ConstantU32(ctx.state, info.component_bit_offset[physical]),
+		                              ConstantU32(ctx.state, info.component_bits[physical]));
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                               values[0], values[1], values[2], values[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              values[0], values[1], values[2], values[3]);
 	return result;
 }
 
 uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                                     uint32_t coord, Prospero::TextureNumericClass numeric_class) {
 	auto& state = ctx.state;
-	state.builder.RequireCapability(CapabilityImageQuery);
+	state.builder.RequireCapability(spv::CapabilityImageQuery);
 	const auto image = LoadSampledImageDescriptor(state, mem.resource);
 	const auto width = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)});
+	state.builder.AddFunction(spv::OpImageQuerySizeLod, TypeU32(state), width, image,
+	                          ConstantU32(state, 0));
 	const auto width_f32 = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), width_f32, width});
+	state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), width_f32, width);
 	const auto left = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, TypeF32(state), left, GlslStd450(state), GlslFloor,
-	                           Binary(state, OpFSub, TypeF32(state),
-	                                  Binary(state, OpFMul, TypeF32(state), coord, width_f32),
-	                                  ConstantF32(state, 0x3f000000u))});
+	state.builder.AddFunction(spv::OpExtInst, TypeF32(state), left, GlslStd450(state),
+	                          GLSLstd450Floor,
+	                          Binary(state, spv::OpFSub, TypeF32(state),
+	                                 Binary(state, spv::OpFMul, TypeF32(state), coord, width_f32),
+	                                 ConstantF32(state, 0x3f000000u)));
 
 	const auto sampled     = MakeSampledImage(state, mem.resource, mem.sampler);
 	const auto vector_type = ImageVectorType(state, numeric_class, 4);
@@ -450,20 +455,20 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 	uint32_t values[2] {};
 	for (uint32_t index = 0; index < 2u; index++) {
 		const auto sample_coord =
-		    Binary(state, OpFDiv, TypeF32(state),
-		           Binary(state, OpFAdd, TypeF32(state), left,
+		    Binary(state, spv::OpFDiv, TypeF32(state),
+		           Binary(state, spv::OpFAdd, TypeF32(state), left,
 		                  ConstantF32(state, index == 0u ? 0x3f000000u : 0x3fc00000u)),
 		           width_f32);
 		const auto texel = state.builder.AllocateId();
-		state.builder.AddFunction({OpImageSampleExplicitLod, vector_type, texel, sampled,
-		                           sample_coord, ImageOperandsLodMask, ZeroF32(state)});
+		state.builder.AddFunction(spv::OpImageSampleExplicitLod, vector_type, texel, sampled,
+		                          sample_coord, spv::ImageOperandsLodMask, ZeroF32(state));
 		values[index] = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeExtract, scalar_type, values[index], texel, component});
+		state.builder.AddFunction(spv::OpCompositeExtract, scalar_type, values[index], texel,
+		                          component);
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpCompositeConstruct, vector_type, result, values[0], values[1], values[1], values[0]});
+	state.builder.AddFunction(spv::OpCompositeConstruct, vector_type, result, values[0], values[1],
+	                          values[1], values[0]);
 	return result;
 }
 
@@ -474,27 +479,28 @@ uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32
 	auto packed = ConstantU32(ctx.state, 0u);
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		const auto value = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), value, texel,
+		                              component);
 		const auto maximum =
 		    ConstantU32(ctx.state, info.component_bits[component] == 32u
 		                               ? UINT32_MAX
 		                               : (1u << info.component_bits[component]) - 1u);
-		const auto within  = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum);
+		const auto within =
+		    Binary(ctx.state, spv::OpULessThan, TypeBool(ctx.state), value, maximum);
 		const auto clamped = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(
-		    {OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
+		ctx.state.builder.AddFunction(spv::OpSelect, TypeU32(ctx.state), clamped, within, value,
+		                              maximum);
 		const auto shifted =
 		    info.component_bit_offset[component] == 0u
 		        ? clamped
-		        : Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped,
+		        : Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), clamped,
 		                 ConstantU32(ctx.state, info.component_bit_offset[component]));
-		packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
+		packed = Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
 	}
 	const auto zero   = ConstantU32(ctx.state, 0u);
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(
-	    {OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                              packed, zero, zero, zero);
 	return result;
 }
 
@@ -508,83 +514,80 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 		if (source < 4u && ((dmask >> source) & 1u) != 0u) {
 			const auto packed_index = DmaskComponentIndex(dmask, source);
 			raw                     = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
-			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			                              mem.data_bits == 16u ? packed_index / 2u : packed_index);
 			if (mem.data_bits == 16u) {
 				if ((packed_index & 1u) != 0u) {
-					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					raw = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), raw,
 					             ConstantU32(ctx.state, 16u));
 				}
-				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				raw = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), raw,
 				             ConstantU32(ctx.state, 0xffffu));
 			}
 		}
 		values[component] = integer ? raw
 		                    : mem.data_bits == 16u
 		                        ? EmitF16BitsToF32(ctx.state, raw)
-		                        : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
+		                        : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), raw);
 	}
 	const auto texel = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(
-	    {OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
-	     texel, values[0], values[1], values[2], values[3]});
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
+	                              integer ? TypeU32Vector(ctx.state, 4)
+	                                      : TypeF32Vector(ctx.state, 4),
+	                              texel, values[0], values[1], values[2], values[3]);
 	return PackImageTexel(ctx, mem, texel);
 }
 
-uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
+spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) {
 	switch (opcode) {
-		case IR::ValueOpcode::ImageAtomicSwap32: return OpAtomicExchange;
-		case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
-		case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
-		case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
-		case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
-		case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
-		case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
-		default: return 0;
+		case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
+		case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
+		case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
+		default: return spv::OpNop;
 	}
 }
 
 } // namespace
 
-bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
+void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto op         = inst.GetOpcode();
 	const auto image_info = IR::ImageOpcodeInfoOf(op);
-	if (image_info.access == IR::ImageAccess::None) {
-		return false;
-	}
 	auto&       state     = ctx.state;
 	const auto& mem       = ctx.Memory(inst);
 	const auto  image_arg = inst.Arg(0);
 	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
 	const auto& image   = state.program.info.images.at(mem.resource);
 	const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
-	if (address == nullptr) return true;
 	if (op == IR::ValueOpcode::ImageQueryDimensions) {
-		state.builder.RequireCapability(CapabilityImageQuery);
+		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		ctx.Define(inst, QueryDimensions(ctx, mem, *address));
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageQueryLod) {
-		state.builder.RequireCapability(CapabilityImageQuery);
+		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		const auto dimension = image.dimension;
 		const auto sampled   = MakeSampledImage(state, mem.resource, mem.sampler);
 		const auto lod       = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
-		     CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components)});
+		    spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
+		    CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components));
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
 			const auto component = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index});
+			state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), component, lod,
+			                          index);
 			values[index] = FloatBits(ctx, component);
 		}
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0],
-		                           values[1], values[2], values[3]});
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result,
+		                          values[0], values[1], values[2], values[3]);
 		ctx.Define(inst, result);
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageRead) {
 		const auto  dimension      = image.dimension;
@@ -601,19 +604,19 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			        const auto coord      = CoordU32(ctx, mem, *address, dimension);
 			        if (dimension_info.multisampled != 0u) {
 				        state.builder.AddFunction(
-				            {OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
-				             descriptor, coord, ImageOperandsSampleMask,
-				             AddressU32(ctx, mem, *address, dimension_info.coordinate_components)});
+				            spv::OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
+				            descriptor, coord, spv::ImageOperandsSampleMask,
+				            AddressU32(ctx, mem, *address, dimension_info.coordinate_components));
 			        } else {
-				        state.builder.AddFunction({OpImageFetch,
-				                                   ImageVectorType(state, numeric_class, 4), color,
-				                                   descriptor, coord, ImageOperandsLodMask,
-				                                   LodU32(ctx, mem, *address, dimension)});
+				        state.builder.AddFunction(spv::OpImageFetch,
+				                                  ImageVectorType(state, numeric_class, 4), color,
+				                                  descriptor, coord, spv::ImageOperandsLodMask,
+				                                  LodU32(ctx, mem, *address, dimension));
 			        }
 			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class,
 			                            false, mem);
 		        }));
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageWrite) {
 		const bool uint_image = image.numeric_class == Prospero::TextureNumericClass::Uint;
@@ -627,7 +630,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
 			EmitStorageImageWrite(state, mem.resource, mip_lod, coord, texel);
 		});
-		return true;
+		return;
 	}
 	if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
 		const auto  dimension      = image.dimension;
@@ -638,7 +641,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (dref && state.program.info.images[mem.resource].conversion_format !=
 		                Prospero::BufferFormat::kInvalid) {
 			ctx.Fail(inst, "uses depth comparison with a packed integer image");
-			return true;
+			return;
 		}
 		const auto coord =
 		    CoordF32(ctx, mem, *address, layout.coord, dimension_info.coordinate_components);
@@ -648,16 +651,16 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				    HasFlag(mem, Decoder::ImageSampleFlagOffset) ||
 				    HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
 					ctx.Fail(inst, "has an unsupported 1D gather variant");
-					return true;
+					return;
 				}
 				const auto sample = EmitOneDimensionalGatherLz(ctx, mem, coord, numeric_class);
 				ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 				                              numeric_class, false, mem, true));
-				return true;
+				return;
 			}
 			if (dimension == ImageDimension::Dim1DArray) {
 				ctx.Fail(inst, "has an unsupported 1D-array gather");
-				return true;
+				return;
 			}
 			const auto            sampled = MakeSampledImage(state, mem.resource, mem.sampler);
 			const auto            sample  = state.builder.AllocateId();
@@ -667,22 +670,29 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				if (layout.dref != NoImageComponent) {
 					dref_value = AddressF32(ctx, mem, *address, layout.dref);
 				}
-				words = {OpImageDrefGather, TypeF32Vector(state, 4), sample, sampled, coord,
+				words = {spv::OpImageDrefGather,
+				         TypeF32Vector(state, 4),
+				         sample,
+				         sampled,
+				         coord,
 				         dref_value};
 			} else {
 				uint32_t component = 0;
 				if (ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid) {
 					component = ImageGatherComponent(mem.dmask);
 				}
-				words = {OpImageGather, ImageVectorType(state, numeric_class, 4),
-				         sample,        sampled,
-				         coord,         ConstantU32(state, component)};
+				words = {spv::OpImageGather,
+				         ImageVectorType(state, numeric_class, 4),
+				         sample,
+				         sampled,
+				         coord,
+				         ConstantU32(state, component)};
 			}
 			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
-				words.push_back(ImageOperandsConstOffsetsMask);
+				words.push_back(spv::ImageOperandsConstOffsetsMask);
 				words.push_back(HorizontalOffsets(state, dimension));
 			} else if (layout.offset != NoImageComponent) {
-				words.push_back(ImageOperandsOffsetMask);
+				words.push_back(spv::ImageOperandsOffsetMask);
 				words.push_back(PackedOffset(ctx, mem, *address, layout, dimension));
 			}
 			state.builder.AddFunction(words);
@@ -692,17 +702,17 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			}
 			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 			                              result_numeric_class, false, mem, true));
-			return true;
+			return;
 		}
 		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
 		                          HasFlag(mem, Decoder::ImageSampleFlagLod) ||
 		                          HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
-		                          state.stage != ShaderType::Pixel;
-		uint32_t opcode = OpImageSampleImplicitLod;
+		                          state.program.stage != ShaderType::Pixel;
+		auto       opcode       = spv::OpImageSampleImplicitLod;
 		if (explicit_lod) {
-			opcode = dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
+			opcode = dref ? spv::OpImageSampleDrefExplicitLod : spv::OpImageSampleExplicitLod;
 		} else if (dref) {
-			opcode = OpImageSampleDrefImplicitLod;
+			opcode = spv::OpImageSampleDrefImplicitLod;
 		}
 		uint32_t result_type = ImageVectorType(state, numeric_class, 4);
 		uint32_t dref_value  = 0;
@@ -716,34 +726,34 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		uint32_t              operand_mask = 0;
 		std::vector<uint32_t> operands;
 		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
-			operand_mask |= ImageOperandsGradMask;
+			operand_mask |= spv::ImageOperandsGradMask;
 			operands.push_back(
 			    CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components));
 			operands.push_back(
 			    CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components));
 		} else if (explicit_lod) {
-			operand_mask |= ImageOperandsLodMask;
+			operand_mask |= spv::ImageOperandsLodMask;
 			auto lod = ZeroF32(state);
 			if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) {
 				lod = AddressF32(ctx, mem, *address, layout.lod);
 			}
 			operands.push_back(lod);
 		} else if (layout.bias != NoImageComponent) {
-			operand_mask |= ImageOperandsBiasMask;
+			operand_mask |= spv::ImageOperandsBiasMask;
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
 		const auto EmitSample = [&](uint32_t resource) {
 			const auto            sampled = MakeSampledImage(state, resource, mem.sampler);
 			const auto            sample  = state.builder.AllocateId();
-			std::vector<uint32_t> words {opcode, result_type, sample, sampled, coord};
+			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};
 			if (dref) {
-				words.push_back(dref_value);
+				sample_operands.push_back(dref_value);
 			}
 			if (operand_mask != 0u) {
-				words.push_back(operand_mask);
-				words.insert(words.end(), operands.begin(), operands.end());
+				sample_operands.push_back(operand_mask);
+				sample_operands.insert(sample_operands.end(), operands.begin(), operands.end());
 			}
-			state.builder.AddFunction(words);
+			state.builder.AddFunction(opcode, sample_operands);
 			return sample;
 		};
 		if (image.indirect_root != mem.resource) {
@@ -753,7 +763,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				result = UnpackImageTexel(ctx, mem, sample);
 			}
 			ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-			return true;
+			return;
 		}
 		const auto* handle = image_arg.ResolveInstruction();
 		const auto* source = image.source < state.program.descriptor_sources.size()
@@ -762,21 +772,21 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
 		    source->indirect_image->key_arg >= handle->NumArgs()) {
 			ctx.Fail(inst, "has invalid indirect image key provenance");
-			return true;
+			return;
 		}
 		const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg));
 		if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u ||
 		    image.indirect_resources.size() < 2u) {
 			ctx.Fail(inst, "has no indirect image runtime mapping");
-			return true;
+			return;
 		}
 		const auto LoadMapping = [&](uint32_t index) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state),
-			                           pointer, state.flattened_srt_variable, ConstantU32(state, 0),
-			                           index});
+			state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+			                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+			                          index);
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+			state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
 			return value;
 		};
 		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
@@ -785,61 +795,63 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		auto       selected = ConstantU32(state, 0u);
 		for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
 		     iteration++) {
-			const auto active = Binary(state, OpULessThan, TypeBool(state), low, high);
-			const auto mid =
-			    Binary(state, OpShiftRightLogical, TypeU32(state),
-			           Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u));
-			const auto probe = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
-			const auto entry      = Binary(state, OpIAdd, TypeU32(state), mapping,
-			                               Binary(state, OpIAdd, TypeU32(state),
-			                                      Binary(state, OpShiftLeftLogical, TypeU32(state),
-			                                             probe, ConstantU32(state, 1u)),
-			                                      ConstantU32(state, 1u)));
+			const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
+			const auto mid    = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+			                           Binary(state, spv::OpIAdd, TypeU32(state), low, high),
+			                           ConstantU32(state, 1u));
+			const auto probe  = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid,
+			                          ConstantU32(state, 0u));
+			const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
+			                          Binary(state, spv::OpIAdd, TypeU32(state),
+			                                 Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
+			                                        probe, ConstantU32(state, 1u)),
+			                                 ConstantU32(state, 1u)));
 			const auto mapped_key = LoadMapping(entry);
-			const auto candidate =
-			    LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
-			const auto equal         = Binary(state, OpIEqual, TypeBool(state), mapped_key, key);
-			const auto match         = Binary(state, OpLogicalAnd, TypeBool(state), active, equal);
+			const auto candidate  = LoadMapping(
+			    Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
+			const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
+			const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
 			const auto next_selected = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), next_selected, match, candidate, selected});
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match,
+			                          candidate, selected);
 			selected              = next_selected;
-			const auto less       = Binary(state, OpULessThan, TypeBool(state), mapped_key, key);
-			const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less);
-			const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active,
-			                               Unary(state, OpLogicalNot, TypeBool(state), less));
+			const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
+			const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
+			const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
+			                               Unary(state, spv::OpLogicalNot, TypeBool(state), less));
 			const auto next_low   = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, TypeU32(state), next_low, take_upper,
-			     Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
+			    spv::OpSelect, TypeU32(state), next_low, take_upper,
+			    Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
 			low                  = next_low;
 			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high});
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid,
+			                          high);
 			high = next_high;
 		}
 		const auto            default_label = state.builder.AllocateId();
 		const auto            merge_label   = state.builder.AllocateId();
 		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
-		std::vector<uint32_t> switch_words {OpSwitch, selected, default_label};
+		std::vector<uint32_t> switch_words {spv::OpSwitch, selected, default_label};
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
 			labels[candidate - 1u] = state.builder.AllocateId();
 			switch_words.push_back(candidate);
 			switch_words.push_back(labels[candidate - 1u]);
 		}
-		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+		state.builder.AddFunction(spv::OpSelectionMerge, merge_label,
+		                          spv::SelectionControlMaskNone);
 		state.builder.AddFunction(switch_words);
-		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
+		std::vector<uint32_t> phi_words {spv::OpPhi, result_type, state.builder.AllocateId()};
 		EmitLabel(state, default_label);
 		phi_words.push_back(EmitSample(image.indirect_resources[0]));
 		phi_words.push_back(default_label);
-		state.builder.AddFunction({OpBranch, merge_label});
+		state.builder.AddFunction(spv::OpBranch, merge_label);
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
 			EmitLabel(state, labels[candidate - 1u]);
 			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
 			phi_words.push_back(labels[candidate - 1u]);
-			state.builder.AddFunction({OpBranch, merge_label});
+			state.builder.AddFunction(spv::OpBranch, merge_label);
 		}
 		EmitLabel(state, merge_label);
 		state.builder.AddFunction(phi_words);
@@ -848,30 +860,30 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			result = UnpackImageTexel(ctx, mem, result);
 		}
 		ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-		return true;
+		return;
 	}
 	const auto atomic_opcode = ImageAtomicOpcode(op);
-	if (atomic_opcode != 0u) {
+	if (atomic_opcode != spv::OpNop) {
 		const auto dimension = image.dimension;
 		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
-			           const auto pointer = state.builder.AllocateId();
-			           const auto pointer_type =
-			               state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)});
-			           state.builder.AddFunction(
-			               {OpImageTexelPointer, pointer_type, pointer,
-			                StorageImageDescriptorPointer(state, mem.resource),
-			                CoordU32(ctx, mem, *address, dimension), ConstantU32(state, 0)});
+			           const auto pointer      = state.builder.AllocateId();
+			           const auto pointer_type = state.builder.Type(
+			               spv::OpTypePointer, spv::StorageClassImage, TypeU32(state));
+			           state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
+			                                     StorageImageDescriptorPointer(state, mem.resource),
+			                                     CoordU32(ctx, mem, *address, dimension),
+			                                     ConstantU32(state, 0));
 			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
-			                                      ConstantU32(state, ScopeDevice),
-			                                      ConstantU32(state, MemorySemanticsNone),
-			                                      ctx.Arg(inst, 2)});
+			           state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer,
+			                                     ConstantU32(state, spv::ScopeDevice),
+			                                     ConstantU32(state, spv::MemorySemanticsMaskNone),
+			                                     ctx.Arg(inst, 2));
 			           EmitDeviceAtomicMemoryBarrier(state);
 			           return old;
 		           }));
-		return true;
+		return;
 	}
-	return false;
+	ctx.Fail(inst, "has no image SPIR-V emitter");
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -0,0 +1,304 @@
+#pragma once
+
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+// Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
+// (IR::Value), and instructions whose metadata or operand loads must stay lazy.
+#define EMIT_NATIVE(name, opcode, type, ...)                                                       \
+	inline constexpr auto Emit##name = EmitNative<spv::opcode, IR::Type::type, __VA_ARGS__>;
+EMIT_NATIVE(BitCastU16F16, OpBitcast, U32, uint32_t)
+inline constexpr auto EmitBitCastF16U16 = EmitBitCastU16F16;
+inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16;
+inline constexpr auto EmitConvertU32U8  = EmitBitCastU16F16;
+inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16;
+EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t)
+uint32_t              EmitConvertU16U32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertU8U32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertF16F32(EmitterState& state, uint32_t arg0);
+inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32;
+uint32_t              EmitConvertS32F32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertU32F32(EmitterState& state, uint32_t arg0);
+uint32_t              EmitConvertF32S32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
+EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
+EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
+            uint32_t)
+uint32_t              EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
+inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
+inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
+inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
+inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
+inline constexpr auto EmitPackSnorm2x16 =
+    EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
+inline constexpr auto EmitPackUnorm2x16 =
+    EmitGlsl<GLSLstd450PackUnorm2x16, IR::Type::U32, uint32_t>;
+uint32_t              EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1);
+inline constexpr auto EmitFPAbs32 = EmitFAbsValue;
+inline constexpr auto EmitFPNeg32 = EmitFNegateValue;
+uint32_t              EmitFPSaturate32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(BitFieldInsert, OpBitFieldInsert, U32, uint32_t, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(BitFieldUExtract, OpBitFieldUExtract, U32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectU1, OpSelect, U1, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectU32, OpSelect, U32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(SelectF32, OpSelect, F32, uint32_t, uint32_t, uint32_t)
+EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
+EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
+EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
+EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
+uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
+uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
+EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
+EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
+uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
+EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
+EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
+EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
+uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
+EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
+uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
+uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0);
+uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+EMIT_NATIVE(SLessThan32, OpSLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThan32, OpULessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(IEqual32, OpIEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SLessThanEqual32, OpSLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThanEqual32, OpULessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SGreaterThan32, OpSGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
+uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
+EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
+EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t)
+EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
+uint32_t              EmitFPIsNan32(EmitterState& state, uint32_t arg0);
+inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
+EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
+inline constexpr auto EmitFPFma32 =
+    EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
+uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0);
+uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1);
+inline constexpr auto EmitFPRoundEven32 = EmitGlsl<GLSLstd450RoundEven, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFloor32     = EmitGlsl<GLSLstd450Floor, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPCeil32      = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPTrunc32     = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFract32     = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
+uint32_t              EmitFPSin(EmitterState& state, uint32_t arg0);
+uint32_t              EmitFPCos(EmitterState& state, uint32_t arg0);
+#undef EMIT_NATIVE
+
+uint32_t              EmitIdentity(ValueEmitContext& ctx, uint32_t value);
+void                  EmitVoid(ValueEmitContext&);
+inline constexpr auto EmitReference    = EmitVoid;
+inline constexpr auto EmitReferenceU32 = EmitVoid;
+inline constexpr auto EmitControlNop   = EmitVoid;
+inline constexpr auto EmitWaitcnt      = EmitVoid;
+inline constexpr auto EmitSendmsg      = EmitVoid;
+inline constexpr auto EmitTtraceData   = EmitVoid;
+inline constexpr auto EmitInstPrefetch = EmitVoid;
+void                  EmitBarrier(EmitterState& state);
+void                  EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
+uint32_t              EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
+uint32_t              EmitUndefU1(EmitterState& state, const IR::Inst& inst);
+inline constexpr auto EmitUndefU8  = EmitUndefU1;
+inline constexpr auto EmitUndefU16 = EmitUndefU1;
+inline constexpr auto EmitUndefU32 = EmitUndefU1;
+inline constexpr auto EmitUndefU64 = EmitUndefU1;
+uint32_t              EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitWqmU64(EmitterState& state, uint32_t value);
+uint32_t              EmitLaneId(EmitterState& state);
+uint32_t              EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
+uint32_t              EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitGetShaderBase(ValueEmitContext& ctx);
+inline constexpr auto EmitGetSrtResource     = EmitVoid;
+inline constexpr auto EmitGetBufferResource  = EmitGetSrtResource;
+inline constexpr auto EmitGetAddressResource = EmitGetSrtResource;
+inline constexpr auto EmitGetScratchResource = EmitGetSrtResource;
+inline constexpr auto EmitGetImageResource   = EmitGetSrtResource;
+inline constexpr auto EmitGetSamplerResource = EmitGetSrtResource;
+inline constexpr auto EmitMakeImageAddress   = EmitGetSrtResource;
+void                  EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);
+inline constexpr auto EmitLoadAddressU8         = EmitLoadMemory;
+inline constexpr auto EmitLoadAddressU16        = EmitLoadMemory;
+inline constexpr auto EmitLoadAddressU32        = EmitLoadMemory;
+inline constexpr auto EmitStoreAddressU8        = EmitStoreMemory;
+inline constexpr auto EmitStoreAddressU16       = EmitStoreMemory;
+inline constexpr auto EmitStoreAddressU32       = EmitStoreMemory;
+inline constexpr auto EmitLoadBufferU8          = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU16         = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32         = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32x2       = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32x3       = EmitLoadMemory;
+inline constexpr auto EmitLoadBufferU32x4       = EmitLoadMemory;
+inline constexpr auto EmitStoreBufferU8         = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU16        = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32        = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32x2      = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32x3      = EmitStoreMemory;
+inline constexpr auto EmitStoreBufferU32x4      = EmitStoreMemory;
+inline constexpr auto EmitBufferAtomicSwap32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicSwap64    = EmitBufferAtomic64;
+inline constexpr auto EmitBufferAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicISub32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicSMin32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
+inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicFMin32    = EmitBufferFloatAtomic;
+inline constexpr auto EmitBufferAtomicFMax32    = EmitBufferFloatAtomic;
+inline constexpr auto EmitLoadSharedU8          = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU16         = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32         = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32x2       = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32x3       = EmitLoadMemory;
+inline constexpr auto EmitLoadSharedU32x4       = EmitLoadMemory;
+inline constexpr auto EmitWriteSharedU8         = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU16        = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32        = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32x2      = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32x3      = EmitStoreMemory;
+inline constexpr auto EmitWriteSharedU32x4      = EmitStoreMemory;
+inline constexpr auto EmitSharedAtomicFMin32    = EmitSharedFloatAtomic;
+inline constexpr auto EmitSharedAtomicFMax32    = EmitSharedFloatAtomic;
+inline constexpr auto EmitSharedAtomicSwap32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicISub32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicInc32     = EmitSharedIncDec;
+inline constexpr auto EmitSharedAtomicDec32     = EmitSharedIncDec;
+inline constexpr auto EmitSharedAtomicSMin32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicUMin32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicXor32     = EmitAtomic32;
+inline constexpr auto EmitDataAppend            = EmitAppendConsume;
+inline constexpr auto EmitDataConsume           = EmitAppendConsume;
+uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
+inline constexpr auto EmitImageQueryDimensions = EmitImage;
+inline constexpr auto EmitImageQueryLod        = EmitImage;
+inline constexpr auto EmitImageRead            = EmitImage;
+inline constexpr auto EmitImageWrite           = EmitImage;
+inline constexpr auto EmitImageSampleRaw       = EmitImage;
+inline constexpr auto EmitImageGatherRaw       = EmitImage;
+inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
+inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
+inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
+inline constexpr auto EmitImageAtomicOr32      = EmitImage;
+inline constexpr auto EmitImageAtomicXor32     = EmitImage;
+void                  EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);
+inline constexpr auto EmitPhi                        = EmitUnreachable;
+inline constexpr auto EmitTessellationBase           = EmitUnreachable;
+inline constexpr auto EmitGetThreadBitScalarRegister = EmitUnreachable;
+inline constexpr auto EmitSetThreadBitScalarRegister = EmitUnreachable;
+inline constexpr auto EmitGetScalarMaskTag           = EmitUnreachable;
+inline constexpr auto EmitSetScalarMaskTag           = EmitUnreachable;
+inline constexpr auto EmitGetScalarRegister          = EmitUnreachable;
+inline constexpr auto EmitSetScalarRegister          = EmitUnreachable;
+inline constexpr auto EmitGetVectorRegister          = EmitUnreachable;
+inline constexpr auto EmitSetVectorRegister          = EmitUnreachable;
+inline constexpr auto EmitGetGotoVariable            = EmitUnreachable;
+inline constexpr auto EmitSetGotoVariable            = EmitUnreachable;
+inline constexpr auto EmitGetScc                     = EmitUnreachable;
+inline constexpr auto EmitSetScc                     = EmitUnreachable;
+inline constexpr auto EmitGetExec                    = EmitUnreachable;
+inline constexpr auto EmitSetExec                    = EmitUnreachable;
+inline constexpr auto EmitGetExecLo                  = EmitUnreachable;
+inline constexpr auto EmitSetExecLo                  = EmitUnreachable;
+inline constexpr auto EmitGetExecHi                  = EmitUnreachable;
+inline constexpr auto EmitSetExecHi                  = EmitUnreachable;
+inline constexpr auto EmitGetVcc                     = EmitUnreachable;
+inline constexpr auto EmitSetVcc                     = EmitUnreachable;
+inline constexpr auto EmitGetVccLo                   = EmitUnreachable;
+inline constexpr auto EmitSetVccLo                   = EmitUnreachable;
+inline constexpr auto EmitGetVccHi                   = EmitUnreachable;
+inline constexpr auto EmitSetVccHi                   = EmitUnreachable;
+inline constexpr auto EmitGetM0                      = EmitUnreachable;
+inline constexpr auto EmitSetM0                      = EmitUnreachable;
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -17,6 +17,7 @@
 #include <cstring>
 #include <iterator>
 #include <map>
+#include <spirv/unified1/GLSL.std.450.h>
 #include <string>
 #include <unordered_map>
 #include <utility>
@@ -24,307 +25,13 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
-enum : uint32_t {
-	ExecutionModelVertex                     = 0,
-	ExecutionModelFragment                   = 4,
-	ExecutionModelGLCompute                  = 5,
-	ExecutionModeOriginUpperLeft             = 7,
-	ExecutionModeEarlyFragmentTests          = 9,
-	ExecutionModeDepthReplacing              = 12,
-	ExecutionModeLocalSize                   = 17,
-	ExecutionModeSignedZeroInfNanPreserve    = 4461,
-	ExecutionModeDerivativeGroupQuadsKHR     = 5289,
-	AddressingModelLogical                   = 0,
-	AddressingModelPhysicalStorageBuffer64   = 5348,
-	MemoryModelGLSL450                       = 1,
-	CapabilityShader                         = 1,
-	CapabilityInt64                          = 11,
-	CapabilityInt64Atomics                   = 12,
-	CapabilityImageGatherExtended            = 25,
-	CapabilityClipDistance                   = 32,
-	CapabilityCullDistance                   = 33,
-	CapabilitySampleRateShading              = 35,
-	CapabilitySampled1D                      = 43,
-	CapabilityImage1D                        = 44,
-	CapabilityImageQuery                     = 50,
-	CapabilityStorageImageWriteWithoutFormat = 56,
-	CapabilityGroupNonUniform                = 61,
-	CapabilityGroupNonUniformBallot          = 64,
-	CapabilityGroupNonUniformShuffle         = 65,
-	CapabilityShaderLayer                    = 69,
-	CapabilityShaderViewportIndex            = 70,
-	CapabilitySignedZeroInfNanPreserve       = 4466,
-	CapabilityFragmentBarycentricKHR         = 5284,
-	CapabilityComputeDerivativeGroupQuadsKHR = 5288,
-	CapabilityPhysicalStorageBufferAddresses = 5347,
-	StorageClassUniformConstant              = 0,
-	StorageClassInput                        = 1,
-	StorageClassOutput                       = 3,
-	StorageClassWorkgroup                    = 4,
-	StorageClassPrivate                      = 6,
-	StorageClassFunction                     = 7,
-	StorageClassPushConstant                 = 9,
-	StorageClassImage                        = 11,
-	StorageClassStorageBuffer                = 12,
-	StorageClassPhysicalStorageBuffer        = 5349,
-	FunctionControlNone                      = 0,
-	SelectionControlNone                     = 0,
-	LoopControlNone                          = 0,
+struct InputBinding : IR::StageInput {
+	uint32_t variable_id = 0;
 };
 
-enum : uint32_t {
-	DecorationBlock         = 2,
-	DecorationBuiltIn       = 11,
-	DecorationNoPerspective = 13,
-	DecorationFlat          = 14,
-	DecorationAliased       = 20,
-	DecorationLocation      = 30,
-	DecorationArrayStride   = 6,
-	DecorationBinding       = 33,
-	DecorationDescriptorSet = 34,
-	DecorationOffset        = 35,
-	DecorationPerVertexKHR  = 5285,
-};
-
-enum : uint32_t {
-	BuiltInPosition                  = 0,
-	BuiltInPointSize                 = 1,
-	BuiltInClipDistance              = 3,
-	BuiltInCullDistance              = 4,
-	BuiltInLayer                     = 9,
-	BuiltInViewportIndex              = 10,
-	BuiltInFragCoord                 = 15,
-	BuiltInFrontFacing               = 17,
-	BuiltInSampleId                  = 18,
-	BuiltInSampleMask                = 20,
-	BuiltInFragDepth                 = 22,
-	BuiltInWorkgroupId               = 26,
-	BuiltInLocalInvocationId         = 27,
-	BuiltInGlobalInvocationId        = 28,
-	BuiltInLocalInvocationIndex      = 29,
-	BuiltInSubgroupLocalInvocationId = 41,
-	BuiltInVertexIndex               = 42,
-	BuiltInInstanceIndex             = 43,
-	BuiltInBaryCoordKHR              = 5286,
-	BuiltInBaryCoordNoPerspKHR       = 5287,
-};
-
-enum : uint32_t {
-	Dim1D              = 0,
-	Dim3D              = 2,
-	Dim2D              = 1,
-	ImageFormatUnknown = 0,
-	ImageFormatRgba32f = 1,
-	ImageFormatR32ui   = 33,
-};
-
-enum : uint32_t {
-	ImageOperandsBiasMask         = 0x00000001u,
-	ImageOperandsLodMask          = 0x00000002u,
-	ImageOperandsGradMask         = 0x00000004u,
-	ImageOperandsOffsetMask       = 0x00000010u,
-	ImageOperandsConstOffsetsMask = 0x00000020u,
-	ImageOperandsSampleMask       = 0x00000040u,
-};
-
-enum : uint32_t {
-	ScopeDevice                    = 1,
-	ScopeWorkgroup                 = 2,
-	ScopeSubgroup                  = 3,
-	MemorySemanticsNone            = 0,
-	MemorySemanticsAcquireRelease  = 0x00000008u,
-	MemorySemanticsUniformMemory   = 0x00000040u,
-	MemorySemanticsWorkgroupMemory = 0x00000100u,
-	MemorySemanticsImageMemory     = 0x00000800u,
-};
-
-enum : uint32_t {
-	OpExtInst                      = 12,
-	OpTypeVoid                     = 19,
-	OpTypeBool                     = 20,
-	OpTypeInt                      = 21,
-	OpTypeFloat                    = 22,
-	OpTypeVector                   = 23,
-	OpTypeImage                    = 25,
-	OpTypeSampler                  = 26,
-	OpTypeSampledImage             = 27,
-	OpTypeArray                    = 28,
-	OpTypeRuntimeArray             = 29,
-	OpTypeStruct                   = 30,
-	OpTypePointer                  = 32,
-	OpTypeFunction                 = 33,
-	OpConstantTrue                 = 41,
-	OpConstantFalse                = 42,
-	OpConstant                     = 43,
-	OpConstantComposite            = 44,
-	OpUndef                        = 1,
-	OpFunction                     = 54,
-	OpFunctionParameter            = 55,
-	OpFunctionEnd                  = 56,
-	OpFunctionCall                 = 57,
-	OpVariable                     = 59,
-	OpImageTexelPointer            = 60,
-	OpLoad                         = 61,
-	OpStore                        = 62,
-	OpAccessChain                  = 65,
-	OpArrayLength                  = 68,
-	OpDecorate                     = 71,
-	OpMemberDecorate               = 72,
-	OpVectorShuffle                = 79,
-	OpCompositeConstruct           = 80,
-	OpCompositeExtract             = 81,
-	OpCopyObject                   = 83,
-	OpSampledImage                 = 86,
-	OpImageSampleImplicitLod       = 87,
-	OpImageSampleExplicitLod       = 88,
-	OpImageSampleDrefImplicitLod   = 89,
-	OpImageSampleDrefExplicitLod   = 90,
-	OpImageFetch                   = 95,
-	OpImageGather                  = 96,
-	OpImageDrefGather              = 97,
-	OpImageWrite                   = 99,
-	OpImageQuerySizeLod            = 103,
-	OpImageQueryLod                = 105,
-	OpImageQuerySize               = 104,
-	OpImageQueryLevels             = 106,
-	OpConvertFToU                  = 109,
-	OpConvertFToS                  = 110,
-	OpConvertSToF                  = 111,
-	OpConvertUToF                  = 112,
-	OpUConvert                     = 113,
-	OpConvertUToPtr                = 120,
-	OpBitcast                      = 124,
-	OpSNegate                      = 126,
-	OpFNegate                      = 127,
-	OpIAdd                         = 128,
-	OpFAdd                         = 129,
-	OpISub                         = 130,
-	OpFSub                         = 131,
-	OpIMul                         = 132,
-	OpFMul                         = 133,
-	OpUDiv                         = 134,
-	OpUMod                         = 137,
-	OpFDiv                         = 136,
-	OpIAddCarry                    = 149,
-	OpUMulExtended                 = 151,
-	OpSMulExtended                 = 152,
-	OpAny                          = 154,
-	OpAll                          = 155,
-	OpLogicalNotEqual              = 165,
-	OpLogicalOr                    = 166,
-	OpLogicalAnd                   = 167,
-	OpLogicalNot                   = 168,
-	OpSelect                       = 169,
-	OpIEqual                       = 170,
-	OpINotEqual                    = 171,
-	OpUGreaterThan                 = 172,
-	OpSGreaterThan                 = 173,
-	OpUGreaterThanEqual            = 174,
-	OpSGreaterThanEqual            = 175,
-	OpULessThan                    = 176,
-	OpSLessThan                    = 177,
-	OpULessThanEqual               = 178,
-	OpSLessThanEqual               = 179,
-	OpFOrdEqual                    = 180,
-	OpFUnordEqual                  = 181,
-	OpFOrdNotEqual                 = 182,
-	OpFUnordNotEqual               = 183,
-	OpFOrdLessThan                 = 184,
-	OpFUnordLessThan               = 185,
-	OpFOrdGreaterThan              = 186,
-	OpFUnordGreaterThan            = 187,
-	OpFOrdLessThanEqual            = 188,
-	OpFUnordLessThanEqual          = 189,
-	OpFOrdGreaterThanEqual         = 190,
-	OpFUnordGreaterThanEqual       = 191,
-	OpShiftRightLogical            = 194,
-	OpShiftRightArithmetic         = 195,
-	OpShiftLeftLogical             = 196,
-	OpBitwiseOr                    = 197,
-	OpBitwiseXor                   = 198,
-	OpBitwiseAnd                   = 199,
-	OpNot                          = 200,
-	OpBitFieldInsert               = 201,
-	OpBitFieldSExtract             = 202,
-	OpBitFieldUExtract             = 203,
-	OpBitReverse                   = 204,
-	OpBitCount                     = 205,
-	OpControlBarrier               = 224,
-	OpMemoryBarrier                = 225,
-	OpAtomicLoad                   = 227,
-	OpAtomicExchange               = 229,
-	OpAtomicCompareExchange        = 230,
-	OpAtomicIAdd                   = 234,
-	OpAtomicISub                   = 235,
-	OpAtomicSMin                   = 236,
-	OpAtomicUMin                   = 237,
-	OpAtomicSMax                   = 238,
-	OpAtomicUMax                   = 239,
-	OpAtomicAnd                    = 240,
-	OpAtomicOr                     = 241,
-	OpAtomicXor                    = 242,
-	OpPhi                          = 245,
-	OpLoopMerge                    = 246,
-	OpSelectionMerge               = 247,
-	OpLabel                        = 248,
-	OpBranch                       = 249,
-	OpBranchConditional            = 250,
-	OpSwitch                       = 251,
-	OpKill                         = 252,
-	OpReturn                       = 253,
-	OpReturnValue                  = 254,
-	OpGroupNonUniformBallot        = 339,
-	OpGroupNonUniformBallotFindLSB = 343,
-	OpGroupNonUniformShuffle       = 345,
-};
-
-enum : uint32_t {
-	MemoryAccessAlignedMask = 0x2,
-};
-
-enum : uint32_t {
-	GlslRoundEven       = 2,
-	GlslTrunc           = 3,
-	GlslFAbs            = 4,
-	GlslFloor           = 8,
-	GlslCeil            = 9,
-	GlslFract           = 10,
-	GlslSin             = 13,
-	GlslCos             = 14,
-	GlslExp2            = 29,
-	GlslLog2            = 30,
-	GlslSqrt            = 31,
-	GlslInverseSqrt     = 32,
-	GlslFMin            = 37,
-	GlslFMax            = 40,
-	GlslFClamp          = 43,
-	GlslLdexp           = 53,
-	GlslFma             = 50,
-	GlslPackSnorm2x16   = 56,
-	GlslPackUnorm2x16   = 57,
-	GlslPackHalf2x16    = 58,
-	GlslUnpackUnorm2x16 = 61,
-	GlslUnpackHalf2x16  = 62,
-	GlslFindILsb        = 73,
-	GlslFindUMsb        = 75,
-};
-
-struct InputBinding {
-	IR::StageInputKind kind            = IR::StageInputKind::VertexIndex;
-	uint32_t           location        = 0;
-	uint32_t           component_count = 1;
-	uint32_t           variable_id     = 0;
-	std::string        debug_name;
-	bool               per_vertex = false;
-};
-
-struct OutputBinding {
-	IR::StageOutputKind kind        = IR::StageOutputKind::Parameter;
-	uint32_t            index       = 0;
-	uint32_t            location    = 0;
-	uint32_t            variable_id = 0;
-	std::string         debug_name;
-	uint32_t            mesh_data_variable = 0;
+struct OutputBinding : IR::StageOutput {
+	uint32_t variable_id        = 0;
+	uint32_t mesh_data_variable = 0;
 };
 
 using ImageDimension = Decoder::ImageDimension;
@@ -339,27 +46,45 @@ struct ImageDimensionInfo {
 };
 
 constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
-    {ImageDimension::Dim1D, Dim1D, 1, 1, 0, 0},
-    {ImageDimension::Dim1DArray, Dim1D, 2, 1, 1, 0},
-    {ImageDimension::Dim2D, Dim2D, 2, 2, 0, 0},
-    {ImageDimension::Dim2DArray, Dim2D, 3, 2, 1, 0},
-    {ImageDimension::Dim3D, Dim3D, 3, 3, 0, 0},
-    {ImageDimension::Dim2DMsaa, Dim2D, 2, 2, 0, 1},
-    {ImageDimension::Dim2DMsaaArray, Dim2D, 3, 2, 1, 1},
+    {ImageDimension::Dim1D, spv::Dim1D, 1, 1, 0, 0},
+    {ImageDimension::Dim1DArray, spv::Dim1D, 2, 1, 1, 0},
+    {ImageDimension::Dim2D, spv::Dim2D, 2, 2, 0, 0},
+    {ImageDimension::Dim2DArray, spv::Dim2D, 3, 2, 1, 0},
+    {ImageDimension::Dim3D, spv::Dim3D, 3, 3, 0, 0},
+    {ImageDimension::Dim2DMsaa, spv::Dim2D, 2, 2, 0, 1},
+    {ImageDimension::Dim2DMsaaArray, spv::Dim2D, 3, 2, 1, 1},
 }};
 
 const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 
+struct SpirvRequirements {
+	bool subgroup_ballot              = false;
+	bool subgroup_shuffle             = false;
+	bool subgroup_local_invocation_id = false;
+	bool compute_derivatives          = false;
+	bool image_gather_extended        = false;
+	bool function_lds                 = false;
+	bool function_scratch             = false;
+	bool pixel_valid_mask             = false;
+	bool buffer_int64_atomics         = false;
+};
+
+SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
+
 struct EmitterState {
 	EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
 	    : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
-	      program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {}
+	      program(program_), input_info(input_info_),
+	      requirements(AnalyzeProgramRequirements(program_)) {}
 
 	Builder                                          builder;
 	const IR::Program&                               program;
 	ShaderStageInputInfo                             input_info;
-	const IR::SpirvRequirements&                     requirements;
-	ShaderType                                       stage                   = ShaderType::Unknown;
+	std::array<uint32_t, 6>                          tess_variables {};
+	uint32_t                                         tess_inner_variable = 0;
+	uint32_t                                         tess_patch_base     = 0;
+
+	const SpirvRequirements                          requirements;
 	uint32_t                                         lane_count              = 1;
 	uint32_t                                         lane_half               = 0;
 	uint32_t                                         storage_buffer_variable = 0;
@@ -391,11 +116,10 @@ struct EmitterState {
 	uint32_t                   per_vertex_variable                   = 0;
 	uint32_t                   point_size_variable                   = 0;
 	uint32_t                   clip_distance_variable                = 0;
+	uint32_t                   invalid_position_clip_distance        = UINT32_MAX;
 	uint32_t                   cull_distance_variable                = 0;
 	uint32_t                   layer_variable                        = 0;
 	uint32_t                   viewport_index_variable               = 0;
-	uint32_t                   clip_distance_count                   = 0;
-	uint32_t                   cull_distance_count                   = 0;
 	uint32_t                   depth_variable                        = 0;
 	uint32_t                   sample_mask_variable                  = 0;
 	std::vector<InputBinding>  inputs;
@@ -409,7 +133,6 @@ uint32_t TypeBool(EmitterState& state);
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
 uint32_t TypeU64(EmitterState& state);
-uint32_t TypeDeviceAddress(EmitterState& state);
 uint32_t TypeScalarU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
@@ -420,40 +143,56 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
 uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
-uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee);
+uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
 uint32_t TypeFunction(EmitterState& state);
 uint32_t TypeStorageBufferPointer(EmitterState& state);
 uint32_t TypeStorageBufferElementPointer(EmitterState& state);
 uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
 uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
-uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
 uint32_t TypePhysicalU32Pointer(EmitterState& state);
 uint32_t TypePushConstantElementPointer(EmitterState& state);
-uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
-uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
+uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
+uint32_t TypeU32ElementPointer(EmitterState& state, spv::StorageClass storage_class);
 
 inline void EmitLabel(EmitterState& state, uint32_t label) {
 	state.current_label = label;
-	state.builder.AddFunction({OpLabel, label});
+	state.builder.AddFunction(spv::OpLabel, label);
 }
 
-inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
+uint32_t TypeId(EmitterState& state, IR::Type type);
+
+// Shared instruction construction; typed aliases add no forwarding functions.
+template <spv::Op opcode, IR::Type type, typename... Args>
+uint32_t EmitNative(EmitterState& state, Args... args) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, type, result, value});
+	state.builder.AddFunction(opcode, TypeId(state, type), result, args...);
 	return result;
 }
 
-inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
-                            uint32_t rhs) {
+uint32_t GlslStd450(EmitterState& state);
+
+template <GLSLstd450 opcode, IR::Type type, typename... Args>
+uint32_t EmitGlsl(EmitterState& state, Args... args) {
+	return EmitNative<spv::OpExtInst, type>(state, GlslStd450(state), opcode, args...);
+}
+
+inline uint32_t Unary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t value) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, type, result, lhs, rhs});
+	state.builder.AddFunction(opcode, type, result, value);
 	return result;
 }
 
-inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition,
-                            uint32_t true_value, uint32_t false_value) {
+inline uint32_t Binary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t lhs,
+                       uint32_t rhs) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
+	state.builder.AddFunction(opcode, type, result, lhs, rhs);
+	return result;
+}
+
+inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
+                       uint32_t false_value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpSelect, type, result, condition, true_value, false_value);
 	return result;
 }
 
@@ -467,9 +206,6 @@ struct ValueEmitContext {
 	uint32_t              FirstLane(uint32_t ballot);
 	uint32_t              Shuffle(const IR::Inst& inst, size_t index, uint32_t lane);
 	uint32_t              Result(const IR::Inst& inst);
-	uint32_t              TypeId(IR::Type type) const;
-	uint32_t              Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
-	                           std::initializer_list<uint32_t> args);
 	uint32_t              Define(const IR::Inst& inst, uint32_t value);
 	uint32_t              ResourceIndex(IR::Value value, IR::ValueOpcode opcode);
 	const IR::Inst*       ImageAddress(IR::Value value);
@@ -525,12 +261,6 @@ uint32_t VertexParameterComponentCount(const InputBinding& input);
 
 uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind);
 
-uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
-                                         uint32_t components);
-
-bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
-
-void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
 
@@ -570,7 +300,7 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource);
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
                            uint32_t texel);
 
-uint32_t ExecutionModelForStage(ShaderType stage);
+spv::ExecutionModel ExecutionModelForStage(ShaderType stage);
 
 uint32_t ConstantU32(EmitterState& state, uint32_t value);
 
@@ -588,24 +318,11 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value);
 
 uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
 
-uint32_t GlslStd450(EmitterState& state);
-
-void AllocateInputVariables(EmitterState& state);
-
-void AllocateOutputVariables(EmitterState& state);
-
-uint32_t BuiltInForInput(IR::StageInputKind kind);
-
-void AddInputAnnotationsAndNames(EmitterState& state);
-
-void AddOutputAnnotationsAndNames(EmitterState& state);
-
-void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name,
-                        IR::DescriptorBindingKind kind);
-
-void AddDescriptorAnnotationsAndNames(EmitterState& state);
-
-void DefineModule(EmitterState& state);
+uint32_t DefineInterfaceVariable(EmitterState& state, uint32_t type, spv::StorageClass storage,
+                                 const char* name);
+void     DefineModule(EmitterState& state);
+void     DefineTessellationInterfaces(EmitterState& state);
+void     DefineTessellationExecutionModes(EmitterState& state);
 void     DefineMeshOutputs(EmitterState& state);
 void     EmitMeshEntryPoint(EmitterState& state);
 void     EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -640,9 +357,9 @@ uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane);
 
-uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitAddU32 = EmitNative<spv::OpIAdd, IR::Type::U32, uint32_t, uint32_t>;
 
-uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
+uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32_t rhs);
 
 uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
 
@@ -665,10 +382,9 @@ struct MemoryResourceAccess {
 
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
-MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
-                                                         const IR::MemoryInfo& mem,
-                                                         uint32_t variable,
-                                                         uint32_t pointer_type);
+MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState&         state,
+                                                        const IR::MemoryInfo& mem,
+                                                        uint32_t variable, uint32_t pointer_type);
 
 uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
                                 uint32_t raw_index);
@@ -679,14 +395,13 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
 uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
                                   uint32_t index);
 
-uint32_t EmitStorageBufferElementPointer(EmitterState& state,
-                                         const MemoryResourceAccess& access, uint32_t index,
-                                         uint32_t pointer_type);
+uint32_t EmitStorageBufferElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                         uint32_t index, uint32_t pointer_type);
 
 uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value);
 
-uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
-                              uint32_t false_value);
+inline constexpr auto EmitTBufferSelectF32 =
+    EmitNative<spv::OpSelect, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
 
 bool IsSignedFormatComponent(Format::ComponentType type);
 
@@ -702,22 +417,16 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
 
-uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
-                            uint32_t false_value);
-
-void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
-                                   uint32_t& out_low, uint32_t& out_high);
-
-void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
-                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high);
+inline constexpr auto EmitSelectValueU32 =
+    EmitNative<spv::OpSelect, IR::Type::U32, uint32_t, uint32_t, uint32_t>;
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
 
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
 
-uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitOrU32 = EmitNative<spv::OpBitwiseOr, IR::Type::U32, uint32_t, uint32_t>;
 
-uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+uint32_t EmitCompareU32Constant(EmitterState& state, spv::Op opcode, uint32_t value,
                                 uint32_t constant);
 
 uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value);
@@ -728,17 +437,19 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 
 uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
-uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value);
+inline constexpr auto EmitBitcastF32ToU32 = EmitNative<spv::OpBitcast, IR::Type::U32, uint32_t>;
 
-uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value);
+inline constexpr auto EmitBitcastU32ToF32 = EmitNative<spv::OpBitcast, IR::Type::F32, uint32_t>;
 
-uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitAndU32 = EmitNative<spv::OpBitwiseAnd, IR::Type::U32, uint32_t, uint32_t>;
 
-uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitLogicalAndBool =
+    EmitNative<spv::OpLogicalAnd, IR::Type::U1, uint32_t, uint32_t>;
 
-uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
+inline constexpr auto EmitLogicalOrBool =
+    EmitNative<spv::OpLogicalOr, IR::Type::U1, uint32_t, uint32_t>;
 
-uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value);
+inline constexpr auto EmitLogicalNotBool = EmitNative<spv::OpLogicalNot, IR::Type::U1, uint32_t>;
 
 F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits);
 
@@ -751,26 +462,18 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
 
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
-uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value);
+inline constexpr auto EmitTruncF32Value = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
 
 uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
 
 uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
 
-uint32_t EmitFNegateValue(EmitterState& state, uint32_t value);
+inline constexpr auto EmitFNegateValue = EmitNative<spv::OpFNegate, IR::Type::F32, uint32_t>;
 
-uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
+inline constexpr auto EmitFAbsValue = EmitGlsl<GLSLstd450FAbs, IR::Type::F32, uint32_t>;
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 
-bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
-
-bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst);
-
-bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
-
-bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
-
 void EmitProgram(EmitterState& state);
 
 void DefineGetBdaPointer(EmitterState& state);
@@ -780,11 +483,11 @@ template <typename Fn>
 void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
 	const auto then_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label});
+	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
+	state.builder.AddFunction(spv::OpBranchConditional, condition, then_label, merge_label);
 	EmitLabel(state, then_label);
 	fn();
-	state.builder.AddFunction({OpBranch, merge_label});
+	state.builder.AddFunction(spv::OpBranch, merge_label);
 	EmitLabel(state, merge_label);
 }
 
@@ -795,19 +498,19 @@ uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition,
 	const auto then_exit   = state.builder.AllocateId();
 	const auto else_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
+	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
+	state.builder.AddFunction(spv::OpBranchConditional, condition, then_label, else_label);
 	EmitLabel(state, then_label);
 	const auto then_value = fn();
-	state.builder.AddFunction({OpBranch, then_exit});
+	state.builder.AddFunction(spv::OpBranch, then_exit);
 	EmitLabel(state, then_exit);
-	state.builder.AddFunction({OpBranch, merge_label});
+	state.builder.AddFunction(spv::OpBranch, merge_label);
 	EmitLabel(state, else_label);
-	state.builder.AddFunction({OpBranch, merge_label});
+	state.builder.AddFunction(spv::OpBranch, merge_label);
 	EmitLabel(state, merge_label);
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpPhi, type, value, then_value, then_exit, default_value, else_label});
+	state.builder.AddFunction(spv::OpPhi, type, value, then_value, then_exit, default_value,
+	                          else_label);
 	return value;
 }
 
@@ -819,12 +522,12 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
 
 template <typename Fn>
 uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
-	const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
+	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto memory = [&] {
 		switch (kind) {
-			case IR::ResourceKind::Lds: return MemorySemanticsWorkgroupMemory;
-			case IR::ResourceKind::Image: return MemorySemanticsImageMemory;
-			default: return MemorySemanticsUniformMemory;
+			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
+			default: return spv::MemorySemanticsUniformMemoryMask;
 		}
 	}();
 	const auto preheader = state.builder.AllocateId();
@@ -834,27 +537,29 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
 	const auto initial   = state.builder.AllocateId();
 	const auto observed  = state.builder.AllocateId();
 	const auto exchanged = state.builder.AllocateId();
-	state.builder.AddFunction({OpBranch, preheader});
+	state.builder.AddFunction(spv::OpBranch, preheader);
 	EmitLabel(state, preheader);
-	state.builder.AddFunction({OpAtomicLoad, TypeU32(state), initial, pointer,
-	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
-	state.builder.AddFunction({OpBranch, header});
+	state.builder.AddFunction(spv::OpAtomicLoad, TypeU32(state), initial, pointer,
+	                          ConstantU32(state, scope),
+	                          ConstantU32(state, spv::MemorySemanticsMaskNone));
+	state.builder.AddFunction(spv::OpBranch, header);
 	EmitLabel(state, header);
-	state.builder.AddFunction(
-	    {OpPhi, TypeU32(state), observed, initial, preheader, exchanged, cont});
+	state.builder.AddFunction(spv::OpPhi, TypeU32(state), observed, initial, preheader, exchanged,
+	                          cont);
 	const auto next = desired(observed);
-	state.builder.AddFunction({OpAtomicCompareExchange, TypeU32(state), exchanged, pointer,
-	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
-	                           ConstantU32(state, MemorySemanticsNone), next, observed});
+	state.builder.AddFunction(spv::OpAtomicCompareExchange, TypeU32(state), exchanged, pointer,
+	                          ConstantU32(state, scope),
+	                          ConstantU32(state, spv::MemorySemanticsMaskNone),
+	                          ConstantU32(state, spv::MemorySemanticsMaskNone), next, observed);
 	const auto success = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, TypeBool(state), success, exchanged, observed});
-	state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone});
-	state.builder.AddFunction({OpBranchConditional, success, merge, cont});
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), success, exchanged, observed);
+	state.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlMaskNone);
+	state.builder.AddFunction(spv::OpBranchConditional, success, merge, cont);
 	EmitLabel(state, cont);
-	state.builder.AddFunction({OpBranch, header});
+	state.builder.AddFunction(spv::OpBranch, header);
 	EmitLabel(state, merge);
-	state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope),
-	                           ConstantU32(state, MemorySemanticsAcquireRelease | memory)});
+	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
+	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
 	return observed;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -7,33 +7,27 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
 		dword_index += state.program.bindings.push_data_start_dword;
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer,
-		                           state.push_constant_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		state.builder.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state),
+		                          pointer, state.push_constant_variable, ConstantU32(state, 0),
+		                          ConstantU32(state, dword_index));
+		state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
 		return value;
 	}
 	if (state.shader_data_storage_variable != 0) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
-		                           state.shader_data_storage_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+		                          pointer, state.shader_data_storage_variable,
+		                          ConstantU32(state, 0), ConstantU32(state, dword_index));
+		state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
 		return value;
 	}
 	return ConstantU32(state, 0);
 }
 
-uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAdd, TypeU32(state), ret, lhs, rhs});
-	return ret;
-}
-
-uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
+	state.builder.AddFunction(opcode, TypeU32(state), ret, lhs, rhs);
 	return ret;
 }
 
@@ -59,13 +53,13 @@ void EmitMemoryOffsets(EmitterState& state) {
 		    EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
 		const auto shift             = ConstantU32(state, (i % 4u) * 8u);
 		state.memory_byte_offsets[i] = EmitBinaryU32(
-		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    state, spv::OpBitwiseAnd, EmitBinaryU32(state, spv::OpShiftRightLogical, word, shift),
 		    ConstantU32(state, 0xffu));
 	}
 }
 
 uint32_t LdsDwordCount(const EmitterState& state) {
-	const auto* workgroup = ShaderWorkgroupInput(state.stage, state.input_info);
+	const auto* workgroup = ShaderWorkgroupInput(state.program.stage, state.input_info);
 	return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u;
 }
 
@@ -73,12 +67,12 @@ static void EnsureLdsStorage(EmitterState& state) {
 	if (state.lds_variable != 0) {
 		return;
 	}
-	if (ShaderWorkgroupInput(state.stage, state.input_info) == nullptr) {
+	if (ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
 		EXIT("function LDS was not prepared before SPIR-V function emission\n");
 	}
 	state.lds_variable = state.builder.DefineGlobalVariable(
-	    TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)),
-	    StorageClassWorkgroup);
+	    TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsDwordCount(state)),
+	    spv::StorageClassWorkgroup);
 	state.builder.AddName(state.lds_variable, "lds_dwords");
 }
 
@@ -94,12 +88,12 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
 	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
 	MemoryResourceAccess access {.kind = mem.kind};
 	access.object_pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, pointer_type, access.object_pointer, variable,
-	                           ConstantU32(state, array_index)});
+	state.builder.AddFunction(spv::OpAccessChain, pointer_type, access.object_pointer, variable,
+	                          ConstantU32(state, array_index));
 	access.byte_offset = state.memory_byte_offsets[array_index];
 	access.length      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
+	                          access.object_pointer, 0);
 	return access;
 }
 
@@ -137,17 +131,16 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 		case IR::ResourceKind::Buffer: {
 			access = PrepareStorageBufferResourceAccess(
 			    state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
-			access.index_offset =
-			    EmitBinaryU32(state, OpShiftRightLogical, access.byte_offset,
-			                  ConstantU32(state, 2u));
+			access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
+			                                    ConstantU32(state, 2u));
 			access.add_index_offset = true;
 			return access;
 		}
 		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
 	}
 	access.length = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
+	                          access.object_pointer, 0);
 	return access;
 }
 
@@ -159,7 +152,7 @@ uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess&
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
                                    uint32_t index) {
 	const auto in_bounds = state.builder.AllocateId();
-	state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length});
+	state.builder.AddFunction(spv::OpULessThan, TypeBool(state), in_bounds, index, access.length);
 	return in_bounds;
 }
 
@@ -167,12 +160,13 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
                                   uint32_t index) {
 	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
 		const auto pointer = state.builder.AllocateId();
-		const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
-		                           : ShaderWorkgroupInput(state.stage, state.input_info) != nullptr
-		                               ? StorageClassWorkgroup
-		                               : StorageClassFunction;
-		state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
-		                           pointer, access.object_pointer, index});
+		const auto storage_class =
+		    access.kind == IR::ResourceKind::Scratch ? spv::StorageClassFunction
+		    : ShaderWorkgroupInput(state.program.stage, state.input_info) != nullptr
+		        ? spv::StorageClassWorkgroup
+		        : spv::StorageClassFunction;
+		state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class),
+		                          pointer, access.object_pointer, index);
 		return pointer;
 	}
 	return EmitStorageBufferElementPointer(state, access, index,
@@ -183,21 +177,14 @@ uint32_t EmitStorageBufferElementPointer(EmitterState& state,
                                          const MemoryResourceAccess& access, uint32_t index,
                                          uint32_t pointer_type) {
 	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, pointer_type, pointer, access.object_pointer,
-	                           ConstantU32(state, 0), index});
+	state.builder.AddFunction(spv::OpAccessChain, pointer_type, pointer, access.object_pointer,
+	                          ConstantU32(state, 0), index);
 	return pointer;
 }
 
 uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, TypeI32(state), ret, value});
-	return ret;
-}
-
-uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
-                              uint32_t false_value) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, TypeF32(state), ret, condition, true_value, false_value});
+	state.builder.AddFunction(spv::OpBitcast, TypeI32(state), ret, value);
 	return ret;
 }
 
@@ -210,33 +197,34 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
 	const auto mantissa_bits = bits == 11u ? 6u : 5u;
 	const auto mantissa_mask = (1u << mantissa_bits) - 1u;
 	const auto mantissa =
-	    EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
+	    EmitBinaryU32(state, spv::OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
 	const auto exponent = EmitBinaryU32(
-	    state, OpBitwiseAnd,
-	    EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
+	    state, spv::OpBitwiseAnd,
+	    EmitBinaryU32(state, spv::OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
 	    ConstantU32(state, 0x1fu));
 
 	const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
 	const auto exponent_bits =
-	    EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
-	const auto mantissa_bits_32 =
-	    EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits));
-	const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32);
+	    EmitBinaryU32(state, spv::OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
+	const auto mantissa_bits_32 = EmitBinaryU32(state, spv::OpShiftLeftLogical, mantissa,
+	                                            ConstantU32(state, 23u - mantissa_bits));
+	const auto normal_bits =
+	    EmitBinaryU32(state, spv::OpBitwiseOr, exponent_bits, mantissa_bits_32);
 	const auto normal      = EmitBitcastU32ToF32(state, normal_bits);
 
 	const auto special_bits =
-	    EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
+	    EmitBinaryU32(state, spv::OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
 	const auto special = EmitBitcastU32ToF32(state, special_bits);
 
 	const auto mantissa_f32 = state.builder.AllocateId();
 	const auto subnormal    = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
+	state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), mantissa_f32, mantissa);
 	state.builder.AddFunction(
-	    {OpFMul, TypeF32(state), subnormal, mantissa_f32,
-	     ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
+	    spv::OpFMul, TypeF32(state), subnormal, mantissa_f32,
+	    ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits))));
 
-	const auto zero_exp    = EmitCompareU32Constant(state, OpIEqual, exponent, 0);
-	const auto special_exp = EmitCompareU32Constant(state, OpIEqual, exponent, 31);
+	const auto zero_exp    = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0);
+	const auto special_exp = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 31);
 	const auto finite      = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
 	const auto result      = EmitTBufferSelectF32(state, special_exp, special, finite);
 	return EmitBitcastF32ToU32(state, result);
@@ -250,22 +238,22 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 		case Format::ComponentType::Sint: return raw;
 		case Format::ComponentType::Uscaled: {
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
+			state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), value, raw);
 			return EmitBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Sscaled: {
 			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
 			const auto value      = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
+			state.builder.AddFunction(spv::OpConvertSToF, TypeF32(state), value, signed_raw);
 			return EmitBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Unorm: {
 			const auto value      = state.builder.AllocateId();
 			const auto normalized = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << bits) - 1u);
-			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
-			state.builder.AddFunction(
-			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
+			state.builder.AddFunction(spv::OpConvertUToF, TypeF32(state), value, raw);
+			state.builder.AddFunction(spv::OpFDiv, TypeF32(state), normalized, value,
+			                          ConstantF32Value(state, max_value));
 			return EmitBitcastF32ToU32(state, normalized);
 		}
 		case Format::ComponentType::Snorm: {
@@ -274,11 +262,11 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			const auto normalized = state.builder.AllocateId();
 			const auto clamped    = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << (bits - 1u)) - 1u);
-			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
-			state.builder.AddFunction(
-			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
-			state.builder.AddFunction({OpExtInst, TypeF32(state), clamped, GlslStd450(state),
-			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			state.builder.AddFunction(spv::OpConvertSToF, TypeF32(state), value, signed_raw);
+			state.builder.AddFunction(spv::OpFDiv, TypeF32(state), normalized, value,
+			                          ConstantF32Value(state, max_value));
+			state.builder.AddFunction(spv::OpExtInst, TypeF32(state), clamped, GlslStd450(state),
+			                          GLSLstd450FMax, normalized, ConstantF32Value(state, -1.0f));
 			return EmitBitcastF32ToU32(state, clamped);
 		}
 		case Format::ComponentType::Float:
@@ -294,9 +282,10 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 }
 
 void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
-	const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory;
-	state.builder.AddFunction(
-	    {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
+	const auto semantics =
+	    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask;
+	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice),
+	                          ConstantU32(state, semantics));
 }
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
@@ -308,12 +297,12 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	};
 	const auto classify = [&](uint32_t bits) {
 		const auto cls = EmitClassifyF32Bits(state, bits);
-		const auto negative = EmitCompareU32Constant(
-		    state, OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u);
+		const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
+		                                             EmitAndConstant(state, bits, 0x80000000u), 0u);
 		const auto negative_key = state.builder.AllocateId();
-		state.builder.AddFunction({OpNot, TypeU32(state), negative_key, bits});
-		const auto positive_key = EmitBinaryU32(
-		    state, OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
+		state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
+		const auto positive_key =
+		    EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
 		return OrderedBits {
 		    cls.nan, cls.zero,
 		    EmitSelectValueU32(state, negative, negative_key, positive_key)};
@@ -324,8 +313,8 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	    state, EmitLogicalOrBool(state, source_class.nan, old_class.nan),
 	    EmitLogicalAndBool(state, source_class.zero, old_class.zero));
 	const auto compare = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), compare,
-	                           source_class.key, old_class.key});
+	state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state),
+	                          compare, source_class.key, old_class.key);
 	return EmitSelectValueU32(
 	    state, EmitLogicalAndBool(state, EmitLogicalNotBool(state, unordered), compare), source,
 	    old);
@@ -353,17 +342,17 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 		const auto perm   = state.builder.AllocateId();
 		const auto base   = state.builder.AllocateId();
 		const auto target = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
-		state.builder.AddFunction(
-		    {OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
-		state.builder.AddFunction(
-		    {OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
-		state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, perm});
+		state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane2, subid,
+		                          ConstantU32(state, 3));
+		state.builder.AddFunction(spv::OpShiftLeftLogical, TypeU32(state), shift, lane2,
+		                          ConstantU32(state, 1));
+		state.builder.AddFunction(spv::OpShiftRightLogical, TypeU32(state), perm0,
+		                          ConstantU32(state, control), shift);
+		state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), perm, perm0,
+		                          ConstantU32(state, 3));
+		state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), base, subid,
+		                          ConstantU32(state, 0xfffffffcu));
+		state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, base, perm);
 		return target;
 	}
 
@@ -373,16 +362,17 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
-	state.builder.AddFunction(
-	    {OpBitwiseOr, TypeU32(state), ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
-	state.builder.AddFunction(
-	    {OpBitwiseXor, TypeU32(state), xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
-	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, xored});
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), lane, subid,
+	                          ConstantU32(state, 31));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), masked, lane,
+	                          ConstantU32(state, control & 0x1fu));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), ored, masked,
+	                          ConstantU32(state, (control >> 5u) & 0x1fu));
+	state.builder.AddFunction(spv::OpBitwiseXor, TypeU32(state), xored, ored,
+	                          ConstantU32(state, (control >> 10u) & 0x1fu));
+	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), base, subid,
+	                          ConstantU32(state, 0xffffffe0u));
+	state.builder.AddFunction(spv::OpBitwiseOr, TypeU32(state), target, base, xored);
 	return target;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -3,24 +3,24 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
 
-uint32_t MeshArray(EmitterState& state, uint32_t storage, uint32_t type, uint32_t count) {
-	const auto array = state.builder.Type(OpTypeArray, {type, ConstantU32(state, count)});
+uint32_t MeshArray(EmitterState& state, spv::StorageClass storage, uint32_t type, uint32_t count) {
+	const auto array = state.builder.Type(spv::OpTypeArray, type, ConstantU32(state, count));
 	return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage);
 }
 
-uint32_t MeshElement(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type,
-                     uint32_t index) {
+uint32_t MeshElement(EmitterState& state, uint32_t variable, spv::StorageClass storage,
+                     uint32_t type, uint32_t index) {
 	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, TypePointer(state, storage, type), pointer, variable, index});
+	state.builder.AddFunction(spv::OpAccessChain, TypePointer(state, storage, type), pointer,
+	                          variable, index);
 	return pointer;
 }
 
-uint32_t MeshLoad(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type,
+uint32_t MeshLoad(EmitterState& state, uint32_t variable, spv::StorageClass storage, uint32_t type,
                   uint32_t index) {
 	const auto pointer = MeshElement(state, variable, storage, type, index);
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, type, value, pointer});
+	state.builder.AddFunction(spv::OpLoad, type, value, pointer);
 	return value;
 }
 
@@ -40,40 +40,44 @@ void DefineMeshOutputs(EmitterState& state) {
 		}
 		const auto type    = MeshOutputType(state, output.kind);
 		output.variable_id = MeshArray(
-		    state, StorageClassOutput, type,
+		    state, spv::StorageClassOutput, type,
 		    output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
 		// Only Layer is read by another invocation, through the primitive's provoking vertex.
 		const bool shared = output.kind == IR::StageOutputKind::Layer;
-		output.mesh_data_variable = MeshArray(
-		    state, shared ? StorageClassWorkgroup : StorageClassPrivate, type,
-		    shared ? mesh.max_vertices : state.lane_count);
+		output.mesh_data_variable =
+		    MeshArray(state, shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, type,
+		              shared ? mesh.max_vertices : state.lane_count);
 		state.interface_variables.push_back(output.variable_id);
 		state.builder.AddName(output.variable_id, output.debug_name.c_str());
 		if (output.kind == IR::StageOutputKind::Parameter) {
-			state.builder.AddAnnotation(
-			    {OpDecorate, output.variable_id, DecorationLocation, output.location});
+			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
+			                            spv::DecorationLocation, output.location);
 		} else {
-			state.builder.AddAnnotation(
-			    {OpDecorate, output.variable_id, DecorationBuiltIn,
-			     output.kind == IR::StageOutputKind::Layer ? BuiltInLayer : BuiltInPosition});
+			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id, spv::DecorationBuiltIn,
+			                            output.kind == IR::StageOutputKind::Layer
+			                                ? spv::BuiltInLayer
+			                                : spv::BuiltInPosition);
 		}
 		if (output.kind == IR::StageOutputKind::Layer) {
-			state.builder.AddAnnotation({OpDecorate, output.variable_id, 5271u}); // PerPrimitiveEXT
+			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
+			                            spv::DecorationPerPrimitiveEXT); // PerPrimitiveEXT
 		}
 	}
-	state.mesh_allocation = MeshArray(state, StorageClassWorkgroup, TypeU32(state), 2);
+	state.mesh_allocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2);
 	state.mesh_primitive_data =
-	    MeshArray(state, StorageClassPrivate, TypeU32(state), state.lane_count);
+	    MeshArray(state, spv::StorageClassPrivate, TypeU32(state), state.lane_count);
 	state.mesh_primitives =
-	    MeshArray(state, StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
-	state.mesh_cull = MeshArray(state, StorageClassOutput, TypeBool(state), mesh.max_primitives);
+	    MeshArray(state, spv::StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
+	state.mesh_cull =
+	    MeshArray(state, spv::StorageClassOutput, TypeBool(state), mesh.max_primitives);
 	state.interface_variables.push_back(state.mesh_primitives);
 	state.interface_variables.push_back(state.mesh_cull);
-	state.builder.AddAnnotation({OpDecorate, state.mesh_primitives, DecorationBuiltIn,
-	                             5296u}); // PrimitiveTriangleIndicesEXT
 	state.builder.AddAnnotation(
-	    {OpDecorate, state.mesh_cull, DecorationBuiltIn, 5299u}); // CullPrimitiveEXT
-	state.builder.AddAnnotation({OpDecorate, state.mesh_cull, 5271u});
+	    spv::OpDecorate, state.mesh_primitives, spv::DecorationBuiltIn,
+	    spv::BuiltInPrimitiveTriangleIndicesEXT); // PrimitiveTriangleIndicesEXT
+	state.builder.AddAnnotation(spv::OpDecorate, state.mesh_cull, spv::DecorationBuiltIn,
+	                            spv::BuiltInCullPrimitiveEXT); // CullPrimitiveEXT
+	state.builder.AddAnnotation(spv::OpDecorate, state.mesh_cull, spv::DecorationPerPrimitiveEXT);
 }
 
 uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) {
@@ -85,113 +89,118 @@ uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32
 		     index);
 	}
 	const bool shared = kind == IR::StageOutputKind::Layer;
-	return MeshElement(state, output->mesh_data_variable,
-	                   shared ? StorageClassWorkgroup : StorageClassPrivate,
-	                   MeshOutputType(state, kind),
-	                   shared ? EmitLocalInvocationIndex(state)
-	                          : ConstantU32(state, state.lane_half));
+	return MeshElement(
+	    state, output->mesh_data_variable,
+	    shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, MeshOutputType(state, kind),
+	    shared ? EmitLocalInvocationIndex(state) : ConstantU32(state, state.lane_half));
 }
 
 uint32_t MeshPrimitivePointer(EmitterState& state) {
-	return MeshElement(state, state.mesh_primitive_data, StorageClassPrivate, TypeU32(state),
+	return MeshElement(state, state.mesh_primitive_data, spv::StorageClassPrivate, TypeU32(state),
 	                   ConstantU32(state, state.lane_half));
 }
 
 void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&      state = ctx.state;
 	const auto first = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpIEqual, TypeBool(state), first, EmitLocalInvocationIndex(state), ConstantU32(state, 0)});
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), first,
+	                          EmitLocalInvocationIndex(state), ConstantU32(state, 0));
 	EmitIfCondition(state, first, [&] {
 		const auto allocation = ctx.Arg(inst, 0);
 		for (uint32_t field = 0; field < 2; field++) {
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), value, allocation,
-			                           ConstantU32(state, field * 12u),
-			                           ConstantU32(state, field == 0 ? 10u : 11u)});
-			const auto pointer = MeshElement(state, state.mesh_allocation, StorageClassWorkgroup,
-			                                 TypeU32(state), ConstantU32(state, field));
-			state.builder.AddFunction({OpStore, pointer, value});
+			state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), value, allocation,
+			                          ConstantU32(state, field * 12u),
+			                          ConstantU32(state, field == 0 ? 10u : 11u));
+			const auto pointer =
+			    MeshElement(state, state.mesh_allocation, spv::StorageClassWorkgroup,
+			                TypeU32(state), ConstantU32(state, field));
+			state.builder.AddFunction(spv::OpStore, pointer, value);
 		}
 	});
 }
 
 void EmitMeshEntryPoint(EmitterState& state) {
-	state.builder.AddFunction(
-	    {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
+	state.builder.AddFunction(spv::OpFunction, TypeVoid(state), state.main_func,
+	                          spv::FunctionControlMaskNone, TypeFunction(state));
 	EmitLabel(state, state.builder.AllocateId());
-	state.builder.AddFunction(
-	    {OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func});
+	state.builder.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.builder.AllocateId(),
+	                          state.mesh_guest_func);
 	// All guest waves finish before the uniform Vulkan allocation and output stores.
-	state.builder.AddFunction(
-	    {OpControlBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, ScopeWorkgroup),
-	     ConstantU32(state, MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory)});
-	const auto vertices = MeshLoad(state, state.mesh_allocation, StorageClassWorkgroup,
-	                               TypeU32(state), ConstantU32(state, 0));
-	const auto primitives = MeshLoad(state, state.mesh_allocation, StorageClassWorkgroup,
+	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
+	                          ConstantU32(state, spv::ScopeWorkgroup),
+	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask |
+	                                                 spv::MemorySemanticsWorkgroupMemoryMask));
+	const auto vertices   = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
+	                                 TypeU32(state), ConstantU32(state, 0));
+	const auto primitives = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
 	                                 TypeU32(state), ConstantU32(state, 1));
-	state.builder.AddFunction({5295u, vertices, primitives}); // OpSetMeshOutputsEXT
+	state.builder.AddFunction(spv::OpSetMeshOutputsEXT, vertices,
+	                          primitives); // OpSetMeshOutputsEXT
 	for (uint32_t half = 0; half < state.lane_count; half++) {
 		state.lane_half      = half;
 		const auto index     = EmitLocalInvocationIndex(state);
 		const auto is_vertex = state.builder.AllocateId();
-		state.builder.AddFunction({OpULessThan, TypeBool(state), is_vertex, index, vertices});
+		state.builder.AddFunction(spv::OpULessThan, TypeBool(state), is_vertex, index, vertices);
 		EmitIfCondition(state, is_vertex, [&] {
 			for (const auto& output: state.outputs) {
 				if (output.kind == IR::StageOutputKind::Layer) {
 					continue;
 				}
 				const auto type  = MeshOutputType(state, output.kind);
-				const auto value = MeshLoad(state, output.mesh_data_variable, StorageClassPrivate,
-				                            type, ConstantU32(state, half));
+				const auto value =
+				    MeshLoad(state, output.mesh_data_variable, spv::StorageClassPrivate, type,
+				             ConstantU32(state, half));
 				const auto pointer =
-				    MeshElement(state, output.variable_id, StorageClassOutput, type, index);
-				state.builder.AddFunction({OpStore, pointer, value});
+				    MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				state.builder.AddFunction(spv::OpStore, pointer, value);
 			}
 		});
 		const auto is_primitive = state.builder.AllocateId();
-		state.builder.AddFunction({OpULessThan, TypeBool(state), is_primitive, index, primitives});
+		state.builder.AddFunction(spv::OpULessThan, TypeBool(state), is_primitive, index,
+		                          primitives);
 		EmitIfCondition(state, is_primitive, [&] {
-			const auto packed = MeshLoad(state, state.mesh_primitive_data, StorageClassPrivate,
+			const auto packed = MeshLoad(state, state.mesh_primitive_data, spv::StorageClassPrivate,
 			                             TypeU32(state), ConstantU32(state, half));
 			uint32_t   vertex[3] {};
 			for (uint32_t component = 0; component < 3; component++) {
 				vertex[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), vertex[component],
-				                           packed, ConstantU32(state, component * 10u),
-				                           ConstantU32(state, 10)});
+				state.builder.AddFunction(
+				    spv::OpBitFieldUExtract, TypeU32(state), vertex[component], packed,
+				    ConstantU32(state, component * 10u), ConstantU32(state, 10));
 			}
 			const auto triangle = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
-			                           vertex[0], vertex[1], vertex[2]});
-			const auto triangle_pointer = MeshElement(
-			    state, state.mesh_primitives, StorageClassOutput, TypeU32Vector(state, 3), index);
-			state.builder.AddFunction({OpStore, triangle_pointer, triangle});
+			state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
+			                          vertex[0], vertex[1], vertex[2]);
+			const auto triangle_pointer =
+			    MeshElement(state, state.mesh_primitives, spv::StorageClassOutput,
+			                TypeU32Vector(state, 3), index);
+			state.builder.AddFunction(spv::OpStore, triangle_pointer, triangle);
 			const auto null_bit =
-			    EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
+			    EmitBinaryU32(state, spv::OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
 			const auto culled = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)});
-			state.builder.AddFunction(
-			    {OpStore,
-			     MeshElement(state, state.mesh_cull, StorageClassOutput, TypeBool(state), index),
-			     culled});
+			state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), culled, null_bit,
+			                          ConstantU32(state, 0));
+			state.builder.AddFunction(spv::OpStore,
+			                          MeshElement(state, state.mesh_cull, spv::StorageClassOutput,
+			                                      TypeBool(state), index),
+			                          culled);
 			for (const auto& output: state.outputs) {
 				if (output.kind != IR::StageOutputKind::Layer) {
 					continue;
 				}
-				const auto layer = MeshLoad(
-				    state, output.mesh_data_variable, StorageClassWorkgroup, TypeU32(state),
-				    vertex[state.input_info.vertex->mesh.provoking_vertex]);
-				const auto pointer = MeshElement(state, output.variable_id, StorageClassOutput,
+				const auto layer = MeshLoad(state, output.mesh_data_variable,
+				                            spv::StorageClassWorkgroup, TypeU32(state),
+				                            vertex[state.input_info.vertex->mesh.provoking_vertex]);
+				const auto pointer = MeshElement(state, output.variable_id, spv::StorageClassOutput,
 				                                 TypeU32(state), index);
-				state.builder.AddFunction({OpStore, pointer, layer});
+				state.builder.AddFunction(spv::OpStore, pointer, layer);
 			}
 		});
 	}
 	state.lane_half = 0;
-	state.builder.AddFunction({OpReturn});
-	state.builder.AddFunction({OpFunctionEnd});
+	state.builder.AddFunction(spv::OpReturn);
+	state.builder.AddFunction(spv::OpFunctionEnd);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include "common/assert.h"
 
@@ -6,7 +6,9 @@
 #include <bit>
 #include <functional>
 #include <optional>
+#include <type_traits>
 #include <unordered_set>
+#include <utility>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -15,11 +17,11 @@ void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
 	const auto kill_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	const auto inactive    = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, TypeBool(state), inactive, active});
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
+	state.builder.AddFunction(spv::OpLogicalNot, TypeBool(state), inactive, active);
+	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
+	state.builder.AddFunction(spv::OpBranchConditional, inactive, kill_label, merge_label);
 	EmitLabel(state, kill_label);
-	state.builder.AddFunction({OpKill});
+	state.builder.AddFunction(spv::OpKill);
 	EmitLabel(state, merge_label);
 }
 
@@ -30,16 +32,16 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
 
 	const auto mask_value = state.builder.AllocateId();
 	const auto active     = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
+	state.builder.AddFunction(spv::OpLoad, TypeU32(state), mask_value,
+	                          state.pixel_valid_mask_variable);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), active, mask_value,
+	                          ConstantU32(state, 0));
 	EmitKillIfBoolFalse(state, active);
 }
 
 uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
-	const auto value_type = ctx.TypeId(type);
-	return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type);
+	const auto value_type = TypeId(ctx.state, type);
+	return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type);
 }
 
 struct DeferredPhiPatch {
@@ -73,8 +75,8 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
 		}
 		for (size_t index = 0; index < phi.NumArgs(); index++) {
 			if (phi.PhiBlock(index) == from) {
-				ctx.state.builder.AddFunction(
-				    {OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
+				ctx.state.builder.AddFunction(spv::OpStore, dispatcher.spills[ctx.half].at(&phi),
+				                              ctx.Def(phi.Arg(index)));
 				break;
 			}
 		}
@@ -92,11 +94,13 @@ const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
 
 void EmitReturn(ValueEmitContext& ctx) {
 	EmitKillIfPixelValidMaskInactive(ctx.state);
-	ctx.state.builder.AddFunction({OpReturn});
+	ctx.state.builder.AddFunction(spv::OpReturn);
 }
 
 uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
+	// Scalar-instruction conditions already test the full wave's raw register values.
 	if (ctx.other_half == nullptr ||
+	    info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
 	    info.terminator.condition == CFG::BranchCondition::GotoVariable) {
 		return ctx.Def(info.condition);
 	}
@@ -104,15 +108,16 @@ uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
 	const auto low    = ctx.state.builder.AllocateId();
 	const auto high   = ctx.state.builder.AllocateId();
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
-	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
+	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
 	const auto kind     = info.terminator.condition;
 	const bool zero     = kind == CFG::BranchCondition::ExecZero ||
 	                      kind == CFG::BranchCondition::VccZero ||
 	                      kind == CFG::BranchCondition::SccZero;
-	const auto combined = EmitBinaryU32(ctx.state, zero ? OpBitwiseAnd : OpBitwiseOr, low, high);
-	ctx.state.builder.AddFunction({zero ? OpIEqual : OpINotEqual, TypeBool(ctx.state), result,
-	                               combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
+	const auto combined =
+	    EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
+	ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
+	                              result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u));
 	return result;
 }
 
@@ -125,14 +130,14 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 			const auto* merge = TargetBlock(program, term.merge_block);
 			const auto* cont  = TargetBlock(program, term.continue_block);
 			if (merge != nullptr && cont != nullptr) {
-				ctx.state.builder.AddFunction(
-				    {OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone});
+				ctx.state.builder.AddFunction(spv::OpLoopMerge, ctx.Label(merge), ctx.Label(cont),
+				                              spv::LoopControlMaskNone);
 			}
 		} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
 		           term.merge_block != UINT32_MAX) {
 			if (const auto* merge = TargetBlock(program, term.merge_block); merge != nullptr) {
-				ctx.state.builder.AddFunction(
-				    {OpSelectionMerge, ctx.Label(merge), SelectionControlNone});
+				ctx.state.builder.AddFunction(spv::OpSelectionMerge, ctx.Label(merge),
+				                              spv::SelectionControlMaskNone);
 			}
 		}
 	};
@@ -145,7 +150,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				return;
 			}
 			emit_merge();
-			ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
+			ctx.state.builder.AddFunction(spv::OpBranch, ctx.Label(target));
 			return;
 		}
 		case CFG::TerminatorKind::ConditionalBranch: {
@@ -157,8 +162,8 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 			}
 			const auto condition = BranchCondition(ctx, info);
 			emit_merge();
-			ctx.state.builder.AddFunction(
-			    {OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
+			ctx.state.builder.AddFunction(spv::OpBranchConditional, condition,
+			                              ctx.Label(true_block), ctx.Label(false_block));
 			return;
 		}
 		default: EmitReturn(ctx); return;
@@ -187,10 +192,9 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
 			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
-			                               BranchCondition(ctx, info),
-			                               ConstantU32(ctx.state, term.true_block),
-			                               ConstantU32(ctx.state, term.false_block)});
+			ctx.state.builder.AddFunction(
+			    spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info),
+			    ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block));
 			return selected;
 		}
 		case CFG::TerminatorKind::IndirectBranch: {
@@ -209,11 +213,10 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
 					const auto match = ctx.state.builder.AllocateId();
 					const auto next  = ctx.state.builder.AllocateId();
-					ctx.state.builder.AddFunction({OpIEqual, TypeBool(ctx.state), match, selector,
-					                               ConstantU32(ctx.state, values[index])});
-					ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), next, match,
-					                               ConstantU32(ctx.state, targets[index]),
-					                               selected});
+					ctx.state.builder.AddFunction(spv::OpIEqual, TypeBool(ctx.state), match,
+					                              selector, ConstantU32(ctx.state, values[index]));
+					ctx.state.builder.AddFunction(spv::OpSelect, TypeU32(ctx.state), next, match,
+					                              ConstantU32(ctx.state, targets[index]), selected);
 					selected = next;
 				}
 			}
@@ -223,18 +226,60 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
 	}
 }
 
+template <typename T>
+decltype(auto) Arg(ValueEmitContext& ctx, const IR::Inst& inst, size_t index) {
+	if constexpr (std::is_same_v<T, const IR::Inst&>) {
+		return inst;
+	} else if constexpr (std::is_same_v<T, IR::Value>) {
+		return inst.Arg(index);
+	} else if constexpr (std::is_same_v<T, IR::ScalarReg>) {
+		return inst.Arg(index).ScalarRegister();
+	} else {
+		static_assert(std::is_same_v<T, uint32_t>);
+		return ctx.Def(inst.Arg(index));
+	}
+}
+
+template <typename Context, typename Return, typename... Args>
+void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::Inst& inst) {
+	// A full instruction keeps metadata and predicated/lane operand loads lazy.
+	static_assert(std::is_same_v<Context, ValueEmitContext> ||
+	              std::is_same_v<Context, EmitterState>);
+	auto& context = [&]() -> Context& {
+		if constexpr (std::is_same_v<Context, EmitterState>)
+			return ctx.state;
+		else
+			return ctx;
+	}();
+	constexpr bool has_inst = (std::is_same_v<Args, const IR::Inst&> || ...);
+	[&]<size_t... I>(std::index_sequence<I...>) {
+		static_assert(((!std::is_same_v<Args, const IR::Inst&> || I == 0) && ...));
+		const auto call = [&] {
+			return emit(context, Arg<Args>(ctx, inst, I - (has_inst && I != 0))...);
+		};
+		if constexpr (std::is_void_v<Return>) {
+			call();
+		} else {
+			static_assert(std::is_same_v<Return, uint32_t>);
+			ctx.Define(inst, call());
+		}
+	}(std::index_sequence_for<Args...> {});
+}
+
 void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
-	if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
-	    EmitValueImage(ctx, inst)) {
-		return;
+	switch (inst.GetOpcode()) {
+#define VALUE_OPCODE(name, ...)                                                                    \
+	case IR::ValueOpcode::name: return Invoke(Emit##name, ctx, inst);
+#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc"
+#undef VALUE_OPCODE
+		default: ctx.Fail(inst, "has no direct SPIR-V emitter");
 	}
-	ctx.Fail(inst, "has no direct SPIR-V emitter");
 }
 
 void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
                                const IR::Inst& inst) {
 	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
-		const auto type = ctx.TypeId(inst.GetType());
+		const auto type = TypeId(ctx.state, inst.GetType());
 		if (type == 0 || inst.NumArgs() == 0) {
 			ctx.Fail(inst, "has no native SPIR-V representation");
 		}
@@ -255,19 +300,19 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
 void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
                                const IR::Inst& inst) {
 	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
-		const auto type = ctx.TypeId(inst.GetType());
+		const auto type = TypeId(ctx.state, inst.GetType());
 		if (type == 0) {
 			ctx.Fail(inst, "cannot be loaded by the dispatcher");
 		}
-		ctx.state.builder.AddFunction(
-		    {OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
+		ctx.state.builder.AddFunction(spv::OpLoad, type, ctx.Result(inst),
+		                              dispatcher.spills[ctx.half].at(&inst));
 		return;
 	}
 	EmitDirectInstruction(ctx, inst);
 	if (const auto found = dispatcher.spills[ctx.half].find(&inst);
 	    found != dispatcher.spills[ctx.half].end()) {
-		ctx.state.builder.AddFunction(
-		    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
+		ctx.state.builder.AddFunction(spv::OpStore, found->second,
+		                              ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst))));
 	}
 }
 
@@ -314,7 +359,7 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
 void EmitStructuredFunction(ValueEmitContext& ctx) {
 	const auto& program = ctx.state.program;
 	StructuredFunctionState structured;
-	ctx.state.builder.AddFunction({OpBranch, ctx.Label(program.blocks.front())});
+	ctx.state.builder.AddFunction(spv::OpBranch, ctx.Label(program.blocks.front()));
 	for (size_t index = 0; index < program.blocks.size(); index++) {
 		const auto* block = program.blocks[index];
 		EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
@@ -329,38 +374,38 @@ void EmitStructuredFunction(ValueEmitContext& ctx) {
 void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
 	auto&       state = ctx.state;
 	const auto* entry = state.program.blocks.front();
-	state.builder.AddFunction({OpBranch, ctx.Label(entry)});
+	state.builder.AddFunction(spv::OpBranch, ctx.Label(entry));
 	EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
 		EmitDispatcherInstruction(lane, dispatcher, inst);
 	});
 	const auto initial_pc =
 	    EmitDispatcherNextPc(ctx, dispatcher, entry, state.program.block_info.front());
 	const auto initial_parent = state.current_label;
-	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	state.builder.AddFunction(spv::OpBranch, dispatcher.header_label);
 
 	EmitLabel(state, dispatcher.header_label);
 	const auto pc      = state.builder.AllocateId();
 	const auto next_pc = state.builder.AllocateId();
-	state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
-	                           dispatcher.continue_label});
+	state.builder.AddFunction(spv::OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
+	                          dispatcher.continue_label);
 	const auto done = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
-	state.builder.AddFunction(
-	    {OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label, LoopControlNone});
-	state.builder.AddFunction(
-	    {OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
+	state.builder.AddFunction(spv::OpIEqual, TypeBool(state), done, pc,
+	                          ConstantU32(ctx.state, UINT32_MAX));
+	state.builder.AddFunction(spv::OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label,
+	                          spv::LoopControlMaskNone);
+	state.builder.AddFunction(spv::OpBranchConditional, done, dispatcher.merge_label,
+	                          dispatcher.select_label);
 
 	EmitLabel(state, dispatcher.select_label);
-	state.builder.AddFunction(
-	    {OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone});
-	std::vector<uint32_t> words {OpSwitch, pc, dispatcher.after_switch_label};
+	state.builder.AddFunction(spv::OpSelectionMerge, dispatcher.after_switch_label,
+	                          spv::SelectionControlMaskNone);
+	std::vector<uint32_t> words {spv::OpSwitch, pc, dispatcher.after_switch_label};
 	for (size_t index = 1; index < state.program.blocks.size(); index++) {
 		words.push_back(state.program.block_info[index].id);
 		words.push_back(ctx.Label(state.program.blocks[index]));
 	}
 	state.builder.AddFunction(words);
-	std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc,
+	std::vector<uint32_t> next_pc_words {spv::OpPhi, TypeU32(state), next_pc,
 	                                     ConstantU32(state, UINT32_MAX), dispatcher.select_label};
 
 	for (size_t index = 1; index < state.program.blocks.size(); index++) {
@@ -372,20 +417,20 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
 		                                           state.program.block_info[index]);
 		next_pc_words.push_back(selected);
 		next_pc_words.push_back(state.current_label);
-		state.builder.AddFunction({OpBranch, dispatcher.after_switch_label});
+		state.builder.AddFunction(spv::OpBranch, dispatcher.after_switch_label);
 	}
 	EmitLabel(state, dispatcher.after_switch_label);
 	state.builder.AddFunction(next_pc_words);
-	state.builder.AddFunction({OpBranch, dispatcher.continue_label});
+	state.builder.AddFunction(spv::OpBranch, dispatcher.continue_label);
 	EmitLabel(state, dispatcher.continue_label);
-	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	state.builder.AddFunction(spv::OpBranch, dispatcher.header_label);
 	EmitLabel(state, dispatcher.merge_label);
 	EmitReturn(ctx);
 }
 
 } // namespace
 
-uint32_t ValueEmitContext::TypeId(IR::Type type) const {
+uint32_t TypeId(EmitterState& state, IR::Type type) {
 	switch (type) {
 		case IR::Type::U1: return TypeBool(state);
 		case IR::Type::U8:
@@ -430,7 +475,8 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
 				return loaded->second.second;
 			}
 			const auto id = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
+			state.builder.AddFunction(spv::OpLoad, TypeId(state, inst->GetType()), id,
+			                          found->second);
 			dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
 			return id;
 		}
@@ -448,11 +494,11 @@ uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t
 
 uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
 	const auto ballot_type = TypeU32Vector(state, 4);
-	const auto scope       = ConstantU32(state, ScopeSubgroup);
+	const auto scope       = ConstantU32(state, spv::ScopeSubgroup);
 	const auto low         = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpGroupNonUniformBallot, ballot_type, low, scope,
-	     other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)});
+	state.builder.AddFunction(spv::OpGroupNonUniformBallot, ballot_type, low, scope,
+	                          other_half == nullptr || half == 0 ? Def(predicate)
+	                                                             : other_half->Def(predicate));
 	if (other_half == nullptr) {
 		return low;
 	}
@@ -460,20 +506,20 @@ uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
 	const auto low_word  = state.builder.AllocateId();
 	const auto high_word = state.builder.AllocateId();
 	const auto ballot    = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, ballot_type, high, scope,
-	                           half == 1 ? Def(predicate) : other_half->Def(predicate)});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low_word, low, 0});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high_word, high, 0});
-	state.builder.AddFunction({OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
-	                           ConstantU32(state, 0), ConstantU32(state, 0)});
+	state.builder.AddFunction(spv::OpGroupNonUniformBallot, ballot_type, high, scope,
+	                          half == 1 ? Def(predicate) : other_half->Def(predicate));
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low_word, low, 0);
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high_word, high, 0);
+	state.builder.AddFunction(spv::OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
+	                          ConstantU32(state, 0), ConstantU32(state, 0));
 	return ballot;
 }
 
 uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
 	if (other_half == nullptr) {
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
-		                           ConstantU32(state, ScopeSubgroup), ballot});
+		state.builder.AddFunction(spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
+		                          ConstantU32(state, spv::ScopeSubgroup), ballot);
 		return result;
 	}
 	const auto low        = state.builder.AllocateId();
@@ -482,40 +528,41 @@ uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
 	const auto high_first = state.builder.AllocateId();
 	const auto low_active = state.builder.AllocateId();
 	const auto result     = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
-	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
-	state.builder.AddFunction(
-	    {OpExtInst, TypeU32(state), low_first, GlslStd450(state), GlslFindILsb, low});
-	state.builder.AddFunction(
-	    {OpExtInst, TypeU32(state), high_first, GlslStd450(state), GlslFindILsb, high});
-	state.builder.AddFunction(
-	    {OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, TypeU32(state), result, low_active, low_first,
-	                           EmitAddU32(state, high_first, ConstantU32(state, 32))});
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, ballot, 0);
+	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, ballot, 1);
+	state.builder.AddFunction(spv::OpExtInst, TypeU32(state), low_first, GlslStd450(state),
+	                          GLSLstd450FindILsb, low);
+	state.builder.AddFunction(spv::OpExtInst, TypeU32(state), high_first, GlslStd450(state),
+	                          GLSLstd450FindILsb, high);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), low_active, low,
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpSelect, TypeU32(state), result, low_active, low_first,
+	                          EmitAddU32(state, high_first, ConstantU32(state, 32)));
 	return result;
 }
 
 uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
-	const auto type  = TypeId(inst.Arg(index).GetType());
-	const auto scope = ConstantU32(state, ScopeSubgroup);
+	const auto type  = TypeId(state, inst.Arg(index).GetType());
+	const auto scope = ConstantU32(state, spv::ScopeSubgroup);
 	const auto low   = state.builder.AllocateId();
 	if (other_half == nullptr) {
-		state.builder.AddFunction(
-		    {OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
+		state.builder.AddFunction(spv::OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index),
+		                          lane);
 		return low;
 	}
-	const auto physical_lane = EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 31));
+	const auto physical_lane =
+	    EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31));
 	const auto high          = state.builder.AllocateId();
 	const auto in_high       = state.builder.AllocateId();
 	const auto value         = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
-	state.builder.AddFunction(
-	    {OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
-	state.builder.AddFunction({OpINotEqual, TypeBool(state), in_high,
-	                           EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 32)),
-	                           ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, type, value, in_high, high, low});
+	state.builder.AddFunction(spv::OpGroupNonUniformShuffle, type, low, scope,
+	                          HalfArg(inst, index, 0), physical_lane);
+	state.builder.AddFunction(spv::OpGroupNonUniformShuffle, type, high, scope,
+	                          HalfArg(inst, index, 1), physical_lane);
+	state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), in_high,
+	                          EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 32)),
+	                          ConstantU32(state, 0));
+	state.builder.AddFunction(spv::OpSelect, type, value, in_high, high, low);
 	return value;
 }
 
@@ -528,20 +575,11 @@ uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
 	return id;
 }
 
-uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
-                                std::initializer_list<uint32_t> args) {
-	const auto type_id = TypeId(type);
-	const auto result  = Result(inst);
-	std::vector<uint32_t> words {opcode, type_id, result};
-	words.insert(words.end(), args.begin(), args.end());
-	state.builder.AddFunction(words);
-	return result;
-}
-
 uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
 	if (const auto found = definitions.find(&inst); found != definitions.end()) {
 		if (found->second != value) {
-			state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value});
+			state.builder.AddFunction(spv::OpCopyObject, TypeId(state, inst.GetType()),
+			                          found->second, value);
 		}
 		return found->second;
 	}
@@ -600,7 +638,7 @@ void EmitProgram(EmitterState& state) {
 		high.half       = 1;
 	}
 	std::optional<DispatcherFunctionState> dispatcher;
-	if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
+	if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
 		state.pixel_valid_mask_variable = state.builder.AllocateId();
 		state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
 	}
@@ -677,27 +715,28 @@ void EmitProgram(EmitterState& state) {
 			break;
 		}
 	}
-	state.builder.AddFunction({OpFunction, TypeVoid(state),
-	                           state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
-	                           FunctionControlNone, TypeFunction(state)});
+	state.builder.AddFunction(spv::OpFunction, TypeVoid(state),
+	                          state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
+	                          spv::FunctionControlMaskNone, TypeFunction(state));
 	EmitLabel(state, state.entry_label);
 	if (state.requirements.function_lds) {
 		state.builder.AddFunction(
-		    {OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)),
-		     state.lds_variable, StorageClassFunction});
+		    spv::OpVariable,
+		    TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)),
+		    state.lds_variable, spv::StorageClassFunction);
 	}
 	if (state.requirements.function_scratch) {
 		for (uint32_t half = 0; half < state.lane_count; half++) {
 			state.builder.AddFunction(
-			    {OpVariable,
-			     TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
-			     state.scratch_variable[half], StorageClassFunction});
+			    spv::OpVariable,
+			    TypeU32ArrayPointer(state, spv::StorageClassFunction, state.program.scratch_dwords),
+			    state.scratch_variable[half], spv::StorageClassFunction);
 		}
 	}
 	if (state.pixel_valid_mask_variable != 0) {
-		state.builder.AddFunction({OpVariable,
-		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
-		                           state.pixel_valid_mask_variable, StorageClassFunction});
+		state.builder.AddFunction(spv::OpVariable,
+		                          TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
+		                          state.pixel_valid_mask_variable, spv::StorageClassFunction);
 	}
 	for (uint32_t half = 0; half < state.lane_count; half++) {
 		auto& lane = half == 0 ? ctx : high;
@@ -706,27 +745,27 @@ void EmitProgram(EmitterState& state) {
 				for (const auto& inst: *block) {
 					if (const auto found = dispatcher->spills[half].find(&inst);
 					    found != dispatcher->spills[half].end()) {
-						state.builder.AddFunction({OpVariable,
-						                           SpillPointerType(lane, inst.GetType()),
-						                           found->second, StorageClassFunction});
+						state.builder.AddFunction(spv::OpVariable,
+						                          SpillPointerType(lane, inst.GetType()),
+						                          found->second, spv::StorageClassFunction);
 					}
 				}
 			}
 		}
 		if (lane.scratch_u32_variable != 0) {
-			state.builder.AddFunction({OpVariable,
-			                           TypePointer(state, StorageClassFunction, TypeU32(state)),
-			                           lane.scratch_u32_variable, StorageClassFunction});
+			state.builder.AddFunction(spv::OpVariable,
+			                          TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
+			                          lane.scratch_u32_variable, spv::StorageClassFunction);
 		}
 	}
 	if (state.gds_variable != 0) {
 		state.gds_length = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
+		state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), state.gds_length,
+		                          state.gds_variable, 0);
 	}
 	if (state.pixel_valid_mask_variable != 0) {
-		state.builder.AddFunction(
-		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
+		state.builder.AddFunction(spv::OpStore, state.pixel_valid_mask_variable,
+		                          ConstantU32(state, 1));
 	}
 	EmitMemoryOffsets(state);
 	if (program.blocks.empty()) {
@@ -736,8 +775,8 @@ void EmitProgram(EmitterState& state) {
 	} else {
 		EmitStructuredFunction(ctx);
 	}
-	state.builder.AddFunction({OpFunctionEnd});
-	if (state.stage == ShaderType::Mesh) {
+	state.builder.AddFunction(spv::OpFunctionEnd);
+	if (state.program.stage == ShaderType::Mesh) {
 		EmitMeshEntryPoint(state);
 	}
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp

@@ -0,0 +1,161 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t TessellationPointer(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& state          = ctx.state;
+	using Attribute      = IR::TessellationAttribute;
+	const auto  kind     = static_cast<Attribute>(inst.Arg(0).U32());
+	const auto& tess     = state.input_info.vertex->tess;
+	const auto  variable = state.tess_variables.at(static_cast<uint32_t>(kind));
+	EXIT_IF(variable == 0);
+	const auto input   = kind == Attribute::ControlInput || kind == Attribute::EvaluationInput;
+	const auto storage = input ? spv::StorageClassInput : spv::StorageClassOutput;
+	const auto pointer = state.builder.AllocateId();
+	if (kind == Attribute::Factor) {
+		EXIT_NOT_IMPLEMENTED(!inst.Arg(1).IsImmediate());
+		const auto index = inst.Arg(1).U32() / 4u;
+		const auto outer = index < 3u;
+		EXIT_NOT_IMPLEMENTED(index >= 4u);
+		state.builder.AddFunction(spv::OpAccessChain, TypePointer(state, storage, TypeF32(state)),
+		                          pointer, outer ? variable : state.tess_inner_variable,
+		                          ConstantU32(state, outer ? index : index - 3u));
+		return pointer;
+	}
+	auto address = ctx.Arg(inst, 1);
+	if (kind == Attribute::PatchOutput) {
+		address =
+		    EmitBinaryU32(state, spv::OpISub, address, ConstantU32(state, state.tess_patch_base));
+	}
+	const bool local  = kind == Attribute::LocalOutput || kind == Attribute::ControlInput;
+	const auto stride = local ? tess.ls_stride : tess.hs_stride;
+	const auto offset = kind == Attribute::PatchOutput ? address
+	                                                   : EmitBinaryU32(state, spv::OpUMod, address,
+	                                                                   ConstantU32(state, stride));
+	const auto attribute =
+	    EmitBinaryU32(state, spv::OpShiftRightLogical, offset, ConstantU32(state, 4));
+	const auto component =
+	    EmitBinaryU32(state, spv::OpBitwiseAnd,
+	                  EmitBinaryU32(state, spv::OpShiftRightLogical, offset, ConstantU32(state, 2)),
+	                  ConstantU32(state, 3));
+	const auto type = TypePointer(state, storage, TypeU32(state));
+	if (kind == Attribute::LocalOutput || kind == Attribute::PatchOutput) {
+		state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, attribute,
+		                          component);
+	} else {
+		const auto vertex =
+		    kind == Attribute::ControlOutput
+		        ? EmitLaneId(state)
+		        : EmitBinaryU32(state, spv::OpUDiv, address, ConstantU32(state, stride));
+		state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, vertex, attribute,
+		                          component);
+	}
+	return pointer;
+}
+
+} // namespace
+
+void DefineTessellationInterfaces(EmitterState& state) {
+	using Attribute = IR::TessellationAttribute;
+	std::array<bool, 6> used {};
+	uint32_t            patch_begin = UINT32_MAX, patch_end = 0;
+	for (const auto* block: state.program.blocks) {
+		for (const auto& inst: *block) {
+			if (inst.GetOpcode() != IR::ValueOpcode::GetTessellationAttribute &&
+			    inst.GetOpcode() != IR::ValueOpcode::SetTessellationAttribute)
+				continue;
+			const auto kind = inst.Arg(0).U32();
+			used.at(kind)   = true;
+			if (kind == static_cast<uint32_t>(Attribute::PatchOutput)) {
+				EXIT_NOT_IMPLEMENTED(!inst.Arg(1).IsImmediate());
+				patch_begin = std::min(patch_begin, inst.Arg(1).U32());
+				patch_end   = std::max(patch_end, inst.Arg(1).U32() + 4u);
+			}
+		}
+	}
+	if (std::ranges::none_of(used, [](bool value) { return value; })) return;
+	const auto& tess  = state.input_info.vertex->tess;
+	const auto  array = [&](uint32_t type, uint32_t count) {
+		return state.builder.Type(spv::OpTypeArray, type, ConstantU32(state, count));
+	};
+	for (uint32_t index = 0; index < used.size(); index++) {
+		if (!used[index]) continue;
+		const auto kind    = static_cast<Attribute>(index);
+		const bool input   = kind == Attribute::ControlInput || kind == Attribute::EvaluationInput;
+		const auto storage = input ? spv::StorageClassInput : spv::StorageClassOutput;
+		uint32_t   type;
+		if (kind == Attribute::Factor) {
+			type = array(TypeF32(state), 4u);
+		} else if (kind == Attribute::PatchOutput) {
+			state.tess_patch_base = patch_begin;
+			type = array(TypeU32Vector(state, 4), (patch_end - patch_begin + 15u) / 16u);
+		} else {
+			const bool local  = kind == Attribute::LocalOutput || kind == Attribute::ControlInput;
+			const auto stride = local ? tess.ls_stride : tess.hs_stride;
+			type              = array(TypeU32Vector(state, 4), (stride + 15u) / 16u);
+			if (kind != Attribute::LocalOutput) {
+				type = array(type, local ? tess.input_control_points : tess.output_control_points);
+			}
+		}
+		auto& variable = state.tess_variables[index];
+		variable       = DefineInterfaceVariable(state, type, storage, "tess_attributes");
+		if (kind == Attribute::Factor) {
+			state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn,
+			                            spv::BuiltInTessLevelOuter);
+			state.tess_inner_variable =
+			    DefineInterfaceVariable(state, array(TypeF32(state), 2u), storage, "tess_inner");
+			state.builder.AddAnnotation(spv::OpDecorate, state.tess_inner_variable,
+			                            spv::DecorationBuiltIn, spv::BuiltInTessLevelInner);
+			state.builder.AddAnnotation(spv::OpDecorate, state.tess_inner_variable,
+			                            spv::DecorationPatch);
+		} else {
+			state.builder.AddAnnotation(
+			    spv::OpDecorate, variable, spv::DecorationLocation,
+			    kind == Attribute::PatchOutput ? (tess.hs_stride + 15u) / 16u : 0u);
+		}
+		if (kind == Attribute::Factor || kind == Attribute::PatchOutput) {
+			state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationPatch);
+		}
+	}
+}
+
+void DefineTessellationExecutionModes(EmitterState& state) {
+	const auto& tess = state.input_info.vertex->tess;
+	EXIT_NOT_IMPLEMENTED(tess.domain != 1u || tess.partitioning != 2u ||
+	                     tess.output_topology != 2u);
+	state.builder.RequireCapability(spv::CapabilityTessellation);
+	if (state.program.stage == ShaderType::TessellationControl) {
+		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeOutputVertices,
+		                               tess.output_control_points);
+	} else {
+		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeTriangles);
+		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSpacingFractionalOdd);
+		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeVertexOrderCw);
+	}
+}
+
+uint32_t EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, 2), [&] {
+		const auto value = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value,
+		                              TessellationPointer(ctx, inst));
+		return value;
+	});
+}
+
+void EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
+	EmitIfCondition(ctx.state, ctx.Arg(inst, 3), [&] {
+		auto value = ctx.Arg(inst, 2);
+		if (inst.Arg(0).U32() == static_cast<uint32_t>(IR::TessellationAttribute::Factor)) {
+			const auto floating = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(spv::OpBitcast, TypeF32(ctx.state), floating, value);
+			value = floating;
+		}
+		ctx.state.builder.AddFunction(spv::OpStore, TessellationPointer(ctx, inst), value);
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter