update screenshot

Co-authored-by: Behzad <behzadkhan1736@gmail.com>

3rdparty/cpuinfo/src/arm/mach/init.c

@@ -14,7 +14,7 @@
 #include <cpuinfo/internal-api.h>
 #include <cpuinfo/log.h>
 
-/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */
+/* Polyfill recent CPUFAMILY_ARM_* values for older platform headers */
 #ifndef CPUFAMILY_ARM_MONSOON_MISTRAL
 	#define CPUFAMILY_ARM_MONSOON_MISTRAL   0xE81E7EF6
 #endif

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.18...4.3)
+cmake_minimum_required(VERSION 3.19...4.3)
 
 if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
 	set(CMAKE_POLICY_VERSION_MINIMUM 3.10 CACHE STRING "Minimum policy version for bundled dependencies")
@@ -244,7 +244,7 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
 	"${KYTY_SOURCE_DIR}/loader/*.h"
 )
 
-find_program(KYTY_GLSLANG_VALIDATOR glslangValidator REQUIRED)
+find_program(KYTY_GLSLANG_VALIDATOR NAMES glslangValidator glslang REQUIRED)
 set(gpu_tiler_shader_dir "${KYTY_SOURCE_DIR}/graphics/host_gpu/shaders")
 set(gpu_tiler_generated_dir "${PROJECT_BINARY_DIR}/gpu_tiler_shaders")
 set(gpu_tiler_shader_names
@@ -259,7 +259,7 @@ set(gpu_tiler_shader_names
 	depth
 	promote_d16
 	demote_d16
-	swap_bgra16
+	color_transform
 )
 file(GLOB gpu_tiler_shader_includes CONFIGURE_DEPENDS "${gpu_tiler_shader_dir}/gpu_tiler_*.inc")
 foreach(shader_name IN LISTS gpu_tiler_shader_names)
@@ -393,8 +393,10 @@ add_kyty_full_emulator_test(shader_cfg_tests "${KYTY_TESTS_DIR}/shaderCfgTests.c
 
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ScalarProvenanceTests.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 )
 target_link_libraries(scalar_provenance_tests fmt::fmt)
@@ -413,7 +415,16 @@ add_executable(archive_file_tests EXCLUDE_FROM_ALL
 target_link_libraries(archive_file_tests common ZArchive::zarchive)
 target_include_directories(archive_file_tests PRIVATE ${inc_headers})
 
-add_executable(avplayer_file_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp")
+add_executable(trophy_system_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/TrophySystemTests.cpp"
+)
+target_link_libraries(trophy_system_tests common)
+target_include_directories(trophy_system_tests PRIVATE ${inc_headers})
+
+add_executable(avplayer_file_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
 target_link_libraries(avplayer_file_tests common FFmpeg::ffmpeg ZArchive::zarchive)
 target_include_directories(avplayer_file_tests PRIVATE ${inc_headers})
 
@@ -468,7 +479,6 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/BindingLayout.cpp"
 )
 target_link_libraries(resource_tracking_tests fmt::fmt)
 target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
@@ -606,6 +616,7 @@ if(BUILD_TESTING)
 	add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
 	add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
 	add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
+	add_test(NAME trophy_system COMMAND $<TARGET_FILE:trophy_system_tests>)
 	add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
 	add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
 	add_test(NAME lru_cache COMMAND $<TARGET_FILE:lru_cache_tests>)
@@ -664,15 +675,12 @@ if(BUILD_TESTING)
 	add_test(NAME color_render_target_1d
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --rt1d-only)
 
-	if(WIN32)
-		# These tests still depend on the Windows multisample-depth path.
-		add_test(NAME texture_cache_image_overlap
-			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
-		add_test(NAME texture_cache_htile_clear
-			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
-		add_test(NAME buffer_cache_ranges
-			COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
-	endif()
+	add_test(NAME texture_cache_image_overlap
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --image-overlap-only)
+	add_test(NAME texture_cache_htile_clear
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --htile-clear-only)
+	add_test(NAME buffer_cache_ranges
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-range-only)
 
 	add_custom_target(kyty_tests DEPENDS
 		ime_dialog_tests
@@ -682,6 +690,7 @@ if(BUILD_TESTING)
 		memory_tracker_tests
 		page_manager_tests
 		archive_file_tests
+		trophy_system_tests
 		avplayer_file_tests
 		bit_array_tests
 		lru_cache_tests
@@ -712,8 +721,9 @@ if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
 	add_executable(macos_sse4a_tests EXCLUDE_FROM_ALL
 		"${KYTY_TESTS_DIR}/macosSse4aTests.cpp"
 		"${KYTY_SOURCE_DIR}/loader/x64InstructionEmulator.cpp"
+		"${KYTY_SOURCE_DIR}/loader/guestInstructionPatcher.cpp"
 	)
-	target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis)
+	target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis xbyak::xbyak)
 	target_include_directories(macos_sse4a_tests PRIVATE ${inc_headers})
 	if(BUILD_TESTING)
 		add_test(NAME macos_sse4a COMMAND $<TARGET_FILE:macos_sse4a_tests>)
@@ -725,6 +735,15 @@ endif()
 add_executable(kyty_emulator "${KYTY_SOURCE_DIR}/main.cpp" ${kyty_emulator_src})
 configure_macos_guest_address_space(kyty_emulator)
 
+set(KYTY_TROPHY_UNLOCK_SOUND "${CMAKE_CURRENT_SOURCE_DIR}/assets/sounds/trophy-unlock.wav")
+add_custom_target(kyty_trophy_sound
+	COMMAND ${CMAKE_COMMAND} -E make_directory "$<TARGET_FILE_DIR:kyty_emulator>/assets/sounds"
+	COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_TROPHY_UNLOCK_SOUND}"
+		"$<TARGET_FILE_DIR:kyty_emulator>/assets/sounds/trophy-unlock.wav"
+	VERBATIM)
+add_dependencies(kyty_emulator kyty_trophy_sound)
+install(FILES "${KYTY_TROPHY_UNLOCK_SOUND}" DESTINATION assets/sounds)
+
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)
 	target_link_libraries(kyty_emulator opengl32)

README.md

@@ -1,8 +1,8 @@
 # KytyPS5
 
-[![Build KytyPS5 (Windows)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (Linux)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (macOS)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (Windows)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28Windows%29&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (Linux)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28Linux%29&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (macOS)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28macOS%29&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
 [![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20Linux%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
 [![Status](https://img.shields.io/badge/status-active%20development-orange.svg)](#current-status)
 [![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
@@ -52,8 +52,8 @@ or graphical glitches, so please include the version you tested when reporting a
       <img src="docs/screenshots/ps5-01.png" width="300" alt="Astro Bot running in KytyPS5">
     </td>
     <td align="center">
-      <strong>Dreaming Sarah</strong><br>
-      <img src="docs/screenshots/ps5-03.png" width="300" alt="Dreaming Sarah running in KytyPS5">
+      <strong>Saros</strong><br>
+      <img src="docs/screenshots/ps5-03.png" width="300" alt="Saros running in KytyPS5">
     </td>
   </tr>
   <tr>
@@ -72,8 +72,8 @@ or graphical glitches, so please include the version you tested when reporting a
       <img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
     </td>
     <td align="center">
-      <strong>UFC 5</strong><br>
-      <img src="docs/screenshots/ps5-06.png" width="300" alt="UFC 5 running in KytyPS5">
+      <strong>UFC 6</strong><br>
+      <img src="docs/screenshots/ps5-06.jpg" width="300" alt="UFC 6 running in KytyPS5">
     </td>
   </tr>
 </table>
@@ -141,7 +141,8 @@ the Vulkan/SPIR-V validation rules.
 - Visual Studio 2022 or Build Tools 2022 with the **Desktop development with C++** workload and
   **C++ Clang tools for Windows** component
 - Qt 6 for MSVC 2022 64-bit, including Concurrent, Network, and Widgets
-- [glslang](https://github.com/KhronosGroup/glslang/releases) (`glslangValidator`) on `PATH`
+- [glslang](https://github.com/KhronosGroup/glslang/releases) (`glslang` or `glslangValidator`) on `PATH`
+- Python 3 on `PATH`; the bundled SPIRV-Tools runs it at configure time
 
 The Microsoft C++ compiler (`cl.exe`) is not supported; use `clang-cl`.
 
@@ -176,7 +177,7 @@ build has no working sound and no gamepad hotplug:
 
 ```bash
 sudo apt-get install --no-install-recommends \
-  clang lld ninja-build cmake git glslang-tools pkg-config \
+  clang lld ninja-build cmake git glslang-tools python3 pkg-config \
   libgl1-mesa-dev libx11-dev libxcursor-dev libxext-dev libxfixes-dev \
   libxi-dev libxrandr-dev libxss-dev libxtst-dev libxkbcommon-dev \
   libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
@@ -251,7 +252,7 @@ Requirements:
 
 - An Apple Silicon Mac with Rosetta 2 installed (`softwareupdate --install-rosetta`)
 - Xcode (or the Command Line Tools)
-- Homebrew packages: `brew install cmake ninja glslang`
+- Homebrew packages: `brew install cmake ninja glslang python`
 - Qt 6 (Concurrent, Network, Widgets) with x86-64 support. The official Qt installation is
   universal and works; Homebrew's Qt is arm64-only and will not link
 

src/common/CMakeLists.txt

@@ -27,6 +27,7 @@ target_link_libraries(common
 		kyty_git_version
 		spdlog::spdlog
 		ZArchive::zarchive
+		nlohmann_json::nlohmann_json
 )
 
 get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES)

src/common/emulatorConfig.cpp

@@ -152,14 +152,26 @@ bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }
 
+bool SyncRawImageBuffersEnabled() {
+	return g_config->sync_raw_image_buffers;
+}
+
 bool TessellationEnabled() {
 	return g_config->tessellation_enabled;
 }
 
+bool TrophyEnabled() {
+	return g_config->trophy_enabled;
+}
+
 bool PlayGoHackEnabled() {
 	return g_config->playgo_hack_enabled;
 }
 
+bool SkipNoticeScreen() {
+	return g_config->skip_notice_screen;
+}
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled() {
 	return g_config->red_zone_protection_enabled;

src/common/emulatorConfig.h

@@ -48,7 +48,7 @@ struct ConfigOptions {
 	int32_t                user_id                     = DEFAULT_USER_ID;
 	std::string            audio_input_device;
 	std::optional<ControllerColor> controller_color;
-	uint32_t               controller_speaker_volume      = 100;
+	uint32_t               controller_speaker_volume      = 50;
 	uint32_t               controller_vibration_intensity = 100;
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
@@ -73,8 +73,11 @@ struct ConfigOptions {
 	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
+	bool                   sync_raw_image_buffers      = false;
 	bool                   tessellation_enabled        = false;
+	bool                   trophy_enabled              = true;
 	bool                   playgo_hack_enabled         = false;
+	bool                   skip_notice_screen          = false;
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	bool red_zone_protection_enabled = false;
 #endif
@@ -122,8 +125,11 @@ bool GpuAssistedValidationEnabled();
 
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
+bool SyncRawImageBuffersEnabled();
 bool TessellationEnabled();
+bool TrophyEnabled();
 bool PlayGoHackEnabled();
+bool SkipNoticeScreen();
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled();
 #endif

src/common/trophies.cpp

@@ -0,0 +1,453 @@
+#include "common/trophies.h"
+
+#include "common/file.h"
+
+#include <algorithm>
+#include <array>
+#include <charconv>
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <span>
+
+namespace Common::Trophies {
+namespace {
+
+using Json  = nlohmann::json;
+using Files = std::map<std::string_view, std::span<const std::byte>>;
+
+constexpr uint64_t MaxUnlocksSize = uint64_t {1} << 20u;
+constexpr uint64_t MaxPackageSize = uint64_t {1} << 30u;
+
+uint64_t ReadBigEndian(const std::byte* data, size_t size) {
+	uint64_t value = 0;
+	for (size_t i = 0; i < size; ++i) {
+		value = (value << 8u) | std::to_integer<uint8_t>(data[i]);
+	}
+	return value;
+}
+
+int ReadId(const Json& value) {
+	if (value.is_number_integer()) {
+		const auto number = value.get<int64_t>();
+		return number >= 0 && number <= INT32_MAX ? static_cast<int>(number) : -1;
+	}
+	if (!value.is_string()) {
+		return -1;
+	}
+	const auto& text        = value.get_ref<const std::string&>();
+	int         id          = -1;
+	const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), id);
+	return error == std::errc {} && end == text.data() + text.size() && id >= 0 ? id : -1;
+}
+
+std::optional<uint64_t> ReadUnsigned(const Json& value) {
+	if (value.is_number_unsigned()) {
+		return value.get<uint64_t>();
+	}
+	if (value.is_number_integer()) {
+		const auto number = value.get<int64_t>();
+		return number >= 0 ? std::optional<uint64_t>(static_cast<uint64_t>(number)) : std::nullopt;
+	}
+	if (!value.is_string()) {
+		return std::nullopt;
+	}
+	const auto& text        = value.get_ref<const std::string&>();
+	uint64_t    number      = 0;
+	const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), number);
+	return error == std::errc {} && end == text.data() + text.size()
+	           ? std::optional<uint64_t>(number)
+	           : std::nullopt;
+}
+
+Json ReadJson(const Files& files, std::string_view name) {
+	const auto entry = files.find(name);
+	if (entry == files.end()) {
+		return {};
+	}
+	return Json::parse(entry->second.begin(), entry->second.end(), nullptr, false);
+}
+
+bool ReadFiles(std::span<const std::byte> data, Files& files, uint64_t& size) {
+	if (data.size() < 0x40 || data.size() > MaxPackageSize ||
+	    ReadBigEndian(data.data(), 4) != 0xb228c60a || ReadBigEndian(data.data() + 4, 4) != 1) {
+		return false;
+	}
+	size             = ReadBigEndian(data.data() + 8, 8);
+	const auto count = ReadBigEndian(data.data() + 0x10, 4);
+	const auto toc   = ReadBigEndian(data.data() + 0x14, 4);
+	if (size < 0x40 || size > data.size() || count > 4096 || toc > size ||
+	    0x20 + count * 0x40 > size - toc) {
+		return false;
+	}
+	for (uint64_t i = 0; i < count; ++i) {
+		const auto*            entry = data.data() + toc + 0x20 + i * 0x40;
+		const auto*            name  = reinterpret_cast<const char*>(entry);
+		const std::string_view filename(name, std::find(name, name + 0x20, '\0') - name);
+		const auto             offset = ReadBigEndian(entry + 0x20, 8);
+		const auto             length = ReadBigEndian(entry + 0x28, 8);
+		if (offset > size || length > size - offset) {
+			return false;
+		}
+		files.emplace(filename, data.subspan(offset, length));
+	}
+	return true;
+}
+
+Package ParsePackage(std::span<const std::byte> data, int console_language) {
+	Files    files;
+	uint64_t size = 0;
+	if (!ReadFiles(data, files, size)) {
+		return {};
+	}
+	const auto conf = ReadJson(files, "tropconf.json");
+	if (!conf.is_object() || !conf.contains("trophies") || !conf["trophies"].is_array() ||
+	    conf["trophies"].size() > 1000 || !conf.contains("defaultLanguage") ||
+	    !conf["defaultLanguage"].is_string()) {
+		return {};
+	}
+	static constexpr std::array<std::string_view, 30> locales = {
+	    "ja-JP", "en-US", "fr-FR",   "es-ES",   "de-DE",  "it-IT", "nl-NL", "pt-PT",
+	    "ru-RU", "ko-KR", "zh-Hant", "zh-Hans", "fi-FI",  "sv-SE", "da-DK", "no-NO",
+	    "pl-PL", "pt-BR", "en-GB",   "tr-TR",   "es-419", "ar-AE", "fr-CA", "cs-CZ",
+	    "hu-HU", "el-GR", "ro-RO",   "th-TH",   "vi-VN",  "id-ID"};
+	const auto locale =
+	    locales[console_language >= 0 && console_language < locales.size() ? console_language : 1];
+	auto meta = ReadJson(files, fmt::format("tropmeta_{}.json", locale));
+	if (meta.is_null()) {
+		meta = ReadJson(files, fmt::format("tropmeta_{}.json",
+		                                   conf["defaultLanguage"].get_ref<const std::string&>()));
+	}
+	if (!meta.is_object() || !meta.contains("metadata") || !meta["metadata"].is_object()) {
+		return {};
+	}
+	const auto& texts = meta["metadata"];
+	if (!texts.contains("titleMetadata") || !texts["titleMetadata"].is_object() ||
+	    !texts["titleMetadata"].contains("name") || !texts["titleMetadata"]["name"].is_string() ||
+	    !texts.contains("trophyMetadata") || !texts["trophyMetadata"].is_array()) {
+		return {};
+	}
+	Package package;
+	package.title = texts["titleMetadata"]["name"].get<std::string>();
+	package.groups.emplace(-1, package.title);
+	uint64_t icon_bytes = 0;
+	for (const auto& definition: conf["trophies"]) {
+		if (!definition.is_object() || !definition.contains("id") ||
+		    !definition.contains("grade") || !definition["grade"].is_string()) {
+			return {};
+		}
+		Trophy trophy;
+		trophy.id              = ReadId(definition["id"]);
+		const auto grade       = definition["grade"].get<std::string>();
+		const auto grade_index = std::string_view("PGSB").find(grade);
+		if (trophy.id < 0 || grade.size() != 1 || grade_index == std::string_view::npos) {
+			return {};
+		}
+		trophy.grade = static_cast<int>(grade_index) + 1;
+		if (definition.contains("groupId")) {
+			trophy.group_id = ReadId(definition["groupId"]);
+			if (trophy.group_id < 0) {
+				return {};
+			}
+		}
+		if (definition.contains("platinumTrophyId")) {
+			trophy.platinum_id = ReadId(definition["platinumTrophyId"]);
+		}
+		trophy.hidden     = definition.contains("hidden") && definition["hidden"] == true;
+		trophy.has_reward = definition.contains("hasReward") && definition["hasReward"] == true;
+		if (definition.contains("unlockCondition")) {
+			const auto& condition = definition["unlockCondition"];
+			if (condition.is_object()) {
+				trophy.progressive =
+				    condition.contains("progressive") && condition["progressive"] == true;
+				if (condition.contains("udsStatId")) {
+					trophy.uds_stat_id = ReadUnsigned(condition["udsStatId"]);
+					if (!trophy.uds_stat_id) {
+						return {};
+					}
+				}
+				if (condition.contains("targetValue")) {
+					trophy.target = ReadUnsigned(condition["targetValue"]);
+				}
+				const auto comparison = condition.find("comparator");
+				if (comparison != condition.end()) {
+					if (*comparison == "ge")
+						trophy.comparison = Comparison::GreaterEqual;
+					else if (*comparison == "gt")
+						trophy.comparison = Comparison::Greater;
+					else if (*comparison == "le")
+						trophy.comparison = Comparison::LessEqual;
+					else if (*comparison == "lt")
+						trophy.comparison = Comparison::Less;
+				}
+			}
+		}
+		const auto icon = files.find(fmt::format("trop{:04}.png", trophy.id));
+		if (icon != files.end()) {
+			if (icon->second.size() > size - icon_bytes) {
+				return {};
+			}
+			trophy.icon_png.assign(icon->second.begin(), icon->second.end());
+			icon_bytes += icon->second.size();
+		}
+		package.groups.try_emplace(trophy.group_id);
+		if (package.groups.size() > 50 ||
+		    !package.trophies.emplace(trophy.id, std::move(trophy)).second) {
+			return {};
+		}
+	}
+	if (texts.contains("groupMetadata") && texts["groupMetadata"].is_array()) {
+		for (const auto& group: texts["groupMetadata"]) {
+			if (group.is_object() && group.contains("id") && group.contains("name") &&
+			    group["name"].is_string()) {
+				const auto found = package.groups.find(ReadId(group["id"]));
+				if (found != package.groups.end()) {
+					found->second = group["name"].get<std::string>();
+				}
+			}
+		}
+	}
+	for (const auto& text: texts["trophyMetadata"]) {
+		if (!text.is_object() || !text.contains("id")) {
+			continue;
+		}
+		const auto found = package.trophies.find(ReadId(text["id"]));
+		if (found == package.trophies.end()) {
+			continue;
+		}
+		for (const auto& [key, field]: {std::pair {"name", &found->second.name},
+		                                {"detail", &found->second.description},
+		                                {"reward", &found->second.reward}}) {
+			if (text.contains(key) && text[key].is_string()) {
+				*field = text[key].get<std::string>();
+			}
+		}
+	}
+	return package;
+}
+
+} // namespace
+
+Package LoadPackage(const std::filesystem::path& path, int console_language) {
+	File file(path, File::Mode::Read);
+	if (file.IsInvalid() || file.Size() > MaxPackageSize) {
+		return {};
+	}
+	return ParsePackage(file.ReadWholeBuffer(), console_language);
+}
+
+std::filesystem::path PackagePath(uint32_t service_label) {
+	return std::filesystem::path(PackageDirectory) / fmt::format("trophy{:02}.ucp", service_label);
+}
+
+std::filesystem::path UnlocksPath(const std::filesystem::path& root, std::string_view title_id,
+                                  int user_id, uint32_t service_label) {
+	if (title_id.empty() || !std::all_of(title_id.begin(), title_id.end(), [](char c) {
+		    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
+		           c == '_' || c == '-';
+	    })) {
+		return {};
+	}
+	return root / "_Trophies" / title_id /
+	       fmt::format("trophies_{}_{}.json", user_id, service_label);
+}
+
+UnlockData LoadUnlockData(const std::filesystem::path& path) {
+	File file(path, File::Mode::Read);
+	if (file.IsInvalid() || file.Size() > MaxUnlocksSize) {
+		return {};
+	}
+	const auto bytes = file.ReadWholeBuffer();
+	const auto json  = Json::parse(bytes.begin(), bytes.end(), nullptr, false);
+	if (!json.is_object() || !json.contains("unlockedTrophies") ||
+	    !json["unlockedTrophies"].is_array()) {
+		return {};
+	}
+	UnlockData unlocks;
+	for (const auto& value: json["unlockedTrophies"]) {
+		const auto id = ReadId(value);
+		if (id >= 0) {
+			unlocks.unlocked.insert(id);
+		}
+	}
+	if (json.contains("unlockedAt") && json["unlockedAt"].is_object()) {
+		for (auto it = json["unlockedAt"].begin(); it != json["unlockedAt"].end(); ++it) {
+			const auto id = ReadId(Json(it.key()));
+			if (id >= 0 && unlocks.unlocked.contains(id)) {
+				const auto tick = ReadUnsigned(it.value());
+				if (tick && *tick >= UnixEpochTick && *tick <= 315537897599999999ULL) {
+					unlocks.timestamps.emplace(id, *tick);
+				}
+			}
+		}
+	}
+	return unlocks;
+}
+
+bool SaveUnlockData(const std::filesystem::path& path, const UnlockData& unlocks) {
+	if (path.empty() || !File::CreateDirectories(path.parent_path())) {
+		return false;
+	}
+	Json timestamps = Json::object();
+	for (const auto& [id, tick]: unlocks.timestamps) {
+		if (unlocks.unlocked.contains(id)) {
+			timestamps[std::to_string(id)] = tick;
+		}
+	}
+	const auto text =
+	    Json {{"unlockedTrophies", unlocks.unlocked}, {"unlockedAt", timestamps}}.dump();
+	auto temporary = path;
+	temporary += ".tmp";
+	File file;
+	if (text.size() > MaxUnlocksSize || !file.Create(temporary)) {
+		return false;
+	}
+	uint32_t written = 0;
+	file.Write(text.data(), static_cast<uint32_t>(text.size()), &written);
+	const bool complete = written == text.size() && file.Flush();
+	file.Close();
+	std::error_code error;
+	if (complete) {
+		std::filesystem::rename(temporary, path, error);
+		if (!error) {
+			return true;
+		}
+	}
+	std::filesystem::remove(temporary, error);
+	return false;
+}
+
+UdsRules LoadUdsRules(const std::filesystem::path& path) {
+	File file(path, File::Mode::Read);
+	if (file.IsInvalid() || file.Size() > MaxPackageSize) {
+		return {};
+	}
+	const auto bytes = file.ReadWholeBuffer();
+	Files      files;
+	uint64_t   size = 0;
+	if (!ReadFiles(bytes, files, size)) {
+		return {};
+	}
+	const auto definitions = ReadJson(files, "stats_definition.json");
+	const auto extraction  = ReadJson(files, "stats_extraction.json");
+	if (!definitions.is_object() || !definitions.contains("statDefinitionArray") ||
+	    !definitions["statDefinitionArray"].is_array() || !extraction.is_object() ||
+	    !extraction.contains("statsExtractionRuleArray") ||
+	    !extraction["statsExtractionRuleArray"].is_array()) {
+		return {};
+	}
+	// Only direct unsigned latest-value stats are evaluated here. Aggregated stats
+	// and filtered/nested extraction require UDS state and event types we do not implement.
+	std::map<uint64_t, bool> stats;
+	for (const auto& definition: definitions["statDefinitionArray"]) {
+		if (!definition.is_object() || !definition.contains("statId") ||
+		    !definition.contains("aggregation") || definition["aggregation"] != "latest" ||
+		    !definition.contains("dataType")) {
+			continue;
+		}
+		const auto  id   = ReadUnsigned(definition["statId"]);
+		const auto& type = definition["dataType"];
+		if (!id || (type != "uint64" && type != "uint32")) {
+			continue;
+		}
+		const uint64_t maximum = type == "uint32" ? UINT32_MAX : UINT64_MAX;
+		if ((definition.contains("minValue") && ReadUnsigned(definition["minValue"]) != 0) ||
+		    (definition.contains("maxValue") && ReadUnsigned(definition["maxValue"]) != maximum)) {
+			continue;
+		}
+		stats.emplace(*id, type == "uint32");
+	}
+	UdsRules events;
+	for (const auto& rule: extraction["statsExtractionRuleArray"]) {
+		if (!rule.is_object() || !rule.contains("condition") || !rule.contains("action")) {
+			continue;
+		}
+		const auto& condition = rule["condition"];
+		const auto& action    = rule["action"];
+		if (!condition.is_object() || condition.size() != 1 || !condition.contains("eventName") ||
+		    !condition["eventName"].is_string() || !action.is_object() ||
+		    !action.contains("input") || !action["input"].is_string() ||
+		    !action.contains("output") || !action["output"].is_object() ||
+		    !action["output"].contains("statId")) {
+			continue;
+		}
+		const auto  id    = ReadUnsigned(action["output"]["statId"]);
+		const auto& input = action["input"].get_ref<const std::string&>();
+		if (!id || !stats.contains(*id) || !input.starts_with("$.") || input.size() == 2 ||
+		    input.find_first_not_of(
+		        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_", 2) !=
+		        std::string::npos) {
+			continue;
+		}
+		events[condition["eventName"].get<std::string>()].push_back(
+		    {*id, input.substr(2), stats.at(*id)});
+	}
+	return events;
+}
+
+std::filesystem::path UdsPackagePath(uint32_t service_label) {
+	return std::filesystem::path("sce_sys/uds") / fmt::format("uds{:02}.ucp", service_label);
+}
+
+std::vector<int> FindUdsTrophies(const Package& package, std::string_view event_name,
+                                 const std::map<std::string, UdsInteger>& properties) {
+	const auto rules = package.event_rules.find(std::string(event_name));
+	if (rules == package.event_rules.end()) {
+		return {};
+	}
+	std::vector<int> ids;
+	for (const auto& [id, trophy]: package.trophies) {
+		if (trophy.grade == 1 || !trophy.uds_stat_id || !trophy.target) {
+			continue;
+		}
+		for (const auto& rule: rules->second) {
+			const auto value = properties.find(rule.input);
+			if (rule.stat_id != *trophy.uds_stat_id || value == properties.end()) {
+				continue;
+			}
+			std::optional<uint64_t> number;
+			if (rule.is_uint32) {
+				if (const auto* input = std::get_if<uint32_t>(&value->second)) number = *input;
+			} else if (const auto* input = std::get_if<uint64_t>(&value->second)) {
+				number = *input;
+			}
+			if (!number) continue;
+			bool matched = false;
+			switch (trophy.comparison) {
+				case Comparison::GreaterEqual: matched = *number >= *trophy.target; break;
+				case Comparison::Greater: matched = *number > *trophy.target; break;
+				case Comparison::LessEqual: matched = *number <= *trophy.target; break;
+				case Comparison::Less: matched = *number < *trophy.target; break;
+				case Comparison::None: break;
+			}
+			if (matched) {
+				ids.push_back(id);
+				break;
+			}
+		}
+	}
+	return ids;
+}
+
+uint32_t Progress::Percentage() const {
+	const auto points        = total_grade[2] * 6 + total_grade[3] * 2 + total_grade[4];
+	const auto earned_points = earned_grade[2] * 6 + earned_grade[3] * 2 + earned_grade[4];
+	return points == 0 ? 0 : earned_points * 100 / points;
+}
+
+Progress GetProgress(const Package& package, const UnlockData& unlocks, std::optional<int> group) {
+	Progress progress;
+	for (const auto& [id, trophy]: package.trophies) {
+		if (group && trophy.group_id != *group) {
+			continue;
+		}
+		++progress.total;
+		++progress.total_grade[trophy.grade];
+		if (unlocks.unlocked.contains(id)) {
+			++progress.earned;
+			++progress.earned_grade[trophy.grade];
+		}
+	}
+	return progress;
+}
+
+} // namespace Common::Trophies

src/common/trophies.h

@@ -0,0 +1,87 @@
+#ifndef KYTY_COMMON_TROPHIES_H_
+#define KYTY_COMMON_TROPHIES_H_
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <string_view>
+#include <variant>
+#include <vector>
+
+namespace Common::Trophies {
+
+inline constexpr char     PackageDirectory[] = "sce_sys/trophy2";
+inline constexpr uint64_t UnixEpochTick      = 62135596800000000ULL;
+
+enum class Comparison { None, GreaterEqual, Greater, LessEqual, Less };
+
+struct Trophy {
+	int                     id          = 0;
+	int                     group_id    = -1;
+	int                     platinum_id = -1;
+	int                     grade       = 0;
+	std::optional<uint64_t> target;
+	std::optional<uint64_t> uds_stat_id;
+	Comparison              comparison  = Comparison::None;
+	bool                    progressive = false;
+	std::string             name;
+	std::string             description;
+	std::string             reward;
+	std::vector<std::byte>  icon_png;
+	bool                    hidden     = false;
+	bool                    has_reward = false;
+};
+
+struct UdsRule {
+	uint64_t    stat_id;
+	std::string input;
+	bool        is_uint32;
+};
+
+using UdsRules   = std::map<std::string, std::vector<UdsRule>>;
+using UdsInteger = std::variant<int32_t, uint32_t, uint64_t>;
+
+struct Package {
+	std::string                title;
+	std::map<int, std::string> groups;
+	std::map<int, Trophy>      trophies;
+	UdsRules                   event_rules;
+};
+
+struct UnlockData {
+	std::set<int>           unlocked;
+	std::map<int, uint64_t> timestamps;
+};
+
+struct Progress {
+	std::array<uint32_t, 5> total_grade {};
+	std::array<uint32_t, 5> earned_grade {};
+	uint32_t                total  = 0;
+	uint32_t                earned = 0;
+
+	[[nodiscard]] uint32_t Percentage() const;
+};
+
+[[nodiscard]] Progress GetProgress(const Package& package, const UnlockData& unlocks,
+                                   std::optional<int> group = std::nullopt);
+
+[[nodiscard]] Package LoadPackage(const std::filesystem::path& path, int console_language);
+[[nodiscard]] std::filesystem::path PackagePath(uint32_t service_label);
+[[nodiscard]] std::filesystem::path UdsPackagePath(uint32_t service_label);
+[[nodiscard]] UdsRules              LoadUdsRules(const std::filesystem::path& path);
+[[nodiscard]] std::vector<int> FindUdsTrophies(const Package& package, std::string_view event_name,
+                                               const std::map<std::string, UdsInteger>& properties);
+[[nodiscard]] std::filesystem::path UnlocksPath(const std::filesystem::path& root,
+                                                std::string_view title_id, int user_id,
+                                                uint32_t service_label);
+[[nodiscard]] UnlockData            LoadUnlockData(const std::filesystem::path& path);
+[[nodiscard]] bool SaveUnlockData(const std::filesystem::path& path, const UnlockData& unlocks);
+
+} // namespace Common::Trophies
+
+#endif // KYTY_COMMON_TROPHIES_H_

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -23,6 +23,12 @@ enum class ContextStateOperation : uint32_t {
 	PushClear = 3,
 };
 
+struct IndirectDrawRegisters {
+	uint32_t vertex_offset;
+	uint32_t instance_offset;
+	uint32_t index_offset;
+};
+
 class Pm4Execution {
 public:
 	[[nodiscard]] bool MadeProgress() const noexcept { return m_made_progress; }
@@ -79,10 +85,11 @@ public:
 	void DrawIndex(DrawIndexArgs args);
 	void DrawIndexOffset(uint32_t index_offset, uint32_t index_count);
 	void DrawIndexAuto(DrawAutoArgs args);
-	void DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed);
+	void DrawIndirect(uint32_t data_offset, IndirectDrawRegisters registers,
+	                  uint32_t draw_initiator, bool indexed);
 	void DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
 	                       const volatile uint32_t* count_addr, uint32_t stride_in_bytes,
-	                       uint32_t draw_initiator, bool indexed);
+	                       IndirectDrawRegisters registers, uint32_t draw_initiator, bool indexed);
 	void WriteAtEndOfPipe32(uint32_t cache_policy, uint32_t event_write_dest,
 	                        uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index,
 	                        uint32_t event_write_source, void* dst_gpu_addr, uint32_t value,

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -184,7 +184,6 @@ static HW::RenderControl DecodeRenderControl(uint32_t value) {
 	    KYTY_PM4_GET(value, DB_RENDER_CONTROL, COPY_STENCIL_TO_COLOR) != 0;
 	r.copy_centroid          = KYTY_PM4_GET(value, DB_RENDER_CONTROL, COPY_CENTROID) != 0;
 	r.copy_sample            = KYTY_PM4_GET(value, DB_RENDER_CONTROL, COPY_SAMPLE);
-	EXIT_NOT_IMPLEMENTED(r.copy_depth_to_color || r.copy_stencil_to_color);
 
 	return r;
 }
@@ -1642,8 +1641,12 @@ KYTY_CP_OP_PARSER(CpOpDrawIndirect) {
 	const auto data_offset    = buffer[0];
 	const auto draw_initiator = buffer[3];
 	const bool indexed        = (cmd_id == 0xc0032500);
+	// The native indexed packet enables the high-half first-index destination in word 3.
+	const IndirectDrawRegisters registers {
+	    buffer[1] & 0xffffu, buffer[2] & 0xffffu,
+	    indexed && (buffer[2] & (1u << 28u)) != 0 ? buffer[1] >> 16u : Pm4::SH_NOP};
 
-	cp.DrawIndirect(data_offset, draw_initiator, indexed);
+	cp.DrawIndirect(data_offset, registers, draw_initiator, indexed);
 
 	return 4;
 }
@@ -1661,13 +1664,17 @@ KYTY_CP_OP_PARSER(CpOpDrawIndirectMulti) {
 	const auto stride_in_bytes = buffer[7];
 	const auto draw_initiator  = buffer[8];
 	const bool indexed         = (cmd_id == 0xc0083800);
+	// Multi-draw moves the first-index enable to word 4; the destinations stay in 2/3.
+	const IndirectDrawRegisters registers {
+	    buffer[1] & 0xffffu, buffer[2] & 0xffffu,
+	    indexed && (buffer[3] & (1u << 28u)) != 0 ? buffer[1] >> 16u : Pm4::SH_NOP};
 
 	if (count_indirect == 0) {
 		count_addr = nullptr;
 	}
 
 	cp.DrawIndirectMulti(data_offset, max_count_or_count, count_addr, stride_in_bytes,
-	                     draw_initiator, indexed);
+	                     registers, draw_initiator, indexed);
 
 	return 9;
 }

src/graphics/guest_gpu/gpu_format.cpp

@@ -20,7 +20,8 @@ struct FormatInfo {
 
 constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k8UNorm, 1, 0, 1, true, false},
-	{BufferFormat::k8SNorm, 0, 0, 1, false, false},
+	{BufferFormat::k8UScaled, 1, 0, 0, true, false},
+	{BufferFormat::k8SNorm, 1, 0, 1, false, false},
 	{BufferFormat::k8UInt, 1, 0, 1, true, true},
 	{BufferFormat::k16UNorm, 2, 0, 2, true, false},
 	{BufferFormat::k16SNorm, 2, 0, 2, true, false},
@@ -28,6 +29,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k16SInt, 2, 0, 2, true, false, true},
 	{BufferFormat::k16Float, 2, 0, 2, true, false},
 	{BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UScaled, 2, 0, 0, true, false},
 	{BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
 	{BufferFormat::k8_8UInt, 2, 0, 2, true, true},
 	{BufferFormat::k8_8SInt, 2, 0, 2, true, false, true},
@@ -41,6 +43,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k16_16Float, 4, 0, 4, true, false},
 	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
 	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
+	{BufferFormat::k10_11_11Float, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
 	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
@@ -67,7 +70,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
 	{BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
 	{BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
-	{BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
+	{BufferFormat::k1_5_5_5UNorm, 2, 0, 2, false, false},
 	{BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
 	{BufferFormat::kFmask8_S2_F1, 1, 0, 1, true, false},
 	{BufferFormat::kFmask8_S4_F1, 1, 0, 1, true, false},
@@ -231,7 +234,12 @@ TextureNumericClass SampledTextureNumericClass(BufferFormat format) {
 }
 
 BufferFormat RemapTextureFormat(BufferFormat format) {
-	return format == BufferFormat::k11_11_10UInt ? BufferFormat::k32UInt : format;
+	switch (format) {
+		case BufferFormat::k8UScaled: return BufferFormat::k8UNorm;
+		case BufferFormat::k8_8UScaled: return BufferFormat::k8_8UNorm;
+		case BufferFormat::k11_11_10UInt: return BufferFormat::k32UInt;
+		default: return format;
+	}
 }
 
 } // namespace Libs::Graphics::Prospero

src/graphics/guest_gpu/graphicsRun.cpp

@@ -689,7 +689,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution) {
 		const auto        opcode        = (packet_header >> 8u) & 0xffu;
 		EXIT_NOT_IMPLEMENTED(remaining_dw > total_dw);
 
-		if (packet_header == 0x80000000u) {
+		if (packet_header == 0x80000000u || Pm4::PacketSizeDw(packet_header) == 1u) {
 			cursor.offset_dw++;
 			execution.m_made_progress = true;
 			continue;
@@ -875,65 +875,33 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou
 	DrawIndex({.index_count = index_count, .index_addr = index_addr});
 }
 
-void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed) {
-	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u);
-	EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
-
-	const auto* args_addr =
-	    reinterpret_cast<const void*>(m_draw_indirect_args_base_addr + data_offset);
-
-	if (!indexed) {
-		DrawIndirectArgs args {};
-		std::memcpy(&args, args_addr, sizeof(args));
-		m_num_instances = args.instance_count;
-		DrawIndexAuto({.vertex_count   = args.vertex_count_per_instance,
-		               .instance_count = args.instance_count,
-		               .first_vertex   = args.start_vertex_location,
-		               .first_instance = args.start_instance_location,
-		               .offset_source  = DrawOffsetSource::IndirectArgs});
-		return;
-	}
-
-	DrawIndexedIndirectArgs args {};
-	std::memcpy(&args, args_addr, sizeof(args));
-
-	uint64_t index_size = 0;
-	switch (m_index_type_and_size) {
-		case 0: index_size = 2; break;
-		case 1: index_size = 4; break;
-		case 2: index_size = 1; break;
-		default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
-	}
-
-	auto* index_addr = reinterpret_cast<const void*>(
-	    m_index_base_addr + static_cast<uint64_t>(args.start_index_location) * index_size);
-
-	const uint32_t index_count =
-	    (m_index_buffer_size != 0 ? std::min(args.index_count_per_instance, m_index_buffer_size)
-	                              : args.index_count_per_instance);
-	if (GraphicsRunDebugDumpEnabled() && index_count != args.index_count_per_instance) {
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
-			LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32
-			     " using INDEX_BUFFER_SIZE\n",
-			     args.index_count_per_instance, index_count);
+static void PatchIndirectDrawOffsets(CommandProcessor& cp, IndirectDrawRegisters registers,
+                                     uint32_t vertex_offset, uint32_t instance_offset,
+                                     uint32_t first_index) {
+	const auto write = [&](uint32_t location, uint32_t value) {
+		if (location == Pm4::SH_NOP) {
+			return;
 		}
-	}
+		EXIT_NOT_IMPLEMENTED(location >= Pm4::SH_NUM ||
+		                     g_hw_sh_indirect_func[location] == nullptr);
+		g_hw_sh_indirect_func[location](cp, location, value);
+	};
+	write(registers.vertex_offset, vertex_offset);
+	write(registers.instance_offset, instance_offset);
+	write(registers.index_offset, first_index);
+}
 
-	m_num_instances = args.instance_count;
-	DrawIndex({.index_count    = index_count,
-	           .index_addr     = index_addr,
-	           .instance_count = args.instance_count,
-	           .base_vertex    = static_cast<int32_t>(args.base_vertex_location),
-	           .first_instance = args.start_instance_location,
-	           .offset_source  = DrawOffsetSource::IndirectArgs});
+void CommandProcessor::DrawIndirect(uint32_t data_offset, IndirectDrawRegisters registers,
+                                    uint32_t draw_initiator, bool indexed) {
+	const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs);
+	DrawIndirectMulti(data_offset, 1, nullptr, args_size, registers, draw_initiator, indexed);
 }
 
 void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
                                          const volatile uint32_t* count_addr,
-                                         uint32_t stride_in_bytes, uint32_t draw_initiator,
-                                         bool indexed) {
-	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u);
+                                         uint32_t stride_in_bytes, IndirectDrawRegisters registers,
+                                         uint32_t draw_initiator, bool indexed) {
+	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != (indexed ? 0u : 2u));
 	EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
 
 	uint32_t draw_count = max_count_or_count;
@@ -967,6 +935,8 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 		if (!indexed) {
 			auto* args = reinterpret_cast<const DrawIndirectArgs*>(args_addr);
+			PatchIndirectDrawOffsets(*this, registers, args->start_vertex_location,
+			                         args->start_instance_location, 0);
 			m_num_instances = args->instance_count;
 			DrawIndexAuto({.vertex_count   = args->vertex_count_per_instance,
 			               .instance_count = args->instance_count,
@@ -977,6 +947,8 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 		}
 
 		auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr);
+		PatchIndirectDrawOffsets(*this, registers, args->base_vertex_location,
+		                         args->start_instance_location, args->start_index_location);
 
 		auto* index_addr = reinterpret_cast<const void*>(
 		    m_index_base_addr + static_cast<uint64_t>(args->start_index_location) * index_size);
@@ -1031,7 +1003,7 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 		}
 
 		m_renderer.GetRenderExecutor().DispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x,
-		                                              thread_group_y, thread_group_z, mode);
+		                                              thread_group_y, thread_group_z, mode, IsAsyncComputeQueue());
 	}
 }
 
@@ -1043,7 +1015,8 @@ void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {
 		return;
 	}
 	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
-	m_renderer.GetRenderExecutor().DispatchIndirect(m_submit_id, CurrentBuffer(), args_addr, mode);
+	m_renderer.GetRenderExecutor().DispatchIndirect(m_submit_id, CurrentBuffer(), args_addr, mode,
+	                                                IsAsyncComputeQueue());
 }
 
 void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {

src/graphics/guest_gpu/pm4.cpp

@@ -120,15 +120,15 @@ void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_
 		uint32_t len = 0;
 
 		const auto packet_type = static_cast<PacketType>(cmd_id >> 30u);
-		// Type-2 packets are header-only padding; every other packet type requires a body.
-		EXIT_NOT_IMPLEMENTED(dw < 2 && packet_type != PacketType::Type2);
+		EXIT_NOT_IMPLEMENTED(dw < 2 && packet_type != PacketType::Type2 &&
+		                     PacketSizeDw(cmd_id) != 1u);
 
 		switch (packet_type) {
 			case PacketType::Type3: {
 				const bool    sh_gx = (cmd_id & 0x2u) == 0;
 				const uint8_t op    = ((cmd_id >> 8u) & 0xffu);
 				const auto    r     = KYTY_PM4_R(cmd_id);
-				len                 = ((cmd_id >> 16u) & 0x3fffu) + 1;
+				len                 = PacketSizeDw(cmd_id) - 1u;
 
 				EXIT_NOT_IMPLEMENTED(len >= dw);
 

src/graphics/guest_gpu/pm4.h

@@ -19,6 +19,10 @@ class File;
 
 namespace Libs::Graphics::Pm4 {
 
+constexpr uint32_t PacketSizeDw(uint32_t header) {
+	return (header & 0x3fffff00u) == 0x3fff1000u ? 1u : KYTY_PM4_LEN(header);
+}
+
 constexpr uint32_t IT_NOP                       = 0x10;
 constexpr uint32_t IT_SET_BASE                  = 0x11;
 constexpr uint32_t IT_CLEAR_STATE               = 0x12;

src/graphics/host_gpu/memoryTracker.h

@@ -82,9 +82,8 @@ public:
 		static_assert(std::is_nothrow_invocable_v<RangeFunc&, uint64_t, uint64_t>);
 		static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
 		CheckNotInUploadCallback();
-		Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
 		const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
-		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+		Iterate<true>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 			manager->lock.lock();
 			manager->ForEachModifiedRange<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset,
 			                                                      bytes, range_func);

src/graphics/host_gpu/pageManager.cpp

@@ -15,6 +15,12 @@
 #include <mutex>
 #include <vector>
 
+#if defined(_MSC_VER)
+#include <intrin.h>
+#elif defined(__x86_64__)
+#include <xmmintrin.h>
+#endif
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #ifndef NOMINMAX
 #define NOMINMAX
@@ -62,7 +68,13 @@ class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
 		while (m_lock.test_and_set(std::memory_order_acquire)) {
-			std::atomic_signal_fence(std::memory_order_seq_cst);
+#if defined(__x86_64__) || defined(_M_X64)
+			_mm_pause();
+#elif defined(_M_ARM64)
+			__yield();
+#elif defined(__aarch64__)
+			asm volatile("yield");
+#endif
 		}
 	}
 	~SpinGuard() { m_lock.clear(std::memory_order_release); }

src/graphics/host_gpu/regionManager.h

@@ -9,6 +9,12 @@
 #include <mutex>
 #include <utility>
 
+#if defined(_MSC_VER)
+#include <intrin.h>
+#elif defined(__x86_64__)
+#include <xmmintrin.h>
+#endif
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #ifndef NOMINMAX
 #define NOMINMAX
@@ -32,7 +38,13 @@ public:
 		const auto thread = CurrentThread();
 		while (m_lock.test_and_set(std::memory_order_acquire)) {
 			EXIT_NOT_IMPLEMENTED(m_owner.load(std::memory_order_relaxed) == thread);
-			std::atomic_signal_fence(std::memory_order_seq_cst);
+#if defined(__x86_64__) || defined(_M_X64)
+			_mm_pause();
+#elif defined(_M_ARM64)
+			__yield();
+#elif defined(__aarch64__)
+			asm volatile("yield");
+#endif
 		}
 		m_owner.store(thread, std::memory_order_relaxed);
 	}
@@ -48,7 +60,9 @@ private:
 		return GetCurrentThreadId();
 #elif defined(__APPLE__)
 		// mach thread port is a nonzero per-thread id (0 is the "no owner" sentinel).
-		return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
+		static thread_local const uint32_t tid =
+		    static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
+		return tid;
 #elif defined(__linux__)
 		static thread_local const uint32_t tid = static_cast<uint32_t>(::syscall(SYS_gettid));
 		return tid;

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -34,9 +34,36 @@ void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void*
 	auto* bytes = static_cast<const uint8_t*>(source);
 	while (size != 0) {
 		const auto chunk  = std::min(size, m_staging_buffer.Size());
+		// Copy waits for ring reuse and flushes the fresh host data before submission.
 		const auto offset = m_staging_buffer.Copy(bytes, chunk, 4);
-		buffer.CopyFrom(m_scheduler.Current(), m_staging_buffer, offset, buffer.Offset(address),
-		                chunk, vk::AccessFlagBits::eHostWrite);
+		const auto destination_offset = buffer.Offset(address);
+		EXIT_IF(destination_offset > buffer.Size() || chunk > buffer.Size() - destination_offset);
+		m_scheduler.EndRendering();
+		const auto command = m_scheduler.Current().Handle();
+		vk::BufferMemoryBarrier2 before {};
+		before.srcStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
+		before.srcAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
+		before.dstStageMask  = vk::PipelineStageFlagBits2::eTransfer;
+		before.dstAccessMask = vk::AccessFlagBits2::eTransferWrite;
+		before.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+		before.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+		before.buffer        = buffer.Handle();
+		before.offset        = destination_offset;
+		before.size          = chunk;
+		vk::DependencyInfo dependency {};
+		dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
+		dependency.bufferMemoryBarrierCount = 1;
+		dependency.pBufferMemoryBarriers    = &before;
+		command.pipelineBarrier2(dependency);
+		const vk::BufferCopy copy {offset, destination_offset, chunk};
+		command.copyBuffer(m_staging_buffer.Handle(), buffer.Handle(), 1, &copy);
+		auto after          = before;
+		after.srcStageMask  = vk::PipelineStageFlagBits2::eTransfer;
+		after.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
+		after.dstStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
+		after.dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
+		dependency.pBufferMemoryBarriers = &after;
+		command.pipelineBarrier2(dependency);
 		bytes += chunk;
 		address += chunk;
 		size -= chunk;
@@ -258,15 +285,8 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		}
 		auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
 
-		// Widen nearby CPU reads so they share one GPU drain.
-		constexpr uint64_t WindowSize   = 512 * 1024;
-		const auto         buffer_begin = buffer.CpuAddress();
-		const auto         buffer_end   = buffer_begin + buffer.Size();
-		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
-		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
-
-		if (DownloadBufferMemory<false>(buffer, window_begin, window_end - window_begin)) {
-			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
+		if (DownloadBufferMemory<false>(buffer, vaddr, size)) {
+			m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
 		}
 		if (is_write) {
 			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -195,6 +195,9 @@ bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& request
 	if (cached.tile_mode != requested.tile_mode) {
 		return false;
 	}
+	if (cached.GetColorTransform() != requested.GetColorTransform()) {
+		return false;
+	}
 	if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format) ||
 	    cached.type != requested.type) {
 		return false;
@@ -634,6 +637,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 	const bool source_depth = source.info.IsDepth();
 	const bool dest_depth   = destination.info.IsDepth();
 	const bool direct_copy =
+	    source.info.GetColorTransform() == destination.info.GetColorTransform() &&
 	    (source.backing.image_type == destination.backing.image_type ||
 	     (source.backing.image_type != vk::ImageType::e1D &&
 	      destination.backing.image_type != vk::ImageType::e1D)) &&
@@ -647,7 +651,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 			EXIT("TextureCache: cross-format multisample image copy is unsupported\n");
 		}
 		auto& copy_buffer = m_buffer_cache.GetUtilityBuffer(MemoryUsage::DeviceLocal);
-		destination.CopyImageWithBuffer(source, copy_buffer);
+		destination.CopyImageWithBuffer(source, copy_buffer, m_tiler);
 	}
 	if (source.IsGpuModified()) {
 		destination.MarkGpuModified();
@@ -690,7 +694,9 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 	    requested.type == cached.info.type && requested.pitch == cached.info.pitch &&
 	    !requested.HasStencil() && !cached.info.HasStencil() && !requested.HasMetadata() &&
 	    !cached.info.HasMetadata();
-	// PPSA04264
+	// PPSA04264, PPSA04288
+	// A partial view retains the entire matching array layout. HTile belongs to
+	// the depth source, independently of the data slices copied into color storage.
 	const bool retain_cached_layout =
 	    requested.samples == 1 && cached.info.samples == 1 && cached.backing.samples == 1 &&
 	    requested.bytes_per_block == cached.info.bytes_per_block &&
@@ -709,8 +715,10 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 	    requested.data.size / requested.resources.layers ==
 	        cached.info.data.size / cached.info.resources.layers &&
 	    !requested.HasStencil() && !cached.info.HasStencil() && !requested.HasMetadata() &&
-	    !cached.info.HasMetadata();
-	bool recreate = cached.info.resources < requested.resources;
+	    (cached.info.metadata.kind == ImageMetadataKind::None ||
+	     cached.info.metadata.kind == ImageMetadataKind::Htile);
+	bool recreate = cached.info.resources < requested.resources ||
+	                requested.IsVolume() != cached.info.IsVolume();
 	switch (binding) {
 		case BindingType::Texture:
 			recreate |= requested.IsDepth() && !cached.info.IsDepth();
@@ -733,8 +741,6 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 		info.data       = cached.info.data;
 		info.resources  = cached.info.resources;
 		info.mip_layout = cached.info.mip_layout;
-	} else {
-		info.resources = std::max(requested.resources, cached.info.resources);
 	}
 	info.htile_clear_mask     = 0;
 	const auto replacement_id = InsertImage(info);
@@ -801,6 +807,9 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 			}
 			return {merged_id};
 		}
+		if (requested.GetColorTransform() != cached.info.GetColorTransform()) {
+			return {ExpandImage(requested, cached_id)};
+		}
 		if (requested.IsBlock() && !cached.info.IsBlock()) {
 			return {ExpandImage(requested, cached_id)};
 		}
@@ -836,6 +845,9 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		      requested.resources.layers == cached.info.resources.layers))) {
 			return {ExpandImage(requested, cached_id)};
 		}
+		if (requested.type != cached.info.type) {
+			return {merged_id};
+		}
 		if (requested.pixel_format != cached.info.pixel_format ||
 		    requested.data.size <= cached.info.data.size) {
 			const auto result_id = merged_id ? merged_id : cached_id;
@@ -905,8 +917,8 @@ struct TextureCache::TextureTransfer {
 	TextureUploadLayout              layout;
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
-	bool                             swap_bgra16 = false;
-	bool                             valid       = false;
+	ColorTransform                   color_transform = ColorTransform::None;
+	bool                             valid           = false;
 
 	[[nodiscard]] uint64_t LinearSize() const {
 		uint64_t size = 0;
@@ -935,7 +947,7 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 	const char* owner            = "TextureCache readback";
 
 	TextureTransfer transfer;
-	transfer.swap_bgra16 = info.bgra16;
+	transfer.color_transform = info.GetColorTransform();
 	if (upload) {
 		if ((render_target || video_out) &&
 		    (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
@@ -1023,10 +1035,9 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
 		if (!transfer.tiles.empty()) {
 			linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size,
-			                        transfer.LinearSize(), transfer.tiles);
-		}
-		if (transfer.swap_bgra16) {
-			linear = m_tiler.SwapBgra16(linear);
+			                        transfer.LinearSize(), transfer.tiles, transfer.color_transform);
+		} else if (transfer.color_transform != ColorTransform::None) {
+			linear = m_tiler.TransformColor(linear, transfer.color_transform, true);
 		}
 		upload(transfer.regions, linear);
 		return;
@@ -1037,6 +1048,8 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	auto info = destination.info;
 	if (image.depth_id) {
 		info.data            = image.info.data;
+		info.resources       = {image.stencil_subresources.level_count,
+		                        image.stencil_subresources.layer_count};
 		info.guest_format    = Prospero::BufferFormat::k8UInt;
 		info.bytes_per_block = 1;
 		if (info.IsTiled()) info.pitch = TileGetDepthPitch(info.extent.width, 1, 0);
@@ -1051,6 +1064,11 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	auto copies = BuildDepthCopies(info, full_slice_size, image.depth_id
 	                                                        ? vk::ImageAspectFlagBits::eStencil
 	                                                        : vk::ImageAspectFlagBits::eDepth);
+	if (image.depth_id) {
+		for (auto& copy: copies) {
+			copy.imageSubresource.baseArrayLayer += image.stencil_subresources.base_layer;
+		}
+	}
 	TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset};
 	if (info.IsTiled()) {
 		const auto tiles = BuildDepthTiles(info);
@@ -1239,7 +1257,8 @@ ImageId TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
 	}
 	auto& record = m_slot_images[association];
 	TouchImage(record);
-	record.depth_id = depth_id;
+	record.depth_id             = depth_id;
+	record.stencil_subresources = depth.stencil_subresources;
 	return association;
 }
 
@@ -1445,6 +1464,18 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	TouchImage(image);
 	image.MarkGpuModified();
 	image.usage.depth_target = true;
+	if (desc.info.HasStencil()) {
+		const auto layers = image.info.resources.layers;
+		EXIT_IF(layers == 0 || image.info.data.size % layers != 0 ||
+		        desc.info.data.address < image.info.data.address);
+		const auto slice_size = image.info.data.size / layers;
+		const auto offset     = desc.info.data.address - image.info.data.address;
+		EXIT_IF(slice_size == 0 || offset % slice_size != 0 || offset / slice_size >= layers ||
+		        desc.info.resources.layers > layers - offset / slice_size);
+		image.stencil_subresources = {0, desc.info.resources.levels,
+		                             static_cast<uint32_t>(offset / slice_size),
+		                             desc.info.resources.layers};
+	}
 	image.info.stencil = desc.info.stencil;
 	image.info.metadata = desc.info.metadata;
 	if (desc.info.HasMetadata()) {
@@ -1491,9 +1522,11 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid image clear\n");
 	}
-	std::scoped_lock     lock {m_lock};
-	ImageId              selected {};
-	vk::ImageAspectFlags aspect {};
+	std::scoped_lock      lock {m_lock};
+	ImageId               selected {};
+	ImageId               stencil_id {};
+	vk::ImageAspectFlags  aspect {};
+	ImageSubresourceRange subresources;
 	for (const auto id: FindImagesInRegion(address, size, false)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {
@@ -1501,10 +1534,13 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		}
 		vk::ImageAspectFlags candidate {};
 		ImageId              candidate_id = id;
+		ImageSubresourceRange candidate_subresources {
+		    0, owner->info.resources.levels, 0, owner->info.TransferLayers()};
 		if (owner->depth_id && owner->info.data.address == address &&
 		    owner->info.data.size == size) {
-			candidate    = vk::ImageAspectFlagBits::eStencil;
-			candidate_id = owner->depth_id;
+			candidate              = vk::ImageAspectFlagBits::eStencil;
+			candidate_id           = owner->depth_id;
+			candidate_subresources = owner->stencil_subresources;
 			owner        = m_slot_images.try_get(candidate_id);
 			if (owner == nullptr || owner->backing.image == nullptr || !owner->info.HasStencil()) {
 				continue;
@@ -1517,11 +1553,13 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		if (!candidate) {
 			continue;
 		}
-		if (selected && selected != candidate_id) {
+		if (selected && (selected != candidate_id || subresources != candidate_subresources)) {
 			return false;
 		}
-		selected = candidate_id;
-		aspect   = candidate;
+		selected     = candidate_id;
+		stencil_id   = candidate == vk::ImageAspectFlagBits::eStencil ? id : ImageId {};
+		aspect       = candidate;
+		subresources = candidate_subresources;
 	}
 	if (!selected) {
 		return false;
@@ -1543,7 +1581,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		clear.depthStencil.stencil = stencil_clear;
 	}
 	ClearImage(command, selected, image.backing.format,
-	           {aspect, 0, image.info.resources.levels, 0, image.info.TransferLayers()}, clear);
+	           {aspect, subresources.base_level, subresources.level_count,
+	            subresources.base_layer, subresources.layer_count}, clear);
+	if (stencil_id) {
+		TrackImage(stencil_id);
+		CommitGpuWrite(m_slot_images[stencil_id]);
+	}
 	return true;
 }
 
@@ -1709,14 +1752,14 @@ void TextureCache::DownloadImage(Image& image, Buffer& destination, uint64_t des
 	}
 
 	auto&      texture   = transfer.texture;
-	const auto transform = texture.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16
-	                                           : TileManager::ColorTransform::None;
+	const auto transform = texture.color_transform;
 	if (texture.tiles.empty()) {
-		if (transform == TileManager::ColorTransform::SwapBgra16) {
+		if (transform != ColorTransform::None) {
 			auto linear = m_tiler.GetScratchBuffer(destination_size);
 			image.Download(texture.regions, linear.buffer, 0, linear.size);
-			m_tiler.SwapBgra16(linear,
-			                   {destination.Handle(), destination_offset, destination_size});
+			m_tiler.TransformColor(linear,
+			                       {destination.Handle(), destination_offset, destination_size},
+			                       transform, false);
 			return;
 		}
 		for (auto& copy: texture.regions) {
@@ -1749,14 +1792,12 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
 	const auto available  = buffer.Size() - buf_offset;
 	uint32_t   levels     = 0;
 	uint64_t   copy_size  = 0;
-	if (image.info.IsVolume()) {
-		// Volume mips contain strided block slices, so a mip's linear span cannot prove that
-		// every retained slice fits. Keep volume synchronization whole-image only.
-		if (!buffer.IsInBounds(image.info.data.address, image.info.data.size)) {
-			return false;
-		}
+	if (buffer.IsInBounds(image.info.data.address, image.info.data.size)) {
 		levels    = image.info.resources.levels;
 		copy_size = image.info.data.size;
+	} else if (image.info.IsVolume() || image.info.resources.layers > 1) {
+		// Array and volume slices are strided across the full mip chain.
+		return false;
 	} else {
 		for (; levels < image.info.resources.levels; ++levels) {
 			const auto& mip = image.info.mip_layout[levels];
@@ -1856,23 +1897,6 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) {
 	}
 }
 
-bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
-	if (!GuestRange {address, size}.Valid()) {
-		return false;
-	}
-	std::scoped_lock lock {m_lock};
-	for (const auto id: FindImagesInRegion(address, size, false)) {
-		const auto& image = m_slot_images[id];
-		// PPSA17168: S_LOAD_DWORD reads shader data at an address overlapping an old
-		// render target whose memory the CPU has reused. The cached image still retains
-		// its earlier GPU-modified flag.
-		if (!image.depth_id && image.IsGpuModified() && !image.IsDefinitelyCpuDirty()) {
-			return true;
-		}
-	}
-	return false;
-}
-
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
 	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);
@@ -1937,6 +1961,11 @@ bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) {
 	return true;
 }
 
+bool TextureCache::IsRegionRegistered(uint64_t address, uint64_t size) {
+	std::scoped_lock lock {m_lock};
+	return !FindImagesInRegion(address, size, false).empty();
+}
+
 void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 	if (!GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid unmap range\n");

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -61,7 +61,7 @@ public:
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
 	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
-	[[nodiscard]] bool IsRegionGpuModified(uint64_t address, uint64_t size);
+	[[nodiscard]] bool IsRegionRegistered(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
 	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -264,7 +264,7 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
 	desc.type              = TextureCache::BindingType::RenderTarget;
 	desc.info.data         = {rt.base.addr, backing_size};
 	desc.info.pixel_format = target_format.format;
-	desc.info.guest_format = transfer_format;
+	desc.info.guest_format = target_format.guest_format;
 	desc.info.type         = image_type;
 	desc.info.extent       = {width, height, depth};
 	desc.info.resources    = {levels, volume ? 1u : view.image_layers};

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -226,43 +226,37 @@ void CommandScheduler::PopPendingOperations() {
 }
 
 void CommandScheduler::DeferOperation(Common::UniqueFunction<void>&& operation) {
+	QueueOperation(std::move(operation), false);
+}
+
+void CommandScheduler::DeferPriorityOperation(Common::UniqueFunction<void>&& operation) {
+	QueueOperation(std::move(operation), true);
+}
+
+void CommandScheduler::QueueOperation(Common::UniqueFunction<void>&& operation, bool priority) {
 	CheckActive();
 	EXIT_IF(!operation);
 	std::unique_lock lock(m_operation_mutex);
 	if (m_operation_state == OperationState::Open) {
-		m_pending_operations.push({std::move(operation), CurrentTick()});
-		return;
-	}
-	if (g_deferred_callback_scheduler == this) {
+		auto& queue = priority ? m_priority_operations : m_pending_operations;
+		queue.push({std::move(operation), CurrentTick()});
 		lock.unlock();
-		operation();
+		if (priority) {
+			m_operation_available.notify_one();
+		}
 		return;
 	}
-	m_operation_available.wait(lock,
-	                           [this] { return m_operation_state == OperationState::Closed; });
+	if (g_deferred_callback_scheduler != this) {
+		m_operation_available.wait(lock,
+		                           [this] { return m_operation_state == OperationState::Closed; });
+	}
 	lock.unlock();
 	operation();
 }
 
-void CommandScheduler::DeferPriorityOperation(Common::UniqueFunction<void>&& operation) {
-	CheckActive();
-	EXIT_IF(!operation);
-	std::unique_lock lock(m_operation_mutex);
-	if (m_operation_state == OperationState::Open) {
-		m_priority_operations.push({std::move(operation), CurrentTick()});
-		lock.unlock();
-		m_operation_available.notify_one();
-		return;
-	}
-	if (g_deferred_callback_scheduler == this) {
-		lock.unlock();
-		operation();
-		return;
-	}
-	m_operation_available.wait(lock,
-	                           [this] { return m_operation_state == OperationState::Closed; });
-	lock.unlock();
-	operation();
+bool CommandScheduler::HasPendingPriorityOperations() {
+	std::lock_guard lock(m_operation_mutex);
+	return !m_priority_operations.empty() || m_priority_active;
 }
 
 void CommandScheduler::PriorityOperationsThread(std::stop_token stop) {

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -38,8 +38,10 @@ public:
 	void                      PopPendingOperations();
 	void                      DrainPriorityOperations();
 	void                      WaitPriorityOperations(uint64_t tick);
+	// Guest-memory completions use the priority queue; normal callbacks maintain GPU resources.
 	void                      DeferOperation(Common::UniqueFunction<void>&& operation);
 	void                      DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
+	[[nodiscard]] bool        HasPendingPriorityOperations();
 	[[nodiscard]] static bool InDeferredOperation() noexcept;
 
 	[[nodiscard]] bool Active() const noexcept { return m_command.m_registers != nullptr; }
@@ -82,6 +84,7 @@ private:
 
 	void BeginNext();
 	void PriorityOperationsThread(std::stop_token stop);
+	void QueueOperation(Common::UniqueFunction<void>&& operation, bool priority);
 	void RunOperation(Common::UniqueFunction<void>&& operation);
 
 	MasterSemaphore              m_master;

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -160,8 +160,7 @@ static TextureCache::ImageDesc MakeDepthTargetDesc(const CommandBuffer& buffer,
 			           z.depth_view.slice_max);
 	}
 	// EXPCLEAR permits an HTile acceleration state; the host attachment is already expanded.
-	if (z.z_info.partially_resident ||
-	    z.stencil_info.partially_resident || z.z_info.max_mip_level != 0 ||
+	if (z.z_info.max_mip_level != 0 ||
 	    z.depth_view.current_mip_level != 0 || unsupported_shading_rate_encoding ||
 	    depth_address == 0 || (depth_address & 0xffffu) != 0) {
 		DepthFatal("unsupported depth register state");
@@ -316,8 +315,6 @@ void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepth
 	r.depth_compare_op        = static_cast<vk::CompareOp>(dc.zfunc);
 
 	r.depth_bounds_test_enable = dc.depth_bounds_enable;
-	r.depth_min_bounds         = hw.GetDepthBoundsMin();
-	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
 	r.stencil_clear_enable =
 	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -29,8 +29,6 @@ struct RenderDepthInfo {
 	bool                        depth_write_enable       = false;
 	vk::CompareOp               depth_compare_op         = vk::CompareOp::eNever;
 	bool                        depth_bounds_test_enable = false;
-	float                       depth_min_bounds         = 0.0f;
-	float                       depth_max_bounds         = 0.0f;
 	bool                        stencil_clear_enable     = false;
 	uint8_t                     stencil_clear_value      = 0;
 	bool                        stencil_test_enable      = false;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -6,6 +6,7 @@
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "kernel/memory.h"
 
@@ -456,7 +457,7 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
 	return static_cast<uint32_t>(std::min<uint64_t>(rows, capacity / row_size));
 }
 
-void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
+void Image::CopyImageWithBuffer(Image& source, Buffer& buffer, TileManager& tiler) {
 	EXIT_IF(buffer.Handle() == nullptr || source.backing.samples != 1 || backing.samples != 1);
 	m_scheduler.EndRendering();
 	const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
@@ -474,6 +475,8 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 	const uint32_t destination_block = info.IsBlock() ? 4u : 1u;
 	EXIT_IF(levels == 0 || source_bytes == 0 || source_bytes != destination_bytes ||
 	        source_block != destination_block);
+	const auto source_transform = source.info.GetColorTransform();
+	const auto target_transform = info.GetColorTransform();
 
 	vk::BufferMemoryBarrier2 barrier {};
 	barrier.srcStageMask        = vk::PipelineStageFlagBits2::eTransfer;
@@ -535,9 +538,19 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 				command.copyImageToBuffer(source.backing.image,
 				                          vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
 				                          source_copy);
-				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
-				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
-				command.pipelineBarrier2(dependency);
+				if (source_transform != target_transform) {
+					const TileManager::Result bytes {buffer.Handle(), 0, copy_size};
+					if (source_transform != ColorTransform::None) {
+						tiler.TransformColor(bytes, bytes, source_transform, false);
+					}
+					if (target_transform != ColorTransform::None) {
+						tiler.TransformColor(bytes, bytes, target_transform, true);
+					}
+				} else {
+					barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
+					barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
+					command.pipelineBarrier2(dependency);
+				}
 				command.copyBufferToImage(buffer.Handle(), backing.image,
 				                          vk::ImageLayout::eTransferDstOptimal, destination_copy);
 			}
@@ -673,7 +686,9 @@ Prospero::BufferFormat RenderTargetTransferFormat(uint32_t bytes_per_element) {
 } // namespace ImageOps
 
 Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
-    : info(image_info), m_graphics(graphics), m_scheduler(scheduler) {
+    : info(image_info),
+      stencil_subresources {0, image_info.resources.levels, 0, image_info.resources.layers},
+      m_graphics(graphics), m_scheduler(scheduler) {
 	KYTY_PROFILER_FUNCTION();
 	ImageOps::Validate(info);
 	m_cpu_dirty =

src/graphics/host_gpu/renderer/image/image.h

@@ -18,6 +18,7 @@ namespace Libs::Graphics {
 
 class Buffer;
 class CommandScheduler;
+class TileManager;
 struct ImageTestAccess;
 
 using ImageId = Common::SlotId;
@@ -66,7 +67,7 @@ public:
 	void CopyImage(Image& source);
 	void Resolve(Image& source, const ImageSubresourceRange& source_range,
 	             const ImageSubresourceRange& destination_range);
-	void CopyImageWithBuffer(Image& source, Buffer& buffer);
+	void CopyImageWithBuffer(Image& source, Buffer& buffer, TileManager& tiler);
 	void CopyMip(Image& source, uint32_t mip, uint32_t layer);
 
 	void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) {
@@ -148,6 +149,8 @@ public:
 	uint64_t         track_addr     = 0;
 	uint64_t         track_addr_end = 0;
 	ImageId          depth_id {};
+	// The current stencil plane's mapping into the depth image, also retained by its association.
+	ImageSubresourceRange stencil_subresources;
 	uint64_t         tick_accessed_last = 0;
 	size_t           lru_id             = 0;
 

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -15,6 +15,8 @@
 
 namespace Libs::Graphics {
 
+enum class ColorTransform : uint32_t { None, SwapBgra16, Reverse10_11_11 };
+
 enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported };
 
 enum class ImageMetadataKind : uint8_t { None, Htile, Dcc, Cmask };
@@ -70,6 +72,14 @@ struct ImageInfo {
 	bool                         bgra16          = false;
 	std::array<ImageMipInfo, 16> mip_layout {};
 
+	[[nodiscard]] ColorTransform GetColorTransform() const noexcept {
+		if (bgra16) return ColorTransform::SwapBgra16;
+		switch (guest_format) {
+			case Prospero::BufferFormat::k10_11_11Float:
+				return ColorTransform::Reverse10_11_11;
+			default: return ColorTransform::None;
+		}
+	}
 	[[nodiscard]] constexpr bool HasStencil() const noexcept { return !stencil.Empty(); }
 	[[nodiscard]] constexpr bool HasMetadata() const noexcept {
 		return metadata.kind != ImageMetadataKind::None;
@@ -91,7 +101,8 @@ struct ImageInfo {
 	}
 	[[nodiscard]] bool IsCompatible(const ImageInfo& other) const noexcept {
 		return pixel_format == other.pixel_format && samples == other.samples &&
-		       bytes_per_block == other.bytes_per_block;
+		       bytes_per_block == other.bytes_per_block &&
+		       GetColorTransform() == other.GetColorTransform();
 	}
 	[[nodiscard]] int32_t MipOf(const ImageInfo& container) const noexcept {
 		if (container.resources.levels > container.mip_layout.size()) {
@@ -379,7 +390,28 @@ inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {
 		const auto encoded = static_cast<float>(value & 0xffu) / 255.0f;
 		return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f);
 	};
+	const auto float16 = [](uint32_t value) {
+		const auto sign = (value & 0x8000u) << 16u;
+		const auto exponent = (value >> 10u) & 0x1fu;
+		const auto mantissa = value & 0x3ffu;
+		if (exponent == 0) {
+			return std::copysign(static_cast<float>(mantissa) * 0x1p-24f,
+			                     std::bit_cast<float>(sign));
+		}
+		return std::bit_cast<float>(sign | ((exponent == 31 ? 255u : exponent + 112u) << 23u) |
+		                            (mantissa << 13u));
+	};
 	switch (format) {
+		case vk::Format::eR16G16Sfloat:
+			next.float32[1] = float16(packed >> 16u);
+			[[fallthrough]];
+		case vk::Format::eR16Sfloat: next.float32[0] = float16(packed); break;
+		case vk::Format::eR16G16Unorm:
+			next.float32[1] = static_cast<float>(packed >> 16u) / 65535.0f;
+			[[fallthrough]];
+		case vk::Format::eR16Unorm:
+			next.float32[0] = static_cast<float>(packed & 0xffffu) / 65535.0f;
+			break;
 		// A single-plane float target carries its clear as raw float bits, the same encoding the
 		// depth decoder below uses. Without this the clear is discarded and the target keeps stale
 		// contents.
@@ -432,6 +464,10 @@ inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {
 // Keep this separate: the first register word alone cannot describe a 64-bit color.
 [[nodiscard]] inline bool DecodeColorDwordFill(vk::Format format, uint32_t packed,
                                                vk::ClearColorValue& clear) {
+	if ((format == vk::Format::eR16Sfloat || format == vk::Format::eR16Unorm) &&
+	    (packed & 0xffffu) != (packed >> 16u)) {
+		return false;
+	}
 	if (format == vk::Format::eR16G16B16A16Sfloat) {
 		if (packed != 0) {
 			return false;

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -35,6 +35,9 @@ struct HostFormatInfo {
 
 HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
                                  Prospero::ChannelOrder order) {
+	if (guest_format == Prospero::BufferFormat::k10_11_11Float) {
+		return {vk::Format::eB10G11R11UfloatPack32, Prospero::ColorMappingBgra};
+	}
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k8_8_8_8UNorm:
@@ -131,7 +134,8 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layo
 		if (host_format.format != vk::Format::eUndefined && bytes != 0) {
 			const auto order_mapping =
 			    kRenderTargetColorMappings[static_cast<size_t>(order)][encoding.components - 1u];
-			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping)};
+			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping),
+			        encoding.buffer_format};
 		}
 	}
 	EXIT("unsupported render-target format combination: layout=%u type=%u order=%u\n",

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -15,6 +15,7 @@ struct RenderTargetFormatInfo {
 	vk::Format                      format            = vk::Format::eUndefined;
 	uint32_t                        bytes_per_element = 0;
 	Prospero::ColorComponentMapping export_mapping;
+	Prospero::BufferFormat          guest_format = Prospero::BufferFormat::kInvalid;
 };
 
 struct SurfaceFormatInfo {

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -13,7 +13,7 @@
 #include "gpu_tiler_shaders/gpu_tiler_standard4_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_standard64_3d_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h"
-#include "gpu_tiler_shaders/gpu_tiler_swap_bgra16_spv.h"
+#include "gpu_tiler_shaders/gpu_tiler_color_transform_spv.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
@@ -31,7 +31,7 @@ TileManager::TileManager(GraphicContext& graphics, CommandScheduler& scheduler,
                          StreamBuffer& stream_buffer)
     : m_graphics(graphics), m_scheduler(scheduler), m_stream_buffer(stream_buffer) {
 	static_assert(FamilyCount == 9);
-	static_assert(sizeof(Push) == 52);
+	static_assert(sizeof(Push) == 60);
 	std::array<vk::DescriptorSetLayoutBinding, 3> bindings {};
 	for (uint32_t index = 0; index < 2; index++) {
 		bindings[index] = {index, vk::DescriptorType::eStorageBuffer, 1,
@@ -77,8 +77,8 @@ TileManager::~TileManager() {
 	if (m_d32_to_d16 != nullptr) {
 		m_graphics.device.destroyPipeline(m_d32_to_d16, nullptr);
 	}
-	if (m_swap_bgra16 != nullptr) {
-		m_graphics.device.destroyPipeline(m_swap_bgra16, nullptr);
+	if (m_color_transform != nullptr) {
+		m_graphics.device.destroyPipeline(m_color_transform, nullptr);
 	}
 	if (m_pipeline_layout != nullptr) {
 		m_graphics.device.destroyPipelineLayout(m_pipeline_layout, nullptr);
@@ -90,7 +90,8 @@ TileManager::~TileManager() {
 
 void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
                           std::span<const GpuTileInfo> infos, uint64_t source_base,
-                          uint64_t target_base, std::vector<Dispatch>& dispatches) {
+                          uint64_t target_base, std::vector<Dispatch>& dispatches,
+                          ColorTransform transform) {
 	EXIT_IF(infos.empty() || tiled_capacity == 0 || linear_capacity == 0);
 	const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	EXIT_NOT_IMPLEMENTED(tiled_capacity > UINT32_MAX || linear_capacity > UINT32_MAX);
@@ -108,6 +109,8 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 	dispatches.clear();
 	dispatches.reserve(infos.size());
 	for (const auto& info: infos) {
+		EXIT_IF((transform == ColorTransform::SwapBgra16 && info.bytes_per_element != 8u) ||
+		        (transform == ColorTransform::Reverse10_11_11 && info.bytes_per_element != 4u));
 		TileBlockLayout block {};
 		const uint32_t  tiled_width  = info.tiled_width != 0 ? info.tiled_width : info.pitch;
 		const uint32_t  tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height;
@@ -193,6 +196,7 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 		dispatch.push.tail_x           = info.tail_x;
 		dispatch.push.tail_y           = info.tail_y;
 		dispatch.push.tail             = info.tail;
+		dispatch.push.color_transform  = static_cast<uint32_t>(transform);
 		dispatches.push_back(dispatch);
 	}
 
@@ -338,13 +342,14 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset,
 
 TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
                                         uint64_t tiled_capacity, uint64_t linear_capacity,
-                                        std::span<const GpuTileInfo> infos) {
+                                        std::span<const GpuTileInfo> infos,
+                                        ColorTransform transform) {
 	const auto&    limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	const uint64_t descriptor_alignment =
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
+	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches, transform);
 	auto scratch = GetScratchBuffer(linear_capacity, tiled);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
@@ -375,15 +380,11 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        target_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	// Reserve stream parameters before recording the image download and conversion.
-	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
+	// Reserve stream parameters before recording the image download.
+	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches, transform);
 	auto linear = GetScratchBuffer(linear_capacity, tiled);
 	image.Download(regions, linear.buffer, 0, linear.size);
-	Result source {linear.buffer, 0, linear.size};
-	if (transform == ColorTransform::SwapBgra16) {
-		source = SwapBgra16(source);
-	}
-	Record(source.buffer, source.offset, linear_capacity, tiled, tiled_offset, tiled_capacity,
+	Record(linear.buffer, 0, linear_capacity, tiled, tiled_offset, tiled_capacity,
 	       dispatches, false);
 }
 
@@ -582,9 +583,20 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	                        &barriers[1], 0, nullptr);
 }
 
-void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
-	if (m_swap_bgra16 == nullptr) {
-		const auto module = CompileSPV(GPU_TILER_SWAP_BGRA16_SPV, m_graphics.device);
+void TileManager::TransformColor(Result input, Result output, ColorTransform transform,
+                                 bool to_host) {
+	uint32_t element_bytes = 0;
+	switch (transform) {
+		case ColorTransform::SwapBgra16: element_bytes = 8; break;
+		case ColorTransform::Reverse10_11_11: element_bytes = 4; break;
+		default: EXIT("invalid color transform\n");
+	}
+	EXIT_IF(input.size == 0 || input.size % element_bytes != 0 ||
+	        input.size / element_bytes > UINT32_MAX ||
+	        output.size < input.size);
+	const auto pixels = static_cast<uint32_t>(input.size / element_bytes);
+	if (m_color_transform == nullptr) {
+		const auto module = CompileSPV(GPU_TILER_COLOR_TRANSFORM_SPV, m_graphics.device);
 		vk::PipelineShaderStageCreateInfo stage {};
 		stage.stage  = vk::ShaderStageFlagBits::eCompute;
 		stage.module = module;
@@ -593,14 +605,12 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 		create.stage  = stage;
 		create.layout = m_pipeline_layout;
 		const auto result =
-		    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_swap_bgra16);
+		    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_color_transform);
 		m_graphics.device.destroyShaderModule(module, nullptr);
-		RequireVulkanSuccess(result, "create BGRA16 swap pipeline");
+		RequireVulkanSuccess(result, "create color transform pipeline");
 	}
-	const uint64_t bytes = static_cast<uint64_t>(pixels) * 8u;
-	EXIT_IF(pixels == 0);
-	const auto input_binding  = BindStorage(input, bytes);
-	const auto output_binding = BindStorage(output, bytes);
+	const auto input_binding  = BindStorage(input, input.size);
+	const auto output_binding = BindStorage(output, input.size);
 
 	const vk::DescriptorBufferInfo infos[] {
 	    input_binding.info,
@@ -631,13 +641,15 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 	command.pipelineBarrier(
 	    vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost,
 	    vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 2, barriers, 0, nullptr);
-	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_swap_bgra16);
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_color_transform);
 	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0,
 	                             static_cast<uint32_t>(writes.size()), writes.data());
 	Push push {};
-	push.src_base = input_binding.base;
-	push.dst_base = output_binding.base;
-	push.width    = pixels;
+	push.src_base        = input_binding.base;
+	push.dst_base        = output_binding.base;
+	push.width           = pixels;
+	push.color_transform = static_cast<uint32_t>(transform);
+	push.to_host         = to_host;
 	command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, sizeof(push),
 	                      &push);
 	command.dispatch((pixels + 63u) / 64u, 1, 1);
@@ -648,17 +660,11 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 	                        0, nullptr);
 }
 
-TileManager::Result TileManager::SwapBgra16(Result input) {
-	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX);
+TileManager::Result TileManager::TransformColor(Result input, ColorTransform transform,
+                                                bool to_host) {
 	auto result = GetScratchBuffer(input.size, input.buffer);
-	SwapBgra16(input, result, static_cast<uint32_t>(input.size / 8u));
+	TransformColor(input, result, transform, to_host);
 	return result;
 }
 
-void TileManager::SwapBgra16(Result input, Result output) {
-	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX ||
-	                     output.size < input.size);
-	SwapBgra16(input, output, static_cast<uint32_t>(input.size / 8u));
-}
-
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/image/tiler.h

@@ -3,6 +3,7 @@
 
 #include "common/common.h"
 #include "graphics/guest_gpu/tile.h"
+#include "graphics/host_gpu/renderer/image/imageInfo.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
@@ -42,7 +43,6 @@ struct GpuTileInfo {
 class TileManager final {
 public:
 	enum class D16Direction { Promote, Demote };
-	enum class ColorTransform { None, SwapBgra16 };
 
 	struct Result {
 		vk::Buffer buffer = nullptr;
@@ -65,7 +65,8 @@ public:
 
 	// Consume scratch results before the next acquisition, or pass their buffer as input.
 	[[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity,
-	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos);
+	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos,
+	                            ColorTransform transform = ColorTransform::None);
 	void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled,
 	          uint64_t tiled_offset, uint64_t tiled_capacity, std::span<const GpuTileInfo> infos);
 	void TileImage(Image& image, std::span<const vk::BufferImageCopy> regions, vk::Buffer tiled,
@@ -75,8 +76,8 @@ public:
 	[[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr);
 	void                 ConvertD16(Result source, Result target, D16Direction direction, bool d32,
 	                                const D16Layout& layout);
-	[[nodiscard]] Result SwapBgra16(Result input);
-	void                 SwapBgra16(Result input, Result output);
+	[[nodiscard]] Result TransformColor(Result input, ColorTransform transform, bool to_host);
+	void TransformColor(Result input, Result output, ColorTransform transform, bool to_host);
 
 private:
 	friend struct TileManagerTestAccess;
@@ -100,6 +101,8 @@ private:
 		uint32_t tail_x;
 		uint32_t tail_y;
 		uint32_t tail;
+		uint32_t color_transform;
+		uint32_t to_host;
 	};
 	struct Dispatch {
 		Push     push {};
@@ -118,12 +121,11 @@ private:
 	                                             uint32_t max_groups) noexcept;
 	void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
 	             std::span<const GpuTileInfo> infos, uint64_t source_base, uint64_t target_base,
-	             std::vector<Dispatch>& dispatches);
+	             std::vector<Dispatch>& dispatches, ColorTransform transform = ColorTransform::None);
 	void Record(vk::Buffer source, uint64_t source_offset, uint64_t source_capacity,
 	            vk::Buffer target, uint64_t target_offset, uint64_t target_capacity,
 	            std::span<Dispatch> dispatches, bool clear_target);
 	[[nodiscard]] vk::Pipeline GetPipeline(uint32_t slot);
-	void                       SwapBgra16(Result input, Result output, uint32_t pixels);
 
 	GraphicContext&                         m_graphics;
 	CommandScheduler&                       m_scheduler;
@@ -132,11 +134,11 @@ private:
 	vk::DescriptorSetLayout                 m_descriptor_layout = nullptr;
 	vk::PipelineLayout                      m_pipeline_layout   = nullptr;
 	std::array<vk::Pipeline, PipelineCount> m_pipelines {};
-	vk::Pipeline                            m_d16_to_d24  = nullptr;
-	vk::Pipeline                            m_d16_to_d32  = nullptr;
-	vk::Pipeline                            m_d24_to_d16  = nullptr;
-	vk::Pipeline                            m_d32_to_d16  = nullptr;
-	vk::Pipeline                            m_swap_bgra16 = nullptr;
+	vk::Pipeline                            m_d16_to_d24      = nullptr;
+	vk::Pipeline                            m_d16_to_d32      = nullptr;
+	vk::Pipeline                            m_d24_to_d16      = nullptr;
+	vk::Pipeline                            m_d32_to_d16      = nullptr;
+	vk::Pipeline                            m_color_transform = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/blendMapping.cpp

@@ -30,6 +30,18 @@ BlendMappingSupport ClassifyBlendMapping(const HW::BlendControl&
 	if (mapping.Map(3) == 3u) {
 		return BlendMappingSupport::Direct;
 	}
+	if (blend.separate_alpha_blend &&
+	    blend.color_srcblend == static_cast<uint8_t>(Prospero::BlendFactor::kSrcAlpha) &&
+	    blend.color_destblend == static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrcAlpha) &&
+	    blend.alpha_destblend == blend.color_destblend &&
+	    blend.color_comb_fcn == static_cast<uint8_t>(Prospero::BlendOp::kAdd) &&
+	    blend.alpha_comb_fcn == blend.color_comb_fcn) {
+		switch (static_cast<Prospero::BlendFactor>(blend.alpha_srcblend)) {
+			case Prospero::BlendFactor::kZero: return BlendMappingSupport::SourceAlphaZero;
+			case Prospero::BlendFactor::kOne: return BlendMappingSupport::SourceAlphaOne;
+			default: break;
+		}
+	}
 	// Moving alpha requires the same equation for all channels.
 	if (blend.separate_alpha_blend && (blend.alpha_srcblend != blend.color_srcblend ||
 	                                   blend.alpha_destblend != blend.color_destblend ||

src/graphics/host_gpu/renderer/pipeline/blendMapping.h

@@ -15,6 +15,8 @@ struct ColorComponentMapping;
 enum class BlendMappingSupport {
 	Direct,
 	SourceAlpha, // Requires logical alpha in the second blend source.
+	SourceAlphaOne, // Source alpha for RGB, one for alpha, with a shared destination equation.
+	SourceAlphaZero, // Source alpha for RGB, zero for alpha, with a shared destination equation.
 	Unsupported,
 };
 

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -3,6 +3,7 @@
 #include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common.h"
+#include "common/emulatorConfig.h"
 #include "common/file.h"
 #include "common/logging/log.h"
 #include "common/profiler.h"
@@ -25,7 +26,6 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 #include "kernel/memory.h"
@@ -68,16 +68,13 @@ vk::DescriptorType NativeDescriptorType(BindingKind kind) {
 		case BindingKind::BdaPagetable:
 		case BindingKind::FaultBuffer:
 		case BindingKind::FlattenedSrt:
-		case BindingKind::ShaderData: return vk::DescriptorType::eStorageBuffer;
+		case BindingKind::ShaderData:
+		case BindingKind::SharedMemory: return vk::DescriptorType::eStorageBuffer;
 		case BindingKind::Count: EXIT("invalid native descriptor binding kind");
 	}
 	EXIT("invalid native descriptor binding kind");
 }
 
-uint32_t NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
-	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
-}
-
 vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element) {
 	vk::ImageView view = nullptr;
 	if (texture.mip_views.empty()) {
@@ -125,12 +122,13 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	if (size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange) {
 		EXIT("storage buffer range is unsupported\n");
 	}
-	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
-	                                                              resource.formatted, id);
+	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(
+	    address, size, resource.written,
+	    resource.formatted || (resource.read && Config::SyncRawImageBuffersEnabled()), id);
 	const auto aligned_offset = Common::AlignDown(offset, alignment);
 	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
-	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
+	if (adjustment >= 256 || size > max_range - adjustment) {
 		EXIT("storage buffer offset adjustment is unsupported\n");
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
@@ -779,6 +777,7 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 	prepared.gds = {nullptr, 0, VK_WHOLE_SIZE};
 	prepared.flattened_srt = {};
 	prepared.shader_data_buffer = {};
+	prepared.shared_memory = {};
 	prepared.images.resize(program.info.images.size());
 	prepared.samplers.clear();
 	prepared.shader_data.clear();
@@ -795,40 +794,62 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 		    m_context, program, i, snapshot.samplers[program.info.samplers[i].snapshot_index]));
 	}
 	prepared.shader_data.reserve(program.bindings.ShaderDataDwords());
-	for (const auto reg: program.bindings.user_data_registers) {
+	for (const auto reg: program.info.user_data_registers) {
 		prepared.shader_data.push_back(snapshot.user_data[reg - program.user_data_base]);
 	}
 	prepared.shader_data.resize(program.bindings.ShaderDataDwords());
-	if (ShaderRecompiler::IR::FindBinding(
-	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) {
+	if (program.bindings.descriptor_counts[static_cast<size_t>(BindingKind::Gds)] != 0) {
 		prepared.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle();
 	}
 }
 
-void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
+void RenderExecutor::FindBuffers(std::span<PreparedBindings* const> stages) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
-	const auto& program  = *prepared.runtime->program;
-	const auto& snapshot = *prepared.runtime->resources;
-	auto&       cache    = m_context.GetBufferCache();
-
-	prepared.buffer_sources.clear();
-	const auto& layout = program.bindings;
-	if (layout.memory_offset_count == 0) {
-		return;
-	}
-	const auto& resources = layout.descriptors.front().resources;
-	prepared.buffer_sources.reserve(resources.size());
-	for (const auto resource: resources) {
-		auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[resource]);
-		const auto address = descriptor.Base48();
-		const auto requested_size = descriptor.GetSize();
-		if (address == 0 || requested_size == 0) {
-			prepared.buffer_sources.push_back({});
+	auto& cache = m_context.GetBufferCache();
+	for (auto* stage: stages) {
+		auto& prepared = *stage;
+		EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
+		const auto& program  = *prepared.runtime->program;
+		const auto& snapshot = *prepared.runtime->resources;
+		const auto& layout = program.bindings;
+		prepared.buffer_sources.resize(layout.descriptor_counts[static_cast<size_t>(BindingKind::Buffers)]);
+		if (layout.descriptor_counts[static_cast<size_t>(BindingKind::Buffers)] == 0) {
 			continue;
 		}
-		const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
-		prepared.buffer_sources.push_back({address, size, cache.FindBuffer(address, size)});
+		for (uint32_t resource = 0; resource < program.info.buffers.size(); ++resource) {
+			const auto& metadata = program.info.buffers[resource];
+			if (metadata.descriptor_index == UINT32_MAX) continue;
+			auto& source = prepared.buffer_sources[metadata.descriptor_index];
+			const auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[resource]);
+			const auto address = descriptor.Base48();
+			auto size = descriptor.GetSize();
+			if (address == 0 || size == 0) {
+				source = {};
+				continue;
+			}
+			if (descriptor.NumRecords() == UINT32_MAX) {
+				if (program.info.buffers[resource].written) {
+					// Sentinel write ranges end before tables captured by any bound stage.
+					for (const auto* reader: stages) {
+						for (const auto [read_address, read_size]: reader->runtime->resources->specialization_reads) {
+							if (read_size == 0) continue;
+							if (read_address > address) {
+								size = std::min(size, read_address - address);
+							} else if (address - read_address < read_size) {
+								EXIT("scalar resource reads overlap a shader buffer write\n");
+							}
+						}
+					}
+				} else {
+					const auto& graphics = m_context.GetGraphics();
+					const auto limit = uint64_t {graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange} -
+					                   (graphics.StorageMinAlignment() - 1);
+					size = std::min({size, uint64_t {256} * 1024 * 1024, limit});
+				}
+			}
+			size = Libs::LibKernel::Memory::ClampRangeSize(address, size);
+			source = {address, size, cache.FindBuffer(address, size)};
+		}
 	}
 }
 
@@ -838,10 +859,9 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	const auto& program   = *prepared.runtime->program;
 	const auto& snapshot  = *prepared.runtime->resources;
 	const auto& layout    = program.bindings;
-	EXIT_IF(prepared.buffer_sources.size() != layout.memory_offset_count);
+	EXIT_IF(prepared.buffer_sources.size() != layout.descriptor_counts[static_cast<size_t>(BindingKind::Buffers)]);
 
-	prepared.buffers.clear();
-	prepared.buffers.reserve(layout.memory_offset_count);
+	prepared.buffers.resize(layout.descriptor_counts[static_cast<size_t>(BindingKind::Buffers)]);
 	EXIT_IF(prepared.shader_data.size() != layout.ShaderDataDwords());
 	std::fill(prepared.shader_data.begin() + layout.memory_offset_dword,
 	          prepared.shader_data.end(), 0);
@@ -850,20 +870,18 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 		const auto shift = (index % 4u) * 8u;
 		prepared.shader_data[dword] |= offset << shift;
 	};
-	for (uint32_t i = 0; i < layout.memory_offset_count; i++) {
-		const auto resource = layout.descriptors.front().resources[i];
+	for (const auto& resource: program.info.buffers) {
+		const auto index = resource.descriptor_index;
+		if (index == UINT32_MAX) continue;
 		uint32_t buffer_offset = 0;
-		prepared.buffers.push_back(NativeStorageBuffer(m_context, prepared.buffer_sources[i],
-		                                               program.info.buffers[resource],
-		                                               buffer_offset));
-		pack_memory_offset(i, buffer_offset);
+		prepared.buffers[index] = NativeStorageBuffer(m_context, prepared.buffer_sources[index],
+		                                              resource, buffer_offset);
+		pack_memory_offset(index, buffer_offset);
 	}
-	if (ShaderRecompiler::IR::FindBinding(
-	        layout, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
+	if (layout.descriptor_counts[static_cast<size_t>(BindingKind::FlattenedSrt)] != 0) {
 		prepared.flattened_srt = NativeUpload(m_context, snapshot.flattened_srt);
 	}
-	if (ShaderRecompiler::IR::FindBinding(
-	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::ShaderData) != nullptr) {
+	if (layout.descriptor_counts[static_cast<size_t>(BindingKind::ShaderData)] != 0) {
 		prepared.shader_data_buffer = NativeUpload(m_context, prepared.shader_data);
 	}
 }
@@ -876,6 +894,7 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	auto&       images   = prepared.images;
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
+	// Acquire each view before a later overlapping descriptor can replace its image.
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
@@ -887,8 +906,6 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
 		}
-	}
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		auto& binding = images[i];
 		binding.mip_views.clear();
 		const auto& resource = program.info.images[i];
@@ -918,8 +935,8 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
                                              std::span<RenderColorInfo> colors) {
 	bool uses_dma = false;
+	FindBuffers(stages);
 	for (auto* stage: stages) {
-		FindBuffers(*stage);
 		uses_dma |= stage->runtime->program->info.uses_dma;
 	}
 	if (uses_dma) {
@@ -956,7 +973,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
                                     std::span<PreparedBindings* const> prepared_bindings) {
 	KYTY_PROFILER_FUNCTION();
 	auto   vk_buffer        = buffer.Handle();
-	size_t descriptor_count = 0;
+	size_t image_descriptor_count = 0;
 	size_t write_count      = 0;
 	ShaderRecompiler::IR::PushData push_data;
 	bool                           has_push_data = false;
@@ -970,9 +987,14 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 	for (const auto* prepared: prepared_bindings) {
 		EXIT_IF(prepared == nullptr || prepared->runtime == nullptr || !*prepared->runtime);
 		const auto& program = *prepared->runtime->program;
-		write_count += program.bindings.descriptors.size();
-		for (const auto& binding: program.bindings.descriptors) {
-			descriptor_count += NativeDescriptorCount(binding);
+		write_count += std::popcount(program.bindings.descriptor_mask);
+		for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
+			const auto index = std::countr_zero(mask);
+			const auto count = program.bindings.descriptor_counts[index];
+			if (index >= ShaderRecompiler::IR::FirstImageBinding &&
+			    index <= static_cast<size_t>(BindingKind::Samplers)) {
+				image_descriptor_count += count;
+			}
 		}
 		const auto shader_stage = NativeShaderStage(program.stage);
 		push_stages |= shader_stage;
@@ -1004,10 +1026,9 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 			}
 			for (const auto* writer: prepared_bindings) {
 				const auto& program = *writer->runtime->program;
-				for (uint32_t i = 0; i < writer->buffer_sources.size(); ++i) {
-					const auto resource = program.bindings.descriptors.front().resources[i];
-					if (!program.info.buffers[resource].written) continue;
-					const auto& written = writer->buffer_sources[i];
+				for (const auto& resource: program.info.buffers) {
+					if (!resource.written || resource.descriptor_index == UINT32_MAX) continue;
+					const auto& written = writer->buffer_sources[resource.descriptor_index];
 					if (written.size != 0 && ImageRangeOverlaps(address, size,
 					                                          written.address, written.size)) {
 						EXIT("scalar resource reads overlap a shader buffer write\n");
@@ -1016,12 +1037,11 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 			}
 		}
 	}
-	m_descriptor_buffers.clear();
-	m_descriptor_images.clear();
+	m_descriptor_images.resize(image_descriptor_count);
 	m_descriptor_writes.clear();
-	m_descriptor_buffers.reserve(descriptor_count);
-	m_descriptor_images.reserve(descriptor_count);
 	m_descriptor_writes.reserve(write_count);
+	size_t image_cursor = 0;
+	std::array<vk::DescriptorBufferInfo, 2> address_buffers;
 
 	for (auto* prepared: prepared_bindings) {
 		const auto& program       = *prepared->runtime->program;
@@ -1086,77 +1106,79 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 			binding.layout = image.backing.state.layout;
 		}
 
-		m_image_occurrences.assign(descriptors.images.size(), 0);
-		for (const auto& binding: program.bindings.descriptors) {
+		std::array<size_t, ShaderRecompiler::IR::ImageBindingCount> image_starts {};
+		for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
+			const auto index = std::countr_zero(mask);
+			const auto kind = static_cast<BindingKind>(index);
 			vk::WriteDescriptorSet write {};
-			write.dstBinding     = ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind);
-			write.descriptorType = NativeDescriptorType(binding.kind);
-			write.descriptorCount   = NativeDescriptorCount(binding);
-			const auto buffer_start = m_descriptor_buffers.size();
-			const auto image_start  = m_descriptor_images.size();
-			if (ShaderRecompiler::IR::ImageBindingResourceClass(binding.kind) !=
+			write.dstBinding = ShaderRecompiler::IR::NativeBinding(program.stage, kind);
+			write.descriptorType = NativeDescriptorType(kind);
+			write.descriptorCount = program.bindings.descriptor_counts[index];
+			const auto image_start = image_cursor;
+			if (ShaderRecompiler::IR::ImageBindingResourceClass(kind) !=
 			    ShaderRecompiler::IR::ImageResourceClass::None) {
-				for (const auto resource: binding.resources) {
-					m_descriptor_images.push_back(MakeImageInfo(
-					    descriptors.images.at(resource), m_image_occurrences.at(resource)++));
-				}
+				image_starts[ShaderRecompiler::IR::ImageBindingIndex(kind)] = image_start;
+				image_cursor += write.descriptorCount;
 			} else {
-				switch (binding.kind) {
+				switch (kind) {
 					case BindingKind::Buffers:
-						EXIT_IF(descriptors.buffers.size() != binding.resources.size());
+						EXIT_IF(descriptors.buffers.size() != write.descriptorCount);
 						for (const auto& view: descriptors.buffers) {
 							EXIT_IF(view.buffer == nullptr);
-							m_descriptor_buffers.push_back(view);
 						}
+						write.pBufferInfo = descriptors.buffers.data();
 						break;
 					case BindingKind::BdaPagetable:
 					case BindingKind::FaultBuffer: {
-						auto&       cache      = m_context.GetBufferCache();
-						const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable
+						auto& cache = m_context.GetBufferCache();
+						const auto* bda_buffer = kind == BindingKind::BdaPagetable
 						                             ? cache.GetBdaPageTableBuffer()
 						                             : cache.GetFaultBuffer();
-						m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0,
-						                                  bda_buffer->Size());
+						auto& view = address_buffers[kind == BindingKind::FaultBuffer];
+						view = {bda_buffer->Handle(), 0, bda_buffer->Size()};
+						write.pBufferInfo = &view;
 						break;
 					}
 					case BindingKind::FlattenedSrt:
 					case BindingKind::ShaderData:
+					case BindingKind::SharedMemory:
 					case BindingKind::Gds: {
 						const vk::DescriptorBufferInfo* view = &descriptors.gds;
-						if (binding.kind == BindingKind::FlattenedSrt) {
+						if (kind == BindingKind::FlattenedSrt) {
 							view = &descriptors.flattened_srt;
-						} else if (binding.kind == BindingKind::ShaderData) {
+						} else if (kind == BindingKind::ShaderData) {
 							view = &descriptors.shader_data_buffer;
+						} else if (kind == BindingKind::SharedMemory) {
+							view = &descriptors.shared_memory;
 						}
 						EXIT_IF(view->buffer == nullptr);
-						m_descriptor_buffers.push_back(*view);
+						write.pBufferInfo = view;
 						break;
 					}
 					case BindingKind::Samplers:
-						for (const auto resource: binding.resources) {
-							const auto sampler = descriptors.samplers.at(resource);
+						for (const auto sampler: descriptors.samplers) {
 							EXIT_IF(sampler == nullptr);
-							m_descriptor_images.emplace_back(sampler, nullptr,
-							                                 vk::ImageLayout::eUndefined);
+							m_descriptor_images[image_cursor++] = {sampler, nullptr, vk::ImageLayout::eUndefined};
 						}
 						break;
 					case BindingKind::Count: EXIT("invalid descriptor binding kind");
 				}
 			}
-			if (m_descriptor_buffers.size() != buffer_start) {
-				write.pBufferInfo = m_descriptor_buffers.data() + buffer_start;
-			}
-			if (m_descriptor_images.size() != image_start) {
+			if (image_cursor != image_start) {
 				write.pImageInfo = m_descriptor_images.data() + image_start;
 			}
 			m_descriptor_writes.push_back(write);
 		}
-		for (uint32_t i = 0; i < descriptors.images.size(); i++) {
-			const auto expected =
-			    descriptors.images[i].mip_views.empty()
-			        ? 1u
-			        : static_cast<uint32_t>(descriptors.images[i].mip_views.size());
-			EXIT_IF(m_image_occurrences[i] != expected);
+		for (uint32_t i = 0; i < program.info.images.size(); ++i) {
+			const auto& resource = program.info.images[i];
+			const auto& texture = descriptors.images[i];
+			EXIT_IF(resource.mip_count != (texture.mip_views.empty() ? 1u : texture.mip_views.size()));
+			const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(resource);
+			const auto start = image_starts[ShaderRecompiler::IR::ImageBindingIndex(kind)] +
+			                   resource.descriptor_index;
+			for (uint32_t mip = 0; mip < resource.mip_count; ++mip) {
+				m_descriptor_images[start + mip] = MakeImageInfo(texture, mip);
+			}
 		}
 
 		const auto shader_data_dwords = program.bindings.ShaderDataDwords();

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -44,13 +44,12 @@ struct PreparedBindings {
 	vk::DescriptorBufferInfo              gds {nullptr, 0, VK_WHOLE_SIZE};
 	vk::DescriptorBufferInfo              flattened_srt;
 	vk::DescriptorBufferInfo              shader_data_buffer;
+	vk::DescriptorBufferInfo              shared_memory;
 	std::vector<uint32_t>                 shader_data;
 };
 
 [[nodiscard]] vk::DescriptorType
 NativeDescriptorType(ShaderRecompiler::IR::DescriptorBindingKind kind);
-[[nodiscard]] uint32_t
-NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding);
 [[nodiscard]] vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
                                                     uint32_t              element = 0);
 

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -39,6 +39,16 @@ namespace Libs::Graphics {
 
 namespace {
 
+uint8_t RemapSourceAlphaFactor(uint8_t factor) {
+	switch (static_cast<Prospero::BlendFactor>(factor)) {
+		case Prospero::BlendFactor::kSrcAlpha:
+			return static_cast<uint8_t>(Prospero::BlendFactor::kSrc1Alpha);
+		case Prospero::BlendFactor::kOneMinusSrcAlpha:
+			return static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrc1Alpha);
+		default: return factor;
+	}
+}
+
 vk::PolygonMode ResolvePolygonMode(const HW::ModeControl& mode, bool cull_front, bool cull_back) {
 	// CxPrimitiveSetup::PolygonMode disables both per-face modes when it is zero.
 	if (mode.poly_mode == 0) {
@@ -94,8 +104,10 @@ void PipelineCacheLog(fmt::format_string<Args...> format, Args&&... args) {
 }
 
 bool ReadShaderGuestMemory(void*, uint64_t address, std::span<uint32_t> values) {
+	// Scalar and unformatted buffer dependencies use the same backing as native raw loads.
+	// Image synchronization belongs to formatted buffer bindings, not these reads.
 	return !values.empty() &&
-	       Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, values.data(), values.size_bytes());
+	       Libs::LibKernel::Memory::TryReadBufferBacking(address, values.data(), values.size_bytes());
 }
 
 void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
@@ -205,17 +217,53 @@ struct PipelineCache::ProgramCache {
 	};
 
 	struct Permutation {
+		std::vector<uint32_t>                       function_code;
 		ShaderRecompiler::IR::ResourceSpecialization specialization;
 		ShaderRecompiler::IR::CompiledShaderInfo     program;
 		ShaderProgram                                handle;
 	};
 
 	struct SourceEntry {
-		explicit SourceEntry(ShaderRecompiler::IR::ResourcePlan plan)
-		    : resource_plan(std::move(plan)) {
+		SourceEntry(ShaderRecompiler::IR::ResourcePlan plan,
+		            std::unique_ptr<ShaderRecompiler::ShaderSource> call_source)
+		    : call_source(std::move(call_source)), resource_plan(std::move(plan)) {
 			permutations.reserve(8);
 		}
 
+		std::span<const uint32_t> FunctionCode() const {
+			return call_source
+			    ? std::span<const uint32_t>(call_source->linked->code).subspan(call_source->code.size())
+			    : std::span<const uint32_t>{};
+		}
+
+		bool RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) {
+			const auto revision = call_source->revision;
+			ShaderRecompiler::RefreshShaderSource(*call_source, runtime);
+			resource_plan.source_reads = call_source->reads;
+			return revision != call_source->revision;
+		}
+
+		template <bool HasCalls>
+		Permutation* FindPermutation(const ShaderRecompiler::IR::SrtRuntime& runtime,
+		                             uint32_t push_data_cursor) {
+			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
+			    resource_plan, runtime, resources, specialization));
+			for (auto& candidate: permutations) {
+				const auto& layout = candidate.program.bindings;
+				if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor(
+				        push_data_cursor, layout.ShaderDataDwords()) ||
+				    candidate.specialization != specialization) {
+					continue;
+				}
+				if constexpr (HasCalls) {
+					if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue;
+				}
+				return &candidate;
+			}
+			return nullptr;
+		}
+
+		std::unique_ptr<ShaderRecompiler::ShaderSource> call_source;
 		ShaderRecompiler::IR::ResourcePlan           resource_plan;
 		ShaderRecompiler::IR::ResourceSnapshot       resources;
 		ShaderRecompiler::IR::ResourceSpecialization specialization;
@@ -244,7 +292,8 @@ struct PipelineCache::ProgramCache {
 	                               const ShaderRecompiler::CompileOptions&      options,
 	                               ShaderRecompiler::TranslateResult            translated,
 	                               ShaderRecompiler::IR::ResourceSpecialization specialization,
-	                               uint32_t push_data_start_dword) {
+	                               uint32_t push_data_start_dword,
+	                               std::span<const uint32_t> function_code) {
 		auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
 		                                               specialization, push_data_start_dword);
 		if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) {
@@ -261,6 +310,7 @@ struct PipelineCache::ProgramCache {
 			     static_cast<uint64_t>(result.spirv.size()), options.wave_size);
 		}
 		return {
+		    .function_code  = {function_code.begin(), function_code.end()},
 		    .specialization = std::move(specialization),
 		    .program        = std::move(result.program).TakeCompiledInfo(),
 		    .handle         = {.id = ++next_shader_id, .module = module},
@@ -287,26 +337,26 @@ struct PipelineCache::ProgramCache {
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
-		const ShaderRecompiler::IR::SrtRuntime       runtime {
+		ShaderRecompiler::IR::SrtRuntime             runtime {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
 		    .read_specialization_memory = ReadShaderGuestMemory,
 		};
+		if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
+			runtime.workgroup_counts = input_info.workgroup_counts;
+		}
+		bool source_changed = false;
 		if (entry != programs.end()) {
-			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
-			    entry->second.resource_plan, runtime, entry->second.resources,
-			    entry->second.specialization));
-			if (const auto permutation = std::ranges::find_if(
-			        entry->second.permutations, [&](const Permutation& candidate) {
-				        const auto& layout = candidate.program.bindings;
-				        return layout.push_data_start_dword ==
-				                   ShaderRecompiler::IR::PushData::StartFor(
-				                       push_data_cursor, layout.ShaderDataDwords()) &&
-				               candidate.specialization == entry->second.specialization;
-			        });
-			    permutation != entry->second.permutations.end()) {
-				input_info.stage = {.program   = &permutation->program,
-				                    .resources = &entry->second.resources};
+			auto& cached = entry->second;
+			Permutation* permutation = nullptr;
+			if (cached.call_source) {
+				source_changed = cached.RefreshCallSource(runtime);
+				if (!source_changed) permutation = cached.FindPermutation<true>(runtime, push_data_cursor);
+			} else {
+				permutation = cached.FindPermutation<false>(runtime, push_data_cursor);
+			}
+			if (permutation != nullptr) {
+				input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources};
 				permutation->program.bindings.AdvancePushData(push_data_cursor);
 				return permutation->handle;
 			}
@@ -353,16 +403,39 @@ struct PipelineCache::ProgramCache {
 			options.wave_size = input_info.wave_size;
 		}
 		DumpShaderOriginal(stage_name, options.shader_hash, params.code);
-		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
+		ShaderRecompiler::ShaderSource source;
+		if (entry == programs.end()) source = ShaderRecompiler::PrepareShaderSource(params.code, options);
+		auto* current = entry == programs.end() ? &source : entry->second.call_source.get();
+		ShaderRecompiler::TranslateResult translated;
+		if (current != nullptr) {
+			if (current->call) {
+				if (entry == programs.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime);
+				translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options);
+				translated.program.source_reads = current->reads;
+			} else {
+				translated = ShaderRecompiler::TranslateProgram(current->decoded, options);
+			}
+		} else {
+			translated = ShaderRecompiler::TranslateProgram(params.code, options);
+		}
 		if (entry == programs.end()) {
+			auto retained = source.call ? std::make_unique<ShaderRecompiler::ShaderSource>(std::move(source)) : nullptr;
 			entry = programs.try_emplace(lookup_key,
-			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
+			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program), std::move(retained)).first;
 			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
 			    entry->second.resource_plan, runtime, entry->second.resources,
 			    entry->second.specialization));
+		} else if (source_changed) {
+			entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
+			if (const auto* permutation = entry->second.FindPermutation<true>(runtime, push_data_cursor)) {
+				input_info.stage = {.program = &permutation->program, .resources = &entry->second.resources};
+				permutation->program.bindings.AdvancePushData(push_data_cursor);
+				return permutation->handle;
+			}
 		}
 		entry->second.permutations.push_back(CompilePermutation(
-		    stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor));
+		    stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor,
+		    entry->second.FunctionCode()));
 		const auto& permutation = entry->second.permutations.back();
 		input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
 		permutation.program.bindings.AdvancePushData(push_data_cursor);
@@ -602,6 +675,13 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 	ShaderParams pixel_params;
 	if (pixel_active) {
 		pixel_params      = PrepareProgram(pixel_regs, sh, target_export_mapping, pixel_info);
+		if (!mesh_active && !tess_active && Prospero::IsRectList(user_config.GetPrimType())) {
+			pixel_info.parameter_mode = ShaderPixelParameterMode::Rectangle;
+		} else if (context.GetModeControl().provoking_vtx_last) {
+			pixel_info.parameter_mode = ShaderPixelParameterMode::LastVertex;
+		} else {
+			pixel_info.parameter_mode = ShaderPixelParameterMode::FirstVertex;
+		}
 		const auto& blend = context.GetBlendControl(0);
 		pixel_info.dual_source_blending =
 		    blend.enable && !context.GetRenderTarget(0).info.blend_bypass &&
@@ -617,14 +697,24 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 		           pixel_info.target_output_mode[0] != 0 && pixel_info.target_output_mode[0] != 7 &&
 		           std::all_of(std::begin(pixel_info.target_output_mode) + 1,
 		                       std::end(pixel_info.target_output_mode),
-		                       [](uint8_t mode) { return mode == 0; }) &&
-		           ClassifyBlendMapping(blend, pixel_info.target_export_mapping[0]) ==
-		               BlendMappingSupport::SourceAlpha) {
-			// Preserve logical alpha when the export mapping moves it.
-			pixel_info.alpha_blend_source_remap = true;
-			pixel_info.dual_source_blending     = true;
-			pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
-			pixel_info.target_export_mapping[1] = {};
+		                       [](uint8_t mode) { return mode == 0; })) {
+			switch (ClassifyBlendMapping(blend, pixel_info.target_export_mapping[0])) {
+				case BlendMappingSupport::SourceAlpha:
+					pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlpha;
+					break;
+				case BlendMappingSupport::SourceAlphaOne:
+					pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlphaOne;
+					break;
+				case BlendMappingSupport::SourceAlphaZero:
+					pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlphaZero;
+					break;
+				default: break;
+			}
+			if (pixel_info.alpha_blend_source != ShaderAlphaBlendSource::None) {
+				pixel_info.dual_source_blending     = true;
+				pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
+				pixel_info.target_export_mapping[1] = {};
+			}
 		}
 	}
 	if (context.GetClipControl().clip_disable) {
@@ -658,19 +748,8 @@ ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs
                                                ShaderComputeInputInfo&      input_info) {
 	input_info.host_subgroup_size = m_graphics.SupportsComputeWave64() ? 64u : 32u;
 	const auto        params      = PrepareProgram(regs, sh, input_info);
-	// Use one effective size for the cache key, LDS declaration, and access bounds.
-	const auto max_lds_dwords =
-	    m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize / 4u;
-	if (input_info.lds_size_dwords > max_lds_dwords) {
-		static std::atomic_bool warned = false;
-		if (!warned.exchange(true, std::memory_order_relaxed)) {
-			PipelineCacheLog("GPU warning: game compute shader requests {} bytes of LDS, but "
-			                 "the Vulkan device limit is {} bytes. Clamping LDS; rendering may "
-			                 "be incorrect.",
-			                 input_info.lds_size_dwords * 4u, max_lds_dwords * 4u);
-		}
-	}
-	input_info.lds_size_dwords = std::min(input_info.lds_size_dwords, max_lds_dwords);
+	input_info.lds_storage = input_info.lds_size_dwords * 4u >
+	    m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize;
 	uint32_t          push_data_cursor = 0;
 	return m_program_cache->Get(params, input_info, push_data_cursor);
 }
@@ -727,10 +806,12 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		}
 		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
 		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		const bool alpha_remap =
-		    slot == 0 && ps_input_info != nullptr && ps_input_info->alpha_blend_source_remap;
+		auto alpha_source = ShaderAlphaBlendSource::None;
+		if (slot == 0 && ps_input_info != nullptr) {
+			alpha_source = ps_input_info->alpha_blend_source;
+		}
 		static_params.blend_enable[slot] = bc.enable && !rt.info.blend_bypass;
-		if (static_params.blend_enable[slot] && !alpha_remap &&
+		if (static_params.blend_enable[slot] && alpha_source == ShaderAlphaBlendSource::None &&
 		    ClassifyBlendMapping(bc, colors[i].export_mapping) != BlendMappingSupport::Direct) {
 			static_params.blend_enable[slot] = false;
 			static std::atomic_bool warned = false;
@@ -742,18 +823,32 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 				    bc.alpha_srcblend, bc.alpha_destblend, bc.separate_alpha_blend ? 1 : 0));
 			}
 		}
-		if (alpha_remap) {
-			static_params.blend_alpha_source_remap = true;
-		}
 		if (static_params.blend_enable[slot]) {
-			static_params.color_srcblend[slot]       = bc.color_srcblend;
-			static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
-			static_params.color_destblend[slot]      = bc.color_destblend;
-			static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
-			if (bc.separate_alpha_blend) {
-				static_params.alpha_srcblend[slot]  = bc.alpha_srcblend;
-				static_params.alpha_comb_fcn[slot]  = bc.alpha_comb_fcn;
-				static_params.alpha_destblend[slot] = bc.alpha_destblend;
+			auto blend = bc;
+			switch (alpha_source) {
+				case ShaderAlphaBlendSource::SourceAlpha:
+					blend.color_srcblend  = RemapSourceAlphaFactor(blend.color_srcblend);
+					blend.color_destblend = RemapSourceAlphaFactor(blend.color_destblend);
+					blend.separate_alpha_blend = false;
+					break;
+				case ShaderAlphaBlendSource::SourceAlphaOne:
+				case ShaderAlphaBlendSource::SourceAlphaZero:
+					// The second source carries the mapped source factor; its alpha stays logical Sa.
+					blend.color_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kSrc1Color);
+					blend.color_destblend =
+					    static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrc1Alpha);
+					blend.separate_alpha_blend = false;
+					break;
+				case ShaderAlphaBlendSource::None: break;
+			}
+			static_params.color_srcblend[slot]       = blend.color_srcblend;
+			static_params.color_comb_fcn[slot]       = blend.color_comb_fcn;
+			static_params.color_destblend[slot]      = blend.color_destblend;
+			static_params.separate_alpha_blend[slot] = blend.separate_alpha_blend;
+			if (blend.separate_alpha_blend) {
+				static_params.alpha_srcblend[slot]  = blend.alpha_srcblend;
+				static_params.alpha_comb_fcn[slot]  = blend.alpha_comb_fcn;
+				static_params.alpha_destblend[slot] = blend.alpha_destblend;
 			}
 		}
 	}
@@ -801,9 +896,6 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 	if (static_params.sample_shading_enable && !m_graphics.sample_rate_shading_enabled) {
 		EXIT("Pipeline: sample-rate shading is required but unsupported by the host\n");
 	}
-	static_params.depth_bounds_test_enable = depth.depth_bounds_test_enable;
-	static_params.depth_min_bounds         = depth.depth_min_bounds;
-	static_params.depth_max_bounds         = depth.depth_max_bounds;
 	const bool rect_list = Prospero::IsRectList(command.GetUserConfig().GetPrimType());
 	static_params.cull_back  = !rect_list && mc.cull_back;
 	static_params.cull_front = !rect_list && mc.cull_front;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -38,9 +38,6 @@ struct PipelineStaticParameters {
 	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
-	bool                       depth_bounds_test_enable = false;
-	float                      depth_min_bounds         = 0.0f;
-	float                      depth_max_bounds         = 0.0f;
 	uint32_t                   color_mask[RENDER_COLOR_ATTACHMENTS_MAX]           = {};
 	bool                       cull_front                                         = false;
 	bool                       cull_back                                          = false;
@@ -55,7 +52,6 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
-	bool                       blend_alpha_source_remap                           = false;
 
 	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
@@ -65,7 +61,7 @@ struct PipelineStaticParameters {
 static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
 static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 126);
+static_assert(sizeof(PipelineStaticParameters) == 116);
 
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -19,6 +19,7 @@
 #include "graphics/shader/shader.h"
 
 #include <algorithm>
+#include <bit>
 #include <limits>
 #include <span>
 #include <vector>
@@ -137,17 +138,14 @@ static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format,
 	}
 }
 
-static vk::BlendFactor GetBlendFactor(uint32_t factor, bool remap_source_alpha) {
+static vk::BlendFactor GetBlendFactor(uint32_t factor) {
 	switch (static_cast<Prospero::BlendFactor>(factor)) {
 		case Prospero::BlendFactor::kZero: return vk::BlendFactor::eZero;
 		case Prospero::BlendFactor::kOne: return vk::BlendFactor::eOne;
 		case Prospero::BlendFactor::kSrcColor: return vk::BlendFactor::eSrcColor;
 		case Prospero::BlendFactor::kOneMinusSrcColor: return vk::BlendFactor::eOneMinusSrcColor;
-		case Prospero::BlendFactor::kSrcAlpha:
-			return remap_source_alpha ? vk::BlendFactor::eSrc1Color : vk::BlendFactor::eSrcAlpha;
-		case Prospero::BlendFactor::kOneMinusSrcAlpha:
-			return remap_source_alpha ? vk::BlendFactor::eOneMinusSrc1Color
-			                          : vk::BlendFactor::eOneMinusSrcAlpha;
+		case Prospero::BlendFactor::kSrcAlpha: return vk::BlendFactor::eSrcAlpha;
+		case Prospero::BlendFactor::kOneMinusSrcAlpha: return vk::BlendFactor::eOneMinusSrcAlpha;
 		case Prospero::BlendFactor::kDstAlpha: return vk::BlendFactor::eDstAlpha;
 		case Prospero::BlendFactor::kOneMinusDstAlpha: return vk::BlendFactor::eOneMinusDstAlpha;
 		case Prospero::BlendFactor::kDstColor: return vk::BlendFactor::eDstColor;
@@ -183,10 +181,12 @@ static vk::BlendOp GetBlendOp(uint32_t op) {
 static void AddLayoutBindings(std::vector<vk::DescriptorSetLayoutBinding>& descriptor_bindings,
                               const ShaderRecompiler::IR::CompiledShaderInfo& program,
                               vk::ShaderStageFlagBits              stage) {
-	for (const auto& binding: program.bindings.descriptors) {
+	for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) {
+		const auto index = std::countr_zero(mask);
+		const auto kind = static_cast<ShaderRecompiler::IR::DescriptorBindingKind>(index);
 		descriptor_bindings.push_back(
-		    {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind),
-		     NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), stage, nullptr});
+		    {ShaderRecompiler::IR::NativeBinding(program.stage, kind), NativeDescriptorType(kind),
+		     program.bindings.descriptor_counts[index], stage, nullptr});
 	}
 }
 
@@ -397,20 +397,18 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		color_blend_attachment[i].colorWriteMask =
 		    vk::ColorComponentFlags {static_params.color_mask[i]};
 		color_blend_attachment[i].blendEnable = static_params.blend_enable[i] ? VK_TRUE : VK_FALSE;
-		// Only target 0 can use the second blend source carrying logical alpha.
-		const bool remap_source_alpha         = i == 0 && static_params.blend_alpha_source_remap;
 		color_blend_attachment[i].srcColorBlendFactor =
-		    GetBlendFactor(static_params.color_srcblend[i], remap_source_alpha);
+		    GetBlendFactor(static_params.color_srcblend[i]);
 		color_blend_attachment[i].dstColorBlendFactor =
-		    GetBlendFactor(static_params.color_destblend[i], remap_source_alpha);
+		    GetBlendFactor(static_params.color_destblend[i]);
 		color_blend_attachment[i].colorBlendOp = GetBlendOp(static_params.color_comb_fcn[i]);
 		color_blend_attachment[i].srcAlphaBlendFactor =
 		    (static_params.separate_alpha_blend[i]
-		         ? GetBlendFactor(static_params.alpha_srcblend[i], remap_source_alpha)
+		         ? GetBlendFactor(static_params.alpha_srcblend[i])
 		         : color_blend_attachment[i].srcColorBlendFactor);
 		color_blend_attachment[i].dstAlphaBlendFactor =
 		    (static_params.separate_alpha_blend[i]
-		         ? GetBlendFactor(static_params.alpha_destblend[i], remap_source_alpha)
+		         ? GetBlendFactor(static_params.alpha_destblend[i])
 		         : color_blend_attachment[i].dstColorBlendFactor);
 		color_blend_attachment[i].alphaBlendOp =
 		    (static_params.separate_alpha_blend[i] ? GetBlendOp(static_params.alpha_comb_fcn[i])
@@ -476,14 +474,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr);
 
 	vk::PipelineDepthStencilStateCreateInfo depth_stencil_info {};
-	depth_stencil_info.depthBoundsTestEnable =
-#if defined(__APPLE__)
-	    VK_FALSE; // MoltenVK lacks the depthBounds feature; depth-bounds testing is disabled
-#else
-	    (static_params.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
-#endif
-	depth_stencil_info.minDepthBounds    = static_params.depth_min_bounds;
-	depth_stencil_info.maxDepthBounds    = static_params.depth_max_bounds;
 
 	std::vector<vk::DynamicState> dynamic_states {
 	    vk::DynamicState::eViewportWithCount,
@@ -502,6 +492,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	    vk::DynamicState::eBlendConstants,
 	};
 #if !defined(__APPLE__)
+	dynamic_states.push_back(vk::DynamicState::eDepthBoundsTestEnable);
+	dynamic_states.push_back(vk::DynamicState::eDepthBounds);
 	if (rendering.color_count != 0) {
 		dynamic_states.push_back(vk::DynamicState::eColorWriteEnableEXT);
 	}

src/graphics/host_gpu/renderer/render.h

@@ -156,12 +156,12 @@ public:
 	KYTY_CLASS_NO_COPY(RenderExecutor);
 
 	void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x,
-	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode);
+	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode, bool async_compute);
 	void DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, uint64_t args_addr,
-	                      uint32_t mode);
+	                      uint32_t mode, bool async_compute);
 
 	void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared);
-	void                           FindBuffers(PreparedBindings& bindings);
+	void                           FindBuffers(std::span<PreparedBindings* const> stages);
 	void                           RebindBuffers(PreparedBindings& bindings);
 	void                           RebindImages(PreparedBindings& bindings);
 	void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
@@ -213,10 +213,8 @@ private:
 	GraphicsBindings                     m_graphics_bindings;
 	PreparedBindings                     m_compute_bindings;
 	std::vector<ImageId>                  m_bound_images;
-	std::vector<vk::DescriptorBufferInfo> m_descriptor_buffers;
 	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;
 	std::vector<vk::WriteDescriptorSet>   m_descriptor_writes;
-	std::vector<uint32_t>                 m_image_occurrences;
 
 	friend class CommandProcessor;
 	friend struct RenderExecutorTestAccess;

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -195,9 +195,38 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 	return true;
 }
 
+static void BindSharedMemory(RenderContext& context, ShaderComputeInputInfo& input,
+                             PreparedBindings& bindings, uint64_t indirect_args = 0) {
+	if (input.stage.program->bindings.descriptor_counts[
+	        static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::SharedMemory)] == 0) {
+		return;
+	}
+	auto& cache = context.GetBufferCache();
+	if (indirect_args != 0) {
+		cache.ReadMemory(indirect_args, sizeof(vk::DispatchIndirectCommand));
+		std::memcpy(input.workgroup_counts, reinterpret_cast<const void*>(indirect_args),
+		            sizeof(input.workgroup_counts));
+	}
+	// LDS has no contents to preserve between dispatches. The existing shader hazard
+	// barriers also order other users of this GPU-only utility buffer.
+	auto& storage = cache.GetUtilityBuffer(MemoryUsage::DeviceLocal);
+	const auto limit = std::min<uint64_t>(storage.Size(),
+	    context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
+	uint64_t size = sizeof(uint32_t);
+	if (std::ranges::find(input.workgroup_counts, 0u) == std::end(input.workgroup_counts)) {
+		size = uint64_t {input.lds_size_dwords} * sizeof(uint32_t);
+		EXIT_IF(size == 0 || size > limit);
+		for (const auto count: input.workgroup_counts) {
+			EXIT_IF(size > limit / count);
+			size *= count;
+		}
+	}
+	bindings.shared_memory = {storage.Handle(), 0, size};
+}
+
 void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
                                     uint32_t thread_group_x, uint32_t thread_group_y,
-                                    uint32_t thread_group_z, uint32_t mode) {
+                                    uint32_t thread_group_z, uint32_t mode, bool async_compute) {
 	EXIT_IF(buffer.IsInvalid());
 	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ctx    = buffer.GetRegisters();
@@ -252,8 +281,21 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	const auto& sh_regs = ctx.GetShaderRegisters();
 
 	ShaderComputeInputInfo input_info {};
+	input_info.async_compute = async_compute;
 	const bool use_thread_dimensions = (mode & DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0;
 	input_info.dispatch_thread_dimensions = use_thread_dimensions;
+	input_info.workgroup_counts[0] = thread_group_x;
+	input_info.workgroup_counts[1] = thread_group_y;
+	input_info.workgroup_counts[2] = thread_group_z;
+	if (use_thread_dimensions) {
+		const uint32_t group_sizes[] = {cs_regs.cs_regs.num_thread_x, cs_regs.cs_regs.num_thread_y,
+		                                cs_regs.cs_regs.num_thread_z};
+		for (uint32_t axis = 0; axis < 3u; ++axis) {
+			const auto size = std::max(group_sizes[axis], 1u);
+			const auto threads = input_info.workgroup_counts[axis];
+			input_info.workgroup_counts[axis] = threads / size + (threads % size != 0u);
+		}
+	}
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
 	if (use_thread_dimensions) {
@@ -272,22 +314,75 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	const bool large_workgroup =
-	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
-	const bool                   has_sampler = !program.info.samplers.empty();
-	static std::atomic<uint32_t> dispatch_log_count {0};
-	if ((large_workgroup || has_sampler) &&
-	    dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) {
+	if (use_thread_dimensions) {
+		const uint32_t old_x = thread_group_x;
+		const uint32_t old_y = thread_group_y;
+		const uint32_t old_z = thread_group_z;
+		thread_group_x       = input_info.workgroup_counts[0];
+		thread_group_y       = input_info.workgroup_counts[1];
+		thread_group_z       = input_info.workgroup_counts[2];
+
+		static std::atomic<uint32_t> log_count {0};
+		if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
+			LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> "
+			     "groups %ux%ux%u\n",
+			     old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u),
+			     std::max(cs_regs.cs_regs.num_thread_y, 1u),
+			     std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y,
+			     thread_group_z);
+		}
+	}
+
+	buffer.EndRendering();
+	auto& pipeline =
+	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
+	auto& bindings = m_compute_bindings;
+	PrepareBindings(input_info.stage, bindings);
+	if (program.bindings.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
+		std::copy(std::begin(input_info.dispatch_threads_num), std::end(input_info.dispatch_threads_num),
+		          bindings.shader_data.begin() + program.bindings.dispatch_thread_dword);
+	}
+	PreparedBindings* descriptor_stage = &bindings;
+	FindBuffers(std::span {&descriptor_stage, 1u});
+	if (program.info.uses_dma) {
+		m_context.PrepareBda();
+	}
+	RebindImages(bindings);
+	BindSharedMemory(m_context, input_info, bindings);
+	RebindBuffers(bindings);
+
+	auto              vk_buffer        = buffer.Handle();
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
+	bool has_storage_writes = bindings.shared_memory.buffer != nullptr ||
+	    HasShaderBufferWrites(input_info.stage);
+	has_storage_writes =
+	    std::any_of(program.info.images.begin(), program.info.images.end(),
+	                [](const auto& image) {
+		                return image.written &&
+		                       image.resource_class ==
+		                           ShaderRecompiler::IR::ImageResourceClass::Storage;
+	                }) ||
+	    has_storage_writes;
+	if (has_storage_writes) {
+		// A host fence used to serialize every dispatch. Preserve its read-before-write ordering
+		// while allowing the queue to execute asynchronously.
+		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
+	}
+	if (Config::GraphicsDebugDumpEnabled()) {
 		const auto sampled_images = std::count_if(
 		    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
 			    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
 		    });
 		const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
 		LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64
+		     " hash=0x%016" PRIx64 " tick=%" PRIu64
 		     " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u "
 		     "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n",
-		     frame_num, sh_ctx.GetCs().cs_regs.data_addr, thread_group_x, thread_group_y,
-		     thread_group_z, mode, input_info.threads_num[0], input_info.threads_num[1],
+		     frame_num, sh_ctx.GetCs().cs_regs.data_addr, program.shader_hash,
+		     m_context.GetCommandScheduler().CurrentTick(),
+		     thread_group_x, thread_group_y, thread_group_z, mode,
+		     input_info.threads_num[0], input_info.threads_num[1],
 		     input_info.threads_num[2], program.info.buffers.size(), program.info.images.size(),
 		     sampled_images, program.info.images.size() - sampled_images,
 		     program.info.samplers.size(),
@@ -334,61 +429,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		}
 	}
 
-	if (use_thread_dimensions) {
-		auto groups_from_threads = [](uint32_t threads, uint32_t group_size) {
-			return (threads == 0
-			            ? 0u
-			            : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u));
-		};
-
-		const uint32_t old_x = thread_group_x;
-		const uint32_t old_y = thread_group_y;
-		const uint32_t old_z = thread_group_z;
-		thread_group_x       = groups_from_threads(thread_group_x, cs_regs.cs_regs.num_thread_x);
-		thread_group_y       = groups_from_threads(thread_group_y, cs_regs.cs_regs.num_thread_y);
-		thread_group_z       = groups_from_threads(thread_group_z, cs_regs.cs_regs.num_thread_z);
-
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
-			LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> "
-			     "groups %ux%ux%u\n",
-			     old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u),
-			     std::max(cs_regs.cs_regs.num_thread_y, 1u),
-			     std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y,
-			     thread_group_z);
-		}
-	}
-
-	buffer.EndRendering();
-	auto& pipeline =
-	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
-	auto& bindings = m_compute_bindings;
-	PrepareBindings(input_info.stage, bindings);
-	FindBuffers(bindings);
-	if (program.info.uses_dma) {
-		m_context.PrepareBda();
-	}
-	RebindImages(bindings);
-	RebindBuffers(bindings);
-
-	auto              vk_buffer        = buffer.Handle();
-	PreparedBindings* descriptor_stage = &bindings;
-	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
-	               std::span {&descriptor_stage, 1u});
-	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
-	has_storage_writes =
-	    std::any_of(program.info.images.begin(), program.info.images.end(),
-	                [](const auto& image) {
-		                return image.written &&
-		                       image.resource_class ==
-		                           ShaderRecompiler::IR::ImageResourceClass::Storage;
-	                }) ||
-	    has_storage_writes;
-	if (has_storage_writes) {
-		// A host fence used to serialize every dispatch. Preserve its read-before-write ordering
-		// while allowing the queue to execute asynchronously.
-		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
-	}
 	vk_buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline);
 	vk_buffer.dispatch(thread_group_x, thread_group_y, thread_group_z);
 
@@ -398,7 +438,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 }
 
 void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
-                                      uint64_t args_addr, uint32_t mode) {
+                                      uint64_t args_addr, uint32_t mode, bool async_compute) {
 	EXIT_IF(buffer.IsInvalid() || args_addr == 0 || (args_addr & 3u) != 0 ||
 	        (mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0);
 	m_context.GetCommandScheduler().PopPendingOperations();
@@ -411,28 +451,31 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 	ShaderComputeInputInfo input_info {};
+	input_info.async_compute = async_compute;
 	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
 	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
 	buffer.EndRendering();
 	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto& bindings = m_compute_bindings;
 	PrepareBindings(input_info.stage, bindings);
-	FindBuffers(bindings);
+	PreparedBindings* descriptor_stage = &bindings;
+	FindBuffers(std::span {&descriptor_stage, 1u});
 	const auto& program = *input_info.stage.program;
 	if (program.info.uses_dma) {
 		m_context.PrepareBda();
 	}
+	BindSharedMemory(m_context, input_info, bindings, args_addr);
 	RebindImages(bindings);
 	// Acquiring arguments can merge cache buffers; finalize shader bindings afterward.
 	const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer(
 	    args_addr, sizeof(vk::DispatchIndirectCommand), false);
 	EXIT_IF(args_buffer == nullptr || (args_offset & 3u) != 0);
 	RebindBuffers(bindings);
-	PreparedBindings* descriptor_stage = &bindings;
 	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
 	               std::span {&descriptor_stage, 1u});
 	const auto vk_buffer = buffer.Handle();
-	const bool has_storage_writes = HasShaderBufferWrites(input_info.stage) ||
+	const bool has_storage_writes = bindings.shared_memory.buffer != nullptr ||
+	    HasShaderBufferWrites(input_info.stage) ||
 	    std::any_of(program.info.images.begin(), program.info.images.end(), [](const auto& image) {
 		    return image.written && image.resource_class ==
 		                                ShaderRecompiler::IR::ImageResourceClass::Storage;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -99,7 +99,12 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		     vaddr, size);
 	}
 	const auto unmap = [this, vaddr, size] {
-		if (m_command_scheduler.Active()) {
+		// Check cache ownership on the GPU thread. Guest-memory callbacks can still
+		// access a range with no cached data, so they must finish before it is unmapped.
+		if (m_command_scheduler.Active() &&
+		    (m_buffer_cache.IsRegionRegistered(vaddr, size) ||
+		     m_texture_cache.IsRegionRegistered(vaddr, size) ||
+		     m_command_scheduler.HasPendingPriorityOperations())) {
 			const auto tick = m_command_scheduler.CurrentTick();
 			m_command_scheduler.Finish();
 			m_command_scheduler.WaitPriorityOperations(tick);

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -379,6 +379,12 @@ static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuf
 	vk_buffer.setDepthTestEnable(depth.depth_test_enable ? VK_TRUE : VK_FALSE);
 	vk_buffer.setDepthWriteEnable(depth.depth_write_enable ? VK_TRUE : VK_FALSE);
 	vk_buffer.setDepthCompareOp(depth.depth_compare_op);
+#if !defined(__APPLE__)
+	vk_buffer.setDepthBoundsTestEnable(depth.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
+	if (depth.depth_bounds_test_enable) {
+		vk_buffer.setDepthBounds(ctx.GetDepthBoundsMin(), ctx.GetDepthBoundsMax());
+	}
+#endif
 
 	const auto& mode              = ctx.GetModeControl();
 	const auto& poly_offset       = ctx.GetPolyOffset();
@@ -583,8 +589,12 @@ static uint32_t DrawColorOutputMask(const HW::Context& ctx) {
 	const auto& sh_regs     = ctx.GetShaderRegisters();
 	const auto  write_mask  = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask;
 	uint32_t    output_mask = 0;
-	for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
-		if (sh_regs.target_output_mode[slot] != 0 &&
+	for (uint32_t index = 0; index < RENDER_COLOR_ATTACHMENTS_MAX; index++) {
+		const auto slot = ShaderPixelExportTarget(sh_regs.m_cbShaderMask, index);
+		if (slot >= RENDER_COLOR_ATTACHMENTS_MAX) {
+			break;
+		}
+		if (sh_regs.target_output_mode[index] != 0 &&
 		    render_target_mask_slot(write_mask, slot) != 0) {
 			output_mask |= 1u << slot;
 		}
@@ -784,6 +794,7 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, vk::PrimitiveTopology& t
 			topology = vk::PrimitiveTopology::eTriangleList;
 			break;
 		case Prospero::PrimitiveType::kTriFan:
+		case Prospero::PrimitiveType::kPolygon:
 			topology = vk::PrimitiveTopology::eTriangleFan;
 			break;
 		case Prospero::PrimitiveType::kTriStrip:
@@ -895,7 +906,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 	if (state.ps_active) {
 		for (const auto& output: state.ps_input_info.stage.program->info.outputs) {
 			if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt) {
-				mrt_mask |= 1u << output.index;
+				mrt_mask |= 1u << output.location;
 			}
 		}
 	}
@@ -994,6 +1005,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
 		case Prospero::PrimitiveType::kRectList:
+		case Prospero::PrimitiveType::kPolygon:
 		case Prospero::PrimitiveType::kRectListLegacy:
 		case Prospero::PrimitiveType::kPatch:
 			if (draw.IsIndexed()) {
@@ -1272,8 +1284,10 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
 
 	DrawEmitInfo emit {};
-	emit.vertex_offset  = vertex_offset + args.base_vertex;
-	emit.first_instance = instance_offset;
+	// Native fetches consume the patched SGPRs; rewritten fetches need Vulkan offsets.
+	const bool native_indirect = indirect && !state.vertex_info[0].fetch_embedded;
+	emit.vertex_offset  = native_indirect ? 0 : vertex_offset + args.base_vertex;
+	emit.first_instance = native_indirect ? 0 : instance_offset;
 
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
 	                    primitive_restart);
@@ -1361,8 +1375,10 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	    indirect ? std::pair<int32_t, uint32_t> {0, args.first_instance}
 	             : ResolveDrawOffsets(ucfg.GetIndexOffset(), state.vertex_info[0]);
 	DrawEmitInfo emit {};
-	emit.first_vertex = static_cast<uint32_t>(vertex_offset + static_cast<int32_t>(args.first_vertex));
-	emit.first_instance = instance_offset;
+	const bool native_indirect = indirect && !state.vertex_info[0].fetch_embedded;
+	emit.first_vertex = native_indirect ? 0 :
+	    static_cast<uint32_t>(vertex_offset + static_cast<int32_t>(args.first_vertex));
+	emit.first_instance = native_indirect ? 0 : instance_offset;
 
 	DrawIndexBufferSource index_source {};
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false);

src/graphics/host_gpu/shaders/gpu_tiler_color.inc

@@ -0,0 +1,20 @@
+// ColorTransform values match imageInfo.h. Preserve the floating-point bit fields exactly.
+uvec2 transform_color(uvec2 value, uint transform, bool to_host) {
+	switch (transform) {
+		case 1u:
+			return uvec2((value.y & 0xffffu) | (value.x & 0xffff0000u),
+			             (value.x & 0xffffu) | (value.y & 0xffff0000u));
+		case 2u: {
+			uint packed = value.x;
+			if (to_host) {
+				value.x = (packed >> 21u) | (((packed >> 10u) & 0x7ffu) << 11u) |
+				          ((packed & 0x3ffu) << 22u);
+			} else {
+				value.x = (packed >> 22u) | (((packed >> 11u) & 0x7ffu) << 10u) |
+				          ((packed & 0x7ffu) << 21u);
+			}
+			break;
+		}
+	}
+	return value;
+}

src/graphics/host_gpu/shaders/gpu_tiler_color_transform.comp

@@ -1,4 +1,5 @@
 #version 450
+#extension GL_GOOGLE_include_directive : require
 
 layout(local_size_x = 64) in;
 
@@ -19,17 +20,23 @@ layout(push_constant, std430) uniform Push {
 	uint tail_x;
 	uint tail_y;
 	uint tail;
+	uint color_transform;
+	uint to_host;
 } params;
 
+#include "gpu_tiler_color.inc"
+
 void main() {
 	uint pixel = gl_GlobalInvocationID.x;
 	if (pixel >= params.width) {
 		return;
 	}
-	uint src = (params.src_base >> 2u) + pixel * 2u;
-	uint dst = (params.dst_base >> 2u) + pixel * 2u;
-	uint bg = input_buffer.data[src];
-	uint ra = input_buffer.data[src + 1u];
-	output_buffer.data[dst] = (ra & 0x0000ffffu) | (bg & 0xffff0000u);
-	output_buffer.data[dst + 1u] = (bg & 0x0000ffffu) | (ra & 0xffff0000u);
+	uint words = params.color_transform == 1u ? 2u : 1u;
+	uint src = (params.src_base >> 2u) + pixel * words;
+	uint dst = (params.dst_base >> 2u) + pixel * words;
+	uvec2 value = uvec2(input_buffer.data[src], 0u);
+	if (words == 2u) value.y = input_buffer.data[src + 1u];
+	value = transform_color(value, params.color_transform, params.to_host != 0u);
+	output_buffer.data[dst] = value.x;
+	if (words == 2u) output_buffer.data[dst + 1u] = value.y;
 }

src/graphics/host_gpu/shaders/gpu_tiler_common.inc

@@ -17,10 +17,19 @@ layout(set = 0, binding = 2, std140) uniform Push {
 	uint tail_x;
 	uint tail_y;
 	uint tail;
+	uint color_transform;
 } params;
 
+#include "gpu_tiler_color.inc"
+
 void copy_element(uint src, uint dst) {
-	if (ELEMENT_BYTES >= 4) {
+	if ((ELEMENT_BYTES == 4u || ELEMENT_BYTES == 8u) && params.color_transform != 0u) {
+		uvec2 value = uvec2(input_buffer.data[src >> 2], 0u);
+		if (ELEMENT_BYTES == 8u) value.y = input_buffer.data[(src >> 2) + 1u];
+		value = transform_color(value, params.color_transform, TILE == 0u);
+		output_buffer.data[dst >> 2] = value.x;
+		if (ELEMENT_BYTES == 8u) output_buffer.data[(dst >> 2) + 1u] = value.y;
+	} else if (ELEMENT_BYTES >= 4) {
 		for (uint i = 0; i < ELEMENT_BYTES; i += 4) {
 			output_buffer.data[(dst + i) >> 2] = input_buffer.data[(src + i) >> 2];
 		}

src/graphics/host_gpu/vulkanCommon.cpp

@@ -48,6 +48,7 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
     {Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
     {Prospero::BufferFormat::k10_10_10_2SNorm, vk::Format::eA2B10G10R10SnormPack32},
+    {Prospero::BufferFormat::k10_10_10_2UScaled, vk::Format::eA2B10G10R10UscaledPack32},
     {Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
     {Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
     {Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},

src/graphics/presentation/systemOverlay.cpp

@@ -3,14 +3,17 @@
 #include <SDL3/SDL.h>
 
 #include "common/assert.h"
+#include "common/logging/log.h"
 #include "common/stringUtils.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "imgui.h"
+#include "imgui_internal.h"
 #include "imgui_impl_vulkan.h"
 #include "libs/controller.h"
 #include "libs/dialog.h"
 #include "libs/ime.h"
 #include "libs/imeDialog.h"
+#include "stb_image.h"
 
 #include <algorithm>
 #include <array>
@@ -18,12 +21,17 @@
 #include <chrono>
 #include <cmath>
 #include <cstdint>
+#include <cstring>
 #include <deque>
+#include <limits>
+#include <memory>
 #include <mutex>
+#include <optional>
 #include <span>
 #include <string>
 #include <string_view>
 #include <utility>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -178,6 +186,45 @@ bool                         g_input_lifecycle_active     = false;
 bool                         g_controller_captured        = false;
 OverlaySession               g_session;
 SDL_Window*                  g_input_window               = nullptr;
+struct TrophyNotification {
+	std::string            name;
+	int32_t                grade;
+	std::vector<std::byte> icon_png;
+};
+std::mutex                     g_trophy_notification_mutex;
+std::deque<TrophyNotification> g_trophy_notifications;
+std::atomic<bool>              g_trophy_notification_active {false};
+
+std::vector<std::string_view> WrapTrophyTitle(std::string_view text, float font_size, float width) {
+	std::vector<std::string_view> lines;
+	auto* const                  font = ImGui::GetFont();
+	for (;;) {
+		const auto newline   = text.find('\n');
+		const auto paragraph = text.substr(0, newline);
+		auto       line      = paragraph.substr(0, 0);
+		for (auto start = paragraph.find_first_not_of(" \t\r"); start != std::string_view::npos;) {
+			const auto end  = std::min(paragraph.find_first_of(" \t\r", start), paragraph.size());
+			const auto word = paragraph.substr(start, end - start);
+			const std::string_view candidate(line.empty() ? word.begin() : line.begin(),
+			                                 word.end());
+			// Keep an oversized word intact; the toast's clip rectangle handles overflow.
+			if (!line.empty() && font->CalcTextSizeA(font_size, std::numeric_limits<float>::max(),
+			                                       0.0f, candidate.data(),
+			                                       candidate.data() + candidate.size()).x > width) {
+				lines.push_back(line);
+				line = word;
+			} else {
+				line = candidate;
+			}
+			start = paragraph.find_first_not_of(" \t\r", end);
+		}
+		lines.push_back(line);
+		if (newline == std::string_view::npos) {
+			return lines;
+		}
+		text.remove_prefix(newline + 1);
+	}
+}
 
 void ClearInputEvents() {
 	std::scoped_lock lock(g_input_mutex);
@@ -467,8 +514,9 @@ SystemOverlayVisualState GetSystemOverlayVisualState() noexcept {
 	const auto core   = CoreIme::GetVisualState();
 	const auto dialog = DialogIme::GetVisualState();
 	const auto system = SystemDialog::GetVisualState();
-	return {core.active || dialog.active || system.active,
-	        core.revision + dialog.revision + system.revision};
+	const bool trophy_active = g_trophy_notification_active.load(std::memory_order_acquire);
+	return {core.active || dialog.active || system.active || trophy_active,
+	        core.revision + dialog.revision + system.revision + (trophy_active ? 1u : 0u)};
 }
 
 bool ProcessSystemOverlayInput(const SDL_Event& event) {
@@ -599,12 +647,34 @@ bool ProcessSystemOverlayInput(const SDL_Event& event) {
 	}
 }
 
+void NotifyTrophyUnlocked(std::string_view name, int32_t grade,
+                          std::span<const std::byte> icon_png) {
+	std::scoped_lock lock(g_trophy_notification_mutex);
+	g_trophy_notifications.push_back(
+	    {std::string(name), grade, std::vector<std::byte>(icon_png.begin(), icon_png.end())});
+	g_trophy_notification_active.store(true, std::memory_order_release);
+}
+
 struct SystemOverlay::Impl {
+	using Clock = std::chrono::steady_clock;
+
 	explicit Impl(GraphicContext& context): graphics(context) {}
 
 	~Impl() {
+		SDL_DestroyAudioStream(trophy_sound_stream);
+		SDL_free(trophy_sound_data);
+		if (trophy_sound_initialized) {
+			SDL_QuitSubSystem(SDL_INIT_AUDIO);
+		}
 		ReleaseVulkan();
 		if (imgui_context != nullptr) {
+			ImGui::SetCurrentContext(imgui_context);
+			if (trophy_notification_image) {
+				ImGui::UnregisterUserTexture(trophy_notification_image.get());
+			}
+			for (const auto& texture: retired_trophy_images) {
+				ImGui::UnregisterUserTexture(texture.get());
+			}
 			ImGui::DestroyContext(imgui_context);
 		}
 	}
@@ -631,6 +701,75 @@ struct SystemOverlay::Impl {
 		style.ItemSpacing    = {8.0f, 8.0f};
 	}
 
+	void PlayTrophySound() {
+		if (!trophy_sound_attempted) {
+			trophy_sound_attempted   = true;
+			trophy_sound_initialized = SDL_InitSubSystem(SDL_INIT_AUDIO);
+			if (!trophy_sound_initialized) {
+				LOGF("[Trophy] could not initialize audio: %s\n", SDL_GetError());
+				return;
+			}
+			const char* base_path = SDL_GetBasePath();
+			if (base_path == nullptr) {
+				LOGF("[Trophy] could not locate sound: %s\n", SDL_GetError());
+				return;
+			}
+			const std::string path = std::string(base_path) + "assets/sounds/trophy-unlock.wav";
+			SDL_AudioSpec     spec {};
+			if (!SDL_LoadWAV(path.c_str(), &spec, &trophy_sound_data, &trophy_sound_size)) {
+				LOGF("[Trophy] could not load %s: %s\n", path.c_str(), SDL_GetError());
+				return;
+			}
+			if (trophy_sound_size > static_cast<uint32_t>(std::numeric_limits<int>::max())) {
+				LOGF("[Trophy] sound is too large: %s\n", path.c_str());
+				return;
+			}
+			trophy_sound_stream = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK,
+			                                                &spec, nullptr, nullptr);
+			if (trophy_sound_stream == nullptr) {
+				LOGF("[Trophy] could not open audio stream: %s\n", SDL_GetError());
+				return;
+			}
+		}
+		if (trophy_sound_stream != nullptr &&
+		    (!SDL_PutAudioStreamData(trophy_sound_stream, trophy_sound_data,
+		                             static_cast<int>(trophy_sound_size)) ||
+		     !SDL_ResumeAudioStreamDevice(trophy_sound_stream))) {
+			LOGF("[Trophy] could not play sound: %s\n", SDL_GetError());
+		}
+	}
+
+	std::unique_ptr<ImTextureData> LoadTrophyIcon(std::span<const std::byte> encoded) {
+		if (encoded.empty() ||
+		    encoded.size() > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
+			return nullptr;
+		}
+
+		const auto* data   = reinterpret_cast<const stbi_uc*>(encoded.data());
+		const int   size   = static_cast<int>(encoded.size());
+		int         width  = 0;
+		int         height = 0;
+		if (!stbi_info_from_memory(data, size, &width, &height, nullptr) || width <= 0 ||
+		    height <= 0 || width > 512 || height > 512) {
+			LOGF("[Trophy] invalid icon or dimensions exceed 512x512\n");
+			return nullptr;
+		}
+		auto* pixels = stbi_load_from_memory(data, size, &width, &height, nullptr, 4);
+		if (pixels == nullptr) {
+			LOGF("[Trophy] could not decode trophy icon: %s\n", stbi_failure_reason());
+			return nullptr;
+		}
+
+		auto texture = std::make_unique<ImTextureData>();
+		texture->Create(ImTextureFormat_RGBA32, width, height);
+		std::memcpy(texture->Pixels, pixels,
+		            static_cast<std::size_t>(width) * static_cast<std::size_t>(height) * 4);
+		texture->UseColors = true;
+		stbi_image_free(pixels);
+		ImGui::RegisterUserTexture(texture.get());
+		return texture;
+	}
+
 	void EnsureVulkan(vk::Format format, uint32_t image_count) {
 		EnsureContext();
 		if (vulkan_initialized) {
@@ -952,9 +1091,131 @@ struct SystemOverlay::Impl {
 		}
 	}
 
+	void DrawTrophyNotification(vk::Extent2D frame_extent, Clock::time_point now) {
+		std::erase_if(retired_trophy_images, [](const auto& texture) {
+			if (texture->Status != ImTextureStatus_Destroyed) {
+				return false;
+			}
+			ImGui::UnregisterUserTexture(texture.get());
+			return true;
+		});
+		constexpr float TOAST_LIFETIME = 5.0f;
+		if (trophy_notification &&
+		    std::chrono::duration<float>(now - trophy_notification_born).count() >=
+		        TOAST_LIFETIME) {
+			trophy_notification.reset();
+			if (trophy_notification_image) {
+				// Keep the texture registered until the backend retires in-flight frames.
+				trophy_notification_image->WantDestroyNextFrame = true;
+				retired_trophy_images.push_back(std::move(trophy_notification_image));
+			}
+		}
+		if (!trophy_notification) {
+			{
+				std::scoped_lock lock(g_trophy_notification_mutex);
+				if (g_trophy_notifications.empty()) {
+					g_trophy_notification_active.store(!retired_trophy_images.empty(),
+					                                   std::memory_order_release);
+					return;
+				}
+				trophy_notification = std::move(g_trophy_notifications.front());
+				g_trophy_notifications.pop_front();
+			}
+			trophy_notification_born  = now;
+			trophy_notification_image = LoadTrophyIcon(trophy_notification->icon_png);
+			PlayTrophySound();
+		}
+		const float age = std::chrono::duration<float>(now - trophy_notification_born).count();
+		const float screen_scale =
+		    std::clamp(std::min(static_cast<float>(frame_extent.width) / 1920.0f,
+		                        static_cast<float>(frame_extent.height) / 1080.0f),
+		               0.65f, 1.25f);
+		const float width = std::max(
+		    0.0f, std::min(520.0f * screen_scale, static_cast<float>(frame_extent.width) - 32.0f));
+		constexpr std::array GRADE_COLORS {
+		    IM_COL32(161, 225, 245, 255), IM_COL32(255, 207, 82, 255), IM_COL32(211, 220, 232, 255),
+		    IM_COL32(211, 142, 91, 255)};
+		const int   grade        = trophy_notification->grade;
+		const bool  has_grade    = grade > 0 && grade <= static_cast<int>(GRADE_COLORS.size());
+		const float pad          = 10.0f * screen_scale;
+		const float base_height  = 104.0f * screen_scale;
+		const float icon_size    = base_height - 2.0f * pad;
+		const float text_offset  = trophy_notification_image != nullptr
+		                               ? pad + icon_size + 18.0f * screen_scale
+		                               : 20.0f * screen_scale;
+		const float title_offset = text_offset + (has_grade ? 30.0f * screen_scale : 0.0f);
+		const float title_size   = 26.0f * screen_scale;
+		const float line_height  = 30.0f * screen_scale;
+		const float text_padding = 16.0f * screen_scale;
+		const auto  title_lines  = WrapTrophyTitle(trophy_notification->name, title_size,
+		                                           width - title_offset - text_padding);
+		const float block_height =
+		    static_cast<float>(title_lines.size()) * line_height + 28.0f * screen_scale;
+		const float  height   = std::max(base_height, block_height + 2.0f * text_padding);
+		const float  rounding = 14.0f * screen_scale;
+		const float  fade_in  = std::clamp(age / 0.22f, 0.0f, 1.0f);
+		const float  fade_out = std::clamp((TOAST_LIFETIME - age) / 0.4f, 0.0f, 1.0f);
+		const float  alpha    = std::min(fade_in, fade_out);
+		const float  slide    = (1.0f - fade_in) * (width + 24.0f);
+		const ImVec2 top_left {static_cast<float>(frame_extent.width) - 32.0f - width + slide,
+		                       32.0f};
+		const ImVec2 bottom_right {top_left.x + width, top_left.y + height};
+		auto*        draw = ImGui::GetForegroundDrawList();
+		const auto   fade = [alpha](int r, int g, int b, int a) {
+            return IM_COL32(r, g, b, static_cast<int>(static_cast<float>(a) * alpha));
+		};
+		draw->AddRectFilled({top_left.x + 3.0f, top_left.y + 5.0f},
+		                    {bottom_right.x + 3.0f, bottom_right.y + 5.0f}, fade(0, 0, 0, 90),
+		                    rounding);
+		draw->AddRectFilled(top_left, bottom_right, fade(24, 26, 32, 235), rounding);
+		draw->AddRect(top_left, bottom_right, fade(255, 255, 255, 28), rounding,
+		              ImDrawFlags_RoundCornersAll, 1.0f);
+
+		const float text_left = top_left.x + text_offset;
+		if (trophy_notification_image != nullptr) {
+			const ImVec2 icon_min {top_left.x + pad, top_left.y + (height - icon_size) * 0.5f};
+			const ImVec2 icon_max {icon_min.x + icon_size, icon_min.y + icon_size};
+			draw->AddImageRounded(trophy_notification_image->GetTexRef(), icon_min, icon_max,
+			                      {0, 0}, {1, 1}, fade(255, 255, 255, 255), 8.0f * screen_scale);
+			draw->AddRect(icon_min, icon_max, fade(255, 255, 255, 40), 8.0f * screen_scale,
+			              ImDrawFlags_RoundCornersAll, 1.0f);
+		}
+
+		const float title_y = top_left.y + (height - block_height) * 0.5f;
+		const float title_x = top_left.x + title_offset;
+		if (has_grade) {
+			// Small cup glyph tinted with the trophy grade.
+			const ImU32 cup = ImGui::GetColorU32(GRADE_COLORS[grade - 1], alpha);
+			const float u   = screen_scale;
+			const float cx  = text_left + 9.0f * u;
+			const float cy  = title_y + 15.0f * u;
+			draw->AddRectFilled({cx - 8.0f * u, cy - 11.0f * u}, {cx + 8.0f * u, cy + 2.0f * u},
+			                    cup, 2.0f * u, ImDrawFlags_RoundCornersBottom);
+			draw->AddRect({cx - 11.0f * u, cy - 9.0f * u}, {cx - 6.0f * u, cy - 3.0f * u}, cup,
+			              2.0f * u, ImDrawFlags_RoundCornersLeft, 1.8f * u);
+			draw->AddRect({cx + 6.0f * u, cy - 9.0f * u}, {cx + 11.0f * u, cy - 3.0f * u}, cup,
+			              2.0f * u, ImDrawFlags_RoundCornersRight, 1.8f * u);
+			draw->AddRectFilled({cx - 1.5f * u, cy + 2.0f * u}, {cx + 1.5f * u, cy + 7.0f * u}, cup);
+			draw->AddRectFilled({cx - 5.0f * u, cy + 7.0f * u}, {cx + 5.0f * u, cy + 10.0f * u}, cup,
+			                    1.5f * u);
+		}
+		draw->PushClipRect(top_left, {bottom_right.x - text_padding, bottom_right.y}, true);
+		float line_y = title_y;
+		for (const auto line : title_lines) {
+			draw->AddText(nullptr, title_size, {title_x, line_y}, fade(255, 255, 255, 255),
+			              line.data(), line.data() + line.size());
+			line_y += line_height;
+		}
+		draw->AddText(nullptr, 20.0f * screen_scale, {text_left, line_y + 8.0f * screen_scale},
+		              fade(176, 180, 192, 255), "Trophy earned!");
+		draw->PopClipRect();
+	}
+
 	bool PrepareFrame(vk::Extent2D frame_extent, vk::Format format, uint32_t image_count) {
 		OverlaySnapshot snapshot;
-		if (!GetOverlaySnapshot(&snapshot)) {
+		const bool      has_overlay = GetOverlaySnapshot(&snapshot);
+		const auto now = Clock::now();
+		if (!has_overlay && !g_trophy_notification_active.load(std::memory_order_acquire)) {
 			return false;
 		}
 		const auto prepared_session = snapshot.session;
@@ -977,7 +1238,6 @@ struct SystemOverlay::Impl {
 		auto& io       = ImGui::GetIO();
 		io.DisplaySize = {static_cast<float>(frame_extent.width),
 		                  static_cast<float>(frame_extent.height)};
-		const auto now = std::chrono::steady_clock::now();
 		io.DeltaTime   = last_frame == std::chrono::steady_clock::time_point {}
 		                     ? 1.0f / 60.0f
 		                     : std::clamp(std::chrono::duration<float>(now - last_frame).count(),
@@ -985,15 +1245,19 @@ struct SystemOverlay::Impl {
 		last_frame     = now;
 		ImGui_ImplVulkan_NewFrame();
 		ImGui::NewFrame();
-		if (!GetOverlaySnapshot(&snapshot) || snapshot.session != prepared_session) {
+		GetOverlaySnapshot(&snapshot);
+		if (snapshot.session != prepared_session) {
 			ImGui::EndFrame();
 			return false;
 		}
-		if (snapshot.session.kind == OverlayKind::Dialog) {
-			DrawDialog(snapshot.dialog, frame_extent);
-		} else {
-			DrawIme(snapshot.ime, frame_extent);
+		if (has_overlay) {
+			if (snapshot.session.kind == OverlayKind::Dialog) {
+				DrawDialog(snapshot.dialog, frame_extent);
+			} else {
+				DrawIme(snapshot.ime, frame_extent);
+			}
 		}
+		DrawTrophyNotification(frame_extent, now);
 		ImGui::Render();
 		extent = frame_extent;
 		return true;
@@ -1040,9 +1304,18 @@ struct SystemOverlay::Impl {
 	float                                 button_height      = 42.0f;
 	ImVec2                                panel_offset {};
 	ImVec2                                right_stick {};
-	OverlaySession                        session;
-	vk::Extent2D                          extent {};
-	std::chrono::steady_clock::time_point last_frame;
+	OverlaySession                              session;
+	std::unique_ptr<ImTextureData>              trophy_notification_image;
+	std::optional<TrophyNotification>           trophy_notification;
+	Clock::time_point                           trophy_notification_born;
+	std::vector<std::unique_ptr<ImTextureData>> retired_trophy_images;
+	SDL_AudioStream*                            trophy_sound_stream      = nullptr;
+	uint8_t*                                    trophy_sound_data        = nullptr;
+	uint32_t                                    trophy_sound_size        = 0;
+	bool                                        trophy_sound_attempted   = false;
+	bool                                        trophy_sound_initialized = false;
+	vk::Extent2D                                extent {};
+	std::chrono::steady_clock::time_point       last_frame;
 };
 
 SystemOverlay::SystemOverlay(GraphicContext& graphics): m_impl(std::make_unique<Impl>(graphics)) {}

src/graphics/presentation/systemOverlay.h

@@ -4,7 +4,10 @@
 #include "common/common.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
+#include <cstddef>
 #include <memory>
+#include <span>
+#include <string_view>
 
 union SDL_Event;
 struct SDL_Window;
@@ -22,6 +25,8 @@ void                     InitializeSystemOverlayInput(SDL_Window* window);
 void                     ShutdownSystemOverlayInput();
 bool                     ProcessSystemOverlayInput(const SDL_Event& event);
 SystemOverlayVisualState GetSystemOverlayVisualState() noexcept;
+void                     NotifyTrophyUnlocked(std::string_view name, int32_t grade,
+                                              std::span<const std::byte> icon_png);
 
 class SystemOverlay final {
 public:

src/graphics/presentation/window/hostInput.cpp

@@ -454,7 +454,7 @@ void HostInputToggleMouseToJoystick() {
 	LOGF("Mouse to right stick: enabled (F7 to release)\n");
 }
 
-bool HostInputWaitEvent(SDL_Event* event) {
+bool HostInputWaitEvent(SDL_Event* event, int max_wait_ms) {
 	int timeout = -1;
 	if (!g_mouse.enabled || SDL_GetKeyboardFocus() != g_mouse_window) {
 		g_mouse.next_poll = 0;
@@ -473,6 +473,7 @@ bool HostInputWaitEvent(SDL_Event* event) {
 		    now_ms < g_cursor_hide_at ? static_cast<int>(g_cursor_hide_at - now_ms) : 0;
 		timeout = timeout < 0 ? cursor_timeout : std::min(timeout, cursor_timeout);
 	}
+	timeout              = timeout < 0 ? max_wait_ms : std::min(timeout, max_wait_ms);
 	const bool has_event = SDL_WaitEventTimeout(event, timeout);
 
 	if (Config::HideCursorEnabled()) {

src/graphics/presentation/window/hostInput.h

@@ -13,7 +13,7 @@ void               HostInputShutdown();
 void               HostInputKey(int key_code, bool down);
 void               HostInputMouseButton(uint8_t mouse_button, bool down);
 void               HostInputToggleMouseToJoystick();
-[[nodiscard]] bool HostInputWaitEvent(SDL_Event* event);
+[[nodiscard]] bool HostInputWaitEvent(SDL_Event* event, int max_wait_ms);
 
 } // namespace Libs::Graphics
 

src/graphics/presentation/window/swapchain.cpp

@@ -998,7 +998,7 @@ void Presenter::Impl::Present() {
 		}
 
 		presented_overlay_revision.store(overlay_visual.revision, std::memory_order_release);
-		window.UpdateTitle();
+		window.loop.presented_frames.fetch_add(1, std::memory_order_relaxed);
 		return;
 	}
 	LOGF("Vulkan presentation retry exhausted; dropping frame\n");

src/graphics/presentation/window/vulkanWindow.cpp

@@ -47,6 +47,12 @@ struct VulkanExtensions {
 	std::vector<vk::LayerProperties>     available_layers;
 };
 
+vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
+	vk::PhysicalDeviceVulkan11Features features {};
+	features.storageBuffer16BitAccess = VK_TRUE;
+	return features;
+}
+
 vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
 	vk::PhysicalDeviceVulkan12Features features {};
 	features.samplerMirrorClampToEdge  = VK_TRUE;
@@ -55,6 +61,8 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 	features.shaderOutputViewportIndex = VK_TRUE;
 	features.bufferDeviceAddress       = VK_TRUE;
 	features.shaderBufferInt64Atomics  = VK_TRUE;
+	features.storageBuffer8BitAccess   = VK_TRUE;
+	features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
 	return features;
 }
 
@@ -206,6 +214,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		depth_clip_control.pNext = &depth_clip_enable;
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
+		vk::PhysicalDeviceVulkan11Features features11 {};
 #if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
 #else
@@ -213,6 +222,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
+		features11.pNext       = features12.pNext;
+		features12.pNext       = &features11;
 		features13.pNext       = &features12;
 		device_features2.pNext = &features13;
 
@@ -256,6 +267,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge",
 		              required_features12.samplerMirrorClampToEdge);
+		check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
+		              WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
+		check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
+		              required_features12.storageBuffer8BitAccess);
 		check_feature(features12.timelineSemaphore, "timelineSemaphore",
 		              required_features12.timelineSemaphore);
 		check_feature(features12.shaderOutputLayer, "shaderOutputLayer",
@@ -266,6 +281,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		              required_features12.bufferDeviceAddress);
 		check_feature(features12.shaderBufferInt64Atomics, "shaderBufferInt64Atomics",
 		              required_features12.shaderBufferInt64Atomics);
+		check_feature(features12.shaderSampledImageArrayNonUniformIndexing,
+		              "shaderSampledImageArrayNonUniformIndexing",
+		              required_features12.shaderSampledImageArrayNonUniformIndexing);
 		check_feature(features13.robustImageAccess, "robustImageAccess");
 		check_feature(features13.dynamicRendering, "dynamicRendering",
 		              required_features13.dynamicRendering);
@@ -478,6 +496,9 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	workgroup_layout.pNext = &depth_clip_control;
 	features12.pNext = workgroup_layout_extension ? static_cast<void*>(&workgroup_layout)
 	                                             : static_cast<void*>(&depth_clip_control);
+	auto features11 = WindowContext::RequiredVulkan11Features();
+	features11.pNext = features12.pNext;
+	features12.pNext = &features11;
 	if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) {
 		Log::WriteToConsoleAndLog(fmt::format(
 		    "WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, "
@@ -560,8 +581,7 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.shaderInt64 = VK_TRUE;
 	device_features.shaderFloat64 =
 	    supported_features2.features.shaderFloat64 &&
-	    float_controls.shaderSignedZeroInfNanPreserveFloat64 &&
-	    float_controls.shaderRoundingModeRTEFloat32;
+	    float_controls.shaderSignedZeroInfNanPreserveFloat64;
 	// if (device_features.shaderFloat64 && !float_controls.shaderDenormPreserveFloat64) {
 	// 	Log::WriteToConsoleAndLog(
 	// 	    "WARNING: Vulkan device does not guarantee FP64 denormal preservation; "

src/graphics/presentation/window/window.cpp

@@ -362,11 +362,13 @@ static void GameEventDidEnterForeground(WindowLoopState& game) {
 	SetPause(game, false);
 }
 
-void WindowContext::Resize(uint32_t new_width, uint32_t new_height) {
-	EXIT_IF(new_width == 0 || new_height == 0);
+void WindowContext::Resize(int new_width, int new_height) {
+	if (new_width <= 0 || new_height <= 0) {
+		return;
+	}
 	Common::LockGuard lock(mutex);
-	graphic_ctx.screen_width  = new_width;
-	graphic_ctx.screen_height = new_height;
+	graphic_ctx.screen_width  = static_cast<uint32_t>(new_width);
+	graphic_ctx.screen_height = static_cast<uint32_t>(new_height);
 }
 
 void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {
@@ -720,6 +722,10 @@ void WindowContext::Run() {
 	loop.need_exit = false;
 	loop.paused.store(false, std::memory_order_release);
 
+	constexpr uint64_t title_interval_ms = 1000;
+	auto               title_time        = SDL_GetTicks();
+	auto               title_frames      = loop.presented_frames.load(std::memory_order_relaxed);
+
 	while (!loop.need_exit) {
 		if (loop.paused.load(std::memory_order_acquire)) {
 			if (!timer.IsPaused()) {
@@ -729,7 +735,20 @@ void WindowContext::Run() {
 			timer.Resume();
 		}
 
-		if (!HostInputWaitEvent(&loop.event)) {
+		// Refresh the title on the main thread without making presentation wait for it.
+		const auto now     = SDL_GetTicks();
+		const auto elapsed = now - title_time;
+		if (elapsed >= title_interval_ms) {
+			const auto frames = loop.presented_frames.load(std::memory_order_relaxed);
+			if (frames != 0) {
+				UpdateTitle(frames, static_cast<double>(frames - title_frames) * 1000.0 /
+				                        static_cast<double>(elapsed));
+			}
+			title_time   = now;
+			title_frames = frames;
+		}
+		const auto wait_ms = static_cast<int>(title_interval_ms - (now - title_time));
+		if (!HostInputWaitEvent(&loop.event, wait_ms)) {
 			continue;
 		}
 		ProcessEvent(timer.GetTimeS());
@@ -911,7 +930,7 @@ void WindowContext::UpdateIcon() {
 	}
 }
 
-void WindowContext::UpdateTitle() {
+void WindowContext::UpdateTitle(uint64_t frame_num, double current_fps) {
 	static char title[128];
 	static char title_id[12];
 	static char app_ver[12];
@@ -921,10 +940,6 @@ void WindowContext::UpdateTitle() {
 	static bool has_app_ver =
 	    Loader::SystemContentParamSfoGetString("APP_VER", app_ver, sizeof(app_ver));
 	static const std::string processor_name = Common::GetSystemInfo().ProcessorName;
-	static uint64_t fps_start   = Common::Timer::QueryPerformanceCounter();
-	static uint64_t frame_num   = 0;
-	static uint64_t fps_frames  = 0;
-	static double   current_fps = 0.0;
 
 #if KYTY_BUILD == KYTY_BUILD_DEBUG
 	static constexpr auto build_type = "Debug";
@@ -934,17 +949,6 @@ void WindowContext::UpdateTitle() {
 	static constexpr auto build_type = "Unknown";
 #endif
 
-	const auto now       = Common::Timer::QueryPerformanceCounter();
-	const auto frequency = Common::Timer::QueryPerformanceFrequency();
-	frame_num++;
-	fps_frames++;
-	if (now - fps_start >= frequency) {
-		current_fps = static_cast<double>(fps_frames) * static_cast<double>(frequency) /
-		              static_cast<double>(now - fps_start);
-		fps_start   = now;
-		fps_frames  = 0;
-	}
-
 	const auto* device_name = graphic_ctx.GetPhysicalDeviceProperties().deviceName.data();
 	auto text = fmt::format(
 	    "[{} | {}] {}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:.0f}", KYTY_BUILD_LABEL, build_type,
@@ -952,16 +956,7 @@ void WindowContext::UpdateTitle() {
 	    (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""), (has_app_ver ? " " : ""),
 	    device_name, processor_name, frame_num, current_fps);
 
-	struct TitleUpdate {
-		SDL_Window*  window;
-		std::string* text;
-	} update {window, &text};
-	EXIT_IF(!SDL_RunOnMainThread(
-	    [](void* data) {
-		    auto& title = *static_cast<TitleUpdate*>(data);
-		    SDL_SetWindowTitle(title.window, title.text->c_str());
-	    },
-	    &update, true));
+	SDL_SetWindowTitle(window, text.c_str());
 }
 
 } // namespace Libs::Graphics

src/graphics/presentation/window/windowInternal.h

@@ -27,6 +27,7 @@ struct WindowLoopState {
 	SDL_Event        event {};
 	bool             need_exit = false;
 	std::atomic_bool paused    = false;
+	std::atomic<uint64_t> presented_frames {0};
 };
 
 struct WindowContext {
@@ -34,6 +35,7 @@ struct WindowContext {
 	~WindowContext();
 	KYTY_CLASS_NO_COPY(WindowContext);
 
+	[[nodiscard]] static vk::PhysicalDeviceVulkan11Features RequiredVulkan11Features() noexcept;
 	[[nodiscard]] static vk::PhysicalDeviceVulkan12Features RequiredVulkan12Features() noexcept;
 	[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
 	[[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept;
@@ -41,8 +43,8 @@ struct WindowContext {
 	void                                                    RecreateSurface();
 	void                                                    RefreshSurfaceCapabilities();
 	void                                                    UpdateIcon();
-	void                                                    UpdateTitle();
-	void                                                    Resize(uint32_t width, uint32_t height);
+	void UpdateTitle(uint64_t frame_num, double current_fps);
+	void                                                    Resize(int width, int height);
 	void ProcessWindowEvent(const SDL_WindowEvent& event);
 	void ProcessDisplayEvent(const SDL_DisplayEvent& event);
 	void ProcessEvent(double time_seconds);

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -7,8 +7,8 @@
 #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/recompiler/frontend/translate/Translate.h"
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
@@ -21,6 +21,7 @@
 #include <array>
 #include <chrono>
 #include <fmt/format.h>
+#include <functional>
 #include <map>
 #include <span>
 #include <utility>
@@ -467,6 +468,7 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
 	Decoder::DecodeInstruction(joined_code, front_words - 1u, result.instructions.back());
 	Decoder::Program back_program;
 	Decoder::DecodeProgram(back, back_program);
+	result.has_swap_pc |= back_program.has_swap_pc;
 	const auto back_pc = front_words * sizeof(uint32_t);
 	for (auto& inst: back_program.instructions) {
 		// A back-stage PC-relative data reference requires its guest code address.
@@ -479,11 +481,227 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
 	return result;
 }
 
+void PrepareCallTarget(ShaderSource& source, const CompileOptions& options) {
+	using namespace IR;
+	for (uint32_t i = 0; i < source.decoded.instructions.size(); ++i) {
+		if (source.decoded.instructions[i].opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
+		EXIT_NOT_IMPLEMENTED(source.call.has_value() || options.stage != ShaderType::Compute);
+		source.call = ShaderSource::Call {.instruction = i};
+	}
+	EXIT_IF(options.input_info.compute == nullptr);
+	const auto& call = source.decoded.instructions[source.call->instruction];
+	EXIT_NOT_IMPLEMENTED(call.src0.kind != Decoder::OperandKind::Sgpr ||
+	                     call.dst.kind != Decoder::OperandKind::Sgpr ||
+	                     call.src0.reg != call.dst.reg || call.dst.reg >= 105u);
+	const auto graph = CFG::BuildGraph(source.decoded);
+	Program query;
+	query.stage = options.stage;
+	query.user_data_base = options.user_data_base;
+	query.user_data_count = static_cast<uint32_t>(options.user_data.size());
+	query.wave_size = options.wave_size;
+	auto& block = *query.block_storage.emplace_back(std::make_unique<Block>());
+	query.blocks.push_back(&block);
+	struct Producer { bool ready = false; std::map<uint32_t, Value> values; };
+	std::map<uint32_t, Producer> producers;
+	std::function<Value(uint32_t, uint32_t)> Register;
+	std::function<Value(uint32_t, Value)> Resolve = [&](uint32_t before, Value value) -> Value {
+		value = value.Resolve();
+		auto* inst = value.TryInstruction();
+		if (inst == nullptr) return value;
+		uint32_t code = UINT32_MAX;
+		switch (inst->GetOpcode()) {
+			case ValueOpcode::GetScalarRegister: code = static_cast<uint32_t>(inst->Arg(0).ScalarRegister()); break;
+			case ValueOpcode::GetVccLo: code = 106u; break;
+			case ValueOpcode::GetVccHi: code = 107u; break;
+			default: break;
+		}
+		if (code != UINT32_MAX) {
+			const auto result = Register(before, code);
+			inst->ReplaceUsesWith(result);
+			return result;
+		}
+		for (size_t i = 0; i < inst->NumArgs(); ++i) inst->SetArg(i, Resolve(before, inst->Arg(i)));
+		return value;
+	};
+	Register = [&](uint32_t before, uint32_t code) -> Value {
+		const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code);
+		if (index == UINT32_MAX) {
+			EXIT_IF(code < query.user_data_base || code - query.user_data_base >= query.user_data_count);
+			return IREmitter(&block).GetUserData(static_cast<ScalarReg>(code));
+		}
+		auto [entry, inserted] = producers.try_emplace(index);
+		auto& producer = entry->second;
+		if (inserted) {
+			Block translated;
+			Frontend::Translator translator(query, &translated, 1u,
+			    (options.input_info.compute->float_mode & 0x10u) == 0u,
+			    !options.input_info.compute->async_compute);
+			translator.TranslateInstruction(source.decoded.instructions[index]);
+			for (const auto& inst: translated) {
+				uint32_t destination = UINT32_MAX;
+				Value value;
+				switch (inst.GetOpcode()) {
+					case ValueOpcode::SetScalarRegister:
+						destination = static_cast<uint32_t>(inst.Arg(0).ScalarRegister()); value = inst.Arg(1); break;
+					case ValueOpcode::SetVccLo: destination = 106u; value = inst.Arg(0); break;
+					case ValueOpcode::SetVccHi: destination = 107u; value = inst.Arg(0); break;
+					default: break;
+				}
+				if (destination != UINT32_MAX) producer.values.insert_or_assign(destination, value);
+			}
+			for (auto& [destination, value]: producer.values) value = Resolve(index, value);
+			for (auto& inst: translated) inst.SetParent(&block);
+			block.Instructions().splice(block.end(), translated.Instructions());
+			producer.ready = true;
+		}
+		EXIT_IF(!producer.ready || !producer.values.contains(code));
+		return producer.values.at(code).Resolve();
+	};
+	for (uint32_t i = 0; i < 2u; ++i) {
+		const auto value = Register(source.call->instruction, call.src0.reg + i);
+		EXIT_IF(!ValidateRuntimeValue(query, value, RuntimeValueType::Integer));
+		auto& root = query.value_storage.emplace_back(ValueOpcode::ReferenceU32);
+		root.SetArg(0, value);
+		source.call->target[i] = Value(&root);
+	}
+	RewriteToSsa(query.blocks);
+	ConstantPropagationPass(query.blocks, query.wave_size);
+	RemoveIdentities(query.blocks);
+	EliminateDeadCode(query.blocks);
+	for (auto& value: source.call->target) value = value.Instruction()->Arg(0).Resolve();
+	query.value_storage.clear();
+	for (auto& inst: block) {
+		inst.SetParent(nullptr);
+		if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 || inst.GetOpcode() == ValueOpcode::ReadConstBuffer)
+			inst.SetFlags(SrtReadFlags {.index = inst.Flags<MemoryFlags>().index});
+	}
+	query.value_storage.splice(query.value_storage.end(), block.Instructions());
+	source.call_targets = std::move(static_cast<ResourcePlan&>(query));
+}
+
 } // namespace
 
+ShaderSource PrepareShaderSource(std::span<const uint32_t> code, const CompileOptions& options) {
+	EXIT_IF(code.empty());
+	ShaderSource source;
+	if (!options.back_code.empty()) {
+		source.decoded = DecodeFusedProgram(code, options.back_code, source.code);
+	} else if (options.stage == ShaderType::Local) {
+		source.decoded = Decoder::DecodeFrontProgram(code);
+		auto& handoff = source.decoded.instructions.back();
+		handoff.opcode = Decoder::Opcode::S_ENDPGM;
+		handoff.src_count = 0;
+	} else {
+		Decoder::DecodeProgram(code, source.decoded);
+	}
+	if (source.decoded.has_swap_pc) {
+		PrepareCallTarget(source, options);
+		source.code.assign(code.begin(), code.end());
+		source.decoded.code = source.code;
+	}
+	return source;
+}
+
+const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime) {
+	auto& reads = source.reads;
+	reads.clear();
+	IR::SrtReadCapture capture(runtime, reads);
+	const auto clean = IR::CleanRuntime(capture.ObservedRuntime());
+	IR::SrtWalker walker(source.call_targets, clean);
+	uint32_t low = 0, high = 0;
+	const auto& call = source.decoded.instructions[source.call->instruction];
+	if (!walker.Evaluate(source.call->target[0], low) || !walker.Evaluate(source.call->target[1], high))
+		EXIT("shader call at pc 0x%08x has an unavailable scalar target", call.pc);
+	// SWAPPC reads the old aliased pair and ignores the target's low two bits.
+	const auto address = ((uint64_t {high} << 32u) | low) & ~uint64_t {3};
+	EXIT_IF(address == 0u);
+	const auto query_reads = reads.size();
+	if (source.linked) {
+		auto& linked = *source.linked;
+		const auto function = std::span(linked.code).subspan(source.code.size());
+		if (clean.read_memory(clean.userdata, address, linked.observed_function) &&
+		    std::ranges::equal(function, linked.observed_function)) {
+			return linked.decoded;
+		}
+		reads.resize(query_reads);
+	}
+	ShaderSource::Linked linked {.code = source.code,
+	                             .decoded = source.decoded};
+	const auto& last = source.decoded.instructions.back();
+	EXIT_NOT_IMPLEMENTED(last.opcode != Decoder::Opcode::S_ENDPGM);
+	// Keep the caller's data/footer bytes at their native offsets. The leaf's virtual CFG
+	// PCs follow decoded instructions; its physical code is independently read and owned.
+	const auto function_pc = last.pc + last.word_count * sizeof(uint32_t);
+	for (uint32_t index = 0;;) {
+		// The observed queue helper is a scalar leaf. Scalar encodings need at most one literal.
+		EXIT_IF(index >= 64u || address > UINT64_MAX - (uint64_t {index} + 2u) * sizeof(uint32_t));
+		std::array<uint32_t, 2> words {};
+		if (!clean.read_memory(clean.userdata, address + uint64_t {index} * sizeof(uint32_t),
+		                       std::span(words).first(1)))
+			EXIT("shader call at pc 0x%08x cannot read leaf code", call.pc);
+		const auto family = Decoder::GetInstructionFamily(words[0]);
+		EXIT_NOT_IMPLEMENTED(family != Decoder::Family::SOP1 && family != Decoder::Family::SOP2 &&
+		                     family != Decoder::Family::SOPC && family != Decoder::Family::SOPK &&
+		                     family != Decoder::Family::SOPP);
+		Decoder::Instruction inst;
+		Decoder::DecodeInstruction(words, 0u, inst);
+		if (inst.word_count == 2u) {
+			if (!clean.read_memory(clean.userdata, address + uint64_t {index + 1u} * sizeof(uint32_t),
+			                       std::span(words).last(1)))
+				EXIT("shader call at pc 0x%08x cannot read leaf literal", call.pc);
+			inst = {};
+			Decoder::DecodeInstruction(words, 0u, inst);
+		}
+		inst.pc = index * sizeof(uint32_t);
+		if (Decoder::IsDirectBranch(inst.opcode)) inst.branch_target += inst.pc;
+		linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
+		index += inst.word_count;
+		EXIT_NOT_IMPLEMENTED(inst.opcode == Decoder::Opcode::S_SWAPPC_B64 ||
+		                     inst.opcode == Decoder::Opcode::S_GETPC_B64 ||
+		                     inst.opcode == Decoder::Opcode::S_ENDPGM);
+		EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) || CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
+		const bool returns = inst.opcode == Decoder::Opcode::S_SETPC_B64;
+		if (returns) {
+			EXIT_NOT_IMPLEMENTED(inst.src0.kind != Decoder::OperandKind::Sgpr || inst.src0.reg != call.dst.reg);
+			inst.opcode = Decoder::Opcode::S_BRANCH;
+			inst.branch_target = call.pc + call.word_count * sizeof(uint32_t);
+		} else if (Decoder::IsDirectBranch(inst.opcode)) {
+			EXIT_IF(inst.branch_target > UINT32_MAX - function_pc);
+			inst.branch_target += function_pc;
+		}
+		inst.pc += function_pc;
+		linked.decoded.instructions.push_back(std::move(inst));
+		if (returns) break;
+	}
+	const auto function_end = linked.decoded.instructions.back().pc +
+	                          linked.decoded.instructions.back().word_count * sizeof(uint32_t);
+	for (size_t i = source.decoded.instructions.size(); i + 1u < linked.decoded.instructions.size(); ++i) {
+		const auto& inst = linked.decoded.instructions[i];
+		if (Decoder::IsDirectBranch(inst.opcode))
+			EXIT_IF(inst.branch_target < function_pc || inst.branch_target >= function_end);
+	}
+	reads.resize(query_reads);
+	reads.emplace_back(address, (linked.code.size() - source.code.size()) * sizeof(uint32_t));
+	linked.decoded.instructions[source.call->instruction].branch_target = function_pc;
+	linked.decoded.code = linked.code;
+	linked.observed_function.resize(linked.code.size() - source.code.size());
+	source.linked = std::move(linked);
+	++source.revision;
+	return source.linked->decoded;
+}
+
 TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOptions& options) {
-	if (code.empty()) {
-		EXIT("shader recompiler input is empty\n");
+	auto source = PrepareShaderSource(code, options);
+	return TranslateProgram(source.decoded, options);
+}
+
+TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileOptions& options) {
+	EXIT_IF(decoded.instructions.empty());
+	if (decoded.has_swap_pc) {
+		for (const auto& inst: decoded.instructions) {
+			if (inst.opcode == Decoder::Opcode::S_SWAPPC_B64 && inst.branch_target == UINT32_MAX)
+				EXIT("shader call at pc 0x%08x must be linked before translation", inst.pc);
+		}
 	}
 	if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
 	    options.stage != ShaderType::Pixel && options.stage != ShaderType::Mesh &&
@@ -494,34 +712,11 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	}
 
 	const auto compile_begin = std::chrono::steady_clock::now();
-	const auto phase_ms      = [&compile_begin]() {
+	const auto phase_ms = [&compile_begin]() {
 		return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
-		                                 std::chrono::steady_clock::now() - compile_begin)
-		                                 .count());
+		    std::chrono::steady_clock::now() - compile_begin).count());
 	};
 
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " code_words=%" PRIu64 " decode\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-	     static_cast<uint64_t>(code.size()));
-
-	Decoder::Program decoded;
-	std::vector<uint32_t> joined_code;
-	if (!options.back_code.empty()) {
-		decoded = DecodeFusedProgram(code, options.back_code, joined_code);
-	} else if (options.stage == ShaderType::Local) {
-		decoded = Decoder::DecodeFrontProgram(code);
-		// The separately compiled hull half runs in the next Vulkan stage.
-		auto& handoff     = decoded.instructions.back();
-		handoff.opcode    = Decoder::Opcode::S_ENDPGM;
-		handoff.src_count = 0;
-	} else {
-		Decoder::DecodeProgram(code, decoded);
-	}
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64
-	     " elapsed_ms=%" PRIu64 "\n",
-	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-	     static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
-
 	std::string decoded_dump;
 	if (options.dump_ir) {
 		decoded_dump = Decoder::ProgramToString(decoded);
@@ -607,6 +802,12 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		IR::RemoveIdentities(ir.blocks);
 		IR::EliminateDeadCode(ir.blocks);
 	}
+	if (decoded.has_swap_pc) {
+		// Leaf returns are resolved CFG edges; native PC values must not escape into GPU data.
+		for (const auto* block: ir.blocks)
+			for (const auto& inst: *block)
+				EXIT_NOT_IMPLEMENTED(inst.GetOpcode() == IR::ValueOpcode::GetShaderBase);
+	}
 	LowerTessellationMemory(ir, options);
 	std::string cfg_dump;
 	if (options.dump_ir) {
@@ -648,11 +849,20 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 				})) {
 					continue;
 				}
+				const auto resource = inst.Flags<uint32_t>();
+				first               = resource < ir.info.buffers.size() &&
+				                              ir.info.buffers[resource].indirect_root == resource
+				                          ? 1u
+				                          : 0u;
 			} else if (op == IR::ValueOpcode::GetImageResource) {
 				const auto resource = inst.Flags<uint32_t>();
 				first = resource < ir.info.images.size() &&
 				                ir.info.images[resource].indirect_root == resource ? 1u : 0u;
-			} else if (op != IR::ValueOpcode::GetSamplerResource) {
+			} else if (op == IR::ValueOpcode::GetSamplerResource) {
+				const auto resource = inst.Flags<uint32_t>();
+				first = resource < ir.info.samplers.size() &&
+				                !ir.info.samplers[resource].indirect_resources.empty() ? 1u : 0u;
+			} else {
 				continue;
 			}
 			for (size_t index = first; index < inst.NumArgs(); index++) {
@@ -665,18 +875,17 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	IR::EliminateDeadCode(ir.blocks);
 
 	IR::CollectShaderInfo(ir, options.input_info);
-	IR::AllocateBindings(ir, push_data_start_dword);
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(translated.cfg_dump, ir);
 		if (options.early_dump) {
-			LOGF("%s native IR and bindings (early):\n%s", GetDumpLabel(options), ir_dump.c_str());
+			LOGF("%s native IR (early):\n%s", GetDumpLabel(options), ir_dump.c_str());
 		}
 	}
 
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
 	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
-	auto spirv = Spirv::EmitProgram(ir, options.input_info);
+	auto spirv = Spirv::EmitProgram(ir, options.input_info, push_data_start_dword);
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -4,8 +4,11 @@
 #include "common/common.h"
 #include "common/stringUtils.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/shader.h"
 
+#include <array>
+#include <optional>
 #include <span>
 #include <vector>
 
@@ -37,6 +40,32 @@ struct CompileResult {
 	IR::Program            program;
 };
 
+// Decoded source and typed scalar call queries are retained once per cache source.
+struct ShaderSource {
+	struct Call {
+		uint32_t instruction;
+		std::array<IR::Value, 2> target;
+	};
+	struct Linked {
+		std::vector<uint32_t> code;
+		Decoder::Program decoded;
+		std::vector<uint32_t> observed_function;
+	};
+	std::vector<uint32_t> code;
+	Decoder::Program decoded;
+	IR::ResourcePlan call_targets;
+	std::optional<Call> call;
+	std::optional<Linked> linked;
+	uint64_t revision = 0;
+	std::vector<std::pair<uint64_t, uint64_t>> reads;
+};
+
+[[nodiscard]] ShaderSource PrepareShaderSource(std::span<const uint32_t> code,
+	                                           const CompileOptions& options);
+const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtRuntime& runtime);
+[[nodiscard]] TranslateResult TranslateProgram(const Decoder::Program& decoded,
+	                                           const CompileOptions& options);
+
 [[nodiscard]] TranslateResult TranslateProgram(std::span<const uint32_t> code,
                                                const CompileOptions& options);
 [[nodiscard]] CompileResult CompileProgram(TranslateResult translated,

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -171,10 +171,12 @@ void Builder::AddFunction(std::span<const uint32_t> words) {
 }
 
 DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
-	std::vector<uint32_t> words {spv::OpPhi, type, result};
-	words.resize(words.size() + incoming_count * 2u);
 	const DeferredPhi phi {m_functions.size()};
-	AddFunction(words);
+	const auto word_count = static_cast<uint32_t>(3u + incoming_count * 2u);
+	m_functions.push_back((word_count << spv::WordCountShift) | spv::OpPhi);
+	m_functions.push_back(type);
+	m_functions.push_back(result);
+	m_functions.resize(m_functions.size() + incoming_count * 2u);
 	m_unpatched_phi_incomings += incoming_count;
 	return phi;
 }

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -3,7 +3,6 @@
 #include "common/assert.h"
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 
 #include <algorithm>
 #include <array>
@@ -18,86 +17,15 @@ namespace {
 	std::abort();
 }
 
-void ValidateNativeProgram(const IR::Program& program) {
-	using Kind                                             = IR::DescriptorBindingKind;
-	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
-	std::array<std::vector<uint32_t>, KindCount> expected;
-	std::array<bool, KindCount>                  present {};
-	const auto                                   Dense = [](size_t size) {
-		std::vector<uint32_t> values(size);
-		for (uint32_t i = 0; i < values.size(); i++) {
-			values[i] = i;
-		}
-		return values;
+void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
+	uint64_t live_buffers = 0;
+	bool uses_lds = false;
+	bool uses_gds = false;
+	const auto uses_mapping = [](const auto& resource) {
+		return resource.indirect_root != UINT32_MAX;
 	};
-	auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
-		const auto index = static_cast<size_t>(kind);
-		present[index]   = true;
-		expected[index]  = std::move(resources);
-	};
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
-		if (!kind.has_value()) {
-			Fail(program, "native shader plan has an invalid image class");
-		}
-		present[static_cast<size_t>(*kind)] = true;
-		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::Dynamic;
-		const auto count   = dynamic ? program.info.images[i].mip_count : 1u;
-		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
-			Fail(program, "native shader plan has an invalid image mip descriptor count");
-		}
-		expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(),
-		                                            count, i);
-	}
-	if (!program.info.samplers.empty()) {
-		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
-	}
-	auto& buffers = expected[static_cast<size_t>(Kind::Buffers)];
-	const bool uses_gds = IR::CollectMemoryResources(program, buffers);
-	present[static_cast<size_t>(Kind::Buffers)] = !buffers.empty();
-	if (uses_gds) {
-		Expect(Kind::Gds);
-	}
-	if (program.info.uses_dma) {
-		Expect(Kind::BdaPagetable);
-		Expect(Kind::FaultBuffer);
-	}
-	if (IR::UsesFlattenedSrt(program)) {
-		Expect(Kind::FlattenedSrt);
-	}
-	if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) {
-		Expect(Kind::ShaderData);
-	}
-
-	std::array<bool, KindCount> seen {};
-	for (const auto& binding: program.bindings.descriptors) {
-		const auto kind = static_cast<size_t>(binding.kind);
-		if (kind >= KindCount || seen[kind] || !present[kind] ||
-		    binding.resources != expected[kind]) {
-			Fail(program, "native descriptor groups do not match shader topology");
-		}
-		seen[kind] = true;
-	}
-	for (size_t i = 0; i < KindCount; i++) {
-		if (present[i] != seen[i]) {
-			Fail(program, "native shader plan is missing a required descriptor group");
-		}
-	}
-	const auto has_shader_data_storage = present[static_cast<size_t>(Kind::ShaderData)];
-	const auto shader_data_dwords = program.bindings.ShaderDataDwords();
-	if ((program.bindings.UsesPushData() &&
-	     !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
-	    program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
-	    program.bindings.memory_offset_count != buffers.size() ||
-	    has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) ||
-	    !std::is_sorted(program.bindings.user_data_registers.begin(),
-	                    program.bindings.user_data_registers.end()) ||
-	    std::adjacent_find(program.bindings.user_data_registers.begin(),
-	                       program.bindings.user_data_registers.end()) !=
-	        program.bindings.user_data_registers.end()) {
-		Fail(program, "native shader-data layout is inconsistent");
-	}
-
+	bool uses_flattened_srt = std::ranges::any_of(program.info.buffers, uses_mapping) ||
+	                          std::ranges::any_of(program.info.images, uses_mapping);
 	const auto planning_only_handle = [&](const IR::Inst& handle) {
 		return !handle.Uses().empty() &&
 		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
@@ -134,6 +62,39 @@ void ValidateNativeProgram(const IR::Program& program) {
 	};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
+			const auto op = inst.GetOpcode();
+			uses_flattened_srt |= op == IR::ValueOpcode::ReadConst;
+			if (IR::BufferAccessOf(op) != IR::BufferAccess::None ||
+			    IR::SharedAccessOf(op) != IR::SharedAccess::None ||
+			    IR::AddressOpcodeInfoOf(op).access != IR::AddressAccess::None) {
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.memory_info.size()) {
+					Fail(program, "typed shader contains invalid memory metadata");
+				}
+				const auto& memory = program.memory_info[index];
+				if (!memory.planning_only) {
+					uses_lds |= memory.kind == IR::ResourceKind::FlatLocal;
+					if (IR::SharedAccessOf(op) != IR::SharedAccess::None) {
+						if (memory.kind != IR::ResourceKind::Lds && memory.kind != IR::ResourceKind::Gds) {
+							Fail(program, "typed shader contains invalid shared-memory metadata");
+						}
+						uses_gds |= memory.kind == IR::ResourceKind::Gds;
+						uses_lds |= memory.kind == IR::ResourceKind::Lds;
+					} else if (memory.kind == IR::ResourceKind::Buffer ||
+					           memory.kind == IR::ResourceKind::ScalarBuffer) {
+						if (memory.resource >= program.info.buffers.size()) {
+							Fail(program, "typed shader contains an invalid buffer resource");
+						}
+						live_buffers |= uint64_t{1} << memory.resource;
+						for (const auto child: program.info.buffers[memory.resource].indirect_resources) {
+							if (child >= program.info.buffers.size()) {
+								Fail(program, "typed shader contains an invalid indirect buffer resource");
+							}
+							live_buffers |= uint64_t{1} << child;
+						}
+					}
+				}
+			}
 			const auto dense = inst.Flags<uint32_t>();
 			switch (inst.GetOpcode()) {
 				case IR::ValueOpcode::GetBufferResource:
@@ -179,149 +140,64 @@ void ValidateNativeProgram(const IR::Program& program) {
 			}
 		}
 	}
+	using Kind = IR::DescriptorBindingKind;
+	std::array<uint32_t, static_cast<size_t>(Kind::Count)> expected {};
+	auto& buffer_count = expected[static_cast<size_t>(Kind::Buffers)];
+	for (uint32_t index = 0; index < program.info.buffers.size(); ++index) {
+		const auto slot = (live_buffers & (uint64_t{1} << index)) != 0 ? buffer_count++ : UINT32_MAX;
+		if (program.info.buffers[index].descriptor_index != slot) {
+			Fail(program, "native buffer descriptor index does not match shader topology");
+		}
+	}
+	for (const auto& image: program.info.images) {
+		const auto kind = IR::DescriptorBindingForImage(image);
+		if (!kind.has_value()) {
+			Fail(program, "native shader plan has an invalid image class");
+		}
+		if (image.mip_count == 0u ||
+		    (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
+			Fail(program, "native shader plan has an invalid image mip descriptor count");
+		}
+		auto& count = expected[static_cast<size_t>(*kind)];
+		if (image.descriptor_index != count) {
+			Fail(program, "native image descriptor index does not match shader topology");
+		}
+		count += image.mip_count;
+	}
+	expected[static_cast<size_t>(Kind::Samplers)] = static_cast<uint32_t>(program.info.samplers.size());
+	expected[static_cast<size_t>(Kind::Gds)] = uses_gds;
+	expected[static_cast<size_t>(Kind::SharedMemory)] = uses_lds && lds_storage;
+	expected[static_cast<size_t>(Kind::BdaPagetable)] = program.info.uses_dma;
+	expected[static_cast<size_t>(Kind::FaultBuffer)] = program.info.uses_dma;
+	expected[static_cast<size_t>(Kind::FlattenedSrt)] = uses_flattened_srt;
+	expected[static_cast<size_t>(Kind::ShaderData)] =
+	    program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData();
+	uint64_t descriptor_mask = 0;
+	for (uint32_t index = 0; index < expected.size(); ++index) {
+		if (expected[index] != 0) descriptor_mask |= uint64_t{1} << index;
+	}
+	if (program.bindings.descriptor_counts != expected ||
+	    program.bindings.descriptor_mask != descriptor_mask) {
+		Fail(program, "native descriptor counts do not match shader topology");
+	}
+	const auto shader_data_dwords = program.bindings.ShaderDataDwords();
+	const auto& registers = program.info.user_data_registers;
+	const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart;
+	if ((program.bindings.UsesPushData() &&
+	     !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
+	    (has_dispatch_threads && (program.stage != ShaderType::Compute ||
+	                              program.bindings.dispatch_thread_dword != registers.size())) ||
+	    program.bindings.memory_offset_dword != registers.size() + (has_dispatch_threads ? 3u : 0u) ||
+	    !std::is_sorted(registers.begin(), registers.end()) ||
+	    std::adjacent_find(registers.begin(), registers.end()) != registers.end()) {
+		Fail(program, "native shader-data layout is inconsistent");
+	}
 }
 
 } // namespace
 
-Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) {
-	SpirvRequirements requirements {};
-	for (const auto* block: program.blocks) {
-		for (const auto& inst: *block) {
-			requirements.float64 |= inst.GetType() == IR::Type::F64;
-			if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
-			    inst.GetType() == IR::Type::U64) {
-				requirements.buffer_int64_atomics = true;
-			}
-			const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
-			if (address_access != IR::AddressAccess::None) {
-				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
-				if (memory_index >= program.memory_info.size()) {
-					Fail(program, "address operation has invalid memory metadata");
-				}
-				const auto kind = program.memory_info[memory_index].kind;
-				if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
-					if (program.scratch_dwords == 0) {
-						Fail(program, "scratch operation has no per-thread storage");
-					}
-					requirements.function_scratch = true;
-					if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute &&
-					    program.stage != ShaderType::Mesh) {
-						requirements.function_lds = true;
-					}
-				} else if (address_access == IR::AddressAccess::Write) {
-					Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
-				}
-			}
-			if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
-				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
-				if (memory_index >= program.memory_info.size()) {
-					Fail(program, "buffer operation has invalid memory metadata");
-				}
-				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind == IR::ResourceKind::IndirectBuffer) {
-					requirements.subgroup_local_invocation_id = true;
-				}
-				if (memory.kind == IR::ResourceKind::Buffer) {
-					requirements.coherent_buffers |= memory.coherent;
-					if (memory.resource >= program.info.buffers.size()) {
-						Fail(program, "buffer operation has invalid resource metadata");
-					}
-					if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
-						if (program.stage != ShaderType::Compute) {
-							Fail(program, "buffer ADD_TID is only valid for compute shaders");
-						}
-						requirements.subgroup_local_invocation_id = true;
-					}
-				}
-			}
-			const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
-			if (shared_access != IR::SharedAccess::None) {
-				const auto index = inst.Flags<IR::MemoryFlags>().index;
-				if (index >= program.memory_info.size()) {
-					Fail(program, "shared operation has invalid memory metadata");
-				}
-				const auto kind = program.memory_info[index].kind;
-				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
-					Fail(program, "shared operation has invalid resource kind");
-				}
-				if (shared_access == IR::SharedAccess::Atomic &&
-				    IR::SharedComponentCount(inst.GetOpcode()) == 2u) {
-					if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) {
-						Fail(program, "64-bit shared atomics require compute LDS");
-					}
-					requirements.shared_int64_atomics = true;
-				}
-				if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh &&
-				    kind == IR::ResourceKind::Lds) {
-					requirements.function_lds = true;
-				}
-				if (shared_access == IR::SharedAccess::Append ||
-				    shared_access == IR::SharedAccess::Consume) {
-					requirements.subgroup_ballot              = true;
-					requirements.subgroup_shuffle             = true;
-					requirements.subgroup_local_invocation_id = true;
-				}
-			}
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
-				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
-				case IR::ValueOpcode::DppMoveU32:
-				case IR::ValueOpcode::ReadFirstLane:
-				case IR::ValueOpcode::ReadLane: {
-					requirements.subgroup_ballot  = true;
-					requirements.subgroup_shuffle = true;
-					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
-						requirements.subgroup_local_invocation_id = true;
-					}
-					break;
-				}
-				case IR::ValueOpcode::DppUpdateU32:
-				case IR::ValueOpcode::WriteLane: {
-					requirements.subgroup_ballot              = true;
-					requirements.subgroup_local_invocation_id = true;
-					break;
-				}
-				case IR::ValueOpcode::Permlane16U32: {
-					requirements.subgroup_ballot              = true;
-					requirements.subgroup_shuffle             = true;
-					requirements.subgroup_local_invocation_id = true;
-					break;
-				}
-				case IR::ValueOpcode::SwizzleU32:
-				case IR::ValueOpcode::BpermuteU32: {
-					requirements.subgroup_ballot              = true;
-					requirements.subgroup_shuffle             = true;
-					requirements.subgroup_local_invocation_id = true;
-					break;
-				}
-				case IR::ValueOpcode::LaneId:
-					requirements.subgroup_local_invocation_id |=
-					    program.stage != ShaderType::TessellationControl;
-					break;
-				case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
-				case IR::ValueOpcode::ImageGatherRaw:
-					requirements.image_gather_extended = true;
-					break;
-				case IR::ValueOpcode::SetAttribute: {
-					const auto index = inst.Flags<IR::ExportFlags>().index;
-					if (index >= program.export_info.size()) {
-						Fail(program, "attribute export has invalid metadata");
-					}
-					if (program.stage == ShaderType::Pixel &&
-					    program.export_info[index].vm) {
-						requirements.pixel_valid_mask = true;
-					}
-					break;
-				}
-				default: break;
-			}
-		}
-	}
-	return requirements;
-}
-
-std::vector<uint32_t> EmitProgram(const IR::Program& program,
-                                  ShaderStageInputInfo input_info) {
+std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo input_info,
+                                  uint32_t push_data_start_dword) {
 	using namespace Emitter;
 
 	if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
@@ -331,11 +207,14 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
 		Fail(program, "binary SPIR-V emitter received an unsupported shader stage");
 	}
 	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete) {
+	    !program.shader_info_complete) {
 		Fail(program, "SPIR-V emitter requires a fully planned native shader program");
 	}
-	ValidateNativeProgram(program);
+	if (program.info.buffers.size() > IR::ShaderInfo::MaxBuffers) {
+		Fail(program, "native shader exceeds the buffer resource limit");
+	}
 	IR::ValidateProgram(program, true);
+	program.bindings = {.push_data_start_dword = push_data_start_dword};
 	EmitterState state(program, input_info);
 	const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
 	state.lane_count =
@@ -343,6 +222,8 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
 	        ? 2u
 	        : 1u;
 	DefineModule(state);
+	ValidateNativeProgram(program, program.stage == ShaderType::Compute &&
+	                                   input_info.compute != nullptr && input_info.compute->lds_storage);
 	EmitProgram(state);
 	state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func,
 	                            "main", state.interface_variables);

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,8 +9,9 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-std::vector<uint32_t> EmitProgram(const IR::Program& program,
-                                  ShaderStageInputInfo input_info);
+// Resets and writes backend IDs and descriptor indices; do not emit the same program concurrently.
+std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo input_info,
+                                  uint32_t push_data_start_dword = 0);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -17,28 +17,26 @@ Pair ExtractPair(EmitterState& state, uint32_t value) {
 	return result;
 }
 
-uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpCompositeConstruct, TypeU64(state), result, low, high);
-	return result;
-}
+uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto bits_type = TypeU32Vector(state, 2);
+	const auto lhs_bits = Unary(state, spv::OpBitcast, bits_type, lhs);
+	const auto rhs_bits = Unary(state, spv::OpBitcast, bits_type, rhs);
+	const auto ordered = Binary(state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
+	                            TypeBool(state), lhs, rhs);
+	auto result = Select(state, TypeF64(state), ordered, lhs, rhs);
 
-uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                        bool not_equal) {
-	const auto compare = Binary(state, not_equal ? spv::OpINotEqual : spv::OpIEqual,
-	                            TypeBoolVector(state, 2), lhs_value, rhs_value);
-	return Unary(state, not_equal ? spv::OpAny : spv::OpAll, TypeBool(state), compare);
-}
+	// Equal values have identical bits except signed zero: min chooses -0, max chooses +0.
+	const auto equal = Binary(state, spv::OpFOrdEqual, TypeBool(state), lhs, rhs);
+	const auto equal_bits = Binary(state, max_value ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
+	                               bits_type, lhs_bits, rhs_bits);
+	result = Select(state, TypeF64(state), equal,
+	                Unary(state, spv::OpBitcast, TypeF64(state), equal_bits), result);
 
-uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                          spv::Op high_compare, spv::Op low_compare) {
-	const auto lhs         = ExtractPair(state, lhs_value);
-	const auto rhs         = ExtractPair(state, rhs_value);
-	const auto high_equal  = Binary(state, spv::OpIEqual, TypeBool(state), lhs.high, rhs.high);
-	const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
-	const auto low_result  = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
-	const auto low_path = Binary(state, spv::OpLogicalAnd, TypeBool(state), high_equal, low_result);
-	return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
+	// Non-IEEE mode selects the other operand for NaN, including rhs when both are NaN.
+	result = Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), rhs),
+	                lhs, result);
+	return Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), lhs),
+	              rhs, result);
 }
 
 uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
@@ -58,113 +56,14 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
 	return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
 }
 
-uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
-	const auto pair       = ExtractPair(state, value);
-	const auto amount     = EmitAndConstant(state, shift, 63u);
-	const auto word_shift = EmitAndConstant(state, amount, 31u);
-	const auto at_least_32 =
-	    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
-	const auto nonzero =
-	    Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
-	const auto carry_count = EmitAndConstant(
-	    state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u);
-	if (opcode == spv::OpShiftLeftLogical) {
-		const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift);
-		const auto carry =
-		    Select(state, TypeU32(state), nonzero,
-		           Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count),
-		           ConstantU32(state, 0));
-		const auto high =
-		    Binary(state, spv::OpBitwiseOr, TypeU32(state),
-		           Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry);
-		return MakePair(state,
-		                Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low),
-		                Select(state, TypeU32(state), at_least_32, low, high));
-	}
-	const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift);
-	const auto carry =
-	    Select(state, TypeU32(state), nonzero,
-	           Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
-	           ConstantU32(state, 0));
-	const auto low = Binary(
-	    state, spv::OpBitwiseOr, TypeU32(state),
-	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry);
-	const auto fill = opcode == spv::OpShiftRightArithmetic
-	                      ? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31))
-	                      : ConstantU32(state, 0);
-	return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low),
-	                Select(state, TypeU32(state), at_least_32, fill, high));
-}
-
-uint32_t EmitConstantShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
-	shift &= 63u;
-	if (shift == 0u) {
-		return value;
-	}
-	const auto pair = ExtractPair(state, value);
-	if (opcode == spv::OpShiftLeftLogical) {
-		if (shift < 32u) {
-			const auto low  = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.low,
-			                         ConstantU32(state, shift));
-			const auto high = Binary(state, spv::OpBitwiseOr, TypeU32(state),
-			                         Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-			                                pair.high, ConstantU32(state, shift)),
-			                         Binary(state, spv::OpShiftRightLogical, TypeU32(state),
-			                                pair.low, ConstantU32(state, 32u - shift)));
-			return MakePair(state, low, high);
-		}
-		return MakePair(state, ConstantU32(state, 0),
-		                shift == 32u ? pair.low
-		                             : Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-		                                      pair.low, ConstantU32(state, shift - 32u)));
-	}
-	if (shift < 32u) {
-		const auto low = Binary(state, spv::OpBitwiseOr, TypeU32(state),
-		                        Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low,
-		                               ConstantU32(state, shift)),
-		                        Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high,
-		                               ConstantU32(state, 32u - shift)));
-		const auto high =
-		    Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
-		return MakePair(state, low, high);
-	}
-	const auto high = opcode == spv::OpShiftRightArithmetic
-	                      ? Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high,
-	                               ConstantU32(state, 31u))
-	                      : ConstantU32(state, 0);
-	const auto low  = shift == 32u ? pair.high
-	                               : Binary(state, opcode, TypeU32(state), pair.high,
-	                                        ConstantU32(state, shift - 32u));
-	return MakePair(state, low, high);
-}
-
-uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
-                     bool max_value) {
-	const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
-	                             : EmitMinMaxU32Value(state, a, b, max_value);
-	return signed_value ? EmitMinMaxI32Value(state, ab, c, max_value)
-	                    : EmitMinMaxU32Value(state, ab, c, max_value);
-}
-
-uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
-	const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
-	const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
-	const auto ab      = Binary(state, spv::OpIAdd, TypeU32(state), a, b);
-	const auto abc     = Binary(state, spv::OpIAdd, TypeU32(state), ab, c);
-	return Binary(state, spv::OpISub, TypeU32(state),
-	              Binary(state, spv::OpISub, TypeU32(state), abc, minimum), maximum);
-}
-
 uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
 	return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
 }
 
 uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
                  std::initializer_list<uint32_t> args) {
-	const auto            result = state.builder.AllocateId();
-	std::vector<uint32_t> words {spv::OpExtInst, type, result, GlslStd450(state), opcode};
-	words.insert(words.end(), args.begin(), args.end());
-	state.builder.AddFunction(words);
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpExtInst, type, result, GlslStd450(state), opcode, args);
 	return result;
 }
 
@@ -201,6 +100,28 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
 }
 
 } // namespace
+
+uint32_t EmitFPFma32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
+	const auto result = EmitGlsl<GLSLstd450Fma, IR::Type::F32>(state, a, b, c);
+	state.builder.AddAnnotation(spv::OpDecorate, result, spv::DecorationNoContraction);
+	return result;
+}
+
+uint32_t EmitFPMad32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
+	// Explicit denormal checks typically drop 3DMiniGolf menu FPS from 28-29 to 21-24
+	// on NVIDIA RTX 5080 Laptop GPU, so leave them disabled for this experiment.
+	// a = EmitFlushF32DenormToSignedZero(state, a);
+	// b = EmitFlushF32DenormToSignedZero(state, b);
+	// c = EmitFlushF32DenormToSignedZero(state, c);
+	const auto product = EmitFPMul32(state, a, b);
+	state.builder.AddAnnotation(spv::OpDecorate, product, spv::DecorationNoContraction);
+	// const auto sum = EmitFPAdd32(state, EmitFlushF32DenormToSignedZero(state, product), c);
+	const auto sum = EmitFPAdd32(state, product, c);
+	state.builder.AddAnnotation(spv::OpDecorate, sum, spv::DecorationNoContraction);
+	// return EmitFlushF32DenormToSignedZero(state, sum);
+	return sum;
+}
+
 uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
 	const auto min_ab   = EmitMinMaxF32Value(state, a, b, false);
 	const auto min3     = EmitMinMaxF32Value(state, min_ab, c, false);
@@ -215,57 +136,15 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c)
 }
 
 uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
-	const auto pair   = ExtractPair(state, value);
-	const auto high_i = state.builder.AllocateId();
-	const auto low_i  = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state),
-	                          GLSLstd450FindUMsb, pair.high);
-	state.builder.AddFunction(spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state),
-	                          GLSLstd450FindUMsb, pair.low);
-	const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
-	const auto low  = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
-	const auto high_nonzero =
-	    Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
-	return Select(state, TypeU32(state), high_nonzero,
-	              Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
-}
-
-uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs   = ExtractPair(state, lhs_value);
-	const auto rhs   = ExtractPair(state, rhs_value);
-	const auto low   = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
-	const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
-	const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
-	const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
-	return MakePair(state, low,
-	                Binary(state, spv::OpIAdd, TypeU32(state),
-	                       Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
-}
-
-uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs    = ExtractPair(state, lhs_value);
-	const auto rhs    = ExtractPair(state, rhs_value);
-	const auto low    = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
-	const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
-	const auto borrow_u32 =
-	    Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
-	const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
-	const auto high  = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
-	return MakePair(state, low, high);
-}
-
-uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs      = ExtractPair(state, lhs_value);
-	const auto rhs      = ExtractPair(state, rhs_value);
-	const auto low_pair = state.builder.AllocateId();
-	const auto low      = state.builder.AllocateId();
-	const auto carry    = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1);
-	const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
-	const auto high  = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
-	return MakePair(state, low, high);
+	// GLSL FindUMsb is restricted to 32-bit components.
+	const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), value);
+	const auto pair =
+	    ExtractPair(state, EmitExt(state, TypeU32Vector(state, 2), GLSLstd450FindUMsb, {unpacked}));
+	const auto high_found =
+	    Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, UINT32_MAX));
+	return Select(state, TypeU32(state), high_found,
+	              Binary(state, spv::OpIAdd, TypeU32(state), pair.high, ConstantU32(state, 32)),
+	              pair.low);
 }
 
 uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
@@ -294,7 +173,7 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
 uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
 	const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0);
 	const auto source    = EmitNative<spv::OpCompositeExtract, IR::Type::U32>(
-	    state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u);
+	    state, Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0), 1u);
 	const auto exponent = EmitAndConstant(state, source, 0x7ff00000u);
 	const auto overflow =
 	    Binary(state, spv::OpLogicalAnd, TypeBool(state),
@@ -312,7 +191,21 @@ uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
 	                                                  EmitFlushF32DenormToSignedZero(state, arg0));
 }
 
+uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
+	arg1 = arg1.Resolve();
+	return arg1.IsImmediate() && arg1.U32() == 0u
+	           ? Unary(ctx.state, spv::OpUConvert, TypeU64(ctx.state), arg0)
+	           : PackU64(ctx.state, arg0, ctx.Def(arg1));
+}
+
 uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
+	const auto value = arg1.U32() == 0u ? arg0
+	                                    : Binary(state, spv::OpShiftRightLogical, TypeU64(state),
+	                                             arg0, ConstantU32(state, 32));
+	return Unary(state, spv::OpUConvert, TypeU32(state), value);
+}
+
+uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1) {
 	return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
 }
 
@@ -336,128 +229,38 @@ uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return EmitMulHigh(state, arg0, arg1, false);
 }
 
-uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) {
-	const auto value = arg0;
-	const auto neg   = Unary(state, spv::OpSNegate, TypeU32(state), value);
-	const auto negative =
-	    Binary(state, spv::OpSLessThan, TypeBool(state), value, ConstantU32(state, 0));
-	return Select(state, TypeU32(state), negative, neg, value);
-}
-
-uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
-	auto& state = ctx.state;
-	return arg1.Resolve().IsImmediate()
-	           ? EmitConstantShift64(state, spv::OpShiftLeftLogical, arg0, arg1.Resolve().U32())
-	           : EmitShift64(state, spv::OpShiftLeftLogical, arg0, ctx.Def(arg1));
-}
-
-uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
-	auto& state = ctx.state;
-	return arg1.Resolve().IsImmediate()
-	           ? EmitConstantShift64(state, spv::OpShiftRightLogical, arg0, arg1.Resolve().U32())
-	           : EmitShift64(state, spv::OpShiftRightLogical, arg0, ctx.Def(arg1));
-}
-
-uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
-	auto& state = ctx.state;
-	return arg1.Resolve().IsImmediate()
-	           ? EmitConstantShift64(state, spv::OpShiftRightArithmetic, arg0, arg1.Resolve().U32())
-	           : EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1));
-}
-
-uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1);
-}
-
 uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
-	const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
+	// Vulkan bit-count operands remain 32-bit, including when counting a U64.
+	const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0);
+	const auto pair =
+	    ExtractPair(state, Unary(state, spv::OpBitCount, TypeU32Vector(state, 2), unpacked));
 	return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
 }
 
-uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0) {
-	const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindILsb, {arg0});
-	return Unary(state, spv::OpBitcast, TypeU32(state), value);
-}
-
-uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0) {
-	const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindUMsb, {arg0});
-	return Unary(state, spv::OpBitcast, TypeU32(state), value);
-}
-
-uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return EmitMinMaxI32Value(state, arg0, arg1, false);
-}
-
-uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return EmitMinMaxI32Value(state, arg0, arg1, true);
-}
-
-uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return EmitMinMaxU32Value(state, arg0, arg1, false);
-}
-
-uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return EmitMinMaxU32Value(state, arg0, arg1, true);
-}
-
 uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
-	return EmitMinMax3(state, arg0, arg1, arg2, true, false);
+	return EmitSMin32(state, arg0, EmitSMin32(state, arg1, arg2));
 }
 
 uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
-	return EmitMinMax3(state, arg0, arg1, arg2, true, true);
+	return EmitSMax32(state, arg0, EmitSMax32(state, arg1, arg2));
 }
 
 uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
-	return EmitMinMax3(state, arg0, arg1, arg2, false, false);
+	return EmitUMin32(state, arg0, EmitUMin32(state, arg1, arg2));
 }
 
 uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
-	return EmitMinMax3(state, arg0, arg1, arg2, false, true);
+	return EmitUMax32(state, arg0, EmitUMax32(state, arg1, arg2));
 }
 
 uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
-	return EmitMed3(state, arg0, arg1, arg2, true);
+	const auto high_min = EmitSMin32(state, EmitSMax32(state, arg0, arg1), arg2);
+	return EmitSMax32(state, EmitSMin32(state, arg0, arg1), high_min);
 }
 
 uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
-	return EmitMed3(state, arg0, arg1, arg2, false);
-}
-
-uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareEqual64(state, arg0, arg1, false);
-}
-
-uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareEqual64(state, arg0, arg1, true);
-}
-
-uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan);
-}
-
-uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan);
-}
-
-uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
-}
-
-uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
-}
-
-uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
-}
-
-uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
-}
-
-uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
-	return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
+	const auto high_min = EmitUMin32(state, EmitUMax32(state, arg0, arg1), arg2);
+	return EmitUMax32(state, EmitUMin32(state, arg0, arg1), high_min);
 }
 
 uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
@@ -468,6 +271,14 @@ uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return EmitMinMaxF32Value(state, arg0, arg1, true);
 }
 
+uint32_t EmitFPMin64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return EmitMinMaxF64(state, arg0, arg1, false);
+}
+
+uint32_t EmitFPMax64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return EmitMinMaxF64(state, arg0, arg1, true);
+}
+
 uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
 	return EmitFMinMax3(state, arg0, arg1, arg2, false);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -82,35 +82,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu);
 }
 
-uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
-	const auto cond = state.builder.AllocateId();
-	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state),
-	                          cond, lhs, rhs);
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs);
-	return ret;
-}
-
-uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
-	const auto cond = state.builder.AllocateId();
-	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction(max_value ? spv::OpSGreaterThan : spv::OpSLessThan, TypeBool(state),
-	                          cond, lhs, rhs);
-	state.builder.AddFunction(spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs);
-	return ret;
-}
-
 F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
 	F32Class cls;
-	cls.bits                 = bits;
-	const auto abs_bits      = EmitAndConstant(state, cls.bits, 0x7fffffffu);
-	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
-	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
-	const auto exponent_max =
-	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
-	const auto mantissa_nonzero = EmitCompareU32Constant(state, spv::OpINotEqual, mantissa_bits, 0);
-	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
-	cls.zero                    = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
+	cls.bits            = bits;
+	const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu);
+	cls.nan  = EmitCompareU32Constant(state, spv::OpUGreaterThan, abs_bits, 0x7f800000u);
+	cls.zero = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
 	return cls;
 }
 
@@ -125,20 +102,23 @@ uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
 	return EmitLogicalAndBool(state, selected, class_match);
 }
 
-uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
-	const auto bits          = EmitBitcastF32ToU32(state, value);
-	const auto sign_bits     = EmitAndConstant(state, bits, 0x80000000u);
-	const auto abs_bits      = EmitAndConstant(state, bits, 0x7fffffffu);
-	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
-	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
-	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
+static uint32_t EmitClassMaskBits(EmitterState& state, uint32_t bits, uint32_t mask, bool half) {
+	const uint32_t sign_mask     = half ? 0x8000u : 0x80000000u;
+	const uint32_t exponent_mask = half ? 0x7c00u : 0x7f800000u;
+	const uint32_t mantissa_mask = half ? 0x03ffu : 0x007fffffu;
+	const uint32_t quiet_mask    = half ? 0x0200u : 0x00400000u;
+	const auto     sign_bits     = EmitAndConstant(state, bits, sign_mask);
+	const auto     abs_bits      = EmitAndConstant(state, bits, sign_mask - 1u);
+	const auto     exponent_bits = EmitAndConstant(state, abs_bits, exponent_mask);
+	const auto     mantissa_bits = EmitAndConstant(state, abs_bits, mantissa_mask);
+	const auto     quiet_bits    = EmitAndConstant(state, mantissa_bits, quiet_mask);
 
 	const auto sign             = EmitCompareU32Constant(state, spv::OpINotEqual, sign_bits, 0);
 	const auto positive         = EmitLogicalNotBool(state, sign);
 	const auto exponent_zero    = EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0);
 	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
 	const auto exponent_inf =
-	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, exponent_mask);
 	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
 	const auto mantissa_zero    = EmitCompareU32Constant(state, spv::OpIEqual, mantissa_bits, 0);
 	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
@@ -178,6 +158,14 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	    EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive)));
 }
 
+uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
+	return EmitClassMaskBits(state, EmitBitcastF32ToU32(state, value), mask, false);
+}
+
+uint32_t EmitClassMaskF16(EmitterState& state, uint32_t bits, uint32_t mask) {
+	return EmitClassMaskBits(state, bits, mask, true);
+}
+
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto lhs_class = EmitClassifyF32(state, lhs);
 	const auto rhs_class = EmitClassifyF32(state, rhs);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -59,21 +59,6 @@ uint32_t          ConstantU32(EmitterState& state, uint32_t value);
 	std::abort();
 }
 
-uint32_t ResourceForDescriptor(const EmitterState& state, IR::DescriptorBindingKind kind,
-                               uint32_t resource) {
-	const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
-	if (descriptor == nullptr) {
-		ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
-	}
-	const auto found =
-	    std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
-	if (found == descriptor->resources.end()) {
-		ExitDescriptorBindingFailure(state, kind, resource,
-		                             "resource is absent from descriptor group");
-	}
-	return static_cast<uint32_t>(found - descriptor->resources.begin());
-}
-
 uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
                                   uint32_t variable_id, uint32_t array_index,
                                   IR::DescriptorBindingKind kind, uint32_t resource,
@@ -83,7 +68,7 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
 	}
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpAccessChain, result_ptr_type, pointer, variable_id,
-	                          ConstantU32(state, array_index));
+	                          array_index);
 	return pointer;
 }
 
@@ -136,8 +121,8 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 		EXIT("invalid image resource class");
 	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
-	const auto scalar_type = image.atomic64 ? TypeScalarU64(state)
-	                                        : ImageScalarType(state, image.numeric_class);
+	const auto  scalar_type =
+	    image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
 	return state.builder.Type(spv::OpTypeImage, scalar_type,
 	                          info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
 	                          info.multisampled, sampled, format);
@@ -152,49 +137,74 @@ uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
 	}
 }
 
-uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip) {
-	const auto pointer = ImageDescriptorPointer(state, resource, mip);
+namespace {
+void DecorateNonUniformDescriptor(EmitterState& state, uint32_t id) {
+	state.builder.RequireExtension("SPV_EXT_descriptor_indexing");
+	state.builder.RequireCapability(spv::CapabilityShaderNonUniform);
+	state.builder.RequireCapability(spv::CapabilitySampledImageArrayNonUniformIndexing);
+	state.builder.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform);
+}
+} // namespace
+
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip,
+                             uint32_t array_index) {
+	const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index);
 	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpLoad, ImageType(state, state.program.info.images.at(resource)),
+	const auto kind = *IR::DescriptorBindingForImage(state.program.info.images[resource]);
+	state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type,
 	                          image, pointer);
+	if (array_index != 0u) {
+		DecorateNonUniformDescriptor(state, image);
+	}
 	return image;
 }
 
-uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
-	const auto array_index =
-	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
-	const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
-	const auto pointer_type =
-	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type);
+uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t array_index) {
+	if (sampler >= state.program.info.samplers.size()) {
+		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Samplers, sampler,
+		                             "sampler resource index is out of range");
+	}
 	const auto pointer = DescriptorElementPointer(
-	    state, pointer_type, state.sampler_variable, array_index,
+	    state, state.sampler_pointer_type, state.sampler_variable,
+	    array_index == 0u ? ConstantU32(state, sampler) : array_index,
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer);
+	state.builder.AddFunction(spv::OpLoad, state.sampler_type, sampler_id, pointer);
+	if (array_index != 0u) DecorateNonUniformDescriptor(state, sampler_id);
 	return sampler_id;
 }
 
-uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip) {
+uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip,
+                          uint32_t array_index, bool sampler_dynamic) {
 	const auto& image_resource = state.program.info.images.at(resource);
 	EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
-	const auto  image          = LoadImageDescriptor(state, resource, mip);
+	const auto  image          = LoadImageDescriptor(state, resource, mip, array_index);
 	const auto  sampled_image = state.builder.AllocateId();
-	const auto  sampled_type =
-	    state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource));
+	const auto kind = *IR::DescriptorBindingForImage(image_resource);
+	auto& sampled_type = state.images[IR::ImageBindingIndex(kind)].sampled_type;
+	if (sampled_type == 0) {
+		sampled_type = state.builder.Type(
+		    spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type);
+	}
 	state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
+	if (array_index != 0u || sampler_dynamic) {
+		DecorateNonUniformDescriptor(state, sampled_image);
+	}
 	return sampled_image;
 }
 
-uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip) {
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip,
+                                uint32_t array_index) {
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(mip >= image.mip_count);
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
-	const auto array_index  = ResourceForDescriptor(state, *kind, resource) + mip;
-	const auto pointer_type = state.builder.Type(
-	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
-	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
-	return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
+	if (array_index == 0u) {
+		array_index = ConstantU32(state, image.descriptor_index + mip);
+	}
+	const auto& definition = state.images[IR::ImageBindingIndex(*kind)];
+	return DescriptorElementPointer(state, definition.pointer_type, definition.variable,
+	                                array_index, *kind, resource,
 	                                "image descriptor array was not emitted");
 }
 
@@ -213,7 +223,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 		EmitWrite(0);
 		return;
 	}
-	EmitImageMipSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
+	EmitIndexSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
 }
 
 spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -19,8 +19,8 @@ uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> compon
 
 uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
 	const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
-	const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
-	                            Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
+	const auto address =
+	    Binary(s, spv::OpIAdd, TypeU64(s), block, Unary(s, spv::OpUConvert, TypeU64(s), offset));
 	const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
 	const auto result = s.builder.AllocateId();
 	s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
@@ -223,16 +223,16 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
 } // namespace
 
 void DefineBvhIntersect(EmitterState& s) {
-	if (!s.requirements.bvh) return;
-	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
+	if (!s.program.info.bvh) return;
+	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s);
 	const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
-	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3);
 	s.bvh_intersect_function = s.builder.AllocateId();
 	s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
 	s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
 	                      spv::FunctionControlMaskNone, signature);
 	std::array<uint32_t, 6> args;
-	const std::array types {vec4, u, f, vec3, vec3, vec3};
+	const std::array types {vec4, wide, f, vec3, vec3, vec3};
 	for (uint32_t i = 0; i < args.size(); ++i) {
 		args[i] = s.builder.AllocateId();
 		s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
@@ -253,29 +253,31 @@ void DefineBvhIntersect(EmitterState& s) {
 	const auto shr = [&](uint32_t value, uint32_t shift) {
 		return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
 	};
-	const auto kind = and_bits(node, 7);
+	const auto kind = and_bits(Unary(s, spv::OpUConvert, u, node), 7);
 	const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
 	const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
 	const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
 	const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
 	const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
 	const auto grow = and_bits(shr(words[1], 23), 0xff);
-	const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
-	    DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
-	const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
+	const auto base  = Binary(s, spv::OpShiftLeftLogical, wide,
+	                          PackU64(s, words[0], and_bits(words[1], 0xff)), ConstantU32(s, 8));
+	const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantU32(s, 3));
 	const auto last_index = Binary(s, spv::OpIAdd, wide, index,
-	    Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
+	                               Select(s, wide, full, ConstantU64(s, 1), ConstantU64(s, 0)));
 	const auto address = Binary(s, spv::OpIAdd, wide, base,
-	    Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
-	const auto last_address = Binary(s, spv::OpIAdd, wide, base,
-	    Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
+	                            Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantU32(s, 6)));
+	const auto last_address =
+	    Binary(s, spv::OpIAdd, wide, base,
+	           Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantU32(s, 6)));
 	auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
 	const auto require = [&](uint32_t condition) {
 		valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
 	};
 	require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
-	require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
-	require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
+	require(Binary(s, spv::OpULessThanEqual, b, last_index,
+	               PackU64(s, words[2], and_bits(words[3], 0x3ff))));
+	require(Binary(s, spv::OpULessThan, b, last_address, ConstantU64(s, uint64_t {1} << 48)));
 	require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
 	for (uint32_t i = 0; i < 3; ++i) {
 		for (const auto value: {origin[i], direction[i], inverse[i]})
@@ -292,13 +294,13 @@ void DefineBvhIntersect(EmitterState& s) {
 	const auto invalid = Vector(s, vec4, invalid_words);
 	const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
 		const auto first = GetBdaPointer(s, address);
-		const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
-			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
+		const auto second  = EmitValueOrDefaultIfCondition(s, full, wide, ConstantU64(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantU64(s, 64)));
 		});
-		const auto present = Binary(s, spv::OpLogicalAnd, b,
-		    Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
+		const auto present = Binary(
+		    s, spv::OpLogicalAnd, b, Binary(s, spv::OpINotEqual, b, first, ConstantU64(s, 0)),
 		    Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
-		        Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
+		           Binary(s, spv::OpINotEqual, b, second, ConstantU64(s, 0))));
 		return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
 			const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
 			const auto merge = s.builder.AllocateId();
@@ -325,17 +327,22 @@ void DefineBvhIntersect(EmitterState& s) {
 uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& s = ctx.state;
 	const auto* ray = ctx.ImageAddress(inst.Arg(1));
-	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
-		ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
+	const auto node_words = inst.Flags<uint32_t>();
+	if (ray == nullptr || (node_words != 1u && node_words != 2u) ||
+	    ray->NumArgs() < node_words + 10u || s.bvh_intersect_function == 0)
+		ctx.Fail(inst, "BVH intersection requires its node pointer, ray components and shared function");
 	const auto vector = [&](uint32_t first) {
 		Vec3 values;
 		for (uint32_t i = 0; i < 3; ++i)
 			values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
 		return Vector(s, TypeF32Vector(s, 3), values);
 	};
-	const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
-	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
-	const auto origin = vector(2), direction = vector(5), inverse = vector(8);
+	const auto descriptor = ctx.Arg(inst, 0);
+	const auto node   = node_words == 2u ? PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
+	                                     : Unary(s, spv::OpUConvert, TypeU64(s), ctx.Arg(*ray, 0));
+	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words));
+	const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u),
+	           inverse = vector(node_words + 7u);
 	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
 	    ConstantU32CompositeZero(s, 4), [&] {
 		const auto result = s.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -10,7 +10,7 @@ namespace {
 
 bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) {
 	const auto register_index = IR::RegIndex(reg);
-	const auto& registers = state.program.bindings.user_data_registers;
+	const auto& registers = state.program.info.user_data_registers;
 	const auto  found     = std::lower_bound(registers.begin(), registers.end(), register_index);
 	if (found == registers.end() || *found != register_index) {
 		return false;
@@ -42,8 +42,9 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
 		                  EmitBinaryU32(state, spv::OpIMul, group, ConstantU32(state, size)));
 	}
 	const bool centroid = kind == IR::StageInputKind::BaryCoordSmoothCentroid;
+	const bool sample = kind == IR::StageInputKind::BaryCoordSmoothSample;
 	const auto variable = InputVariableForKind(
-	    state, centroid ? IR::StageInputKind::BaryCoordSmooth : kind);
+	    state, centroid || sample ? IR::StageInputKind::BaryCoordSmooth : kind);
 	if (variable == 0) {
 		return ConstantU32(state, 0);
 	}
@@ -76,15 +77,24 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
 		state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
 		return bits;
 	}
-	if (centroid || kind == IR::StageInputKind::BaryCoordSmooth ||
+	if (centroid || sample || kind == IR::StageInputKind::BaryCoordSmooth ||
 	    kind == IR::StageInputKind::BaryCoordNoPerspective) {
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		if (centroid) {
+		if (centroid || sample) {
 			const auto coordinates = state.builder.AllocateId();
 			state.builder.RequireCapability(spv::CapabilityInterpolationFunction);
-			state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 3), coordinates,
-			                          GlslStd450(state), GLSLstd450InterpolateAtCentroid, variable);
+			if (sample) {
+				const auto sample_id = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpLoad, TypeI32(state), sample_id,
+				                          InputVariableForKind(state, IR::StageInputKind::SampleId));
+				state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 3), coordinates,
+				                          GlslStd450(state), GLSLstd450InterpolateAtSample,
+				                          variable, sample_id);
+			} else {
+				state.builder.AddFunction(spv::OpExtInst, TypeF32Vector(state, 3), coordinates,
+				                          GlslStd450(state), GLSLstd450InterpolateAtCentroid, variable);
+			}
 			state.builder.AddFunction(spv::OpCompositeExtract, TypeF32(state), value,
 			                          coordinates, component + 1u);
 		} else {
@@ -167,6 +177,13 @@ uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
 		return value;
 	};
 	if (input->per_vertex) {
+		if (PixelParameterIsFlat(state, attr)) {
+			const auto value = load_per_vertex(
+			    state.input_info.pixel->parameter_mode == ShaderPixelParameterMode::LastVertex ? 2u : 0u);
+			const auto bits  = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
+			return bits;
+		}
 		const auto barycentric_kind = state.input_info.pixel->ps_no_perspective
 		                                  ? IR::StageInputKind::BaryCoordNoPerspective
 		                                  : IR::StageInputKind::BaryCoordSmooth;
@@ -221,8 +238,12 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
 		return value;
 	};
 
-	const auto selected_vertex = (mode + 1u) % 3u;
-	uint32_t   value           = load_vertex(selected_vertex);
+	uint32_t selected_vertex = (mode + 1u) % 3u;
+	if (mode == 2u && PixelParameterIsFlat(state, attr)) {
+		selected_vertex =
+		    state.input_info.pixel->parameter_mode == ShaderPixelParameterMode::LastVertex ? 2u : 0u;
+	}
+	uint32_t value = load_vertex(selected_vertex);
 	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
 		const auto delta = state.builder.AllocateId();
 		state.builder.AddFunction(spv::OpFSub, TypeF32(state), delta, value, load_vertex(0));
@@ -252,8 +273,12 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
                       bool uint_output) {
 	auto& state = ctx.state;
 	if (exp.compr && !uint_output) {
-		const auto unpack =
-		    MrtOutputMode(state, exp) == 5u ? GLSLstd450UnpackUnorm2x16 : GLSLstd450UnpackHalf2x16;
+		auto unpack = GLSLstd450UnpackHalf2x16;
+		switch (MrtOutputMode(state, exp)) {
+			case 5: unpack = GLSLstd450UnpackUnorm2x16; break;
+			case 6: unpack = GLSLstd450UnpackSnorm2x16; break;
+			default: break;
+		}
 		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
 		                   ConstantF32(state, 0x3f800000u)};
 		for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -279,50 +304,46 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 		                          f32[0], f32[1], f32[2], f32[3]);
 		return vector;
 	}
-	uint32_t raw[4] = {
-	    ConstantU32(state, 0),
-	    ConstantU32(state, 0),
-	    ConstantU32(state, 0),
-	    ConstantU32(state, uint_output ? 1u : 0x3f800000u),
-	};
-	if (exp.compr) {
-		for (uint32_t pair = 0; pair < 2u; pair++) {
-			if ((exp.en & (3u << (pair * 2u))) == 0u) {
-				continue;
-			}
-			const auto packed = ExportRawComponent(ctx, data, pair);
-			for (uint32_t lane = 0; lane < 2u; lane++) {
-				const auto component = pair * 2u + lane;
-				if (((exp.en >> component) & 1u) == 0u) {
+	if (exp.compr || exp.en != 0xfu) {
+		uint32_t raw[4] = {
+		    ConstantU32(state, 0),
+		    ConstantU32(state, 0),
+		    ConstantU32(state, 0),
+		    ConstantU32(state, uint_output ? 1u : 0x3f800000u),
+		};
+		if (exp.compr) {
+			for (uint32_t pair = 0; pair < 2u; pair++) {
+				if ((exp.en & (3u << (pair * 2u))) == 0u) {
 					continue;
 				}
-				raw[component] = state.builder.AllocateId();
-				state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), raw[component],
-				                          packed, ConstantU32(state, lane * 16u),
-				                          ConstantU32(state, 16));
+				const auto packed = ExportRawComponent(ctx, data, pair);
+				for (uint32_t lane = 0; lane < 2u; lane++) {
+					const auto component = pair * 2u + lane;
+					if (((exp.en >> component) & 1u) == 0u) {
+						continue;
+					}
+					raw[component] = state.builder.AllocateId();
+					state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), raw[component],
+					                          packed, ConstantU32(state, lane * 16u),
+					                          ConstantU32(state, 16));
+				}
 			}
-		}
-	} else {
-		for (uint32_t component = 0; component < 4u; component++) {
-			if (((exp.en >> component) & 1u) != 0u) {
-				raw[component] = ExportRawComponent(ctx, data, component);
+		} else {
+			for (uint32_t component = 0; component < 4u; component++) {
+				if (((exp.en >> component) & 1u) != 0u) {
+					raw[component] = ExportRawComponent(ctx, data, component);
+				}
 			}
 		}
+		data = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), data,
+		                          raw[0], raw[1], raw[2], raw[3]);
 	}
 	if (uint_output) {
-		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), vector,
-		                          raw[0], raw[1], raw[2], raw[3]);
-		return vector;
-	}
-	uint32_t f32[4] {};
-	for (uint32_t component = 0; component < 4u; component++) {
-		f32[component] = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpBitcast, TypeF32(state), f32[component], raw[component]);
+		return data;
 	}
 	const auto vector = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
-	                          f32[1], f32[2], f32[3]);
+	state.builder.AddFunction(spv::OpBitcast, TypeF32Vector(state, 4), vector, data);
 	return vector;
 }
 
@@ -366,18 +387,23 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 			state.builder.AddFunction(spv::OpStore, state.point_size_variable, f32);
 			continue;
 		}
-		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
+		auto StoreDistance = [&](IR::StageOutputKind kind, uint32_t variable, uint32_t index) {
 			if (index == UINT32_MAX) {
 				return;
 			}
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain,
-			                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
-			                          pointer, variable, ConstantU32(state, index));
+			uint32_t pointer;
+			if (state.program.stage == ShaderType::Mesh) {
+				pointer = MeshOutputPointer(state, kind, index);
+			} else {
+				pointer = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpAccessChain,
+				                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+				                          pointer, variable, ConstantU32(state, index));
+			}
 			state.builder.AddFunction(spv::OpStore, pointer, f32);
 		};
-		StoreDistance(state.clip_distance_variable, output.clip_distance);
-		StoreDistance(state.cull_distance_variable, output.cull_distance);
+		StoreDistance(IR::StageOutputKind::ClipDistance, state.clip_distance_variable, output.clip_distance);
+		StoreDistance(IR::StageOutputKind::CullDistance, state.cull_distance_variable, output.cull_distance);
 	}
 }
 
@@ -425,16 +451,16 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
 	const auto quad_bits   = state.builder.AllocateId();
 	const auto result      = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
-	                          ConstantU64(state, 0x0000000100000001ull));
+	                          ConstantU32(state, 1));
 	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one);
 	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
-	                          ConstantU64(state, 0x0000000200000002ull));
+	                          ConstantU32(state, 2));
 	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one,
 	                          shifted_two);
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
 	                          ConstantU64(state, 0x1111111111111111ull));
 	state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits,
-	                          ConstantU64(state, 0x0000000f0000000full));
+	                          ConstantU64(state, 15));
 	return result;
 }
 
@@ -442,7 +468,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&       state = ctx.state;
 	const auto& exp   = ctx.Export(inst);
 	const auto  exec  = ctx.Arg(inst, 1);
-	if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
+	if (state.program.stage == ShaderType::Pixel && exp.vm && state.program.info.pixel_valid_mask &&
 	    state.pixel_valid_mask_variable != 0) {
 		const auto value = state.builder.AllocateId();
 		state.builder.AddFunction(spv::OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1),
@@ -454,7 +480,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	// Skip dormant color exports after their valid mask; MRT1 is reserved for logical alpha.
 	if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-	    exp.index != 0 && state.input_info.pixel->alpha_blend_source_remap) {
+	    exp.index != 0 && state.input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None) {
 		return;
 	}
 	EmitIfCondition(state, exec, [&]() {
@@ -498,14 +524,36 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
 		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-		    exp.index == 0 && !uint_output && state.input_info.pixel->alpha_blend_source_remap) {
+		    exp.index == 0 && !uint_output &&
+		    state.input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None) {
 			// Broadcast logical alpha before swizzling the primary output.
 			const auto blend_output =
 			    OutputVariableForExport(state, {.kind = IR::ExportTargetKind::Mrt, .index = 1});
 			if (blend_output != 0) {
-				const auto alpha = state.builder.AllocateId();
+				auto alpha = state.builder.AllocateId();
 				state.builder.AddFunction(spv::OpVectorShuffle, vector_type, alpha, value, value,
 				                          3u, 3u, 3u, 3u);
+				uint32_t alpha_factor = 0;
+				switch (state.input_info.pixel->alpha_blend_source) {
+					case ShaderAlphaBlendSource::SourceAlphaOne:
+						alpha_factor = ConstantF32Value(state, 1.0f);
+						break;
+					case ShaderAlphaBlendSource::SourceAlphaZero:
+						alpha_factor = ConstantF32Value(state, 0.0f);
+						break;
+					default: break;
+				}
+				if (alpha_factor != 0) {
+					const auto mapping = state.input_info.pixel->target_export_mapping[0];
+					for (uint32_t component = 0; component < 4; component++) {
+						if (mapping.Map(component) != 3u) continue;
+						const auto factors = state.builder.AllocateId();
+						state.builder.AddFunction(spv::OpCompositeInsert, vector_type, factors,
+						                          alpha_factor, alpha, component);
+						alpha = factors;
+						break;
+					}
+				}
 				state.builder.AddFunction(spv::OpStore, blend_output, alpha);
 			}
 		}
@@ -574,6 +622,14 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
 
 void EmitVoid(ValueEmitContext&) {}
 
+void EmitStoreCompletion(EmitterState& state) {
+	state.builder.AddFunction(
+	    spv::OpControlBarrier, ConstantU32(state, spv::ScopeSubgroup),
+	    ConstantU32(state, spv::ScopeDevice),
+	    ConstantU32(state, spv::MemorySemanticsReleaseMask | spv::MemorySemanticsUniformMemoryMask |
+	                          spv::MemorySemanticsImageMemoryMask));
+}
+
 void EmitBarrier(EmitterState& state) {
 	const auto tessellation = state.program.stage == ShaderType::TessellationControl;
 	if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
@@ -581,9 +637,11 @@ void EmitBarrier(EmitterState& state) {
 		return;
 	}
 	const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
-	const auto semantics    = tessellation ? spv::MemorySemanticsMaskNone
-	                                       : spv::MemorySemanticsAcquireReleaseMask |
-	                                             spv::MemorySemanticsWorkgroupMemoryMask;
+	const auto memory = state.lds_storage_class == spv::StorageClassStorageBuffer
+	                        ? spv::MemorySemanticsUniformMemoryMask
+	                        : spv::MemorySemanticsWorkgroupMemoryMask;
+	const auto semantics = tessellation ? spv::MemorySemanticsMaskNone
+	                                    : spv::MemorySemanticsAcquireReleaseMask | memory;
 	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
 	                          ConstantU32(state, memory_scope), ConstantU32(state, semantics));
 }
@@ -623,6 +681,14 @@ uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index)
 	return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
 }
 
+uint32_t EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto start = ctx.state.program.bindings.dispatch_thread_dword;
+	if (start == IR::PushData::NoStart || !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() >= 3u) {
+		ctx.Fail(inst, "invalid dispatch thread extent");
+	}
+	return EmitShaderDataDwordLoad(ctx.state, start + inst.Arg(0).U32());
+}
+
 uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
 	const auto result = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpUndef, TypeId(state, inst.GetType()), result);
@@ -654,26 +720,18 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 }
 
 uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) {
-	if (ctx.other_half == nullptr) return ctx.Arg(inst, 0);
-	// A native scalar branch makes one decision for both emulated wave halves.
-	if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst));
+	// Scalar branches include lanes disabled by EXEC, which the branch body may restore.
+	if (ctx.other_half != nullptr && ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst));
 	const auto kind = inst.Flags<CFG::BranchCondition>();
 	if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0);
-	const auto ballot = ctx.Ballot(inst.Arg(0));
-	const auto low = ctx.state.builder.AllocateId();
-	const auto high = ctx.state.builder.AllocateId();
-	const auto combined = ctx.state.builder.AllocateId();
-	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
 	const bool zero = kind == CFG::BranchCondition::ExecZero ||
 	                  kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero;
-	ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
-	                              TypeU32(ctx.state), combined, low, high);
-	ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual,
-	                              TypeBool(ctx.state), result, combined,
-	                              ConstantU32(ctx.state, zero ? ~0u : 0u));
-	return result;
+	const auto ballot = ctx.Ballot(inst.Arg(0));
+	const auto reference = zero ? ctx.Ballot(IR::Value(true))
+	                            : ConstantU32CompositeZero(ctx.state, 4);
+	const auto compare = Binary(ctx.state, zero ? spv::OpIEqual : spv::OpINotEqual,
+	                            TypeBoolVector(ctx.state, 4), ballot, reference);
+	return Unary(ctx.state, zero ? spv::OpAll : spv::OpAny, TypeBool(ctx.state), compare);
 }
 
 uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -391,28 +391,50 @@ Format::BufferFormatInfo ImageConversionFormat(const EmitterState&   state,
 	const auto format = state.program.info.images[mem.resource].conversion_format;
 	if (format == Prospero::BufferFormat::kInvalid) return {};
 	const auto info = Format::GetFormatInfo(format);
-	EXIT_IF(Prospero::SampledTextureNumericClass(format) != Prospero::TextureNumericClass::Uint ||
-	        Prospero::RemapTextureFormat(format) == format ||
-	        info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
-	        info.byte_size != sizeof(uint32_t) || info.component_count == 0u ||
+	EXIT_IF(Prospero::RemapTextureFormat(format) == format || info.component_count == 0u ||
 	        info.component_count > 4u);
+	EXIT_IF(info.type != Format::ComponentType::Uscaled &&
+	        (info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
+	         info.byte_size != sizeof(uint32_t)));
 	return info;
 }
 
+uint32_t ImageGatherSource(const EmitterState& state, const IR::MemoryInfo& mem) {
+	const auto component = ImageGatherComponent(mem.dmask);
+	const auto info      = ImageConversionFormat(state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return component;
+	if (info.packed_bitfield) return 0u;
+	const auto selector =
+	    (state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	return selector >= 4u ? (selector - 4u) % info.component_count : 0u;
+}
+
 uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
 	const auto info = ImageConversionFormat(ctx.state, mem);
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
 
-	const auto packed = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u);
-	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
-	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
+	const auto scalar_type   = ImageScalarType(ctx.state, numeric_class);
+	uint32_t   packed        = 0;
+	if (info.packed_bitfield) {
+		packed = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type, packed, texel, 0u);
+	}
+	uint32_t components[4] {};
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		components[component] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(ctx.state),
-		                              components[component], packed,
-		                              ConstantU32(ctx.state, info.component_bit_offset[component]),
-		                              ConstantU32(ctx.state, info.component_bits[component]));
+		if (info.packed_bitfield) {
+			ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, scalar_type,
+			                              components[component], packed,
+			                              ConstantU32(ctx.state, info.component_bit_offset[component]),
+			                              ConstantU32(ctx.state, info.component_bits[component]));
+		} else {
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type,
+			                              components[component], texel, component);
+			// UNorm backing preserves the guest's filtering; scaling precedes swizzle constants.
+			components[component] = Binary(ctx.state, spv::OpFMul, scalar_type, components[component],
+			                               ConstantF32Value(ctx.state, 255.0f));
+		}
 	}
 	for (uint32_t component = info.component_count; component < 4u; component++) {
 		components[component] = components[component % info.component_count];
@@ -423,16 +445,19 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 	for (uint32_t component = 0; component < 4u; component++) {
 		const auto selector = (swizzle >> (component * 3u)) & 7u;
 		if (selector == 1u) {
-			selected[component] = ConstantU32(ctx.state, 1u);
+			selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 1u)
+			                                          : ConstantF32Value(ctx.state, 1.0f);
 		} else if (selector >= 4u) {
 			selected[component] = components[selector - 4u];
 		} else {
-			selected[component] = ConstantU32(ctx.state, 0u);
+			selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 0u)
+			                                          : ZeroF32(ctx.state);
 		}
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                              selected[0], selected[1], selected[2], selected[3]);
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
+	                              ImageVectorType(ctx.state, numeric_class, 4), result, selected[0],
+	                              selected[1], selected[2], selected[3]);
 	return result;
 }
 
@@ -443,10 +468,24 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 	const auto component = ImageGatherComponent(mem.dmask);
 	const auto selector =
 	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
+	const auto vector_type   = ImageVectorType(ctx.state, numeric_class, 4);
 	if (selector < 4u) {
-		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
-		return ctx.state.builder.Constant(spv::OpConstantComposite, TypeU32Vector(ctx.state, 4),
-		                                  value, value, value, value);
+		uint32_t value;
+		switch (info.type) {
+			case Format::ComponentType::Uscaled:
+				value = ConstantF32Value(ctx.state, selector == 1u ? 1.0f : 0.0f);
+				break;
+			default: value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u); break;
+		}
+		return ctx.state.builder.Constant(spv::OpConstantComposite, vector_type, value, value,
+		                                  value, value);
+	}
+	if (!info.packed_bitfield) {
+		const auto scale  = ConstantF32Value(ctx.state, 255.0f);
+		const auto scales = ctx.state.builder.Constant(spv::OpConstantComposite, vector_type,
+		                                                scale, scale, scale, scale);
+		return Binary(ctx.state, spv::OpFMul, vector_type, gathered, scales);
 	}
 
 	uint32_t values[4] {};
@@ -488,10 +527,7 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 	                                     LoadSamplerDescriptor(state, mem.sampler));
 	const auto vector_type = ImageVectorType(state, numeric_class, 4);
 	const auto scalar_type = ImageScalarType(state, numeric_class);
-	const auto component =
-	    ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid
-	        ? ImageGatherComponent(mem.dmask)
-	        : 0u;
+	const auto component = ImageGatherSource(state, mem);
 	uint32_t values[2] {};
 	for (uint32_t index = 0; index < 2u; index++) {
 		const auto sample_coord =
@@ -515,6 +551,7 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
 	const auto info = ImageConversionFormat(ctx.state, mem);
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+	EXIT_IF(info.type != Format::ComponentType::Uint || !info.packed_bitfield);
 
 	auto packed = ConstantU32(ctx.state, 0u);
 	for (uint32_t component = 0; component < info.component_count; component++) {
@@ -578,6 +615,90 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 	return PackImageTexel(ctx, mem, texel);
 }
 
+template <typename Emit>
+uint32_t EmitImageAccess(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t result_type,
+                         const Emit& emit) {
+	auto& state = ctx.state;
+	const auto& mem = ctx.Memory(inst);
+	const auto& image = state.program.info.images.at(mem.resource);
+	if (image.indirect_root != mem.resource) return emit(mem.resource, 0u);
+	const auto* handle = inst.Arg(0).ResolveInstruction();
+	const auto* source = image.source < state.program.descriptor_sources.size()
+	                         ? &state.program.descriptor_sources[image.source]
+	                         : nullptr;
+	if (handle == nullptr || source == nullptr || !source->indirect_descriptor.has_value() ||
+	    handle->NumArgs() == 0u) {
+		ctx.Fail(inst, "has invalid indirect image key provenance");
+	}
+	const auto key = ctx.Def(handle->Arg(0));
+	if (state.flattened_srt_variable == 0 ||
+	    image.indirect_resources.size() < 2u) {
+		ctx.Fail(inst, "has no indirect image runtime mapping");
+	}
+	const auto selected = EmitIndirectResourceIndex(
+	    state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
+	struct ImageRun {
+		uint32_t first;
+		uint32_t count;
+		uint32_t resource;
+		uint32_t slot_bias;
+	};
+	std::vector<ImageRun> runs;
+	for (uint32_t ordinal = 0; ordinal < image.indirect_resources.size(); ++ordinal) {
+		const auto resource = image.indirect_resources[ordinal];
+		const auto& candidate = state.program.info.images[resource];
+		const auto kind = *IR::DescriptorBindingForImage(candidate);
+		if (!runs.empty()) {
+			auto& run = runs.back();
+			const auto& first = state.program.info.images[run.resource];
+			const auto next_slot = run.slot_bias + ordinal;
+			if (candidate.dimension == first.dimension && candidate.cube == first.cube &&
+			    IR::DescriptorBindingForImage(first) == kind &&
+			    candidate.mip_count == 1u && first.mip_count == 1u &&
+			    (ordinal == 1u || candidate.descriptor_index == next_slot)) {
+				// Native roots precede appended children; only the root may have a slot gap.
+				if (ordinal == 1u)
+					run.slot_bias = candidate.descriptor_index - ordinal;
+				++run.count;
+				continue;
+			}
+		}
+		runs.push_back({ordinal, 1u, resource,
+		                candidate.descriptor_index - ordinal});
+	}
+	const auto EmitRun = [&](uint32_t index) {
+		const auto& run = runs[index];
+		if (run.count == 1u) return emit(run.resource, 0u);
+		auto slot = Binary(state, spv::OpIAdd, TypeU32(state), selected,
+		                   ConstantU32(state, run.slot_bias));
+		if (run.first == 0u) {
+			const auto root_slot = image.descriptor_index;
+			if (root_slot != run.slot_bias) {
+				const auto is_root = Binary(state, spv::OpIEqual, TypeBool(state), selected,
+				                            ConstantU32(state, 0u));
+				const auto root_or_child = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpSelect, TypeU32(state), root_or_child,
+				                          is_root, ConstantU32(state, root_slot), slot);
+				slot = root_or_child;
+			}
+		}
+		return emit(run.resource, slot);
+	};
+	auto run_index = ConstantU32(state, 0u);
+	for (uint32_t index = 1; index < runs.size(); ++index) {
+		const auto at_or_after = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state),
+		                               selected, ConstantU32(state, runs[index].first));
+		const auto next = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpSelect, TypeU32(state), next, at_or_after,
+		                          ConstantU32(state, index), run_index);
+		run_index = next;
+	}
+	return runs.size() == 1u
+	           ? EmitRun(0u)
+	           : EmitIndexSwitch(state, run_index, static_cast<uint32_t>(runs.size()),
+	                             result_type, EmitRun);
+}
+
 } // namespace
 
 void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -619,29 +740,35 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		return;
 	}
 	if (op == IR::ValueOpcode::ImageRead) {
-		const auto  dimension      = image.dimension;
-		const auto& dimension_info = ImageDimensionInfoFor(dimension);
-		const auto  numeric_class  = image.numeric_class;
-		const auto  condition      = ctx.Arg(inst, 2);
+		const auto numeric_class = image.numeric_class;
+		const auto condition = ctx.Arg(inst, 2);
 		ctx.Define(
 		    inst,
 		    EmitValueOrDefaultIfCondition(
 		        state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4),
 		        [&]() {
-			        const auto descriptor = LoadImageDescriptor(state, mem.resource);
-			        const auto color      = state.builder.AllocateId();
-			        const auto coord      = CoordU32(ctx, mem, *address, dimension);
-			        if (dimension_info.multisampled != 0u) {
-				        state.builder.AddFunction(
-				            spv::OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
-				            descriptor, coord, spv::ImageOperandsSampleMask,
-				            AddressU32(ctx, mem, *address, dimension_info.coordinate_components));
-			        } else {
-				        state.builder.AddFunction(spv::OpImageFetch,
-				                                  ImageVectorType(state, numeric_class, 4), color,
-				                                  descriptor, coord, spv::ImageOperandsLodMask,
-				                                  LodU32(ctx, mem, *address, dimension));
-			        }
+			        const auto color = EmitImageAccess(
+			            ctx, inst, ImageVectorType(state, numeric_class, 4),
+			            [&](uint32_t resource, uint32_t array_index) {
+				        const auto& candidate = state.program.info.images[resource];
+				        const auto dimension = candidate.dimension;
+				        const auto& dimension_info = ImageDimensionInfoFor(dimension);
+				        const auto descriptor = LoadImageDescriptor(state, resource, 0u, array_index);
+				        const auto fetched = state.builder.AllocateId();
+				        const auto coord = CoordU32(ctx, mem, *address, dimension);
+				        if (dimension_info.multisampled != 0u) {
+					        state.builder.AddFunction(
+					            spv::OpImageFetch, ImageVectorType(state, candidate.numeric_class, 4), fetched,
+					            descriptor, coord, spv::ImageOperandsSampleMask,
+					            AddressU32(ctx, mem, *address, dimension_info.coordinate_components));
+				        } else {
+					        state.builder.AddFunction(
+					            spv::OpImageFetch, ImageVectorType(state, candidate.numeric_class, 4), fetched,
+					            descriptor, coord, spv::ImageOperandsLodMask,
+					            LodU32(ctx, mem, *address, mem.image_dimension));
+				        }
+				        return fetched;
+			            });
 			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class,
 			                            false, mem);
 		        }));
@@ -669,7 +796,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const bool  dref           = HasFlag(mem, Decoder::ImageSampleFlagCompare);
 		if (dref && state.program.info.images[mem.resource].conversion_format !=
 		                Prospero::BufferFormat::kInvalid) {
-			ctx.Fail(inst, "uses depth comparison with a packed integer image");
+			ctx.Fail(inst, "uses depth comparison with a converted image");
 			return;
 		}
 		if (op == IR::ValueOpcode::ImageGatherRaw) {
@@ -700,9 +827,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				                        ? AddressF32(ctx, mem, *address, layout.dref)
 				                        : ZeroF32(state);
 			} else {
-				component_or_dref = ConstantU32(
-				    state, ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid
-				               ? ImageGatherComponent(mem.dmask) : 0u);
+				component_or_dref = ConstantU32(state, ImageGatherSource(state, mem));
 			}
 			uint32_t operand_mask = 0;
 			uint32_t offset = 0;
@@ -717,18 +842,15 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto EmitGather = [&](uint32_t mip) {
 				const auto sampled = MakeSampledImage(state, mem.resource, sampler_id, mip);
 				const auto sample = state.builder.AllocateId();
-				std::vector<uint32_t> words {
-				    static_cast<uint32_t>(dref ? spv::OpImageDrefGather : spv::OpImageGather), result_type,
-				    sample, sampled, coord, component_or_dref};
-				if (operand_mask != 0u) {
-					words.push_back(operand_mask);
-					words.push_back(offset);
-				}
-				state.builder.AddFunction(words);
+				const std::array operands {operand_mask, offset};
+				state.builder.AddFunction(
+				    dref ? spv::OpImageDrefGather : spv::OpImageGather, result_type,
+				    sample, sampled, coord, component_or_dref,
+				    std::span(operands).first(operand_mask != 0u ? 2u : 0u));
 				return sample;
 			};
 			const auto sample = image.mip_mode == IR::ImageMipMode::Dynamic && image.mip_count > 1u
-			                        ? EmitImageMipSwitch(
+			                        ? EmitIndexSwitch(
 			                              state, GatherMip(ctx, inst, mem, *address, layout,
 			                                               image.mip_count),
 			                              image.mip_count, result_type, EmitGather)
@@ -756,145 +878,52 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				dref_value = AddressF32(ctx, mem, *address, layout.dref);
 			}
 		}
-		uint32_t              operand_mask = 0;
-		std::vector<uint32_t> operands;
+		std::array<uint32_t, 3> operands {};
+		size_t operand_count = 0;
 		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
-			operand_mask |= spv::ImageOperandsGradMask;
-			operands.push_back(
-			    CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components));
-			operands.push_back(
-			    CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components));
+			operands[0] = spv::ImageOperandsGradMask;
+			operands[1] = CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components);
+			operands[2] = CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components);
+			operand_count = 3;
 		} else if (explicit_lod) {
-			operand_mask |= spv::ImageOperandsLodMask;
 			auto lod = ZeroF32(state);
 			if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) {
 				lod = AddressF32(ctx, mem, *address, layout.lod);
 			}
-			operands.push_back(lod);
+			operands[0] = spv::ImageOperandsLodMask;
+			operands[1] = lod;
+			operand_count = 2;
 		} else if (layout.bias != NoImageComponent) {
-			operand_mask |= spv::ImageOperandsBiasMask;
-			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
+			operands[0] = spv::ImageOperandsBiasMask;
+			operands[1] = AddressF32(ctx, mem, *address, layout.bias);
+			operand_count = 2;
 		}
-		const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler);
-		const auto EmitSample = [&](uint32_t resource) {
+		const auto& samplers = state.program.info.samplers.at(mem.sampler).indirect_resources;
+		uint32_t sampler_index = 0;
+		if (!samplers.empty()) {
+			const auto* handle = inst.Arg(1).ResolveInstruction();
+			if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetSamplerResource)
+				ctx.Fail(inst, "has invalid finite sampler key provenance");
+			sampler_index = EmitIndexSwitch(
+			    state, ctx.Def(handle->Arg(0)), static_cast<uint32_t>(samplers.size()),
+			    TypeU32(state), [&](uint32_t ordinal) { return ConstantU32(state, samplers[ordinal]); });
+		}
+		const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler, sampler_index);
+		const auto EmitSample = [&](uint32_t resource, uint32_t array_index) {
 			const auto& candidate = state.program.info.images[resource];
 			const auto coord =
 			    CoordF32(ctx, mem, *address, layout.coord,
 			             ImageDimensionInfoFor(candidate.dimension).coordinate_components,
 			             candidate.cube);
-			const auto            sampled = MakeSampledImage(state, resource, sampler_id);
-			const auto            sample  = state.builder.AllocateId();
-			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};
-			if (dref) {
-				sample_operands.push_back(dref_value);
-			}
-			if (operand_mask != 0u) {
-				sample_operands.push_back(operand_mask);
-				sample_operands.insert(sample_operands.end(), operands.begin(), operands.end());
-			}
-			state.builder.AddFunction(opcode, sample_operands);
+			const auto sampled = MakeSampledImage(state, resource, sampler_id, 0u, array_index,
+			                                      sampler_index != 0u);
+			const auto sample = state.builder.AllocateId();
+			state.builder.AddFunction(opcode, result_type, sample, sampled, coord,
+			                          std::span(&dref_value, dref ? 1u : 0u),
+			                          std::span<const uint32_t>(operands).first(operand_count));
 			return sample;
 		};
-		if (image.indirect_root != mem.resource) {
-			const auto sample = EmitSample(mem.resource);
-			auto       result = sample;
-			if (!dref) {
-				result = UnpackImageTexel(ctx, mem, sample);
-			}
-			ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
-			return;
-		}
-		const auto* handle = image_arg.ResolveInstruction();
-		const auto* source = image.source < state.program.descriptor_sources.size()
-		                         ? &state.program.descriptor_sources[image.source]
-		                         : nullptr;
-		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
-		    handle->NumArgs() == 0u) {
-			ctx.Fail(inst, "has invalid indirect image key provenance");
-			return;
-		}
-		const auto key = ctx.Def(handle->Arg(0));
-		if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u ||
-		    image.indirect_resources.size() < 2u) {
-			ctx.Fail(inst, "has no indirect image runtime mapping");
-			return;
-		}
-		const auto LoadMapping = [&](uint32_t index) {
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
-			                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
-			                          index);
-			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
-			return value;
-		};
-		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
-		auto       low      = ConstantU32(state, 0u);
-		auto       high     = LoadMapping(mapping);
-		auto       selected = ConstantU32(state, 0u);
-		for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
-		     iteration++) {
-			const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
-			const auto mid    = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
-			                           Binary(state, spv::OpIAdd, TypeU32(state), low, high),
-			                           ConstantU32(state, 1u));
-			const auto probe  = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid,
-			                          ConstantU32(state, 0u));
-			const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
-			                          Binary(state, spv::OpIAdd, TypeU32(state),
-			                                 Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-			                                        probe, ConstantU32(state, 1u)),
-			                                 ConstantU32(state, 1u)));
-			const auto mapped_key = LoadMapping(entry);
-			const auto candidate  = LoadMapping(
-			    Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
-			const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
-			const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
-			const auto next_selected = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match,
-			                          candidate, selected);
-			selected              = next_selected;
-			const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
-			const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
-			const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
-			                               Unary(state, spv::OpLogicalNot, TypeBool(state), less));
-			const auto next_low   = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    spv::OpSelect, TypeU32(state), next_low, take_upper,
-			    Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
-			low                  = next_low;
-			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid,
-			                          high);
-			high = next_high;
-		}
-		const auto            default_label = state.builder.AllocateId();
-		const auto            merge_label   = state.builder.AllocateId();
-		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
-		std::vector<uint32_t> switch_words {spv::OpSwitch, selected, default_label};
-		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
-			labels[candidate - 1u] = state.builder.AllocateId();
-			switch_words.push_back(candidate);
-			switch_words.push_back(labels[candidate - 1u]);
-		}
-		state.builder.AddFunction(spv::OpSelectionMerge, merge_label,
-		                          spv::SelectionControlMaskNone);
-		state.builder.AddFunction(switch_words);
-		std::vector<uint32_t> phi_words {spv::OpPhi, result_type, state.builder.AllocateId()};
-		EmitLabel(state, default_label);
-		phi_words.push_back(EmitSample(image.indirect_resources[0]));
-		phi_words.push_back(default_label);
-		state.builder.AddFunction(spv::OpBranch, merge_label);
-		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
-			EmitLabel(state, labels[candidate - 1u]);
-			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
-			phi_words.push_back(labels[candidate - 1u]);
-			state.builder.AddFunction(spv::OpBranch, merge_label);
-		}
-		EmitLabel(state, merge_label);
-		state.builder.AddFunction(phi_words);
-		auto result = phi_words[2];
+		auto result = EmitImageAccess(ctx, inst, result_type, EmitSample);
 		if (!dref) {
 			result = UnpackImageTexel(ctx, mem, result);
 		}
@@ -908,9 +937,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(inst, EmitValueOrDefaultIfCondition(
 		                     state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() {
 			           const auto pointer      = state.builder.AllocateId();
-			           const auto pointer_type = state.builder.Type(
-			               spv::OpTypePointer, spv::StorageClassImage,
-			               image.atomic64 ? TypeScalarU64(state) : TypeU32(state));
+			           const auto pointer_type =
+			               state.builder.Type(spv::OpTypePointer, spv::StorageClassImage,
+			                                  image.atomic64 ? TypeU64(state) : TypeU32(state));
 			           state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
 			                                     ImageDescriptorPointer(state, mem.resource),
 			                                     CoordU32(ctx, mem, *address, dimension),

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -23,27 +23,22 @@ uint32_t              EmitConvertF16F32(EmitterState& state, uint32_t arg0);
 inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32;
 uint32_t              EmitConvertS32F32(EmitterState& state, uint32_t arg0);
 uint32_t              EmitConvertU32F32(EmitterState& state, uint32_t arg0);
-template <IR::Type type>
-uint32_t EmitConvertSigned32ToFloat(EmitterState& state, uint32_t arg0) {
-	const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0);
-	return EmitNative<spv::OpConvertSToF, type>(state, signed_value);
-}
-inline constexpr auto EmitConvertF32S32 = EmitConvertSigned32ToFloat<IR::Type::F32>;
-inline constexpr auto EmitConvertF64S32 = EmitConvertSigned32ToFloat<IR::Type::F64>;
+EMIT_NATIVE(ConvertF32S32, OpConvertSToF, F32, uint32_t)
+EMIT_NATIVE(ConvertF64S32, OpConvertSToF, F64, uint32_t)
 uint32_t              EmitConvertF32F64(EmitterState& state, uint32_t arg0);
 uint32_t              EmitConvertF64F32(EmitterState& state, uint32_t arg0);
 EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
 EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
-EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
+uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
 EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
             uint32_t)
 uint32_t              EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
-inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
-inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
-inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
+uint32_t              EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1);
+inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU32x2;
+inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU32x2;
 inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
 inline constexpr auto EmitPackSnorm2x16 =
     EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
@@ -63,34 +58,46 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
 EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
 EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
-uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(IAdd64, OpIAdd, U64, uint32_t, uint32_t)
+EMIT_NATIVE(ISub64, OpISub, U64, uint32_t, uint32_t)
+EMIT_NATIVE(IMul64, OpIMul, U64, uint32_t, uint32_t)
 EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
 uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
+inline constexpr auto EmitIAbs32 = EmitGlsl<GLSLstd450SAbs, IR::Type::U32, uint32_t>;
 EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
-uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
-uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
-uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+template <spv::Op opcode>
+uint32_t EmitShift64(ValueEmitContext& ctx, uint32_t value, IR::Value shift) {
+	shift = shift.Resolve();
+	if (shift.IsImmediate()) {
+		const auto amount = shift.U32() & 63u;
+		return amount == 0u ? value
+		                    : Binary(ctx.state, opcode, TypeU64(ctx.state), value,
+		                             ConstantU32(ctx.state, amount));
+	}
+	return Binary(ctx.state, opcode, TypeU64(ctx.state), value,
+	              EmitAndConstant(ctx.state, ctx.Def(shift), 63u));
+}
+inline constexpr auto EmitShiftLeftLogical64     = EmitShift64<spv::OpShiftLeftLogical>;
+inline constexpr auto EmitShiftRightLogical64    = EmitShift64<spv::OpShiftRightLogical>;
+inline constexpr auto EmitShiftRightArithmetic64 = EmitShift64<spv::OpShiftRightArithmetic>;
 EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
 EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
 EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
 EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
-uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(BitwiseAnd64, OpBitwiseAnd, U64, uint32_t, uint32_t)
 EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
 EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
 uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
-uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0);
-uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0);
+inline constexpr auto EmitFindILsb32 = EmitGlsl<GLSLstd450FindILsb, IR::Type::U32, uint32_t>;
+inline constexpr auto EmitFindUMsb32 = EmitGlsl<GLSLstd450FindUMsb, IR::Type::U32, uint32_t>;
 uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0);
-uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+inline constexpr auto EmitSMin32 = EmitGlsl<GLSLstd450SMin, IR::Type::U32, uint32_t, uint32_t>;
+inline constexpr auto EmitSMax32 = EmitGlsl<GLSLstd450SMax, IR::Type::U32, uint32_t, uint32_t>;
+inline constexpr auto EmitUMin32 = EmitGlsl<GLSLstd450UMin, IR::Type::U32, uint32_t, uint32_t>;
+inline constexpr auto EmitUMax32 = EmitGlsl<GLSLstd450UMax, IR::Type::U32, uint32_t, uint32_t>;
 uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
@@ -107,14 +114,14 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
 EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
 EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
 EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
-uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(IEqual64, OpIEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(INotEqual64, OpINotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThan64, OpULessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SLessThan64, OpSLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SLessThanEqual64, OpSLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThanEqual64, OpULessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThanEqual64, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThan64, OpUGreaterThan, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
@@ -150,8 +157,9 @@ inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32<spv::OpFOrdL
 inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32<spv::OpFUnordLessThanEqual>;
 inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32<spv::OpFOrdGreaterThanEqual>;
 inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32<spv::OpFUnordGreaterThanEqual>;
-uint32_t              EmitFPIsNan32(EmitterState& state, uint32_t arg0);
+EMIT_NATIVE(FPIsNan32, OpIsNan, U1, uint32_t)
 inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
+inline constexpr auto EmitFPCmpClass16 = EmitClassMaskF16;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
@@ -160,10 +168,12 @@ EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t)
 inline constexpr auto EmitFPFma64 =
     EmitGlsl<GLSLstd450Fma, IR::Type::F64, uint32_t, uint32_t, uint32_t>;
 uint32_t              EmitFPRecip64(EmitterState& state, uint32_t arg0);
-inline constexpr auto EmitFPFma32 =
-    EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
+uint32_t EmitFPFma32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPMad32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMin64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMax64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
@@ -178,7 +188,11 @@ inline constexpr auto EmitFPRoundEven32 = EmitGlsl<GLSLstd450RoundEven, IR::Type
 inline constexpr auto EmitFPFloor32     = EmitGlsl<GLSLstd450Floor, IR::Type::F32, uint32_t>;
 inline constexpr auto EmitFPCeil32      = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
 inline constexpr auto EmitFPTrunc32     = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFloor64     = EmitGlsl<GLSLstd450Floor, IR::Type::F64, uint32_t>;
+inline constexpr auto EmitFPCeil64      = EmitGlsl<GLSLstd450Ceil, IR::Type::F64, uint32_t>;
+inline constexpr auto EmitFPTrunc64     = EmitGlsl<GLSLstd450Trunc, IR::Type::F64, uint32_t>;
 inline constexpr auto EmitFPFract32     = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFract64     = EmitGlsl<GLSLstd450Fract, IR::Type::F64, uint32_t>;
 uint32_t              EmitFPSin(EmitterState& state, uint32_t arg0);
 uint32_t              EmitFPCos(EmitterState& state, uint32_t arg0);
 #undef EMIT_NATIVE
@@ -188,16 +202,17 @@ void                  EmitVoid(ValueEmitContext&);
 inline constexpr auto EmitReference    = EmitVoid;
 inline constexpr auto EmitReferenceU32 = EmitVoid;
 inline constexpr auto EmitControlNop   = EmitVoid;
-inline constexpr auto EmitWaitcnt      = EmitVoid;
 inline constexpr auto EmitSendmsg      = EmitVoid;
 inline constexpr auto EmitTtraceData   = EmitVoid;
 inline constexpr auto EmitInstPrefetch = EmitVoid;
 void                  EmitBarrier(EmitterState& state);
+void                  EmitStoreCompletion(EmitterState& state);
 void                  EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
+uint32_t              EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
 uint32_t              EmitUndefU1(EmitterState& state, const IR::Inst& inst);
 inline constexpr auto EmitUndefU8  = EmitUndefU1;
@@ -230,10 +245,12 @@ void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& ins
 uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitPermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -312,15 +329,21 @@ inline constexpr auto EmitImageSampleRaw       = EmitImage;
 inline constexpr auto EmitImageGatherRaw       = EmitImage;
 inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage;
 inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
+inline constexpr auto EmitImageAtomicSwap64    = EmitImage;
 inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicIAdd64    = EmitImage;
 inline constexpr auto EmitImageAtomicSMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMin64    = EmitImage;
 inline constexpr auto EmitImageAtomicSMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax64    = EmitImage;
 inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
+inline constexpr auto EmitImageAtomicAnd64     = EmitImage;
 inline constexpr auto EmitImageAtomicOr32      = EmitImage;
+inline constexpr auto EmitImageAtomicOr64      = EmitImage;
 inline constexpr auto EmitImageAtomicXor32     = EmitImage;
+inline constexpr auto EmitImageAtomicXor64     = EmitImage;
 inline constexpr auto EmitImageAtomicFMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicFMax32    = EmitImage;
 void                  EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -6,7 +6,6 @@
 #include "graphics/shader/recompiler/BufferFormat.h"
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
 #include <algorithm>
@@ -58,37 +57,41 @@ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
 
 const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 
-struct SpirvRequirements {
-	bool bvh                          = false;
-	bool subgroup_ballot              = false;
-	bool subgroup_shuffle             = false;
-	bool subgroup_local_invocation_id = false;
-	bool compute_derivatives          = false;
-	bool image_gather_extended        = false;
-	bool function_lds                 = false;
-	bool function_scratch             = false;
-	bool pixel_valid_mask             = false;
-	bool buffer_int64_atomics         = false;
-	bool shared_int64_atomics         = false;
-	bool coherent_buffers             = false;
-	bool float64                      = false;
+struct BufferDefinition {
+	uint32_t variable             = 0;
+	uint32_t type                 = 0;
+	uint32_t element_type         = 0;
+	uint32_t pointer_type         = 0;
+	uint32_t element_pointer_type = 0;
 };
 
-SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
+struct ImageDefinition {
+	uint32_t variable     = 0;
+	uint32_t type         = 0;
+	uint32_t pointer_type = 0;
+	uint32_t sampled_type = 0;
+};
 
 struct EmitterState {
-	EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
+	EmitterState(IR::Program& program_, ShaderStageInputInfo input_info_)
 	    : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
-	      program(program_), input_info(input_info_),
-	      requirements(AnalyzeProgramRequirements(program_)) {}
+	      program(program_), input_info(input_info_) {
+		if (ShaderWorkgroupInput(program.stage, input_info) != nullptr) {
+			lds_storage_class = spv::StorageClassWorkgroup;
+		}
+		if (program.info.uses_lds && program.stage == ShaderType::Compute &&
+		    input_info.compute != nullptr && input_info.compute->lds_storage) {
+			lds_storage_class = spv::StorageClassStorageBuffer;
+		}
+	}
 
 	Builder                                          builder;
-	const IR::Program&                               program;
+	IR::Program&                                     program;
 	ShaderStageInputInfo                             input_info;
 	uint32_t                                        void_type = 0;
 	uint32_t                                        bool_type = 0;
 	uint32_t                                        u32_type = 0;
-	uint32_t                                        native_u64_type = 0;
+	uint32_t                                        u64_type = 0;
 	uint32_t                                        i32_type = 0;
 	uint32_t                                        f32_type = 0;
 	uint32_t                                        f64_type = 0;
@@ -103,11 +106,9 @@ struct EmitterState {
 	uint32_t                                         tess_inner_variable = 0;
 	uint32_t                                         tess_patch_base     = 0;
 
-	const SpirvRequirements                          requirements;
 	uint32_t                                         lane_count              = 1;
 	uint32_t                                         lane_half               = 0;
-	uint32_t                                         storage_buffer_variable = 0;
-	uint32_t                                         storage_buffer_u64_variable = 0;
+	std::array<BufferDefinition, 4>                  storage_buffers {};
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
 	uint32_t                                         bda_pagetable_variable  = 0;
 	uint32_t                                         fault_buffer_variable   = 0;
@@ -118,11 +119,15 @@ struct EmitterState {
 	uint32_t                                         push_constant_variable  = 0;
 	uint32_t                                         shader_data_storage_variable = 0;
 	uint32_t                                         flattened_srt_variable  = 0;
+	spv::StorageClass                                lds_storage_class = spv::StorageClassFunction;
+	uint32_t                                         lds_base_dwords         = 0;
 	uint32_t                                         lds_variable            = 0;
 	uint32_t                                         lds_u64_variable        = 0;
 	std::array<uint32_t, 2>                          scratch_variable {};
-	std::array<uint32_t, IR::ImageBindingCount>      image_variables {};
+	std::array<ImageDefinition, IR::ImageBindingCount> images {};
 	uint32_t                   sampler_variable                      = 0;
+	uint32_t                   sampler_type                          = 0;
+	uint32_t                   sampler_pointer_type                  = 0;
 	uint32_t                   main_func                             = 0;
 	uint32_t                   mesh_guest_func                       = 0;
 	uint32_t                   mesh_allocation                       = 0;
@@ -146,7 +151,6 @@ struct EmitterState {
 	std::vector<InputBinding>  inputs;
 	std::vector<OutputBinding> outputs;
 	std::vector<uint32_t>      interface_variables;
-	std::unordered_map<const IR::Block*, uint32_t> labels;
 };
 
 uint32_t TypeVoid(EmitterState& state);
@@ -154,7 +158,6 @@ uint32_t TypeBool(EmitterState& state);
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
 uint32_t TypeU64(EmitterState& state);
-uint32_t TypeScalarU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
 uint32_t TypeI32Pair(EmitterState& state);
@@ -167,10 +170,7 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
 uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
 uint32_t TypeFunction(EmitterState& state);
-uint32_t TypeStorageBufferPointer(EmitterState& state);
-uint32_t TypeStorageBufferElementPointer(EmitterState& state);
-uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
-uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
+uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32);
 uint32_t TypePhysicalU32Pointer(EmitterState& state);
 uint32_t TypePushConstantElementPointer(EmitterState& state);
 uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
@@ -208,6 +208,12 @@ inline uint32_t Unary(EmitterState& state, spv::Op opcode, uint32_t type, uint32
 	return result;
 }
 
+inline uint32_t PackU64(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto pair = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), pair, low, high);
+	return Unary(state, spv::OpBitcast, TypeU64(state), pair);
+}
+
 inline uint32_t Binary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t lhs,
                        uint32_t rhs) {
 	const auto result = state.builder.AllocateId();
@@ -237,12 +243,10 @@ struct ValueEmitContext {
 	const IR::Inst*       ImageAddress(IR::Value value);
 	const IR::MemoryInfo& Memory(const IR::Inst& inst) const;
 	const IR::ExportInfo& Export(const IR::Inst& inst) const;
-	uint32_t              Label(const IR::Block* block) const;
 	[[noreturn]] void     Fail(const char* reason) const;
 	[[noreturn]] void     Fail(const IR::Inst& inst, const char* reason) const;
 
 	EmitterState&                                                      state;
-	std::unordered_map<const IR::Inst*, uint32_t>                      definitions;
 	const std::unordered_map<const IR::Inst*, uint32_t>*               dispatcher_spills = nullptr;
 	std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
 	uint32_t                                                           scratch_u32_variable = 0;
@@ -298,9 +302,6 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
                                                IR::DescriptorBindingKind kind, uint32_t resource,
                                                const char* reason);
 
-uint32_t ResourceForDescriptor(const EmitterState& state, IR::DescriptorBindingKind kind,
-                               uint32_t resource);
-
 uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
                                   uint32_t variable_id, uint32_t array_index,
                                   IR::DescriptorBindingKind kind, uint32_t resource,
@@ -315,14 +316,17 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image);
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension);
 
-uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0,
+                                uint32_t array_index = 0);
 
-uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0,
+                             uint32_t array_index = 0);
 
-uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler);
+uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t array_index = 0);
 
 uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id,
-                          uint32_t mip = 0);
+                          uint32_t mip = 0, uint32_t array_index = 0,
+                          bool sampler_dynamic = false);
 
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
                            uint32_t texel);
@@ -348,7 +352,8 @@ uint32_t DefineInterfaceVariable(EmitterState& state, uint32_t type, spv::Storag
 void     DefineModule(EmitterState& state);
 void     DefineTessellationInterfaces(EmitterState& state);
 void     DefineTessellationExecutionModes(EmitterState& state);
-void     DefineMeshOutputs(EmitterState& state);
+void     DefineMeshOutputs(EmitterState& state, uint32_t clip_distance_count,
+                           uint32_t cull_distance_count);
 void     EmitMeshEntryPoint(EmitterState& state);
 void     EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0);
@@ -402,20 +407,18 @@ struct MemoryResourceAccess {
 	IR::ResourceKind      kind             = IR::ResourceKind::None;
 	uint32_t              object_pointer   = 0;
 	uint32_t              length           = 0;
-	uint32_t              index_offset     = 0;
 	uint32_t              byte_offset      = 0;
-	bool                  add_index_offset = false;
+	uint32_t              element_bits     = 32;
 	spv::MemoryAccessMask memory_access    = spv::MemoryAccessMaskNone;
 };
 
+uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
+                                   uint32_t search_iterations, uint32_t default_resource);
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
 MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState&         state,
                                                         const IR::MemoryInfo& mem,
-                                                        uint32_t variable, uint32_t pointer_type);
-
-uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
-                                uint32_t raw_index);
+                                                        BufferDefinition& buffer);
 
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
                                    uint32_t index);
@@ -448,6 +451,9 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind);
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value);
 
+uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
+                                      bool max_value);
+
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
 
 inline constexpr auto EmitSelectValueU32 =
@@ -466,10 +472,6 @@ uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_
 
 uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32);
 
-uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
-
-uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
-
 inline constexpr auto EmitBitcastF32ToU32 = EmitNative<spv::OpBitcast, IR::Type::U32, uint32_t>;
 
 inline constexpr auto EmitBitcastU32ToF32 = EmitNative<spv::OpBitcast, IR::Type::F32, uint32_t>;
@@ -492,6 +494,7 @@ uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
                                uint32_t class_match);
 
 uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
+uint32_t EmitClassMaskF16(EmitterState& state, uint32_t bits, uint32_t mask);
 
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
@@ -512,22 +515,20 @@ void EmitProgram(EmitterState& state);
 void DefineGetBdaPointer(EmitterState& state);
 void DefineBvhIntersect(EmitterState& state);
 uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
-uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
-uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
-auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_count,
+auto EmitIndexSwitch(EmitterState& state, uint32_t index, uint32_t count,
                         uint32_t result_type, Fn&& emit) {
 	constexpr bool has_result = !std::is_void_v<std::invoke_result_t<Fn, uint32_t>>;
-	EXIT_IF(mip_count == 0u);
+	EXIT_IF(count == 0u);
 	const auto merge_label = state.builder.AllocateId();
-	std::vector<uint32_t> labels(mip_count);
-	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
-	for (uint32_t mip = 0; mip < mip_count; mip++) {
-		labels[mip] = state.builder.AllocateId();
-		words.push_back(mip);
-		words.push_back(labels[mip]);
+	std::vector<uint32_t> labels(count);
+	std::vector<uint32_t> words {spv::OpSwitch, index, merge_label};
+	for (uint32_t item = 0; item < count; item++) {
+		labels[item] = state.builder.AllocateId();
+		words.push_back(item);
+		words.push_back(labels[item]);
 	}
 	if constexpr (has_result) {
 		words[2] = labels.front();
@@ -538,13 +539,13 @@ auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_coun
 	if constexpr (has_result) {
 		phi = {spv::OpPhi, result_type, state.builder.AllocateId()};
 	}
-	for (uint32_t mip = 0; mip < mip_count; mip++) {
-		EmitLabel(state, labels[mip]);
+	for (uint32_t item = 0; item < count; item++) {
+		EmitLabel(state, labels[item]);
 		if constexpr (has_result) {
-			phi.push_back(emit(mip));
+			phi.push_back(emit(item));
 			phi.push_back(state.current_label);
 		} else {
-			emit(mip);
+			emit(item);
 		}
 		state.builder.AddFunction(spv::OpBranch, merge_label);
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -31,6 +31,61 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
 	return ret;
 }
 
+uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
+                                   uint32_t search_iterations, uint32_t default_resource) {
+	const auto LoadMapping = [&](uint32_t index) {
+		const auto pointer = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+		                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+		                          index);
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
+		return value;
+	};
+	const auto mapping = ConstantU32(state, mapping_offset);
+	const auto count = LoadMapping(mapping);
+	if (search_iterations == 0u) {
+		const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
+		                          ConstantU32(state, 2u));
+		const auto valid = Binary(state, spv::OpULessThan, TypeBool(state), index, count);
+		const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u));
+		const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe,
+		                          ConstantU32(state, mapping_offset + 1u));
+		return Select(state, TypeU32(state), valid, LoadMapping(entry),
+		              ConstantU32(state, default_resource));
+	}
+	const auto entry_at = [&](uint32_t index) {
+		return Binary(state, spv::OpIAdd, TypeU32(state), mapping,
+		              Binary(state, spv::OpIAdd, TypeU32(state),
+		                     Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
+		                            ConstantU32(state, 1u)), ConstantU32(state, 1u)));
+	};
+	auto low = ConstantU32(state, 0u);
+	auto high = count;
+	for (uint32_t iteration = 0; iteration < search_iterations; ++iteration) {
+		const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
+		const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+		                        Binary(state, spv::OpIAdd, TypeU32(state), low, high),
+		                        ConstantU32(state, 1u));
+		const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
+		const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
+		                         LoadMapping(entry_at(probe)), key);
+		low = Select(state, TypeU32(state),
+		             Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
+		             Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
+		high = Select(state, TypeU32(state),
+		              Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
+		                     Unary(state, spv::OpLogicalNot, TypeBool(state), less)), mid, high);
+	}
+	const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
+	const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
+	const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
+	                          Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
+	const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
+	                                         ConstantU32(state, 1u)));
+	return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
+}
+
 uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
@@ -48,13 +103,14 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
 }
 
 void EmitMemoryOffsets(EmitterState& state) {
-	for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
+	const auto count = state.program.bindings.descriptor_counts[static_cast<size_t>(IR::DescriptorBindingKind::Buffers)];
+	for (uint32_t i = 0; i < count; i += 4u) {
 		const auto word =
 		    EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
-		const auto shift             = ConstantU32(state, (i % 4u) * 8u);
-		state.memory_byte_offsets[i] = EmitBinaryU32(
-		    state, spv::OpBitwiseAnd, EmitBinaryU32(state, spv::OpShiftRightLogical, word, shift),
-		    ConstantU32(state, 0xffu));
+		for (uint32_t lane = 0; lane < std::min(4u, count - i); ++lane) {
+			state.memory_byte_offsets[i + lane] = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
+			    state, word, ConstantU32(state, lane * 8u), ConstantU32(state, 8u));
+		}
 	}
 }
 
@@ -67,8 +123,8 @@ void EnsureLdsStorage(EmitterState& state) {
 	if (state.lds_variable != 0) {
 		return;
 	}
-	if (ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
-		EXIT("function LDS was not prepared before SPIR-V function emission\n");
+	if (state.lds_storage_class != spv::StorageClassWorkgroup) {
+		EXIT("LDS backing was not prepared before SPIR-V function emission\n");
 	}
 	const auto define = [&](uint32_t type, uint32_t bytes) {
 		const auto array = state.builder.DecoratedType(
@@ -82,9 +138,9 @@ void EnsureLdsStorage(EmitterState& state) {
 		state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
 		return variable;
 	};
-	if (state.requirements.shared_int64_atomics) {
+	if (state.program.info.shared_int64_atomics) {
 		state.lds_variable = define(TypeU32(state), 4u);
-		state.lds_u64_variable = define(TypeScalarU64(state), 8u);
+		state.lds_u64_variable = define(TypeU64(state), 8u);
 		state.builder.AddName(state.lds_u64_variable, "lds_qwords");
 	} else {
 		state.lds_variable = state.builder.DefineGlobalVariable(
@@ -96,20 +152,22 @@ void EnsureLdsStorage(EmitterState& state) {
 
 MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
                                                          const IR::MemoryInfo& mem,
-                                                         uint32_t variable,
-                                                         uint32_t pointer_type) {
-	if (variable == 0) {
+                                                         BufferDefinition& buffer) {
+	if (buffer.variable == 0) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
 		                             "storage buffer descriptor array was not emitted");
 	}
+	if (buffer.pointer_type == 0) {
+		buffer.pointer_type = TypePointer(state, spv::StorageClassStorageBuffer, buffer.type);
+	}
 	const auto array_index =
-	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
+	    state.program.info.buffers.at(mem.resource).descriptor_index;
 	MemoryResourceAccess access {
 	    .kind = mem.kind,
 	    .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone};
 	access.object_pointer = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpAccessChain, pointer_type, access.object_pointer, variable,
-	                          ConstantU32(state, array_index));
+	state.builder.AddFunction(spv::OpAccessChain, buffer.pointer_type, access.object_pointer,
+	                          buffer.variable, ConstantU32(state, array_index));
 	access.byte_offset = state.memory_byte_offsets[array_index];
 	access.length      = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
@@ -149,24 +207,20 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 			EXIT("physical address memory must use the BDA emitter\n");
 		case IR::ResourceKind::ScalarBuffer:
 		case IR::ResourceKind::Buffer: {
-			access = PrepareStorageBufferResourceAccess(
-			    state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
-			access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
-			                                    ConstantU32(state, 2u));
-			access.add_index_offset = true;
+			const auto bits = mem.kind == IR::ResourceKind::Buffer
+			                      ? IR::StorageBufferElementBits(state.program, mem) : 32u;
+			auto* buffer = &state.storage_buffers[2];
+			if (bits == 8u) {
+				buffer = &state.storage_buffers[0];
+			} else if (bits == 16u) {
+				buffer = &state.storage_buffers[1];
+			}
+			access = PrepareStorageBufferResourceAccess(state, mem, *buffer);
+			access.element_bits = bits;
 			return access;
 		}
 		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
 	}
-	access.length = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
-	                          access.object_pointer, 0);
-	return access;
-}
-
-uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
-                                uint32_t raw_index) {
-	return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
 }
 
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
@@ -179,13 +233,15 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
 uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
                                   uint32_t index) {
 	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
+		const auto storage_class = access.kind == IR::ResourceKind::Scratch
+		                               ? spv::StorageClassFunction : state.lds_storage_class;
+		if (storage_class == spv::StorageClassStorageBuffer) {
+			return EmitStorageBufferElementPointer(
+			    state, access, EmitAddU32(state, state.lds_base_dwords, index),
+			    TypeStorageBufferElementPointer(state, 32));
+		}
 		const auto pointer = state.builder.AllocateId();
-		const auto storage_class =
-		    access.kind == IR::ResourceKind::Scratch ? spv::StorageClassFunction
-		    : ShaderWorkgroupInput(state.program.stage, state.input_info) != nullptr
-		        ? spv::StorageClassWorkgroup
-		        : spv::StorageClassFunction;
-		if (access.kind == IR::ResourceKind::Lds && state.requirements.shared_int64_atomics) {
+		if (access.kind == IR::ResourceKind::Lds && state.program.info.shared_int64_atomics) {
 			state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class),
 			                          pointer, access.object_pointer, ConstantU32(state, 0), index);
 			return pointer;
@@ -195,7 +251,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
 		return pointer;
 	}
 	return EmitStorageBufferElementPointer(state, access, index,
-	                                       TypeStorageBufferElementPointer(state));
+	                                       TypeStorageBufferElementPointer(state, access.element_bits));
 }
 
 uint32_t EmitStorageBufferElementPointer(EmitterState& state,
@@ -311,10 +367,12 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicCompareSwap32:
 		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
 		case IR::ValueOpcode::ImageAtomicSwap32:
+		case IR::ValueOpcode::ImageAtomicSwap64:
 		case IR::ValueOpcode::BufferAtomicSwap32:
 		case IR::ValueOpcode::BufferAtomicSwap64:
 		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
 		case IR::ValueOpcode::ImageAtomicIAdd32:
+		case IR::ValueOpcode::ImageAtomicIAdd64:
 		case IR::ValueOpcode::BufferAtomicIAdd32:
 		case IR::ValueOpcode::SharedAtomicIAdd64:
 		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
@@ -324,6 +382,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::BufferAtomicSMin32:
 		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
 		case IR::ValueOpcode::ImageAtomicUMin32:
+		case IR::ValueOpcode::ImageAtomicUMin64:
 		case IR::ValueOpcode::BufferAtomicUMin32:
 		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
 		case IR::ValueOpcode::ImageAtomicSMax32:
@@ -334,15 +393,18 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::BufferAtomicUMax32:
 		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
 		case IR::ValueOpcode::ImageAtomicAnd32:
+		case IR::ValueOpcode::ImageAtomicAnd64:
 		case IR::ValueOpcode::BufferAtomicAnd32:
 		case IR::ValueOpcode::BufferAtomicAnd64:
 		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
 		case IR::ValueOpcode::ImageAtomicOr32:
+		case IR::ValueOpcode::ImageAtomicOr64:
 		case IR::ValueOpcode::BufferAtomicOr32:
 		case IR::ValueOpcode::BufferAtomicOr64:
 		case IR::ValueOpcode::SharedAtomicOr64:
 		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
 		case IR::ValueOpcode::ImageAtomicXor32:
+		case IR::ValueOpcode::ImageAtomicXor64:
 		case IR::ValueOpcode::BufferAtomicXor32:
 		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
 		default: return spv::OpNop;
@@ -354,7 +416,7 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
 	const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
 	const auto old    = ctx.state.builder.AllocateId();
 	const bool wide   = inst.GetType() == IR::Type::U64;
-	const auto type   = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state);
+	const auto type   = wide ? TypeU64(ctx.state) : TypeU32(ctx.state);
 	if (opcode == spv::OpAtomicCompareExchange) {
 		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
 		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
@@ -363,20 +425,22 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
 		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
 		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
 	} else {
-		auto value = ctx.Arg(inst, inst.NumArgs() - 2);
-		if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value);
+		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
 		ctx.state.builder.AddFunction(opcode, type, old, pointer,
 		                              ConstantU32(ctx.state, scope),
 		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
 	}
-	return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old;
+	return old;
 }
 
 void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
 	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto memory = [&] {
 		switch (kind) {
-			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Lds:
+				return state.lds_storage_class == spv::StorageClassStorageBuffer
+				           ? spv::MemorySemanticsUniformMemoryMask
+				           : spv::MemorySemanticsWorkgroupMemoryMask;
 			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
 			default: return spv::MemorySemanticsUniformMemoryMask;
 		}
@@ -385,6 +449,20 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
 	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
 }
 
+namespace {
+
+uint32_t FloatOrderKey(EmitterState& state, uint32_t bits) {
+	const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
+	                                             EmitAndConstant(state, bits, 0x80000000u), 0u);
+	const auto negative_key = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
+	const auto positive_key =
+	    EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
+	return EmitSelectValueU32(state, negative, negative_key, positive_key);
+}
+
+} // namespace
+
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value) {
 	struct OrderedBits {
@@ -394,15 +472,7 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	};
 	const auto classify = [&](uint32_t bits) {
 		const auto cls = EmitClassifyF32Bits(state, bits);
-		const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
-		                                             EmitAndConstant(state, bits, 0x80000000u), 0u);
-		const auto negative_key = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
-		const auto positive_key =
-		    EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
-		return OrderedBits {
-		    cls.nan, cls.zero,
-		    EmitSelectValueU32(state, negative, negative_key, positive_key)};
+		return OrderedBits {cls.nan, cls.zero, FloatOrderKey(state, bits)};
 	};
 	const auto source_class = classify(source);
 	const auto old_class    = classify(old);
@@ -417,6 +487,45 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	    old);
 }
 
+uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
+                                      bool max_value) {
+	struct OrderedBits {
+		uint32_t nan;
+		uint32_t signaling_nan;
+		uint32_t key;
+	};
+	const auto classify = [&](uint32_t bits) {
+		const auto abs = EmitAndConstant(state, bits, 0x7fffffffu);
+		const auto nan = EmitCompareU32Constant(state, spv::OpUGreaterThan, abs, 0x7f800000u);
+		const auto signaling_nan = EmitLogicalAndBool(
+		    state, nan, EmitCompareU32Constant(state, spv::OpIEqual,
+		                                      EmitAndConstant(state, bits, 0x00400000u), 0u));
+		const auto small = EmitCompareU32Constant(state, spv::OpULessThan, abs, 0x00800000u);
+		const auto sign = EmitAndConstant(state, bits, 0x80000000u);
+		const auto positive_denorm = EmitLogicalAndBool(
+		    state, EmitCompareU32Constant(state, spv::OpIEqual, sign, 0u),
+		    EmitCompareU32Constant(state, spv::OpINotEqual, abs, 0u));
+		// Negative denorms tie with -0; positive denorms tie only with each other.
+		const auto small_bits = EmitSelectValueU32(
+		    state, positive_denorm, ConstantU32(state, 1u), sign);
+		const auto key_bits = EmitSelectValueU32(state, small, small_bits, bits);
+		return OrderedBits {nan, signaling_nan, FloatOrderKey(state, key_bits)};
+	};
+	const auto lhs = classify(old);
+	const auto rhs = classify(source);
+	const auto choose_source = state.builder.AllocateId();
+	state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThanEqual,
+	                          TypeBool(state), choose_source, rhs.key, lhs.key);
+	// Keep original bits: ordering denorms does not flush the selected operand.
+	auto result = EmitSelectValueU32(state, choose_source, source, old);
+	result = EmitSelectValueU32(state, lhs.nan, source, result);
+	result = EmitSelectValueU32(state, rhs.nan, old, result);
+	result = EmitSelectValueU32(state, rhs.signaling_nan,
+	                            EmitOrU32(state, source, ConstantU32(state, 0x00400000u)), result);
+	return EmitSelectValueU32(state, lhs.signaling_nan,
+	                          EmitOrU32(state, old, ConstantU32(state, 0x00400000u)), result);
+}
+
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
 	if ((control & 0xc000u) == 0xc000u) {
 		const uint32_t mask         = control & 0x1fu;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -25,43 +25,58 @@ uint32_t MeshLoad(EmitterState& state, uint32_t variable, spv::StorageClass stor
 }
 
 uint32_t MeshOutputType(EmitterState& state, IR::StageOutputKind kind) {
-	return kind == IR::StageOutputKind::Layer ? TypeU32(state) : TypeF32Vector(state, 4);
+	switch (kind) {
+		case IR::StageOutputKind::Layer: return TypeU32(state);
+		case IR::StageOutputKind::ClipDistance:
+		case IR::StageOutputKind::CullDistance: return TypeF32(state);
+		case IR::StageOutputKind::Position:
+		case IR::StageOutputKind::Parameter: return TypeF32Vector(state, 4);
+		default: EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(kind));
+	}
 }
 
 } // namespace
 
-void DefineMeshOutputs(EmitterState& state) {
+void DefineMeshOutputs(EmitterState& state, uint32_t clip_distance_count,
+                       uint32_t cull_distance_count) {
 	const auto& mesh = state.input_info.vertex->mesh;
 	for (auto& output: state.outputs) {
-		if (output.kind != IR::StageOutputKind::Position &&
-		    output.kind != IR::StageOutputKind::Parameter &&
-		    output.kind != IR::StageOutputKind::Layer) {
-			EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(output.kind));
+		const auto type = MeshOutputType(state, output.kind);
+		const bool clip = output.kind == IR::StageOutputKind::ClipDistance;
+		const bool cull = output.kind == IR::StageOutputKind::CullDistance;
+		auto& variable = clip ? state.clip_distance_variable
+		                 : cull ? state.cull_distance_variable : output.variable_id;
+		if (variable == 0) {
+			const auto element_type = clip || cull
+			                              ? state.builder.Type(spv::OpTypeArray, type,
+			                                                   ConstantU32(state, clip ? clip_distance_count
+			                                                                           : cull_distance_count))
+			                              : type;
+			variable = MeshArray(state, spv::StorageClassOutput, element_type,
+			                     output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives
+			                                                             : mesh.max_vertices);
+			state.interface_variables.push_back(variable);
+			state.builder.AddName(variable, output.debug_name.c_str());
+			if (output.kind == IR::StageOutputKind::Parameter) {
+				state.builder.AddAnnotation(spv::OpDecorate, variable,
+				                            spv::DecorationLocation, output.location);
+			} else {
+				const auto builtin = clip ? spv::BuiltInClipDistance
+				                     : cull ? spv::BuiltInCullDistance
+				                     : output.kind == IR::StageOutputKind::Layer ? spv::BuiltInLayer
+				                                                                 : spv::BuiltInPosition;
+				state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn, builtin);
+			}
+			if (output.kind == IR::StageOutputKind::Layer) {
+				state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationPerPrimitiveEXT);
+			}
 		}
-		const auto type    = MeshOutputType(state, output.kind);
-		output.variable_id = MeshArray(
-		    state, spv::StorageClassOutput, type,
-		    output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
+		output.variable_id = variable;
 		// Only Layer is read by another invocation, through the primitive's provoking vertex.
 		const bool shared = output.kind == IR::StageOutputKind::Layer;
 		output.mesh_data_variable =
 		    MeshArray(state, shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, type,
 		              shared ? mesh.max_vertices : state.lane_count);
-		state.interface_variables.push_back(output.variable_id);
-		state.builder.AddName(output.variable_id, output.debug_name.c_str());
-		if (output.kind == IR::StageOutputKind::Parameter) {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
-			                            spv::DecorationLocation, output.location);
-		} else {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id, spv::DecorationBuiltIn,
-			                            output.kind == IR::StageOutputKind::Layer
-			                                ? spv::BuiltInLayer
-			                                : spv::BuiltInPosition);
-		}
-		if (output.kind == IR::StageOutputKind::Layer) {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
-			                            spv::DecorationPerPrimitiveEXT); // PerPrimitiveEXT
-		}
 	}
 	state.mesh_allocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2);
 	state.mesh_primitive_data =
@@ -148,8 +163,16 @@ void EmitMeshEntryPoint(EmitterState& state) {
 				const auto value =
 				    MeshLoad(state, output.mesh_data_variable, spv::StorageClassPrivate, type,
 				             ConstantU32(state, half));
-				const auto pointer =
-				    MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				uint32_t pointer;
+				if (output.kind == IR::StageOutputKind::ClipDistance ||
+				    output.kind == IR::StageOutputKind::CullDistance) {
+					pointer = state.builder.AllocateId();
+					state.builder.AddFunction(
+					    spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, type), pointer,
+					    output.variable_id, index, ConstantU32(state, output.index));
+				} else {
+					pointer = MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				}
 				state.builder.AddFunction(spv::OpStore, pointer, value);
 			}
 		});

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -35,14 +35,11 @@ uint32_t TypeU32(EmitterState& state) {
 }
 
 uint32_t TypeU64(EmitterState& state) {
-	return TypeU32Vector(state, 2);
-}
-
-uint32_t TypeScalarU64(EmitterState& state) {
-	if (state.native_u64_type == 0) {
-		state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
+	if (state.u64_type == 0) {
+		state.builder.RequireCapability(spv::CapabilityInt64);
+		state.u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
 	}
-	return state.native_u64_type;
+	return state.u64_type;
 }
 
 uint32_t TypeU32Pair(EmitterState& state) {
@@ -91,7 +88,8 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
 }
 
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
-	EXIT_IF(components < 2u || components > 4u);
+	EXIT_IF(components < 1u || components > 4u);
+	if (components == 1u) return TypeU32(state);
 	return components == 2u ? TypeU32Pair(state) : TypeU32Vector(state, components);
 }
 
@@ -122,46 +120,39 @@ uint32_t TypeFunction(EmitterState& state) {
 	return state.function_type;
 }
 
-uint32_t StorageRuntimeArrayType(EmitterState& state) {
-	return state.builder.DecoratedType(
-	    spv::OpTypeRuntimeArray,
-	    {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}}, TypeU32(state));
+uint32_t StorageBufferType(EmitterState& state, BufferDefinition& buffer, uint32_t bits = 32) {
+	if (buffer.type == 0) {
+		if (bits == 32u) {
+			buffer.element_type = TypeU32(state);
+		} else if (bits == 64u) {
+			buffer.element_type = TypeU64(state);
+		} else {
+			buffer.element_type = state.builder.Type(spv::OpTypeInt, bits, 0);
+		}
+		const auto array = state.builder.DecoratedType(
+		    spv::OpTypeRuntimeArray,
+		    {{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}}, buffer.element_type);
+		buffer.type = state.builder.DecoratedType(
+		    spv::OpTypeStruct, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
+		                        {spv::OpDecorate, {spv::DecorationBlock}}}, array);
+	}
+	return buffer.type;
 }
 
-uint32_t StorageBufferType(EmitterState& state) {
-	return state.builder.DecoratedType(spv::OpTypeStruct,
-	                                   {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
-	                                    {spv::OpDecorate, {spv::DecorationBlock}}},
-	                                   StorageRuntimeArrayType(state));
-}
-
-uint32_t TypeStorageBufferPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state));
-}
-
-uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state));
-}
-
-uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
-	return state.builder.DecoratedType(
-	    spv::OpTypeRuntimeArray,
-	    {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}}, TypeScalarU64(state));
-}
-
-uint32_t StorageBufferU64Type(EmitterState& state) {
-	return state.builder.DecoratedType(spv::OpTypeStruct,
-	                                   {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
-	                                    {spv::OpDecorate, {spv::DecorationBlock}}},
-	                                   StorageU64RuntimeArrayType(state));
-}
-
-uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state));
-}
-
-uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
+uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) {
+	auto* buffer = &state.storage_buffers[2];
+	if (bits == 8u) {
+		buffer = &state.storage_buffers[0];
+	} else if (bits == 16u) {
+		buffer = &state.storage_buffers[1];
+	} else if (bits == 64u) {
+		buffer = &state.storage_buffers[3];
+	}
+	if (buffer->element_pointer_type == 0) {
+		buffer->element_pointer_type =
+		    TypePointer(state, spv::StorageClassStorageBuffer, buffer->element_type);
+	}
+	return buffer->element_pointer_type;
 }
 
 uint32_t TypePhysicalU32Pointer(EmitterState& state) {
@@ -216,6 +207,43 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
 }
 
 void DefineDescriptors(EmitterState& state) {
+	using Kind = IR::DescriptorBindingKind;
+	auto& info = state.program.info;
+	auto& bindings = state.program.bindings;
+	auto& counts = bindings.descriptor_counts;
+	auto& buffer_count = counts[static_cast<size_t>(Kind::Buffers)];
+	for (uint32_t index = 0; index < info.buffers.size(); ++index) {
+		info.buffers[index].descriptor_index = (info.live_buffers & (uint64_t{1} << index)) != 0
+		                                          ? buffer_count++ : UINT32_MAX;
+	}
+	std::array<const IR::ImageResource*, IR::ImageBindingCount> image_types {};
+	for (auto& image: info.images) {
+		const auto kind = IR::DescriptorBindingForImage(image);
+		if (!kind.has_value() || image.mip_count == 0u ||
+		    (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
+			EXIT("shader image has an invalid descriptor type or mip count");
+		}
+		auto& count = counts[static_cast<size_t>(*kind)];
+		image.descriptor_index = count;
+		if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = &image;
+		count += image.mip_count;
+	}
+	counts[static_cast<size_t>(Kind::Samplers)] = static_cast<uint32_t>(info.samplers.size());
+	counts[static_cast<size_t>(Kind::Gds)] = info.uses_gds;
+	counts[static_cast<size_t>(Kind::SharedMemory)] =
+	    info.uses_lds && state.lds_storage_class == spv::StorageClassStorageBuffer;
+	counts[static_cast<size_t>(Kind::BdaPagetable)] = info.uses_dma;
+	counts[static_cast<size_t>(Kind::FaultBuffer)] = info.uses_dma;
+	counts[static_cast<size_t>(Kind::FlattenedSrt)] = info.uses_flattened_srt;
+	bindings.memory_offset_dword = static_cast<uint32_t>(info.user_data_registers.size());
+	if (info.uses_dispatch_threads) {
+		bindings.dispatch_thread_dword = bindings.memory_offset_dword;
+		bindings.memory_offset_dword += 3u;
+	}
+	bindings.push_data_start_dword =
+	    IR::PushData::StartFor(bindings.push_data_start_dword, bindings.ShaderDataDwords());
+	counts[static_cast<size_t>(Kind::ShaderData)] =
+	    bindings.ShaderDataDwords() != 0 && !bindings.UsesPushData();
 	if (state.program.bindings.UsesPushData() || state.program.stage == ShaderType::Mesh) {
 		const auto type              = PushConstantBlockType(state);
 		state.push_constant_variable = state.builder.DefineGlobalVariable(
@@ -223,7 +251,10 @@ void DefineDescriptors(EmitterState& state) {
 		state.builder.AddName(type, "BufferResource");
 		state.builder.AddName(state.push_constant_variable, "vsharp");
 	}
-	for (const auto& binding: state.program.bindings.descriptors) {
+	for (uint32_t index = 0; index < counts.size(); ++index) {
+		if (counts[index] == 0) continue;
+		bindings.descriptor_mask |= uint64_t{1} << index;
+		const auto kind = static_cast<Kind>(index);
 		const auto Define = [&](uint32_t type, const char* name,
 		                        spv::StorageClass storage = spv::StorageClassStorageBuffer) {
 			const auto variable =
@@ -231,65 +262,83 @@ void DefineDescriptors(EmitterState& state) {
 			state.builder.AddName(variable, name);
 			state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0);
 			state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding,
-			                            IR::NativeBinding(state.program.stage, binding.kind));
+			                            IR::NativeBinding(state.program.stage, kind));
 			return variable;
 		};
 		const auto ArrayType = [&](uint32_t type) {
 			return state.builder.Type(
 			    spv::OpTypeArray, type,
-			    ConstantU32(state, static_cast<uint32_t>(binding.resources.size())));
+			    ConstantU32(state, counts[index]));
 		};
-		switch (binding.kind) {
-			case IR::DescriptorBindingKind::Buffers:
-				state.storage_buffer_variable =
-				    Define(ArrayType(StorageBufferType(state)), "buffers");
-				if (state.requirements.buffer_int64_atomics) {
-					state.storage_buffer_u64_variable =
-					    Define(ArrayType(StorageBufferU64Type(state)), "buffers_u64");
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
-					                            spv::DecorationAliased);
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
-					                            spv::DecorationAliased);
-				}
-				if (state.requirements.coherent_buffers) {
+		switch (kind) {
+			case IR::DescriptorBindingKind::Buffers: {
+				const std::array used {info.buffer_u8, info.buffer_u16, true, info.buffer_int64_atomics};
+				constexpr std::array names {"buffers_u8", "buffers_u16", "buffers", "buffers_u64"};
+				for (uint32_t alias = 0; alias < used.size(); ++alias) {
+					if (!used[alias]) continue;
+					auto& buffer = state.storage_buffers[alias];
+					const auto type = StorageBufferType(state, buffer, 8u << alias);
+					buffer.variable = Define(ArrayType(type), names[alias]);
+					if (info.buffer_u8 || info.buffer_u16 || info.buffer_int64_atomics) {
+						state.builder.AddAnnotation(spv::OpDecorate, buffer.variable, spv::DecorationAliased);
+					}
 					// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
 					// must participate in visibility for cache-bypassing polling loads.
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
-					                            spv::DecorationCoherent);
-					if (state.storage_buffer_u64_variable != 0) {
-						state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
-						                            spv::DecorationCoherent);
+					if (info.coherent_buffers) {
+						state.builder.AddAnnotation(spv::OpDecorate, buffer.variable, spv::DecorationCoherent);
 					}
 				}
 				break;
+			}
 			case IR::DescriptorBindingKind::BdaPagetable:
-				state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");
+				state.bda_pagetable_variable =
+				    Define(StorageBufferType(state, state.storage_buffers[3], 64), "bda_pagetable");
 				break;
 			case IR::DescriptorBindingKind::FaultBuffer:
-				state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer");
+				state.fault_buffer_variable =
+				    Define(StorageBufferType(state, state.storage_buffers[2]), "fault_buffer");
 				break;
 			case IR::DescriptorBindingKind::ShaderData:
 				state.shader_data_storage_variable =
-				    Define(StorageBufferType(state), "shader_data");
+				    Define(StorageBufferType(state, state.storage_buffers[2]), "shader_data");
 				break;
 			case IR::DescriptorBindingKind::FlattenedSrt:
-				state.flattened_srt_variable = Define(StorageBufferType(state), "flattened_srt");
+				state.flattened_srt_variable =
+				    Define(StorageBufferType(state, state.storage_buffers[2]), "flattened_srt");
 				break;
 			case IR::DescriptorBindingKind::Samplers:
-				state.sampler_variable = Define(ArrayType(state.builder.Type(spv::OpTypeSampler)),
-				                                "samplers", spv::StorageClassUniformConstant);
+				state.sampler_type = state.builder.Type(spv::OpTypeSampler);
+				state.sampler_pointer_type = TypePointer(
+				    state, spv::StorageClassUniformConstant, state.sampler_type);
+				state.sampler_variable = Define(ArrayType(state.sampler_type), "samplers",
+				                                spv::StorageClassUniformConstant);
 				break;
 			case IR::DescriptorBindingKind::Gds:
-				state.gds_variable = Define(StorageBufferType(state), "gds");
+				state.gds_variable = Define(StorageBufferType(state, state.storage_buffers[2]), "gds");
+				break;
+			case IR::DescriptorBindingKind::SharedMemory:
+				state.lds_variable = Define(StorageBufferType(state, state.storage_buffers[2]), "lds_dwords");
+				state.builder.AddAnnotation(spv::OpDecorate, state.lds_variable, spv::DecorationCoherent);
+				if (state.program.info.shared_int64_atomics) {
+					state.lds_u64_variable =
+					    Define(StorageBufferType(state, state.storage_buffers[3], 64), "lds_qwords");
+					state.builder.AddAnnotation(spv::OpDecorate, state.lds_u64_variable, spv::DecorationCoherent);
+					for (const auto variable: {state.lds_variable, state.lds_u64_variable}) {
+						state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
+					}
+				}
 				break;
 			default: {
-				EXIT_IF(IR::ImageBindingResourceClass(binding.kind) ==
+				EXIT_IF(IR::ImageBindingResourceClass(kind) ==
 				        IR::ImageResourceClass::None);
-				const auto& image = state.program.info.images.at(binding.resources.front());
-				const auto  name  = "image_" + std::to_string(static_cast<uint32_t>(binding.kind));
-				state.image_variables[IR::ImageBindingIndex(binding.kind)] =
-				    Define(ArrayType(ImageType(state, image)), name.c_str(),
-				           spv::StorageClassUniformConstant);
+				const auto& image = *image_types[IR::ImageBindingIndex(kind)];
+				const auto  name  = "image_" + std::to_string(static_cast<uint32_t>(kind));
+				auto& definition = state.images[IR::ImageBindingIndex(kind)];
+				definition.type = ImageType(state, image);
+				definition.variable = Define(ArrayType(definition.type), name.c_str(),
+				                             spv::StorageClassUniformConstant);
+				definition.pointer_type = TypePointer(
+				    state, spv::StorageClassUniformConstant, definition.type);
 				if (image.dimension == ImageDimension::Dim1D ||
 				    image.dimension == ImageDimension::Dim1DArray) {
 					state.builder.RequireCapability(image.resource_class ==
@@ -327,17 +376,17 @@ uint32_t ConstantBool(EmitterState& state, bool value) {
 }
 
 uint32_t ConstantU64(EmitterState& state, uint64_t value) {
-	return state.builder.Constant(spv::OpConstantComposite, TypeU64(state),
-	                              ConstantU32(state, static_cast<uint32_t>(value)),
-	                              ConstantU32(state, static_cast<uint32_t>(value >> 32u)));
+	return state.builder.Constant(spv::OpConstant, TypeU64(state), static_cast<uint32_t>(value),
+	                              static_cast<uint32_t>(value >> 32u));
 }
 
 uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
-	EXIT_IF(components < 2u || components > 4u);
-	const auto            zero = ConstantU32(state, 0);
-	std::vector<uint32_t> values(components, zero);
+	EXIT_IF(components < 1u || components > 4u);
+	const auto zero = ConstantU32(state, 0);
+	if (components == 1u) return zero;
+	const std::array values {zero, zero, zero, zero};
 	return state.builder.Constant(spv::OpConstantComposite, TypeU32Composite(state, components),
-	                              values);
+	                              std::span(values).first(components));
 }
 
 uint32_t GlslStd450(EmitterState& state) {
@@ -414,6 +463,7 @@ uint32_t BuiltInForInput(IR::StageInputKind kind) {
 		case IR::StageInputKind::BaryCoordSmooth: return spv::BuiltInBaryCoordKHR;
 		case IR::StageInputKind::BaryCoordNoPerspective: return spv::BuiltInBaryCoordNoPerspKHR;
 		case IR::StageInputKind::WorkgroupId: return spv::BuiltInWorkgroupId;
+		case IR::StageInputKind::NumWorkgroups: return spv::BuiltInNumWorkgroups;
 		case IR::StageInputKind::LocalInvocationId: return spv::BuiltInLocalInvocationId;
 		case IR::StageInputKind::LocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
 		case IR::StageInputKind::GlobalInvocationId: return spv::BuiltInGlobalInvocationId;
@@ -426,15 +476,19 @@ void DefineInputs(EmitterState& state) {
 	for (const auto& input: state.program.info.inputs) {
 		state.inputs.push_back({input});
 	}
+	const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components,
+	                             const char* name) {
+		if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) {
+			    return input.kind == kind;
+		    })) {
+			state.inputs.push_back({{kind, 0, components, name}});
+		}
+	};
+	if (state.program.info.uses_lds && state.lds_storage_class == spv::StorageClassStorageBuffer) {
+		add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID");
+		add_builtin(IR::StageInputKind::NumWorkgroups, 3, "gl_NumWorkGroups");
+	}
 	if (state.lane_count == 2) {
-		const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components,
-		                             const char* name) {
-			if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) {
-				    return input.kind == kind;
-			    })) {
-				state.inputs.push_back({{kind, 0, components, name}});
-			}
-		};
 		add_builtin(IR::StageInputKind::LocalInvocationIndex, 1, "gl_LocalInvocationIndex");
 		if (std::ranges::any_of(state.inputs, [](const InputBinding& input) {
 			    return input.kind == IR::StageInputKind::GlobalInvocationId;
@@ -465,6 +519,7 @@ void DefineInputs(EmitterState& state) {
 			case IR::StageInputKind::Layer:
 			case IR::StageInputKind::SampleId: type = TypeI32(state); break;
 			case IR::StageInputKind::WorkgroupId:
+			case IR::StageInputKind::NumWorkgroups:
 			case IR::StageInputKind::LocalInvocationId:
 			case IR::StageInputKind::GlobalInvocationId: type = TypeU32Vector(state, 3); break;
 			case IR::StageInputKind::FragCoord: type = TypeF32Vector(state, 4); break;
@@ -516,7 +571,7 @@ void DefineInputs(EmitterState& state) {
 			                            builtin);
 		}
 	}
-	if (state.requirements.subgroup_local_invocation_id) {
+	if (state.program.info.subgroup_local_invocation_id) {
 		const auto variable = DefineInterfaceVariable(state, TypeU32(state), spv::StorageClassInput,
 		                                              "gl_SubgroupInvocationID");
 		state.subgroup_local_invocation_id_variable = variable;
@@ -541,7 +596,7 @@ void DefineOutputs(EmitterState& state) {
 		}
 	}
 	if (state.program.stage == ShaderType::Mesh) {
-		DefineMeshOutputs(state);
+		DefineMeshOutputs(state, clip_distance_count, cull_distance_count);
 		return;
 	}
 	if (state.program.stage == ShaderType::Vertex && clip_distance_count + cull_distance_count < 8u &&
@@ -639,10 +694,10 @@ void DefineModule(EmitterState& state) {
 	DefineOutputs(state);
 	DefineTessellationInterfaces(state);
 	DefineDescriptors(state);
-	if (state.requirements.function_lds) {
+	if (state.program.info.function_lds) {
 		state.lds_variable = state.builder.AllocateId();
 	}
-	if (state.requirements.function_scratch) {
+	if (state.program.info.function_scratch) {
 		for (uint32_t half = 0; half < state.lane_count; half++) {
 			state.scratch_variable[half] = state.builder.AllocateId();
 		}
@@ -667,16 +722,24 @@ void DefineModule(EmitterState& state) {
 
 	state.builder.RequireCapability(spv::CapabilityShader);
 	state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
+	if (state.program.info.buffer_u8) {
+		state.builder.RequireExtension("SPV_KHR_8bit_storage");
+		state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess);
+	}
+	if (state.program.info.buffer_u16) {
+		state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
+	}
 	if (state.program.info.uses_dma) {
 		state.builder.RequireCapability(spv::CapabilityInt64);
 		state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
 		state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
 	}
-	if (state.requirements.buffer_int64_atomics || state.requirements.shared_int64_atomics) {
+	if (state.program.info.buffer_int64_atomics || state.program.info.shared_int64_atomics) {
 		state.builder.RequireCapability(spv::CapabilityInt64);
 		state.builder.RequireCapability(spv::CapabilityInt64Atomics);
 	}
-	if (state.requirements.shared_int64_atomics) {
+	if (state.program.info.shared_int64_atomics &&
+	    state.lds_storage_class == spv::StorageClassWorkgroup) {
 		state.builder.RequireVersion(0x00010400u);
 		state.builder.RequireExtension("SPV_KHR_workgroup_memory_explicit_layout");
 		state.builder.RequireCapability(spv::CapabilityWorkgroupMemoryExplicitLayoutKHR);
@@ -698,20 +761,21 @@ void DefineModule(EmitterState& state) {
 	if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) {
 		state.builder.RequireCapability(spv::CapabilitySampleRateShading);
 	}
-	if (state.requirements.image_gather_extended) {
+	if (state.program.info.image_gather_extended) {
 		state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
 	}
-	if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
-	    state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
+	if (state.lane_count == 2 || state.program.info.subgroup_barrier ||
+	    state.program.info.subgroup_ballot || state.program.info.subgroup_shuffle ||
+	    state.program.info.subgroup_local_invocation_id) {
 		state.builder.RequireCapability(spv::CapabilityGroupNonUniform);
 	}
-	if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
+	if (state.lane_count == 2 || state.program.info.subgroup_ballot) {
 		state.builder.RequireCapability(spv::CapabilityGroupNonUniformBallot);
 	}
-	if (state.requirements.subgroup_shuffle) {
+	if (state.program.info.subgroup_shuffle) {
 		state.builder.RequireCapability(spv::CapabilityGroupNonUniformShuffle);
 	}
-	if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) {
+	if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) {
 		state.builder.RequireCapability(spv::CapabilityComputeDerivativeGroupQuadsKHR);
 		state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives");
 	}
@@ -734,22 +798,20 @@ void DefineModule(EmitterState& state) {
 	// contract prevents host compilers from treating synthesized IEEE values as finite.
 	state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
 	                               32u);
-	if (state.requirements.float64) {
+	if (state.program.info.float64) {
 		EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
 		                     state.input_info.compute->float_mode != 0xc0);
-		// MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals.
+		// Use native rounding for MODE=0xc0, consistent with ordinary FP32 arithmetic.
 		state.builder.RequireCapability(spv::CapabilityFloat64);
-		state.builder.RequireCapability(spv::CapabilityRoundingModeRTE);
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
 		                               64u);
 		// FP64 denormal preservation is temporarily disabled.
 		// state.builder.RequireCapability(spv::CapabilityDenormPreserve);
 		// state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u);
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeRoundingModeRTE, 32u);
 	}
 	if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) {
-		uint32_t    local_x = state.requirements.compute_derivatives ? 2u : 1u;
-		uint32_t    local_y = state.requirements.compute_derivatives ? 2u : 1u;
+		uint32_t    local_x = state.program.info.compute_derivatives ? 2u : 1u;
+		uint32_t    local_y = state.program.info.compute_derivatives ? 2u : 1u;
 		uint32_t    local_z = 1u;
 		local_x             = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
 		local_y             = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
@@ -757,7 +819,7 @@ void DefineModule(EmitterState& state) {
 		if (state.lane_count == 2) {
 			local_x = ((local_x * local_y * local_z + 63u) / 64u) * 32u;
 			local_y = local_z = 1u;
-			if (state.requirements.compute_derivatives) {
+			if (state.program.info.compute_derivatives) {
 				local_y = local_x / 2u;
 				local_x = 2u;
 			}
@@ -776,14 +838,14 @@ void DefineModule(EmitterState& state) {
 			state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeEarlyFragmentTests);
 		}
 	}
-	if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) {
+	if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) {
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDerivativeGroupQuadsKHR);
 	}
 	state.builder.AddName(state.main_func, "main");
-	if (state.requirements.function_lds) {
+	if (state.program.info.function_lds) {
 		state.builder.AddName(state.lds_variable, "lds_dwords");
 	}
-	if (state.requirements.function_scratch) {
+	if (state.program.info.function_scratch) {
 		for (uint32_t half = 0; half < state.lane_count; half++) {
 			state.builder.AddName(state.scratch_variable[half], "scratch_dwords");
 		}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -83,63 +83,50 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
 	}
 }
 
-const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
-	const auto found = std::ranges::find_if(
-	    program.block_info, [&](const IR::BlockInfo& info) { return info.id == id; });
-	if (found == program.block_info.end()) {
-		return nullptr;
-	}
-	return program.blocks[static_cast<size_t>(found - program.block_info.begin())];
-}
-
 void EmitReturn(ValueEmitContext& ctx) {
 	EmitKillIfPixelValidMaskInactive(ctx.state);
 	ctx.state.builder.AddFunction(spv::OpReturn);
 }
 
-void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
-                              const IR::BlockInfo& info) {
-	const auto& program = ctx.state.program;
-	const auto& term       = info.terminator;
+void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block) {
+	const auto& term       = block->terminator;
 	const auto  emit_merge = [&]() {
 		if (term.loop_header) {
-			const auto* merge = TargetBlock(program, term.merge_block);
-			const auto* cont  = TargetBlock(program, term.continue_block);
+			const auto* merge = term.merge_block;
+			const auto* cont  = term.continue_block;
 			if (merge != nullptr && cont != nullptr) {
-				ctx.state.builder.AddFunction(spv::OpLoopMerge, ctx.Label(merge), ctx.Label(cont),
-				                              spv::LoopControlMaskNone);
+				ctx.state.builder.AddFunction(spv::OpLoopMerge, merge->Definition(),
+				                              cont->Definition(), spv::LoopControlMaskNone);
 			}
 		} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
-		           term.merge_block != UINT32_MAX) {
-			if (const auto* merge = TargetBlock(program, term.merge_block); merge != nullptr) {
-				ctx.state.builder.AddFunction(spv::OpSelectionMerge, ctx.Label(merge),
-				                              spv::SelectionControlMaskNone);
-			}
+		           term.merge_block != nullptr) {
+			ctx.state.builder.AddFunction(spv::OpSelectionMerge, term.merge_block->Definition(),
+			                              spv::SelectionControlMaskNone);
 		}
 	};
 
 	switch (term.kind) {
 		case CFG::TerminatorKind::Branch: {
-			const auto* target = TargetBlock(program, term.true_block);
+			const auto* target = term.true_block;
 			if (target == nullptr) {
 				EmitReturn(ctx);
 				return;
 			}
 			emit_merge();
-			ctx.state.builder.AddFunction(spv::OpBranch, ctx.Label(target));
+			ctx.state.builder.AddFunction(spv::OpBranch, target->Definition());
 			return;
 		}
 		case CFG::TerminatorKind::ConditionalBranch: {
-			const auto* true_block  = TargetBlock(program, term.true_block);
-			const auto* false_block = TargetBlock(program, term.false_block);
-			if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) {
+			const auto* true_block  = term.true_block;
+			const auto* false_block = term.false_block;
+			if (true_block == nullptr || false_block == nullptr || block->condition.IsEmpty()) {
 				EmitReturn(ctx);
 				return;
 			}
-			const auto condition = ctx.Def(info.condition);
+			const auto condition = ctx.Def(block->condition);
 			emit_merge();
 			ctx.state.builder.AddFunction(spv::OpBranchConditional, condition,
-			                              ctx.Label(true_block), ctx.Label(false_block));
+			                              true_block->Definition(), false_block->Definition());
 			return;
 		}
 		default: EmitReturn(ctx); return;
@@ -147,8 +134,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 }
 
 void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
-                          const IR::Block* from, uint32_t target) {
-	const auto* block = TargetBlock(ctx.state.program, target);
+                          const IR::Block* from, const IR::Block* block) {
 	if (block != nullptr) {
 		StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
 		if (ctx.other_half != nullptr) {
@@ -158,41 +144,37 @@ void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState&
 }
 
 uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
-                              const IR::Block* block, const IR::BlockInfo& info) {
-	const auto& term = info.terminator;
+                              const IR::Block* block) {
+	const auto& term = block->terminator;
 	switch (term.kind) {
 		case CFG::TerminatorKind::Branch:
 			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
-			return ConstantU32(ctx.state, term.true_block);
+			return ConstantU32(ctx.state, term.true_block->id);
 		case CFG::TerminatorKind::ConditionalBranch: {
 			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
 			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction(
-			    spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition),
-			    ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block));
+			ctx.state.builder.AddFunction(spv::OpSelect, TypeU32(ctx.state), selected,
+			                              ctx.Def(block->condition),
+			                              ConstantU32(ctx.state, term.true_block->id),
+			                              ConstantU32(ctx.state, term.false_block->id));
 			return selected;
 		}
 		case CFG::TerminatorKind::IndirectBranch: {
-			for (const auto target: term.indirect_targets) {
+			for (const auto* target: block->ImmSuccessors()) {
 				EmitDispatcherTarget(ctx, dispatcher, block, target);
 			}
 			uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
-			if (!info.indirect_target.IsEmpty()) {
-				const auto  selector = ctx.Def(info.indirect_target);
-				const auto& values   = term.indirect_selector_code != UINT32_MAX
-				                           ? term.indirect_selector_values
-				                           : term.indirect_target_pcs;
-				const auto& targets  = term.indirect_selector_code != UINT32_MAX
-				                           ? term.indirect_selector_targets
-				                           : term.indirect_targets;
-				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
+			if (!block->indirect_target.IsEmpty()) {
+				const auto selector = ctx.Def(block->indirect_target);
+				for (const auto& branch: term.cases) {
 					const auto match = ctx.state.builder.AllocateId();
 					const auto next  = ctx.state.builder.AllocateId();
 					ctx.state.builder.AddFunction(spv::OpIEqual, TypeBool(ctx.state), match,
-					                              selector, ConstantU32(ctx.state, values[index]));
+					                              selector, ConstantU32(ctx.state, branch.value));
 					ctx.state.builder.AddFunction(spv::OpSelect, TypeU32(ctx.state), next, match,
-					                              ConstantU32(ctx.state, targets[index]), selected);
+					                              ConstantU32(ctx.state, branch.target->id),
+					                              selected);
 					selected = next;
 				}
 			}
@@ -267,7 +249,7 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
 		}
 		for (size_t index = 0; index < inst.NumArgs(); index++) {
 			const auto* predecessor = inst.PhiBlock(index);
-			if (predecessor == nullptr || !ctx.state.labels.contains(predecessor)) {
+			if (predecessor == nullptr || predecessor->Definition() == 0) {
 				ctx.Fail(inst, "has a predecessor outside the structured function");
 			}
 		}
@@ -301,7 +283,7 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt
 template <typename EmitInstruction>
 void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
 	ctx.state.current_block = block;
-	EmitLabel(ctx.state, ctx.Label(block));
+	EmitLabel(ctx.state, block->Definition());
 	bool emitted_non_phi = false;
 	for (const auto& inst: *block) {
 		if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
@@ -315,6 +297,7 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
 			auto& lane          = half == 0 ? ctx : *ctx.other_half;
 			ctx.state.lane_half = half;
 			if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
+			                  inst.GetOpcode() != IR::ValueOpcode::StoreCompletion &&
 			                  inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
 				emit_instruction(lane, inst);
 			}
@@ -341,14 +324,14 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
 void EmitStructuredFunction(ValueEmitContext& ctx) {
 	const auto& program = ctx.state.program;
 	StructuredFunctionState structured;
-	ctx.state.builder.AddFunction(spv::OpBranch, ctx.Label(program.blocks.front()));
+	ctx.state.builder.AddFunction(spv::OpBranch, program.blocks.front()->Definition());
 	for (size_t index = 0; index < program.blocks.size(); index++) {
 		const auto* block = program.blocks[index];
 		EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
 			EmitStructuredInstruction(lane, structured, inst);
 		});
 		structured.block_exit_labels.emplace(block, ctx.state.current_label);
-		EmitStructuredTerminator(ctx, block, program.block_info[index]);
+		EmitStructuredTerminator(ctx, block);
 	}
 	PatchStructuredPhis(ctx, structured);
 }
@@ -356,12 +339,11 @@ void EmitStructuredFunction(ValueEmitContext& ctx) {
 void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
 	auto&       state = ctx.state;
 	const auto* entry = state.program.blocks.front();
-	state.builder.AddFunction(spv::OpBranch, ctx.Label(entry));
+	state.builder.AddFunction(spv::OpBranch, entry->Definition());
 	EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
 		EmitDispatcherInstruction(lane, dispatcher, inst);
 	});
-	const auto initial_pc =
-	    EmitDispatcherNextPc(ctx, dispatcher, entry, state.program.block_info.front());
+	const auto initial_pc     = EmitDispatcherNextPc(ctx, dispatcher, entry);
 	const auto initial_parent = state.current_label;
 	state.builder.AddFunction(spv::OpBranch, dispatcher.header_label);
 
@@ -383,8 +365,8 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
 	                          spv::SelectionControlMaskNone);
 	std::vector<uint32_t> words {spv::OpSwitch, pc, dispatcher.after_switch_label};
 	for (size_t index = 1; index < state.program.blocks.size(); index++) {
-		words.push_back(state.program.block_info[index].id);
-		words.push_back(ctx.Label(state.program.blocks[index]));
+		words.push_back(state.program.blocks[index]->id);
+		words.push_back(state.program.blocks[index]->Definition());
 	}
 	state.builder.AddFunction(words);
 	std::vector<uint32_t> next_pc_words {spv::OpPhi, TypeU32(state), next_pc,
@@ -395,8 +377,7 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
 		          [&](ValueEmitContext& lane, const IR::Inst& inst) {
 			          EmitDispatcherInstruction(lane, dispatcher, inst);
 		          });
-		const auto selected = EmitDispatcherNextPc(ctx, dispatcher, state.program.blocks[index],
-		                                           state.program.block_info[index]);
+		const auto selected = EmitDispatcherNextPc(ctx, dispatcher, state.program.blocks[index]);
 		next_pc_words.push_back(selected);
 		next_pc_words.push_back(state.current_label);
 		state.builder.AddFunction(spv::OpBranch, dispatcher.after_switch_label);
@@ -550,23 +531,23 @@ uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t
 }
 
 uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
-	if (const auto found = definitions.find(&inst); found != definitions.end()) {
-		return found->second;
+	if (const auto id = inst.Definition(half); id != 0) {
+		return id;
 	}
 	const auto id = state.builder.AllocateId();
-	definitions.emplace(&inst, id);
+	inst.SetDefinition(half, id);
 	return id;
 }
 
 uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
-	if (const auto found = definitions.find(&inst); found != definitions.end()) {
-		if (found->second != value) {
+	if (const auto id = inst.Definition(half); id != 0) {
+		if (id != value) {
 			state.builder.AddFunction(spv::OpCopyObject, TypeId(state, inst.GetType()),
-			                          found->second, value);
+			                          id, value);
 		}
-		return found->second;
+		return id;
 	}
-	definitions.emplace(&inst, value);
+	inst.SetDefinition(half, value);
 	return value;
 }
 
@@ -594,10 +575,6 @@ const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const {
 	return state.program.export_info.at(inst.Flags<IR::ExportFlags>().index);
 }
 
-uint32_t ValueEmitContext::Label(const IR::Block* block) const {
-	return state.labels.at(block);
-}
-
 [[noreturn]] void ValueEmitContext::Fail(const char* reason) const {
 	EXIT("SPIR-V emission failed: hash=0x%016" PRIx64 " stage=%u reason=%s\n",
 	     state.program.shader_hash, static_cast<unsigned>(state.program.stage), reason);
@@ -621,13 +598,16 @@ void EmitProgram(EmitterState& state) {
 		high.half       = 1;
 	}
 	std::optional<DispatcherFunctionState> dispatcher;
-	if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
+	if (state.program.stage == ShaderType::Pixel && state.program.info.pixel_valid_mask) {
 		state.pixel_valid_mask_variable = state.builder.AllocateId();
 		state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
 	}
 	for (const auto* block: program.blocks) {
-		const auto label = state.builder.AllocateId();
-		state.labels.emplace(block, label);
+		// Re-emission may use a different host subgroup width and ID allocation order.
+		for (const auto& inst: *block) {
+			inst.ResetDefinitions();
+		}
+		block->SetDefinition(state.builder.AllocateId());
 	}
 	if (state.program.dispatcher_fallback) {
 		auto& dispatch = dispatcher.emplace();
@@ -668,8 +648,8 @@ void EmitProgram(EmitterState& state) {
 			}
 		}
 		for (size_t index = 0; index < program.blocks.size(); index++) {
-			mark_cross_block(program.block_info[index].condition, program.blocks[index]);
-			mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
+			mark_cross_block(program.blocks[index]->condition, program.blocks[index]);
+			mark_cross_block(program.blocks[index]->indirect_target, program.blocks[index]);
 		}
 		dispatch.header_label       = state.builder.AllocateId();
 		dispatch.select_label       = state.builder.AllocateId();
@@ -686,30 +666,23 @@ void EmitProgram(EmitterState& state) {
 	}
 	DefineGetBdaPointer(state);
 	DefineBvhIntersect(state);
-	for (const auto* block: program.blocks) {
-		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
-			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
-		    })) {
-			ctx.scratch_u32_variable = state.builder.AllocateId();
-			if (state.lane_count == 2) {
-				high.scratch_u32_variable = state.builder.AllocateId();
-			}
-			break;
+	if (program.info.uses_swizzle) {
+		ctx.scratch_u32_variable = state.builder.AllocateId();
+		if (state.lane_count == 2) {
+			high.scratch_u32_variable = state.builder.AllocateId();
 		}
 	}
 	state.builder.AddFunction(spv::OpFunction, TypeVoid(state),
 	                          state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
 	                          spv::FunctionControlMaskNone, TypeFunction(state));
 	EmitLabel(state, state.entry_label);
-	if (state.requirements.function_lds) {
+	if (state.program.info.function_lds) {
 		state.builder.AddFunction(
 		    spv::OpVariable,
 		    TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)),
 		    state.lds_variable, spv::StorageClassFunction);
 	}
-	if (state.requirements.function_scratch) {
+	if (state.program.info.function_scratch) {
 		for (uint32_t half = 0; half < state.lane_count; half++) {
 			state.builder.AddFunction(
 			    spv::OpVariable,
@@ -751,6 +724,17 @@ void EmitProgram(EmitterState& state) {
 		state.builder.AddFunction(spv::OpStore, state.pixel_valid_mask_variable,
 		                          ConstantU32(state, 1));
 	}
+	if (state.lds_storage_class == spv::StorageClassStorageBuffer && state.lds_variable != 0) {
+		const auto group_x = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, 0);
+		const auto group_y = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, 1);
+		const auto group_z = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, 2);
+		const auto count_x = EmitInputComponentU32(state, IR::StageInputKind::NumWorkgroups, 0);
+		const auto count_y = EmitInputComponentU32(state, IR::StageInputKind::NumWorkgroups, 1);
+		const auto row = EmitAddU32(state, group_y, EmitBinaryU32(state, spv::OpIMul, group_z, count_y));
+		const auto index = EmitAddU32(state, group_x, EmitBinaryU32(state, spv::OpIMul, row, count_x));
+		state.lds_base_dwords = EmitBinaryU32(state, spv::OpIMul, index,
+		                                     ConstantU32(state, LdsDwordCount(state)));
+	}
 	EmitMemoryOffsets(state);
 	if (program.blocks.empty()) {
 		EmitReturn(ctx);

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -8,7 +8,6 @@
 #include <deque>
 #include <list>
 #include <map>
-#include <set>
 #include <span>
 
 namespace Libs::Graphics::ShaderRecompiler::CFG {
@@ -568,9 +567,8 @@ bool ResolveSetpcTargets(const Decoder::Program& program, uint32_t setpc_index,
 	return false;
 }
 
-bool IsValidTarget(uint32_t target, const std::set<uint32_t>& instruction_pcs, uint32_t first_pc,
-                   uint32_t end_pc) {
-	return target == end_pc || (target >= first_pc && instruction_pcs.contains(target));
+bool IsValidTarget(uint32_t target, std::span<const Instruction> instructions, uint32_t end_pc) {
+	return target == end_pc || std::ranges::binary_search(instructions, target, {}, &Instruction::pc);
 }
 
 std::vector<uint32_t> AllBlockIds(uint32_t count) {
@@ -1455,9 +1453,7 @@ Graph BuildGraph(const Decoder::Program& program) {
 	const auto first_pc = program.instructions.front().pc;
 	const auto end_pc   = ProgramEndPc(program);
 
-	std::set<uint32_t> instruction_pcs;
 	for (const auto& inst: program.instructions) {
-		instruction_pcs.insert(inst.pc);
 		if (inst.opcode == Opcode::UNSUPPORTED) {
 			ExitBuildFailure(
 			    graph, FailureKind::UnsupportedInstruction, UINT32_MAX,
@@ -1466,23 +1462,22 @@ Graph BuildGraph(const Decoder::Program& program) {
 		}
 	}
 
-	std::set<uint32_t> labels;
-	labels.insert(first_pc);
-	labels.insert(end_pc);
+	std::vector<uint32_t> labels {first_pc, end_pc};
 
 	std::map<uint32_t, SetpcTargetInfo> setpc_targets;
 	for (uint32_t i = 0; i < program.instructions.size(); i++) {
 		const auto& inst    = program.instructions[i];
 		const auto  next_pc = InstructionEndPc(inst);
-		if (Decoder::IsDirectBranch(inst.opcode)) {
-			if (!IsValidTarget(inst.branch_target, instruction_pcs, first_pc, end_pc)) {
+		if (Decoder::IsDirectBranch(inst.opcode) ||
+		    (inst.opcode == Opcode::S_SWAPPC_B64 && inst.branch_target != UINT32_MAX)) {
+			if (!IsValidTarget(inst.branch_target, program.instructions, end_pc)) {
 				ExitBuildFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
 				                 fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}",
 				                             inst.pc, inst.branch_target));
 			}
-			labels.insert(inst.branch_target);
+			labels.push_back(inst.branch_target);
 			if (next_pc <= end_pc) {
-				labels.insert(next_pc);
+				labels.push_back(next_pc);
 			}
 		} else if (inst.opcode == Opcode::S_SETPC_B64) {
 			SetpcTargetInfo target_info;
@@ -1495,30 +1490,32 @@ Graph BuildGraph(const Decoder::Program& program) {
 			                            ? std::span<const uint32_t>(target_info.target_pcs)
 			                            : std::span<const uint32_t>(&target_info.target, 1);
 			for (const auto target: target_pcs) {
-				if (!IsValidTarget(target, instruction_pcs, first_pc, end_pc)) {
+				if (!IsValidTarget(target, program.instructions, end_pc)) {
 					ExitBuildFailure(
 					    graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
 					    fmt::format("S_SETPC_B64 at pc 0x{:08x} targets invalid pc 0x{:08x}",
 					                inst.pc, target));
 				}
-				labels.insert(target);
+				labels.push_back(target);
 			}
 			setpc_targets.emplace(inst.pc, std::move(target_info));
 			if (next_pc <= end_pc) {
-				labels.insert(next_pc);
+				labels.push_back(next_pc);
 			}
-		} else if (inst.opcode == Opcode::S_ENDPGM) {
-			labels.insert(next_pc);
+		} else if (inst.opcode == Opcode::S_ENDPGM || inst.opcode == Opcode::S_SWAPPC_B64) {
+			labels.push_back(next_pc);
 		}
 	}
 
-	std::vector<uint32_t> sorted_labels(labels.begin(), labels.end());
-	for (uint32_t i = 0; i < sorted_labels.size(); i++) {
-		const auto start = sorted_labels[i];
+	SortUnique(labels);
+	graph.blocks.reserve(labels.size());
+	for (uint32_t i = 0; i < labels.size(); i++) {
+		const auto start = labels[i];
 		if (start > end_pc) {
 			continue;
 		}
-		if (start != end_pc && !instruction_pcs.contains(start)) {
+		const auto begin = std::ranges::lower_bound(program.instructions, start, {}, &Instruction::pc);
+		if (start != end_pc && (begin == program.instructions.end() || begin->pc != start)) {
 			ExitBuildFailure(
 			    graph, FailureKind::InvalidLabel, UINT32_MAX,
 			    fmt::format("CFG label does not start on an instruction: 0x{:08x}", start));
@@ -1527,11 +1524,8 @@ Graph BuildGraph(const Decoder::Program& program) {
 		BasicBlock block;
 		block.id         = static_cast<uint32_t>(graph.blocks.size());
 		block.start_pc   = start;
-		block.end_pc     = i + 1u < sorted_labels.size() ? sorted_labels[i + 1u] : end_pc;
-		block.inst_begin = static_cast<uint32_t>(
-		    std::lower_bound(program.instructions.begin(), program.instructions.end(), start,
-		                     [](const Instruction& inst, uint32_t pc) { return inst.pc < pc; }) -
-		    program.instructions.begin());
+		block.end_pc     = i + 1u < labels.size() ? labels[i + 1u] : end_pc;
+		block.inst_begin = static_cast<uint32_t>(begin - program.instructions.begin());
 		block.inst_end = static_cast<uint32_t>(
 		    std::lower_bound(program.instructions.begin(), program.instructions.end(), block.end_pc,
 		                     [](const Instruction& inst, uint32_t pc) { return inst.pc < pc; }) -
@@ -1541,10 +1535,10 @@ Graph BuildGraph(const Decoder::Program& program) {
 
 	graph.entry_block = 0;
 
-	std::map<uint32_t, uint32_t> pc_to_block;
-	for (const auto& block: graph.blocks) {
-		pc_to_block.emplace(block.start_pc, block.id);
-	}
+	const auto block_at_pc = [&](uint32_t pc) {
+		const auto found = std::ranges::lower_bound(graph.blocks, pc, {}, &BasicBlock::start_pc);
+		return found != graph.blocks.end() && found->start_pc == pc ? found->id : UINT32_MAX;
+	};
 
 	for (auto& block: graph.blocks) {
 		block.terminator = {};
@@ -1557,6 +1551,9 @@ Graph BuildGraph(const Decoder::Program& program) {
 		const auto  next_pc = InstructionEndPc(last);
 		if (last.opcode == Opcode::S_ENDPGM) {
 			block.terminator.kind = TerminatorKind::Return;
+		} else if (last.opcode == Opcode::S_SWAPPC_B64 && last.branch_target != UINT32_MAX) {
+			block.terminator.kind = TerminatorKind::Branch;
+			block.terminator.true_block = block_at_pc(last.branch_target);
 		} else if (last.opcode == Opcode::S_SETPC_B64) {
 			const auto& target_info = setpc_targets.at(last.pc);
 			if (target_info.indirect) {
@@ -1566,7 +1563,7 @@ Graph BuildGraph(const Decoder::Program& program) {
 				block.terminator.indirect_selector_code = target_info.selector_code;
 				for (const auto target_pc: target_info.target_pcs) {
 					block.terminator.indirect_target_pcs.push_back(target_pc);
-					block.terminator.indirect_targets.push_back(pc_to_block.at(target_pc));
+					block.terminator.indirect_targets.push_back(block_at_pc(target_pc));
 				}
 				const auto selector_count = std::min(target_info.selector_values.size(),
 				                                     target_info.selector_target_pcs.size());
@@ -1574,35 +1571,35 @@ Graph BuildGraph(const Decoder::Program& program) {
 					block.terminator.indirect_selector_values.push_back(
 					    target_info.selector_values[i]);
 					block.terminator.indirect_selector_targets.push_back(
-					    pc_to_block.at(target_info.selector_target_pcs[i]));
+					    block_at_pc(target_info.selector_target_pcs[i]));
 				}
 			} else {
 				block.terminator.kind       = TerminatorKind::Branch;
 				block.terminator.condition  = BranchCondition::Always;
-				block.terminator.true_block = pc_to_block.at(target_info.target);
+				block.terminator.true_block = block_at_pc(target_info.target);
 			}
 		} else if (last.opcode == Opcode::S_BRANCH) {
 			block.terminator.kind       = TerminatorKind::Branch;
 			block.terminator.condition  = BranchCondition::Always;
-			block.terminator.true_block = pc_to_block.at(last.branch_target);
+			block.terminator.true_block = block_at_pc(last.branch_target);
 		} else if (Decoder::IsConditionalBranch(last.opcode)) {
 			block.terminator.kind       = TerminatorKind::ConditionalBranch;
 			block.terminator.condition  = ConditionForOpcode(last.opcode);
-			block.terminator.true_block = pc_to_block.at(last.branch_target);
-			const auto fallthrough      = pc_to_block.find(next_pc);
-			if (fallthrough == pc_to_block.end()) {
+			block.terminator.true_block = block_at_pc(last.branch_target);
+			const auto fallthrough      = block_at_pc(next_pc);
+			if (fallthrough == UINT32_MAX) {
 				ExitBuildFailure(
 				    graph, FailureKind::MissingFallthrough, block.id,
 				    fmt::format("conditional branch at pc 0x{:08x} has no fallthrough block",
 				                last.pc));
 			}
-			block.terminator.false_block = fallthrough->second;
+			block.terminator.false_block = fallthrough;
 		} else {
-			auto next = pc_to_block.find(block.end_pc);
-			if (next != pc_to_block.end() && block.end_pc != block.start_pc) {
+			const auto next = block_at_pc(block.end_pc);
+			if (next != UINT32_MAX && block.end_pc != block.start_pc) {
 				block.terminator.kind       = TerminatorKind::Branch;
 				block.terminator.condition  = BranchCondition::Always;
-				block.terminator.true_block = next->second;
+				block.terminator.true_block = next;
 			} else {
 				block.terminator.kind = TerminatorKind::Return;
 			}
@@ -1683,6 +1680,36 @@ Graph Structurize(const Graph& graph) {
 	return GotoStructurizer(graph).Run();
 }
 
+bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) {
+	uint32_t destination = UINT32_MAX;
+	// Scalar ALU pair widths are not all decoded. The typed producer must confirm this word.
+	return InstructionWritesScalarCode(inst, code) ||
+	       (ScalarOperandCode(inst.dst, destination) && code == destination + 1u);
+}
+
+uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
+                              uint32_t before, uint32_t code) {
+	EXIT_IF(before >= program.instructions.size());
+	const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) {
+		return block.inst_begin <= before && before < block.inst_end;
+	});
+	EXIT_IF(found == graph.blocks.end());
+	const auto* block = &*found;
+	for (size_t depth = 0; depth < graph.blocks.size(); ++depth) {
+		for (uint32_t i = before; i > block->inst_begin;) {
+			const auto& inst = program.instructions[--i];
+			EXIT_IF(inst.opcode == Opcode::S_SWAPPC_B64);
+			if (MayWriteScalarRegister(inst, code)) return i;
+		}
+		if (block->id == graph.entry_block) return UINT32_MAX;
+		EXIT_IF(block->predecessors.size() != 1u);
+		block = graph.FindBlock(block->predecessors[0]);
+		EXIT_IF(block == nullptr);
+		before = block->inst_end;
+	}
+	EXIT("scalar shader call source has cyclic reaching definitions");
+}
+
 std::string BranchConditionToString(BranchCondition condition) {
 	switch (condition) {
 		case BranchCondition::Always: return "always";

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -113,6 +113,9 @@ struct Graph {
 };
 
 Graph       BuildGraph(const Decoder::Program& program);
+bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code);
+uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
+                              uint32_t before, uint32_t code);
 // Returns structured control flow or failure diagnostics without changing the native graph.
 // On failure, failure_block is an original block ID or UINT32_MAX.
 Graph       Structurize(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -20,6 +20,12 @@ struct MimgGatherInfo {
 	uint32_t flags    = 0;
 };
 
+struct MimgAtomicInfo {
+	uint32_t encoding    = 0;
+	Opcode   decoded     = Opcode::UNSUPPORTED;
+	bool     supports_64 = false;
+};
+
 constexpr ImageDimension DecodeImageDimension(uint32_t dim) {
 	switch (dim) {
 		case 0u: return ImageDimension::Dim1D;
@@ -190,17 +196,17 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
     {0x61u, Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal},
 };
 
-constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
-    {0x0fu, Opcode::IMAGE_ATOMIC_SWAP},
+constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = {
+    {0x0fu, Opcode::IMAGE_ATOMIC_SWAP, true},
     {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP},
-    {0x11u, Opcode::IMAGE_ATOMIC_ADD},
+    {0x11u, Opcode::IMAGE_ATOMIC_ADD, true},
     {0x14u, Opcode::IMAGE_ATOMIC_SMIN},
-    {0x15u, Opcode::IMAGE_ATOMIC_UMIN},
+    {0x15u, Opcode::IMAGE_ATOMIC_UMIN, true},
     {0x16u, Opcode::IMAGE_ATOMIC_SMAX},
-    {0x17u, Opcode::IMAGE_ATOMIC_UMAX},
-    {0x18u, Opcode::IMAGE_ATOMIC_AND},
-    {0x19u, Opcode::IMAGE_ATOMIC_OR},
-    {0x1au, Opcode::IMAGE_ATOMIC_XOR},
+    {0x17u, Opcode::IMAGE_ATOMIC_UMAX, true},
+    {0x18u, Opcode::IMAGE_ATOMIC_AND, true},
+    {0x19u, Opcode::IMAGE_ATOMIC_OR, true},
+    {0x1au, Opcode::IMAGE_ATOMIC_XOR, true},
     {0x1eu, Opcode::IMAGE_ATOMIC_FMIN},
     {0x1fu, Opcode::IMAGE_ATOMIC_FMAX},
 };
@@ -210,7 +216,7 @@ constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCO
 constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST);
 
 Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                        const Detail::OpcodeMap* atomic) {
+                        const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return Opcode::IMAGE_SAMPLE;
 	}
@@ -228,7 +234,8 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
-		case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
+		case 0xe6u:
+		case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
@@ -245,7 +252,7 @@ uint32_t DecodeMimgSampleFlags(const MimgSampleInfo* sample, const MimgGatherInf
 
 uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
                                      const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                                     const Detail::OpcodeMap* atomic) {
+                                     const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return ImageSampleAddressComponents(sample->flags, dimension);
 	}
@@ -258,6 +265,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
 
 	switch (opcode) {
 		case 0xe6u: return 11u;
+		case 0xe7u: return 12u;
 		case 0x0eu: return 1u;
 		case 0x01u:
 		case 0x09u: return ImageCoordComponents(dimension) + 1u;
@@ -368,7 +376,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			SetUnsupported(inst, Family::MIMG, opcode, "MIMG image atomic has invalid DMASK");
 		} else if (inst.dmask == mask64) {
 			inst.data_bits = 64u;
-			if (inst.opcode != Opcode::IMAGE_ATOMIC_UMAX) {
+			if (!atomic->supports_64) {
 				SetUnsupported(inst, Family::MIMG, opcode,
 				               "MIMG 64-bit image atomic opcode is not implemented");
 			}
@@ -379,10 +387,10 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
-	if (opcode == 0xe6u &&
+	if (inst.opcode == Opcode::IMAGE_BVH_INTERSECT_RAY &&
 	    (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
 		SetUnsupported(inst, Family::MIMG, opcode,
-		               "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
+		               "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf");
 	}
 
 	DecodeVectorGpr(vdata, inst.dst);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -47,6 +47,8 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x1du, Opcode::BUFFER_STORE_DWORDX2, 2, 32},
     {0x1eu, Opcode::BUFFER_STORE_DWORDX4, 4, 32},
     {0x1fu, Opcode::BUFFER_STORE_DWORDX3, 3, 32},
+    {0x24u, Opcode::BUFFER_LOAD_SHORT_D16, 1, 16},
+    {0x25u, Opcode::BUFFER_LOAD_SHORT_D16_HI, 1, 16},
     {0x30u, Opcode::BUFFER_ATOMIC_SWAP, 1, 32},
     {0x31u, Opcode::BUFFER_ATOMIC_CMPSWAP, 1, 32},
     {0x32u, Opcode::BUFFER_ATOMIC_ADD, 1, 32},
@@ -63,6 +65,8 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
     {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
     {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
+    {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true},
+    {0x84u, Opcode::BUFFER_STORE_FORMAT_D16_X, 1, 16, false, false, true},
 };
 
 constexpr MemoryOpcodeInfo MTBUF_OPCODE_LIST[] = {
@@ -84,6 +88,7 @@ constexpr MemoryOpcodeInfo FLAT_OPCODE_LIST[] = {
     {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},     {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
     {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},   {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
     {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32}, {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
+    {0x24u, Opcode::FLAT_LOAD_SHORT_D16, 1, 16},
 };
 
 constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
@@ -92,7 +97,8 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x05u, Opcode::DS_MIN_I32, 1, 32},          {0x06u, Opcode::DS_MAX_I32, 1, 32},
     {0x07u, Opcode::DS_MIN_U32, 1, 32},          {0x08u, Opcode::DS_MAX_U32, 1, 32},
     {0x09u, Opcode::DS_AND_B32, 1, 32},          {0x0au, Opcode::DS_OR_B32, 1, 32},
-    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0du, Opcode::DS_WRITE_B32, 1, 32},
+    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0cu, Opcode::DS_MSKOR_B32, 1, 32},
+    {0x0du, Opcode::DS_WRITE_B32, 1, 32},
     {0x0eu, Opcode::DS_WRITE2_B32, 2, 32},       {0x0fu, Opcode::DS_WRITE2ST64_B32, 2, 32},
     {0x12u, Opcode::DS_MIN_F32, 1, 32},          {0x13u, Opcode::DS_MAX_F32, 1, 32},
     {0x1eu, Opcode::DS_WRITE_B8, 1, 8},          {0x1fu, Opcode::DS_WRITE_B16, 1, 16},
@@ -107,6 +113,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
+    {0x3fu, Opcode::DS_ORDERED_COUNT, 1, 32},
     {0x40u, Opcode::DS_ADD_U64, 2, 32},         {0x4au, Opcode::DS_OR_B64, 2, 32},
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
@@ -116,7 +123,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0xa6u, Opcode::DS_READ_U16_D16, 1, 16},
     {0xa7u, Opcode::DS_READ_U16_D16_HI, 1, 16},
     {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
-    {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
+    {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32},     {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
     {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
     {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
 };
@@ -192,6 +199,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 		case Opcode::DS_OR_RTN_B32:
 		case Opcode::DS_XOR_B32:
 		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_WRXCHG_RTN_B32: return true;
 		default: return false;
 	}
@@ -203,12 +211,15 @@ uint32_t DsSourceCount(Opcode opcode) {
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
 		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_MSKOR_B32: return 3u;
 		case Opcode::DS_MIN_F32:
-		case Opcode::DS_MAX_F32: return 3u;
+		case Opcode::DS_MAX_F32:
+		case Opcode::DS_PERMUTE_B32:
 		case Opcode::DS_BPERMUTE_B32: return 2u;
 		case Opcode::DS_READ_ADDTID_B32:
 		case Opcode::DS_CONSUME:
 		case Opcode::DS_APPEND: return 0u;
+		case Opcode::DS_ORDERED_COUNT: return 1u;
 		default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u;
 	}
 }
@@ -289,6 +300,11 @@ void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
 	}
 
 	DecodeVectorGpr(vdata, inst.dst);
+	if (inst.opcode == Opcode::BUFFER_LOAD_FORMAT_D16_X ||
+	    inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16 ||
+	    inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16_HI) {
+		inst.dst.sdwa_sel = inst.opcode == Opcode::BUFFER_LOAD_SHORT_D16_HI ? 5u : 4u;
+	}
 	DecodeVectorGpr(vaddr, inst.src0);
 	DecodeScalarSource(srsrc * 4u, pc, inst.src1);
 	DecodeScalarSource(soffset, pc, inst.src2);
@@ -338,7 +354,6 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const uint32_t word0  = code[word_index];
 	const uint32_t word1  = code[word_index + 1u];
 	const uint32_t offset = word0 & 0xfffu;
-	const uint32_t dlc    = (word0 >> 12u) & 1u;
 	const uint32_t lds    = (word0 >> 13u) & 1u;
 	const uint32_t seg    = (word0 >> 14u) & 0x3u;
 	const uint32_t opcode = (word0 >> 18u) & 0x7fu;
@@ -350,6 +365,7 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.pc             = pc;
 	inst.word_count     = 2;
 	inst.offset         = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
+	inst.dlc            = ((word0 >> 12u) & 1u) != 0;
 	inst.glc            = ((word0 >> 16u) & 1u) != 0;
 	inst.slc            = ((word0 >> 17u) & 1u) != 0;
 	inst.family         = Family::FLAT;
@@ -359,7 +375,7 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	ApplyMemoryInfo(inst, info);
 	SetRawWords(inst, code, word_index, 2);
 
-	if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) {
+	if (lds != 0 || inst.glc || (inst.slc && !IsFlatStoreOpcode(inst.opcode)) || seg == 3u) {
 		SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
 		return;
 	}
@@ -369,6 +385,10 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 
 	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst);
+	if (inst.opcode == Opcode::FLAT_LOAD_SHORT_D16) {
+		// D16 loads reuse partial destinations to preserve the untouched high half.
+		inst.dst.sdwa_sel = 4u;
+	}
 	DecodeVectorGpr(addr, inst.src0);
 	inst.src_count = 1;
 	if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
@@ -403,11 +423,20 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (inst.opcode == Opcode::UNSUPPORTED) {
 		SetUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
 	}
+	if (inst.opcode == Opcode::DS_ORDERED_COUNT) {
+		// Keep OFFSET1's wave type as well as its add/swap selector for GDS addressing.
+		inst.offset           = offset0 & 0xfcu;
+		inst.secondary_offset = offset1;
+		if (!inst.gds || ((offset1 >> 4u) & 3u) > 1u || ((offset1 >> 6u) & 3u) != 0u) {
+			SetUnsupported(inst, Family::DS, opcode, "DS ordered count variant is not implemented");
+		}
+	}
 	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
 	if (inst.gds &&
-	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_BPERMUTE_B32 ||
+	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_PERMUTE_B32 ||
+	     inst.opcode == Opcode::DS_BPERMUTE_B32 ||
 	     inst.opcode == Opcode::DS_WRITE_ADDTID_B32 ||
 	     inst.opcode == Opcode::DS_READ_ADDTID_B32)) {
 		SetUnsupported(inst, Family::DS, opcode, "DS lane operation is available only for LDS");

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -53,12 +53,15 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x14u, Opcode::S_FF1_I32_B64},
     {0x15u, Opcode::S_FLBIT_I32_B32},
     {0x16u, Opcode::S_FLBIT_I32_B64},
+    {0x19u, Opcode::S_SEXT_I32_I8},
+    {0x1au, Opcode::S_SEXT_I32_I16},
     {0x1bu, Opcode::S_BITSET0_B32},
     {0x1cu, Opcode::S_BITSET0_B64},
     {0x1du, Opcode::S_BITSET1_B32},
     {0x1eu, Opcode::S_BITSET1_B64},
     {0x1fu, Opcode::S_GETPC_B64},
     {0x20u, Opcode::S_SETPC_B64},
+    {0x21u, Opcode::S_SWAPPC_B64},
     {0x24u, Opcode::S_AND_SAVEEXEC_B64},
     {0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
     {0x2du, Opcode::S_QUADMASK_B64},
@@ -86,7 +89,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = {
     {0x08u, Opcode::S_CMP_LE_I32}, {0x09u, Opcode::S_CMP_EQ_U32}, {0x0au, Opcode::S_CMP_LG_U32},
     {0x0bu, Opcode::S_CMP_GT_U32}, {0x0cu, Opcode::S_CMP_GE_U32}, {0x0du, Opcode::S_CMP_LT_U32},
     {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32},    {0x10u, Opcode::S_MULK_I32},
-    {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT},    {0x18u, Opcode::S_WAITCNT},
+    {0x12u, Opcode::S_GETREG_B32}, {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT_VSCNT}, {0x18u, Opcode::S_WAITCNT},
     {0x19u, Opcode::S_WAITCNT},    {0x1au, Opcode::S_WAITCNT},
     {0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END},
 };
@@ -153,6 +156,10 @@ void DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 
 	switch (inst.opcode) {
+		case Opcode::S_SWAPPC_B64:
+			inst.branch_target = UINT32_MAX;
+			inst.data_dwords = 2;
+			break;
 		case Opcode::S_GETPC_B64:
 			inst.src_count = 0;
 			DecodeScalarDestination(sdst, pc, inst.dst);
@@ -200,7 +207,7 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const uint32_t word   = code[word_index];
 	const uint32_t opcode = (word >> 23u) & 0x1fu;
 	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-	const auto     imm    = opcode >= 0x09u && opcode <= 0x0eu
+	const auto     imm    = (opcode >= 0x09u && opcode <= 0x0eu) || opcode == 0x12u
 	                           ? static_cast<int32_t>(word & 0xffffu)
 	                           : static_cast<int32_t>(static_cast<int16_t>(word & 0xffffu));
 
@@ -220,6 +227,7 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 
 	switch (inst.opcode) {
+		case Opcode::S_GETREG_B32:
 		case Opcode::S_MOVK_I32: DecodeScalarDestination(sdst, pc, inst.dst); return;
 		case Opcode::S_SUBVECTOR_LOOP_BEGIN:
 		case Opcode::S_SUBVECTOR_LOOP_END:
@@ -234,6 +242,14 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			inst.src_count         = 1;
 			return;
 		}
+		case Opcode::S_WAITCNT_VSCNT:
+			inst.dst.kind       = OperandKind::Null;
+			inst.src1           = inst.src0;
+			inst.src1.value     = word & 0xffffu;
+			inst.src1.signed_val = static_cast<int32_t>(inst.src1.value);
+			DecodeScalarSource(sdst, pc, inst.src0);
+			inst.src_count = 2;
+			return;
 		case Opcode::S_SETREG_B32:
 			inst.dst.kind        = OperandKind::Null;
 			inst.src1.kind       = OperandKind::LiteralConstant;

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -389,6 +389,7 @@ Program DecodeFrontProgram(std::span<const uint32_t> front) {
 	while (front_words < front.size()) {
 		auto& inst = result.instructions.emplace_back();
 		DecodeInstruction(front, front_words, inst);
+		result.has_swap_pc |= inst.opcode == Opcode::S_SWAPPC_B64;
 		front_words += inst.word_count;
 		if (inst.opcode == Opcode::S_SETPC_B64) {
 			EXIT_NOT_IMPLEMENTED(inst.src0.kind != OperandKind::Sgpr || inst.src0.reg != 6u);
@@ -406,6 +407,7 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 	program.instructions.clear();
 	program.instructions.reserve(code.size());
 	program.code = code;
+	program.has_swap_pc = false;
 
 	std::vector<bool> branch_targets;
 	for (uint32_t word_index = 0; word_index < code.size();) {
@@ -413,6 +415,7 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 		DecodeInstruction(code, word_index, program.instructions.back());
 
 		const auto& inst = program.instructions.back();
+		program.has_swap_pc |= inst.opcode == Opcode::S_SWAPPC_B64;
 		word_index += inst.word_count;
 
 		if (IsDirectBranch(inst.opcode)) {
@@ -503,6 +506,8 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.dst).c_str(),
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_ABS_I32:
+		case Opcode::S_SEXT_I32_I8:
+		case Opcode::S_SEXT_I32_I16:
 		case Opcode::S_BREV_B32:
 		case Opcode::S_BREV_B64:
 		case Opcode::S_BCNT1_I32_B32:
@@ -535,6 +540,11 @@ std::string InstructionToString(const Instruction& inst) {
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc,
 			                                               OperandToString(inst.src0).c_str(),
 			                                               OperandToString(inst.src1).c_str()));
+		case Opcode::S_WAITCNT_VSCNT:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               OperandToString(inst.src0),
+			                                               OperandToString(inst.src1)));
 		case Opcode::S_NOP:
 		case Opcode::S_WAITCNT:
 		case Opcode::S_WAITCNT_DEPCTR:
@@ -612,12 +622,16 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_LOAD_FORMAT_XY:
 		case Opcode::BUFFER_LOAD_FORMAT_XYZ:
 		case Opcode::BUFFER_LOAD_FORMAT_XYZW:
+		case Opcode::BUFFER_LOAD_FORMAT_D16_X:
 		case Opcode::BUFFER_STORE_FORMAT_X:
 		case Opcode::BUFFER_STORE_FORMAT_XY:
 		case Opcode::BUFFER_STORE_FORMAT_XYZ:
 		case Opcode::BUFFER_STORE_FORMAT_XYZW:
+		case Opcode::BUFFER_STORE_FORMAT_D16_X:
 		case Opcode::BUFFER_LOAD_UBYTE:
 		case Opcode::BUFFER_LOAD_USHORT:
+		case Opcode::BUFFER_LOAD_SHORT_D16:
+		case Opcode::BUFFER_LOAD_SHORT_D16_HI:
 		case Opcode::BUFFER_LOAD_DWORD:
 		case Opcode::BUFFER_LOAD_DWORDX2:
 		case Opcode::BUFFER_LOAD_DWORDX3:
@@ -658,6 +672,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_LOAD_SBYTE:
 		case Opcode::FLAT_LOAD_USHORT:
 		case Opcode::FLAT_LOAD_SSHORT:
+		case Opcode::FLAT_LOAD_SHORT_D16:
 		case Opcode::FLAT_LOAD_DWORD:
 		case Opcode::FLAT_LOAD_DWORDX2:
 		case Opcode::FLAT_LOAD_DWORDX3:
@@ -692,10 +707,12 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_OR_RTN_B32:
 		case Opcode::DS_XOR_B32:
 		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_WRXCHG_RTN_B32:
 		case Opcode::DS_MIN_F32:
 		case Opcode::DS_MAX_F32:
 		case Opcode::DS_SWIZZLE_B32:
+		case Opcode::DS_PERMUTE_B32:
 		case Opcode::DS_BPERMUTE_B32:
 		case Opcode::DS_READ_I8:
 		case Opcode::DS_READ_U8:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -45,6 +45,8 @@ enum class Opcode {
 	S_CMOV_B64,
 	S_MOVK_I32,
 	S_ABS_I32,
+	S_SEXT_I32_I8,
+	S_SEXT_I32_I16,
 	S_ABSDIFF_I32,
 	S_BREV_B32,
 	S_BREV_B64,
@@ -58,6 +60,7 @@ enum class Opcode {
 	S_QUADMASK_B64,
 	S_GETPC_B64,
 	S_SETPC_B64,
+	S_SWAPPC_B64,
 	S_SUBVECTOR_LOOP_BEGIN,
 	S_SUBVECTOR_LOOP_END,
 	S_AND_SAVEEXEC_B32,
@@ -186,10 +189,14 @@ enum class Opcode {
 	V_RCP_F32,
 	V_RCP_IFLAG_F32,
 	V_FRACT_F32,
+	V_FRACT_F64,
 	V_TRUNC_F32,
+	V_TRUNC_F64,
 	V_CEIL_F32,
+	V_CEIL_F64,
 	V_RNDNE_F32,
 	V_FLOOR_F32,
+	V_FLOOR_F64,
 	V_EXP_F32,
 	V_LOG_F32,
 	V_RSQ_F32,
@@ -220,6 +227,9 @@ enum class Opcode {
 	V_MAC_F32,
 	V_MADMK_F32,
 	V_MADAK_F32,
+	V_FMAC_F32,
+	V_FMAMK_F32,
+	V_FMAAK_F32,
 	V_MIN_F32,
 	V_MAX_F32,
 	V_ADD_F16,
@@ -247,6 +257,8 @@ enum class Opcode {
 	V_FMA_F64,
 	V_ADD_F64,
 	V_MUL_F64,
+	V_MIN_F64,
+	V_MAX_F64,
 	V_FMA_F32,
 	V_FMA_F16,
 	V_PACK_B32_F16,
@@ -259,6 +271,7 @@ enum class Opcode {
 	V_MIN3_I32,
 	V_MIN3_U32,
 	V_MIN3_F16,
+	V_MIN3_U16,
 	V_MAX3_F32,
 	V_MAX3_I32,
 	V_MAX3_U32,
@@ -372,6 +385,7 @@ enum class Opcode {
 	V_CMPX_LG_F32,
 	V_CMPX_GE_F32,
 	V_CMPX_O_F32,
+	V_CMPX_U_F32,
 	V_CMPX_NGE_F32,
 	V_CMPX_NLG_F32,
 	V_CMPX_NGT_F32,
@@ -388,6 +402,8 @@ enum class Opcode {
 	V_CMP_T_I32,
 	V_CMP_CLASS_F32,
 	V_CMPX_CLASS_F32,
+	V_CMP_CLASS_F16,
+	V_CMPX_CLASS_F16,
 	V_CMP_LT_I16,
 	V_CMP_EQ_I16,
 	V_CMP_LE_I16,
@@ -432,6 +448,7 @@ enum class Opcode {
 	V_CMPX_LT_U16,
 	V_CMPX_EQ_U16,
 	V_CMPX_GT_U16,
+	V_CMPX_NE_U16,
 	V_CMP_NE_U16,
 	V_CMP_GE_U16,
 	V_CMP_F_U32,
@@ -442,18 +459,34 @@ enum class Opcode {
 	V_CMP_NE_U32,
 	V_CMP_GE_U32,
 	V_CMP_T_U32,
+	V_CMP_F_I64,
 	V_CMP_EQ_I64,
 	V_CMP_LT_I64,
 	V_CMP_LE_I64,
+	V_CMP_NE_I64,
+	V_CMP_T_I64,
+	V_CMP_F_U64,
 	V_CMP_LT_U64,
 	V_CMP_EQ_U64,
 	V_CMP_LE_U64,
 	V_CMP_GT_U64,
 	V_CMP_NE_U64,
 	V_CMP_GE_U64,
+	V_CMP_T_U64,
+	V_CMPX_F_I64,
+	V_CMPX_LT_I64,
+	V_CMPX_EQ_I64,
+	V_CMPX_LE_I64,
 	V_CMPX_NE_I64,
+	V_CMPX_T_I64,
+	V_CMPX_F_U64,
+	V_CMPX_LT_U64,
+	V_CMPX_EQ_U64,
 	V_CMPX_LE_U64,
+	V_CMPX_GT_U64,
 	V_CMPX_NE_U64,
+	V_CMPX_GE_U64,
+	V_CMPX_T_U64,
 	V_CMPX_LT_U32,
 	V_CMPX_EQ_U32,
 	V_CMPX_LE_U32,
@@ -476,14 +509,18 @@ enum class Opcode {
 	BUFFER_LOAD_FORMAT_XY,
 	BUFFER_LOAD_FORMAT_XYZ,
 	BUFFER_LOAD_FORMAT_XYZW,
+	BUFFER_LOAD_FORMAT_D16_X,
 	BUFFER_STORE_FORMAT_X,
 	BUFFER_STORE_FORMAT_XY,
 	BUFFER_STORE_FORMAT_XYZ,
 	BUFFER_STORE_FORMAT_XYZW,
+	BUFFER_STORE_FORMAT_D16_X,
 	BUFFER_LOAD_UBYTE,
 	BUFFER_LOAD_SBYTE,
 	BUFFER_LOAD_USHORT,
 	BUFFER_LOAD_SSHORT,
+	BUFFER_LOAD_SHORT_D16,
+	BUFFER_LOAD_SHORT_D16_HI,
 	BUFFER_LOAD_DWORD,
 	BUFFER_LOAD_DWORDX2,
 	BUFFER_LOAD_DWORDX3,
@@ -522,6 +559,7 @@ enum class Opcode {
 	FLAT_LOAD_SBYTE,
 	FLAT_LOAD_USHORT,
 	FLAT_LOAD_SSHORT,
+	FLAT_LOAD_SHORT_D16,
 	FLAT_LOAD_DWORD,
 	FLAT_LOAD_DWORDX2,
 	FLAT_LOAD_DWORDX3,
@@ -556,13 +594,16 @@ enum class Opcode {
 	DS_OR_RTN_B32,
 	DS_XOR_B32,
 	DS_XOR_RTN_B32,
+	DS_MSKOR_B32,
 	DS_WRXCHG_RTN_B32,
 	DS_MIN_F32,
 	DS_MAX_F32,
 	DS_SWIZZLE_B32,
+	DS_PERMUTE_B32,
 	DS_BPERMUTE_B32,
 	DS_CONSUME,
 	DS_APPEND,
+	DS_ORDERED_COUNT,
 	DS_READ_I8,
 	DS_READ_U8,
 	DS_READ_I16,
@@ -625,6 +666,7 @@ enum class Opcode {
 
 	S_NOP,
 	S_WAITCNT,
+	S_WAITCNT_VSCNT,
 	S_WAITCNT_DEPCTR,
 	S_BARRIER,
 	S_BRANCH,
@@ -637,6 +679,7 @@ enum class Opcode {
 	S_CBRANCH_CDBGSYS,
 	S_CBRANCH_CDBGSYS_OR_USER,
 	S_SENDMSG,
+	S_GETREG_B32,
 	S_SETREG_B32,
 	S_SLEEP,
 	S_TRAP,
@@ -775,6 +818,7 @@ struct Instruction {
 struct Program {
 	std::span<const uint32_t> code;
 	std::vector<Instruction>  instructions;
+	bool                     has_swap_pc = false;
 };
 
 // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -35,8 +35,8 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
     {0x04u, Opcode::V_SUB_F32, Vop2SdwaProfile::Float32},
     {0x05u, Opcode::V_SUBREV_F32},
     {0x08u, Opcode::V_MUL_F32, Vop2SdwaProfile::Float32},
-    {0x09u, Opcode::V_MUL_I32_I24, Vop2SdwaProfile::IntegerFullDestination},
-    {0x0bu, Opcode::V_MUL_U32_U24, Vop2SdwaProfile::IntegerFullDestination},
+    {0x09u, Opcode::V_MUL_I32_I24, Vop2SdwaProfile::IntegerPartialDestination},
+    {0x0bu, Opcode::V_MUL_U32_U24, Vop2SdwaProfile::IntegerPartialDestination},
     {0x0fu, Opcode::V_MIN_F32},
     {0x10u, Opcode::V_MAX_F32},
     {0x11u, Opcode::V_MIN_I32},
@@ -65,9 +65,9 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
     {0x2au, Opcode::V_SUBREV_CO_CI_U32},
     {0x26u, Opcode::V_SUB_NC_U32, Vop2SdwaProfile::IntegerPartialDestination},
     {0x27u, Opcode::V_SUBREV_NC_U32, Vop2SdwaProfile::IntegerFullDestination},
-    {0x2bu, Opcode::V_MAC_F32},
-    {0x2cu, Opcode::V_MADMK_F32},
-    {0x2du, Opcode::V_MADAK_F32},
+    {0x2bu, Opcode::V_FMAC_F32},
+    {0x2cu, Opcode::V_FMAMK_F32},
+    {0x2du, Opcode::V_FMAAK_F32},
     {0x2fu, Opcode::V_CVT_PKRTZ_F16_F32, Vop2SdwaProfile::PackedFloat16},
     {0x32u, Opcode::V_ADD_F16, Vop2SdwaProfile::Float16},
     {0x33u, Opcode::V_SUB_F16, Vop2SdwaProfile::Float16},
@@ -91,6 +91,9 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x0fu, Opcode::V_CVT_F32_F64},
     {0x10u, Opcode::V_CVT_F64_F32},
     {0x16u, Opcode::V_CVT_F64_U32},
+    {0x17u, Opcode::V_TRUNC_F64},
+    {0x18u, Opcode::V_CEIL_F64},
+    {0x1au, Opcode::V_FLOOR_F64},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -123,6 +126,7 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x39u, Opcode::V_FFBH_U32},
     {0x3au, Opcode::V_FFBL_B32},
     {0x3bu, Opcode::V_FFBH_I32},
+    {0x3eu, Opcode::V_FRACT_F64},
     {0x3fu, Opcode::V_FREXP_EXP_I32_F32},
     {0x40u, Opcode::V_FREXP_MANT_F32},
     {0x42u, Opcode::V_MOVRELD_B32},
@@ -155,6 +159,9 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x0fu, Opcode::V_CVT_F32_F64},
     {0x10u, Opcode::V_CVT_F64_F32},
     {0x16u, Opcode::V_CVT_F64_U32},
+    {0x17u, Opcode::V_TRUNC_F64},
+    {0x18u, Opcode::V_CEIL_F64},
+    {0x1au, Opcode::V_FLOOR_F64},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -182,6 +189,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x39u, Opcode::V_FFBH_U32},
     {0x3au, Opcode::V_FFBL_B32},
     {0x3bu, Opcode::V_FFBH_I32},
+    {0x3eu, Opcode::V_FRACT_F64},
     {0x3fu, Opcode::V_FREXP_EXP_I32_F32},
     {0x40u, Opcode::V_FREXP_MANT_F32},
     {0x42u, Opcode::V_MOVRELD_B32},
@@ -224,7 +232,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x11u, Opcode::V_CMPX_LT_F32},        {0x12u, Opcode::V_CMPX_EQ_F32},
     {0x13u, Opcode::V_CMPX_LE_F32},        {0x14u, Opcode::V_CMPX_GT_F32},
     {0x15u, Opcode::V_CMPX_LG_F32},        {0x16u, Opcode::V_CMPX_GE_F32},
-    {0x17u, Opcode::V_CMPX_O_F32},
+    {0x17u, Opcode::V_CMPX_O_F32},         {0x18u, Opcode::V_CMPX_U_F32},
     {0x19u, Opcode::V_CMPX_NGE_F32},       {0x1au, Opcode::V_CMPX_NLG_F32},
     {0x1bu, Opcode::V_CMPX_NGT_F32},       {0x1cu, Opcode::V_CMPX_NLE_F32},
     {0x1du, Opcode::V_CMPX_NEQ_F32},       {0x1eu, Opcode::V_CMPX_NLT_F32},
@@ -237,27 +245,39 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x88u, Opcode::V_CMP_CLASS_F32},      {0x89u, Opcode::V_CMP_LT_I16},
     {0x8au, Opcode::V_CMP_EQ_I16},         {0x8bu, Opcode::V_CMP_LE_I16},
     {0x8cu, Opcode::V_CMP_GT_I16},         {0x8du, Opcode::V_CMP_NE_I16},
-    {0x8eu, Opcode::V_CMP_GE_I16},         {0x91u, Opcode::V_CMPX_LT_I32},
+    {0x8eu, Opcode::V_CMP_GE_I16},         {0x8fu, Opcode::V_CMP_CLASS_F16, false},
+    {0x91u, Opcode::V_CMPX_LT_I32},
     {0x92u, Opcode::V_CMPX_EQ_I32},        {0x93u, Opcode::V_CMPX_LE_I32},
     {0x94u, Opcode::V_CMPX_GT_I32},        {0x95u, Opcode::V_CMPX_NE_I32},
     {0x96u, Opcode::V_CMPX_GE_I32},        {0x98u, Opcode::V_CMPX_CLASS_F32},
     {0x99u, Opcode::V_CMPX_LT_I16, false}, {0x9au, Opcode::V_CMPX_EQ_I16, false},
     {0x9bu, Opcode::V_CMPX_LE_I16, false}, {0x9cu, Opcode::V_CMPX_GT_I16, false},
     {0x9du, Opcode::V_CMPX_NE_I16, false}, {0x9eu, Opcode::V_CMPX_GE_I16, false},
+    {0x9fu, Opcode::V_CMPX_CLASS_F16, false},
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
     {0xb9u, Opcode::V_CMPX_LT_U16, false}, {0xbau, Opcode::V_CMPX_EQ_U16, false},
-    {0xbcu, Opcode::V_CMPX_GT_U16},        {0xc0u, Opcode::V_CMP_F_U32},
+    {0xbcu, Opcode::V_CMPX_GT_U16},        {0xbdu, Opcode::V_CMPX_NE_U16, false},
+    {0xc0u, Opcode::V_CMP_F_U32},
     {0xc1u, Opcode::V_CMP_LT_U32},         {0xc2u, Opcode::V_CMP_EQ_U32},
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
     {0xc5u, Opcode::V_CMP_NE_U32},         {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},          {0xa2u, Opcode::V_CMP_EQ_I64, false},
     {0xa1u, Opcode::V_CMP_LT_I64, false},  {0xa3u, Opcode::V_CMP_LE_I64, false},
+    {0xa5u, Opcode::V_CMP_NE_I64, false},
+    {0xa0u, Opcode::V_CMP_F_I64, false},   {0xa7u, Opcode::V_CMP_T_I64, false},
+    {0xb0u, Opcode::V_CMPX_F_I64, false},  {0xb1u, Opcode::V_CMPX_LT_I64, false},
+    {0xb2u, Opcode::V_CMPX_EQ_I64, false}, {0xb3u, Opcode::V_CMPX_LE_I64, false},
+    {0xb7u, Opcode::V_CMPX_T_I64, false},
     {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
     {0xd2u, Opcode::V_CMPX_EQ_U32},        {0xd3u, Opcode::V_CMPX_LE_U32},
     {0xd4u, Opcode::V_CMPX_GT_U32},        {0xd5u, Opcode::V_CMPX_NE_U32},
     {0xd6u, Opcode::V_CMPX_GE_U32},        {0xe1u, Opcode::V_CMP_LT_U64, false},
+    {0xe0u, Opcode::V_CMP_F_U64, false},   {0xe7u, Opcode::V_CMP_T_U64, false},
+    {0xf0u, Opcode::V_CMPX_F_U64, false},  {0xf1u, Opcode::V_CMPX_LT_U64, false},
+    {0xf2u, Opcode::V_CMPX_EQ_U64, false}, {0xf4u, Opcode::V_CMPX_GT_U64, false},
+    {0xf6u, Opcode::V_CMPX_GE_U64, false}, {0xf7u, Opcode::V_CMPX_T_U64, false},
     {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe3u, Opcode::V_CMP_LE_U64, false},
     {0xe4u, Opcode::V_CMP_GT_U64, false},  {0xe5u, Opcode::V_CMP_NE_U64, false},
     {0xe6u, Opcode::V_CMP_GE_U64, false},  {0xf3u, Opcode::V_CMPX_LE_U64, false},
@@ -289,6 +309,8 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x14cu, Opcode::V_FMA_F64},
     {0x164u, Opcode::V_ADD_F64},
     {0x165u, Opcode::V_MUL_F64},
+    {0x166u, Opcode::V_MIN_F64},
+    {0x167u, Opcode::V_MAX_F64},
     {0x148u, Opcode::V_BFE_U32},
     {0x149u, Opcode::V_BFE_I32},
     {0x14au, Opcode::V_BFI_B32},
@@ -297,6 +319,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x152u, Opcode::V_MIN3_I32},
     {0x153u, Opcode::V_MIN3_U32},
     {0x351u, Opcode::V_MIN3_F16},
+    {0x353u, Opcode::V_MIN3_U16},
     {0x154u, Opcode::V_MAX3_F32},
     {0x155u, Opcode::V_MAX3_I32},
     {0x156u, Opcode::V_MAX3_U32},
@@ -447,8 +470,9 @@ bool IsNativeVop3F16TernaryOpcode(Opcode opcode) {
 	       opcode == Opcode::V_MED3_F16 || opcode == Opcode::V_FMA_F16;
 }
 
-bool IsNativeVop3I16TernaryOpcode(Opcode opcode) {
-	return opcode == Opcode::V_MED3_I16 || opcode == Opcode::V_MAD_I16;
+bool IsNativeVop3B16TernaryOpcode(Opcode opcode) {
+	return opcode == Opcode::V_MIN3_U16 || opcode == Opcode::V_MED3_I16 ||
+	       opcode == Opcode::V_MAD_I16;
 }
 
 bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
@@ -472,11 +496,23 @@ bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
 bool IsVop1FloatResultOpcode(Opcode opcode);
 bool IsVopcCompareExec(Opcode opcode);
 
+bool IsVop1Float64Opcode(Opcode opcode) {
+	return opcode == Opcode::V_CVT_F64_I32 || opcode == Opcode::V_CVT_F32_F64 ||
+	       opcode == Opcode::V_CVT_F64_F32 || opcode == Opcode::V_CVT_F64_U32 ||
+	       opcode == Opcode::V_RCP_F64 || opcode == Opcode::V_FRACT_F64 ||
+	       opcode == Opcode::V_TRUNC_F64 || opcode == Opcode::V_CEIL_F64 ||
+	       opcode == Opcode::V_FLOOR_F64;
+}
+
 bool IsVop1FloatSourceOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_CVT_F32_F64:
 		case Opcode::V_CVT_F64_F32:
 		case Opcode::V_RCP_F64:
+		case Opcode::V_FRACT_F64:
+		case Opcode::V_TRUNC_F64:
+		case Opcode::V_CEIL_F64:
+		case Opcode::V_FLOOR_F64:
 		case Opcode::V_MOV_B32:
 		case Opcode::V_CVT_F32_F16:
 		case Opcode::V_CVT_U32_F32:
@@ -543,7 +579,7 @@ struct Vop1SdwaRule {
 
 constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
     {Opcode::V_MOV_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(),
-     SdwaSelWords() | SdwaSelFull(), false},
+     SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
     {Opcode::V_CVT_F32_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_I32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_UBYTE0, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
@@ -551,6 +587,7 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
     // partial destination may combine with any source selector.
     {Opcode::V_NOT_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(),
      SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
+    {Opcode::V_BFREV_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_FFBL_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_FFBH_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
@@ -647,6 +684,10 @@ bool SupportsVop1Clamp(Opcode opcode) {
 }
 
 bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
+	if (IsVop1Float64Opcode(inst.opcode)) {
+		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support SDWA");
+		return false;
+	}
 	const auto dst_sel   = (modifier >> 8u) & 0x7u;
 	const auto dst_u     = (modifier >> 11u) & 0x3u;
 	const auto clamp     = (modifier >> 13u) & 0x1u;
@@ -735,9 +776,7 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
 
 void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                    uint32_t opcode, uint32_t vdst, Instruction& inst) {
-	if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 ||
-	    inst.opcode == Opcode::V_CVT_F64_F32 || inst.opcode == Opcode::V_CVT_F64_U32 ||
-	    inst.opcode == Opcode::V_RCP_F64) {
+	if (IsVop1Float64Opcode(inst.opcode)) {
 		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP");
 		return;
 	}
@@ -774,6 +813,9 @@ bool IsVop2FloatOpcode(Opcode opcode) {
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MADMK_F32:
 		case Opcode::V_MADAK_F32:
+		case Opcode::V_FMAC_F32:
+		case Opcode::V_FMAMK_F32:
+		case Opcode::V_FMAAK_F32:
 		case Opcode::V_CVT_PKRTZ_F16_F32:
 		case Opcode::V_ADD_F16:
 		case Opcode::V_SUB_F16:
@@ -861,6 +903,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_LG_F32:
 		case Opcode::V_CMPX_GE_F32:
 		case Opcode::V_CMPX_O_F32:
+		case Opcode::V_CMPX_U_F32:
 		case Opcode::V_CMPX_NGE_F32:
 		case Opcode::V_CMPX_NLG_F32:
 		case Opcode::V_CMPX_NGT_F32:
@@ -887,6 +930,8 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_NEQ_F16:
 		case Opcode::V_CMPX_NLT_F16:
 		case Opcode::V_CMP_CLASS_F32:
+		case Opcode::V_CMP_CLASS_F16:
+		case Opcode::V_CMPX_CLASS_F16:
 		case Opcode::V_CMPX_CLASS_F32: return true;
 		default: return false;
 	}
@@ -952,7 +997,8 @@ constexpr Vop2SdwaRule VOP2_SDWA_RULES[] = {
     {SdwaSelAll(), SdwaSelAll(), SdwaSelAll(), true, false},
     {SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false},
     {SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), false, false},
-    {SdwaSelWords() | SdwaSelFull(), SdwaSelAll(), SdwaSelAll(), true, false},
+    // Bitwise: WriteRawU32 inserts a byte or word destination like any other partial write.
+    {SdwaSelAll(), SdwaSelAll(), SdwaSelAll(), true, false},
 };
 static_assert(sizeof(VOP2_SDWA_RULES) / sizeof(VOP2_SDWA_RULES[0]) ==
               static_cast<size_t>(Vop2SdwaProfile::Count));
@@ -1031,6 +1077,7 @@ void FinalizeVop2Instruction(std::span<const uint32_t> code, uint32_t word_index
                              Instruction& inst) {
 	switch (inst.opcode) {
 		case Opcode::V_MADMK_F32:
+		case Opcode::V_FMAMK_F32:
 		case Opcode::V_FMAMK_F16:
 			inst.src2      = inst.src1;
 			inst.src1      = {};
@@ -1038,6 +1085,7 @@ void FinalizeVop2Instruction(std::span<const uint32_t> code, uint32_t word_index
 			inst.src_count = 3;
 			break;
 		case Opcode::V_MADAK_F32:
+		case Opcode::V_FMAAK_F32:
 		case Opcode::V_FMAAK_F16:
 			inst.src2      = {};
 			inst.src2.kind = OperandKind::LiteralConstant;
@@ -1100,8 +1148,10 @@ void DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 	DecodeVectorGpr(vsrc1, inst.src1);
 	DecodeScalarSource(src0 + 256u, pc, inst.src0);
 	ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu);
-	inst.src1.negate       = ((modifier >> 22u) & 0x1u) != 0u;
-	inst.src1.absolute     = ((modifier >> 23u) & 0x1u) != 0u;
+	if (!inst.src0.dpp8) {
+		inst.src1.negate   = ((modifier >> 22u) & 0x1u) != 0u;
+		inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
+	}
 	const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16;
 	if (packed_fmac) {
 		inst.src0.negate_hi = inst.src0.negate;
@@ -1152,7 +1202,8 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) {
 }
 
 bool SupportsVopcSdwa(Opcode opcode) {
-	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 &&
+	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_NE_I64 &&
+	       opcode != Opcode::V_CMP_EQ_F64 &&
 	       opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 &&
 	       opcode != Opcode::V_CMPX_GE_F64;
 }
@@ -1221,6 +1272,8 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
+		case Opcode::V_MIN_F64:
+		case Opcode::V_MAX_F64:
 		case Opcode::V_MUL_LO_U32:
 		case Opcode::V_MUL_HI_U32:
 		case Opcode::V_MUL_LO_I32:
@@ -1327,7 +1380,7 @@ void ApplyNativeVop3TernaryModifiers(Instruction& inst, uint32_t op_sel, uint32_
 	inst.dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u;
 }
 
-void ApplyNativeVop3I16TernarySelectors(Instruction& inst, uint32_t op_sel) {
+void ApplyNativeVop3B16TernarySelectors(Instruction& inst, uint32_t op_sel) {
 	Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2};
 	for (uint32_t i = 0; i < 3u; i++) {
 		sources[i]->op_sel = ((op_sel >> i) & 1u) != 0;
@@ -1370,9 +1423,12 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
 		case Opcode::V_MIN_F32:
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
+		case Opcode::V_FMAC_F32:
 		case Opcode::V_MAD_F32:
 		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
+		case Opcode::V_MIN_F64:
+		case Opcode::V_MAX_F64:
 		case Opcode::V_FMA_F64:
 		case Opcode::V_FMA_F32:
 		case Opcode::V_PACK_B32_F16:
@@ -1403,9 +1459,13 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
 		case Opcode::V_MIN_F32:
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
+		case Opcode::V_FMAC_F32:
 		case Opcode::V_MAD_F32:
 		case Opcode::V_FMA_F32:
 		case Opcode::V_FMA_F16:
+		case Opcode::V_MIN3_F16:
+		case Opcode::V_MAX3_F16:
+		case Opcode::V_MED3_F16:
 		case Opcode::V_CUBEID_F32:
 		case Opcode::V_CVT_PKRTZ_F16_F32:
 		case Opcode::V_LDEXP_F32:
@@ -1436,9 +1496,10 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
 		return clamp != 0u || omod != 0u;
 	}
 	if (IsNativeVop3F16TernaryOpcode(opcode)) {
-		return opcode != Opcode::V_FMA_F16 && (clamp != 0u || omod != 0u);
+		// Clamp is applied by WriteF16 for every f16 ternary; omod only for V_FMA_F16.
+		return opcode != Opcode::V_FMA_F16 && omod != 0u;
 	}
-	if (IsNativeVop3I16TernaryOpcode(opcode)) {
+	if (IsNativeVop3B16TernaryOpcode(opcode)) {
 		return abs != 0u || (clamp != 0u && !clamp_modifier) || omod != 0u || neg != 0u;
 	}
 	if (IsNativeVop3B16BinaryOpcode(opcode)) {
@@ -1486,6 +1547,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_LG_F32:
 		case Opcode::V_CMPX_GE_F32:
 		case Opcode::V_CMPX_O_F32:
+		case Opcode::V_CMPX_U_F32:
 		case Opcode::V_CMPX_NGE_F32:
 		case Opcode::V_CMPX_NLG_F32:
 		case Opcode::V_CMPX_NGT_F32:
@@ -1499,15 +1561,27 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_I32:
 		case Opcode::V_CMPX_GE_I32:
 		case Opcode::V_CMPX_CLASS_F32:
+		case Opcode::V_CMPX_CLASS_F16:
 		case Opcode::V_CMPX_LT_U32:
 		case Opcode::V_CMPX_EQ_U32:
 		case Opcode::V_CMPX_LE_U32:
 		case Opcode::V_CMPX_GT_U32:
 		case Opcode::V_CMPX_NE_U32:
 		case Opcode::V_CMPX_GE_U32:
+		case Opcode::V_CMPX_F_I64:
+		case Opcode::V_CMPX_LT_I64:
+		case Opcode::V_CMPX_EQ_I64:
+		case Opcode::V_CMPX_LE_I64:
 		case Opcode::V_CMPX_NE_I64:
+		case Opcode::V_CMPX_T_I64:
+		case Opcode::V_CMPX_F_U64:
+		case Opcode::V_CMPX_LT_U64:
+		case Opcode::V_CMPX_EQ_U64:
 		case Opcode::V_CMPX_LE_U64:
+		case Opcode::V_CMPX_GT_U64:
 		case Opcode::V_CMPX_NE_U64:
+		case Opcode::V_CMPX_GE_U64:
+		case Opcode::V_CMPX_T_U64:
 		case Opcode::V_CMPX_LT_I16:
 		case Opcode::V_CMPX_EQ_I16:
 		case Opcode::V_CMPX_LE_I16:
@@ -1517,6 +1591,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_LT_U16:
 		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:
+		case Opcode::V_CMPX_NE_U16:
 		case Opcode::V_CMPX_LT_F16:
 		case Opcode::V_CMPX_EQ_F16:
 		case Opcode::V_CMPX_LE_F16:
@@ -1551,6 +1626,7 @@ void DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		return;
 	}
 	switch (src0) {
+		case 233u: DecodeVop2Dpp(pc, code, word_index, opcode, vdst, vsrc1, inst); return;
 		case 249u: DecodeVop2Sdwa(pc, code, word_index, opcode, vdst, vsrc1, inst); return;
 		case 250u: DecodeVop2Dpp(pc, code, word_index, opcode, vdst, vsrc1, inst); return;
 		default: break;
@@ -1660,7 +1736,7 @@ void DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const bool vop3b_uses_sdst = carry_in_out || vop3b_carry_out || vop3b_mad_u64;
 	const bool mad_mix         = false;
 	const bool f16_ternary     = IsNativeVop3F16TernaryOpcode(inst.opcode);
-	const bool i16_ternary     = IsNativeVop3I16TernaryOpcode(inst.opcode);
+	const bool b16_ternary     = IsNativeVop3B16TernaryOpcode(inst.opcode);
 	const bool b16_binary      = IsNativeVop3B16BinaryOpcode(inst.opcode);
 	const bool pack_b32_f16    = inst.opcode == Opcode::V_PACK_B32_F16;
 	const bool permlane        = IsPermlaneOpcode(inst.opcode);
@@ -1771,8 +1847,8 @@ void DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		ApplyNativeVop3TernaryModifiers(inst, op_sel, abs, neg);
 	} else if (f16_ternary) {
 		ApplyNativeVop3TernaryModifiers(inst, op_sel, abs, neg);
-	} else if (i16_ternary) {
-		ApplyNativeVop3I16TernarySelectors(inst, op_sel);
+	} else if (b16_ternary) {
+		ApplyNativeVop3B16TernarySelectors(inst, op_sel);
 	} else if (b16_binary) {
 		ApplyNativeVop3B16BinaryModifiers(inst, op_sel);
 	} else if (pack_b32_f16) {

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -31,6 +31,8 @@ void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueO
 	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
 	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
 	                       inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMPX_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMP_NE_I64 ||
 	                       inst.opcode == Decoder::Opcode::V_CMPX_NE_I64;
 	const auto read = [&](const Decoder::Operand& operand) {
 		// RDNA2 expands signed 64-bit integer literals by sign extension.
@@ -71,11 +73,13 @@ void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool
 	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, cmpx);
 }
 
-void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx) {
-	const auto value = ReadOperand(inst.src0, IR::Type::F32);
-	const auto mask  = ReadOperand(inst.src1, IR::Type::U32);
-	EmitCompareResult(inst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})), false,
-	                  cmpx);
+void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half) {
+	const auto value = half ? IR::Value(Read16LaneBits(inst.src0, false))
+	                        : ReadOperand(inst.src0, IR::Type::F32);
+	const auto mask = half ? IR::Value(Read16LaneBits(inst.src1, false))
+	                      : ReadOperand(inst.src1, IR::Type::U32);
+	const auto opcode = half ? IR::ValueOpcode::FPCmpClass16 : IR::ValueOpcode::FPCmpClass32;
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {value, mask})), false, cmpx);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -181,8 +181,12 @@ void Translator::EmitControlNop() {
 	ir.Emit(IR::ValueOpcode::ControlNop);
 }
 
-void Translator::EmitWaitcnt() {
-	ir.Emit(IR::ValueOpcode::Waitcnt);
+void Translator::S_WAITCNT_VSCNT(const Decoder::Instruction& inst) {
+	const auto count = inst.src1.value & 63u;
+	if (inst.src0.kind != Decoder::OperandKind::Null || (count != 0u && count != 63u)) {
+		EXIT("unsupported partial or register-based S_WAITCNT_VSCNT at 0x%08x", inst.pc);
+	}
+	if (count == 0u) ir.Emit(IR::ValueOpcode::StoreCompletion);
 }
 
 void Translator::S_BARRIER() {
@@ -244,6 +248,7 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
 
 void Translator::ScalarSelect64(const Decoder::Instruction& inst,
                                  const Decoder::Operand& false_source) {
+	// Preserve per-word expressions for descriptor tracking and mask provenance.
 	const auto condition     = ir.GetScc();
 	const auto lhs           = ReadU32Pair(inst.src0);
 	const auto rhs           = ReadU32Pair(false_source);
@@ -252,13 +257,14 @@ void Translator::ScalarSelect64(const Decoder::Instruction& inst,
 	const auto selected_mask_valid =
 	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
 	                   {condition, ReadMaskValid(inst.src0), ReadMaskValid(false_source)}));
-	if (IsExecOrVcc(inst.dst)) {
-		WriteMask(inst.dst, selected_mask, true);
-		return;
-	}
 	WriteU32Pair(inst.dst,
 	             {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])});
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+	// Scalar selects preserve the raw pair even when EXEC/VCC holds an address.
+	if (inst.dst.kind == Decoder::OperandKind::ExecLo) {
+		ir.SetExec(selected_mask);
+	} else if (inst.dst.kind == Decoder::OperandKind::VccLo) {
+		ir.SetVcc(selected_mask);
+	} else if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
 		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
 		ir.SetThreadBitScalarReg(dst, selected_mask);
 		ir.SetScalarMaskTag(dst, selected_mask_valid);

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -8,13 +8,13 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
 void Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                bool accumulator, bool quiet_snan) {
 	const auto translate_lane = [&](bool high) {
-		const auto lhs = ReadF16LaneAsF32(inst.src0, high, true);
-		const auto rhs = ReadF16LaneAsF32(inst.src1, high, true);
+		const auto lhs = ReadF16LaneAsF32(inst.src0, high);
+		const auto rhs = ReadF16LaneAsF32(inst.src1, high);
 		IR::F32    result;
 		if (accumulator) {
-			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high, true)}));
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high)}));
 		} else if (inst.src_count == 3u) {
-			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high, true)}));
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high)}));
 		} else {
 			result = IR::F32(ir.Emit(opcode, {lhs, rhs}));
 		}

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -31,11 +31,12 @@ void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
 	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
 }
 
-void Translator::V_MED3_I16(const Decoder::Instruction& inst) {
+void Translator::Integer16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+                                  bool sign) {
 	const auto result = IR::U32(ir.Emit(
-	    IR::ValueOpcode::SMedTri32, {ReadU16AsU32(inst.src0, true), ReadU16AsU32(inst.src1, true),
-	                                 ReadU16AsU32(inst.src2, true)}));
-	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
+	    opcode, {ReadU16AsU32(inst.src0, sign), ReadU16AsU32(inst.src1, sign),
+	             ReadU16AsU32(inst.src2, sign)}));
+	Write16Bits(DestinationOperand(inst), result);
 }
 
 void Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -114,6 +115,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
 		return;
 	}
 
+	// Descriptor tracking reasons about each scalar word independently.
 	const auto lhs        = ReadU32Pair(inst.src0);
 	auto       mask_valid = ReadMaskValid(inst.src0);
 	if (inst.src_count > 1u) {
@@ -269,22 +271,18 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
 }
 
 void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
-	const auto lhs       = ReadU32(inst.src0);
-	const auto rhs       = ReadU32(inst.src1);
-	const auto add       = ExtractU64(ReadU64(inst.src2));
-	const auto mul_low   = ir.IMul(lhs, rhs);
-	const auto mul_high  = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
-	const auto low       = ir.IAdd(mul_low, add[0]);
-	const auto carry_low = ir.ULessThan(low, mul_low);
-	const auto high0     = ir.IAdd(mul_high, add[1]);
-	const auto carry0    = ir.ULessThan(high0, mul_high);
-	const auto high =
-	    ir.IAdd(high0, ir.Select(carry_low, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))));
-	const auto carry1 = ir.ULessThan(high, high0);
-	WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high));
+	const auto lhs = ReadU32(inst.src0);
+	const auto rhs = ReadU32(inst.src1);
+	// Keep the 32x32 product in words: native widening IMul64 fails NVIDIA compilation
+	// in large vertex shaders (Astro's Playroom). The 64-bit addition stays native.
+	const auto product = ir.ConstructU64(
+	    ir.IMul(lhs, rhs), IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})));
+	const auto result  = ir.Emit(IR::ValueOpcode::IAdd64, {product, ReadU64(inst.src2)});
+	WriteOperand(DestinationOperand(inst), result);
 	if (inst.dst2.kind != Decoder::OperandKind::Null &&
 	    inst.dst2.kind != Decoder::OperandKind::Unknown) {
-		WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
+		// The 32x32 product fits in 64 bits, so only the addition can carry.
+		WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product})));
 	}
 }
 
@@ -407,9 +405,12 @@ IR::U64 Translator::RightMask64(IR::U32 count) {
 }
 
 void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
-	const auto count  = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
-	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
-	const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset});
+	const auto count  = ReadU32(inst.src0);
+	const auto offset = ReadU32(inst.src1);
+	const auto one    = IR::Value(uint64_t {1});
+	const auto limit  = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {one, count});
+	const auto mask   = ir.Emit(IR::ValueOpcode::ISub64, {limit, one});
+	const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {mask, offset});
 	WriteOperand(DestinationOperand(inst), result);
 }
 
@@ -483,10 +484,11 @@ void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
 	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
 }
 
-void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
-	const auto hi      = ReadU32(inst.src0);
-	const auto lo      = ReadU32(inst.src1);
-	const auto shift   = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
+void Translator::V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset) {
+	const auto hi = ReadU32(inst.src0);
+	const auto lo = ReadU32(inst.src1);
+	auto shift    = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(byte_offset ? 3u : 31u)));
+	if (byte_offset) shift = ir.ShiftLeftLogical(shift, IR::U32(IR::Value(3u)));
 	const auto lo_part = ir.ShiftRightLogical(lo, shift);
 	const auto inverse =
 	    ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(32u)), shift), IR::U32(IR::Value(31u)));
@@ -496,21 +498,6 @@ void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(lo_part, hi_part));
 }
 
-void Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
-	const auto hi           = ReadU32(inst.src0);
-	const auto lo           = ReadU32(inst.src1);
-	const auto byte_offset  = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
-	const auto bit_offset   = ir.ShiftLeftLogical(byte_offset, IR::U32(IR::Value(3u)));
-	const auto concatenated = ir.ConstructU64(lo, hi);
-	const auto shifted =
-	    IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64,
-	                    {concatenated, ir.BitwiseAnd(bit_offset, IR::U32(IR::Value(63u)))}));
-	const auto in_range =
-	    IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, IR::Value(8u)}));
-	WriteOperand(DestinationOperand(inst),
-	             ir.Select(in_range, ExtractU64(shifted)[0], IR::U32(IR::Value(0u))));
-}
-
 void Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
 	const auto shift  = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -129,8 +129,9 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
 	// Vector loads use GLC/DLC to bypass L0/GL1; atomics use GLC only to return data.
 	const bool buffer_atomic = decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP &&
 	                           decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX;
-	memory.coherent = memory.kind == ResourceKind::Buffer && !buffer_atomic &&
-	                  (decoded.glc || decoded.dlc);
+	memory.coherent = (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::Flat ||
+	                   memory.kind == ResourceKind::Global) &&
+	                  !buffer_atomic && (decoded.glc || decoded.dlc);
 	memory.resource      = ResourceIndexFromOperand(decoded.src1);
 	memory.sampler       = ResourceIndexFromOperand(decoded.src2);
 	if (memory.kind == ResourceKind::ScalarBuffer) {
@@ -186,7 +187,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 	if (decoded.family == Decoder::Family::MUBUF || decoded.family == Decoder::Family::MTBUF) {
 		const bool store_or_atomic =
 		    (decoded.opcode >= Decoder::Opcode::BUFFER_STORE_FORMAT_X &&
-		     decoded.opcode <= Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW) ||
+		     decoded.opcode <= Decoder::Opcode::BUFFER_STORE_FORMAT_D16_X) ||
 		    (decoded.opcode >= Decoder::Opcode::BUFFER_STORE_BYTE &&
 		     decoded.opcode <= Decoder::Opcode::BUFFER_STORE_DWORDX4) ||
 		    (decoded.opcode >= Decoder::Opcode::TBUFFER_STORE_FORMAT_X &&
@@ -239,9 +240,6 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 			case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
 			case Decoder::Opcode::DS_WRITE_ADDTID_B32:
 				return index == 0u ? decoded.src1 : MakeM0Operand();
-			case Decoder::Opcode::DS_MIN_F32:
-			case Decoder::Opcode::DS_MAX_F32:
-				return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
 			case Decoder::Opcode::DS_WRITE_B8:
 			case Decoder::Opcode::DS_WRITE_B16:
 			case Decoder::Opcode::DS_WRITE_B8_D16_HI:
@@ -255,12 +253,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 			case Decoder::Opcode::DS_WRITE2_B64:
 			case Decoder::Opcode::DS_WRITE2ST64_B64:
 				return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
-			default:
-				if (decoded.opcode >= Decoder::Opcode::DS_ADD_U32 &&
-				    decoded.opcode <= Decoder::Opcode::DS_WRXCHG_RTN_B32) {
-					return index == 0u ? decoded.src1 : decoded.src0;
-				}
-				return decoded.src0;
+			default: return decoded.src0;
 		}
 	}
 	return DecodedSourceAt(decoded, index);
@@ -434,7 +427,7 @@ void Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
 void Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
 	const auto      memory = MemoryInfoFromDecoded(inst);
 	IR::ValueOpcode opcode;
-	const auto      bits = memory.data_bits;
+	const auto      bits = memory.formatted ? 32u : memory.data_bits;
 	const auto      sign = memory.data_signed;
 	switch (bits) {
 		case 8u: opcode = IR::ValueOpcode::LoadBufferU8; break;
@@ -480,7 +473,8 @@ void Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
 	const auto      data     = ReadU32(data_src);
 	IR::ValueOpcode opcode;
 	IR::Value       value;
-	switch (memory.data_bits) {
+	// Formatted stores carry packed VGPR bits; the format determines their conversion.
+	switch (memory.formatted ? 32u : memory.data_bits) {
 		case 8u:
 			opcode = IR::ValueOpcode::StoreBufferU8;
 			value  = NarrowSubdword(data, 8u);
@@ -554,7 +548,10 @@ void Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	}
 }
 
-void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32,
+                               IR::ValueOpcode opcode64) {
+	const auto opcode = inst.data_bits == 64u ? opcode64 : opcode32;
+	EXIT_IF(opcode == IR::ValueOpcode::Count);
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
@@ -580,12 +577,13 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                            bool returns_value) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
-	const auto address = ReadU32(MemorySourceAt(inst, 1));
-	const auto data_src = MemorySourceAt(inst, 0);
+	const auto address = ReadU32(inst.src0);
 	const IR::Value value =
-	    memory.data_dwords == 2u ? IR::Value(ReadU64(data_src)) : IR::Value(ReadU32(data_src));
-	const auto result  = ir.Emit(opcode, {address, value, ir.GetExec()},
-	                             AddMemoryInfo(memory, inst.pc));
+	    memory.data_dwords == 2u ? IR::Value(ReadU64(inst.src1)) : IR::Value(ReadU32(inst.src1));
+	const auto flags = AddMemoryInfo(memory, inst.pc);
+	const auto result = inst.src_count == 3u
+	                        ? ir.Emit(opcode, {address, value, ReadU32(inst.src2), ir.GetExec()}, flags)
+	                        : ir.Emit(opcode, {address, value, ir.GetExec()}, flags);
 	if (returns_value) {
 		WriteOperand(inst.dst, result);
 	}
@@ -857,14 +855,6 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
 	}
 }
 
-void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
-	const auto memory = MemoryInfoFromDecoded(inst);
-	ir.Emit(opcode,
-	        {ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)),
-	         ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()},
-	        AddMemoryInfo(memory, inst.pc));
-}
-
 void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	WriteOperand(inst.dst, ir.Emit(opcode,
@@ -894,9 +884,54 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
 	                                ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()}));
 }
 
-void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
+void Translator::DS_ORDERED_COUNT(const Decoder::Instruction& inst) {
+	// Approximate ordered counting with one GDS atomic per wave. Wave-launch ordering and
+	// release/done synchronization are not emulated.
+	const auto memory = MemoryInfoFromDecoded(inst);
+	const auto m0 = ir.GetM0();
+	auto address = ir.BitwiseAnd(ir.ShiftRightLogical(m0, IR::U32(IR::Value(16u))),
+	                             IR::U32(IR::Value(0xfffcu)));
+	if (((inst.secondary_offset >> 2u) & 3u) == 1u) {
+		const auto packer = ir.BitwiseAnd(m0, IR::U32(IR::Value(0xfu)));
+		address = ir.IAdd(address, ir.ShiftLeftLogical(packer, IR::U32(IR::Value(2u))));
+	}
+	const auto exec   = ir.GetExec();
+	const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {exec});
+	const auto low = ir.CompositeExtract(ballot, 0u);
+	const auto high = ir.CompositeExtract(ballot, 1u);
+	const auto active = ir.INotEqual(ir.BitwiseOr(low, high), IR::U32(IR::Value(0u)));
+	const auto first = ir.Select(
+	    ir.INotEqual(low, IR::U32(IR::Value(0u))),
+	    IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {low})),
+	    ir.IAdd(IR::U32(IR::Value(32u)), IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {high}))));
+	// An empty EXEC still returns the counter. Elect lane zero and perform an add of zero.
+	const auto selected = ir.Select(active, first, IR::U32(IR::Value(0u)));
+	const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
+	const auto is_first = ir.IEqual(lane, selected);
+	const auto source = IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane,
+	                                   {ReadU32(inst.src0), is_first}));
+	const auto value = ir.Select(active, source, IR::U32(IR::Value(0u)));
+	const auto flags = AddMemoryInfo(memory, inst.pc);
+	IR::U32 result;
+	if (((inst.secondary_offset >> 4u) & 3u) == 1u) {
+		const auto swapped = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicSwap32,
+		    {address, value, ir.LogicalAnd(is_first, active)}, flags));
+		const auto unchanged = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32,
+		    {address, IR::Value(0u), ir.LogicalAnd(is_first, ir.LogicalNot(active))}, flags));
+		result = ir.Select(active, swapped, unchanged);
+	} else {
+		result = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32,
+		                        {address, value, is_first}, flags));
+	}
+	// DS_ORDERED_COUNT writes every destination lane, regardless of EXEC.
+	ir.SetVectorReg(static_cast<IR::VectorReg>(inst.dst.reg),
+	    IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane, {result, is_first})));
+}
+
+void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) {
 	const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset)));
-	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32,
+	WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32
+	                                       : IR::ValueOpcode::PermuteU32,
 	                               {ReadU32(inst.src1), address, ir.GetExec()}));
 }
 
@@ -904,7 +939,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
 		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
 			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
-			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()},
+			    inst.image_address_components - 10u);
 			for (uint32_t component = 0; component < 4; ++component) {
 				WriteOperand(OffsetOperand(inst.dst, component),
 				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
@@ -937,6 +973,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::BUFFER_LOAD_SBYTE:
 		case Decoder::Opcode::BUFFER_LOAD_USHORT:
 		case Decoder::Opcode::BUFFER_LOAD_SSHORT:
+		case Decoder::Opcode::BUFFER_LOAD_SHORT_D16:
+		case Decoder::Opcode::BUFFER_LOAD_SHORT_D16_HI:
 		case Decoder::Opcode::BUFFER_LOAD_DWORD:
 		case Decoder::Opcode::BUFFER_LOAD_DWORDX2:
 		case Decoder::Opcode::BUFFER_LOAD_DWORDX3:
@@ -945,6 +983,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY:
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ:
 		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_D16_X:
 		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_X:
 		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XY:
 		case Decoder::Opcode::TBUFFER_LOAD_FORMAT_XYZ:
@@ -960,6 +999,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_XY:
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZ:
 		case Decoder::Opcode::BUFFER_STORE_FORMAT_XYZW:
+		case Decoder::Opcode::BUFFER_STORE_FORMAT_D16_X:
 		case Decoder::Opcode::TBUFFER_STORE_FORMAT_X:
 		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XY:
 		case Decoder::Opcode::TBUFFER_STORE_FORMAT_XYZ:
@@ -1046,30 +1086,38 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, false);
 		case Decoder::Opcode::DS_XOR_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, true);
+		case Decoder::Opcode::DS_MSKOR_B32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicMaskedOr32, false);
 		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
 
 		case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_SWAP:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32,
+			                    IR::ValueOpcode::ImageAtomicSwap64);
 		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32,
+			                    IR::ValueOpcode::ImageAtomicIAdd64);
 		case Decoder::Opcode::IMAGE_ATOMIC_SMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32,
+			                    IR::ValueOpcode::ImageAtomicUMin64);
 		case Decoder::Opcode::IMAGE_ATOMIC_SMAX:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
-			return IMAGE_ATOMIC(inst, inst.data_bits == 64u ? IR::ValueOpcode::ImageAtomicUMax64
-			                                                : IR::ValueOpcode::ImageAtomicUMax32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32,
+			                    IR::ValueOpcode::ImageAtomicUMax64);
 		case Decoder::Opcode::IMAGE_ATOMIC_AND:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32,
+			                    IR::ValueOpcode::ImageAtomicAnd64);
 		case Decoder::Opcode::IMAGE_ATOMIC_OR:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32,
+			                    IR::ValueOpcode::ImageAtomicOr64);
 		case Decoder::Opcode::IMAGE_ATOMIC_XOR:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32,
+			                    IR::ValueOpcode::ImageAtomicXor64);
 		case Decoder::Opcode::IMAGE_ATOMIC_FMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicFMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_FMAX:
@@ -1079,6 +1127,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::FLAT_LOAD_SBYTE:
 		case Decoder::Opcode::FLAT_LOAD_USHORT:
 		case Decoder::Opcode::FLAT_LOAD_SSHORT:
+		case Decoder::Opcode::FLAT_LOAD_SHORT_D16:
 		case Decoder::Opcode::FLAT_LOAD_DWORD:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX2:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX3:
@@ -1108,11 +1157,13 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst);
 
 		case Decoder::Opcode::DS_MIN_F32:
-			return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMin32);
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false);
 		case Decoder::Opcode::DS_MAX_F32:
-			return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32);
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMax32, false);
 		case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst);
-		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_BPERMUTE_B32(inst);
+		case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false);
+		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true);
+		case Decoder::Opcode::DS_ORDERED_COUNT: return DS_ORDERED_COUNT(inst);
 		case Decoder::Opcode::DS_CONSUME:
 			return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume);
 		case Decoder::Opcode::DS_APPEND:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -10,15 +10,24 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_MOV_B64: S_MOV_B64(inst); return;
 		case O::S_WQM_B32: S_WQM(inst, false); return;
 		case O::S_WQM_B64: S_WQM(inst, true); return;
-		case O::S_GETPC_B64: S_GETPC_B64(inst); return;
+		case O::S_GETPC_B64:
+		case O::S_SWAPPC_B64: S_GETPC_B64(inst); return;
 		case O::S_SETPC_B64: return;
 		case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return;
 		case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return;
 		case O::S_CSELECT_B32: S_CSELECT_B32(inst); return;
 		case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return;
 		case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return;
+		case O::S_GETREG_B32:
+			if (!graphics_compute || (inst.src0.value != 0x1818u && inst.src0.value != 0x0a18u))
+				EXIT("unsupported S_GETREG_B32 at pc 0x%08x: selector=0x%04x graphics_compute=%u",
+				     inst.pc, inst.src0.value, static_cast<unsigned>(graphics_compute));
+			// HW_ID2 identifies graphics compute as QUEUE=6, ME=0.
+			WriteOperand(inst.dst, IR::Value(inst.src0.value == 0x1818u ? 6u : 0u));
+			return;
 		case O::S_SETREG_B32: EmitControlNop(); return;
-		case O::S_WAITCNT: EmitWaitcnt(); return;
+		case O::S_WAITCNT_VSCNT: S_WAITCNT_VSCNT(inst); return;
+		case O::S_WAITCNT: return;
 
 		case O::S_AND_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
@@ -133,6 +142,14 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 
 		case O::S_ABS_I32:
 			return SimpleInteger(inst, IR::ValueOpcode::IAbs32, IR::Type::U32, false, false, true);
+		case O::S_SEXT_I32_I8:
+		case O::S_SEXT_I32_I16: {
+			// Sign extension leaves SCC unchanged.
+			const auto bits = inst.opcode == O::S_SEXT_I32_I8 ? 8u : 16u;
+			WriteOperand(inst.dst, IR::U32(ir.Emit(IR::ValueOpcode::BitFieldSExtract,
+			    {ReadU32(inst.src0), IR::Value(0u), IR::Value(bits)})));
+			return;
+		}
 		case O::S_MUL_I32:
 		case O::S_MULK_I32:
 			return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false);
@@ -164,9 +181,16 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BCNT1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
 			                     true);
-		case O::S_BCNT1_I32_B64:
-			return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false,
-			                     true);
+		case O::S_BCNT1_I32_B64: {
+			// Vulkan bit counts operate on 32-bit words; avoid packing only to split again.
+			const auto value  = ReadU32Pair(inst.src0);
+			const auto low    = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]}));
+			const auto high   = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]}));
+			const auto result = ir.IAdd(low, high);
+			WriteOperand(inst.dst, result);
+			ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
+			return;
+		}
 		case O::S_FF1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false,
 			                     false);
@@ -224,7 +248,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_SLEEP:
 		case O::S_SETPRIO:
 		case O::S_TRAP: EmitControlNop(); return;
-		case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return;
+		case O::S_WAITCNT_DEPCTR: return;
 		case O::S_BARRIER: S_BARRIER(); return;
 		case O::S_SENDMSG: S_SENDMSG(inst); return;
 		case O::S_TTRACEDATA: S_TTRACEDATA(); return;

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -503,41 +503,11 @@ IR::U64 Translator::ReadU64(const Decoder::Operand& operand) {
 	return IR::U64(ReadOperand(operand, IR::Type::U64));
 }
 
-IR::F32 Translator::ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed) {
-	if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
-		const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel);
-		auto       value    = use_zero ? IR::F32(IR::Value::F32(0.0f))
-		                               : ir.BitCastF32(IR::U32(IR::Value(operand.value)));
-		const auto half     = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
-		value               = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
-		if (operand.absolute) {
-			value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
-		}
-		if (high_lane ? operand.negate_hi : operand.negate) {
-			value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
-		}
-		return value;
-	}
-	auto raw_operand      = operand;
-	raw_operand.sdwa_sel  = 6;
-	raw_operand.sdwa_sext = false;
-	const auto bits       = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand));
-	uint32_t   offset     = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
-	if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
-		offset = operand.sdwa_sel == 5u ? 16u : 0u;
-	}
-	const auto half_u32 = IR::U32(
-	    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(16u)}));
-	const auto half_u16 = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, {half_u32}));
+IR::F32 Translator::ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane) {
+	const auto bits     = Read16LaneBits(operand, high_lane);
+	const auto half_u16 = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, {bits}));
 	const auto half     = IR::F16(ir.Emit(IR::ValueOpcode::BitCastF16U16, {half_u16}));
-	auto       value    = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
-	if (operand.absolute) {
-		value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
-	}
-	if (high_lane ? operand.negate_hi : operand.negate) {
-		value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
-	}
-	return value;
+	return IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
 }
 
 IR::F32 Translator::ReadF16AsF32(const Decoder::Operand& operand) {
@@ -555,29 +525,6 @@ IR::F32 Translator::ReadMixF32(const Decoder::Operand& operand) {
 	return IR::F32(ReadOperand(value_operand, IR::Type::F32));
 }
 
-IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane) {
-	auto raw_operand      = operand;
-	raw_operand.sdwa_sel  = 6;
-	raw_operand.sdwa_sext = false;
-	const auto bits       = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand));
-	uint32_t   offset     = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
-	uint32_t   width      = 16u;
-	if (operand.sdwa_sel <= 3u) {
-		offset = operand.sdwa_sel * 8u;
-		width  = 8u;
-	} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
-		offset = operand.sdwa_sel == 5u ? 16u : 0u;
-	}
-	if (width == 8u && operand.sdwa_sext) {
-		// Extend the selected byte into the 16-bit operand before halfword modifiers.
-		const auto value = IR::U32(
-		    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {bits, IR::Value(offset), IR::Value(width)}));
-		return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
-	}
-	return IR::U32(
-	    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)}));
-}
-
 IR::U32 Translator::ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane,
                                      bool sign_extend) {
 	auto value = Read16LaneBits(operand, high_lane);
@@ -593,12 +540,36 @@ IR::U32 Translator::ReadU16AsU32(const Decoder::Operand& operand, bool sign_exte
 }
 
 IR::U32 Translator::Read16LaneBits(const Decoder::Operand& operand, bool high_lane) {
-	auto value = ReadU16LaneRaw(operand, high_lane);
+	auto raw_operand      = operand;
+	raw_operand.sdwa_sel  = 6;
+	raw_operand.sdwa_sext = false;
+	auto bits             = ReadRawU32(operand);
+	if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
+		// Inline floats encode the operand's width before SDWA or OP_SEL selects bits.
+		const auto half      = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {ir.BitCastF32(bits)}));
+		const auto half_bits = IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {half}));
+		bits                = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {half_bits}));
+	}
+	bits = ApplyBitSourceModifiers(raw_operand, bits);
+	uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
+	uint32_t width  = 16u;
+	if (operand.sdwa_sel <= 3u) {
+		offset = operand.sdwa_sel * 8u;
+		width  = 8u;
+	} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
+		offset = operand.sdwa_sel == 5u ? 16u : 0u;
+	}
+	const auto extract = width == 8u && operand.sdwa_sext
+	                         ? IR::ValueOpcode::BitFieldSExtract
+	                         : IR::ValueOpcode::BitFieldUExtract;
+	auto value = IR::U32(ir.Emit(extract, {bits, IR::Value(offset), IR::Value(width)}));
+	if (width == 8u && operand.sdwa_sext) {
+		value = ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
+	}
 	if (operand.absolute) {
 		value = ir.BitwiseAnd(value, IR::U32(IR::Value(0x7fffu)));
 	}
 	if (high_lane ? operand.negate_hi : operand.negate) {
-		// RDNA2 source NEG flips the sign bit, including packed integer operands.
 		value = ir.BitwiseXor(value, IR::U32(IR::Value(0x8000u)));
 	}
 	return value;
@@ -764,7 +735,7 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
 }
 
 void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
-                                    IR::BlockInfo& info) {
+                                    IR::Block& info) {
 	const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 {
 		IR::U1 condition;
 		switch (kind) {
@@ -895,8 +866,14 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_ADD_F64:
+		case Decoder::Opcode::V_MIN_F64:
+		case Decoder::Opcode::V_MAX_F64:
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
-		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
+		case Decoder::Opcode::V_RCP_F64:
+		case Decoder::Opcode::V_TRUNC_F64:
+		case Decoder::Opcode::V_CEIL_F64:
+		case Decoder::Opcode::V_FLOOR_F64:
+		case Decoder::Opcode::V_FRACT_F64: include_vector(inst.dst, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
 		case Decoder::Opcode::V_CMP_EQ_F64:
 		case Decoder::Opcode::V_CMP_LE_F64:
@@ -905,15 +882,23 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CMP_EQ_I64:
 		case Decoder::Opcode::V_CMP_LT_I64:
 		case Decoder::Opcode::V_CMP_LE_I64:
+		case Decoder::Opcode::V_CMP_NE_I64:
 		case Decoder::Opcode::V_CMP_LT_U64:
 		case Decoder::Opcode::V_CMP_EQ_U64:
 		case Decoder::Opcode::V_CMP_LE_U64:
 		case Decoder::Opcode::V_CMP_GT_U64:
 		case Decoder::Opcode::V_CMP_NE_U64:
 		case Decoder::Opcode::V_CMP_GE_U64:
+		case Decoder::Opcode::V_CMPX_LT_I64:
+		case Decoder::Opcode::V_CMPX_EQ_I64:
+		case Decoder::Opcode::V_CMPX_LE_I64:
 		case Decoder::Opcode::V_CMPX_NE_I64:
+		case Decoder::Opcode::V_CMPX_LT_U64:
+		case Decoder::Opcode::V_CMPX_EQ_U64:
 		case Decoder::Opcode::V_CMPX_LE_U64:
+		case Decoder::Opcode::V_CMPX_GT_U64:
 		case Decoder::Opcode::V_CMPX_NE_U64:
+		case Decoder::Opcode::V_CMPX_GE_U64:
 			include_vector(inst.src0, 2u);
 			include_vector(inst.src1, 2u);
 			break;
@@ -1038,45 +1023,67 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 
 	result.block_storage.reserve(cfg.blocks.size() + 1u);
 	result.blocks.reserve(cfg.blocks.size() + 1u);
-	result.block_info.reserve(cfg.blocks.size() + 1u);
 	const auto max_id = std::ranges::max_element(cfg.blocks, {}, [](const CFG::BasicBlock& block) {
 		                    return block.id;
 	                    })->id;
 	if (max_id == UINT32_MAX) {
 		EXIT("cannot allocate a typed entry block id");
 	}
-	CFG::Terminator terminator;
-	terminator.kind       = CFG::TerminatorKind::Branch;
-	terminator.true_block = cfg.blocks.front().id;
 	result.block_storage.push_back(std::make_unique<IR::Block>());
 	result.blocks.push_back(result.block_storage.back().get());
-	result.block_info.push_back({max_id + 1u, cfg.blocks.front().start_pc,
-	                             cfg.blocks.front().start_pc, std::move(terminator)});
+	auto& entry    = *result.blocks.front();
+	entry.id       = max_id + 1u;
+	entry.start_pc = entry.end_pc = cfg.blocks.front().start_pc;
+	entry.terminator.kind         = CFG::TerminatorKind::Branch;
 
-	std::unordered_map<uint32_t, size_t> block_indices;
-	block_indices.reserve(cfg.blocks.size());
-	for (const auto& source_block: cfg.blocks) {
-		if (!block_indices.emplace(source_block.id, result.blocks.size()).second) {
-			EXIT("CFG contains duplicate block id %u", source_block.id);
+	std::unordered_map<uint32_t, IR::Block*> blocks_by_id;
+	blocks_by_id.reserve(cfg.blocks.size());
+	for (const auto& source: cfg.blocks) {
+		auto& block = *result.block_storage.emplace_back(std::make_unique<IR::Block>());
+		if (!blocks_by_id.emplace(source.id, &block).second) {
+			EXIT("CFG contains duplicate block id %u", source.id);
 		}
-		result.block_storage.push_back(std::make_unique<IR::Block>());
-		result.blocks.push_back(result.block_storage.back().get());
-		result.block_info.push_back(
-		    {source_block.id, source_block.start_pc, source_block.end_pc, source_block.terminator});
+		result.blocks.push_back(&block);
+		block.id       = source.id;
+		block.start_pc = source.start_pc;
+		block.end_pc   = source.end_pc;
 	}
-	for (const auto& source_block: cfg.blocks) {
-		const auto source_index = block_indices.at(source_block.id);
-		for (const auto successor: source_block.successors) {
-			const auto target = block_indices.find(successor);
-			if (target == block_indices.end()) {
-				EXIT("CFG block %u has unknown successor %u", source_block.id, successor);
+	const auto target_block = [&](uint32_t id) -> IR::Block* {
+		return id == UINT32_MAX ? nullptr : blocks_by_id.at(id);
+	};
+	entry.terminator.true_block = target_block(cfg.blocks.front().id);
+	for (const auto& source: cfg.blocks) {
+		auto&       block               = *blocks_by_id.at(source.id);
+		const auto& term                = source.terminator;
+		block.terminator.kind           = term.kind;
+		block.terminator.true_block     = target_block(term.true_block);
+		block.terminator.false_block    = target_block(term.false_block);
+		block.terminator.merge_block    = target_block(term.merge_block);
+		block.terminator.continue_block = target_block(term.continue_block);
+		block.terminator.loop_header    = term.loop_header;
+		block.terminator.indexed        = term.indirect_selector_code != UINT32_MAX;
+		const auto& values =
+		    block.terminator.indexed ? term.indirect_selector_values : term.indirect_target_pcs;
+		const auto& targets =
+		    block.terminator.indexed ? term.indirect_selector_targets : term.indirect_targets;
+		if (values.size() != targets.size()) {
+			EXIT("CFG block %u has inconsistent indirect targets", source.id);
+		}
+		block.terminator.cases.reserve(values.size());
+		for (size_t index = 0; index < values.size(); ++index) {
+			block.terminator.cases.push_back({values[index], target_block(targets[index])});
+		}
+		for (const auto successor: source.successors) {
+			const auto target = blocks_by_id.find(successor);
+			if (target == blocks_by_id.end()) {
+				EXIT("CFG block %u has unknown successor %u", source.id, successor);
 			}
-			result.blocks[source_index]->AddBranch(result.blocks[target->second]);
+			block.AddBranch(target->second);
 		}
 	}
 	{
-		result.blocks.front()->AddBranch(result.blocks.at(block_indices.at(cfg.blocks.front().id)));
-		IR::IREmitter entry_ir(result.blocks.front());
+		entry.AddBranch(entry.terminator.true_block);
+		IR::IREmitter entry_ir(&entry);
 		const auto    builtin = [&](IR::StageInputKind kind, uint32_t component = 0u) {
 			return IR::U32(
 			    entry_ir.Emit(IR::ValueOpcode::GetBuiltin,
@@ -1104,6 +1111,16 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				                                  IR::U32(IR::Value(total_threads)));
 			}
 		}
+		if (options.stage == ShaderType::Compute &&
+		    options.input_info.compute->dispatch_thread_dimensions) {
+			for (uint32_t axis = 0; axis < 3u; axis++) {
+				const auto extent = IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::GetDispatchThreadExtent, {IR::Value(axis)}));
+				initial_exec = entry_ir.LogicalAnd(
+				    initial_exec,
+				    entry_ir.ULessThan(builtin(IR::StageInputKind::GlobalInvocationId, axis), extent));
+			}
+		}
 		entry_ir.SetExec(initial_exec);
 		const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec});
 		entry_ir.SetExecLo(entry_ir.CompositeExtract(initial_mask, 0));
@@ -1142,7 +1159,8 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			}
 		} else if (options.stage == ShaderType::Mesh) {
 			const auto& mesh = options.input_info.vertex->mesh;
-			EXIT_NOT_IMPLEMENTED(mesh.primitives_per_group == 0u || mesh.vertices_per_group > 64u ||
+			EXIT_NOT_IMPLEMENTED(mesh.primitives_per_group == 0u || mesh.vertices_per_group > 0x1ffu ||
+			                     mesh.primitives_per_group > 0x1ffu ||
 			                     total_threads > 15u * options.wave_size);
 			const auto u32  = [](uint32_t value) { return IR::U32(IR::Value(value)); };
 			const auto draw = [&](uint32_t index) {
@@ -1178,6 +1196,10 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			const auto wave_info = entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(wave, u32(24)),
 			                                          u32(((total_threads + options.wave_size - 1u) /
 			                                               options.wave_size) << 28u));
+			entry_ir.SetScalarReg(
+			    static_cast<IR::ScalarReg>(2),
+			    entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(vertices, u32(12)),
+			                       entry_ir.ShiftLeftLogical(primitives, u32(22))));
 			entry_ir.SetScalarReg(
 			    static_cast<IR::ScalarReg>(3),
 			    entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(
@@ -1286,6 +1308,8 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				}
 			};
 			barycentric_pair(ps->ps_perspective_center_vgpr, IR::StageInputKind::BaryCoordSmooth);
+			barycentric_pair(ps->ps_perspective_sample_vgpr,
+			                 IR::StageInputKind::BaryCoordSmoothSample);
 			barycentric_pair(ps->ps_perspective_centroid_vgpr,
 			                 IR::StageInputKind::BaryCoordSmoothCentroid);
 			uint32_t reg = ps->ps_system_input_base;
@@ -1329,10 +1353,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 		}
 	}
 	const bool flush_f32_inputs = options.stage == ShaderType::Compute &&
-	                             (options.input_info.compute->float_mode & 0x10u) == 0;
+	                              (options.input_info.compute->float_mode & 0x10u) == 0;
 	for (const auto& cfg_block: cfg.blocks) {
-		const auto typed_index = block_indices.at(cfg_block.id);
-		Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs);
+		auto*      block = blocks_by_id.at(cfg_block.id);
+		Translator translator(result, block, vector_limit, flush_f32_inputs,
+		    options.stage == ShaderType::Compute && !options.input_info.compute->async_compute);
 		for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
 			const auto& instruction = decoded.instructions[index];
 			if (IsCodeTableLoad(cfg, instruction.pc)) {
@@ -1355,7 +1380,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			}
 			translator.TranslateInstruction(instruction);
 		}
-		translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]);
+		translator.AddBranchCondition(cfg, cfg_block, *block);
 	}
 	IR::ValidateProgram(result, false);
 	return result;

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -10,14 +10,15 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
 class Translator {
 public:
 	Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit,
-	           bool flush_f32_inputs)
+	           bool flush_f32_inputs, bool graphics_compute)
 	    : program(program), ir(block), current_vector_limit(vector_limit),
-	      flush_f32_inputs(flush_f32_inputs) {}
+	      flush_f32_inputs(flush_f32_inputs), graphics_compute(graphics_compute) {}
 
 	void TranslateInstruction(const Decoder::Instruction& inst);
 	void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
 	                            uint32_t component_count, const ShaderBufferResource& resource);
-	void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info);
+	void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
+	                        IR::Block& info);
 
 private:
 	const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index);
@@ -40,10 +41,9 @@ private:
 	IR::U32                ReadU32(const Decoder::Operand& operand);
 	std::array<IR::U32, 2> ReadU32Pair(const Decoder::Operand& operand);
 	IR::U64                ReadU64(const Decoder::Operand& operand);
-	IR::F32 ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed = false);
+	IR::F32 ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane);
 	IR::F32 ReadF16AsF32(const Decoder::Operand& operand);
 	IR::F32 ReadMixF32(const Decoder::Operand& operand);
-	IR::U32 ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane);
 	IR::U32 ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane, bool sign_extend);
 	IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend);
 	IR::U32 Read16LaneBits(const Decoder::Operand& operand, bool high_lane);
@@ -86,7 +86,8 @@ private:
 	void          BUFFER_LOAD(const Decoder::Instruction& inst);
 	void          BUFFER_STORE(const Decoder::Instruction& inst);
 	void          BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32,
+	                           IR::ValueOpcode opcode64 = IR::ValueOpcode::Count);
 	void DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
 	void FLAT_LOAD(const Decoder::Instruction& inst);
 	void FLAT_STORE(const Decoder::Instruction& inst);
@@ -105,11 +106,11 @@ private:
 	void DS_READ2(const Decoder::Instruction& inst);
 	void DS_WRITE(const Decoder::Instruction& inst);
 	void DS_WRITE2(const Decoder::Instruction& inst);
-	void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void DS_ADDTID(const Decoder::Instruction& inst, bool write);
 	void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
-	void DS_BPERMUTE_B32(const Decoder::Instruction& inst);
+	void DS_PERMUTE(const Decoder::Instruction& inst, bool backward);
+	void DS_ORDERED_COUNT(const Decoder::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -127,7 +128,7 @@ private:
 	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
 	                      bool cmpx);
 	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx);
-	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx);
+	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half = false);
 	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
 	void V_CVT_F32_U32(const Decoder::Instruction& inst);
 	void V_CVT_F32_I32(const Decoder::Instruction& inst);
@@ -166,8 +167,8 @@ private:
 	void FloatCube(const Decoder::Instruction& inst, uint32_t result_kind);
 	void Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
 	void Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
+	void Integer16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
 	void V_MAD_I16(const Decoder::Instruction& inst);
-	void V_MED3_I16(const Decoder::Instruction& inst);
 	void PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
 	                          bool arithmetic);
 	void PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
@@ -208,8 +209,7 @@ private:
 	void    V_BFI_B32(const Decoder::Instruction& inst);
 	void    S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
 	void    S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
-	void    V_ALIGNBIT_B32(const Decoder::Instruction& inst);
-	void    V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
+	void    V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset);
 	void    V_LSHL_ADD_U32(const Decoder::Instruction& inst);
 	void    V_ADD_LSHL_U32(const Decoder::Instruction& inst);
 	void    V_XAD_U32(const Decoder::Instruction& inst);
@@ -229,7 +229,7 @@ private:
 	void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode,
 	                    IR::ValueOpcode compare_opcode);
 	void EmitControlNop();
-	void EmitWaitcnt();
+	void S_WAITCNT_VSCNT(const Decoder::Instruction& inst);
 	void S_BARRIER();
 	void S_SENDMSG(const Decoder::Instruction& inst);
 	void S_TTRACEDATA();
@@ -264,6 +264,7 @@ private:
 	uint32_t        current_pc           = 0;
 	uint32_t        current_vector_limit = 1;
 	bool            flush_f32_inputs;
+	bool            graphics_compute;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -96,16 +96,35 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMPX_LE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true);
 			return;
+		case O::V_CMP_F_I64:
+		case O::V_CMP_F_U64: EmitCompareConstant(inst, false, false, false); return;
+		case O::V_CMP_T_I64:
+		case O::V_CMP_T_U64: EmitCompareConstant(inst, true, false, false); return;
+		case O::V_CMPX_F_I64:
+		case O::V_CMPX_F_U64: EmitCompareConstant(inst, false, false, true); return;
+		case O::V_CMPX_T_I64:
+		case O::V_CMPX_T_U64: EmitCompareConstant(inst, true, false, true); return;
 		case O::V_CMP_EQ_I64:
 		case O::V_CMP_EQ_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMPX_EQ_I64:
+		case O::V_CMPX_EQ_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, true);
+			return;
 		case O::V_CMP_LT_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMPX_LT_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, true);
+			return;
 		case O::V_CMP_GT_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMPX_GT_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, true);
+			return;
+		case O::V_CMP_NE_I64:
 		case O::V_CMP_NE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
 			return;
@@ -117,9 +136,15 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_LT_I64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMPX_LT_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true);
+			return;
 		case O::V_CMP_LE_I64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMPX_LE_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true);
+			return;
 		case O::V_CMP_LE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false);
 			return;
@@ -129,6 +154,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_GE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMPX_GE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, true);
+			return;
 
 		case O::V_CMP_EQ_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
@@ -142,6 +170,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_NE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
 			return;
+		case O::V_CMPX_NE_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, true);
+			return;
 		case O::V_CMP_NE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
 			return;
@@ -340,8 +371,11 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true, false); return;
 		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false, false); return;
 		case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true, true); return;
+		case O::V_CMPX_U_F32: EmitFloatOrderedCompare(inst, false, true); return;
 		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return;
 		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return;
+		case O::V_CMP_CLASS_F16: EmitFloatClassCompare(inst, false, true); return;
+		case O::V_CMPX_CLASS_F16: EmitFloatClassCompare(inst, true, true); return;
 
 		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return;
 		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return;
@@ -389,7 +423,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_SUB_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::ISub32, false);
 		case O::V_MUL_LO_U16: return Integer16Binary(inst, IR::ValueOpcode::IMul32, false);
 		case O::V_MAD_I16: return V_MAD_I16(inst);
-		case O::V_MED3_I16: return V_MED3_I16(inst);
+		case O::V_MED3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMedTri32, true);
+		case O::V_MIN3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMinTri32, false);
 		case O::V_MIN_I16: return Integer16Binary(inst, IR::ValueOpcode::SMin32, true);
 		case O::V_MAX_I16: return Integer16Binary(inst, IR::ValueOpcode::SMax32, true);
 		case O::V_MIN_U16: return Integer16Binary(inst, IR::ValueOpcode::UMin32, false);
@@ -453,10 +488,14 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_RCP_F64: return FloatUnary(inst, IR::ValueOpcode::FPRecip64);
 		case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32);
 		case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32);
+		case O::V_FRACT_F64: return FloatUnary(inst, IR::ValueOpcode::FPFract64);
 		case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32);
+		case O::V_TRUNC_F64: return FloatUnary(inst, IR::ValueOpcode::FPTrunc64);
 		case O::V_CEIL_F32: return FloatUnary(inst, IR::ValueOpcode::FPCeil32);
+		case O::V_CEIL_F64: return FloatUnary(inst, IR::ValueOpcode::FPCeil64);
 		case O::V_RNDNE_F32: return FloatUnary(inst, IR::ValueOpcode::FPRoundEven32);
 		case O::V_FLOOR_F32: return FloatUnary(inst, IR::ValueOpcode::FPFloor32);
+		case O::V_FLOOR_F64: return FloatUnary(inst, IR::ValueOpcode::FPFloor64);
 		case O::V_EXP_F32: return FloatUnary(inst, IR::ValueOpcode::FPExp2);
 		case O::V_LOG_F32: return FloatUnary(inst, IR::ValueOpcode::FPLog2);
 		case O::V_RSQ_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipSqrt32);
@@ -469,13 +508,18 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
 		case O::V_ADD_F64: return FloatBinary(inst, IR::ValueOpcode::FPAdd64, false);
 		case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false);
+		case O::V_MIN_F64: return FloatBinary(inst, IR::ValueOpcode::FPMin64, false);
+		case O::V_MAX_F64: return FloatBinary(inst, IR::ValueOpcode::FPMax64, false);
 		case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
 		case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);
 		case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false);
-		case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true);
+		case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPMad32, true, true);
 		case O::V_MADMK_F32:
 		case O::V_MADAK_F32:
-		case O::V_MAD_F32:
+		case O::V_MAD_F32: return FloatTernary(inst, IR::ValueOpcode::FPMad32, false, true);
+		case O::V_FMAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true);
+		case O::V_FMAMK_F32:
+		case O::V_FMAAK_F32:
 		case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true);
 		case O::V_FMA_F64: return FloatTernary(inst, IR::ValueOpcode::FPFma64, false, false);
 		case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false);
@@ -591,8 +635,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_BFE_U32: return V_BFE_U32(inst, false);
 		case O::V_BFE_I32: return V_BFE_U32(inst, true);
 		case O::V_BFI_B32: return V_BFI_B32(inst);
-		case O::V_ALIGNBIT_B32: return V_ALIGNBIT_B32(inst);
-		case O::V_ALIGNBYTE_B32: return V_ALIGNBYTE_B32(inst);
+		case O::V_ALIGNBIT_B32: return V_ALIGN_B32(inst, false);
+		case O::V_ALIGNBYTE_B32: return V_ALIGN_B32(inst, true);
 		case O::V_LSHL_ADD_U32: return V_LSHL_ADD_U32(inst);
 		case O::V_ADD_LSHL_U32: return V_ADD_LSHL_U32(inst);
 		case O::V_XAD_U32: return V_XAD_U32(inst);

src/graphics/shader/recompiler/ir/Block.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 #include "graphics/shader/recompiler/ir/Reg.h"
 #include "graphics/shader/recompiler/ir/Value.h"
 
@@ -37,6 +38,8 @@ public:
 	[[nodiscard]] const InstructionList&  Instructions() const;
 	[[nodiscard]] std::span<Block* const> ImmPredecessors() const;
 	[[nodiscard]] std::span<Block* const> ImmSuccessors() const;
+	[[nodiscard]] uint32_t Definition() const { return definition; }
+	void SetDefinition(uint32_t value) const { definition = value; }
 
 	void               SsaSeal();
 	[[nodiscard]] bool IsSsaSealed() const;
@@ -47,6 +50,27 @@ public:
 	const_iterator end() const;
 	bool           empty() const;
 
+	struct SwitchCase {
+		uint32_t value;
+		Block*   target;
+	};
+	struct Terminator {
+		CFG::TerminatorKind     kind           = CFG::TerminatorKind::Return;
+		Block*                  true_block     = nullptr;
+		Block*                  false_block    = nullptr;
+		Block*                  merge_block    = nullptr;
+		Block*                  continue_block = nullptr;
+		std::vector<SwitchCase> cases;
+		bool                    indexed     = false;
+		bool                    loop_header = false;
+	};
+	uint32_t   id       = 0;
+	uint32_t   start_pc = 0;
+	uint32_t   end_pc   = 0;
+	Terminator terminator;
+	Value      condition;
+	Value      indirect_target;
+
 	std::array<Value, NumScalarRegs> ssa_sreg_values {};
 	std::array<Value, NumScalarRegs> ssa_thread_bit_sreg_values {};
 	std::array<Value, NumScalarRegs> ssa_sreg_mask_tags {};
@@ -57,6 +81,7 @@ private:
 	std::vector<Block*> predecessors;
 	std::vector<Block*> successors;
 	bool                ssa_sealed = false;
+	mutable uint32_t    definition = 0;
 };
 
 using BlockList = std::vector<Block*>;

src/graphics/shader/recompiler/ir/Program.cpp

@@ -1,5 +1,6 @@
 #include "common/assert.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/BufferFormat.h"
 
 #include <fmt/format.h>
 #include <map>
@@ -190,6 +191,28 @@ Value ResolveInvariantPhi(const ResourcePlan& program, Value value) {
 	return invariant;
 }
 
+Value ResolveActiveU32(Value value, Value active) {
+	for (uint32_t depth = 0; depth <= 32; ++depth) {
+		value = value.Resolve();
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 ||
+		    inst->Arg(0).Resolve() != active)
+			return value;
+		value = inst->Arg(1);
+	}
+	return {};
+}
+
+uint32_t StorageBufferElementBits(const Program& program, const MemoryInfo& memory) {
+	if (!memory.formatted) return memory.data_bits;
+	const auto format = memory.typed
+	                        ? Format::DecodeTBufferFormat(memory.data_format, memory.number_format)
+	                        : program.info.buffers[memory.resource].descriptor_format;
+	const auto info = Format::GetFormatInfo(format);
+	return info.type == Format::ComponentType::Unknown || info.packed_bitfield
+	           ? 32u : info.component_bits[0];
+}
+
 bool HasShaderMemoryWrites(const Program& program) {
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
@@ -207,12 +230,11 @@ bool HasShaderMemoryWrites(const Program& program) {
 }
 
 void ValidateProgram(const Program& program, bool require_ssa) {
-	if (program.blocks.size() != program.block_info.size() ||
-	    program.blocks.size() != program.block_storage.size()) {
+	if (program.blocks.size() != program.block_storage.size()) {
 		return Fail("value IR block storage is inconsistent");
 	}
 	std::unordered_map<const Block*, size_t>   block_indices;
-	std::unordered_map<uint32_t, const Block*> blocks_by_id;
+	std::unordered_set<uint32_t>               block_ids;
 	std::unordered_map<const Inst*, size_t>    instruction_positions;
 	for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) {
 		const auto* block = program.blocks[block_index];
@@ -223,10 +245,10 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 		if (!block_indices.emplace(block, block_index).second) {
 			return Fail("value IR block pointer is duplicated");
 		}
-		if (program.block_info[block_index].id == UINT32_MAX) {
+		if (block->id == UINT32_MAX) {
 			return Fail("value IR block uses the reserved exit id");
 		}
-		if (!blocks_by_id.emplace(program.block_info[block_index].id, block).second) {
+		if (!block_ids.insert(block->id).second) {
 			return Fail("value IR block id is duplicated");
 		}
 		size_t position = 0;
@@ -271,15 +293,14 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 		}
 
 		std::unordered_set<const Block*> expected_successors;
-		const auto                       add_target = [&](uint32_t id) {
-			const auto found = blocks_by_id.find(id);
-			if (found == blocks_by_id.end()) {
+		const auto                       add_target = [&](const Block* target) {
+			if (!block_indices.contains(target)) {
 				return false;
 			}
-			expected_successors.insert(found->second);
+			expected_successors.insert(target);
 			return true;
 		};
-		const auto& terminator             = program.block_info[block_index].terminator;
+		const auto& terminator             = block->terminator;
 		const auto  validate_control_value = [&](Value value, Type type) {
 			if (value.IsEmpty() || value.GetType() != type) {
 				return false;
@@ -297,44 +318,32 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 				if (!add_target(terminator.true_block) || !add_target(terminator.false_block)) {
 					return Fail("value IR conditional branch target is missing");
 				}
-				if (!validate_control_value(program.block_info[block_index].condition, Type::U1)) {
+				if (!validate_control_value(block->condition, Type::U1)) {
 					return Fail("value IR conditional branch condition is invalid");
 				}
 				break;
 			case CFG::TerminatorKind::IndirectBranch: {
-				if (!validate_control_value(program.block_info[block_index].indirect_target,
-				                            Type::U32)) {
+				if (!validate_control_value(block->indirect_target, Type::U32)) {
 					return Fail("value IR indirect branch selector is invalid");
 				}
-				std::unordered_set<uint32_t> indirect_targets;
-				for (const auto target: terminator.indirect_targets) {
-					if (!indirect_targets.insert(target).second) {
-						return Fail("value IR indirect branch target is duplicated");
+				std::unordered_set<uint32_t> values;
+				for (const auto& branch: terminator.cases) {
+					if (!values.insert(branch.value).second) {
+						return Fail("value IR indirect selector value is duplicated");
 					}
-					if (!add_target(target)) {
+					if (!add_target(branch.target)) {
 						return Fail("value IR indirect branch target is missing");
 					}
 				}
-				if (terminator.indirect_selector_values.size() !=
-				    terminator.indirect_selector_targets.size()) {
-					return Fail("value IR indirect selector table is inconsistent");
-				}
-				for (const auto target: terminator.indirect_selector_targets) {
-					const auto found = blocks_by_id.find(target);
-					if (found == blocks_by_id.end() ||
-					    !expected_successors.contains(found->second)) {
-						return Fail("value IR indirect selector target is not a CFG successor");
-					}
-				}
 				break;
 			}
 			case CFG::TerminatorKind::Return:
 			case CFG::TerminatorKind::Unsupported: break;
 		}
-		if ((terminator.merge_block != UINT32_MAX &&
-		     !blocks_by_id.contains(terminator.merge_block)) ||
-		    (terminator.continue_block != UINT32_MAX &&
-		     !blocks_by_id.contains(terminator.continue_block))) {
+		if ((terminator.merge_block != nullptr &&
+		     !block_indices.contains(terminator.merge_block)) ||
+		    (terminator.continue_block != nullptr &&
+		     !block_indices.contains(terminator.continue_block))) {
 			return Fail("value IR structured control target is missing");
 		}
 		if (successors != expected_successors) {
@@ -399,7 +408,8 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
 				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind != ResourceKind::ScalarBuffer) {
+				if (memory.kind != ResourceKind::ScalarBuffer &&
+				    memory.kind != ResourceKind::IndirectBuffer) {
 					return Fail(fmt::format("{} has an invalid scalar-memory resource kind",
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
@@ -457,7 +467,7 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 				}
 				if (memory.kind == ResourceKind::IndirectBuffer &&
 				    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) {
-					return Fail("indirect buffer requires a raw DWORD x2/x3/x4 load");
+					return Fail("indirect buffer requires a scalar, raw DWORD x1/x2/x3/x4, or formatted X load");
 				}
 				if (buffer_components > 1u &&
 				    (!vector_buffer || memory.data_bits != 32u ||
@@ -640,7 +650,7 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 				}
 			}
 		}
-		const auto& info = program.block_info[block_index];
+		const auto& info = *program.blocks[block_index];
 		if (!info.condition.IsEmpty() && !control_dominates(info.condition, block)) {
 			return Fail("value IR branch condition definition does not dominate its use");
 		}
@@ -651,9 +661,9 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 }
 
 void ResolveControlFlowIdentities(Program& program) {
-	for (auto& info: program.block_info) {
-		info.condition       = info.condition.Resolve();
-		info.indirect_target = info.indirect_target.Resolve();
+	for (auto* block: program.blocks) {
+		block->condition       = block->condition.Resolve();
+		block->indirect_target = block->indirect_target.Resolve();
 	}
 }
 
@@ -688,9 +698,9 @@ std::string ProgramToString(const Program& program) {
 
 	std::string text;
 	for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) {
-		text += fmt::format("Block ${} pc=0x{:08x}..0x{:08x}\n", block_index,
-		                    program.block_info[block_index].start_pc,
-		                    program.block_info[block_index].end_pc);
+		text +=
+		    fmt::format("Block ${} pc=0x{:08x}..0x{:08x}\n", block_index,
+		                program.blocks[block_index]->start_pc, program.blocks[block_index]->end_pc);
 		for (const auto& inst: *program.blocks[block_index]) {
 			const auto type = inst.GetType();
 			if (type != Type::Void) {

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -73,9 +73,13 @@ struct MemoryInfo {
 	bool                    planning_only                                         = false;
 
 	[[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const {
-		return !formatted && !typed && data_bits == 32u &&
-		       (opcode == ValueOpcode::LoadBufferU32x2 || opcode == ValueOpcode::LoadBufferU32x3 ||
-		        opcode == ValueOpcode::LoadBufferU32x4);
+		return !typed && data_bits == 32u &&
+		       (formatted ? opcode == ValueOpcode::LoadBufferU32
+		                  : opcode == ValueOpcode::ReadConstBuffer ||
+		                        opcode == ValueOpcode::LoadBufferU32 ||
+		                        opcode == ValueOpcode::LoadBufferU32x2 ||
+		                        opcode == ValueOpcode::LoadBufferU32x3 ||
+		                        opcode == ValueOpcode::LoadBufferU32x4);
 	}
 
 	bool operator==(const MemoryInfo& other) const = default;
@@ -97,8 +101,10 @@ struct ExportInfo {
 
 struct BufferResource {
 	static constexpr uint32_t NoImageAlias = UINT32_MAX;
+	static constexpr uint32_t NoIndirectBuffer = UINT32_MAX;
 
 	uint32_t               source             = 0;
+	uint32_t               descriptor_index   = UINT32_MAX;
 	uint32_t               first_use_pc       = 0;
 	uint32_t               max_byte_extent    = 0;
 	uint32_t               packed_stride      = 0;
@@ -110,6 +116,10 @@ struct BufferResource {
 	bool                   atomic             = false;
 	bool                   formatted          = false;
 	bool                   scalar             = false;
+	uint32_t               indirect_root              = NoIndirectBuffer;
+	uint32_t               indirect_mapping_offset    = 0;
+	uint32_t               indirect_search_iterations = 0;
+	std::vector<uint32_t>  indirect_resources;
 
 	bool operator==(const BufferResource& other) const = default;
 };
@@ -122,6 +132,7 @@ struct ImageResource {
 	static constexpr uint32_t NoIndirectImage = UINT32_MAX;
 
 	uint32_t                      source            = 0;
+	uint32_t                      descriptor_index  = UINT32_MAX;
 	uint32_t                      first_use_pc      = 0;
 	ImageResourceClass            resource_class    = ImageResourceClass::None;
 	Prospero::TextureNumericClass numeric_class     = Prospero::TextureNumericClass::Unsupported;
@@ -150,6 +161,7 @@ struct SamplerResource {
 	uint32_t first_use_pc          = 0;
 	// Native filtering/border variants share the original sampler's runtime descriptor.
 	uint32_t snapshot_index        = 0;
+	std::vector<uint32_t> indirect_resources;
 	bool     force_point_filtering = false;
 	bool     depth_compare         = false;
 	bool     integer_border        = false;
@@ -187,9 +199,11 @@ enum class StageInputKind {
 	Layer,
 	SampleId,
 	BaryCoordSmooth,
+	BaryCoordSmoothSample,
 	BaryCoordSmoothCentroid,
 	BaryCoordNoPerspective,
 	WorkgroupId,
+	NumWorkgroups,
 	LocalInvocationId,
 	LocalInvocationIndex,
 	GlobalInvocationId,
@@ -292,11 +306,12 @@ enum class DescriptorBindingKind : uint32_t {
 	FaultBuffer,
 	FlattenedSrt,
 	ShaderData,
+	SharedMemory,
 	Count,
 };
 
 static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u);
-static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 55u);
+static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u);
 
 struct PushData {
 	static constexpr uint32_t DwordCount = 32;
@@ -412,22 +427,16 @@ DescriptorBindingForImage(const ImageResource& image) {
 	return static_cast<DescriptorBindingKind>(base + dimension);
 }
 
-struct DescriptorBinding {
-	DescriptorBindingKind kind = DescriptorBindingKind::Buffers;
-	std::vector<uint32_t> resources;
-
-	bool operator==(const DescriptorBinding& other) const = default;
-};
-
-struct BindingLayout {
-	uint32_t                       push_data_start_dword = PushData::NoStart;
-	uint32_t                       memory_offset_dword = 0;
-	uint32_t                       memory_offset_count = 0;
-	std::vector<uint32_t>          user_data_registers;
-	std::vector<DescriptorBinding> descriptors;
+struct Bindings {
+	std::array<uint32_t, static_cast<size_t>(DescriptorBindingKind::Count)> descriptor_counts {};
+	uint64_t descriptor_mask = 0;
+	uint32_t push_data_start_dword = PushData::NoStart;
+	uint32_t dispatch_thread_dword = PushData::NoStart;
+	uint32_t memory_offset_dword = 0;
 
 	[[nodiscard]] uint32_t ShaderDataDwords() const {
-		return memory_offset_dword + (memory_offset_count + 3u) / 4u;
+		return memory_offset_dword +
+		       (descriptor_counts[static_cast<size_t>(DescriptorBindingKind::Buffers)] + 3u) / 4u;
 	}
 	[[nodiscard]] bool UsesPushData() const {
 		return push_data_start_dword != PushData::NoStart;
@@ -438,7 +447,7 @@ struct BindingLayout {
 		}
 	}
 
-	bool operator==(const BindingLayout& other) const = default;
+	bool operator==(const Bindings& other) const = default;
 };
 
 struct ShaderInfo {
@@ -453,41 +462,66 @@ struct ShaderInfo {
 	std::vector<SampledResourcePair> sampled_pairs;
 	std::vector<StageInput>          inputs;
 	std::vector<StageOutput>         outputs;
+	std::vector<uint32_t>            user_data_registers;
+	uint64_t                        live_buffers = 0;
 	std::array<uint8_t, 32>          vertex_fetch_components {};
 	int32_t                          vertex_offset_sgpr = -1;
 	int32_t                          instance_offset_sgpr = -1;
-	bool                             has_bitwise_xor    = false;
-	bool                             uses_dma           = false;
+	bool                            has_bitwise_xor               = false;
+	bool                            uses_dma                      = false;
+	bool                            uses_swizzle                  = false;
+	bool                            uses_lds                      = false;
+	bool                            uses_gds                      = false;
+	bool                            uses_flattened_srt            = false;
+	bool                            uses_dispatch_threads         = false;
+	bool                            bvh                           = false;
+	bool                            subgroup_ballot               = false;
+	bool                            subgroup_barrier              = false;
+	bool                            subgroup_shuffle              = false;
+	bool                            subgroup_local_invocation_id  = false;
+	bool                            compute_derivatives           = false;
+	bool                            image_gather_extended         = false;
+	bool                            function_lds                  = false;
+	bool                            function_scratch              = false;
+	bool                            pixel_valid_mask              = false;
+	bool                            buffer_int64_atomics          = false;
+	bool                            buffer_u8                     = false;
+	bool                            buffer_u16                    = false;
+	bool                            shared_int64_atomics          = false;
+	bool                            coherent_buffers              = false;
+	bool                            float64                       = false;
 
 	bool operator==(const ShaderInfo& other) const = default;
 };
 
-struct BlockInfo {
-	uint32_t        id       = 0;
-	uint32_t        start_pc = 0;
-	uint32_t        end_pc   = 0;
-	CFG::Terminator terminator;
-	Value           condition;
-	Value           indirect_target;
-};
-
 struct DescriptorSource {
-	struct IndirectImage {
-		uint32_t material_source = UINT32_MAX;
+	struct IndirectDescriptor {
+		struct SelectorRead {
+			uint32_t source = UINT32_MAX;
+			uint32_t stride = 0;
+			uint32_t offset = 0;
+
+			bool operator==(const SelectorRead& other) const = default;
+		};
+		std::optional<SelectorRead> selector;
 		uint32_t table_source    = 0;
-		uint32_t selector_stride = 0;
-		uint32_t selector_offset = 0;
 		uint32_t table_offset    = 0;
+		uint32_t table_immediate = 0;
+		uint32_t table_stride    = 0;
+		uint32_t table_record_bytes = 0;
+		bool     table_scalar = false;
+		uint32_t workgroup_axis  = UINT32_MAX;
 		Value    key_count;
+		Value                 selector_first;
 		Value    selector_mask;
 		std::vector<uint32_t> sources;
 
-		bool operator==(const IndirectImage& other) const = default;
+		bool operator==(const IndirectDescriptor& other) const = default;
 	};
 
 	std::array<Value, 8>         dwords {};
 	uint32_t                     dword_count = 0;
-	std::optional<IndirectImage> indirect_image;
+	std::optional<IndirectDescriptor> indirect_descriptor;
 
 	bool operator==(const DescriptorSource& other) const = default;
 };
@@ -518,7 +552,7 @@ struct CompiledShaderInfo {
 	uint32_t                      param_export_mask   = 0;
 	bool                          has_address_writes  = false;
 	ShaderInfo                    info;
-	BindingLayout                 bindings;
+	Bindings                      bindings;
 };
 
 struct UniformFillPlan {
@@ -556,8 +590,8 @@ struct ResourcePlan {
 	std::vector<DescriptorSource>       descriptor_sources;
 	std::vector<ResourceBlock>          control_flow;
 	std::vector<SrtRead>                srt_reads;
-	std::vector<uint8_t>                clean_flat_slots;
 	bool                                requires_specialization_memory = false;
+	std::vector<std::pair<uint64_t, uint64_t>> source_reads;
 	bool                                capture_specialization_reads = false;
 	bool                                srt_plan_complete          = false;
 	bool                                resource_tracking_complete = false;
@@ -590,7 +624,6 @@ struct Program: ResourcePlan {
 	bool                          dispatcher_fallback = false;
 	CFG::FailureKind              cfg_failure_kind    = CFG::FailureKind::None;
 	std::string                   fallback_reason;
-	std::vector<BlockInfo>        block_info;
 	struct ScalarWrite { uint32_t pc; ScalarReg reg; };
 	std::vector<ScalarWrite>      scalar_writes;
 	// Typed memory and export instructions reference shader-local metadata by dense index.
@@ -598,11 +631,12 @@ struct Program: ResourcePlan {
 	std::vector<ExportInfo>       export_info;
 	bool                          has_address_writes = false;
 	bool                          shader_info_complete = false;
-	BindingLayout                 bindings;
-	bool                          binding_layout_complete = false;
+	Bindings                      bindings;
 
 };
 
+uint32_t StorageBufferElementBits(const Program& program, const MemoryInfo& memory);
+
 std::string ProgramToString(const Program& program);
 bool        HasShaderMemoryWrites(const Program& program);
 
@@ -610,6 +644,7 @@ void  ValidateProgram(const Program& program, bool require_ssa);
 void  ResolveControlFlowIdentities(Program& program);
 bool  EquivalentValue(const ResourcePlan& program, Value left, Value right);
 Value ResolveInvariantPhi(const ResourcePlan& program, Value value);
+Value ResolveActiveU32(Value value, Value active);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 

src/graphics/shader/recompiler/ir/Value.h

@@ -129,6 +129,12 @@ public:
 		}
 		return evaluation_index;
 	}
+	// Backend definitions belong to one emission. Wave64 emulation needs one per half.
+	[[nodiscard]] uint32_t Definition(uint32_t half) const { return definitions[half]; }
+	void SetDefinition(uint32_t half, uint32_t definition) const {
+		definitions[half] = definition;
+	}
+	void ResetDefinitions() const { definitions = {}; }
 
 	void SetParent(Block* block);
 	void SetArg(size_t index, Value value);
@@ -165,6 +171,7 @@ private:
 	uint8_t             num_args;
 	bool                live = false;
 	mutable uint32_t    evaluation_index = UINT32_MAX;
+	mutable std::array<uint32_t, 2> definitions {};
 	uint64_t            flags;
 	Block*              parent = nullptr;
 	union {
@@ -175,6 +182,6 @@ private:
 	std::vector<Use>    uses;
 };
 
-static_assert(sizeof(Inst) <= 112, "Inst operand storage unintentionally increased");
+static_assert(sizeof(Inst) <= 120, "Inst operand and backend storage unintentionally increased");
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -93,6 +93,7 @@ bool HasSideEffects(ValueOpcode opcode) {
 		case ValueOpcode::SetAttribute:
 		case ValueOpcode::SetTessellationAttribute:
 		case ValueOpcode::MeshAllocate:
+		case ValueOpcode::StoreCompletion:
 		case ValueOpcode::Barrier: return true;
 		default: return false;
 	}
@@ -177,7 +178,8 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicAnd32:
 		case ValueOpcode::SharedAtomicOr32:
 		case ValueOpcode::SharedAtomicOr64:
-		case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic;
+		case ValueOpcode::SharedAtomicXor32:
+		case ValueOpcode::SharedAtomicMaskedOr32: return SharedAccess::Atomic;
 		case ValueOpcode::DataAppend: return SharedAccess::Append;
 		case ValueOpcode::DataConsume: return SharedAccess::Consume;
 		default: return SharedAccess::None;
@@ -222,15 +224,21 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
 			return {ImageAccess::Write, ImageResourceClass::Storage, false};
 		case ValueOpcode::ImageAtomicCompareSwap32:
 		case ValueOpcode::ImageAtomicSwap32:
+		case ValueOpcode::ImageAtomicSwap64:
 		case ValueOpcode::ImageAtomicIAdd32:
+		case ValueOpcode::ImageAtomicIAdd64:
 		case ValueOpcode::ImageAtomicSMin32:
 		case ValueOpcode::ImageAtomicUMin32:
+		case ValueOpcode::ImageAtomicUMin64:
 		case ValueOpcode::ImageAtomicSMax32:
 		case ValueOpcode::ImageAtomicUMax32:
 		case ValueOpcode::ImageAtomicUMax64:
 		case ValueOpcode::ImageAtomicAnd32:
+		case ValueOpcode::ImageAtomicAnd64:
 		case ValueOpcode::ImageAtomicOr32:
+		case ValueOpcode::ImageAtomicOr64:
 		case ValueOpcode::ImageAtomicXor32:
+		case ValueOpcode::ImageAtomicXor64:
 		case ValueOpcode::ImageAtomicFMin32:
 		case ValueOpcode::ImageAtomicFMax32:
 			return {ImageAccess::Atomic, ImageResourceClass::Storage, false};