launcher: add per-game config import and export

PthreadCreate carries a guest thread name, but RunThread never handed it to
the host, so every guest thread showed up as the same generic host thread
name in core dumps, perf samples and htop.

Call pthread_setname_np() on Linux so host-side diagnostics can attribute
work to the guest thread that caused it.

3rdparty/cpuinfo/src/arm/mach/init.c

@@ -14,7 +14,7 @@
 #include <cpuinfo/internal-api.h>
 #include <cpuinfo/log.h>
 
-/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */
+/* Polyfill recent CPUFAMILY_ARM_* values for older platform headers */
 #ifndef CPUFAMILY_ARM_MONSOON_MISTRAL
 	#define CPUFAMILY_ARM_MONSOON_MISTRAL   0xE81E7EF6
 #endif

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.18...4.3)
+cmake_minimum_required(VERSION 3.19...4.3)
 
 if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
 	set(CMAKE_POLICY_VERSION_MINIMUM 3.10 CACHE STRING "Minimum policy version for bundled dependencies")
@@ -259,7 +259,7 @@ set(gpu_tiler_shader_names
 	depth
 	promote_d16
 	demote_d16
-	swap_bgra16
+	color_transform
 )
 file(GLOB gpu_tiler_shader_includes CONFIGURE_DEPENDS "${gpu_tiler_shader_dir}/gpu_tiler_*.inc")
 foreach(shader_name IN LISTS gpu_tiler_shader_names)
@@ -413,7 +413,16 @@ add_executable(archive_file_tests EXCLUDE_FROM_ALL
 target_link_libraries(archive_file_tests common ZArchive::zarchive)
 target_include_directories(archive_file_tests PRIVATE ${inc_headers})
 
-add_executable(avplayer_file_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp")
+add_executable(trophy_system_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/TrophySystemTests.cpp"
+)
+target_link_libraries(trophy_system_tests common)
+target_include_directories(trophy_system_tests PRIVATE ${inc_headers})
+
+add_executable(avplayer_file_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
 target_link_libraries(avplayer_file_tests common FFmpeg::ffmpeg ZArchive::zarchive)
 target_include_directories(avplayer_file_tests PRIVATE ${inc_headers})
 
@@ -606,6 +615,7 @@ if(BUILD_TESTING)
 	add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
 	add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
 	add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
+	add_test(NAME trophy_system COMMAND $<TARGET_FILE:trophy_system_tests>)
 	add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
 	add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
 	add_test(NAME lru_cache COMMAND $<TARGET_FILE:lru_cache_tests>)
@@ -682,6 +692,7 @@ if(BUILD_TESTING)
 		memory_tracker_tests
 		page_manager_tests
 		archive_file_tests
+		trophy_system_tests
 		avplayer_file_tests
 		bit_array_tests
 		lru_cache_tests
@@ -712,8 +723,9 @@ if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
 	add_executable(macos_sse4a_tests EXCLUDE_FROM_ALL
 		"${KYTY_TESTS_DIR}/macosSse4aTests.cpp"
 		"${KYTY_SOURCE_DIR}/loader/x64InstructionEmulator.cpp"
+		"${KYTY_SOURCE_DIR}/loader/guestInstructionPatcher.cpp"
 	)
-	target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis)
+	target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis xbyak::xbyak)
 	target_include_directories(macos_sse4a_tests PRIVATE ${inc_headers})
 	if(BUILD_TESTING)
 		add_test(NAME macos_sse4a COMMAND $<TARGET_FILE:macos_sse4a_tests>)
@@ -725,6 +737,15 @@ endif()
 add_executable(kyty_emulator "${KYTY_SOURCE_DIR}/main.cpp" ${kyty_emulator_src})
 configure_macos_guest_address_space(kyty_emulator)
 
+set(KYTY_TROPHY_UNLOCK_SOUND "${CMAKE_CURRENT_SOURCE_DIR}/assets/sounds/trophy-unlock.wav")
+add_custom_target(kyty_trophy_sound
+	COMMAND ${CMAKE_COMMAND} -E make_directory "$<TARGET_FILE_DIR:kyty_emulator>/assets/sounds"
+	COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_TROPHY_UNLOCK_SOUND}"
+		"$<TARGET_FILE_DIR:kyty_emulator>/assets/sounds/trophy-unlock.wav"
+	VERBATIM)
+add_dependencies(kyty_emulator kyty_trophy_sound)
+install(FILES "${KYTY_TROPHY_UNLOCK_SOUND}" DESTINATION assets/sounds)
+
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)
 	target_link_libraries(kyty_emulator opengl32)

README.md

@@ -1,8 +1,8 @@
 # KytyPS5
 
-[![Build KytyPS5 (Windows)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (Linux)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (macOS)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (Windows)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28Windows%29&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (Linux)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28Linux%29&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (macOS)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28macOS%29&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
 [![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20Linux%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
 [![Status](https://img.shields.io/badge/status-active%20development-orange.svg)](#current-status)
 [![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
@@ -72,8 +72,8 @@ or graphical glitches, so please include the version you tested when reporting a
       <img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
     </td>
     <td align="center">
-      <strong>UFC 5</strong><br>
-      <img src="docs/screenshots/ps5-06.png" width="300" alt="UFC 5 running in KytyPS5">
+      <strong>UFC 6</strong><br>
+      <img src="docs/screenshots/ps5-06.jpg" width="300" alt="UFC 6 running in KytyPS5">
     </td>
   </tr>
 </table>

src/common/CMakeLists.txt

@@ -27,6 +27,7 @@ target_link_libraries(common
 		kyty_git_version
 		spdlog::spdlog
 		ZArchive::zarchive
+		nlohmann_json::nlohmann_json
 )
 
 get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES)

src/common/emulatorConfig.cpp

@@ -156,6 +156,10 @@ bool TessellationEnabled() {
 	return g_config->tessellation_enabled;
 }
 
+bool TrophyEnabled() {
+	return g_config->trophy_enabled;
+}
+
 bool PlayGoHackEnabled() {
 	return g_config->playgo_hack_enabled;
 }

src/common/emulatorConfig.h

@@ -48,7 +48,7 @@ struct ConfigOptions {
 	int32_t                user_id                     = DEFAULT_USER_ID;
 	std::string            audio_input_device;
 	std::optional<ControllerColor> controller_color;
-	uint32_t               controller_speaker_volume      = 100;
+	uint32_t               controller_speaker_volume      = 50;
 	uint32_t               controller_vibration_intensity = 100;
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
@@ -74,6 +74,7 @@ struct ConfigOptions {
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
 	bool                   tessellation_enabled        = false;
+	bool                   trophy_enabled              = true;
 	bool                   playgo_hack_enabled         = false;
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	bool red_zone_protection_enabled = false;
@@ -123,6 +124,7 @@ bool GpuAssistedValidationEnabled();
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
 bool TessellationEnabled();
+bool TrophyEnabled();
 bool PlayGoHackEnabled();
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled();

src/common/trophies.cpp

@@ -0,0 +1,453 @@
+#include "common/trophies.h"
+
+#include "common/file.h"
+
+#include <algorithm>
+#include <array>
+#include <charconv>
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <span>
+
+namespace Common::Trophies {
+namespace {
+
+using Json  = nlohmann::json;
+using Files = std::map<std::string_view, std::span<const std::byte>>;
+
+constexpr uint64_t MaxUnlocksSize = uint64_t {1} << 20u;
+constexpr uint64_t MaxPackageSize = uint64_t {1} << 30u;
+
+uint64_t ReadBigEndian(const std::byte* data, size_t size) {
+	uint64_t value = 0;
+	for (size_t i = 0; i < size; ++i) {
+		value = (value << 8u) | std::to_integer<uint8_t>(data[i]);
+	}
+	return value;
+}
+
+int ReadId(const Json& value) {
+	if (value.is_number_integer()) {
+		const auto number = value.get<int64_t>();
+		return number >= 0 && number <= INT32_MAX ? static_cast<int>(number) : -1;
+	}
+	if (!value.is_string()) {
+		return -1;
+	}
+	const auto& text        = value.get_ref<const std::string&>();
+	int         id          = -1;
+	const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), id);
+	return error == std::errc {} && end == text.data() + text.size() && id >= 0 ? id : -1;
+}
+
+std::optional<uint64_t> ReadUnsigned(const Json& value) {
+	if (value.is_number_unsigned()) {
+		return value.get<uint64_t>();
+	}
+	if (value.is_number_integer()) {
+		const auto number = value.get<int64_t>();
+		return number >= 0 ? std::optional<uint64_t>(static_cast<uint64_t>(number)) : std::nullopt;
+	}
+	if (!value.is_string()) {
+		return std::nullopt;
+	}
+	const auto& text        = value.get_ref<const std::string&>();
+	uint64_t    number      = 0;
+	const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), number);
+	return error == std::errc {} && end == text.data() + text.size()
+	           ? std::optional<uint64_t>(number)
+	           : std::nullopt;
+}
+
+Json ReadJson(const Files& files, std::string_view name) {
+	const auto entry = files.find(name);
+	if (entry == files.end()) {
+		return {};
+	}
+	return Json::parse(entry->second.begin(), entry->second.end(), nullptr, false);
+}
+
+bool ReadFiles(std::span<const std::byte> data, Files& files, uint64_t& size) {
+	if (data.size() < 0x40 || data.size() > MaxPackageSize ||
+	    ReadBigEndian(data.data(), 4) != 0xb228c60a || ReadBigEndian(data.data() + 4, 4) != 1) {
+		return false;
+	}
+	size             = ReadBigEndian(data.data() + 8, 8);
+	const auto count = ReadBigEndian(data.data() + 0x10, 4);
+	const auto toc   = ReadBigEndian(data.data() + 0x14, 4);
+	if (size < 0x40 || size > data.size() || count > 4096 || toc > size ||
+	    0x20 + count * 0x40 > size - toc) {
+		return false;
+	}
+	for (uint64_t i = 0; i < count; ++i) {
+		const auto*            entry = data.data() + toc + 0x20 + i * 0x40;
+		const auto*            name  = reinterpret_cast<const char*>(entry);
+		const std::string_view filename(name, std::find(name, name + 0x20, '\0') - name);
+		const auto             offset = ReadBigEndian(entry + 0x20, 8);
+		const auto             length = ReadBigEndian(entry + 0x28, 8);
+		if (offset > size || length > size - offset) {
+			return false;
+		}
+		files.emplace(filename, data.subspan(offset, length));
+	}
+	return true;
+}
+
+Package ParsePackage(std::span<const std::byte> data, int console_language) {
+	Files    files;
+	uint64_t size = 0;
+	if (!ReadFiles(data, files, size)) {
+		return {};
+	}
+	const auto conf = ReadJson(files, "tropconf.json");
+	if (!conf.is_object() || !conf.contains("trophies") || !conf["trophies"].is_array() ||
+	    conf["trophies"].size() > 1000 || !conf.contains("defaultLanguage") ||
+	    !conf["defaultLanguage"].is_string()) {
+		return {};
+	}
+	static constexpr std::array<std::string_view, 30> locales = {
+	    "ja-JP", "en-US", "fr-FR",   "es-ES",   "de-DE",  "it-IT", "nl-NL", "pt-PT",
+	    "ru-RU", "ko-KR", "zh-Hant", "zh-Hans", "fi-FI",  "sv-SE", "da-DK", "no-NO",
+	    "pl-PL", "pt-BR", "en-GB",   "tr-TR",   "es-419", "ar-AE", "fr-CA", "cs-CZ",
+	    "hu-HU", "el-GR", "ro-RO",   "th-TH",   "vi-VN",  "id-ID"};
+	const auto locale =
+	    locales[console_language >= 0 && console_language < locales.size() ? console_language : 1];
+	auto meta = ReadJson(files, fmt::format("tropmeta_{}.json", locale));
+	if (meta.is_null()) {
+		meta = ReadJson(files, fmt::format("tropmeta_{}.json",
+		                                   conf["defaultLanguage"].get_ref<const std::string&>()));
+	}
+	if (!meta.is_object() || !meta.contains("metadata") || !meta["metadata"].is_object()) {
+		return {};
+	}
+	const auto& texts = meta["metadata"];
+	if (!texts.contains("titleMetadata") || !texts["titleMetadata"].is_object() ||
+	    !texts["titleMetadata"].contains("name") || !texts["titleMetadata"]["name"].is_string() ||
+	    !texts.contains("trophyMetadata") || !texts["trophyMetadata"].is_array()) {
+		return {};
+	}
+	Package package;
+	package.title = texts["titleMetadata"]["name"].get<std::string>();
+	package.groups.emplace(-1, package.title);
+	uint64_t icon_bytes = 0;
+	for (const auto& definition: conf["trophies"]) {
+		if (!definition.is_object() || !definition.contains("id") ||
+		    !definition.contains("grade") || !definition["grade"].is_string()) {
+			return {};
+		}
+		Trophy trophy;
+		trophy.id              = ReadId(definition["id"]);
+		const auto grade       = definition["grade"].get<std::string>();
+		const auto grade_index = std::string_view("PGSB").find(grade);
+		if (trophy.id < 0 || grade.size() != 1 || grade_index == std::string_view::npos) {
+			return {};
+		}
+		trophy.grade = static_cast<int>(grade_index) + 1;
+		if (definition.contains("groupId")) {
+			trophy.group_id = ReadId(definition["groupId"]);
+			if (trophy.group_id < 0) {
+				return {};
+			}
+		}
+		if (definition.contains("platinumTrophyId")) {
+			trophy.platinum_id = ReadId(definition["platinumTrophyId"]);
+		}
+		trophy.hidden     = definition.contains("hidden") && definition["hidden"] == true;
+		trophy.has_reward = definition.contains("hasReward") && definition["hasReward"] == true;
+		if (definition.contains("unlockCondition")) {
+			const auto& condition = definition["unlockCondition"];
+			if (condition.is_object()) {
+				trophy.progressive =
+				    condition.contains("progressive") && condition["progressive"] == true;
+				if (condition.contains("udsStatId")) {
+					trophy.uds_stat_id = ReadUnsigned(condition["udsStatId"]);
+					if (!trophy.uds_stat_id) {
+						return {};
+					}
+				}
+				if (condition.contains("targetValue")) {
+					trophy.target = ReadUnsigned(condition["targetValue"]);
+				}
+				const auto comparison = condition.find("comparator");
+				if (comparison != condition.end()) {
+					if (*comparison == "ge")
+						trophy.comparison = Comparison::GreaterEqual;
+					else if (*comparison == "gt")
+						trophy.comparison = Comparison::Greater;
+					else if (*comparison == "le")
+						trophy.comparison = Comparison::LessEqual;
+					else if (*comparison == "lt")
+						trophy.comparison = Comparison::Less;
+				}
+			}
+		}
+		const auto icon = files.find(fmt::format("trop{:04}.png", trophy.id));
+		if (icon != files.end()) {
+			if (icon->second.size() > size - icon_bytes) {
+				return {};
+			}
+			trophy.icon_png.assign(icon->second.begin(), icon->second.end());
+			icon_bytes += icon->second.size();
+		}
+		package.groups.try_emplace(trophy.group_id);
+		if (package.groups.size() > 50 ||
+		    !package.trophies.emplace(trophy.id, std::move(trophy)).second) {
+			return {};
+		}
+	}
+	if (texts.contains("groupMetadata") && texts["groupMetadata"].is_array()) {
+		for (const auto& group: texts["groupMetadata"]) {
+			if (group.is_object() && group.contains("id") && group.contains("name") &&
+			    group["name"].is_string()) {
+				const auto found = package.groups.find(ReadId(group["id"]));
+				if (found != package.groups.end()) {
+					found->second = group["name"].get<std::string>();
+				}
+			}
+		}
+	}
+	for (const auto& text: texts["trophyMetadata"]) {
+		if (!text.is_object() || !text.contains("id")) {
+			continue;
+		}
+		const auto found = package.trophies.find(ReadId(text["id"]));
+		if (found == package.trophies.end()) {
+			continue;
+		}
+		for (const auto& [key, field]: {std::pair {"name", &found->second.name},
+		                                {"detail", &found->second.description},
+		                                {"reward", &found->second.reward}}) {
+			if (text.contains(key) && text[key].is_string()) {
+				*field = text[key].get<std::string>();
+			}
+		}
+	}
+	return package;
+}
+
+} // namespace
+
+Package LoadPackage(const std::filesystem::path& path, int console_language) {
+	File file(path, File::Mode::Read);
+	if (file.IsInvalid() || file.Size() > MaxPackageSize) {
+		return {};
+	}
+	return ParsePackage(file.ReadWholeBuffer(), console_language);
+}
+
+std::filesystem::path PackagePath(uint32_t service_label) {
+	return std::filesystem::path(PackageDirectory) / fmt::format("trophy{:02}.ucp", service_label);
+}
+
+std::filesystem::path UnlocksPath(const std::filesystem::path& root, std::string_view title_id,
+                                  int user_id, uint32_t service_label) {
+	if (title_id.empty() || !std::all_of(title_id.begin(), title_id.end(), [](char c) {
+		    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
+		           c == '_' || c == '-';
+	    })) {
+		return {};
+	}
+	return root / "_Trophies" / title_id /
+	       fmt::format("trophies_{}_{}.json", user_id, service_label);
+}
+
+UnlockData LoadUnlockData(const std::filesystem::path& path) {
+	File file(path, File::Mode::Read);
+	if (file.IsInvalid() || file.Size() > MaxUnlocksSize) {
+		return {};
+	}
+	const auto bytes = file.ReadWholeBuffer();
+	const auto json  = Json::parse(bytes.begin(), bytes.end(), nullptr, false);
+	if (!json.is_object() || !json.contains("unlockedTrophies") ||
+	    !json["unlockedTrophies"].is_array()) {
+		return {};
+	}
+	UnlockData unlocks;
+	for (const auto& value: json["unlockedTrophies"]) {
+		const auto id = ReadId(value);
+		if (id >= 0) {
+			unlocks.unlocked.insert(id);
+		}
+	}
+	if (json.contains("unlockedAt") && json["unlockedAt"].is_object()) {
+		for (auto it = json["unlockedAt"].begin(); it != json["unlockedAt"].end(); ++it) {
+			const auto id = ReadId(Json(it.key()));
+			if (id >= 0 && unlocks.unlocked.contains(id)) {
+				const auto tick = ReadUnsigned(it.value());
+				if (tick && *tick >= UnixEpochTick && *tick <= 315537897599999999ULL) {
+					unlocks.timestamps.emplace(id, *tick);
+				}
+			}
+		}
+	}
+	return unlocks;
+}
+
+bool SaveUnlockData(const std::filesystem::path& path, const UnlockData& unlocks) {
+	if (path.empty() || !File::CreateDirectories(path.parent_path())) {
+		return false;
+	}
+	Json timestamps = Json::object();
+	for (const auto& [id, tick]: unlocks.timestamps) {
+		if (unlocks.unlocked.contains(id)) {
+			timestamps[std::to_string(id)] = tick;
+		}
+	}
+	const auto text =
+	    Json {{"unlockedTrophies", unlocks.unlocked}, {"unlockedAt", timestamps}}.dump();
+	auto temporary = path;
+	temporary += ".tmp";
+	File file;
+	if (text.size() > MaxUnlocksSize || !file.Create(temporary)) {
+		return false;
+	}
+	uint32_t written = 0;
+	file.Write(text.data(), static_cast<uint32_t>(text.size()), &written);
+	const bool complete = written == text.size() && file.Flush();
+	file.Close();
+	std::error_code error;
+	if (complete) {
+		std::filesystem::rename(temporary, path, error);
+		if (!error) {
+			return true;
+		}
+	}
+	std::filesystem::remove(temporary, error);
+	return false;
+}
+
+UdsRules LoadUdsRules(const std::filesystem::path& path) {
+	File file(path, File::Mode::Read);
+	if (file.IsInvalid() || file.Size() > MaxPackageSize) {
+		return {};
+	}
+	const auto bytes = file.ReadWholeBuffer();
+	Files      files;
+	uint64_t   size = 0;
+	if (!ReadFiles(bytes, files, size)) {
+		return {};
+	}
+	const auto definitions = ReadJson(files, "stats_definition.json");
+	const auto extraction  = ReadJson(files, "stats_extraction.json");
+	if (!definitions.is_object() || !definitions.contains("statDefinitionArray") ||
+	    !definitions["statDefinitionArray"].is_array() || !extraction.is_object() ||
+	    !extraction.contains("statsExtractionRuleArray") ||
+	    !extraction["statsExtractionRuleArray"].is_array()) {
+		return {};
+	}
+	// Only direct unsigned latest-value stats are evaluated here. Aggregated stats
+	// and filtered/nested extraction require UDS state and event types we do not implement.
+	std::map<uint64_t, bool> stats;
+	for (const auto& definition: definitions["statDefinitionArray"]) {
+		if (!definition.is_object() || !definition.contains("statId") ||
+		    !definition.contains("aggregation") || definition["aggregation"] != "latest" ||
+		    !definition.contains("dataType")) {
+			continue;
+		}
+		const auto  id   = ReadUnsigned(definition["statId"]);
+		const auto& type = definition["dataType"];
+		if (!id || (type != "uint64" && type != "uint32")) {
+			continue;
+		}
+		const uint64_t maximum = type == "uint32" ? UINT32_MAX : UINT64_MAX;
+		if ((definition.contains("minValue") && ReadUnsigned(definition["minValue"]) != 0) ||
+		    (definition.contains("maxValue") && ReadUnsigned(definition["maxValue"]) != maximum)) {
+			continue;
+		}
+		stats.emplace(*id, type == "uint32");
+	}
+	UdsRules events;
+	for (const auto& rule: extraction["statsExtractionRuleArray"]) {
+		if (!rule.is_object() || !rule.contains("condition") || !rule.contains("action")) {
+			continue;
+		}
+		const auto& condition = rule["condition"];
+		const auto& action    = rule["action"];
+		if (!condition.is_object() || condition.size() != 1 || !condition.contains("eventName") ||
+		    !condition["eventName"].is_string() || !action.is_object() ||
+		    !action.contains("input") || !action["input"].is_string() ||
+		    !action.contains("output") || !action["output"].is_object() ||
+		    !action["output"].contains("statId")) {
+			continue;
+		}
+		const auto  id    = ReadUnsigned(action["output"]["statId"]);
+		const auto& input = action["input"].get_ref<const std::string&>();
+		if (!id || !stats.contains(*id) || !input.starts_with("$.") || input.size() == 2 ||
+		    input.find_first_not_of(
+		        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_", 2) !=
+		        std::string::npos) {
+			continue;
+		}
+		events[condition["eventName"].get<std::string>()].push_back(
+		    {*id, input.substr(2), stats.at(*id)});
+	}
+	return events;
+}
+
+std::filesystem::path UdsPackagePath(uint32_t service_label) {
+	return std::filesystem::path("sce_sys/uds") / fmt::format("uds{:02}.ucp", service_label);
+}
+
+std::vector<int> FindUdsTrophies(const Package& package, std::string_view event_name,
+                                 const std::map<std::string, UdsInteger>& properties) {
+	const auto rules = package.event_rules.find(std::string(event_name));
+	if (rules == package.event_rules.end()) {
+		return {};
+	}
+	std::vector<int> ids;
+	for (const auto& [id, trophy]: package.trophies) {
+		if (trophy.grade == 1 || !trophy.uds_stat_id || !trophy.target) {
+			continue;
+		}
+		for (const auto& rule: rules->second) {
+			const auto value = properties.find(rule.input);
+			if (rule.stat_id != *trophy.uds_stat_id || value == properties.end()) {
+				continue;
+			}
+			std::optional<uint64_t> number;
+			if (rule.is_uint32) {
+				if (const auto* input = std::get_if<uint32_t>(&value->second)) number = *input;
+			} else if (const auto* input = std::get_if<uint64_t>(&value->second)) {
+				number = *input;
+			}
+			if (!number) continue;
+			bool matched = false;
+			switch (trophy.comparison) {
+				case Comparison::GreaterEqual: matched = *number >= *trophy.target; break;
+				case Comparison::Greater: matched = *number > *trophy.target; break;
+				case Comparison::LessEqual: matched = *number <= *trophy.target; break;
+				case Comparison::Less: matched = *number < *trophy.target; break;
+				case Comparison::None: break;
+			}
+			if (matched) {
+				ids.push_back(id);
+				break;
+			}
+		}
+	}
+	return ids;
+}
+
+uint32_t Progress::Percentage() const {
+	const auto points        = total_grade[2] * 6 + total_grade[3] * 2 + total_grade[4];
+	const auto earned_points = earned_grade[2] * 6 + earned_grade[3] * 2 + earned_grade[4];
+	return points == 0 ? 0 : earned_points * 100 / points;
+}
+
+Progress GetProgress(const Package& package, const UnlockData& unlocks, std::optional<int> group) {
+	Progress progress;
+	for (const auto& [id, trophy]: package.trophies) {
+		if (group && trophy.group_id != *group) {
+			continue;
+		}
+		++progress.total;
+		++progress.total_grade[trophy.grade];
+		if (unlocks.unlocked.contains(id)) {
+			++progress.earned;
+			++progress.earned_grade[trophy.grade];
+		}
+	}
+	return progress;
+}
+
+} // namespace Common::Trophies

src/common/trophies.h

@@ -0,0 +1,87 @@
+#ifndef KYTY_COMMON_TROPHIES_H_
+#define KYTY_COMMON_TROPHIES_H_
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <string_view>
+#include <variant>
+#include <vector>
+
+namespace Common::Trophies {
+
+inline constexpr char     PackageDirectory[] = "sce_sys/trophy2";
+inline constexpr uint64_t UnixEpochTick      = 62135596800000000ULL;
+
+enum class Comparison { None, GreaterEqual, Greater, LessEqual, Less };
+
+struct Trophy {
+	int                     id          = 0;
+	int                     group_id    = -1;
+	int                     platinum_id = -1;
+	int                     grade       = 0;
+	std::optional<uint64_t> target;
+	std::optional<uint64_t> uds_stat_id;
+	Comparison              comparison  = Comparison::None;
+	bool                    progressive = false;
+	std::string             name;
+	std::string             description;
+	std::string             reward;
+	std::vector<std::byte>  icon_png;
+	bool                    hidden     = false;
+	bool                    has_reward = false;
+};
+
+struct UdsRule {
+	uint64_t    stat_id;
+	std::string input;
+	bool        is_uint32;
+};
+
+using UdsRules   = std::map<std::string, std::vector<UdsRule>>;
+using UdsInteger = std::variant<int32_t, uint32_t, uint64_t>;
+
+struct Package {
+	std::string                title;
+	std::map<int, std::string> groups;
+	std::map<int, Trophy>      trophies;
+	UdsRules                   event_rules;
+};
+
+struct UnlockData {
+	std::set<int>           unlocked;
+	std::map<int, uint64_t> timestamps;
+};
+
+struct Progress {
+	std::array<uint32_t, 5> total_grade {};
+	std::array<uint32_t, 5> earned_grade {};
+	uint32_t                total  = 0;
+	uint32_t                earned = 0;
+
+	[[nodiscard]] uint32_t Percentage() const;
+};
+
+[[nodiscard]] Progress GetProgress(const Package& package, const UnlockData& unlocks,
+                                   std::optional<int> group = std::nullopt);
+
+[[nodiscard]] Package LoadPackage(const std::filesystem::path& path, int console_language);
+[[nodiscard]] std::filesystem::path PackagePath(uint32_t service_label);
+[[nodiscard]] std::filesystem::path UdsPackagePath(uint32_t service_label);
+[[nodiscard]] UdsRules              LoadUdsRules(const std::filesystem::path& path);
+[[nodiscard]] std::vector<int> FindUdsTrophies(const Package& package, std::string_view event_name,
+                                               const std::map<std::string, UdsInteger>& properties);
+[[nodiscard]] std::filesystem::path UnlocksPath(const std::filesystem::path& root,
+                                                std::string_view title_id, int user_id,
+                                                uint32_t service_label);
+[[nodiscard]] UnlockData            LoadUnlockData(const std::filesystem::path& path);
+[[nodiscard]] bool SaveUnlockData(const std::filesystem::path& path, const UnlockData& unlocks);
+
+} // namespace Common::Trophies
+
+#endif // KYTY_COMMON_TROPHIES_H_

src/graphics/guest_gpu/gpu_format.cpp

@@ -20,7 +20,8 @@ struct FormatInfo {
 
 constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k8UNorm, 1, 0, 1, true, false},
-	{BufferFormat::k8SNorm, 0, 0, 1, false, false},
+	{BufferFormat::k8UScaled, 1, 0, 0, true, false},
+	{BufferFormat::k8SNorm, 1, 0, 1, false, false},
 	{BufferFormat::k8UInt, 1, 0, 1, true, true},
 	{BufferFormat::k16UNorm, 2, 0, 2, true, false},
 	{BufferFormat::k16SNorm, 2, 0, 2, true, false},
@@ -28,6 +29,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k16SInt, 2, 0, 2, true, false, true},
 	{BufferFormat::k16Float, 2, 0, 2, true, false},
 	{BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UScaled, 2, 0, 0, true, false},
 	{BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
 	{BufferFormat::k8_8UInt, 2, 0, 2, true, true},
 	{BufferFormat::k8_8SInt, 2, 0, 2, true, false, true},
@@ -41,6 +43,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k16_16Float, 4, 0, 4, true, false},
 	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
 	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
+	{BufferFormat::k10_11_11Float, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
 	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
@@ -67,7 +70,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
 	{BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
 	{BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
-	{BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
+	{BufferFormat::k1_5_5_5UNorm, 2, 0, 2, false, false},
 	{BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
 	{BufferFormat::kFmask8_S2_F1, 1, 0, 1, true, false},
 	{BufferFormat::kFmask8_S4_F1, 1, 0, 1, true, false},
@@ -231,7 +234,12 @@ TextureNumericClass SampledTextureNumericClass(BufferFormat format) {
 }
 
 BufferFormat RemapTextureFormat(BufferFormat format) {
-	return format == BufferFormat::k11_11_10UInt ? BufferFormat::k32UInt : format;
+	switch (format) {
+		case BufferFormat::k8UScaled: return BufferFormat::k8UNorm;
+		case BufferFormat::k8_8UScaled: return BufferFormat::k8_8UNorm;
+		case BufferFormat::k11_11_10UInt: return BufferFormat::k32UInt;
+		default: return format;
+	}
 }
 
 } // namespace Libs::Graphics::Prospero

src/graphics/guest_gpu/graphicsRun.cpp

@@ -876,64 +876,15 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou
 }
 
 void CommandProcessor::DrawIndirect(uint32_t data_offset, uint32_t draw_initiator, bool indexed) {
-	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u);
-	EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
-
-	const auto* args_addr =
-	    reinterpret_cast<const void*>(m_draw_indirect_args_base_addr + data_offset);
-
-	if (!indexed) {
-		DrawIndirectArgs args {};
-		std::memcpy(&args, args_addr, sizeof(args));
-		m_num_instances = args.instance_count;
-		DrawIndexAuto({.vertex_count   = args.vertex_count_per_instance,
-		               .instance_count = args.instance_count,
-		               .first_vertex   = args.start_vertex_location,
-		               .first_instance = args.start_instance_location,
-		               .offset_source  = DrawOffsetSource::IndirectArgs});
-		return;
-	}
-
-	DrawIndexedIndirectArgs args {};
-	std::memcpy(&args, args_addr, sizeof(args));
-
-	uint64_t index_size = 0;
-	switch (m_index_type_and_size) {
-		case 0: index_size = 2; break;
-		case 1: index_size = 4; break;
-		case 2: index_size = 1; break;
-		default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
-	}
-
-	auto* index_addr = reinterpret_cast<const void*>(
-	    m_index_base_addr + static_cast<uint64_t>(args.start_index_location) * index_size);
-
-	const uint32_t index_count =
-	    (m_index_buffer_size != 0 ? std::min(args.index_count_per_instance, m_index_buffer_size)
-	                              : args.index_count_per_instance);
-	if (GraphicsRunDebugDumpEnabled() && index_count != args.index_count_per_instance) {
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
-			LOGF("\t DrawIndexIndirect: clamped index_count from %" PRIu32 " to %" PRIu32
-			     " using INDEX_BUFFER_SIZE\n",
-			     args.index_count_per_instance, index_count);
-		}
-	}
-
-	m_num_instances = args.instance_count;
-	DrawIndex({.index_count    = index_count,
-	           .index_addr     = index_addr,
-	           .instance_count = args.instance_count,
-	           .base_vertex    = static_cast<int32_t>(args.base_vertex_location),
-	           .first_instance = args.start_instance_location,
-	           .offset_source  = DrawOffsetSource::IndirectArgs});
+	const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs);
+	DrawIndirectMulti(data_offset, 1, nullptr, args_size, draw_initiator, indexed);
 }
 
 void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
                                          const volatile uint32_t* count_addr,
                                          uint32_t stride_in_bytes, uint32_t draw_initiator,
                                          bool indexed) {
-	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != 2u);
+	EXIT_NOT_IMPLEMENTED((draw_initiator & ~0x20u) != (indexed ? 0u : 2u));
 	EXIT_NOT_IMPLEMENTED(m_draw_indirect_args_base_addr == 0);
 
 	uint32_t draw_count = max_count_or_count;

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -258,15 +258,8 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		}
 		auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
 
-		// Widen nearby CPU reads so they share one GPU drain.
-		constexpr uint64_t WindowSize   = 512 * 1024;
-		const auto         buffer_begin = buffer.CpuAddress();
-		const auto         buffer_end   = buffer_begin + buffer.Size();
-		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
-		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
-
-		if (DownloadBufferMemory<false>(buffer, window_begin, window_end - window_begin)) {
-			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
+		if (DownloadBufferMemory<false>(buffer, vaddr, size)) {
+			m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
 		}
 		if (is_write) {
 			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -195,6 +195,9 @@ bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& request
 	if (cached.tile_mode != requested.tile_mode) {
 		return false;
 	}
+	if (cached.GetColorTransform() != requested.GetColorTransform()) {
+		return false;
+	}
 	if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format) ||
 	    cached.type != requested.type) {
 		return false;
@@ -634,6 +637,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 	const bool source_depth = source.info.IsDepth();
 	const bool dest_depth   = destination.info.IsDepth();
 	const bool direct_copy =
+	    source.info.GetColorTransform() == destination.info.GetColorTransform() &&
 	    (source.backing.image_type == destination.backing.image_type ||
 	     (source.backing.image_type != vk::ImageType::e1D &&
 	      destination.backing.image_type != vk::ImageType::e1D)) &&
@@ -647,7 +651,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 			EXIT("TextureCache: cross-format multisample image copy is unsupported\n");
 		}
 		auto& copy_buffer = m_buffer_cache.GetUtilityBuffer(MemoryUsage::DeviceLocal);
-		destination.CopyImageWithBuffer(source, copy_buffer);
+		destination.CopyImageWithBuffer(source, copy_buffer, m_tiler);
 	}
 	if (source.IsGpuModified()) {
 		destination.MarkGpuModified();
@@ -710,7 +714,8 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 	        cached.info.data.size / cached.info.resources.layers &&
 	    !requested.HasStencil() && !cached.info.HasStencil() && !requested.HasMetadata() &&
 	    !cached.info.HasMetadata();
-	bool recreate = cached.info.resources < requested.resources;
+	bool recreate = cached.info.resources < requested.resources ||
+	                requested.IsVolume() != cached.info.IsVolume();
 	switch (binding) {
 		case BindingType::Texture:
 			recreate |= requested.IsDepth() && !cached.info.IsDepth();
@@ -801,6 +806,9 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 			}
 			return {merged_id};
 		}
+		if (requested.GetColorTransform() != cached.info.GetColorTransform()) {
+			return {ExpandImage(requested, cached_id)};
+		}
 		if (requested.IsBlock() && !cached.info.IsBlock()) {
 			return {ExpandImage(requested, cached_id)};
 		}
@@ -836,6 +844,9 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		      requested.resources.layers == cached.info.resources.layers))) {
 			return {ExpandImage(requested, cached_id)};
 		}
+		if (requested.type != cached.info.type) {
+			return {merged_id};
+		}
 		if (requested.pixel_format != cached.info.pixel_format ||
 		    requested.data.size <= cached.info.data.size) {
 			const auto result_id = merged_id ? merged_id : cached_id;
@@ -905,8 +916,8 @@ struct TextureCache::TextureTransfer {
 	TextureUploadLayout              layout;
 	std::vector<vk::BufferImageCopy> regions;
 	std::vector<GpuTileInfo>         tiles;
-	bool                             swap_bgra16 = false;
-	bool                             valid       = false;
+	ColorTransform                   color_transform = ColorTransform::None;
+	bool                             valid           = false;
 
 	[[nodiscard]] uint64_t LinearSize() const {
 		uint64_t size = 0;
@@ -935,7 +946,7 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 	const char* owner            = "TextureCache readback";
 
 	TextureTransfer transfer;
-	transfer.swap_bgra16 = info.bgra16;
+	transfer.color_transform = info.GetColorTransform();
 	if (upload) {
 		if ((render_target || video_out) &&
 		    (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
@@ -1023,10 +1034,9 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 		TileManager::Result linear {source.Handle(), source_offset, info.data.size};
 		if (!transfer.tiles.empty()) {
 			linear = m_tiler.Detile(source.Handle(), source_offset, info.data.size,
-			                        transfer.LinearSize(), transfer.tiles);
-		}
-		if (transfer.swap_bgra16) {
-			linear = m_tiler.SwapBgra16(linear);
+			                        transfer.LinearSize(), transfer.tiles, transfer.color_transform);
+		} else if (transfer.color_transform != ColorTransform::None) {
+			linear = m_tiler.TransformColor(linear, transfer.color_transform, true);
 		}
 		upload(transfer.regions, linear);
 		return;
@@ -1037,6 +1047,8 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	auto info = destination.info;
 	if (image.depth_id) {
 		info.data            = image.info.data;
+		info.resources       = {image.stencil_subresources.level_count,
+		                        image.stencil_subresources.layer_count};
 		info.guest_format    = Prospero::BufferFormat::k8UInt;
 		info.bytes_per_block = 1;
 		if (info.IsTiled()) info.pitch = TileGetDepthPitch(info.extent.width, 1, 0);
@@ -1051,6 +1063,11 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	auto copies = BuildDepthCopies(info, full_slice_size, image.depth_id
 	                                                        ? vk::ImageAspectFlagBits::eStencil
 	                                                        : vk::ImageAspectFlagBits::eDepth);
+	if (image.depth_id) {
+		for (auto& copy: copies) {
+			copy.imageSubresource.baseArrayLayer += image.stencil_subresources.base_layer;
+		}
+	}
 	TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset};
 	if (info.IsTiled()) {
 		const auto tiles = BuildDepthTiles(info);
@@ -1239,7 +1256,8 @@ ImageId TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
 	}
 	auto& record = m_slot_images[association];
 	TouchImage(record);
-	record.depth_id = depth_id;
+	record.depth_id             = depth_id;
+	record.stencil_subresources = depth.stencil_subresources;
 	return association;
 }
 
@@ -1445,6 +1463,18 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	TouchImage(image);
 	image.MarkGpuModified();
 	image.usage.depth_target = true;
+	if (desc.info.HasStencil()) {
+		const auto layers = image.info.resources.layers;
+		EXIT_IF(layers == 0 || image.info.data.size % layers != 0 ||
+		        desc.info.data.address < image.info.data.address);
+		const auto slice_size = image.info.data.size / layers;
+		const auto offset     = desc.info.data.address - image.info.data.address;
+		EXIT_IF(slice_size == 0 || offset % slice_size != 0 || offset / slice_size >= layers ||
+		        desc.info.resources.layers > layers - offset / slice_size);
+		image.stencil_subresources = {0, desc.info.resources.levels,
+		                             static_cast<uint32_t>(offset / slice_size),
+		                             desc.info.resources.layers};
+	}
 	image.info.stencil = desc.info.stencil;
 	image.info.metadata = desc.info.metadata;
 	if (desc.info.HasMetadata()) {
@@ -1491,9 +1521,11 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid image clear\n");
 	}
-	std::scoped_lock     lock {m_lock};
-	ImageId              selected {};
-	vk::ImageAspectFlags aspect {};
+	std::scoped_lock      lock {m_lock};
+	ImageId               selected {};
+	ImageId               stencil_id {};
+	vk::ImageAspectFlags  aspect {};
+	ImageSubresourceRange subresources;
 	for (const auto id: FindImagesInRegion(address, size, false)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {
@@ -1501,10 +1533,13 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		}
 		vk::ImageAspectFlags candidate {};
 		ImageId              candidate_id = id;
+		ImageSubresourceRange candidate_subresources {
+		    0, owner->info.resources.levels, 0, owner->info.TransferLayers()};
 		if (owner->depth_id && owner->info.data.address == address &&
 		    owner->info.data.size == size) {
-			candidate    = vk::ImageAspectFlagBits::eStencil;
-			candidate_id = owner->depth_id;
+			candidate              = vk::ImageAspectFlagBits::eStencil;
+			candidate_id           = owner->depth_id;
+			candidate_subresources = owner->stencil_subresources;
 			owner        = m_slot_images.try_get(candidate_id);
 			if (owner == nullptr || owner->backing.image == nullptr || !owner->info.HasStencil()) {
 				continue;
@@ -1517,11 +1552,13 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		if (!candidate) {
 			continue;
 		}
-		if (selected && selected != candidate_id) {
+		if (selected && (selected != candidate_id || subresources != candidate_subresources)) {
 			return false;
 		}
-		selected = candidate_id;
-		aspect   = candidate;
+		selected     = candidate_id;
+		stencil_id   = candidate == vk::ImageAspectFlagBits::eStencil ? id : ImageId {};
+		aspect       = candidate;
+		subresources = candidate_subresources;
 	}
 	if (!selected) {
 		return false;
@@ -1543,7 +1580,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		clear.depthStencil.stencil = stencil_clear;
 	}
 	ClearImage(command, selected, image.backing.format,
-	           {aspect, 0, image.info.resources.levels, 0, image.info.TransferLayers()}, clear);
+	           {aspect, subresources.base_level, subresources.level_count,
+	            subresources.base_layer, subresources.layer_count}, clear);
+	if (stencil_id) {
+		TrackImage(stencil_id);
+		CommitGpuWrite(m_slot_images[stencil_id]);
+	}
 	return true;
 }
 
@@ -1709,14 +1751,14 @@ void TextureCache::DownloadImage(Image& image, Buffer& destination, uint64_t des
 	}
 
 	auto&      texture   = transfer.texture;
-	const auto transform = texture.swap_bgra16 ? TileManager::ColorTransform::SwapBgra16
-	                                           : TileManager::ColorTransform::None;
+	const auto transform = texture.color_transform;
 	if (texture.tiles.empty()) {
-		if (transform == TileManager::ColorTransform::SwapBgra16) {
+		if (transform != ColorTransform::None) {
 			auto linear = m_tiler.GetScratchBuffer(destination_size);
 			image.Download(texture.regions, linear.buffer, 0, linear.size);
-			m_tiler.SwapBgra16(linear,
-			                   {destination.Handle(), destination_offset, destination_size});
+			m_tiler.TransformColor(linear,
+			                       {destination.Handle(), destination_offset, destination_size},
+			                       transform, false);
 			return;
 		}
 		for (auto& copy: texture.regions) {
@@ -1856,23 +1898,6 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) {
 	}
 }
 
-bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
-	if (!GuestRange {address, size}.Valid()) {
-		return false;
-	}
-	std::scoped_lock lock {m_lock};
-	for (const auto id: FindImagesInRegion(address, size, false)) {
-		const auto& image = m_slot_images[id];
-		// PPSA17168: S_LOAD_DWORD reads shader data at an address overlapping an old
-		// render target whose memory the CPU has reused. The cached image still retains
-		// its earlier GPU-modified flag.
-		if (!image.depth_id && image.IsGpuModified() && !image.IsDefinitelyCpuDirty()) {
-			return true;
-		}
-	}
-	return false;
-}
-
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
 	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);
@@ -1937,6 +1962,11 @@ bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) {
 	return true;
 }
 
+bool TextureCache::IsRegionRegistered(uint64_t address, uint64_t size) {
+	std::scoped_lock lock {m_lock};
+	return !FindImagesInRegion(address, size, false).empty();
+}
+
 void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 	if (!GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid unmap range\n");

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -61,7 +61,7 @@ public:
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
 	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
-	[[nodiscard]] bool IsRegionGpuModified(uint64_t address, uint64_t size);
+	[[nodiscard]] bool IsRegionRegistered(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
 	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -264,7 +264,7 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
 	desc.type              = TextureCache::BindingType::RenderTarget;
 	desc.info.data         = {rt.base.addr, backing_size};
 	desc.info.pixel_format = target_format.format;
-	desc.info.guest_format = transfer_format;
+	desc.info.guest_format = target_format.guest_format;
 	desc.info.type         = image_type;
 	desc.info.extent       = {width, height, depth};
 	desc.info.resources    = {levels, volume ? 1u : view.image_layers};

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -226,43 +226,37 @@ void CommandScheduler::PopPendingOperations() {
 }
 
 void CommandScheduler::DeferOperation(Common::UniqueFunction<void>&& operation) {
+	QueueOperation(std::move(operation), false);
+}
+
+void CommandScheduler::DeferPriorityOperation(Common::UniqueFunction<void>&& operation) {
+	QueueOperation(std::move(operation), true);
+}
+
+void CommandScheduler::QueueOperation(Common::UniqueFunction<void>&& operation, bool priority) {
 	CheckActive();
 	EXIT_IF(!operation);
 	std::unique_lock lock(m_operation_mutex);
 	if (m_operation_state == OperationState::Open) {
-		m_pending_operations.push({std::move(operation), CurrentTick()});
-		return;
-	}
-	if (g_deferred_callback_scheduler == this) {
+		auto& queue = priority ? m_priority_operations : m_pending_operations;
+		queue.push({std::move(operation), CurrentTick()});
 		lock.unlock();
-		operation();
+		if (priority) {
+			m_operation_available.notify_one();
+		}
 		return;
 	}
-	m_operation_available.wait(lock,
-	                           [this] { return m_operation_state == OperationState::Closed; });
+	if (g_deferred_callback_scheduler != this) {
+		m_operation_available.wait(lock,
+		                           [this] { return m_operation_state == OperationState::Closed; });
+	}
 	lock.unlock();
 	operation();
 }
 
-void CommandScheduler::DeferPriorityOperation(Common::UniqueFunction<void>&& operation) {
-	CheckActive();
-	EXIT_IF(!operation);
-	std::unique_lock lock(m_operation_mutex);
-	if (m_operation_state == OperationState::Open) {
-		m_priority_operations.push({std::move(operation), CurrentTick()});
-		lock.unlock();
-		m_operation_available.notify_one();
-		return;
-	}
-	if (g_deferred_callback_scheduler == this) {
-		lock.unlock();
-		operation();
-		return;
-	}
-	m_operation_available.wait(lock,
-	                           [this] { return m_operation_state == OperationState::Closed; });
-	lock.unlock();
-	operation();
+bool CommandScheduler::HasPendingPriorityOperations() {
+	std::lock_guard lock(m_operation_mutex);
+	return !m_priority_operations.empty() || m_priority_active;
 }
 
 void CommandScheduler::PriorityOperationsThread(std::stop_token stop) {

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -38,8 +38,10 @@ public:
 	void                      PopPendingOperations();
 	void                      DrainPriorityOperations();
 	void                      WaitPriorityOperations(uint64_t tick);
+	// Guest-memory completions use the priority queue; normal callbacks maintain GPU resources.
 	void                      DeferOperation(Common::UniqueFunction<void>&& operation);
 	void                      DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
+	[[nodiscard]] bool        HasPendingPriorityOperations();
 	[[nodiscard]] static bool InDeferredOperation() noexcept;
 
 	[[nodiscard]] bool Active() const noexcept { return m_command.m_registers != nullptr; }
@@ -82,6 +84,7 @@ private:
 
 	void BeginNext();
 	void PriorityOperationsThread(std::stop_token stop);
+	void QueueOperation(Common::UniqueFunction<void>&& operation, bool priority);
 	void RunOperation(Common::UniqueFunction<void>&& operation);
 
 	MasterSemaphore              m_master;

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -160,8 +160,7 @@ static TextureCache::ImageDesc MakeDepthTargetDesc(const CommandBuffer& buffer,
 			           z.depth_view.slice_max);
 	}
 	// EXPCLEAR permits an HTile acceleration state; the host attachment is already expanded.
-	if (z.z_info.partially_resident ||
-	    z.stencil_info.partially_resident || z.z_info.max_mip_level != 0 ||
+	if (z.z_info.max_mip_level != 0 ||
 	    z.depth_view.current_mip_level != 0 || unsupported_shading_rate_encoding ||
 	    depth_address == 0 || (depth_address & 0xffffu) != 0) {
 		DepthFatal("unsupported depth register state");
@@ -316,8 +315,6 @@ void RenderExecutor::ResolveRenderDepthTarget(CommandBuffer& buffer, RenderDepth
 	r.depth_compare_op        = static_cast<vk::CompareOp>(dc.zfunc);
 
 	r.depth_bounds_test_enable = dc.depth_bounds_enable;
-	r.depth_min_bounds         = hw.GetDepthBoundsMin();
-	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
 	r.stencil_clear_enable =
 	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -29,8 +29,6 @@ struct RenderDepthInfo {
 	bool                        depth_write_enable       = false;
 	vk::CompareOp               depth_compare_op         = vk::CompareOp::eNever;
 	bool                        depth_bounds_test_enable = false;
-	float                       depth_min_bounds         = 0.0f;
-	float                       depth_max_bounds         = 0.0f;
 	bool                        stencil_clear_enable     = false;
 	uint8_t                     stencil_clear_value      = 0;
 	bool                        stencil_test_enable      = false;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -6,6 +6,7 @@
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "kernel/memory.h"
 
@@ -456,7 +457,7 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
 	return static_cast<uint32_t>(std::min<uint64_t>(rows, capacity / row_size));
 }
 
-void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
+void Image::CopyImageWithBuffer(Image& source, Buffer& buffer, TileManager& tiler) {
 	EXIT_IF(buffer.Handle() == nullptr || source.backing.samples != 1 || backing.samples != 1);
 	m_scheduler.EndRendering();
 	const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
@@ -474,6 +475,8 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 	const uint32_t destination_block = info.IsBlock() ? 4u : 1u;
 	EXIT_IF(levels == 0 || source_bytes == 0 || source_bytes != destination_bytes ||
 	        source_block != destination_block);
+	const auto source_transform = source.info.GetColorTransform();
+	const auto target_transform = info.GetColorTransform();
 
 	vk::BufferMemoryBarrier2 barrier {};
 	barrier.srcStageMask        = vk::PipelineStageFlagBits2::eTransfer;
@@ -535,9 +538,19 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 				command.copyImageToBuffer(source.backing.image,
 				                          vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
 				                          source_copy);
-				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
-				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
-				command.pipelineBarrier2(dependency);
+				if (source_transform != target_transform) {
+					const TileManager::Result bytes {buffer.Handle(), 0, copy_size};
+					if (source_transform != ColorTransform::None) {
+						tiler.TransformColor(bytes, bytes, source_transform, false);
+					}
+					if (target_transform != ColorTransform::None) {
+						tiler.TransformColor(bytes, bytes, target_transform, true);
+					}
+				} else {
+					barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
+					barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
+					command.pipelineBarrier2(dependency);
+				}
 				command.copyBufferToImage(buffer.Handle(), backing.image,
 				                          vk::ImageLayout::eTransferDstOptimal, destination_copy);
 			}
@@ -673,7 +686,9 @@ Prospero::BufferFormat RenderTargetTransferFormat(uint32_t bytes_per_element) {
 } // namespace ImageOps
 
 Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
-    : info(image_info), m_graphics(graphics), m_scheduler(scheduler) {
+    : info(image_info),
+      stencil_subresources {0, image_info.resources.levels, 0, image_info.resources.layers},
+      m_graphics(graphics), m_scheduler(scheduler) {
 	KYTY_PROFILER_FUNCTION();
 	ImageOps::Validate(info);
 	m_cpu_dirty =

src/graphics/host_gpu/renderer/image/image.h

@@ -18,6 +18,7 @@ namespace Libs::Graphics {
 
 class Buffer;
 class CommandScheduler;
+class TileManager;
 struct ImageTestAccess;
 
 using ImageId = Common::SlotId;
@@ -66,7 +67,7 @@ public:
 	void CopyImage(Image& source);
 	void Resolve(Image& source, const ImageSubresourceRange& source_range,
 	             const ImageSubresourceRange& destination_range);
-	void CopyImageWithBuffer(Image& source, Buffer& buffer);
+	void CopyImageWithBuffer(Image& source, Buffer& buffer, TileManager& tiler);
 	void CopyMip(Image& source, uint32_t mip, uint32_t layer);
 
 	void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) {
@@ -148,6 +149,8 @@ public:
 	uint64_t         track_addr     = 0;
 	uint64_t         track_addr_end = 0;
 	ImageId          depth_id {};
+	// The current stencil plane's mapping into the depth image, also retained by its association.
+	ImageSubresourceRange stencil_subresources;
 	uint64_t         tick_accessed_last = 0;
 	size_t           lru_id             = 0;
 

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -15,6 +15,8 @@
 
 namespace Libs::Graphics {
 
+enum class ColorTransform : uint32_t { None, SwapBgra16, Reverse10_11_11 };
+
 enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported };
 
 enum class ImageMetadataKind : uint8_t { None, Htile, Dcc, Cmask };
@@ -70,6 +72,14 @@ struct ImageInfo {
 	bool                         bgra16          = false;
 	std::array<ImageMipInfo, 16> mip_layout {};
 
+	[[nodiscard]] ColorTransform GetColorTransform() const noexcept {
+		if (bgra16) return ColorTransform::SwapBgra16;
+		switch (guest_format) {
+			case Prospero::BufferFormat::k10_11_11Float:
+				return ColorTransform::Reverse10_11_11;
+			default: return ColorTransform::None;
+		}
+	}
 	[[nodiscard]] constexpr bool HasStencil() const noexcept { return !stencil.Empty(); }
 	[[nodiscard]] constexpr bool HasMetadata() const noexcept {
 		return metadata.kind != ImageMetadataKind::None;
@@ -91,7 +101,8 @@ struct ImageInfo {
 	}
 	[[nodiscard]] bool IsCompatible(const ImageInfo& other) const noexcept {
 		return pixel_format == other.pixel_format && samples == other.samples &&
-		       bytes_per_block == other.bytes_per_block;
+		       bytes_per_block == other.bytes_per_block &&
+		       GetColorTransform() == other.GetColorTransform();
 	}
 	[[nodiscard]] int32_t MipOf(const ImageInfo& container) const noexcept {
 		if (container.resources.levels > container.mip_layout.size()) {
@@ -379,7 +390,28 @@ inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {
 		const auto encoded = static_cast<float>(value & 0xffu) / 255.0f;
 		return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f);
 	};
+	const auto float16 = [](uint32_t value) {
+		const auto sign = (value & 0x8000u) << 16u;
+		const auto exponent = (value >> 10u) & 0x1fu;
+		const auto mantissa = value & 0x3ffu;
+		if (exponent == 0) {
+			return std::copysign(static_cast<float>(mantissa) * 0x1p-24f,
+			                     std::bit_cast<float>(sign));
+		}
+		return std::bit_cast<float>(sign | ((exponent == 31 ? 255u : exponent + 112u) << 23u) |
+		                            (mantissa << 13u));
+	};
 	switch (format) {
+		case vk::Format::eR16G16Sfloat:
+			next.float32[1] = float16(packed >> 16u);
+			[[fallthrough]];
+		case vk::Format::eR16Sfloat: next.float32[0] = float16(packed); break;
+		case vk::Format::eR16G16Unorm:
+			next.float32[1] = static_cast<float>(packed >> 16u) / 65535.0f;
+			[[fallthrough]];
+		case vk::Format::eR16Unorm:
+			next.float32[0] = static_cast<float>(packed & 0xffffu) / 65535.0f;
+			break;
 		// A single-plane float target carries its clear as raw float bits, the same encoding the
 		// depth decoder below uses. Without this the clear is discarded and the target keeps stale
 		// contents.
@@ -432,6 +464,10 @@ inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {
 // Keep this separate: the first register word alone cannot describe a 64-bit color.
 [[nodiscard]] inline bool DecodeColorDwordFill(vk::Format format, uint32_t packed,
                                                vk::ClearColorValue& clear) {
+	if ((format == vk::Format::eR16Sfloat || format == vk::Format::eR16Unorm) &&
+	    (packed & 0xffffu) != (packed >> 16u)) {
+		return false;
+	}
 	if (format == vk::Format::eR16G16B16A16Sfloat) {
 		if (packed != 0) {
 			return false;

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -35,6 +35,9 @@ struct HostFormatInfo {
 
 HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
                                  Prospero::ChannelOrder order) {
+	if (guest_format == Prospero::BufferFormat::k10_11_11Float) {
+		return {vk::Format::eB10G11R11UfloatPack32, Prospero::ColorMappingBgra};
+	}
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k8_8_8_8UNorm:
@@ -131,7 +134,8 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layo
 		if (host_format.format != vk::Format::eUndefined && bytes != 0) {
 			const auto order_mapping =
 			    kRenderTargetColorMappings[static_cast<size_t>(order)][encoding.components - 1u];
-			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping)};
+			return {host_format.format, bytes, host_format.host_to_storage.Then(order_mapping),
+			        encoding.buffer_format};
 		}
 	}
 	EXIT("unsupported render-target format combination: layout=%u type=%u order=%u\n",

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -15,6 +15,7 @@ struct RenderTargetFormatInfo {
 	vk::Format                      format            = vk::Format::eUndefined;
 	uint32_t                        bytes_per_element = 0;
 	Prospero::ColorComponentMapping export_mapping;
+	Prospero::BufferFormat          guest_format = Prospero::BufferFormat::kInvalid;
 };
 
 struct SurfaceFormatInfo {

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -13,7 +13,7 @@
 #include "gpu_tiler_shaders/gpu_tiler_standard4_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_standard64_3d_spv.h"
 #include "gpu_tiler_shaders/gpu_tiler_standard64_spv.h"
-#include "gpu_tiler_shaders/gpu_tiler_swap_bgra16_spv.h"
+#include "gpu_tiler_shaders/gpu_tiler_color_transform_spv.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
@@ -31,7 +31,7 @@ TileManager::TileManager(GraphicContext& graphics, CommandScheduler& scheduler,
                          StreamBuffer& stream_buffer)
     : m_graphics(graphics), m_scheduler(scheduler), m_stream_buffer(stream_buffer) {
 	static_assert(FamilyCount == 9);
-	static_assert(sizeof(Push) == 52);
+	static_assert(sizeof(Push) == 60);
 	std::array<vk::DescriptorSetLayoutBinding, 3> bindings {};
 	for (uint32_t index = 0; index < 2; index++) {
 		bindings[index] = {index, vk::DescriptorType::eStorageBuffer, 1,
@@ -77,8 +77,8 @@ TileManager::~TileManager() {
 	if (m_d32_to_d16 != nullptr) {
 		m_graphics.device.destroyPipeline(m_d32_to_d16, nullptr);
 	}
-	if (m_swap_bgra16 != nullptr) {
-		m_graphics.device.destroyPipeline(m_swap_bgra16, nullptr);
+	if (m_color_transform != nullptr) {
+		m_graphics.device.destroyPipeline(m_color_transform, nullptr);
 	}
 	if (m_pipeline_layout != nullptr) {
 		m_graphics.device.destroyPipelineLayout(m_pipeline_layout, nullptr);
@@ -90,7 +90,8 @@ TileManager::~TileManager() {
 
 void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
                           std::span<const GpuTileInfo> infos, uint64_t source_base,
-                          uint64_t target_base, std::vector<Dispatch>& dispatches) {
+                          uint64_t target_base, std::vector<Dispatch>& dispatches,
+                          ColorTransform transform) {
 	EXIT_IF(infos.empty() || tiled_capacity == 0 || linear_capacity == 0);
 	const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	EXIT_NOT_IMPLEMENTED(tiled_capacity > UINT32_MAX || linear_capacity > UINT32_MAX);
@@ -108,6 +109,8 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 	dispatches.clear();
 	dispatches.reserve(infos.size());
 	for (const auto& info: infos) {
+		EXIT_IF((transform == ColorTransform::SwapBgra16 && info.bytes_per_element != 8u) ||
+		        (transform == ColorTransform::Reverse10_11_11 && info.bytes_per_element != 4u));
 		TileBlockLayout block {};
 		const uint32_t  tiled_width  = info.tiled_width != 0 ? info.tiled_width : info.pitch;
 		const uint32_t  tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height;
@@ -193,6 +196,7 @@ void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_ca
 		dispatch.push.tail_x           = info.tail_x;
 		dispatch.push.tail_y           = info.tail_y;
 		dispatch.push.tail             = info.tail;
+		dispatch.push.color_transform  = static_cast<uint32_t>(transform);
 		dispatches.push_back(dispatch);
 	}
 
@@ -338,13 +342,14 @@ void TileManager::Record(vk::Buffer source, uint64_t source_offset,
 
 TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
                                         uint64_t tiled_capacity, uint64_t linear_capacity,
-                                        std::span<const GpuTileInfo> infos) {
+                                        std::span<const GpuTileInfo> infos,
+                                        ColorTransform transform) {
 	const auto&    limits = m_graphics.GetPhysicalDeviceProperties().limits;
 	const uint64_t descriptor_alignment =
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
+	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches, transform);
 	auto scratch = GetScratchBuffer(linear_capacity, tiled);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
@@ -375,15 +380,11 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        target_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	// Reserve stream parameters before recording the image download and conversion.
-	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
+	// Reserve stream parameters before recording the image download.
+	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches, transform);
 	auto linear = GetScratchBuffer(linear_capacity, tiled);
 	image.Download(regions, linear.buffer, 0, linear.size);
-	Result source {linear.buffer, 0, linear.size};
-	if (transform == ColorTransform::SwapBgra16) {
-		source = SwapBgra16(source);
-	}
-	Record(source.buffer, source.offset, linear_capacity, tiled, tiled_offset, tiled_capacity,
+	Record(linear.buffer, 0, linear_capacity, tiled, tiled_offset, tiled_capacity,
 	       dispatches, false);
 }
 
@@ -582,9 +583,20 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	                        &barriers[1], 0, nullptr);
 }
 
-void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
-	if (m_swap_bgra16 == nullptr) {
-		const auto module = CompileSPV(GPU_TILER_SWAP_BGRA16_SPV, m_graphics.device);
+void TileManager::TransformColor(Result input, Result output, ColorTransform transform,
+                                 bool to_host) {
+	uint32_t element_bytes = 0;
+	switch (transform) {
+		case ColorTransform::SwapBgra16: element_bytes = 8; break;
+		case ColorTransform::Reverse10_11_11: element_bytes = 4; break;
+		default: EXIT("invalid color transform\n");
+	}
+	EXIT_IF(input.size == 0 || input.size % element_bytes != 0 ||
+	        input.size / element_bytes > UINT32_MAX ||
+	        output.size < input.size);
+	const auto pixels = static_cast<uint32_t>(input.size / element_bytes);
+	if (m_color_transform == nullptr) {
+		const auto module = CompileSPV(GPU_TILER_COLOR_TRANSFORM_SPV, m_graphics.device);
 		vk::PipelineShaderStageCreateInfo stage {};
 		stage.stage  = vk::ShaderStageFlagBits::eCompute;
 		stage.module = module;
@@ -593,14 +605,12 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 		create.stage  = stage;
 		create.layout = m_pipeline_layout;
 		const auto result =
-		    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_swap_bgra16);
+		    m_graphics.device.createComputePipelines(nullptr, 1, &create, nullptr, &m_color_transform);
 		m_graphics.device.destroyShaderModule(module, nullptr);
-		RequireVulkanSuccess(result, "create BGRA16 swap pipeline");
+		RequireVulkanSuccess(result, "create color transform pipeline");
 	}
-	const uint64_t bytes = static_cast<uint64_t>(pixels) * 8u;
-	EXIT_IF(pixels == 0);
-	const auto input_binding  = BindStorage(input, bytes);
-	const auto output_binding = BindStorage(output, bytes);
+	const auto input_binding  = BindStorage(input, input.size);
+	const auto output_binding = BindStorage(output, input.size);
 
 	const vk::DescriptorBufferInfo infos[] {
 	    input_binding.info,
@@ -631,13 +641,15 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 	command.pipelineBarrier(
 	    vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost,
 	    vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 2, barriers, 0, nullptr);
-	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_swap_bgra16);
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_color_transform);
 	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0,
 	                             static_cast<uint32_t>(writes.size()), writes.data());
 	Push push {};
-	push.src_base = input_binding.base;
-	push.dst_base = output_binding.base;
-	push.width    = pixels;
+	push.src_base        = input_binding.base;
+	push.dst_base        = output_binding.base;
+	push.width           = pixels;
+	push.color_transform = static_cast<uint32_t>(transform);
+	push.to_host         = to_host;
 	command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, sizeof(push),
 	                      &push);
 	command.dispatch((pixels + 63u) / 64u, 1, 1);
@@ -648,17 +660,11 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 	                        0, nullptr);
 }
 
-TileManager::Result TileManager::SwapBgra16(Result input) {
-	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX);
+TileManager::Result TileManager::TransformColor(Result input, ColorTransform transform,
+                                                bool to_host) {
 	auto result = GetScratchBuffer(input.size, input.buffer);
-	SwapBgra16(input, result, static_cast<uint32_t>(input.size / 8u));
+	TransformColor(input, result, transform, to_host);
 	return result;
 }
 
-void TileManager::SwapBgra16(Result input, Result output) {
-	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX ||
-	                     output.size < input.size);
-	SwapBgra16(input, output, static_cast<uint32_t>(input.size / 8u));
-}
-
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/image/tiler.h

@@ -3,6 +3,7 @@
 
 #include "common/common.h"
 #include "graphics/guest_gpu/tile.h"
+#include "graphics/host_gpu/renderer/image/imageInfo.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
@@ -42,7 +43,6 @@ struct GpuTileInfo {
 class TileManager final {
 public:
 	enum class D16Direction { Promote, Demote };
-	enum class ColorTransform { None, SwapBgra16 };
 
 	struct Result {
 		vk::Buffer buffer = nullptr;
@@ -65,7 +65,8 @@ public:
 
 	// Consume scratch results before the next acquisition, or pass their buffer as input.
 	[[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity,
-	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos);
+	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos,
+	                            ColorTransform transform = ColorTransform::None);
 	void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled,
 	          uint64_t tiled_offset, uint64_t tiled_capacity, std::span<const GpuTileInfo> infos);
 	void TileImage(Image& image, std::span<const vk::BufferImageCopy> regions, vk::Buffer tiled,
@@ -75,8 +76,8 @@ public:
 	[[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr);
 	void                 ConvertD16(Result source, Result target, D16Direction direction, bool d32,
 	                                const D16Layout& layout);
-	[[nodiscard]] Result SwapBgra16(Result input);
-	void                 SwapBgra16(Result input, Result output);
+	[[nodiscard]] Result TransformColor(Result input, ColorTransform transform, bool to_host);
+	void TransformColor(Result input, Result output, ColorTransform transform, bool to_host);
 
 private:
 	friend struct TileManagerTestAccess;
@@ -100,6 +101,8 @@ private:
 		uint32_t tail_x;
 		uint32_t tail_y;
 		uint32_t tail;
+		uint32_t color_transform;
+		uint32_t to_host;
 	};
 	struct Dispatch {
 		Push     push {};
@@ -118,12 +121,11 @@ private:
 	                                             uint32_t max_groups) noexcept;
 	void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
 	             std::span<const GpuTileInfo> infos, uint64_t source_base, uint64_t target_base,
-	             std::vector<Dispatch>& dispatches);
+	             std::vector<Dispatch>& dispatches, ColorTransform transform = ColorTransform::None);
 	void Record(vk::Buffer source, uint64_t source_offset, uint64_t source_capacity,
 	            vk::Buffer target, uint64_t target_offset, uint64_t target_capacity,
 	            std::span<Dispatch> dispatches, bool clear_target);
 	[[nodiscard]] vk::Pipeline GetPipeline(uint32_t slot);
-	void                       SwapBgra16(Result input, Result output, uint32_t pixels);
 
 	GraphicContext&                         m_graphics;
 	CommandScheduler&                       m_scheduler;
@@ -132,11 +134,11 @@ private:
 	vk::DescriptorSetLayout                 m_descriptor_layout = nullptr;
 	vk::PipelineLayout                      m_pipeline_layout   = nullptr;
 	std::array<vk::Pipeline, PipelineCount> m_pipelines {};
-	vk::Pipeline                            m_d16_to_d24  = nullptr;
-	vk::Pipeline                            m_d16_to_d32  = nullptr;
-	vk::Pipeline                            m_d24_to_d16  = nullptr;
-	vk::Pipeline                            m_d32_to_d16  = nullptr;
-	vk::Pipeline                            m_swap_bgra16 = nullptr;
+	vk::Pipeline                            m_d16_to_d24      = nullptr;
+	vk::Pipeline                            m_d16_to_d32      = nullptr;
+	vk::Pipeline                            m_d24_to_d16      = nullptr;
+	vk::Pipeline                            m_d32_to_d16      = nullptr;
+	vk::Pipeline                            m_color_transform = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/blendMapping.cpp

@@ -30,6 +30,18 @@ BlendMappingSupport ClassifyBlendMapping(const HW::BlendControl&
 	if (mapping.Map(3) == 3u) {
 		return BlendMappingSupport::Direct;
 	}
+	if (blend.separate_alpha_blend &&
+	    blend.color_srcblend == static_cast<uint8_t>(Prospero::BlendFactor::kSrcAlpha) &&
+	    blend.color_destblend == static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrcAlpha) &&
+	    blend.alpha_destblend == blend.color_destblend &&
+	    blend.color_comb_fcn == static_cast<uint8_t>(Prospero::BlendOp::kAdd) &&
+	    blend.alpha_comb_fcn == blend.color_comb_fcn) {
+		switch (static_cast<Prospero::BlendFactor>(blend.alpha_srcblend)) {
+			case Prospero::BlendFactor::kZero: return BlendMappingSupport::SourceAlphaZero;
+			case Prospero::BlendFactor::kOne: return BlendMappingSupport::SourceAlphaOne;
+			default: break;
+		}
+	}
 	// Moving alpha requires the same equation for all channels.
 	if (blend.separate_alpha_blend && (blend.alpha_srcblend != blend.color_srcblend ||
 	                                   blend.alpha_destblend != blend.color_destblend ||

src/graphics/host_gpu/renderer/pipeline/blendMapping.h

@@ -15,6 +15,8 @@ struct ColorComponentMapping;
 enum class BlendMappingSupport {
 	Direct,
 	SourceAlpha, // Requires logical alpha in the second blend source.
+	SourceAlphaOne, // Source alpha for RGB, one for alpha, with a shared destination equation.
+	SourceAlphaZero, // Source alpha for RGB, zero for alpha, with a shared destination equation.
 	Unsupported,
 };
 

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -68,7 +68,8 @@ vk::DescriptorType NativeDescriptorType(BindingKind kind) {
 		case BindingKind::BdaPagetable:
 		case BindingKind::FaultBuffer:
 		case BindingKind::FlattenedSrt:
-		case BindingKind::ShaderData: return vk::DescriptorType::eStorageBuffer;
+		case BindingKind::ShaderData:
+		case BindingKind::SharedMemory: return vk::DescriptorType::eStorageBuffer;
 		case BindingKind::Count: EXIT("invalid native descriptor binding kind");
 	}
 	EXIT("invalid native descriptor binding kind");
@@ -130,7 +131,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	const auto aligned_offset = Common::AlignDown(offset, alignment);
 	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
-	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
+	if (adjustment >= 256 || size > max_range - adjustment) {
 		EXIT("storage buffer offset adjustment is unsupported\n");
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
@@ -779,6 +780,7 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 	prepared.gds = {nullptr, 0, VK_WHOLE_SIZE};
 	prepared.flattened_srt = {};
 	prepared.shader_data_buffer = {};
+	prepared.shared_memory = {};
 	prepared.images.resize(program.info.images.size());
 	prepared.samplers.clear();
 	prepared.shader_data.clear();
@@ -805,30 +807,52 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 	}
 }
 
-void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
+void RenderExecutor::FindBuffers(std::span<PreparedBindings* const> stages) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
-	const auto& program  = *prepared.runtime->program;
-	const auto& snapshot = *prepared.runtime->resources;
-	auto&       cache    = m_context.GetBufferCache();
-
-	prepared.buffer_sources.clear();
-	const auto& layout = program.bindings;
-	if (layout.memory_offset_count == 0) {
-		return;
-	}
-	const auto& resources = layout.descriptors.front().resources;
-	prepared.buffer_sources.reserve(resources.size());
-	for (const auto resource: resources) {
-		auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[resource]);
-		const auto address = descriptor.Base48();
-		const auto requested_size = descriptor.GetSize();
-		if (address == 0 || requested_size == 0) {
-			prepared.buffer_sources.push_back({});
+	auto& cache = m_context.GetBufferCache();
+	for (auto* stage: stages) {
+		auto& prepared = *stage;
+		EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
+		const auto& program  = *prepared.runtime->program;
+		const auto& snapshot = *prepared.runtime->resources;
+		prepared.buffer_sources.clear();
+		const auto& layout = program.bindings;
+		if (layout.memory_offset_count == 0) {
 			continue;
 		}
-		const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
-		prepared.buffer_sources.push_back({address, size, cache.FindBuffer(address, size)});
+		const auto& resources = layout.descriptors.front().resources;
+		prepared.buffer_sources.reserve(resources.size());
+		for (const auto resource: resources) {
+			const auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[resource]);
+			const auto address = descriptor.Base48();
+			auto size = descriptor.GetSize();
+			if (address == 0 || size == 0) {
+				prepared.buffer_sources.push_back({});
+				continue;
+			}
+			if (descriptor.NumRecords() == UINT32_MAX) {
+				if (program.info.buffers[resource].written) {
+					// Sentinel write ranges end before tables captured by any bound stage.
+					for (const auto* reader: stages) {
+						for (const auto [read_address, read_size]: reader->runtime->resources->specialization_reads) {
+							if (read_size == 0) continue;
+							if (read_address > address) {
+								size = std::min(size, read_address - address);
+							} else if (address - read_address < read_size) {
+								EXIT("scalar resource reads overlap a shader buffer write\n");
+							}
+						}
+					}
+				} else {
+					const auto& graphics = m_context.GetGraphics();
+					const auto limit = uint64_t {graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange} -
+					                   (graphics.StorageMinAlignment() - 1);
+					size = std::min({size, uint64_t {256} * 1024 * 1024, limit});
+				}
+			}
+			size = Libs::LibKernel::Memory::ClampRangeSize(address, size);
+			prepared.buffer_sources.push_back({address, size, cache.FindBuffer(address, size)});
+		}
 	}
 }
 
@@ -876,6 +900,7 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	auto&       images   = prepared.images;
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
+	// Acquire each view before a later overlapping descriptor can replace its image.
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
@@ -887,8 +912,6 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
 		}
-	}
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		auto& binding = images[i];
 		binding.mip_views.clear();
 		const auto& resource = program.info.images[i];
@@ -918,8 +941,8 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
                                              std::span<RenderColorInfo> colors) {
 	bool uses_dma = false;
+	FindBuffers(stages);
 	for (auto* stage: stages) {
-		FindBuffers(*stage);
 		uses_dma |= stage->runtime->program->info.uses_dma;
 	}
 	if (uses_dma) {
@@ -1121,12 +1144,15 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 					}
 					case BindingKind::FlattenedSrt:
 					case BindingKind::ShaderData:
+					case BindingKind::SharedMemory:
 					case BindingKind::Gds: {
 						const vk::DescriptorBufferInfo* view = &descriptors.gds;
 						if (binding.kind == BindingKind::FlattenedSrt) {
 							view = &descriptors.flattened_srt;
 						} else if (binding.kind == BindingKind::ShaderData) {
 							view = &descriptors.shader_data_buffer;
+						} else if (binding.kind == BindingKind::SharedMemory) {
+							view = &descriptors.shared_memory;
 						}
 						EXIT_IF(view->buffer == nullptr);
 						m_descriptor_buffers.push_back(*view);

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -44,6 +44,7 @@ struct PreparedBindings {
 	vk::DescriptorBufferInfo              gds {nullptr, 0, VK_WHOLE_SIZE};
 	vk::DescriptorBufferInfo              flattened_srt;
 	vk::DescriptorBufferInfo              shader_data_buffer;
+	vk::DescriptorBufferInfo              shared_memory;
 	std::vector<uint32_t>                 shader_data;
 };
 

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -39,6 +39,16 @@ namespace Libs::Graphics {
 
 namespace {
 
+uint8_t RemapSourceAlphaFactor(uint8_t factor) {
+	switch (static_cast<Prospero::BlendFactor>(factor)) {
+		case Prospero::BlendFactor::kSrcAlpha:
+			return static_cast<uint8_t>(Prospero::BlendFactor::kSrc1Alpha);
+		case Prospero::BlendFactor::kOneMinusSrcAlpha:
+			return static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrc1Alpha);
+		default: return factor;
+	}
+}
+
 vk::PolygonMode ResolvePolygonMode(const HW::ModeControl& mode, bool cull_front, bool cull_back) {
 	// CxPrimitiveSetup::PolygonMode disables both per-face modes when it is zero.
 	if (mode.poly_mode == 0) {
@@ -94,8 +104,10 @@ void PipelineCacheLog(fmt::format_string<Args...> format, Args&&... args) {
 }
 
 bool ReadShaderGuestMemory(void*, uint64_t address, std::span<uint32_t> values) {
+	// Scalar and unformatted buffer dependencies use the same backing as native raw loads.
+	// Image synchronization belongs to formatted buffer bindings, not these reads.
 	return !values.empty() &&
-	       Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, values.data(), values.size_bytes());
+	       Libs::LibKernel::Memory::TryReadBufferBacking(address, values.data(), values.size_bytes());
 }
 
 void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
@@ -287,11 +299,14 @@ struct PipelineCache::ProgramCache {
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
-		const ShaderRecompiler::IR::SrtRuntime       runtime {
+		ShaderRecompiler::IR::SrtRuntime             runtime {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
 		    .read_specialization_memory = ReadShaderGuestMemory,
 		};
+		if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
+			runtime.workgroup_counts = input_info.workgroup_counts;
+		}
 		if (entry != programs.end()) {
 			EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(
 			    entry->second.resource_plan, runtime, entry->second.resources,
@@ -617,14 +632,24 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 		           pixel_info.target_output_mode[0] != 0 && pixel_info.target_output_mode[0] != 7 &&
 		           std::all_of(std::begin(pixel_info.target_output_mode) + 1,
 		                       std::end(pixel_info.target_output_mode),
-		                       [](uint8_t mode) { return mode == 0; }) &&
-		           ClassifyBlendMapping(blend, pixel_info.target_export_mapping[0]) ==
-		               BlendMappingSupport::SourceAlpha) {
-			// Preserve logical alpha when the export mapping moves it.
-			pixel_info.alpha_blend_source_remap = true;
-			pixel_info.dual_source_blending     = true;
-			pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
-			pixel_info.target_export_mapping[1] = {};
+		                       [](uint8_t mode) { return mode == 0; })) {
+			switch (ClassifyBlendMapping(blend, pixel_info.target_export_mapping[0])) {
+				case BlendMappingSupport::SourceAlpha:
+					pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlpha;
+					break;
+				case BlendMappingSupport::SourceAlphaOne:
+					pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlphaOne;
+					break;
+				case BlendMappingSupport::SourceAlphaZero:
+					pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlphaZero;
+					break;
+				default: break;
+			}
+			if (pixel_info.alpha_blend_source != ShaderAlphaBlendSource::None) {
+				pixel_info.dual_source_blending     = true;
+				pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
+				pixel_info.target_export_mapping[1] = {};
+			}
 		}
 	}
 	if (context.GetClipControl().clip_disable) {
@@ -658,19 +683,8 @@ ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs
                                                ShaderComputeInputInfo&      input_info) {
 	input_info.host_subgroup_size = m_graphics.SupportsComputeWave64() ? 64u : 32u;
 	const auto        params      = PrepareProgram(regs, sh, input_info);
-	// Use one effective size for the cache key, LDS declaration, and access bounds.
-	const auto max_lds_dwords =
-	    m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize / 4u;
-	if (input_info.lds_size_dwords > max_lds_dwords) {
-		static std::atomic_bool warned = false;
-		if (!warned.exchange(true, std::memory_order_relaxed)) {
-			PipelineCacheLog("GPU warning: game compute shader requests {} bytes of LDS, but "
-			                 "the Vulkan device limit is {} bytes. Clamping LDS; rendering may "
-			                 "be incorrect.",
-			                 input_info.lds_size_dwords * 4u, max_lds_dwords * 4u);
-		}
-	}
-	input_info.lds_size_dwords = std::min(input_info.lds_size_dwords, max_lds_dwords);
+	input_info.lds_storage = input_info.lds_size_dwords * 4u >
+	    m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize;
 	uint32_t          push_data_cursor = 0;
 	return m_program_cache->Get(params, input_info, push_data_cursor);
 }
@@ -727,10 +741,12 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		}
 		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
 		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		const bool alpha_remap =
-		    slot == 0 && ps_input_info != nullptr && ps_input_info->alpha_blend_source_remap;
+		auto alpha_source = ShaderAlphaBlendSource::None;
+		if (slot == 0 && ps_input_info != nullptr) {
+			alpha_source = ps_input_info->alpha_blend_source;
+		}
 		static_params.blend_enable[slot] = bc.enable && !rt.info.blend_bypass;
-		if (static_params.blend_enable[slot] && !alpha_remap &&
+		if (static_params.blend_enable[slot] && alpha_source == ShaderAlphaBlendSource::None &&
 		    ClassifyBlendMapping(bc, colors[i].export_mapping) != BlendMappingSupport::Direct) {
 			static_params.blend_enable[slot] = false;
 			static std::atomic_bool warned = false;
@@ -742,18 +758,32 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 				    bc.alpha_srcblend, bc.alpha_destblend, bc.separate_alpha_blend ? 1 : 0));
 			}
 		}
-		if (alpha_remap) {
-			static_params.blend_alpha_source_remap = true;
-		}
 		if (static_params.blend_enable[slot]) {
-			static_params.color_srcblend[slot]       = bc.color_srcblend;
-			static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
-			static_params.color_destblend[slot]      = bc.color_destblend;
-			static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
-			if (bc.separate_alpha_blend) {
-				static_params.alpha_srcblend[slot]  = bc.alpha_srcblend;
-				static_params.alpha_comb_fcn[slot]  = bc.alpha_comb_fcn;
-				static_params.alpha_destblend[slot] = bc.alpha_destblend;
+			auto blend = bc;
+			switch (alpha_source) {
+				case ShaderAlphaBlendSource::SourceAlpha:
+					blend.color_srcblend  = RemapSourceAlphaFactor(blend.color_srcblend);
+					blend.color_destblend = RemapSourceAlphaFactor(blend.color_destblend);
+					blend.separate_alpha_blend = false;
+					break;
+				case ShaderAlphaBlendSource::SourceAlphaOne:
+				case ShaderAlphaBlendSource::SourceAlphaZero:
+					// The second source carries the mapped source factor; its alpha stays logical Sa.
+					blend.color_srcblend = static_cast<uint8_t>(Prospero::BlendFactor::kSrc1Color);
+					blend.color_destblend =
+					    static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrc1Alpha);
+					blend.separate_alpha_blend = false;
+					break;
+				case ShaderAlphaBlendSource::None: break;
+			}
+			static_params.color_srcblend[slot]       = blend.color_srcblend;
+			static_params.color_comb_fcn[slot]       = blend.color_comb_fcn;
+			static_params.color_destblend[slot]      = blend.color_destblend;
+			static_params.separate_alpha_blend[slot] = blend.separate_alpha_blend;
+			if (blend.separate_alpha_blend) {
+				static_params.alpha_srcblend[slot]  = blend.alpha_srcblend;
+				static_params.alpha_comb_fcn[slot]  = blend.alpha_comb_fcn;
+				static_params.alpha_destblend[slot] = blend.alpha_destblend;
 			}
 		}
 	}
@@ -801,9 +831,6 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 	if (static_params.sample_shading_enable && !m_graphics.sample_rate_shading_enabled) {
 		EXIT("Pipeline: sample-rate shading is required but unsupported by the host\n");
 	}
-	static_params.depth_bounds_test_enable = depth.depth_bounds_test_enable;
-	static_params.depth_min_bounds         = depth.depth_min_bounds;
-	static_params.depth_max_bounds         = depth.depth_max_bounds;
 	const bool rect_list = Prospero::IsRectList(command.GetUserConfig().GetPrimType());
 	static_params.cull_back  = !rect_list && mc.cull_back;
 	static_params.cull_front = !rect_list && mc.cull_front;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -38,9 +38,6 @@ struct PipelineStaticParameters {
 	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
-	bool                       depth_bounds_test_enable = false;
-	float                      depth_min_bounds         = 0.0f;
-	float                      depth_max_bounds         = 0.0f;
 	uint32_t                   color_mask[RENDER_COLOR_ATTACHMENTS_MAX]           = {};
 	bool                       cull_front                                         = false;
 	bool                       cull_back                                          = false;
@@ -55,7 +52,6 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
-	bool                       blend_alpha_source_remap                           = false;
 
 	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
@@ -65,7 +61,7 @@ struct PipelineStaticParameters {
 static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
 static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 126);
+static_assert(sizeof(PipelineStaticParameters) == 116);
 
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -137,17 +137,14 @@ static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format,
 	}
 }
 
-static vk::BlendFactor GetBlendFactor(uint32_t factor, bool remap_source_alpha) {
+static vk::BlendFactor GetBlendFactor(uint32_t factor) {
 	switch (static_cast<Prospero::BlendFactor>(factor)) {
 		case Prospero::BlendFactor::kZero: return vk::BlendFactor::eZero;
 		case Prospero::BlendFactor::kOne: return vk::BlendFactor::eOne;
 		case Prospero::BlendFactor::kSrcColor: return vk::BlendFactor::eSrcColor;
 		case Prospero::BlendFactor::kOneMinusSrcColor: return vk::BlendFactor::eOneMinusSrcColor;
-		case Prospero::BlendFactor::kSrcAlpha:
-			return remap_source_alpha ? vk::BlendFactor::eSrc1Color : vk::BlendFactor::eSrcAlpha;
-		case Prospero::BlendFactor::kOneMinusSrcAlpha:
-			return remap_source_alpha ? vk::BlendFactor::eOneMinusSrc1Color
-			                          : vk::BlendFactor::eOneMinusSrcAlpha;
+		case Prospero::BlendFactor::kSrcAlpha: return vk::BlendFactor::eSrcAlpha;
+		case Prospero::BlendFactor::kOneMinusSrcAlpha: return vk::BlendFactor::eOneMinusSrcAlpha;
 		case Prospero::BlendFactor::kDstAlpha: return vk::BlendFactor::eDstAlpha;
 		case Prospero::BlendFactor::kOneMinusDstAlpha: return vk::BlendFactor::eOneMinusDstAlpha;
 		case Prospero::BlendFactor::kDstColor: return vk::BlendFactor::eDstColor;
@@ -397,20 +394,18 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		color_blend_attachment[i].colorWriteMask =
 		    vk::ColorComponentFlags {static_params.color_mask[i]};
 		color_blend_attachment[i].blendEnable = static_params.blend_enable[i] ? VK_TRUE : VK_FALSE;
-		// Only target 0 can use the second blend source carrying logical alpha.
-		const bool remap_source_alpha         = i == 0 && static_params.blend_alpha_source_remap;
 		color_blend_attachment[i].srcColorBlendFactor =
-		    GetBlendFactor(static_params.color_srcblend[i], remap_source_alpha);
+		    GetBlendFactor(static_params.color_srcblend[i]);
 		color_blend_attachment[i].dstColorBlendFactor =
-		    GetBlendFactor(static_params.color_destblend[i], remap_source_alpha);
+		    GetBlendFactor(static_params.color_destblend[i]);
 		color_blend_attachment[i].colorBlendOp = GetBlendOp(static_params.color_comb_fcn[i]);
 		color_blend_attachment[i].srcAlphaBlendFactor =
 		    (static_params.separate_alpha_blend[i]
-		         ? GetBlendFactor(static_params.alpha_srcblend[i], remap_source_alpha)
+		         ? GetBlendFactor(static_params.alpha_srcblend[i])
 		         : color_blend_attachment[i].srcColorBlendFactor);
 		color_blend_attachment[i].dstAlphaBlendFactor =
 		    (static_params.separate_alpha_blend[i]
-		         ? GetBlendFactor(static_params.alpha_destblend[i], remap_source_alpha)
+		         ? GetBlendFactor(static_params.alpha_destblend[i])
 		         : color_blend_attachment[i].dstColorBlendFactor);
 		color_blend_attachment[i].alphaBlendOp =
 		    (static_params.separate_alpha_blend[i] ? GetBlendOp(static_params.alpha_comb_fcn[i])
@@ -476,14 +471,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	EXIT_NOT_IMPLEMENTED(pipeline.pipeline_layout == nullptr);
 
 	vk::PipelineDepthStencilStateCreateInfo depth_stencil_info {};
-	depth_stencil_info.depthBoundsTestEnable =
-#if defined(__APPLE__)
-	    VK_FALSE; // MoltenVK lacks the depthBounds feature; depth-bounds testing is disabled
-#else
-	    (static_params.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
-#endif
-	depth_stencil_info.minDepthBounds    = static_params.depth_min_bounds;
-	depth_stencil_info.maxDepthBounds    = static_params.depth_max_bounds;
 
 	std::vector<vk::DynamicState> dynamic_states {
 	    vk::DynamicState::eViewportWithCount,
@@ -502,6 +489,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	    vk::DynamicState::eBlendConstants,
 	};
 #if !defined(__APPLE__)
+	dynamic_states.push_back(vk::DynamicState::eDepthBoundsTestEnable);
+	dynamic_states.push_back(vk::DynamicState::eDepthBounds);
 	if (rendering.color_count != 0) {
 		dynamic_states.push_back(vk::DynamicState::eColorWriteEnableEXT);
 	}

src/graphics/host_gpu/renderer/render.h

@@ -161,7 +161,7 @@ public:
 	                      uint32_t mode);
 
 	void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared);
-	void                           FindBuffers(PreparedBindings& bindings);
+	void                           FindBuffers(std::span<PreparedBindings* const> stages);
 	void                           RebindBuffers(PreparedBindings& bindings);
 	void                           RebindImages(PreparedBindings& bindings);
 	void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -20,6 +20,7 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
 #include "kernel/pthread.h"
@@ -195,6 +196,35 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i
 	return true;
 }
 
+static void BindSharedMemory(RenderContext& context, ShaderComputeInputInfo& input,
+                             PreparedBindings& bindings, uint64_t indirect_args = 0) {
+	if (ShaderRecompiler::IR::FindBinding(input.stage.program->bindings,
+	        ShaderRecompiler::IR::DescriptorBindingKind::SharedMemory) == nullptr) {
+		return;
+	}
+	auto& cache = context.GetBufferCache();
+	if (indirect_args != 0) {
+		cache.ReadMemory(indirect_args, sizeof(vk::DispatchIndirectCommand));
+		std::memcpy(input.workgroup_counts, reinterpret_cast<const void*>(indirect_args),
+		            sizeof(input.workgroup_counts));
+	}
+	// LDS has no contents to preserve between dispatches. The existing shader hazard
+	// barriers also order other users of this GPU-only utility buffer.
+	auto& storage = cache.GetUtilityBuffer(MemoryUsage::DeviceLocal);
+	const auto limit = std::min<uint64_t>(storage.Size(),
+	    context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
+	uint64_t size = sizeof(uint32_t);
+	if (std::ranges::find(input.workgroup_counts, 0u) == std::end(input.workgroup_counts)) {
+		size = uint64_t {input.lds_size_dwords} * sizeof(uint32_t);
+		EXIT_IF(size == 0 || size > limit);
+		for (const auto count: input.workgroup_counts) {
+			EXIT_IF(size > limit / count);
+			size *= count;
+		}
+	}
+	bindings.shared_memory = {storage.Handle(), 0, size};
+}
+
 void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
                                     uint32_t thread_group_x, uint32_t thread_group_y,
                                     uint32_t thread_group_z, uint32_t mode) {
@@ -254,6 +284,18 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	ShaderComputeInputInfo input_info {};
 	const bool use_thread_dimensions = (mode & DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0;
 	input_info.dispatch_thread_dimensions = use_thread_dimensions;
+	input_info.workgroup_counts[0] = thread_group_x;
+	input_info.workgroup_counts[1] = thread_group_y;
+	input_info.workgroup_counts[2] = thread_group_z;
+	if (use_thread_dimensions) {
+		const uint32_t group_sizes[] = {cs_regs.cs_regs.num_thread_x, cs_regs.cs_regs.num_thread_y,
+		                                cs_regs.cs_regs.num_thread_z};
+		for (uint32_t axis = 0; axis < 3u; ++axis) {
+			const auto size = std::max(group_sizes[axis], 1u);
+			const auto threads = input_info.workgroup_counts[axis];
+			input_info.workgroup_counts[axis] = threads / size + (threads % size != 0u);
+		}
+	}
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
 	if (use_thread_dimensions) {
@@ -272,22 +314,75 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	const bool large_workgroup =
-	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
-	const bool                   has_sampler = !program.info.samplers.empty();
-	static std::atomic<uint32_t> dispatch_log_count {0};
-	if ((large_workgroup || has_sampler) &&
-	    dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) {
+	if (use_thread_dimensions) {
+		const uint32_t old_x = thread_group_x;
+		const uint32_t old_y = thread_group_y;
+		const uint32_t old_z = thread_group_z;
+		thread_group_x       = input_info.workgroup_counts[0];
+		thread_group_y       = input_info.workgroup_counts[1];
+		thread_group_z       = input_info.workgroup_counts[2];
+
+		static std::atomic<uint32_t> log_count {0};
+		if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
+			LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> "
+			     "groups %ux%ux%u\n",
+			     old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u),
+			     std::max(cs_regs.cs_regs.num_thread_y, 1u),
+			     std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y,
+			     thread_group_z);
+		}
+	}
+
+	buffer.EndRendering();
+	auto& pipeline =
+	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
+	auto& bindings = m_compute_bindings;
+	PrepareBindings(input_info.stage, bindings);
+	if (program.bindings.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
+		std::copy(std::begin(input_info.dispatch_threads_num), std::end(input_info.dispatch_threads_num),
+		          bindings.shader_data.begin() + program.bindings.dispatch_thread_dword);
+	}
+	PreparedBindings* descriptor_stage = &bindings;
+	FindBuffers(std::span {&descriptor_stage, 1u});
+	if (program.info.uses_dma) {
+		m_context.PrepareBda();
+	}
+	RebindImages(bindings);
+	BindSharedMemory(m_context, input_info, bindings);
+	RebindBuffers(bindings);
+
+	auto              vk_buffer        = buffer.Handle();
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
+	bool has_storage_writes = bindings.shared_memory.buffer != nullptr ||
+	    HasShaderBufferWrites(input_info.stage);
+	has_storage_writes =
+	    std::any_of(program.info.images.begin(), program.info.images.end(),
+	                [](const auto& image) {
+		                return image.written &&
+		                       image.resource_class ==
+		                           ShaderRecompiler::IR::ImageResourceClass::Storage;
+	                }) ||
+	    has_storage_writes;
+	if (has_storage_writes) {
+		// A host fence used to serialize every dispatch. Preserve its read-before-write ordering
+		// while allowing the queue to execute asynchronously.
+		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
+	}
+	if (Config::GraphicsDebugDumpEnabled()) {
 		const auto sampled_images = std::count_if(
 		    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
 			    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
 		    });
 		const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
 		LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64
+		     " hash=0x%016" PRIx64 " tick=%" PRIu64
 		     " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u "
 		     "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n",
-		     frame_num, sh_ctx.GetCs().cs_regs.data_addr, thread_group_x, thread_group_y,
-		     thread_group_z, mode, input_info.threads_num[0], input_info.threads_num[1],
+		     frame_num, sh_ctx.GetCs().cs_regs.data_addr, program.shader_hash,
+		     m_context.GetCommandScheduler().CurrentTick(),
+		     thread_group_x, thread_group_y, thread_group_z, mode,
+		     input_info.threads_num[0], input_info.threads_num[1],
 		     input_info.threads_num[2], program.info.buffers.size(), program.info.images.size(),
 		     sampled_images, program.info.images.size() - sampled_images,
 		     program.info.samplers.size(),
@@ -334,61 +429,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		}
 	}
 
-	if (use_thread_dimensions) {
-		auto groups_from_threads = [](uint32_t threads, uint32_t group_size) {
-			return (threads == 0
-			            ? 0u
-			            : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u));
-		};
-
-		const uint32_t old_x = thread_group_x;
-		const uint32_t old_y = thread_group_y;
-		const uint32_t old_z = thread_group_z;
-		thread_group_x       = groups_from_threads(thread_group_x, cs_regs.cs_regs.num_thread_x);
-		thread_group_y       = groups_from_threads(thread_group_y, cs_regs.cs_regs.num_thread_y);
-		thread_group_z       = groups_from_threads(thread_group_z, cs_regs.cs_regs.num_thread_z);
-
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
-			LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> "
-			     "groups %ux%ux%u\n",
-			     old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u),
-			     std::max(cs_regs.cs_regs.num_thread_y, 1u),
-			     std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y,
-			     thread_group_z);
-		}
-	}
-
-	buffer.EndRendering();
-	auto& pipeline =
-	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
-	auto& bindings = m_compute_bindings;
-	PrepareBindings(input_info.stage, bindings);
-	FindBuffers(bindings);
-	if (program.info.uses_dma) {
-		m_context.PrepareBda();
-	}
-	RebindImages(bindings);
-	RebindBuffers(bindings);
-
-	auto              vk_buffer        = buffer.Handle();
-	PreparedBindings* descriptor_stage = &bindings;
-	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
-	               std::span {&descriptor_stage, 1u});
-	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
-	has_storage_writes =
-	    std::any_of(program.info.images.begin(), program.info.images.end(),
-	                [](const auto& image) {
-		                return image.written &&
-		                       image.resource_class ==
-		                           ShaderRecompiler::IR::ImageResourceClass::Storage;
-	                }) ||
-	    has_storage_writes;
-	if (has_storage_writes) {
-		// A host fence used to serialize every dispatch. Preserve its read-before-write ordering
-		// while allowing the queue to execute asynchronously.
-		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
-	}
 	vk_buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline);
 	vk_buffer.dispatch(thread_group_x, thread_group_y, thread_group_z);
 
@@ -417,22 +457,24 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto& bindings = m_compute_bindings;
 	PrepareBindings(input_info.stage, bindings);
-	FindBuffers(bindings);
+	PreparedBindings* descriptor_stage = &bindings;
+	FindBuffers(std::span {&descriptor_stage, 1u});
 	const auto& program = *input_info.stage.program;
 	if (program.info.uses_dma) {
 		m_context.PrepareBda();
 	}
+	BindSharedMemory(m_context, input_info, bindings, args_addr);
 	RebindImages(bindings);
 	// Acquiring arguments can merge cache buffers; finalize shader bindings afterward.
 	const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer(
 	    args_addr, sizeof(vk::DispatchIndirectCommand), false);
 	EXIT_IF(args_buffer == nullptr || (args_offset & 3u) != 0);
 	RebindBuffers(bindings);
-	PreparedBindings* descriptor_stage = &bindings;
 	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
 	               std::span {&descriptor_stage, 1u});
 	const auto vk_buffer = buffer.Handle();
-	const bool has_storage_writes = HasShaderBufferWrites(input_info.stage) ||
+	const bool has_storage_writes = bindings.shared_memory.buffer != nullptr ||
+	    HasShaderBufferWrites(input_info.stage) ||
 	    std::any_of(program.info.images.begin(), program.info.images.end(), [](const auto& image) {
 		    return image.written && image.resource_class ==
 		                                ShaderRecompiler::IR::ImageResourceClass::Storage;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -99,7 +99,12 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		     vaddr, size);
 	}
 	const auto unmap = [this, vaddr, size] {
-		if (m_command_scheduler.Active()) {
+		// Check cache ownership on the GPU thread. Guest-memory callbacks can still
+		// access a range with no cached data, so they must finish before it is unmapped.
+		if (m_command_scheduler.Active() &&
+		    (m_buffer_cache.IsRegionRegistered(vaddr, size) ||
+		     m_texture_cache.IsRegionRegistered(vaddr, size) ||
+		     m_command_scheduler.HasPendingPriorityOperations())) {
 			const auto tick = m_command_scheduler.CurrentTick();
 			m_command_scheduler.Finish();
 			m_command_scheduler.WaitPriorityOperations(tick);

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -379,6 +379,12 @@ static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuf
 	vk_buffer.setDepthTestEnable(depth.depth_test_enable ? VK_TRUE : VK_FALSE);
 	vk_buffer.setDepthWriteEnable(depth.depth_write_enable ? VK_TRUE : VK_FALSE);
 	vk_buffer.setDepthCompareOp(depth.depth_compare_op);
+#if !defined(__APPLE__)
+	vk_buffer.setDepthBoundsTestEnable(depth.depth_bounds_test_enable ? VK_TRUE : VK_FALSE);
+	if (depth.depth_bounds_test_enable) {
+		vk_buffer.setDepthBounds(ctx.GetDepthBoundsMin(), ctx.GetDepthBoundsMax());
+	}
+#endif
 
 	const auto& mode              = ctx.GetModeControl();
 	const auto& poly_offset       = ctx.GetPolyOffset();
@@ -784,6 +790,7 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, vk::PrimitiveTopology& t
 			topology = vk::PrimitiveTopology::eTriangleList;
 			break;
 		case Prospero::PrimitiveType::kTriFan:
+		case Prospero::PrimitiveType::kPolygon:
 			topology = vk::PrimitiveTopology::eTriangleFan;
 			break;
 		case Prospero::PrimitiveType::kTriStrip:
@@ -994,6 +1001,7 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 		case Prospero::PrimitiveType::kTriFan:
 		case Prospero::PrimitiveType::kTriStrip:
 		case Prospero::PrimitiveType::kRectList:
+		case Prospero::PrimitiveType::kPolygon:
 		case Prospero::PrimitiveType::kRectListLegacy:
 		case Prospero::PrimitiveType::kPatch:
 			if (draw.IsIndexed()) {

src/graphics/host_gpu/shaders/gpu_tiler_color.inc

@@ -0,0 +1,20 @@
+// ColorTransform values match imageInfo.h. Preserve the floating-point bit fields exactly.
+uvec2 transform_color(uvec2 value, uint transform, bool to_host) {
+	switch (transform) {
+		case 1u:
+			return uvec2((value.y & 0xffffu) | (value.x & 0xffff0000u),
+			             (value.x & 0xffffu) | (value.y & 0xffff0000u));
+		case 2u: {
+			uint packed = value.x;
+			if (to_host) {
+				value.x = (packed >> 21u) | (((packed >> 10u) & 0x7ffu) << 11u) |
+				          ((packed & 0x3ffu) << 22u);
+			} else {
+				value.x = (packed >> 22u) | (((packed >> 11u) & 0x7ffu) << 10u) |
+				          ((packed & 0x7ffu) << 21u);
+			}
+			break;
+		}
+	}
+	return value;
+}

src/graphics/host_gpu/shaders/gpu_tiler_color_transform.comp

@@ -1,4 +1,5 @@
 #version 450
+#extension GL_GOOGLE_include_directive : require
 
 layout(local_size_x = 64) in;
 
@@ -19,17 +20,23 @@ layout(push_constant, std430) uniform Push {
 	uint tail_x;
 	uint tail_y;
 	uint tail;
+	uint color_transform;
+	uint to_host;
 } params;
 
+#include "gpu_tiler_color.inc"
+
 void main() {
 	uint pixel = gl_GlobalInvocationID.x;
 	if (pixel >= params.width) {
 		return;
 	}
-	uint src = (params.src_base >> 2u) + pixel * 2u;
-	uint dst = (params.dst_base >> 2u) + pixel * 2u;
-	uint bg = input_buffer.data[src];
-	uint ra = input_buffer.data[src + 1u];
-	output_buffer.data[dst] = (ra & 0x0000ffffu) | (bg & 0xffff0000u);
-	output_buffer.data[dst + 1u] = (bg & 0x0000ffffu) | (ra & 0xffff0000u);
+	uint words = params.color_transform == 1u ? 2u : 1u;
+	uint src = (params.src_base >> 2u) + pixel * words;
+	uint dst = (params.dst_base >> 2u) + pixel * words;
+	uvec2 value = uvec2(input_buffer.data[src], 0u);
+	if (words == 2u) value.y = input_buffer.data[src + 1u];
+	value = transform_color(value, params.color_transform, params.to_host != 0u);
+	output_buffer.data[dst] = value.x;
+	if (words == 2u) output_buffer.data[dst + 1u] = value.y;
 }

src/graphics/host_gpu/shaders/gpu_tiler_common.inc

@@ -17,10 +17,19 @@ layout(set = 0, binding = 2, std140) uniform Push {
 	uint tail_x;
 	uint tail_y;
 	uint tail;
+	uint color_transform;
 } params;
 
+#include "gpu_tiler_color.inc"
+
 void copy_element(uint src, uint dst) {
-	if (ELEMENT_BYTES >= 4) {
+	if ((ELEMENT_BYTES == 4u || ELEMENT_BYTES == 8u) && params.color_transform != 0u) {
+		uvec2 value = uvec2(input_buffer.data[src >> 2], 0u);
+		if (ELEMENT_BYTES == 8u) value.y = input_buffer.data[(src >> 2) + 1u];
+		value = transform_color(value, params.color_transform, TILE == 0u);
+		output_buffer.data[dst >> 2] = value.x;
+		if (ELEMENT_BYTES == 8u) output_buffer.data[(dst >> 2) + 1u] = value.y;
+	} else if (ELEMENT_BYTES >= 4) {
 		for (uint i = 0; i < ELEMENT_BYTES; i += 4) {
 			output_buffer.data[(dst + i) >> 2] = input_buffer.data[(src + i) >> 2];
 		}

src/graphics/host_gpu/vulkanCommon.cpp

@@ -48,6 +48,7 @@ constexpr FormatMapping kFormatMappings[] = {
     {Prospero::BufferFormat::k11_11_10Float, vk::Format::eB10G11R11UfloatPack32},
     {Prospero::BufferFormat::k10_10_10_2UNorm, vk::Format::eA2B10G10R10UnormPack32},
     {Prospero::BufferFormat::k10_10_10_2SNorm, vk::Format::eA2B10G10R10SnormPack32},
+    {Prospero::BufferFormat::k10_10_10_2UScaled, vk::Format::eA2B10G10R10UscaledPack32},
     {Prospero::BufferFormat::k10_10_10_2UInt, vk::Format::eA2B10G10R10UintPack32},
     {Prospero::BufferFormat::k8_8_8_8UNorm, vk::Format::eR8G8B8A8Unorm},
     {Prospero::BufferFormat::k8_8_8_8SNorm, vk::Format::eR8G8B8A8Snorm},

src/graphics/presentation/systemOverlay.cpp

@@ -3,14 +3,17 @@
 #include <SDL3/SDL.h>
 
 #include "common/assert.h"
+#include "common/logging/log.h"
 #include "common/stringUtils.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "imgui.h"
+#include "imgui_internal.h"
 #include "imgui_impl_vulkan.h"
 #include "libs/controller.h"
 #include "libs/dialog.h"
 #include "libs/ime.h"
 #include "libs/imeDialog.h"
+#include "stb_image.h"
 
 #include <algorithm>
 #include <array>
@@ -18,12 +21,17 @@
 #include <chrono>
 #include <cmath>
 #include <cstdint>
+#include <cstring>
 #include <deque>
+#include <limits>
+#include <memory>
 #include <mutex>
+#include <optional>
 #include <span>
 #include <string>
 #include <string_view>
 #include <utility>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -178,6 +186,14 @@ bool                         g_input_lifecycle_active     = false;
 bool                         g_controller_captured        = false;
 OverlaySession               g_session;
 SDL_Window*                  g_input_window               = nullptr;
+struct TrophyNotification {
+	std::string            name;
+	int32_t                grade;
+	std::vector<std::byte> icon_png;
+};
+std::mutex                     g_trophy_notification_mutex;
+std::deque<TrophyNotification> g_trophy_notifications;
+std::atomic<bool>              g_trophy_notification_active {false};
 
 void ClearInputEvents() {
 	std::scoped_lock lock(g_input_mutex);
@@ -467,8 +483,9 @@ SystemOverlayVisualState GetSystemOverlayVisualState() noexcept {
 	const auto core   = CoreIme::GetVisualState();
 	const auto dialog = DialogIme::GetVisualState();
 	const auto system = SystemDialog::GetVisualState();
-	return {core.active || dialog.active || system.active,
-	        core.revision + dialog.revision + system.revision};
+	const bool trophy_active = g_trophy_notification_active.load(std::memory_order_acquire);
+	return {core.active || dialog.active || system.active || trophy_active,
+	        core.revision + dialog.revision + system.revision + (trophy_active ? 1u : 0u)};
 }
 
 bool ProcessSystemOverlayInput(const SDL_Event& event) {
@@ -599,12 +616,34 @@ bool ProcessSystemOverlayInput(const SDL_Event& event) {
 	}
 }
 
+void NotifyTrophyUnlocked(std::string_view name, int32_t grade,
+                          std::span<const std::byte> icon_png) {
+	std::scoped_lock lock(g_trophy_notification_mutex);
+	g_trophy_notifications.push_back(
+	    {std::string(name), grade, std::vector<std::byte>(icon_png.begin(), icon_png.end())});
+	g_trophy_notification_active.store(true, std::memory_order_release);
+}
+
 struct SystemOverlay::Impl {
+	using Clock = std::chrono::steady_clock;
+
 	explicit Impl(GraphicContext& context): graphics(context) {}
 
 	~Impl() {
+		SDL_DestroyAudioStream(trophy_sound_stream);
+		SDL_free(trophy_sound_data);
+		if (trophy_sound_initialized) {
+			SDL_QuitSubSystem(SDL_INIT_AUDIO);
+		}
 		ReleaseVulkan();
 		if (imgui_context != nullptr) {
+			ImGui::SetCurrentContext(imgui_context);
+			if (trophy_notification_image) {
+				ImGui::UnregisterUserTexture(trophy_notification_image.get());
+			}
+			for (const auto& texture: retired_trophy_images) {
+				ImGui::UnregisterUserTexture(texture.get());
+			}
 			ImGui::DestroyContext(imgui_context);
 		}
 	}
@@ -631,6 +670,75 @@ struct SystemOverlay::Impl {
 		style.ItemSpacing    = {8.0f, 8.0f};
 	}
 
+	void PlayTrophySound() {
+		if (!trophy_sound_attempted) {
+			trophy_sound_attempted   = true;
+			trophy_sound_initialized = SDL_InitSubSystem(SDL_INIT_AUDIO);
+			if (!trophy_sound_initialized) {
+				LOGF("[Trophy] could not initialize audio: %s\n", SDL_GetError());
+				return;
+			}
+			const char* base_path = SDL_GetBasePath();
+			if (base_path == nullptr) {
+				LOGF("[Trophy] could not locate sound: %s\n", SDL_GetError());
+				return;
+			}
+			const std::string path = std::string(base_path) + "assets/sounds/trophy-unlock.wav";
+			SDL_AudioSpec     spec {};
+			if (!SDL_LoadWAV(path.c_str(), &spec, &trophy_sound_data, &trophy_sound_size)) {
+				LOGF("[Trophy] could not load %s: %s\n", path.c_str(), SDL_GetError());
+				return;
+			}
+			if (trophy_sound_size > static_cast<uint32_t>(std::numeric_limits<int>::max())) {
+				LOGF("[Trophy] sound is too large: %s\n", path.c_str());
+				return;
+			}
+			trophy_sound_stream = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK,
+			                                                &spec, nullptr, nullptr);
+			if (trophy_sound_stream == nullptr) {
+				LOGF("[Trophy] could not open audio stream: %s\n", SDL_GetError());
+				return;
+			}
+		}
+		if (trophy_sound_stream != nullptr &&
+		    (!SDL_PutAudioStreamData(trophy_sound_stream, trophy_sound_data,
+		                             static_cast<int>(trophy_sound_size)) ||
+		     !SDL_ResumeAudioStreamDevice(trophy_sound_stream))) {
+			LOGF("[Trophy] could not play sound: %s\n", SDL_GetError());
+		}
+	}
+
+	std::unique_ptr<ImTextureData> LoadTrophyIcon(std::span<const std::byte> encoded) {
+		if (encoded.empty() ||
+		    encoded.size() > static_cast<std::size_t>(std::numeric_limits<int>::max())) {
+			return nullptr;
+		}
+
+		const auto* data   = reinterpret_cast<const stbi_uc*>(encoded.data());
+		const int   size   = static_cast<int>(encoded.size());
+		int         width  = 0;
+		int         height = 0;
+		if (!stbi_info_from_memory(data, size, &width, &height, nullptr) || width <= 0 ||
+		    height <= 0 || width > 512 || height > 512) {
+			LOGF("[Trophy] invalid icon or dimensions exceed 512x512\n");
+			return nullptr;
+		}
+		auto* pixels = stbi_load_from_memory(data, size, &width, &height, nullptr, 4);
+		if (pixels == nullptr) {
+			LOGF("[Trophy] could not decode trophy icon: %s\n", stbi_failure_reason());
+			return nullptr;
+		}
+
+		auto texture = std::make_unique<ImTextureData>();
+		texture->Create(ImTextureFormat_RGBA32, width, height);
+		std::memcpy(texture->Pixels, pixels,
+		            static_cast<std::size_t>(width) * static_cast<std::size_t>(height) * 4);
+		texture->UseColors = true;
+		stbi_image_free(pixels);
+		ImGui::RegisterUserTexture(texture.get());
+		return texture;
+	}
+
 	void EnsureVulkan(vk::Format format, uint32_t image_count) {
 		EnsureContext();
 		if (vulkan_initialized) {
@@ -952,9 +1060,116 @@ struct SystemOverlay::Impl {
 		}
 	}
 
+	void DrawTrophyNotification(vk::Extent2D frame_extent, Clock::time_point now) {
+		std::erase_if(retired_trophy_images, [](const auto& texture) {
+			if (texture->Status != ImTextureStatus_Destroyed) {
+				return false;
+			}
+			ImGui::UnregisterUserTexture(texture.get());
+			return true;
+		});
+		constexpr float TOAST_LIFETIME = 5.0f;
+		if (trophy_notification &&
+		    std::chrono::duration<float>(now - trophy_notification_born).count() >=
+		        TOAST_LIFETIME) {
+			trophy_notification.reset();
+			if (trophy_notification_image) {
+				// Keep the texture registered until the backend retires in-flight frames.
+				trophy_notification_image->WantDestroyNextFrame = true;
+				retired_trophy_images.push_back(std::move(trophy_notification_image));
+			}
+		}
+		if (!trophy_notification) {
+			{
+				std::scoped_lock lock(g_trophy_notification_mutex);
+				if (g_trophy_notifications.empty()) {
+					g_trophy_notification_active.store(!retired_trophy_images.empty(),
+					                                   std::memory_order_release);
+					return;
+				}
+				trophy_notification = std::move(g_trophy_notifications.front());
+				g_trophy_notifications.pop_front();
+			}
+			trophy_notification_born  = now;
+			trophy_notification_image = LoadTrophyIcon(trophy_notification->icon_png);
+			PlayTrophySound();
+		}
+		const float age = std::chrono::duration<float>(now - trophy_notification_born).count();
+		const float screen_scale =
+		    std::clamp(std::min(static_cast<float>(frame_extent.width) / 1920.0f,
+		                        static_cast<float>(frame_extent.height) / 1080.0f),
+		               0.65f, 1.25f);
+		const float width = std::max(
+		    0.0f, std::min(520.0f * screen_scale, static_cast<float>(frame_extent.width) - 32.0f));
+		const float  height   = 104.0f * screen_scale;
+		const float  rounding = 14.0f * screen_scale;
+		const float  fade_in  = std::clamp(age / 0.22f, 0.0f, 1.0f);
+		const float  fade_out = std::clamp((TOAST_LIFETIME - age) / 0.4f, 0.0f, 1.0f);
+		const float  alpha    = std::min(fade_in, fade_out);
+		const float  slide    = (1.0f - fade_in) * (width + 24.0f);
+		const ImVec2 top_left {static_cast<float>(frame_extent.width) - 32.0f - width + slide,
+		                       32.0f};
+		const ImVec2 bottom_right {top_left.x + width, top_left.y + height};
+		auto*        draw = ImGui::GetForegroundDrawList();
+		const auto   fade = [alpha](int r, int g, int b, int a) {
+            return IM_COL32(r, g, b, static_cast<int>(static_cast<float>(a) * alpha));
+		};
+		draw->AddRectFilled({top_left.x + 3.0f, top_left.y + 5.0f},
+		                    {bottom_right.x + 3.0f, bottom_right.y + 5.0f}, fade(0, 0, 0, 90),
+		                    rounding);
+		draw->AddRectFilled(top_left, bottom_right, fade(24, 26, 32, 235), rounding);
+		draw->AddRect(top_left, bottom_right, fade(255, 255, 255, 28), rounding,
+		              ImDrawFlags_RoundCornersAll, 1.0f);
+
+		const float pad       = 10.0f * screen_scale;
+		const float icon_size = height - 2.0f * pad;
+		float       text_left = top_left.x + 20.0f * screen_scale;
+		if (trophy_notification_image != nullptr) {
+			const ImVec2 icon_min {top_left.x + pad, top_left.y + pad};
+			const ImVec2 icon_max {icon_min.x + icon_size, icon_min.y + icon_size};
+			draw->AddImageRounded(trophy_notification_image->GetTexRef(), icon_min, icon_max,
+			                      {0, 0}, {1, 1}, fade(255, 255, 255, 255), 8.0f * screen_scale);
+			draw->AddRect(icon_min, icon_max, fade(255, 255, 255, 40), 8.0f * screen_scale,
+			              ImDrawFlags_RoundCornersAll, 1.0f);
+			text_left = icon_max.x + 18.0f * screen_scale;
+		}
+
+		constexpr std::array GRADE_COLORS {IM_COL32(161, 225, 245, 255), IM_COL32(255, 207, 82, 255),
+		                                   IM_COL32(211, 220, 232, 255),
+		                                   IM_COL32(211, 142, 91, 255)};
+		const int            grade     = trophy_notification->grade;
+		const float          title_y   = top_left.y + height * 0.5f - 25.0f * screen_scale;
+		float                title_x   = text_left;
+		if (grade > 0 && grade <= static_cast<int>(GRADE_COLORS.size())) {
+			// Small cup glyph tinted with the trophy grade.
+			const ImU32 cup = ImGui::GetColorU32(GRADE_COLORS[grade - 1], alpha);
+			const float u   = screen_scale;
+			const float cx  = text_left + 9.0f * u;
+			const float cy  = title_y + 15.0f * u;
+			draw->AddRectFilled({cx - 8.0f * u, cy - 11.0f * u}, {cx + 8.0f * u, cy + 2.0f * u},
+			                    cup, 2.0f * u, ImDrawFlags_RoundCornersBottom);
+			draw->AddRect({cx - 11.0f * u, cy - 9.0f * u}, {cx - 6.0f * u, cy - 3.0f * u}, cup,
+			              2.0f * u, ImDrawFlags_RoundCornersLeft, 1.8f * u);
+			draw->AddRect({cx + 6.0f * u, cy - 9.0f * u}, {cx + 11.0f * u, cy - 3.0f * u}, cup,
+			              2.0f * u, ImDrawFlags_RoundCornersRight, 1.8f * u);
+			draw->AddRectFilled({cx - 1.5f * u, cy + 2.0f * u}, {cx + 1.5f * u, cy + 7.0f * u}, cup);
+			draw->AddRectFilled({cx - 5.0f * u, cy + 7.0f * u}, {cx + 5.0f * u, cy + 10.0f * u}, cup,
+			                    1.5f * u);
+			title_x += 30.0f * u;
+		}
+		draw->PushClipRect(top_left, {bottom_right.x - 16.0f * screen_scale, bottom_right.y}, true);
+		draw->AddText(nullptr, 26.0f * screen_scale, {title_x, title_y},
+		              fade(255, 255, 255, 255), trophy_notification->name.c_str());
+		draw->AddText(nullptr, 20.0f * screen_scale, {text_left, title_y + 38.0f * screen_scale},
+		              fade(176, 180, 192, 255), "Trophy earned!");
+		draw->PopClipRect();
+	}
+
 	bool PrepareFrame(vk::Extent2D frame_extent, vk::Format format, uint32_t image_count) {
 		OverlaySnapshot snapshot;
-		if (!GetOverlaySnapshot(&snapshot)) {
+		const bool      has_overlay = GetOverlaySnapshot(&snapshot);
+		const auto now = Clock::now();
+		if (!has_overlay && !g_trophy_notification_active.load(std::memory_order_acquire)) {
 			return false;
 		}
 		const auto prepared_session = snapshot.session;
@@ -977,7 +1192,6 @@ struct SystemOverlay::Impl {
 		auto& io       = ImGui::GetIO();
 		io.DisplaySize = {static_cast<float>(frame_extent.width),
 		                  static_cast<float>(frame_extent.height)};
-		const auto now = std::chrono::steady_clock::now();
 		io.DeltaTime   = last_frame == std::chrono::steady_clock::time_point {}
 		                     ? 1.0f / 60.0f
 		                     : std::clamp(std::chrono::duration<float>(now - last_frame).count(),
@@ -985,15 +1199,19 @@ struct SystemOverlay::Impl {
 		last_frame     = now;
 		ImGui_ImplVulkan_NewFrame();
 		ImGui::NewFrame();
-		if (!GetOverlaySnapshot(&snapshot) || snapshot.session != prepared_session) {
+		GetOverlaySnapshot(&snapshot);
+		if (snapshot.session != prepared_session) {
 			ImGui::EndFrame();
 			return false;
 		}
-		if (snapshot.session.kind == OverlayKind::Dialog) {
-			DrawDialog(snapshot.dialog, frame_extent);
-		} else {
-			DrawIme(snapshot.ime, frame_extent);
+		if (has_overlay) {
+			if (snapshot.session.kind == OverlayKind::Dialog) {
+				DrawDialog(snapshot.dialog, frame_extent);
+			} else {
+				DrawIme(snapshot.ime, frame_extent);
+			}
 		}
+		DrawTrophyNotification(frame_extent, now);
 		ImGui::Render();
 		extent = frame_extent;
 		return true;
@@ -1040,9 +1258,18 @@ struct SystemOverlay::Impl {
 	float                                 button_height      = 42.0f;
 	ImVec2                                panel_offset {};
 	ImVec2                                right_stick {};
-	OverlaySession                        session;
-	vk::Extent2D                          extent {};
-	std::chrono::steady_clock::time_point last_frame;
+	OverlaySession                              session;
+	std::unique_ptr<ImTextureData>              trophy_notification_image;
+	std::optional<TrophyNotification>           trophy_notification;
+	Clock::time_point                           trophy_notification_born;
+	std::vector<std::unique_ptr<ImTextureData>> retired_trophy_images;
+	SDL_AudioStream*                            trophy_sound_stream      = nullptr;
+	uint8_t*                                    trophy_sound_data        = nullptr;
+	uint32_t                                    trophy_sound_size        = 0;
+	bool                                        trophy_sound_attempted   = false;
+	bool                                        trophy_sound_initialized = false;
+	vk::Extent2D                                extent {};
+	std::chrono::steady_clock::time_point       last_frame;
 };
 
 SystemOverlay::SystemOverlay(GraphicContext& graphics): m_impl(std::make_unique<Impl>(graphics)) {}

src/graphics/presentation/systemOverlay.h

@@ -4,7 +4,10 @@
 #include "common/common.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
+#include <cstddef>
 #include <memory>
+#include <span>
+#include <string_view>
 
 union SDL_Event;
 struct SDL_Window;
@@ -22,6 +25,8 @@ void                     InitializeSystemOverlayInput(SDL_Window* window);
 void                     ShutdownSystemOverlayInput();
 bool                     ProcessSystemOverlayInput(const SDL_Event& event);
 SystemOverlayVisualState GetSystemOverlayVisualState() noexcept;
+void                     NotifyTrophyUnlocked(std::string_view name, int32_t grade,
+                                              std::span<const std::byte> icon_png);
 
 class SystemOverlay final {
 public:

src/graphics/presentation/window/vulkanWindow.cpp

@@ -47,6 +47,12 @@ struct VulkanExtensions {
 	std::vector<vk::LayerProperties>     available_layers;
 };
 
+vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
+	vk::PhysicalDeviceVulkan11Features features {};
+	features.storageBuffer16BitAccess = VK_TRUE;
+	return features;
+}
+
 vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
 	vk::PhysicalDeviceVulkan12Features features {};
 	features.samplerMirrorClampToEdge  = VK_TRUE;
@@ -55,6 +61,8 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 	features.shaderOutputViewportIndex = VK_TRUE;
 	features.bufferDeviceAddress       = VK_TRUE;
 	features.shaderBufferInt64Atomics  = VK_TRUE;
+	features.storageBuffer8BitAccess   = VK_TRUE;
+	features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
 	return features;
 }
 
@@ -206,6 +214,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		depth_clip_control.pNext = &depth_clip_enable;
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
+		vk::PhysicalDeviceVulkan11Features features11 {};
 #if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
 #else
@@ -213,6 +222,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
+		features11.pNext       = features12.pNext;
+		features12.pNext       = &features11;
 		features13.pNext       = &features12;
 		device_features2.pNext = &features13;
 
@@ -256,6 +267,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge",
 		              required_features12.samplerMirrorClampToEdge);
+		check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
+		              WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
+		check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
+		              required_features12.storageBuffer8BitAccess);
 		check_feature(features12.timelineSemaphore, "timelineSemaphore",
 		              required_features12.timelineSemaphore);
 		check_feature(features12.shaderOutputLayer, "shaderOutputLayer",
@@ -266,6 +281,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		              required_features12.bufferDeviceAddress);
 		check_feature(features12.shaderBufferInt64Atomics, "shaderBufferInt64Atomics",
 		              required_features12.shaderBufferInt64Atomics);
+		check_feature(features12.shaderSampledImageArrayNonUniformIndexing,
+		              "shaderSampledImageArrayNonUniformIndexing",
+		              required_features12.shaderSampledImageArrayNonUniformIndexing);
 		check_feature(features13.robustImageAccess, "robustImageAccess");
 		check_feature(features13.dynamicRendering, "dynamicRendering",
 		              required_features13.dynamicRendering);
@@ -478,6 +496,9 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	workgroup_layout.pNext = &depth_clip_control;
 	features12.pNext = workgroup_layout_extension ? static_cast<void*>(&workgroup_layout)
 	                                             : static_cast<void*>(&depth_clip_control);
+	auto features11 = WindowContext::RequiredVulkan11Features();
+	features11.pNext = features12.pNext;
+	features12.pNext = &features11;
 	if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) {
 		Log::WriteToConsoleAndLog(fmt::format(
 		    "WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, "
@@ -560,8 +581,7 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.shaderInt64 = VK_TRUE;
 	device_features.shaderFloat64 =
 	    supported_features2.features.shaderFloat64 &&
-	    float_controls.shaderSignedZeroInfNanPreserveFloat64 &&
-	    float_controls.shaderRoundingModeRTEFloat32;
+	    float_controls.shaderSignedZeroInfNanPreserveFloat64;
 	// if (device_features.shaderFloat64 && !float_controls.shaderDenormPreserveFloat64) {
 	// 	Log::WriteToConsoleAndLog(
 	// 	    "WARNING: Vulkan device does not guarantee FP64 denormal preservation; "

src/graphics/presentation/window/window.cpp

@@ -362,11 +362,13 @@ static void GameEventDidEnterForeground(WindowLoopState& game) {
 	SetPause(game, false);
 }
 
-void WindowContext::Resize(uint32_t new_width, uint32_t new_height) {
-	EXIT_IF(new_width == 0 || new_height == 0);
+void WindowContext::Resize(int new_width, int new_height) {
+	if (new_width <= 0 || new_height <= 0) {
+		return;
+	}
 	Common::LockGuard lock(mutex);
-	graphic_ctx.screen_width  = new_width;
-	graphic_ctx.screen_height = new_height;
+	graphic_ctx.screen_width  = static_cast<uint32_t>(new_width);
+	graphic_ctx.screen_height = static_cast<uint32_t>(new_height);
 }
 
 void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {

src/graphics/presentation/window/windowInternal.h

@@ -34,6 +34,7 @@ struct WindowContext {
 	~WindowContext();
 	KYTY_CLASS_NO_COPY(WindowContext);
 
+	[[nodiscard]] static vk::PhysicalDeviceVulkan11Features RequiredVulkan11Features() noexcept;
 	[[nodiscard]] static vk::PhysicalDeviceVulkan12Features RequiredVulkan12Features() noexcept;
 	[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
 	[[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept;
@@ -42,7 +43,7 @@ struct WindowContext {
 	void                                                    RefreshSurfaceCapabilities();
 	void                                                    UpdateIcon();
 	void                                                    UpdateTitle();
-	void                                                    Resize(uint32_t width, uint32_t height);
+	void                                                    Resize(int width, int height);
 	void ProcessWindowEvent(const SDL_WindowEvent& event);
 	void ProcessDisplayEvent(const SDL_DisplayEvent& event);
 	void ProcessEvent(double time_seconds);

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -648,6 +648,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 				})) {
 					continue;
 				}
+				const auto resource = inst.Flags<uint32_t>();
+				first               = resource < ir.info.buffers.size() &&
+				                              ir.info.buffers[resource].indirect_root == resource
+				                          ? 1u
+				                          : 0u;
 			} else if (op == IR::ValueOpcode::GetImageResource) {
 				const auto resource = inst.Flags<uint32_t>();
 				first = resource < ir.info.images.size() &&
@@ -665,7 +670,9 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 	IR::EliminateDeadCode(ir.blocks);
 
 	IR::CollectShaderInfo(ir, options.input_info);
-	IR::AllocateBindings(ir, push_data_start_dword);
+	IR::AllocateBindings(ir, push_data_start_dword,
+	                     ir.stage == ShaderType::Compute && options.input_info.compute != nullptr &&
+	                         options.input_info.compute->lds_storage);
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(translated.cfg_dump, ir);

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -18,7 +18,7 @@ namespace {
 	std::abort();
 }
 
-void ValidateNativeProgram(const IR::Program& program) {
+void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
 	using Kind                                             = IR::DescriptorBindingKind;
 	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
 	std::array<std::vector<uint32_t>, KindCount> expected;
@@ -53,11 +53,14 @@ void ValidateNativeProgram(const IR::Program& program) {
 		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
 	}
 	auto& buffers = expected[static_cast<size_t>(Kind::Buffers)];
-	const bool uses_gds = IR::CollectMemoryResources(program, buffers);
+	const auto shared = IR::CollectMemoryResources(program, buffers);
 	present[static_cast<size_t>(Kind::Buffers)] = !buffers.empty();
-	if (uses_gds) {
+	if (shared.gds) {
 		Expect(Kind::Gds);
 	}
+	if (shared.lds && lds_storage) {
+		Expect(Kind::SharedMemory);
+	}
 	if (program.info.uses_dma) {
 		Expect(Kind::BdaPagetable);
 		Expect(Kind::FaultBuffer);
@@ -85,9 +88,13 @@ void ValidateNativeProgram(const IR::Program& program) {
 	}
 	const auto has_shader_data_storage = present[static_cast<size_t>(Kind::ShaderData)];
 	const auto shader_data_dwords = program.bindings.ShaderDataDwords();
+	const auto user_data_dwords = program.bindings.user_data_registers.size();
+	const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart;
 	if ((program.bindings.UsesPushData() &&
 	     !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
-	    program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
+	    (has_dispatch_threads && (program.stage != ShaderType::Compute ||
+	                              program.bindings.dispatch_thread_dword != user_data_dwords)) ||
+	    program.bindings.memory_offset_dword != user_data_dwords + (has_dispatch_threads ? 3u : 0u) ||
 	    program.bindings.memory_offset_count != buffers.size() ||
 	    has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) ||
 	    !std::is_sorted(program.bindings.user_data_registers.begin(),
@@ -198,7 +205,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (memory_index >= program.memory_info.size()) {
 					Fail(program, "address operation has invalid memory metadata");
 				}
-				const auto kind = program.memory_info[memory_index].kind;
+				const auto& memory = program.memory_info[memory_index];
+				const auto kind = memory.kind;
 				if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
 					if (program.scratch_dwords == 0) {
 						Fail(program, "scratch operation has no per-thread storage");
@@ -210,6 +218,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					}
 				} else if (address_access == IR::AddressAccess::Write) {
 					Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
+				} else {
+					// Descriptor stores can alias coherent physical-address loads.
+					requirements.coherent_buffers |= memory.coherent;
 				}
 			}
 			if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
@@ -218,7 +229,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					Fail(program, "buffer operation has invalid memory metadata");
 				}
 				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind == IR::ResourceKind::IndirectBuffer) {
+				if (memory.kind == IR::ResourceKind::IndirectBuffer &&
+				    inst.GetOpcode() != IR::ValueOpcode::ReadConstBuffer) {
 					requirements.subgroup_local_invocation_id = true;
 				}
 				if (memory.kind == IR::ResourceKind::Buffer) {
@@ -226,6 +238,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					if (memory.resource >= program.info.buffers.size()) {
 						Fail(program, "buffer operation has invalid resource metadata");
 					}
+					const auto bits = StorageBufferElementBits(program, memory);
+					requirements.buffer_u8 |= bits == 8u;
+					requirements.buffer_u16 |= bits == 16u;
 					if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
 						if (program.stage != ShaderType::Compute) {
 							Fail(program, "buffer ADD_TID is only valid for compute shaders");
@@ -263,6 +278,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				}
 			}
 			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break;
 				case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
 				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
@@ -288,6 +304,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					break;
 				}
 				case IR::ValueOpcode::SwizzleU32:
+				case IR::ValueOpcode::PermuteU32:
 				case IR::ValueOpcode::BpermuteU32: {
 					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;
@@ -334,7 +351,8 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
 	    !program.shader_info_complete || !program.binding_layout_complete) {
 		Fail(program, "SPIR-V emitter requires a fully planned native shader program");
 	}
-	ValidateNativeProgram(program);
+	ValidateNativeProgram(program, program.stage == ShaderType::Compute &&
+	                                   input_info.compute != nullptr && input_info.compute->lds_storage);
 	IR::ValidateProgram(program, true);
 	EmitterState state(program, input_info);
 	const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -17,28 +17,26 @@ Pair ExtractPair(EmitterState& state, uint32_t value) {
 	return result;
 }
 
-uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpCompositeConstruct, TypeU64(state), result, low, high);
-	return result;
-}
+uint32_t EmitMinMaxF64(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto bits_type = TypeU32Vector(state, 2);
+	const auto lhs_bits = Unary(state, spv::OpBitcast, bits_type, lhs);
+	const auto rhs_bits = Unary(state, spv::OpBitcast, bits_type, rhs);
+	const auto ordered = Binary(state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
+	                            TypeBool(state), lhs, rhs);
+	auto result = Select(state, TypeF64(state), ordered, lhs, rhs);
 
-uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                        bool not_equal) {
-	const auto compare = Binary(state, not_equal ? spv::OpINotEqual : spv::OpIEqual,
-	                            TypeBoolVector(state, 2), lhs_value, rhs_value);
-	return Unary(state, not_equal ? spv::OpAny : spv::OpAll, TypeBool(state), compare);
-}
+	// Equal values have identical bits except signed zero: min chooses -0, max chooses +0.
+	const auto equal = Binary(state, spv::OpFOrdEqual, TypeBool(state), lhs, rhs);
+	const auto equal_bits = Binary(state, max_value ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
+	                               bits_type, lhs_bits, rhs_bits);
+	result = Select(state, TypeF64(state), equal,
+	                Unary(state, spv::OpBitcast, TypeF64(state), equal_bits), result);
 
-uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                          spv::Op high_compare, spv::Op low_compare) {
-	const auto lhs         = ExtractPair(state, lhs_value);
-	const auto rhs         = ExtractPair(state, rhs_value);
-	const auto high_equal  = Binary(state, spv::OpIEqual, TypeBool(state), lhs.high, rhs.high);
-	const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
-	const auto low_result  = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
-	const auto low_path = Binary(state, spv::OpLogicalAnd, TypeBool(state), high_equal, low_result);
-	return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
+	// Non-IEEE mode selects the other operand for NaN, including rhs when both are NaN.
+	result = Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), rhs),
+	                lhs, result);
+	return Select(state, TypeF64(state), Unary(state, spv::OpIsNan, TypeBool(state), lhs),
+	              rhs, result);
 }
 
 uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
@@ -58,86 +56,6 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
 	return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
 }
 
-uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
-	const auto pair       = ExtractPair(state, value);
-	const auto amount     = EmitAndConstant(state, shift, 63u);
-	const auto word_shift = EmitAndConstant(state, amount, 31u);
-	const auto at_least_32 =
-	    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
-	const auto nonzero =
-	    Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
-	const auto carry_count = EmitAndConstant(
-	    state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u);
-	if (opcode == spv::OpShiftLeftLogical) {
-		const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift);
-		const auto carry =
-		    Select(state, TypeU32(state), nonzero,
-		           Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count),
-		           ConstantU32(state, 0));
-		const auto high =
-		    Binary(state, spv::OpBitwiseOr, TypeU32(state),
-		           Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry);
-		return MakePair(state,
-		                Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low),
-		                Select(state, TypeU32(state), at_least_32, low, high));
-	}
-	const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift);
-	const auto carry =
-	    Select(state, TypeU32(state), nonzero,
-	           Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
-	           ConstantU32(state, 0));
-	const auto low = Binary(
-	    state, spv::OpBitwiseOr, TypeU32(state),
-	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry);
-	const auto fill = opcode == spv::OpShiftRightArithmetic
-	                      ? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31))
-	                      : ConstantU32(state, 0);
-	return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low),
-	                Select(state, TypeU32(state), at_least_32, fill, high));
-}
-
-uint32_t EmitConstantShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
-	shift &= 63u;
-	if (shift == 0u) {
-		return value;
-	}
-	const auto pair = ExtractPair(state, value);
-	if (opcode == spv::OpShiftLeftLogical) {
-		if (shift < 32u) {
-			const auto low  = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.low,
-			                         ConstantU32(state, shift));
-			const auto high = Binary(state, spv::OpBitwiseOr, TypeU32(state),
-			                         Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-			                                pair.high, ConstantU32(state, shift)),
-			                         Binary(state, spv::OpShiftRightLogical, TypeU32(state),
-			                                pair.low, ConstantU32(state, 32u - shift)));
-			return MakePair(state, low, high);
-		}
-		return MakePair(state, ConstantU32(state, 0),
-		                shift == 32u ? pair.low
-		                             : Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-		                                      pair.low, ConstantU32(state, shift - 32u)));
-	}
-	if (shift < 32u) {
-		const auto low = Binary(state, spv::OpBitwiseOr, TypeU32(state),
-		                        Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low,
-		                               ConstantU32(state, shift)),
-		                        Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high,
-		                               ConstantU32(state, 32u - shift)));
-		const auto high =
-		    Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
-		return MakePair(state, low, high);
-	}
-	const auto high = opcode == spv::OpShiftRightArithmetic
-	                      ? Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high,
-	                               ConstantU32(state, 31u))
-	                      : ConstantU32(state, 0);
-	const auto low  = shift == 32u ? pair.high
-	                               : Binary(state, opcode, TypeU32(state), pair.high,
-	                                        ConstantU32(state, shift - 32u));
-	return MakePair(state, low, high);
-}
-
 uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
                      bool max_value) {
 	const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
@@ -201,6 +119,28 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
 }
 
 } // namespace
+
+uint32_t EmitFPFma32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
+	const auto result = EmitGlsl<GLSLstd450Fma, IR::Type::F32>(state, a, b, c);
+	state.builder.AddAnnotation(spv::OpDecorate, result, spv::DecorationNoContraction);
+	return result;
+}
+
+uint32_t EmitFPMad32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
+	// Explicit denormal checks typically drop 3DMiniGolf menu FPS from 28-29 to 21-24
+	// on NVIDIA RTX 5080 Laptop GPU, so leave them disabled for this experiment.
+	// a = EmitFlushF32DenormToSignedZero(state, a);
+	// b = EmitFlushF32DenormToSignedZero(state, b);
+	// c = EmitFlushF32DenormToSignedZero(state, c);
+	const auto product = EmitFPMul32(state, a, b);
+	state.builder.AddAnnotation(spv::OpDecorate, product, spv::DecorationNoContraction);
+	// const auto sum = EmitFPAdd32(state, EmitFlushF32DenormToSignedZero(state, product), c);
+	const auto sum = EmitFPAdd32(state, product, c);
+	state.builder.AddAnnotation(spv::OpDecorate, sum, spv::DecorationNoContraction);
+	// return EmitFlushF32DenormToSignedZero(state, sum);
+	return sum;
+}
+
 uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
 	const auto min_ab   = EmitMinMaxF32Value(state, a, b, false);
 	const auto min3     = EmitMinMaxF32Value(state, min_ab, c, false);
@@ -215,57 +155,15 @@ uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c)
 }
 
 uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
-	const auto pair   = ExtractPair(state, value);
-	const auto high_i = state.builder.AllocateId();
-	const auto low_i  = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state),
-	                          GLSLstd450FindUMsb, pair.high);
-	state.builder.AddFunction(spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state),
-	                          GLSLstd450FindUMsb, pair.low);
-	const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
-	const auto low  = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
-	const auto high_nonzero =
-	    Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
-	return Select(state, TypeU32(state), high_nonzero,
-	              Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
-}
-
-uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs   = ExtractPair(state, lhs_value);
-	const auto rhs   = ExtractPair(state, rhs_value);
-	const auto low   = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
-	const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
-	const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
-	const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
-	return MakePair(state, low,
-	                Binary(state, spv::OpIAdd, TypeU32(state),
-	                       Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
-}
-
-uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs    = ExtractPair(state, lhs_value);
-	const auto rhs    = ExtractPair(state, rhs_value);
-	const auto low    = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
-	const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
-	const auto borrow_u32 =
-	    Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
-	const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
-	const auto high  = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
-	return MakePair(state, low, high);
-}
-
-uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs      = ExtractPair(state, lhs_value);
-	const auto rhs      = ExtractPair(state, rhs_value);
-	const auto low_pair = state.builder.AllocateId();
-	const auto low      = state.builder.AllocateId();
-	const auto carry    = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1);
-	const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
-	const auto high  = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
-	return MakePair(state, low, high);
+	// GLSL FindUMsb is restricted to 32-bit components.
+	const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), value);
+	const auto pair =
+	    ExtractPair(state, EmitExt(state, TypeU32Vector(state, 2), GLSLstd450FindUMsb, {unpacked}));
+	const auto high_found =
+	    Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, UINT32_MAX));
+	return Select(state, TypeU32(state), high_found,
+	              Binary(state, spv::OpIAdd, TypeU32(state), pair.high, ConstantU32(state, 32)),
+	              pair.low);
 }
 
 uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
@@ -294,7 +192,7 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
 uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
 	const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0);
 	const auto source    = EmitNative<spv::OpCompositeExtract, IR::Type::U32>(
-	    state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u);
+	    state, Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0), 1u);
 	const auto exponent = EmitAndConstant(state, source, 0x7ff00000u);
 	const auto overflow =
 	    Binary(state, spv::OpLogicalAnd, TypeBool(state),
@@ -312,7 +210,21 @@ uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
 	                                                  EmitFlushF32DenormToSignedZero(state, arg0));
 }
 
+uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
+	arg1 = arg1.Resolve();
+	return arg1.IsImmediate() && arg1.U32() == 0u
+	           ? Unary(ctx.state, spv::OpUConvert, TypeU64(ctx.state), arg0)
+	           : PackU64(ctx.state, arg0, ctx.Def(arg1));
+}
+
 uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
+	const auto value = arg1.U32() == 0u ? arg0
+	                                    : Binary(state, spv::OpShiftRightLogical, TypeU64(state),
+	                                             arg0, ConstantU32(state, 32));
+	return Unary(state, spv::OpUConvert, TypeU32(state), value);
+}
+
+uint32_t EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1) {
 	return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
 }
 
@@ -344,33 +256,11 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) {
 	return Select(state, TypeU32(state), negative, neg, value);
 }
 
-uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
-	auto& state = ctx.state;
-	return arg1.Resolve().IsImmediate()
-	           ? EmitConstantShift64(state, spv::OpShiftLeftLogical, arg0, arg1.Resolve().U32())
-	           : EmitShift64(state, spv::OpShiftLeftLogical, arg0, ctx.Def(arg1));
-}
-
-uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
-	auto& state = ctx.state;
-	return arg1.Resolve().IsImmediate()
-	           ? EmitConstantShift64(state, spv::OpShiftRightLogical, arg0, arg1.Resolve().U32())
-	           : EmitShift64(state, spv::OpShiftRightLogical, arg0, ctx.Def(arg1));
-}
-
-uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
-	auto& state = ctx.state;
-	return arg1.Resolve().IsImmediate()
-	           ? EmitConstantShift64(state, spv::OpShiftRightArithmetic, arg0, arg1.Resolve().U32())
-	           : EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1));
-}
-
-uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1);
-}
-
 uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
-	const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
+	// Vulkan bit-count operands remain 32-bit, including when counting a U64.
+	const auto unpacked = Unary(state, spv::OpBitcast, TypeU32Vector(state, 2), arg0);
+	const auto pair =
+	    ExtractPair(state, Unary(state, spv::OpBitCount, TypeU32Vector(state, 2), unpacked));
 	return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
 }
 
@@ -424,38 +314,6 @@ uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32
 	return EmitMed3(state, arg0, arg1, arg2, false);
 }
 
-uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareEqual64(state, arg0, arg1, false);
-}
-
-uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareEqual64(state, arg0, arg1, true);
-}
-
-uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan);
-}
-
-uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan);
-}
-
-uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
-}
-
-uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
-}
-
-uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
-}
-
-uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
-	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
-}
-
 uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
 	return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
 }
@@ -468,6 +326,14 @@ uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return EmitMinMaxF32Value(state, arg0, arg1, true);
 }
 
+uint32_t EmitFPMin64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return EmitMinMaxF64(state, arg0, arg1, false);
+}
+
+uint32_t EmitFPMax64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return EmitMinMaxF64(state, arg0, arg1, true);
+}
+
 uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
 	return EmitFMinMax3(state, arg0, arg1, arg2, false);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -125,20 +125,23 @@ uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
 	return EmitLogicalAndBool(state, selected, class_match);
 }
 
-uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
-	const auto bits          = EmitBitcastF32ToU32(state, value);
-	const auto sign_bits     = EmitAndConstant(state, bits, 0x80000000u);
-	const auto abs_bits      = EmitAndConstant(state, bits, 0x7fffffffu);
-	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
-	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
-	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
+static uint32_t EmitClassMaskBits(EmitterState& state, uint32_t bits, uint32_t mask, bool half) {
+	const uint32_t sign_mask     = half ? 0x8000u : 0x80000000u;
+	const uint32_t exponent_mask = half ? 0x7c00u : 0x7f800000u;
+	const uint32_t mantissa_mask = half ? 0x03ffu : 0x007fffffu;
+	const uint32_t quiet_mask    = half ? 0x0200u : 0x00400000u;
+	const auto     sign_bits     = EmitAndConstant(state, bits, sign_mask);
+	const auto     abs_bits      = EmitAndConstant(state, bits, sign_mask - 1u);
+	const auto     exponent_bits = EmitAndConstant(state, abs_bits, exponent_mask);
+	const auto     mantissa_bits = EmitAndConstant(state, abs_bits, mantissa_mask);
+	const auto     quiet_bits    = EmitAndConstant(state, mantissa_bits, quiet_mask);
 
 	const auto sign             = EmitCompareU32Constant(state, spv::OpINotEqual, sign_bits, 0);
 	const auto positive         = EmitLogicalNotBool(state, sign);
 	const auto exponent_zero    = EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0);
 	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
 	const auto exponent_inf =
-	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
+	    EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, exponent_mask);
 	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
 	const auto mantissa_zero    = EmitCompareU32Constant(state, spv::OpIEqual, mantissa_bits, 0);
 	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
@@ -178,6 +181,14 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
 	    EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive)));
 }
 
+uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
+	return EmitClassMaskBits(state, EmitBitcastF32ToU32(state, value), mask, false);
+}
+
+uint32_t EmitClassMaskF16(EmitterState& state, uint32_t bits, uint32_t mask) {
+	return EmitClassMaskBits(state, bits, mask, true);
+}
+
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
 	const auto lhs_class = EmitClassifyF32(state, lhs);
 	const auto rhs_class = EmitClassifyF32(state, rhs);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -83,7 +83,7 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
 	}
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpAccessChain, result_ptr_type, pointer, variable_id,
-	                          ConstantU32(state, array_index));
+	                          array_index);
 	return pointer;
 }
 
@@ -136,8 +136,8 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 		EXIT("invalid image resource class");
 	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
-	const auto scalar_type = image.atomic64 ? TypeScalarU64(state)
-	                                        : ImageScalarType(state, image.numeric_class);
+	const auto  scalar_type =
+	    image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
 	return state.builder.Type(spv::OpTypeImage, scalar_type,
 	                          info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
 	                          info.multisampled, sampled, format);
@@ -152,8 +152,9 @@ uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
 	}
 }
 
-uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip) {
-	const auto pointer = ImageDescriptorPointer(state, resource, mip);
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip,
+                             uint32_t array_index) {
+	const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index);
 	const auto image = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpLoad, ImageType(state, state.program.info.images.at(resource)),
 	                          image, pointer);
@@ -167,30 +168,40 @@ uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
 	const auto pointer_type =
 	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type);
 	const auto pointer = DescriptorElementPointer(
-	    state, pointer_type, state.sampler_variable, array_index,
+	    state, pointer_type, state.sampler_variable, ConstantU32(state, array_index),
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer);
 	return sampler_id;
 }
 
-uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip) {
+uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip,
+                          uint32_t array_index) {
 	const auto& image_resource = state.program.info.images.at(resource);
 	EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
-	const auto  image          = LoadImageDescriptor(state, resource, mip);
+	const auto  image          = LoadImageDescriptor(state, resource, mip, array_index);
 	const auto  sampled_image = state.builder.AllocateId();
 	const auto  sampled_type =
 	    state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource));
 	state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
+	if (array_index != 0u) {
+		state.builder.RequireExtension("SPV_EXT_descriptor_indexing");
+		state.builder.RequireCapability(spv::CapabilityShaderNonUniform);
+		state.builder.RequireCapability(spv::CapabilitySampledImageArrayNonUniformIndexing);
+		state.builder.AddAnnotation(spv::OpDecorate, sampled_image, spv::DecorationNonUniform);
+	}
 	return sampled_image;
 }
 
-uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip) {
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip,
+                                uint32_t array_index) {
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(mip >= image.mip_count);
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
-	const auto array_index  = ResourceForDescriptor(state, *kind, resource) + mip;
+	if (array_index == 0u) {
+		array_index = ConstantU32(state, ResourceForDescriptor(state, *kind, resource) + mip);
+	}
 	const auto pointer_type = state.builder.Type(
 	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
@@ -213,7 +224,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 		EmitWrite(0);
 		return;
 	}
-	EmitImageMipSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
+	EmitIndexSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
 }
 
 spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -19,8 +19,8 @@ uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> compon
 
 uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
 	const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
-	const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
-	                            Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
+	const auto address =
+	    Binary(s, spv::OpIAdd, TypeU64(s), block, Unary(s, spv::OpUConvert, TypeU64(s), offset));
 	const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
 	const auto result = s.builder.AllocateId();
 	s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
@@ -224,15 +224,15 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
 
 void DefineBvhIntersect(EmitterState& s) {
 	if (!s.requirements.bvh) return;
-	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
+	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s);
 	const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
-	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3);
 	s.bvh_intersect_function = s.builder.AllocateId();
 	s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
 	s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
 	                      spv::FunctionControlMaskNone, signature);
 	std::array<uint32_t, 6> args;
-	const std::array types {vec4, u, f, vec3, vec3, vec3};
+	const std::array types {vec4, wide, f, vec3, vec3, vec3};
 	for (uint32_t i = 0; i < args.size(); ++i) {
 		args[i] = s.builder.AllocateId();
 		s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
@@ -253,29 +253,31 @@ void DefineBvhIntersect(EmitterState& s) {
 	const auto shr = [&](uint32_t value, uint32_t shift) {
 		return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
 	};
-	const auto kind = and_bits(node, 7);
+	const auto kind = and_bits(Unary(s, spv::OpUConvert, u, node), 7);
 	const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
 	const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
 	const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
 	const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
 	const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
 	const auto grow = and_bits(shr(words[1], 23), 0xff);
-	const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
-	    DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
-	const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
+	const auto base  = Binary(s, spv::OpShiftLeftLogical, wide,
+	                          PackU64(s, words[0], and_bits(words[1], 0xff)), ConstantU32(s, 8));
+	const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantU32(s, 3));
 	const auto last_index = Binary(s, spv::OpIAdd, wide, index,
-	    Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
+	                               Select(s, wide, full, ConstantU64(s, 1), ConstantU64(s, 0)));
 	const auto address = Binary(s, spv::OpIAdd, wide, base,
-	    Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
-	const auto last_address = Binary(s, spv::OpIAdd, wide, base,
-	    Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
+	                            Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantU32(s, 6)));
+	const auto last_address =
+	    Binary(s, spv::OpIAdd, wide, base,
+	           Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantU32(s, 6)));
 	auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
 	const auto require = [&](uint32_t condition) {
 		valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
 	};
 	require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
-	require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
-	require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
+	require(Binary(s, spv::OpULessThanEqual, b, last_index,
+	               PackU64(s, words[2], and_bits(words[3], 0x3ff))));
+	require(Binary(s, spv::OpULessThan, b, last_address, ConstantU64(s, uint64_t {1} << 48)));
 	require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
 	for (uint32_t i = 0; i < 3; ++i) {
 		for (const auto value: {origin[i], direction[i], inverse[i]})
@@ -292,13 +294,13 @@ void DefineBvhIntersect(EmitterState& s) {
 	const auto invalid = Vector(s, vec4, invalid_words);
 	const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
 		const auto first = GetBdaPointer(s, address);
-		const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
-			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
+		const auto second  = EmitValueOrDefaultIfCondition(s, full, wide, ConstantU64(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantU64(s, 64)));
 		});
-		const auto present = Binary(s, spv::OpLogicalAnd, b,
-		    Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
+		const auto present = Binary(
+		    s, spv::OpLogicalAnd, b, Binary(s, spv::OpINotEqual, b, first, ConstantU64(s, 0)),
 		    Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
-		        Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
+		           Binary(s, spv::OpINotEqual, b, second, ConstantU64(s, 0))));
 		return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
 			const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
 			const auto merge = s.builder.AllocateId();
@@ -325,17 +327,22 @@ void DefineBvhIntersect(EmitterState& s) {
 uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& s = ctx.state;
 	const auto* ray = ctx.ImageAddress(inst.Arg(1));
-	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
-		ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
+	const auto node_words = inst.Flags<uint32_t>();
+	if (ray == nullptr || (node_words != 1u && node_words != 2u) ||
+	    ray->NumArgs() < node_words + 10u || s.bvh_intersect_function == 0)
+		ctx.Fail(inst, "BVH intersection requires its node pointer, ray components and shared function");
 	const auto vector = [&](uint32_t first) {
 		Vec3 values;
 		for (uint32_t i = 0; i < 3; ++i)
 			values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
 		return Vector(s, TypeF32Vector(s, 3), values);
 	};
-	const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
-	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
-	const auto origin = vector(2), direction = vector(5), inverse = vector(8);
+	const auto descriptor = ctx.Arg(inst, 0);
+	const auto node   = node_words == 2u ? PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
+	                                     : Unary(s, spv::OpUConvert, TypeU64(s), ctx.Arg(*ray, 0));
+	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words));
+	const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u),
+	           inverse = vector(node_words + 7u);
 	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
 	    ConstantU32CompositeZero(s, 4), [&] {
 		const auto result = s.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -366,18 +366,23 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 			state.builder.AddFunction(spv::OpStore, state.point_size_variable, f32);
 			continue;
 		}
-		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
+		auto StoreDistance = [&](IR::StageOutputKind kind, uint32_t variable, uint32_t index) {
 			if (index == UINT32_MAX) {
 				return;
 			}
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain,
-			                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
-			                          pointer, variable, ConstantU32(state, index));
+			uint32_t pointer;
+			if (state.program.stage == ShaderType::Mesh) {
+				pointer = MeshOutputPointer(state, kind, index);
+			} else {
+				pointer = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpAccessChain,
+				                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+				                          pointer, variable, ConstantU32(state, index));
+			}
 			state.builder.AddFunction(spv::OpStore, pointer, f32);
 		};
-		StoreDistance(state.clip_distance_variable, output.clip_distance);
-		StoreDistance(state.cull_distance_variable, output.cull_distance);
+		StoreDistance(IR::StageOutputKind::ClipDistance, state.clip_distance_variable, output.clip_distance);
+		StoreDistance(IR::StageOutputKind::CullDistance, state.cull_distance_variable, output.cull_distance);
 	}
 }
 
@@ -425,16 +430,16 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
 	const auto quad_bits   = state.builder.AllocateId();
 	const auto result      = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
-	                          ConstantU64(state, 0x0000000100000001ull));
+	                          ConstantU32(state, 1));
 	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one);
 	state.builder.AddFunction(spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
-	                          ConstantU64(state, 0x0000000200000002ull));
+	                          ConstantU32(state, 2));
 	state.builder.AddFunction(spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one,
 	                          shifted_two);
 	state.builder.AddFunction(spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
 	                          ConstantU64(state, 0x1111111111111111ull));
 	state.builder.AddFunction(spv::OpIMul, TypeU64(state), result, quad_bits,
-	                          ConstantU64(state, 0x0000000f0000000full));
+	                          ConstantU64(state, 15));
 	return result;
 }
 
@@ -454,7 +459,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	// Skip dormant color exports after their valid mask; MRT1 is reserved for logical alpha.
 	if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-	    exp.index != 0 && state.input_info.pixel->alpha_blend_source_remap) {
+	    exp.index != 0 && state.input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None) {
 		return;
 	}
 	EmitIfCondition(state, exec, [&]() {
@@ -498,14 +503,36 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
 		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-		    exp.index == 0 && !uint_output && state.input_info.pixel->alpha_blend_source_remap) {
+		    exp.index == 0 && !uint_output &&
+		    state.input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None) {
 			// Broadcast logical alpha before swizzling the primary output.
 			const auto blend_output =
 			    OutputVariableForExport(state, {.kind = IR::ExportTargetKind::Mrt, .index = 1});
 			if (blend_output != 0) {
-				const auto alpha = state.builder.AllocateId();
+				auto alpha = state.builder.AllocateId();
 				state.builder.AddFunction(spv::OpVectorShuffle, vector_type, alpha, value, value,
 				                          3u, 3u, 3u, 3u);
+				uint32_t alpha_factor = 0;
+				switch (state.input_info.pixel->alpha_blend_source) {
+					case ShaderAlphaBlendSource::SourceAlphaOne:
+						alpha_factor = ConstantF32Value(state, 1.0f);
+						break;
+					case ShaderAlphaBlendSource::SourceAlphaZero:
+						alpha_factor = ConstantF32Value(state, 0.0f);
+						break;
+					default: break;
+				}
+				if (alpha_factor != 0) {
+					const auto mapping = state.input_info.pixel->target_export_mapping[0];
+					for (uint32_t component = 0; component < 4; component++) {
+						if (mapping.Map(component) != 3u) continue;
+						const auto factors = state.builder.AllocateId();
+						state.builder.AddFunction(spv::OpCompositeInsert, vector_type, factors,
+						                          alpha_factor, alpha, component);
+						alpha = factors;
+						break;
+					}
+				}
 				state.builder.AddFunction(spv::OpStore, blend_output, alpha);
 			}
 		}
@@ -574,6 +601,14 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
 
 void EmitVoid(ValueEmitContext&) {}
 
+void EmitStoreCompletion(EmitterState& state) {
+	state.builder.AddFunction(
+	    spv::OpControlBarrier, ConstantU32(state, spv::ScopeSubgroup),
+	    ConstantU32(state, spv::ScopeDevice),
+	    ConstantU32(state, spv::MemorySemanticsReleaseMask | spv::MemorySemanticsUniformMemoryMask |
+	                          spv::MemorySemanticsImageMemoryMask));
+}
+
 void EmitBarrier(EmitterState& state) {
 	const auto tessellation = state.program.stage == ShaderType::TessellationControl;
 	if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
@@ -581,9 +616,11 @@ void EmitBarrier(EmitterState& state) {
 		return;
 	}
 	const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
-	const auto semantics    = tessellation ? spv::MemorySemanticsMaskNone
-	                                       : spv::MemorySemanticsAcquireReleaseMask |
-	                                             spv::MemorySemanticsWorkgroupMemoryMask;
+	const auto memory = state.lds_storage_class == spv::StorageClassStorageBuffer
+	                        ? spv::MemorySemanticsUniformMemoryMask
+	                        : spv::MemorySemanticsWorkgroupMemoryMask;
+	const auto semantics = tessellation ? spv::MemorySemanticsMaskNone
+	                                    : spv::MemorySemanticsAcquireReleaseMask | memory;
 	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
 	                          ConstantU32(state, memory_scope), ConstantU32(state, semantics));
 }
@@ -623,6 +660,14 @@ uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index)
 	return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
 }
 
+uint32_t EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto start = ctx.state.program.bindings.dispatch_thread_dword;
+	if (start == IR::PushData::NoStart || !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() >= 3u) {
+		ctx.Fail(inst, "invalid dispatch thread extent");
+	}
+	return EmitShaderDataDwordLoad(ctx.state, start + inst.Arg(0).U32());
+}
+
 uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
 	const auto result = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpUndef, TypeId(state, inst.GetType()), result);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -391,28 +391,50 @@ Format::BufferFormatInfo ImageConversionFormat(const EmitterState&   state,
 	const auto format = state.program.info.images[mem.resource].conversion_format;
 	if (format == Prospero::BufferFormat::kInvalid) return {};
 	const auto info = Format::GetFormatInfo(format);
-	EXIT_IF(Prospero::SampledTextureNumericClass(format) != Prospero::TextureNumericClass::Uint ||
-	        Prospero::RemapTextureFormat(format) == format ||
-	        info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
-	        info.byte_size != sizeof(uint32_t) || info.component_count == 0u ||
+	EXIT_IF(Prospero::RemapTextureFormat(format) == format || info.component_count == 0u ||
 	        info.component_count > 4u);
+	EXIT_IF(info.type != Format::ComponentType::Uscaled &&
+	        (info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
+	         info.byte_size != sizeof(uint32_t)));
 	return info;
 }
 
+uint32_t ImageGatherSource(const EmitterState& state, const IR::MemoryInfo& mem) {
+	const auto component = ImageGatherComponent(mem.dmask);
+	const auto info      = ImageConversionFormat(state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return component;
+	if (info.packed_bitfield) return 0u;
+	const auto selector =
+	    (state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	return selector >= 4u ? (selector - 4u) % info.component_count : 0u;
+}
+
 uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
 	const auto info = ImageConversionFormat(ctx.state, mem);
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
 
-	const auto packed = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u);
-	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
-	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
+	const auto scalar_type   = ImageScalarType(ctx.state, numeric_class);
+	uint32_t   packed        = 0;
+	if (info.packed_bitfield) {
+		packed = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type, packed, texel, 0u);
+	}
+	uint32_t components[4] {};
 	for (uint32_t component = 0; component < info.component_count; component++) {
 		components[component] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(ctx.state),
-		                              components[component], packed,
-		                              ConstantU32(ctx.state, info.component_bit_offset[component]),
-		                              ConstantU32(ctx.state, info.component_bits[component]));
+		if (info.packed_bitfield) {
+			ctx.state.builder.AddFunction(spv::OpBitFieldUExtract, scalar_type,
+			                              components[component], packed,
+			                              ConstantU32(ctx.state, info.component_bit_offset[component]),
+			                              ConstantU32(ctx.state, info.component_bits[component]));
+		} else {
+			ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type,
+			                              components[component], texel, component);
+			// UNorm backing preserves the guest's filtering; scaling precedes swizzle constants.
+			components[component] = Binary(ctx.state, spv::OpFMul, scalar_type, components[component],
+			                               ConstantF32Value(ctx.state, 255.0f));
+		}
 	}
 	for (uint32_t component = info.component_count; component < 4u; component++) {
 		components[component] = components[component % info.component_count];
@@ -423,16 +445,19 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
 	for (uint32_t component = 0; component < 4u; component++) {
 		const auto selector = (swizzle >> (component * 3u)) & 7u;
 		if (selector == 1u) {
-			selected[component] = ConstantU32(ctx.state, 1u);
+			selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 1u)
+			                                          : ConstantF32Value(ctx.state, 1.0f);
 		} else if (selector >= 4u) {
 			selected[component] = components[selector - 4u];
 		} else {
-			selected[component] = ConstantU32(ctx.state, 0u);
+			selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 0u)
+			                                          : ZeroF32(ctx.state);
 		}
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
-	                              selected[0], selected[1], selected[2], selected[3]);
+	ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
+	                              ImageVectorType(ctx.state, numeric_class, 4), result, selected[0],
+	                              selected[1], selected[2], selected[3]);
 	return result;
 }
 
@@ -443,10 +468,24 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
 	const auto component = ImageGatherComponent(mem.dmask);
 	const auto selector =
 	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
+	const auto vector_type   = ImageVectorType(ctx.state, numeric_class, 4);
 	if (selector < 4u) {
-		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
-		return ctx.state.builder.Constant(spv::OpConstantComposite, TypeU32Vector(ctx.state, 4),
-		                                  value, value, value, value);
+		uint32_t value;
+		switch (info.type) {
+			case Format::ComponentType::Uscaled:
+				value = ConstantF32Value(ctx.state, selector == 1u ? 1.0f : 0.0f);
+				break;
+			default: value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u); break;
+		}
+		return ctx.state.builder.Constant(spv::OpConstantComposite, vector_type, value, value,
+		                                  value, value);
+	}
+	if (!info.packed_bitfield) {
+		const auto scale  = ConstantF32Value(ctx.state, 255.0f);
+		const auto scales = ctx.state.builder.Constant(spv::OpConstantComposite, vector_type,
+		                                                scale, scale, scale, scale);
+		return Binary(ctx.state, spv::OpFMul, vector_type, gathered, scales);
 	}
 
 	uint32_t values[4] {};
@@ -488,10 +527,7 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 	                                     LoadSamplerDescriptor(state, mem.sampler));
 	const auto vector_type = ImageVectorType(state, numeric_class, 4);
 	const auto scalar_type = ImageScalarType(state, numeric_class);
-	const auto component =
-	    ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid
-	        ? ImageGatherComponent(mem.dmask)
-	        : 0u;
+	const auto component = ImageGatherSource(state, mem);
 	uint32_t values[2] {};
 	for (uint32_t index = 0; index < 2u; index++) {
 		const auto sample_coord =
@@ -515,6 +551,7 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
 	const auto info = ImageConversionFormat(ctx.state, mem);
 	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+	EXIT_IF(info.type != Format::ComponentType::Uint || !info.packed_bitfield);
 
 	auto packed = ConstantU32(ctx.state, 0u);
 	for (uint32_t component = 0; component < info.component_count; component++) {
@@ -669,7 +706,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const bool  dref           = HasFlag(mem, Decoder::ImageSampleFlagCompare);
 		if (dref && state.program.info.images[mem.resource].conversion_format !=
 		                Prospero::BufferFormat::kInvalid) {
-			ctx.Fail(inst, "uses depth comparison with a packed integer image");
+			ctx.Fail(inst, "uses depth comparison with a converted image");
 			return;
 		}
 		if (op == IR::ValueOpcode::ImageGatherRaw) {
@@ -700,9 +737,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				                        ? AddressF32(ctx, mem, *address, layout.dref)
 				                        : ZeroF32(state);
 			} else {
-				component_or_dref = ConstantU32(
-				    state, ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid
-				               ? ImageGatherComponent(mem.dmask) : 0u);
+				component_or_dref = ConstantU32(state, ImageGatherSource(state, mem));
 			}
 			uint32_t operand_mask = 0;
 			uint32_t offset = 0;
@@ -728,7 +763,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				return sample;
 			};
 			const auto sample = image.mip_mode == IR::ImageMipMode::Dynamic && image.mip_count > 1u
-			                        ? EmitImageMipSwitch(
+			                        ? EmitIndexSwitch(
 			                              state, GatherMip(ctx, inst, mem, *address, layout,
 			                                               image.mip_count),
 			                              image.mip_count, result_type, EmitGather)
@@ -776,13 +811,13 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
 		const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler);
-		const auto EmitSample = [&](uint32_t resource) {
+		const auto EmitSample = [&](uint32_t resource, uint32_t array_index = 0u) {
 			const auto& candidate = state.program.info.images[resource];
 			const auto coord =
 			    CoordF32(ctx, mem, *address, layout.coord,
 			             ImageDimensionInfoFor(candidate.dimension).coordinate_components,
 			             candidate.cube);
-			const auto            sampled = MakeSampledImage(state, resource, sampler_id);
+			const auto sampled = MakeSampledImage(state, resource, sampler_id, 0u, array_index);
 			const auto            sample  = state.builder.AllocateId();
 			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};
 			if (dref) {
@@ -808,93 +843,82 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto* source = image.source < state.program.descriptor_sources.size()
 		                         ? &state.program.descriptor_sources[image.source]
 		                         : nullptr;
-		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
+		if (handle == nullptr || source == nullptr || !source->indirect_descriptor.has_value() ||
 		    handle->NumArgs() == 0u) {
 			ctx.Fail(inst, "has invalid indirect image key provenance");
 			return;
 		}
 		const auto key = ctx.Def(handle->Arg(0));
-		if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u ||
+		if (state.flattened_srt_variable == 0 ||
 		    image.indirect_resources.size() < 2u) {
 			ctx.Fail(inst, "has no indirect image runtime mapping");
 			return;
 		}
-		const auto LoadMapping = [&](uint32_t index) {
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
-			                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
-			                          index);
-			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
-			return value;
+		const auto selected = EmitIndirectResourceIndex(
+		    state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
+		struct SampleRun {
+			uint32_t first;
+			uint32_t count;
+			uint32_t resource;
+			uint32_t slot_bias;
 		};
-		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
-		auto       low      = ConstantU32(state, 0u);
-		auto       high     = LoadMapping(mapping);
-		auto       selected = ConstantU32(state, 0u);
-		for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
-		     iteration++) {
-			const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
-			const auto mid    = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
-			                           Binary(state, spv::OpIAdd, TypeU32(state), low, high),
-			                           ConstantU32(state, 1u));
-			const auto probe  = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid,
-			                          ConstantU32(state, 0u));
-			const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
-			                          Binary(state, spv::OpIAdd, TypeU32(state),
-			                                 Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-			                                        probe, ConstantU32(state, 1u)),
-			                                 ConstantU32(state, 1u)));
-			const auto mapped_key = LoadMapping(entry);
-			const auto candidate  = LoadMapping(
-			    Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
-			const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
-			const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
-			const auto next_selected = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match,
-			                          candidate, selected);
-			selected              = next_selected;
-			const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
-			const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
-			const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
-			                               Unary(state, spv::OpLogicalNot, TypeBool(state), less));
-			const auto next_low   = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    spv::OpSelect, TypeU32(state), next_low, take_upper,
-			    Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
-			low                  = next_low;
-			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid,
-			                          high);
-			high = next_high;
+		std::vector<SampleRun> runs;
+		for (uint32_t ordinal = 0; ordinal < image.indirect_resources.size(); ++ordinal) {
+			const auto resource = image.indirect_resources[ordinal];
+			const auto& candidate = state.program.info.images[resource];
+			const auto kind = *IR::DescriptorBindingForImage(candidate);
+			const auto& binding = *IR::FindBinding(state.program.bindings, kind);
+			if (!runs.empty()) {
+				auto& run = runs.back();
+				const auto& first = state.program.info.images[run.resource];
+				const auto next_slot = run.slot_bias + ordinal;
+				if (candidate.dimension == first.dimension && candidate.cube == first.cube &&
+				    IR::DescriptorBindingForImage(first) == kind &&
+				    candidate.mip_count == 1u && first.mip_count == 1u &&
+				    (ordinal == 1u || (next_slot < binding.resources.size() &&
+				                      binding.resources[next_slot] == resource))) {
+					// Native roots precede appended children; only the root may have a slot gap.
+					if (ordinal == 1u)
+						run.slot_bias = ResourceForDescriptor(state, kind, resource) - ordinal;
+					++run.count;
+					continue;
+				}
+			}
+			runs.push_back({ordinal, 1u, resource,
+			                ResourceForDescriptor(state, kind, resource) - ordinal});
 		}
-		const auto            default_label = state.builder.AllocateId();
-		const auto            merge_label   = state.builder.AllocateId();
-		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
-		std::vector<uint32_t> switch_words {spv::OpSwitch, selected, default_label};
-		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
-			labels[candidate - 1u] = state.builder.AllocateId();
-			switch_words.push_back(candidate);
-			switch_words.push_back(labels[candidate - 1u]);
+		const auto EmitRun = [&](uint32_t index) {
+			const auto& run = runs[index];
+			if (run.count == 1u) return EmitSample(run.resource);
+			auto slot = Binary(state, spv::OpIAdd, TypeU32(state), selected,
+			                   ConstantU32(state, run.slot_bias));
+			if (run.first == 0u) {
+				const auto kind = *IR::DescriptorBindingForImage(image);
+				const auto root_slot = ResourceForDescriptor(state, kind, mem.resource);
+				if (root_slot != run.slot_bias) {
+					const auto is_root = Binary(state, spv::OpIEqual, TypeBool(state), selected,
+					                            ConstantU32(state, 0u));
+					const auto root_or_child = state.builder.AllocateId();
+					state.builder.AddFunction(spv::OpSelect, TypeU32(state), root_or_child,
+					                          is_root, ConstantU32(state, root_slot), slot);
+					slot = root_or_child;
+				}
+			}
+			return EmitSample(run.resource, slot);
+		};
+		auto run_index = ConstantU32(state, 0u);
+		for (uint32_t index = 1; index < runs.size(); ++index) {
+			const auto at_or_after = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state),
+			                               selected, ConstantU32(state, runs[index].first));
+			const auto next = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next, at_or_after,
+			                          ConstantU32(state, index), run_index);
+			run_index = next;
 		}
-		state.builder.AddFunction(spv::OpSelectionMerge, merge_label,
-		                          spv::SelectionControlMaskNone);
-		state.builder.AddFunction(switch_words);
-		std::vector<uint32_t> phi_words {spv::OpPhi, result_type, state.builder.AllocateId()};
-		EmitLabel(state, default_label);
-		phi_words.push_back(EmitSample(image.indirect_resources[0]));
-		phi_words.push_back(default_label);
-		state.builder.AddFunction(spv::OpBranch, merge_label);
-		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
-			EmitLabel(state, labels[candidate - 1u]);
-			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
-			phi_words.push_back(labels[candidate - 1u]);
-			state.builder.AddFunction(spv::OpBranch, merge_label);
-		}
-		EmitLabel(state, merge_label);
-		state.builder.AddFunction(phi_words);
-		auto result = phi_words[2];
+		auto result = runs.size() == 1u
+		                  ? EmitRun(0u)
+		                  : EmitIndexSwitch(state, run_index, static_cast<uint32_t>(runs.size()),
+		                                    result_type, EmitRun);
 		if (!dref) {
 			result = UnpackImageTexel(ctx, mem, result);
 		}
@@ -908,9 +932,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(inst, EmitValueOrDefaultIfCondition(
 		                     state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() {
 			           const auto pointer      = state.builder.AllocateId();
-			           const auto pointer_type = state.builder.Type(
-			               spv::OpTypePointer, spv::StorageClassImage,
-			               image.atomic64 ? TypeScalarU64(state) : TypeU32(state));
+			           const auto pointer_type =
+			               state.builder.Type(spv::OpTypePointer, spv::StorageClassImage,
+			                                  image.atomic64 ? TypeU64(state) : TypeU32(state));
 			           state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
 			                                     ImageDescriptorPointer(state, mem.resource),
 			                                     CoordU32(ctx, mem, *address, dimension),

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -34,16 +34,16 @@ uint32_t              EmitConvertF32F64(EmitterState& state, uint32_t arg0);
 uint32_t              EmitConvertF64F32(EmitterState& state, uint32_t arg0);
 EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
 EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
-EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
+uint32_t EmitCompositeConstructU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
 EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
 EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
             uint32_t)
 uint32_t              EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
-inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
-inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
-inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
+uint32_t              EmitCompositeExtractU32x2(EmitterState& state, uint32_t arg0, IR::Value arg1);
+inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU32x2;
+inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU32x2;
 inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
 inline constexpr auto EmitPackSnorm2x16 =
     EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
@@ -63,9 +63,9 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
 EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
 EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
-uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(IAdd64, OpIAdd, U64, uint32_t, uint32_t)
+EMIT_NATIVE(ISub64, OpISub, U64, uint32_t, uint32_t)
+EMIT_NATIVE(IMul64, OpIMul, U64, uint32_t, uint32_t)
 EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
 uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
@@ -73,14 +73,26 @@ uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
 EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
 EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
-uint32_t EmitShiftLeftLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
-uint32_t EmitShiftRightLogical64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
-uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
+template <spv::Op opcode>
+uint32_t EmitShift64(ValueEmitContext& ctx, uint32_t value, IR::Value shift) {
+	shift = shift.Resolve();
+	if (shift.IsImmediate()) {
+		const auto amount = shift.U32() & 63u;
+		return amount == 0u ? value
+		                    : Binary(ctx.state, opcode, TypeU64(ctx.state), value,
+		                             ConstantU32(ctx.state, amount));
+	}
+	return Binary(ctx.state, opcode, TypeU64(ctx.state), value,
+	              EmitAndConstant(ctx.state, ctx.Def(shift), 63u));
+}
+inline constexpr auto EmitShiftLeftLogical64     = EmitShift64<spv::OpShiftLeftLogical>;
+inline constexpr auto EmitShiftRightLogical64    = EmitShift64<spv::OpShiftRightLogical>;
+inline constexpr auto EmitShiftRightArithmetic64 = EmitShift64<spv::OpShiftRightArithmetic>;
 EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
 EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
 EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
 EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
-uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(BitwiseAnd64, OpBitwiseAnd, U64, uint32_t, uint32_t)
 EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
 EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
 uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
@@ -107,14 +119,14 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
 EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
 EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
 EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
-uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
-uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+EMIT_NATIVE(IEqual64, OpIEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(INotEqual64, OpINotEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThan64, OpULessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SLessThan64, OpSLessThan, U1, uint32_t, uint32_t)
+EMIT_NATIVE(SLessThanEqual64, OpSLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(ULessThanEqual64, OpULessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThanEqual64, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(UGreaterThan64, OpUGreaterThan, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
@@ -152,6 +164,7 @@ inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32<spv::OpFO
 inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32<spv::OpFUnordGreaterThanEqual>;
 uint32_t              EmitFPIsNan32(EmitterState& state, uint32_t arg0);
 inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
+inline constexpr auto EmitFPCmpClass16 = EmitClassMaskF16;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
@@ -160,10 +173,12 @@ EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t)
 inline constexpr auto EmitFPFma64 =
     EmitGlsl<GLSLstd450Fma, IR::Type::F64, uint32_t, uint32_t, uint32_t>;
 uint32_t              EmitFPRecip64(EmitterState& state, uint32_t arg0);
-inline constexpr auto EmitFPFma32 =
-    EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
+uint32_t EmitFPFma32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
+uint32_t EmitFPMad32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMin64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitFPMax64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
 uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
@@ -178,7 +193,11 @@ inline constexpr auto EmitFPRoundEven32 = EmitGlsl<GLSLstd450RoundEven, IR::Type
 inline constexpr auto EmitFPFloor32     = EmitGlsl<GLSLstd450Floor, IR::Type::F32, uint32_t>;
 inline constexpr auto EmitFPCeil32      = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
 inline constexpr auto EmitFPTrunc32     = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFloor64     = EmitGlsl<GLSLstd450Floor, IR::Type::F64, uint32_t>;
+inline constexpr auto EmitFPCeil64      = EmitGlsl<GLSLstd450Ceil, IR::Type::F64, uint32_t>;
+inline constexpr auto EmitFPTrunc64     = EmitGlsl<GLSLstd450Trunc, IR::Type::F64, uint32_t>;
 inline constexpr auto EmitFPFract32     = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFract64     = EmitGlsl<GLSLstd450Fract, IR::Type::F64, uint32_t>;
 uint32_t              EmitFPSin(EmitterState& state, uint32_t arg0);
 uint32_t              EmitFPCos(EmitterState& state, uint32_t arg0);
 #undef EMIT_NATIVE
@@ -188,16 +207,17 @@ void                  EmitVoid(ValueEmitContext&);
 inline constexpr auto EmitReference    = EmitVoid;
 inline constexpr auto EmitReferenceU32 = EmitVoid;
 inline constexpr auto EmitControlNop   = EmitVoid;
-inline constexpr auto EmitWaitcnt      = EmitVoid;
 inline constexpr auto EmitSendmsg      = EmitVoid;
 inline constexpr auto EmitTtraceData   = EmitVoid;
 inline constexpr auto EmitInstPrefetch = EmitVoid;
 void                  EmitBarrier(EmitterState& state);
+void                  EmitStoreCompletion(EmitterState& state);
 void                  EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
+uint32_t              EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
 uint32_t              EmitUndefU1(EmitterState& state, const IR::Inst& inst);
 inline constexpr auto EmitUndefU8  = EmitUndefU1;
@@ -230,10 +250,12 @@ void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& ins
 uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitPermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -312,15 +334,21 @@ inline constexpr auto EmitImageSampleRaw       = EmitImage;
 inline constexpr auto EmitImageGatherRaw       = EmitImage;
 inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage;
 inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
+inline constexpr auto EmitImageAtomicSwap64    = EmitImage;
 inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicIAdd64    = EmitImage;
 inline constexpr auto EmitImageAtomicSMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMin64    = EmitImage;
 inline constexpr auto EmitImageAtomicSMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax64    = EmitImage;
 inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
+inline constexpr auto EmitImageAtomicAnd64     = EmitImage;
 inline constexpr auto EmitImageAtomicOr32      = EmitImage;
+inline constexpr auto EmitImageAtomicOr64      = EmitImage;
 inline constexpr auto EmitImageAtomicXor32     = EmitImage;
+inline constexpr auto EmitImageAtomicXor64     = EmitImage;
 inline constexpr auto EmitImageAtomicFMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicFMax32    = EmitImage;
 void                  EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -61,6 +61,7 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 struct SpirvRequirements {
 	bool bvh                          = false;
 	bool subgroup_ballot              = false;
+	bool subgroup_barrier             = false;
 	bool subgroup_shuffle             = false;
 	bool subgroup_local_invocation_id = false;
 	bool compute_derivatives          = false;
@@ -69,6 +70,8 @@ struct SpirvRequirements {
 	bool function_scratch             = false;
 	bool pixel_valid_mask             = false;
 	bool buffer_int64_atomics         = false;
+	bool buffer_u8                    = false;
+	bool buffer_u16                   = false;
 	bool shared_int64_atomics         = false;
 	bool coherent_buffers             = false;
 	bool float64                      = false;
@@ -80,7 +83,14 @@ struct EmitterState {
 	EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
 	    : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
 	      program(program_), input_info(input_info_),
-	      requirements(AnalyzeProgramRequirements(program_)) {}
+	      requirements(AnalyzeProgramRequirements(program_)) {
+		if (ShaderWorkgroupInput(program.stage, input_info) != nullptr) {
+			lds_storage_class = spv::StorageClassWorkgroup;
+		}
+		if (IR::FindBinding(program.bindings, IR::DescriptorBindingKind::SharedMemory) != nullptr) {
+			lds_storage_class = spv::StorageClassStorageBuffer;
+		}
+	}
 
 	Builder                                          builder;
 	const IR::Program&                               program;
@@ -88,7 +98,7 @@ struct EmitterState {
 	uint32_t                                        void_type = 0;
 	uint32_t                                        bool_type = 0;
 	uint32_t                                        u32_type = 0;
-	uint32_t                                        native_u64_type = 0;
+	uint32_t                                        u64_type = 0;
 	uint32_t                                        i32_type = 0;
 	uint32_t                                        f32_type = 0;
 	uint32_t                                        f64_type = 0;
@@ -107,6 +117,8 @@ struct EmitterState {
 	uint32_t                                         lane_count              = 1;
 	uint32_t                                         lane_half               = 0;
 	uint32_t                                         storage_buffer_variable = 0;
+	uint32_t                                         storage_buffer_u8_variable = 0;
+	uint32_t                                         storage_buffer_u16_variable = 0;
 	uint32_t                                         storage_buffer_u64_variable = 0;
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
 	uint32_t                                         bda_pagetable_variable  = 0;
@@ -118,6 +130,8 @@ struct EmitterState {
 	uint32_t                                         push_constant_variable  = 0;
 	uint32_t                                         shader_data_storage_variable = 0;
 	uint32_t                                         flattened_srt_variable  = 0;
+	spv::StorageClass                                lds_storage_class = spv::StorageClassFunction;
+	uint32_t                                         lds_base_dwords         = 0;
 	uint32_t                                         lds_variable            = 0;
 	uint32_t                                         lds_u64_variable        = 0;
 	std::array<uint32_t, 2>                          scratch_variable {};
@@ -154,7 +168,6 @@ uint32_t TypeBool(EmitterState& state);
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
 uint32_t TypeU64(EmitterState& state);
-uint32_t TypeScalarU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
 uint32_t TypeI32Pair(EmitterState& state);
@@ -167,10 +180,9 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
 uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
 uint32_t TypeFunction(EmitterState& state);
-uint32_t TypeStorageBufferPointer(EmitterState& state);
-uint32_t TypeStorageBufferElementPointer(EmitterState& state);
-uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
-uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
+uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits);
+uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits = 32);
+uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32);
 uint32_t TypePhysicalU32Pointer(EmitterState& state);
 uint32_t TypePushConstantElementPointer(EmitterState& state);
 uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
@@ -208,6 +220,12 @@ inline uint32_t Unary(EmitterState& state, spv::Op opcode, uint32_t type, uint32
 	return result;
 }
 
+inline uint32_t PackU64(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto pair = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), pair, low, high);
+	return Unary(state, spv::OpBitcast, TypeU64(state), pair);
+}
+
 inline uint32_t Binary(EmitterState& state, spv::Op opcode, uint32_t type, uint32_t lhs,
                        uint32_t rhs) {
 	const auto result = state.builder.AllocateId();
@@ -315,14 +333,16 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image);
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension);
 
-uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0,
+                                uint32_t array_index = 0);
 
-uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0,
+                             uint32_t array_index = 0);
 
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler);
 
 uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id,
-                          uint32_t mip = 0);
+                          uint32_t mip = 0, uint32_t array_index = 0);
 
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
                            uint32_t texel);
@@ -348,7 +368,8 @@ uint32_t DefineInterfaceVariable(EmitterState& state, uint32_t type, spv::Storag
 void     DefineModule(EmitterState& state);
 void     DefineTessellationInterfaces(EmitterState& state);
 void     DefineTessellationExecutionModes(EmitterState& state);
-void     DefineMeshOutputs(EmitterState& state);
+void     DefineMeshOutputs(EmitterState& state, uint32_t clip_distance_count,
+                           uint32_t cull_distance_count);
 void     EmitMeshEntryPoint(EmitterState& state);
 void     EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0);
@@ -392,6 +413,7 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
 uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
 
 Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
+uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem);
 
 void EmitMemoryOffsets(EmitterState& state);
 
@@ -402,21 +424,19 @@ struct MemoryResourceAccess {
 	IR::ResourceKind      kind             = IR::ResourceKind::None;
 	uint32_t              object_pointer   = 0;
 	uint32_t              length           = 0;
-	uint32_t              index_offset     = 0;
 	uint32_t              byte_offset      = 0;
-	bool                  add_index_offset = false;
+	uint32_t              element_bits     = 32;
 	spv::MemoryAccessMask memory_access    = spv::MemoryAccessMaskNone;
 };
 
+uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
+                                   uint32_t search_iterations, uint32_t default_resource);
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
 MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState&         state,
                                                         const IR::MemoryInfo& mem,
                                                         uint32_t variable, uint32_t pointer_type);
 
-uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
-                                uint32_t raw_index);
-
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
                                    uint32_t index);
 
@@ -448,6 +468,9 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind);
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value);
 
+uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
+                                      bool max_value);
+
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
 
 inline constexpr auto EmitSelectValueU32 =
@@ -492,6 +515,7 @@ uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
                                uint32_t class_match);
 
 uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
+uint32_t EmitClassMaskF16(EmitterState& state, uint32_t bits, uint32_t mask);
 
 uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
 
@@ -512,22 +536,20 @@ void EmitProgram(EmitterState& state);
 void DefineGetBdaPointer(EmitterState& state);
 void DefineBvhIntersect(EmitterState& state);
 uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
-uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
-uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
-auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_count,
+auto EmitIndexSwitch(EmitterState& state, uint32_t index, uint32_t count,
                         uint32_t result_type, Fn&& emit) {
 	constexpr bool has_result = !std::is_void_v<std::invoke_result_t<Fn, uint32_t>>;
-	EXIT_IF(mip_count == 0u);
+	EXIT_IF(count == 0u);
 	const auto merge_label = state.builder.AllocateId();
-	std::vector<uint32_t> labels(mip_count);
-	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
-	for (uint32_t mip = 0; mip < mip_count; mip++) {
-		labels[mip] = state.builder.AllocateId();
-		words.push_back(mip);
-		words.push_back(labels[mip]);
+	std::vector<uint32_t> labels(count);
+	std::vector<uint32_t> words {spv::OpSwitch, index, merge_label};
+	for (uint32_t item = 0; item < count; item++) {
+		labels[item] = state.builder.AllocateId();
+		words.push_back(item);
+		words.push_back(labels[item]);
 	}
 	if constexpr (has_result) {
 		words[2] = labels.front();
@@ -538,13 +560,13 @@ auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_coun
 	if constexpr (has_result) {
 		phi = {spv::OpPhi, result_type, state.builder.AllocateId()};
 	}
-	for (uint32_t mip = 0; mip < mip_count; mip++) {
-		EmitLabel(state, labels[mip]);
+	for (uint32_t item = 0; item < count; item++) {
+		EmitLabel(state, labels[item]);
 		if constexpr (has_result) {
-			phi.push_back(emit(mip));
+			phi.push_back(emit(item));
 			phi.push_back(state.current_label);
 		} else {
-			emit(mip);
+			emit(item);
 		}
 		state.builder.AddFunction(spv::OpBranch, merge_label);
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -31,6 +31,61 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
 	return ret;
 }
 
+uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
+                                   uint32_t search_iterations, uint32_t default_resource) {
+	const auto LoadMapping = [&](uint32_t index) {
+		const auto pointer = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+		                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+		                          index);
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
+		return value;
+	};
+	const auto mapping = ConstantU32(state, mapping_offset);
+	const auto count = LoadMapping(mapping);
+	if (search_iterations == 0u) {
+		const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
+		                          ConstantU32(state, 2u));
+		const auto valid = Binary(state, spv::OpULessThan, TypeBool(state), index, count);
+		const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u));
+		const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe,
+		                          ConstantU32(state, mapping_offset + 1u));
+		return Select(state, TypeU32(state), valid, LoadMapping(entry),
+		              ConstantU32(state, default_resource));
+	}
+	const auto entry_at = [&](uint32_t index) {
+		return Binary(state, spv::OpIAdd, TypeU32(state), mapping,
+		              Binary(state, spv::OpIAdd, TypeU32(state),
+		                     Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
+		                            ConstantU32(state, 1u)), ConstantU32(state, 1u)));
+	};
+	auto low = ConstantU32(state, 0u);
+	auto high = count;
+	for (uint32_t iteration = 0; iteration < search_iterations; ++iteration) {
+		const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
+		const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+		                        Binary(state, spv::OpIAdd, TypeU32(state), low, high),
+		                        ConstantU32(state, 1u));
+		const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
+		const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
+		                         LoadMapping(entry_at(probe)), key);
+		low = Select(state, TypeU32(state),
+		             Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
+		             Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
+		high = Select(state, TypeU32(state),
+		              Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
+		                     Unary(state, spv::OpLogicalNot, TypeBool(state), less)), mid, high);
+	}
+	const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
+	const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
+	const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
+	                          Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
+	const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
+	                                         ConstantU32(state, 1u)));
+	return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
+}
+
 uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
@@ -47,6 +102,15 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
 	return state.program.info.buffers[mem.resource].descriptor_format;
 }
 
+uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) {
+	if (!mem.formatted) return mem.data_bits;
+	const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
+	                              : program.info.buffers[mem.resource].descriptor_format;
+	const auto info = Format::GetFormatInfo(format);
+	return info.type == Format::ComponentType::Unknown || info.packed_bitfield
+	           ? 32u : info.component_bits[0];
+}
+
 void EmitMemoryOffsets(EmitterState& state) {
 	for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
 		const auto word =
@@ -67,8 +131,8 @@ void EnsureLdsStorage(EmitterState& state) {
 	if (state.lds_variable != 0) {
 		return;
 	}
-	if (ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
-		EXIT("function LDS was not prepared before SPIR-V function emission\n");
+	if (state.lds_storage_class != spv::StorageClassWorkgroup) {
+		EXIT("LDS backing was not prepared before SPIR-V function emission\n");
 	}
 	const auto define = [&](uint32_t type, uint32_t bytes) {
 		const auto array = state.builder.DecoratedType(
@@ -84,7 +148,7 @@ void EnsureLdsStorage(EmitterState& state) {
 	};
 	if (state.requirements.shared_int64_atomics) {
 		state.lds_variable = define(TypeU32(state), 4u);
-		state.lds_u64_variable = define(TypeScalarU64(state), 8u);
+		state.lds_u64_variable = define(TypeU64(state), 8u);
 		state.builder.AddName(state.lds_u64_variable, "lds_qwords");
 	} else {
 		state.lds_variable = state.builder.DefineGlobalVariable(
@@ -149,24 +213,18 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 			EXIT("physical address memory must use the BDA emitter\n");
 		case IR::ResourceKind::ScalarBuffer:
 		case IR::ResourceKind::Buffer: {
+			const auto bits = mem.kind == IR::ResourceKind::Buffer
+			                      ? StorageBufferElementBits(state.program, mem) : 32u;
+			const auto variable = bits == 8u ? state.storage_buffer_u8_variable
+			                      : bits == 16u ? state.storage_buffer_u16_variable
+			                                    : state.storage_buffer_variable;
 			access = PrepareStorageBufferResourceAccess(
-			    state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
-			access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
-			                                    ConstantU32(state, 2u));
-			access.add_index_offset = true;
+			    state, mem, variable, TypeStorageBufferPointer(state, bits));
+			access.element_bits = bits;
 			return access;
 		}
 		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
 	}
-	access.length = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
-	                          access.object_pointer, 0);
-	return access;
-}
-
-uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
-                                uint32_t raw_index) {
-	return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
 }
 
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
@@ -179,12 +237,14 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
 uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
                                   uint32_t index) {
 	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
+		const auto storage_class = access.kind == IR::ResourceKind::Scratch
+		                               ? spv::StorageClassFunction : state.lds_storage_class;
+		if (storage_class == spv::StorageClassStorageBuffer) {
+			return EmitStorageBufferElementPointer(
+			    state, access, EmitAddU32(state, state.lds_base_dwords, index),
+			    TypeStorageBufferElementPointer(state, 32));
+		}
 		const auto pointer = state.builder.AllocateId();
-		const auto storage_class =
-		    access.kind == IR::ResourceKind::Scratch ? spv::StorageClassFunction
-		    : ShaderWorkgroupInput(state.program.stage, state.input_info) != nullptr
-		        ? spv::StorageClassWorkgroup
-		        : spv::StorageClassFunction;
 		if (access.kind == IR::ResourceKind::Lds && state.requirements.shared_int64_atomics) {
 			state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class),
 			                          pointer, access.object_pointer, ConstantU32(state, 0), index);
@@ -195,7 +255,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
 		return pointer;
 	}
 	return EmitStorageBufferElementPointer(state, access, index,
-	                                       TypeStorageBufferElementPointer(state));
+	                                       TypeStorageBufferElementPointer(state, access.element_bits));
 }
 
 uint32_t EmitStorageBufferElementPointer(EmitterState& state,
@@ -311,10 +371,12 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicCompareSwap32:
 		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
 		case IR::ValueOpcode::ImageAtomicSwap32:
+		case IR::ValueOpcode::ImageAtomicSwap64:
 		case IR::ValueOpcode::BufferAtomicSwap32:
 		case IR::ValueOpcode::BufferAtomicSwap64:
 		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
 		case IR::ValueOpcode::ImageAtomicIAdd32:
+		case IR::ValueOpcode::ImageAtomicIAdd64:
 		case IR::ValueOpcode::BufferAtomicIAdd32:
 		case IR::ValueOpcode::SharedAtomicIAdd64:
 		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
@@ -324,6 +386,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::BufferAtomicSMin32:
 		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
 		case IR::ValueOpcode::ImageAtomicUMin32:
+		case IR::ValueOpcode::ImageAtomicUMin64:
 		case IR::ValueOpcode::BufferAtomicUMin32:
 		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
 		case IR::ValueOpcode::ImageAtomicSMax32:
@@ -334,15 +397,18 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::BufferAtomicUMax32:
 		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
 		case IR::ValueOpcode::ImageAtomicAnd32:
+		case IR::ValueOpcode::ImageAtomicAnd64:
 		case IR::ValueOpcode::BufferAtomicAnd32:
 		case IR::ValueOpcode::BufferAtomicAnd64:
 		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
 		case IR::ValueOpcode::ImageAtomicOr32:
+		case IR::ValueOpcode::ImageAtomicOr64:
 		case IR::ValueOpcode::BufferAtomicOr32:
 		case IR::ValueOpcode::BufferAtomicOr64:
 		case IR::ValueOpcode::SharedAtomicOr64:
 		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
 		case IR::ValueOpcode::ImageAtomicXor32:
+		case IR::ValueOpcode::ImageAtomicXor64:
 		case IR::ValueOpcode::BufferAtomicXor32:
 		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
 		default: return spv::OpNop;
@@ -354,7 +420,7 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
 	const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
 	const auto old    = ctx.state.builder.AllocateId();
 	const bool wide   = inst.GetType() == IR::Type::U64;
-	const auto type   = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state);
+	const auto type   = wide ? TypeU64(ctx.state) : TypeU32(ctx.state);
 	if (opcode == spv::OpAtomicCompareExchange) {
 		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
 		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
@@ -363,20 +429,22 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
 		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
 		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
 	} else {
-		auto value = ctx.Arg(inst, inst.NumArgs() - 2);
-		if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value);
+		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
 		ctx.state.builder.AddFunction(opcode, type, old, pointer,
 		                              ConstantU32(ctx.state, scope),
 		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
 	}
-	return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old;
+	return old;
 }
 
 void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
 	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto memory = [&] {
 		switch (kind) {
-			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Lds:
+				return state.lds_storage_class == spv::StorageClassStorageBuffer
+				           ? spv::MemorySemanticsUniformMemoryMask
+				           : spv::MemorySemanticsWorkgroupMemoryMask;
 			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
 			default: return spv::MemorySemanticsUniformMemoryMask;
 		}
@@ -385,6 +453,20 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
 	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
 }
 
+namespace {
+
+uint32_t FloatOrderKey(EmitterState& state, uint32_t bits) {
+	const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
+	                                             EmitAndConstant(state, bits, 0x80000000u), 0u);
+	const auto negative_key = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
+	const auto positive_key =
+	    EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
+	return EmitSelectValueU32(state, negative, negative_key, positive_key);
+}
+
+} // namespace
+
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value) {
 	struct OrderedBits {
@@ -394,15 +476,7 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	};
 	const auto classify = [&](uint32_t bits) {
 		const auto cls = EmitClassifyF32Bits(state, bits);
-		const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
-		                                             EmitAndConstant(state, bits, 0x80000000u), 0u);
-		const auto negative_key = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
-		const auto positive_key =
-		    EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
-		return OrderedBits {
-		    cls.nan, cls.zero,
-		    EmitSelectValueU32(state, negative, negative_key, positive_key)};
+		return OrderedBits {cls.nan, cls.zero, FloatOrderKey(state, bits)};
 	};
 	const auto source_class = classify(source);
 	const auto old_class    = classify(old);
@@ -417,6 +491,45 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
 	    old);
 }
 
+uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
+                                      bool max_value) {
+	struct OrderedBits {
+		uint32_t nan;
+		uint32_t signaling_nan;
+		uint32_t key;
+	};
+	const auto classify = [&](uint32_t bits) {
+		const auto abs = EmitAndConstant(state, bits, 0x7fffffffu);
+		const auto nan = EmitCompareU32Constant(state, spv::OpUGreaterThan, abs, 0x7f800000u);
+		const auto signaling_nan = EmitLogicalAndBool(
+		    state, nan, EmitCompareU32Constant(state, spv::OpIEqual,
+		                                      EmitAndConstant(state, bits, 0x00400000u), 0u));
+		const auto small = EmitCompareU32Constant(state, spv::OpULessThan, abs, 0x00800000u);
+		const auto sign = EmitAndConstant(state, bits, 0x80000000u);
+		const auto positive_denorm = EmitLogicalAndBool(
+		    state, EmitCompareU32Constant(state, spv::OpIEqual, sign, 0u),
+		    EmitCompareU32Constant(state, spv::OpINotEqual, abs, 0u));
+		// Negative denorms tie with -0; positive denorms tie only with each other.
+		const auto small_bits = EmitSelectValueU32(
+		    state, positive_denorm, ConstantU32(state, 1u), sign);
+		const auto key_bits = EmitSelectValueU32(state, small, small_bits, bits);
+		return OrderedBits {nan, signaling_nan, FloatOrderKey(state, key_bits)};
+	};
+	const auto lhs = classify(old);
+	const auto rhs = classify(source);
+	const auto choose_source = state.builder.AllocateId();
+	state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThanEqual,
+	                          TypeBool(state), choose_source, rhs.key, lhs.key);
+	// Keep original bits: ordering denorms does not flush the selected operand.
+	auto result = EmitSelectValueU32(state, choose_source, source, old);
+	result = EmitSelectValueU32(state, lhs.nan, source, result);
+	result = EmitSelectValueU32(state, rhs.nan, old, result);
+	result = EmitSelectValueU32(state, rhs.signaling_nan,
+	                            EmitOrU32(state, source, ConstantU32(state, 0x00400000u)), result);
+	return EmitSelectValueU32(state, lhs.signaling_nan,
+	                          EmitOrU32(state, old, ConstantU32(state, 0x00400000u)), result);
+}
+
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
 	if ((control & 0xc000u) == 0xc000u) {
 		const uint32_t mask         = control & 0x1fu;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -25,43 +25,58 @@ uint32_t MeshLoad(EmitterState& state, uint32_t variable, spv::StorageClass stor
 }
 
 uint32_t MeshOutputType(EmitterState& state, IR::StageOutputKind kind) {
-	return kind == IR::StageOutputKind::Layer ? TypeU32(state) : TypeF32Vector(state, 4);
+	switch (kind) {
+		case IR::StageOutputKind::Layer: return TypeU32(state);
+		case IR::StageOutputKind::ClipDistance:
+		case IR::StageOutputKind::CullDistance: return TypeF32(state);
+		case IR::StageOutputKind::Position:
+		case IR::StageOutputKind::Parameter: return TypeF32Vector(state, 4);
+		default: EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(kind));
+	}
 }
 
 } // namespace
 
-void DefineMeshOutputs(EmitterState& state) {
+void DefineMeshOutputs(EmitterState& state, uint32_t clip_distance_count,
+                       uint32_t cull_distance_count) {
 	const auto& mesh = state.input_info.vertex->mesh;
 	for (auto& output: state.outputs) {
-		if (output.kind != IR::StageOutputKind::Position &&
-		    output.kind != IR::StageOutputKind::Parameter &&
-		    output.kind != IR::StageOutputKind::Layer) {
-			EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(output.kind));
+		const auto type = MeshOutputType(state, output.kind);
+		const bool clip = output.kind == IR::StageOutputKind::ClipDistance;
+		const bool cull = output.kind == IR::StageOutputKind::CullDistance;
+		auto& variable = clip ? state.clip_distance_variable
+		                 : cull ? state.cull_distance_variable : output.variable_id;
+		if (variable == 0) {
+			const auto element_type = clip || cull
+			                              ? state.builder.Type(spv::OpTypeArray, type,
+			                                                   ConstantU32(state, clip ? clip_distance_count
+			                                                                           : cull_distance_count))
+			                              : type;
+			variable = MeshArray(state, spv::StorageClassOutput, element_type,
+			                     output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives
+			                                                             : mesh.max_vertices);
+			state.interface_variables.push_back(variable);
+			state.builder.AddName(variable, output.debug_name.c_str());
+			if (output.kind == IR::StageOutputKind::Parameter) {
+				state.builder.AddAnnotation(spv::OpDecorate, variable,
+				                            spv::DecorationLocation, output.location);
+			} else {
+				const auto builtin = clip ? spv::BuiltInClipDistance
+				                     : cull ? spv::BuiltInCullDistance
+				                     : output.kind == IR::StageOutputKind::Layer ? spv::BuiltInLayer
+				                                                                 : spv::BuiltInPosition;
+				state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn, builtin);
+			}
+			if (output.kind == IR::StageOutputKind::Layer) {
+				state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationPerPrimitiveEXT);
+			}
 		}
-		const auto type    = MeshOutputType(state, output.kind);
-		output.variable_id = MeshArray(
-		    state, spv::StorageClassOutput, type,
-		    output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
+		output.variable_id = variable;
 		// Only Layer is read by another invocation, through the primitive's provoking vertex.
 		const bool shared = output.kind == IR::StageOutputKind::Layer;
 		output.mesh_data_variable =
 		    MeshArray(state, shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, type,
 		              shared ? mesh.max_vertices : state.lane_count);
-		state.interface_variables.push_back(output.variable_id);
-		state.builder.AddName(output.variable_id, output.debug_name.c_str());
-		if (output.kind == IR::StageOutputKind::Parameter) {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
-			                            spv::DecorationLocation, output.location);
-		} else {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id, spv::DecorationBuiltIn,
-			                            output.kind == IR::StageOutputKind::Layer
-			                                ? spv::BuiltInLayer
-			                                : spv::BuiltInPosition);
-		}
-		if (output.kind == IR::StageOutputKind::Layer) {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
-			                            spv::DecorationPerPrimitiveEXT); // PerPrimitiveEXT
-		}
 	}
 	state.mesh_allocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2);
 	state.mesh_primitive_data =
@@ -148,8 +163,16 @@ void EmitMeshEntryPoint(EmitterState& state) {
 				const auto value =
 				    MeshLoad(state, output.mesh_data_variable, spv::StorageClassPrivate, type,
 				             ConstantU32(state, half));
-				const auto pointer =
-				    MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				uint32_t pointer;
+				if (output.kind == IR::StageOutputKind::ClipDistance ||
+				    output.kind == IR::StageOutputKind::CullDistance) {
+					pointer = state.builder.AllocateId();
+					state.builder.AddFunction(
+					    spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, type), pointer,
+					    output.variable_id, index, ConstantU32(state, output.index));
+				} else {
+					pointer = MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				}
 				state.builder.AddFunction(spv::OpStore, pointer, value);
 			}
 		});

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -35,14 +35,11 @@ uint32_t TypeU32(EmitterState& state) {
 }
 
 uint32_t TypeU64(EmitterState& state) {
-	return TypeU32Vector(state, 2);
-}
-
-uint32_t TypeScalarU64(EmitterState& state) {
-	if (state.native_u64_type == 0) {
-		state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
+	if (state.u64_type == 0) {
+		state.builder.RequireCapability(spv::CapabilityInt64);
+		state.u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
 	}
-	return state.native_u64_type;
+	return state.u64_type;
 }
 
 uint32_t TypeU32Pair(EmitterState& state) {
@@ -91,7 +88,8 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
 }
 
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
-	EXIT_IF(components < 2u || components > 4u);
+	EXIT_IF(components < 1u || components > 4u);
+	if (components == 1u) return TypeU32(state);
 	return components == 2u ? TypeU32Pair(state) : TypeU32Vector(state, components);
 }
 
@@ -122,46 +120,32 @@ uint32_t TypeFunction(EmitterState& state) {
 	return state.function_type;
 }
 
-uint32_t StorageRuntimeArrayType(EmitterState& state) {
-	return state.builder.DecoratedType(
-	    spv::OpTypeRuntimeArray,
-	    {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}}, TypeU32(state));
+uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) {
+	return bits == 32u   ? TypeU32(state)
+	       : bits == 64u ? TypeU64(state)
+	                     : state.builder.Type(spv::OpTypeInt, bits, 0);
 }
 
-uint32_t StorageBufferType(EmitterState& state) {
+uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) {
+	return state.builder.DecoratedType(
+	    spv::OpTypeRuntimeArray,
+	    {{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}},
+	    TypeStorageBufferElement(state, bits));
+}
+
+uint32_t StorageBufferType(EmitterState& state, uint32_t bits = 32) {
 	return state.builder.DecoratedType(spv::OpTypeStruct,
 	                                   {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
 	                                    {spv::OpDecorate, {spv::DecorationBlock}}},
-	                                   StorageRuntimeArrayType(state));
+	                                   StorageRuntimeArrayType(state, bits));
 }
 
-uint32_t TypeStorageBufferPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state));
+uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits) {
+	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state, bits));
 }
 
-uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state));
-}
-
-uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
-	return state.builder.DecoratedType(
-	    spv::OpTypeRuntimeArray,
-	    {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}}, TypeScalarU64(state));
-}
-
-uint32_t StorageBufferU64Type(EmitterState& state) {
-	return state.builder.DecoratedType(spv::OpTypeStruct,
-	                                   {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
-	                                    {spv::OpDecorate, {spv::DecorationBlock}}},
-	                                   StorageU64RuntimeArrayType(state));
-}
-
-uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state));
-}
-
-uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
+uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) {
+	return TypePointer(state, spv::StorageClassStorageBuffer, TypeStorageBufferElement(state, bits));
 }
 
 uint32_t TypePhysicalU32Pointer(EmitterState& state) {
@@ -243,27 +227,34 @@ void DefineDescriptors(EmitterState& state) {
 			case IR::DescriptorBindingKind::Buffers:
 				state.storage_buffer_variable =
 				    Define(ArrayType(StorageBufferType(state)), "buffers");
+				if (state.requirements.buffer_u8) {
+					state.storage_buffer_u8_variable =
+					    Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8");
+				}
+				if (state.requirements.buffer_u16) {
+					state.storage_buffer_u16_variable =
+					    Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16");
+				}
 				if (state.requirements.buffer_int64_atomics) {
 					state.storage_buffer_u64_variable =
-					    Define(ArrayType(StorageBufferU64Type(state)), "buffers_u64");
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
-					                            spv::DecorationAliased);
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
-					                            spv::DecorationAliased);
+					    Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64");
 				}
-				if (state.requirements.coherent_buffers) {
+				for (const auto variable: {state.storage_buffer_variable, state.storage_buffer_u8_variable,
+				                           state.storage_buffer_u16_variable, state.storage_buffer_u64_variable}) {
+					if (variable == 0) continue;
+					if (state.storage_buffer_u8_variable != 0 || state.storage_buffer_u16_variable != 0 ||
+					    state.storage_buffer_u64_variable != 0) {
+						state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
+					}
 					// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
 					// must participate in visibility for cache-bypassing polling loads.
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
-					                            spv::DecorationCoherent);
-					if (state.storage_buffer_u64_variable != 0) {
-						state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
-						                            spv::DecorationCoherent);
+					if (state.requirements.coherent_buffers) {
+						state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent);
 					}
 				}
 				break;
 			case IR::DescriptorBindingKind::BdaPagetable:
-				state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");
+				state.bda_pagetable_variable = Define(StorageBufferType(state, 64), "bda_pagetable");
 				break;
 			case IR::DescriptorBindingKind::FaultBuffer:
 				state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer");
@@ -282,6 +273,17 @@ void DefineDescriptors(EmitterState& state) {
 			case IR::DescriptorBindingKind::Gds:
 				state.gds_variable = Define(StorageBufferType(state), "gds");
 				break;
+			case IR::DescriptorBindingKind::SharedMemory:
+				state.lds_variable = Define(StorageBufferType(state), "lds_dwords");
+				state.builder.AddAnnotation(spv::OpDecorate, state.lds_variable, spv::DecorationCoherent);
+				if (state.requirements.shared_int64_atomics) {
+					state.lds_u64_variable = Define(StorageBufferType(state, 64), "lds_qwords");
+					state.builder.AddAnnotation(spv::OpDecorate, state.lds_u64_variable, spv::DecorationCoherent);
+					for (const auto variable: {state.lds_variable, state.lds_u64_variable}) {
+						state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
+					}
+				}
+				break;
 			default: {
 				EXIT_IF(IR::ImageBindingResourceClass(binding.kind) ==
 				        IR::ImageResourceClass::None);
@@ -327,14 +329,14 @@ uint32_t ConstantBool(EmitterState& state, bool value) {
 }
 
 uint32_t ConstantU64(EmitterState& state, uint64_t value) {
-	return state.builder.Constant(spv::OpConstantComposite, TypeU64(state),
-	                              ConstantU32(state, static_cast<uint32_t>(value)),
-	                              ConstantU32(state, static_cast<uint32_t>(value >> 32u)));
+	return state.builder.Constant(spv::OpConstant, TypeU64(state), static_cast<uint32_t>(value),
+	                              static_cast<uint32_t>(value >> 32u));
 }
 
 uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
-	EXIT_IF(components < 2u || components > 4u);
+	EXIT_IF(components < 1u || components > 4u);
 	const auto            zero = ConstantU32(state, 0);
+	if (components == 1u) return zero;
 	std::vector<uint32_t> values(components, zero);
 	return state.builder.Constant(spv::OpConstantComposite, TypeU32Composite(state, components),
 	                              values);
@@ -414,6 +416,7 @@ uint32_t BuiltInForInput(IR::StageInputKind kind) {
 		case IR::StageInputKind::BaryCoordSmooth: return spv::BuiltInBaryCoordKHR;
 		case IR::StageInputKind::BaryCoordNoPerspective: return spv::BuiltInBaryCoordNoPerspKHR;
 		case IR::StageInputKind::WorkgroupId: return spv::BuiltInWorkgroupId;
+		case IR::StageInputKind::NumWorkgroups: return spv::BuiltInNumWorkgroups;
 		case IR::StageInputKind::LocalInvocationId: return spv::BuiltInLocalInvocationId;
 		case IR::StageInputKind::LocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
 		case IR::StageInputKind::GlobalInvocationId: return spv::BuiltInGlobalInvocationId;
@@ -426,15 +429,19 @@ void DefineInputs(EmitterState& state) {
 	for (const auto& input: state.program.info.inputs) {
 		state.inputs.push_back({input});
 	}
+	const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components,
+	                             const char* name) {
+		if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) {
+			    return input.kind == kind;
+		    })) {
+			state.inputs.push_back({{kind, 0, components, name}});
+		}
+	};
+	if (IR::FindBinding(state.program.bindings, IR::DescriptorBindingKind::SharedMemory) != nullptr) {
+		add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID");
+		add_builtin(IR::StageInputKind::NumWorkgroups, 3, "gl_NumWorkGroups");
+	}
 	if (state.lane_count == 2) {
-		const auto add_builtin = [&](IR::StageInputKind kind, uint32_t components,
-		                             const char* name) {
-			if (std::ranges::none_of(state.inputs, [kind](const InputBinding& input) {
-				    return input.kind == kind;
-			    })) {
-				state.inputs.push_back({{kind, 0, components, name}});
-			}
-		};
 		add_builtin(IR::StageInputKind::LocalInvocationIndex, 1, "gl_LocalInvocationIndex");
 		if (std::ranges::any_of(state.inputs, [](const InputBinding& input) {
 			    return input.kind == IR::StageInputKind::GlobalInvocationId;
@@ -465,6 +472,7 @@ void DefineInputs(EmitterState& state) {
 			case IR::StageInputKind::Layer:
 			case IR::StageInputKind::SampleId: type = TypeI32(state); break;
 			case IR::StageInputKind::WorkgroupId:
+			case IR::StageInputKind::NumWorkgroups:
 			case IR::StageInputKind::LocalInvocationId:
 			case IR::StageInputKind::GlobalInvocationId: type = TypeU32Vector(state, 3); break;
 			case IR::StageInputKind::FragCoord: type = TypeF32Vector(state, 4); break;
@@ -541,7 +549,7 @@ void DefineOutputs(EmitterState& state) {
 		}
 	}
 	if (state.program.stage == ShaderType::Mesh) {
-		DefineMeshOutputs(state);
+		DefineMeshOutputs(state, clip_distance_count, cull_distance_count);
 		return;
 	}
 	if (state.program.stage == ShaderType::Vertex && clip_distance_count + cull_distance_count < 8u &&
@@ -667,6 +675,13 @@ void DefineModule(EmitterState& state) {
 
 	state.builder.RequireCapability(spv::CapabilityShader);
 	state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
+	if (state.requirements.buffer_u8) {
+		state.builder.RequireExtension("SPV_KHR_8bit_storage");
+		state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess);
+	}
+	if (state.requirements.buffer_u16) {
+		state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
+	}
 	if (state.program.info.uses_dma) {
 		state.builder.RequireCapability(spv::CapabilityInt64);
 		state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
@@ -676,7 +691,8 @@ void DefineModule(EmitterState& state) {
 		state.builder.RequireCapability(spv::CapabilityInt64);
 		state.builder.RequireCapability(spv::CapabilityInt64Atomics);
 	}
-	if (state.requirements.shared_int64_atomics) {
+	if (state.requirements.shared_int64_atomics &&
+	    state.lds_storage_class == spv::StorageClassWorkgroup) {
 		state.builder.RequireVersion(0x00010400u);
 		state.builder.RequireExtension("SPV_KHR_workgroup_memory_explicit_layout");
 		state.builder.RequireCapability(spv::CapabilityWorkgroupMemoryExplicitLayoutKHR);
@@ -701,8 +717,9 @@ void DefineModule(EmitterState& state) {
 	if (state.requirements.image_gather_extended) {
 		state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
 	}
-	if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
-	    state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
+	if (state.lane_count == 2 || state.requirements.subgroup_barrier ||
+	    state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
+	    state.requirements.subgroup_local_invocation_id) {
 		state.builder.RequireCapability(spv::CapabilityGroupNonUniform);
 	}
 	if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
@@ -737,15 +754,13 @@ void DefineModule(EmitterState& state) {
 	if (state.requirements.float64) {
 		EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
 		                     state.input_info.compute->float_mode != 0xc0);
-		// MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals.
+		// Use native rounding for MODE=0xc0, consistent with ordinary FP32 arithmetic.
 		state.builder.RequireCapability(spv::CapabilityFloat64);
-		state.builder.RequireCapability(spv::CapabilityRoundingModeRTE);
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
 		                               64u);
 		// FP64 denormal preservation is temporarily disabled.
 		// state.builder.RequireCapability(spv::CapabilityDenormPreserve);
 		// state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u);
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeRoundingModeRTE, 32u);
 	}
 	if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) {
 		uint32_t    local_x = state.requirements.compute_derivatives ? 2u : 1u;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -315,6 +315,7 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
 			auto& lane          = half == 0 ? ctx : *ctx.other_half;
 			ctx.state.lane_half = half;
 			if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
+			                  inst.GetOpcode() != IR::ValueOpcode::StoreCompletion &&
 			                  inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
 				emit_instruction(lane, inst);
 			}
@@ -688,9 +689,7 @@ void EmitProgram(EmitterState& state) {
 	DefineBvhIntersect(state);
 	for (const auto* block: program.blocks) {
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
-			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
+			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32;
 		    })) {
 			ctx.scratch_u32_variable = state.builder.AllocateId();
 			if (state.lane_count == 2) {
@@ -751,6 +750,17 @@ void EmitProgram(EmitterState& state) {
 		state.builder.AddFunction(spv::OpStore, state.pixel_valid_mask_variable,
 		                          ConstantU32(state, 1));
 	}
+	if (state.lds_storage_class == spv::StorageClassStorageBuffer && state.lds_variable != 0) {
+		const auto group_x = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, 0);
+		const auto group_y = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, 1);
+		const auto group_z = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, 2);
+		const auto count_x = EmitInputComponentU32(state, IR::StageInputKind::NumWorkgroups, 0);
+		const auto count_y = EmitInputComponentU32(state, IR::StageInputKind::NumWorkgroups, 1);
+		const auto row = EmitAddU32(state, group_y, EmitBinaryU32(state, spv::OpIMul, group_z, count_y));
+		const auto index = EmitAddU32(state, group_x, EmitBinaryU32(state, spv::OpIMul, row, count_x));
+		state.lds_base_dwords = EmitBinaryU32(state, spv::OpIMul, index,
+		                                     ConstantU32(state, LdsDwordCount(state)));
+	}
 	EmitMemoryOffsets(state);
 	if (program.blocks.empty()) {
 		EmitReturn(ctx);

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -20,6 +20,12 @@ struct MimgGatherInfo {
 	uint32_t flags    = 0;
 };
 
+struct MimgAtomicInfo {
+	uint32_t encoding    = 0;
+	Opcode   decoded     = Opcode::UNSUPPORTED;
+	bool     supports_64 = false;
+};
+
 constexpr ImageDimension DecodeImageDimension(uint32_t dim) {
 	switch (dim) {
 		case 0u: return ImageDimension::Dim1D;
@@ -190,17 +196,17 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
     {0x61u, Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal},
 };
 
-constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
-    {0x0fu, Opcode::IMAGE_ATOMIC_SWAP},
+constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = {
+    {0x0fu, Opcode::IMAGE_ATOMIC_SWAP, true},
     {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP},
-    {0x11u, Opcode::IMAGE_ATOMIC_ADD},
+    {0x11u, Opcode::IMAGE_ATOMIC_ADD, true},
     {0x14u, Opcode::IMAGE_ATOMIC_SMIN},
-    {0x15u, Opcode::IMAGE_ATOMIC_UMIN},
+    {0x15u, Opcode::IMAGE_ATOMIC_UMIN, true},
     {0x16u, Opcode::IMAGE_ATOMIC_SMAX},
-    {0x17u, Opcode::IMAGE_ATOMIC_UMAX},
-    {0x18u, Opcode::IMAGE_ATOMIC_AND},
-    {0x19u, Opcode::IMAGE_ATOMIC_OR},
-    {0x1au, Opcode::IMAGE_ATOMIC_XOR},
+    {0x17u, Opcode::IMAGE_ATOMIC_UMAX, true},
+    {0x18u, Opcode::IMAGE_ATOMIC_AND, true},
+    {0x19u, Opcode::IMAGE_ATOMIC_OR, true},
+    {0x1au, Opcode::IMAGE_ATOMIC_XOR, true},
     {0x1eu, Opcode::IMAGE_ATOMIC_FMIN},
     {0x1fu, Opcode::IMAGE_ATOMIC_FMAX},
 };
@@ -210,7 +216,7 @@ constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCO
 constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST);
 
 Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                        const Detail::OpcodeMap* atomic) {
+                        const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return Opcode::IMAGE_SAMPLE;
 	}
@@ -228,7 +234,8 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
-		case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
+		case 0xe6u:
+		case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
@@ -245,7 +252,7 @@ uint32_t DecodeMimgSampleFlags(const MimgSampleInfo* sample, const MimgGatherInf
 
 uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
                                      const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                                     const Detail::OpcodeMap* atomic) {
+                                     const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return ImageSampleAddressComponents(sample->flags, dimension);
 	}
@@ -258,6 +265,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
 
 	switch (opcode) {
 		case 0xe6u: return 11u;
+		case 0xe7u: return 12u;
 		case 0x0eu: return 1u;
 		case 0x01u:
 		case 0x09u: return ImageCoordComponents(dimension) + 1u;
@@ -368,7 +376,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			SetUnsupported(inst, Family::MIMG, opcode, "MIMG image atomic has invalid DMASK");
 		} else if (inst.dmask == mask64) {
 			inst.data_bits = 64u;
-			if (inst.opcode != Opcode::IMAGE_ATOMIC_UMAX) {
+			if (!atomic->supports_64) {
 				SetUnsupported(inst, Family::MIMG, opcode,
 				               "MIMG 64-bit image atomic opcode is not implemented");
 			}
@@ -379,10 +387,10 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
-	if (opcode == 0xe6u &&
+	if (inst.opcode == Opcode::IMAGE_BVH_INTERSECT_RAY &&
 	    (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
 		SetUnsupported(inst, Family::MIMG, opcode,
-		               "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
+		               "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf");
 	}
 
 	DecodeVectorGpr(vdata, inst.dst);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -84,6 +84,7 @@ constexpr MemoryOpcodeInfo FLAT_OPCODE_LIST[] = {
     {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},     {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
     {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},   {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
     {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32}, {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
+    {0x24u, Opcode::FLAT_LOAD_SHORT_D16, 1, 16},
 };
 
 constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
@@ -92,7 +93,8 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x05u, Opcode::DS_MIN_I32, 1, 32},          {0x06u, Opcode::DS_MAX_I32, 1, 32},
     {0x07u, Opcode::DS_MIN_U32, 1, 32},          {0x08u, Opcode::DS_MAX_U32, 1, 32},
     {0x09u, Opcode::DS_AND_B32, 1, 32},          {0x0au, Opcode::DS_OR_B32, 1, 32},
-    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0du, Opcode::DS_WRITE_B32, 1, 32},
+    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0cu, Opcode::DS_MSKOR_B32, 1, 32},
+    {0x0du, Opcode::DS_WRITE_B32, 1, 32},
     {0x0eu, Opcode::DS_WRITE2_B32, 2, 32},       {0x0fu, Opcode::DS_WRITE2ST64_B32, 2, 32},
     {0x12u, Opcode::DS_MIN_F32, 1, 32},          {0x13u, Opcode::DS_MAX_F32, 1, 32},
     {0x1eu, Opcode::DS_WRITE_B8, 1, 8},          {0x1fu, Opcode::DS_WRITE_B16, 1, 16},
@@ -116,7 +118,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0xa6u, Opcode::DS_READ_U16_D16, 1, 16},
     {0xa7u, Opcode::DS_READ_U16_D16_HI, 1, 16},
     {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
-    {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
+    {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32},     {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
     {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
     {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
 };
@@ -192,6 +194,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 		case Opcode::DS_OR_RTN_B32:
 		case Opcode::DS_XOR_B32:
 		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_WRXCHG_RTN_B32: return true;
 		default: return false;
 	}
@@ -203,8 +206,10 @@ uint32_t DsSourceCount(Opcode opcode) {
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
 		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_MSKOR_B32: return 3u;
 		case Opcode::DS_MIN_F32:
-		case Opcode::DS_MAX_F32: return 3u;
+		case Opcode::DS_MAX_F32:
+		case Opcode::DS_PERMUTE_B32:
 		case Opcode::DS_BPERMUTE_B32: return 2u;
 		case Opcode::DS_READ_ADDTID_B32:
 		case Opcode::DS_CONSUME:
@@ -338,7 +343,6 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const uint32_t word0  = code[word_index];
 	const uint32_t word1  = code[word_index + 1u];
 	const uint32_t offset = word0 & 0xfffu;
-	const uint32_t dlc    = (word0 >> 12u) & 1u;
 	const uint32_t lds    = (word0 >> 13u) & 1u;
 	const uint32_t seg    = (word0 >> 14u) & 0x3u;
 	const uint32_t opcode = (word0 >> 18u) & 0x7fu;
@@ -350,6 +354,7 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.pc             = pc;
 	inst.word_count     = 2;
 	inst.offset         = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
+	inst.dlc            = ((word0 >> 12u) & 1u) != 0;
 	inst.glc            = ((word0 >> 16u) & 1u) != 0;
 	inst.slc            = ((word0 >> 17u) & 1u) != 0;
 	inst.family         = Family::FLAT;
@@ -359,7 +364,7 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	ApplyMemoryInfo(inst, info);
 	SetRawWords(inst, code, word_index, 2);
 
-	if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) {
+	if (lds != 0 || inst.glc || inst.slc || seg == 3u) {
 		SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
 		return;
 	}
@@ -369,6 +374,10 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 
 	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst);
+	if (inst.opcode == Opcode::FLAT_LOAD_SHORT_D16) {
+		// D16 loads reuse partial destinations to preserve the untouched high half.
+		inst.dst.sdwa_sel = 4u;
+	}
 	DecodeVectorGpr(addr, inst.src0);
 	inst.src_count = 1;
 	if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
@@ -407,7 +416,8 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
 	if (inst.gds &&
-	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_BPERMUTE_B32 ||
+	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_PERMUTE_B32 ||
+	     inst.opcode == Opcode::DS_BPERMUTE_B32 ||
 	     inst.opcode == Opcode::DS_WRITE_ADDTID_B32 ||
 	     inst.opcode == Opcode::DS_READ_ADDTID_B32)) {
 		SetUnsupported(inst, Family::DS, opcode, "DS lane operation is available only for LDS");

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -53,6 +53,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x14u, Opcode::S_FF1_I32_B64},
     {0x15u, Opcode::S_FLBIT_I32_B32},
     {0x16u, Opcode::S_FLBIT_I32_B64},
+    {0x1au, Opcode::S_SEXT_I32_I16},
     {0x1bu, Opcode::S_BITSET0_B32},
     {0x1cu, Opcode::S_BITSET0_B64},
     {0x1du, Opcode::S_BITSET1_B32},
@@ -86,7 +87,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = {
     {0x08u, Opcode::S_CMP_LE_I32}, {0x09u, Opcode::S_CMP_EQ_U32}, {0x0au, Opcode::S_CMP_LG_U32},
     {0x0bu, Opcode::S_CMP_GT_U32}, {0x0cu, Opcode::S_CMP_GE_U32}, {0x0du, Opcode::S_CMP_LT_U32},
     {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32},    {0x10u, Opcode::S_MULK_I32},
-    {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT},    {0x18u, Opcode::S_WAITCNT},
+    {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT_VSCNT}, {0x18u, Opcode::S_WAITCNT},
     {0x19u, Opcode::S_WAITCNT},    {0x1au, Opcode::S_WAITCNT},
     {0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END},
 };
@@ -234,6 +235,14 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			inst.src_count         = 1;
 			return;
 		}
+		case Opcode::S_WAITCNT_VSCNT:
+			inst.dst.kind       = OperandKind::Null;
+			inst.src1           = inst.src0;
+			inst.src1.value     = word & 0xffffu;
+			inst.src1.signed_val = static_cast<int32_t>(inst.src1.value);
+			DecodeScalarSource(sdst, pc, inst.src0);
+			inst.src_count = 2;
+			return;
 		case Opcode::S_SETREG_B32:
 			inst.dst.kind        = OperandKind::Null;
 			inst.src1.kind       = OperandKind::LiteralConstant;

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -503,6 +503,7 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.dst).c_str(),
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_ABS_I32:
+		case Opcode::S_SEXT_I32_I16:
 		case Opcode::S_BREV_B32:
 		case Opcode::S_BREV_B64:
 		case Opcode::S_BCNT1_I32_B32:
@@ -535,6 +536,11 @@ std::string InstructionToString(const Instruction& inst) {
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc,
 			                                               OperandToString(inst.src0).c_str(),
 			                                               OperandToString(inst.src1).c_str()));
+		case Opcode::S_WAITCNT_VSCNT:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               OperandToString(inst.src0),
+			                                               OperandToString(inst.src1)));
 		case Opcode::S_NOP:
 		case Opcode::S_WAITCNT:
 		case Opcode::S_WAITCNT_DEPCTR:
@@ -658,6 +664,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_LOAD_SBYTE:
 		case Opcode::FLAT_LOAD_USHORT:
 		case Opcode::FLAT_LOAD_SSHORT:
+		case Opcode::FLAT_LOAD_SHORT_D16:
 		case Opcode::FLAT_LOAD_DWORD:
 		case Opcode::FLAT_LOAD_DWORDX2:
 		case Opcode::FLAT_LOAD_DWORDX3:
@@ -692,10 +699,12 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_OR_RTN_B32:
 		case Opcode::DS_XOR_B32:
 		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_WRXCHG_RTN_B32:
 		case Opcode::DS_MIN_F32:
 		case Opcode::DS_MAX_F32:
 		case Opcode::DS_SWIZZLE_B32:
+		case Opcode::DS_PERMUTE_B32:
 		case Opcode::DS_BPERMUTE_B32:
 		case Opcode::DS_READ_I8:
 		case Opcode::DS_READ_U8:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -45,6 +45,7 @@ enum class Opcode {
 	S_CMOV_B64,
 	S_MOVK_I32,
 	S_ABS_I32,
+	S_SEXT_I32_I16,
 	S_ABSDIFF_I32,
 	S_BREV_B32,
 	S_BREV_B64,
@@ -186,10 +187,14 @@ enum class Opcode {
 	V_RCP_F32,
 	V_RCP_IFLAG_F32,
 	V_FRACT_F32,
+	V_FRACT_F64,
 	V_TRUNC_F32,
+	V_TRUNC_F64,
 	V_CEIL_F32,
+	V_CEIL_F64,
 	V_RNDNE_F32,
 	V_FLOOR_F32,
+	V_FLOOR_F64,
 	V_EXP_F32,
 	V_LOG_F32,
 	V_RSQ_F32,
@@ -220,6 +225,9 @@ enum class Opcode {
 	V_MAC_F32,
 	V_MADMK_F32,
 	V_MADAK_F32,
+	V_FMAC_F32,
+	V_FMAMK_F32,
+	V_FMAAK_F32,
 	V_MIN_F32,
 	V_MAX_F32,
 	V_ADD_F16,
@@ -247,6 +255,8 @@ enum class Opcode {
 	V_FMA_F64,
 	V_ADD_F64,
 	V_MUL_F64,
+	V_MIN_F64,
+	V_MAX_F64,
 	V_FMA_F32,
 	V_FMA_F16,
 	V_PACK_B32_F16,
@@ -388,6 +398,7 @@ enum class Opcode {
 	V_CMP_T_I32,
 	V_CMP_CLASS_F32,
 	V_CMPX_CLASS_F32,
+	V_CMPX_CLASS_F16,
 	V_CMP_LT_I16,
 	V_CMP_EQ_I16,
 	V_CMP_LE_I16,
@@ -432,6 +443,7 @@ enum class Opcode {
 	V_CMPX_LT_U16,
 	V_CMPX_EQ_U16,
 	V_CMPX_GT_U16,
+	V_CMPX_NE_U16,
 	V_CMP_NE_U16,
 	V_CMP_GE_U16,
 	V_CMP_F_U32,
@@ -445,6 +457,7 @@ enum class Opcode {
 	V_CMP_EQ_I64,
 	V_CMP_LT_I64,
 	V_CMP_LE_I64,
+	V_CMP_NE_I64,
 	V_CMP_LT_U64,
 	V_CMP_EQ_U64,
 	V_CMP_LE_U64,
@@ -522,6 +535,7 @@ enum class Opcode {
 	FLAT_LOAD_SBYTE,
 	FLAT_LOAD_USHORT,
 	FLAT_LOAD_SSHORT,
+	FLAT_LOAD_SHORT_D16,
 	FLAT_LOAD_DWORD,
 	FLAT_LOAD_DWORDX2,
 	FLAT_LOAD_DWORDX3,
@@ -556,10 +570,12 @@ enum class Opcode {
 	DS_OR_RTN_B32,
 	DS_XOR_B32,
 	DS_XOR_RTN_B32,
+	DS_MSKOR_B32,
 	DS_WRXCHG_RTN_B32,
 	DS_MIN_F32,
 	DS_MAX_F32,
 	DS_SWIZZLE_B32,
+	DS_PERMUTE_B32,
 	DS_BPERMUTE_B32,
 	DS_CONSUME,
 	DS_APPEND,
@@ -625,6 +641,7 @@ enum class Opcode {
 
 	S_NOP,
 	S_WAITCNT,
+	S_WAITCNT_VSCNT,
 	S_WAITCNT_DEPCTR,
 	S_BARRIER,
 	S_BRANCH,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -35,7 +35,7 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
     {0x04u, Opcode::V_SUB_F32, Vop2SdwaProfile::Float32},
     {0x05u, Opcode::V_SUBREV_F32},
     {0x08u, Opcode::V_MUL_F32, Vop2SdwaProfile::Float32},
-    {0x09u, Opcode::V_MUL_I32_I24, Vop2SdwaProfile::IntegerFullDestination},
+    {0x09u, Opcode::V_MUL_I32_I24, Vop2SdwaProfile::IntegerPartialDestination},
     {0x0bu, Opcode::V_MUL_U32_U24, Vop2SdwaProfile::IntegerFullDestination},
     {0x0fu, Opcode::V_MIN_F32},
     {0x10u, Opcode::V_MAX_F32},
@@ -65,9 +65,9 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
     {0x2au, Opcode::V_SUBREV_CO_CI_U32},
     {0x26u, Opcode::V_SUB_NC_U32, Vop2SdwaProfile::IntegerPartialDestination},
     {0x27u, Opcode::V_SUBREV_NC_U32, Vop2SdwaProfile::IntegerFullDestination},
-    {0x2bu, Opcode::V_MAC_F32},
-    {0x2cu, Opcode::V_MADMK_F32},
-    {0x2du, Opcode::V_MADAK_F32},
+    {0x2bu, Opcode::V_FMAC_F32},
+    {0x2cu, Opcode::V_FMAMK_F32},
+    {0x2du, Opcode::V_FMAAK_F32},
     {0x2fu, Opcode::V_CVT_PKRTZ_F16_F32, Vop2SdwaProfile::PackedFloat16},
     {0x32u, Opcode::V_ADD_F16, Vop2SdwaProfile::Float16},
     {0x33u, Opcode::V_SUB_F16, Vop2SdwaProfile::Float16},
@@ -91,6 +91,9 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x0fu, Opcode::V_CVT_F32_F64},
     {0x10u, Opcode::V_CVT_F64_F32},
     {0x16u, Opcode::V_CVT_F64_U32},
+    {0x17u, Opcode::V_TRUNC_F64},
+    {0x18u, Opcode::V_CEIL_F64},
+    {0x1au, Opcode::V_FLOOR_F64},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -123,6 +126,7 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x39u, Opcode::V_FFBH_U32},
     {0x3au, Opcode::V_FFBL_B32},
     {0x3bu, Opcode::V_FFBH_I32},
+    {0x3eu, Opcode::V_FRACT_F64},
     {0x3fu, Opcode::V_FREXP_EXP_I32_F32},
     {0x40u, Opcode::V_FREXP_MANT_F32},
     {0x42u, Opcode::V_MOVRELD_B32},
@@ -155,6 +159,9 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x0fu, Opcode::V_CVT_F32_F64},
     {0x10u, Opcode::V_CVT_F64_F32},
     {0x16u, Opcode::V_CVT_F64_U32},
+    {0x17u, Opcode::V_TRUNC_F64},
+    {0x18u, Opcode::V_CEIL_F64},
+    {0x1au, Opcode::V_FLOOR_F64},
     {0x2fu, Opcode::V_RCP_F64},
     {0x05u, Opcode::V_CVT_F32_I32},
     {0x06u, Opcode::V_CVT_F32_U32},
@@ -182,6 +189,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x39u, Opcode::V_FFBH_U32},
     {0x3au, Opcode::V_FFBL_B32},
     {0x3bu, Opcode::V_FFBH_I32},
+    {0x3eu, Opcode::V_FRACT_F64},
     {0x3fu, Opcode::V_FREXP_EXP_I32_F32},
     {0x40u, Opcode::V_FREXP_MANT_F32},
     {0x42u, Opcode::V_MOVRELD_B32},
@@ -244,16 +252,19 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x99u, Opcode::V_CMPX_LT_I16, false}, {0x9au, Opcode::V_CMPX_EQ_I16, false},
     {0x9bu, Opcode::V_CMPX_LE_I16, false}, {0x9cu, Opcode::V_CMPX_GT_I16, false},
     {0x9du, Opcode::V_CMPX_NE_I16, false}, {0x9eu, Opcode::V_CMPX_GE_I16, false},
+    {0x9fu, Opcode::V_CMPX_CLASS_F16, false},
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
     {0xb9u, Opcode::V_CMPX_LT_U16, false}, {0xbau, Opcode::V_CMPX_EQ_U16, false},
-    {0xbcu, Opcode::V_CMPX_GT_U16},        {0xc0u, Opcode::V_CMP_F_U32},
+    {0xbcu, Opcode::V_CMPX_GT_U16},        {0xbdu, Opcode::V_CMPX_NE_U16, false},
+    {0xc0u, Opcode::V_CMP_F_U32},
     {0xc1u, Opcode::V_CMP_LT_U32},         {0xc2u, Opcode::V_CMP_EQ_U32},
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
     {0xc5u, Opcode::V_CMP_NE_U32},         {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},          {0xa2u, Opcode::V_CMP_EQ_I64, false},
     {0xa1u, Opcode::V_CMP_LT_I64, false},  {0xa3u, Opcode::V_CMP_LE_I64, false},
+    {0xa5u, Opcode::V_CMP_NE_I64, false},
     {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
     {0xd2u, Opcode::V_CMPX_EQ_U32},        {0xd3u, Opcode::V_CMPX_LE_U32},
     {0xd4u, Opcode::V_CMPX_GT_U32},        {0xd5u, Opcode::V_CMPX_NE_U32},
@@ -289,6 +300,8 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x14cu, Opcode::V_FMA_F64},
     {0x164u, Opcode::V_ADD_F64},
     {0x165u, Opcode::V_MUL_F64},
+    {0x166u, Opcode::V_MIN_F64},
+    {0x167u, Opcode::V_MAX_F64},
     {0x148u, Opcode::V_BFE_U32},
     {0x149u, Opcode::V_BFE_I32},
     {0x14au, Opcode::V_BFI_B32},
@@ -472,11 +485,23 @@ bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
 bool IsVop1FloatResultOpcode(Opcode opcode);
 bool IsVopcCompareExec(Opcode opcode);
 
+bool IsVop1Float64Opcode(Opcode opcode) {
+	return opcode == Opcode::V_CVT_F64_I32 || opcode == Opcode::V_CVT_F32_F64 ||
+	       opcode == Opcode::V_CVT_F64_F32 || opcode == Opcode::V_CVT_F64_U32 ||
+	       opcode == Opcode::V_RCP_F64 || opcode == Opcode::V_FRACT_F64 ||
+	       opcode == Opcode::V_TRUNC_F64 || opcode == Opcode::V_CEIL_F64 ||
+	       opcode == Opcode::V_FLOOR_F64;
+}
+
 bool IsVop1FloatSourceOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_CVT_F32_F64:
 		case Opcode::V_CVT_F64_F32:
 		case Opcode::V_RCP_F64:
+		case Opcode::V_FRACT_F64:
+		case Opcode::V_TRUNC_F64:
+		case Opcode::V_CEIL_F64:
+		case Opcode::V_FLOOR_F64:
 		case Opcode::V_MOV_B32:
 		case Opcode::V_CVT_F32_F16:
 		case Opcode::V_CVT_U32_F32:
@@ -551,6 +576,7 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
     // partial destination may combine with any source selector.
     {Opcode::V_NOT_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(),
      SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
+    {Opcode::V_BFREV_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_FFBL_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_FFBH_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
@@ -647,6 +673,10 @@ bool SupportsVop1Clamp(Opcode opcode) {
 }
 
 bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
+	if (IsVop1Float64Opcode(inst.opcode)) {
+		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support SDWA");
+		return false;
+	}
 	const auto dst_sel   = (modifier >> 8u) & 0x7u;
 	const auto dst_u     = (modifier >> 11u) & 0x3u;
 	const auto clamp     = (modifier >> 13u) & 0x1u;
@@ -735,9 +765,7 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
 
 void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                    uint32_t opcode, uint32_t vdst, Instruction& inst) {
-	if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 ||
-	    inst.opcode == Opcode::V_CVT_F64_F32 || inst.opcode == Opcode::V_CVT_F64_U32 ||
-	    inst.opcode == Opcode::V_RCP_F64) {
+	if (IsVop1Float64Opcode(inst.opcode)) {
 		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP");
 		return;
 	}
@@ -774,6 +802,9 @@ bool IsVop2FloatOpcode(Opcode opcode) {
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MADMK_F32:
 		case Opcode::V_MADAK_F32:
+		case Opcode::V_FMAC_F32:
+		case Opcode::V_FMAMK_F32:
+		case Opcode::V_FMAAK_F32:
 		case Opcode::V_CVT_PKRTZ_F16_F32:
 		case Opcode::V_ADD_F16:
 		case Opcode::V_SUB_F16:
@@ -887,6 +918,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_NEQ_F16:
 		case Opcode::V_CMPX_NLT_F16:
 		case Opcode::V_CMP_CLASS_F32:
+		case Opcode::V_CMPX_CLASS_F16:
 		case Opcode::V_CMPX_CLASS_F32: return true;
 		default: return false;
 	}
@@ -1031,6 +1063,7 @@ void FinalizeVop2Instruction(std::span<const uint32_t> code, uint32_t word_index
                              Instruction& inst) {
 	switch (inst.opcode) {
 		case Opcode::V_MADMK_F32:
+		case Opcode::V_FMAMK_F32:
 		case Opcode::V_FMAMK_F16:
 			inst.src2      = inst.src1;
 			inst.src1      = {};
@@ -1038,6 +1071,7 @@ void FinalizeVop2Instruction(std::span<const uint32_t> code, uint32_t word_index
 			inst.src_count = 3;
 			break;
 		case Opcode::V_MADAK_F32:
+		case Opcode::V_FMAAK_F32:
 		case Opcode::V_FMAAK_F16:
 			inst.src2      = {};
 			inst.src2.kind = OperandKind::LiteralConstant;
@@ -1100,8 +1134,10 @@ void DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 	DecodeVectorGpr(vsrc1, inst.src1);
 	DecodeScalarSource(src0 + 256u, pc, inst.src0);
 	ApplyDppModifier(inst.src0, modifier, code[word_index] & 0x1ffu);
-	inst.src1.negate       = ((modifier >> 22u) & 0x1u) != 0u;
-	inst.src1.absolute     = ((modifier >> 23u) & 0x1u) != 0u;
+	if (!inst.src0.dpp8) {
+		inst.src1.negate   = ((modifier >> 22u) & 0x1u) != 0u;
+		inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
+	}
 	const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16;
 	if (packed_fmac) {
 		inst.src0.negate_hi = inst.src0.negate;
@@ -1152,7 +1188,8 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) {
 }
 
 bool SupportsVopcSdwa(Opcode opcode) {
-	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 &&
+	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_NE_I64 &&
+	       opcode != Opcode::V_CMP_EQ_F64 &&
 	       opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 &&
 	       opcode != Opcode::V_CMPX_GE_F64;
 }
@@ -1221,6 +1258,8 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
+		case Opcode::V_MIN_F64:
+		case Opcode::V_MAX_F64:
 		case Opcode::V_MUL_LO_U32:
 		case Opcode::V_MUL_HI_U32:
 		case Opcode::V_MUL_LO_I32:
@@ -1370,9 +1409,12 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
 		case Opcode::V_MIN_F32:
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
+		case Opcode::V_FMAC_F32:
 		case Opcode::V_MAD_F32:
 		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
+		case Opcode::V_MIN_F64:
+		case Opcode::V_MAX_F64:
 		case Opcode::V_FMA_F64:
 		case Opcode::V_FMA_F32:
 		case Opcode::V_PACK_B32_F16:
@@ -1403,6 +1445,7 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
 		case Opcode::V_MIN_F32:
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
+		case Opcode::V_FMAC_F32:
 		case Opcode::V_MAD_F32:
 		case Opcode::V_FMA_F32:
 		case Opcode::V_FMA_F16:
@@ -1499,6 +1542,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_I32:
 		case Opcode::V_CMPX_GE_I32:
 		case Opcode::V_CMPX_CLASS_F32:
+		case Opcode::V_CMPX_CLASS_F16:
 		case Opcode::V_CMPX_LT_U32:
 		case Opcode::V_CMPX_EQ_U32:
 		case Opcode::V_CMPX_LE_U32:
@@ -1517,6 +1561,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_LT_U16:
 		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:
+		case Opcode::V_CMPX_NE_U16:
 		case Opcode::V_CMPX_LT_F16:
 		case Opcode::V_CMPX_EQ_F16:
 		case Opcode::V_CMPX_LE_F16:
@@ -1551,6 +1596,7 @@ void DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		return;
 	}
 	switch (src0) {
+		case 233u: DecodeVop2Dpp(pc, code, word_index, opcode, vdst, vsrc1, inst); return;
 		case 249u: DecodeVop2Sdwa(pc, code, word_index, opcode, vdst, vsrc1, inst); return;
 		case 250u: DecodeVop2Dpp(pc, code, word_index, opcode, vdst, vsrc1, inst); return;
 		default: break;

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -31,6 +31,7 @@ void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueO
 	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
 	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
 	                       inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMP_NE_I64 ||
 	                       inst.opcode == Decoder::Opcode::V_CMPX_NE_I64;
 	const auto read = [&](const Decoder::Operand& operand) {
 		// RDNA2 expands signed 64-bit integer literals by sign extension.
@@ -71,11 +72,13 @@ void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool
 	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, cmpx);
 }
 
-void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx) {
-	const auto value = ReadOperand(inst.src0, IR::Type::F32);
-	const auto mask  = ReadOperand(inst.src1, IR::Type::U32);
-	EmitCompareResult(inst, IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})), false,
-	                  cmpx);
+void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half) {
+	const auto value = half ? IR::Value(Read16LaneBits(inst.src0, false))
+	                        : ReadOperand(inst.src0, IR::Type::F32);
+	const auto mask = half ? IR::Value(Read16LaneBits(inst.src1, false))
+	                      : ReadOperand(inst.src1, IR::Type::U32);
+	const auto opcode = half ? IR::ValueOpcode::FPCmpClass16 : IR::ValueOpcode::FPCmpClass32;
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {value, mask})), false, cmpx);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -181,8 +181,12 @@ void Translator::EmitControlNop() {
 	ir.Emit(IR::ValueOpcode::ControlNop);
 }
 
-void Translator::EmitWaitcnt() {
-	ir.Emit(IR::ValueOpcode::Waitcnt);
+void Translator::S_WAITCNT_VSCNT(const Decoder::Instruction& inst) {
+	const auto count = inst.src1.value & 63u;
+	if (inst.src0.kind != Decoder::OperandKind::Null || (count != 0u && count != 63u)) {
+		EXIT("unsupported partial or register-based S_WAITCNT_VSCNT at 0x%08x", inst.pc);
+	}
+	if (count == 0u) ir.Emit(IR::ValueOpcode::StoreCompletion);
 }
 
 void Translator::S_BARRIER() {
@@ -244,6 +248,7 @@ void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
 
 void Translator::ScalarSelect64(const Decoder::Instruction& inst,
                                  const Decoder::Operand& false_source) {
+	// Preserve per-word expressions for descriptor tracking and mask provenance.
 	const auto condition     = ir.GetScc();
 	const auto lhs           = ReadU32Pair(inst.src0);
 	const auto rhs           = ReadU32Pair(false_source);
@@ -252,13 +257,14 @@ void Translator::ScalarSelect64(const Decoder::Instruction& inst,
 	const auto selected_mask_valid =
 	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
 	                   {condition, ReadMaskValid(inst.src0), ReadMaskValid(false_source)}));
-	if (IsExecOrVcc(inst.dst)) {
-		WriteMask(inst.dst, selected_mask, true);
-		return;
-	}
 	WriteU32Pair(inst.dst,
 	             {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])});
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+	// Scalar selects preserve the raw pair even when EXEC/VCC holds an address.
+	if (inst.dst.kind == Decoder::OperandKind::ExecLo) {
+		ir.SetExec(selected_mask);
+	} else if (inst.dst.kind == Decoder::OperandKind::VccLo) {
+		ir.SetVcc(selected_mask);
+	} else if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
 		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
 		ir.SetThreadBitScalarReg(dst, selected_mask);
 		ir.SetScalarMaskTag(dst, selected_mask_valid);

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -8,13 +8,13 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
 void Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                bool accumulator, bool quiet_snan) {
 	const auto translate_lane = [&](bool high) {
-		const auto lhs = ReadF16LaneAsF32(inst.src0, high, true);
-		const auto rhs = ReadF16LaneAsF32(inst.src1, high, true);
+		const auto lhs = ReadF16LaneAsF32(inst.src0, high);
+		const auto rhs = ReadF16LaneAsF32(inst.src1, high);
 		IR::F32    result;
 		if (accumulator) {
-			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high, true)}));
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.dst, high)}));
 		} else if (inst.src_count == 3u) {
-			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high, true)}));
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src2, high)}));
 		} else {
 			result = IR::F32(ir.Emit(opcode, {lhs, rhs}));
 		}

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -114,6 +114,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
 		return;
 	}
 
+	// Descriptor tracking reasons about each scalar word independently.
 	const auto lhs        = ReadU32Pair(inst.src0);
 	auto       mask_valid = ReadMaskValid(inst.src0);
 	if (inst.src_count > 1u) {
@@ -269,22 +270,18 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
 }
 
 void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
-	const auto lhs       = ReadU32(inst.src0);
-	const auto rhs       = ReadU32(inst.src1);
-	const auto add       = ExtractU64(ReadU64(inst.src2));
-	const auto mul_low   = ir.IMul(lhs, rhs);
-	const auto mul_high  = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
-	const auto low       = ir.IAdd(mul_low, add[0]);
-	const auto carry_low = ir.ULessThan(low, mul_low);
-	const auto high0     = ir.IAdd(mul_high, add[1]);
-	const auto carry0    = ir.ULessThan(high0, mul_high);
-	const auto high =
-	    ir.IAdd(high0, ir.Select(carry_low, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))));
-	const auto carry1 = ir.ULessThan(high, high0);
-	WriteOperand(DestinationOperand(inst), ir.ConstructU64(low, high));
+	const auto lhs = ReadU32(inst.src0);
+	const auto rhs = ReadU32(inst.src1);
+	// Keep the 32x32 product in words: native widening IMul64 fails NVIDIA compilation
+	// in large vertex shaders (Astro's Playroom). The 64-bit addition stays native.
+	const auto product = ir.ConstructU64(
+	    ir.IMul(lhs, rhs), IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})));
+	const auto result  = ir.Emit(IR::ValueOpcode::IAdd64, {product, ReadU64(inst.src2)});
+	WriteOperand(DestinationOperand(inst), result);
 	if (inst.dst2.kind != Decoder::OperandKind::Null &&
 	    inst.dst2.kind != Decoder::OperandKind::Unknown) {
-		WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
+		// The 32x32 product fits in 64 bits, so only the addition can carry.
+		WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product})));
 	}
 }
 
@@ -407,9 +404,12 @@ IR::U64 Translator::RightMask64(IR::U32 count) {
 }
 
 void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
-	const auto count  = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
-	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
-	const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset});
+	const auto count  = ReadU32(inst.src0);
+	const auto offset = ReadU32(inst.src1);
+	const auto one    = IR::Value(uint64_t {1});
+	const auto limit  = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {one, count});
+	const auto mask   = ir.Emit(IR::ValueOpcode::ISub64, {limit, one});
+	const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {mask, offset});
 	WriteOperand(DestinationOperand(inst), result);
 }
 
@@ -483,10 +483,11 @@ void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
 	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
 }
 
-void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
-	const auto hi      = ReadU32(inst.src0);
-	const auto lo      = ReadU32(inst.src1);
-	const auto shift   = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
+void Translator::V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset) {
+	const auto hi = ReadU32(inst.src0);
+	const auto lo = ReadU32(inst.src1);
+	auto shift    = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(byte_offset ? 3u : 31u)));
+	if (byte_offset) shift = ir.ShiftLeftLogical(shift, IR::U32(IR::Value(3u)));
 	const auto lo_part = ir.ShiftRightLogical(lo, shift);
 	const auto inverse =
 	    ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(32u)), shift), IR::U32(IR::Value(31u)));
@@ -496,21 +497,6 @@ void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(lo_part, hi_part));
 }
 
-void Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
-	const auto hi           = ReadU32(inst.src0);
-	const auto lo           = ReadU32(inst.src1);
-	const auto byte_offset  = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
-	const auto bit_offset   = ir.ShiftLeftLogical(byte_offset, IR::U32(IR::Value(3u)));
-	const auto concatenated = ir.ConstructU64(lo, hi);
-	const auto shifted =
-	    IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64,
-	                    {concatenated, ir.BitwiseAnd(bit_offset, IR::U32(IR::Value(63u)))}));
-	const auto in_range =
-	    IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, IR::Value(8u)}));
-	WriteOperand(DestinationOperand(inst),
-	             ir.Select(in_range, ExtractU64(shifted)[0], IR::U32(IR::Value(0u))));
-}
-
 void Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
 	const auto shift  = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -129,8 +129,9 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
 	// Vector loads use GLC/DLC to bypass L0/GL1; atomics use GLC only to return data.
 	const bool buffer_atomic = decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP &&
 	                           decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX;
-	memory.coherent = memory.kind == ResourceKind::Buffer && !buffer_atomic &&
-	                  (decoded.glc || decoded.dlc);
+	memory.coherent = (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::Flat ||
+	                   memory.kind == ResourceKind::Global) &&
+	                  !buffer_atomic && (decoded.glc || decoded.dlc);
 	memory.resource      = ResourceIndexFromOperand(decoded.src1);
 	memory.sampler       = ResourceIndexFromOperand(decoded.src2);
 	if (memory.kind == ResourceKind::ScalarBuffer) {
@@ -239,9 +240,6 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 			case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
 			case Decoder::Opcode::DS_WRITE_ADDTID_B32:
 				return index == 0u ? decoded.src1 : MakeM0Operand();
-			case Decoder::Opcode::DS_MIN_F32:
-			case Decoder::Opcode::DS_MAX_F32:
-				return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
 			case Decoder::Opcode::DS_WRITE_B8:
 			case Decoder::Opcode::DS_WRITE_B16:
 			case Decoder::Opcode::DS_WRITE_B8_D16_HI:
@@ -255,12 +253,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 			case Decoder::Opcode::DS_WRITE2_B64:
 			case Decoder::Opcode::DS_WRITE2ST64_B64:
 				return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
-			default:
-				if (decoded.opcode >= Decoder::Opcode::DS_ADD_U32 &&
-				    decoded.opcode <= Decoder::Opcode::DS_WRXCHG_RTN_B32) {
-					return index == 0u ? decoded.src1 : decoded.src0;
-				}
-				return decoded.src0;
+			default: return decoded.src0;
 		}
 	}
 	return DecodedSourceAt(decoded, index);
@@ -554,7 +547,10 @@ void Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	}
 }
 
-void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32,
+                               IR::ValueOpcode opcode64) {
+	const auto opcode = inst.data_bits == 64u ? opcode64 : opcode32;
+	EXIT_IF(opcode == IR::ValueOpcode::Count);
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
@@ -580,12 +576,13 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                            bool returns_value) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
-	const auto address = ReadU32(MemorySourceAt(inst, 1));
-	const auto data_src = MemorySourceAt(inst, 0);
+	const auto address = ReadU32(inst.src0);
 	const IR::Value value =
-	    memory.data_dwords == 2u ? IR::Value(ReadU64(data_src)) : IR::Value(ReadU32(data_src));
-	const auto result  = ir.Emit(opcode, {address, value, ir.GetExec()},
-	                             AddMemoryInfo(memory, inst.pc));
+	    memory.data_dwords == 2u ? IR::Value(ReadU64(inst.src1)) : IR::Value(ReadU32(inst.src1));
+	const auto flags = AddMemoryInfo(memory, inst.pc);
+	const auto result = inst.src_count == 3u
+	                        ? ir.Emit(opcode, {address, value, ReadU32(inst.src2), ir.GetExec()}, flags)
+	                        : ir.Emit(opcode, {address, value, ir.GetExec()}, flags);
 	if (returns_value) {
 		WriteOperand(inst.dst, result);
 	}
@@ -857,14 +854,6 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
 	}
 }
 
-void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
-	const auto memory = MemoryInfoFromDecoded(inst);
-	ir.Emit(opcode,
-	        {ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)),
-	         ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()},
-	        AddMemoryInfo(memory, inst.pc));
-}
-
 void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	WriteOperand(inst.dst, ir.Emit(opcode,
@@ -894,9 +883,10 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
 	                                ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()}));
 }
 
-void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
+void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) {
 	const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset)));
-	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32,
+	WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32
+	                                       : IR::ValueOpcode::PermuteU32,
 	                               {ReadU32(inst.src1), address, ir.GetExec()}));
 }
 
@@ -904,7 +894,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
 		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
 			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
-			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()},
+			    inst.image_address_components - 10u);
 			for (uint32_t component = 0; component < 4; ++component) {
 				WriteOperand(OffsetOperand(inst.dst, component),
 				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
@@ -1046,30 +1037,38 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, false);
 		case Decoder::Opcode::DS_XOR_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, true);
+		case Decoder::Opcode::DS_MSKOR_B32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicMaskedOr32, false);
 		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
 
 		case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_SWAP:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32,
+			                    IR::ValueOpcode::ImageAtomicSwap64);
 		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32,
+			                    IR::ValueOpcode::ImageAtomicIAdd64);
 		case Decoder::Opcode::IMAGE_ATOMIC_SMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32,
+			                    IR::ValueOpcode::ImageAtomicUMin64);
 		case Decoder::Opcode::IMAGE_ATOMIC_SMAX:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
-			return IMAGE_ATOMIC(inst, inst.data_bits == 64u ? IR::ValueOpcode::ImageAtomicUMax64
-			                                                : IR::ValueOpcode::ImageAtomicUMax32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32,
+			                    IR::ValueOpcode::ImageAtomicUMax64);
 		case Decoder::Opcode::IMAGE_ATOMIC_AND:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32,
+			                    IR::ValueOpcode::ImageAtomicAnd64);
 		case Decoder::Opcode::IMAGE_ATOMIC_OR:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32,
+			                    IR::ValueOpcode::ImageAtomicOr64);
 		case Decoder::Opcode::IMAGE_ATOMIC_XOR:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32);
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32,
+			                    IR::ValueOpcode::ImageAtomicXor64);
 		case Decoder::Opcode::IMAGE_ATOMIC_FMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicFMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_FMAX:
@@ -1079,6 +1078,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::FLAT_LOAD_SBYTE:
 		case Decoder::Opcode::FLAT_LOAD_USHORT:
 		case Decoder::Opcode::FLAT_LOAD_SSHORT:
+		case Decoder::Opcode::FLAT_LOAD_SHORT_D16:
 		case Decoder::Opcode::FLAT_LOAD_DWORD:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX2:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX3:
@@ -1108,11 +1108,12 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst);
 
 		case Decoder::Opcode::DS_MIN_F32:
-			return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMin32);
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false);
 		case Decoder::Opcode::DS_MAX_F32:
-			return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32);
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMax32, false);
 		case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst);
-		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_BPERMUTE_B32(inst);
+		case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false);
+		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true);
 		case Decoder::Opcode::DS_CONSUME:
 			return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume);
 		case Decoder::Opcode::DS_APPEND:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -18,7 +18,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return;
 		case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return;
 		case O::S_SETREG_B32: EmitControlNop(); return;
-		case O::S_WAITCNT: EmitWaitcnt(); return;
+		case O::S_WAITCNT_VSCNT: S_WAITCNT_VSCNT(inst); return;
+		case O::S_WAITCNT: return;
 
 		case O::S_AND_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
@@ -133,6 +134,10 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 
 		case O::S_ABS_I32:
 			return SimpleInteger(inst, IR::ValueOpcode::IAbs32, IR::Type::U32, false, false, true);
+		case O::S_SEXT_I32_I16:
+			WriteOperand(inst.dst, IR::U32(ir.Emit(IR::ValueOpcode::BitFieldSExtract,
+			    {ReadU32(inst.src0), IR::Value(0u), IR::Value(16u)})));
+			return;
 		case O::S_MUL_I32:
 		case O::S_MULK_I32:
 			return SimpleInteger(inst, IR::ValueOpcode::IMul32, IR::Type::U32, false, false, false);
@@ -164,9 +169,16 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BCNT1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
 			                     true);
-		case O::S_BCNT1_I32_B64:
-			return SimpleInteger(inst, IR::ValueOpcode::BitCount64, IR::Type::U64, false, false,
-			                     true);
+		case O::S_BCNT1_I32_B64: {
+			// Vulkan bit counts operate on 32-bit words; avoid packing only to split again.
+			const auto value  = ReadU32Pair(inst.src0);
+			const auto low    = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[0]}));
+			const auto high   = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {value[1]}));
+			const auto result = ir.IAdd(low, high);
+			WriteOperand(inst.dst, result);
+			ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
+			return;
+		}
 		case O::S_FF1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::FindILsb32, IR::Type::U32, false, false,
 			                     false);
@@ -224,7 +236,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_SLEEP:
 		case O::S_SETPRIO:
 		case O::S_TRAP: EmitControlNop(); return;
-		case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return;
+		case O::S_WAITCNT_DEPCTR: return;
 		case O::S_BARRIER: S_BARRIER(); return;
 		case O::S_SENDMSG: S_SENDMSG(inst); return;
 		case O::S_TTRACEDATA: S_TTRACEDATA(); return;

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -503,41 +503,11 @@ IR::U64 Translator::ReadU64(const Decoder::Operand& operand) {
 	return IR::U64(ReadOperand(operand, IR::Type::U64));
 }
 
-IR::F32 Translator::ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed) {
-	if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
-		const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel);
-		auto       value    = use_zero ? IR::F32(IR::Value::F32(0.0f))
-		                               : ir.BitCastF32(IR::U32(IR::Value(operand.value)));
-		const auto half     = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
-		value               = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
-		if (operand.absolute) {
-			value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
-		}
-		if (high_lane ? operand.negate_hi : operand.negate) {
-			value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
-		}
-		return value;
-	}
-	auto raw_operand      = operand;
-	raw_operand.sdwa_sel  = 6;
-	raw_operand.sdwa_sext = false;
-	const auto bits       = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand));
-	uint32_t   offset     = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
-	if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
-		offset = operand.sdwa_sel == 5u ? 16u : 0u;
-	}
-	const auto half_u32 = IR::U32(
-	    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(16u)}));
-	const auto half_u16 = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, {half_u32}));
+IR::F32 Translator::ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane) {
+	const auto bits     = Read16LaneBits(operand, high_lane);
+	const auto half_u16 = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, {bits}));
 	const auto half     = IR::F16(ir.Emit(IR::ValueOpcode::BitCastF16U16, {half_u16}));
-	auto       value    = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
-	if (operand.absolute) {
-		value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
-	}
-	if (high_lane ? operand.negate_hi : operand.negate) {
-		value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
-	}
-	return value;
+	return IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
 }
 
 IR::F32 Translator::ReadF16AsF32(const Decoder::Operand& operand) {
@@ -555,29 +525,6 @@ IR::F32 Translator::ReadMixF32(const Decoder::Operand& operand) {
 	return IR::F32(ReadOperand(value_operand, IR::Type::F32));
 }
 
-IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane) {
-	auto raw_operand      = operand;
-	raw_operand.sdwa_sel  = 6;
-	raw_operand.sdwa_sext = false;
-	const auto bits       = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand));
-	uint32_t   offset     = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
-	uint32_t   width      = 16u;
-	if (operand.sdwa_sel <= 3u) {
-		offset = operand.sdwa_sel * 8u;
-		width  = 8u;
-	} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
-		offset = operand.sdwa_sel == 5u ? 16u : 0u;
-	}
-	if (width == 8u && operand.sdwa_sext) {
-		// Extend the selected byte into the 16-bit operand before halfword modifiers.
-		const auto value = IR::U32(
-		    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {bits, IR::Value(offset), IR::Value(width)}));
-		return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
-	}
-	return IR::U32(
-	    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)}));
-}
-
 IR::U32 Translator::ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane,
                                      bool sign_extend) {
 	auto value = Read16LaneBits(operand, high_lane);
@@ -593,12 +540,36 @@ IR::U32 Translator::ReadU16AsU32(const Decoder::Operand& operand, bool sign_exte
 }
 
 IR::U32 Translator::Read16LaneBits(const Decoder::Operand& operand, bool high_lane) {
-	auto value = ReadU16LaneRaw(operand, high_lane);
+	auto raw_operand      = operand;
+	raw_operand.sdwa_sel  = 6;
+	raw_operand.sdwa_sext = false;
+	auto bits             = ReadRawU32(operand);
+	if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
+		// Inline floats encode the operand's width before SDWA or OP_SEL selects bits.
+		const auto half      = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {ir.BitCastF32(bits)}));
+		const auto half_bits = IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {half}));
+		bits                = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {half_bits}));
+	}
+	bits = ApplyBitSourceModifiers(raw_operand, bits);
+	uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
+	uint32_t width  = 16u;
+	if (operand.sdwa_sel <= 3u) {
+		offset = operand.sdwa_sel * 8u;
+		width  = 8u;
+	} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
+		offset = operand.sdwa_sel == 5u ? 16u : 0u;
+	}
+	const auto extract = width == 8u && operand.sdwa_sext
+	                         ? IR::ValueOpcode::BitFieldSExtract
+	                         : IR::ValueOpcode::BitFieldUExtract;
+	auto value = IR::U32(ir.Emit(extract, {bits, IR::Value(offset), IR::Value(width)}));
+	if (width == 8u && operand.sdwa_sext) {
+		value = ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
+	}
 	if (operand.absolute) {
 		value = ir.BitwiseAnd(value, IR::U32(IR::Value(0x7fffu)));
 	}
 	if (high_lane ? operand.negate_hi : operand.negate) {
-		// RDNA2 source NEG flips the sign bit, including packed integer operands.
 		value = ir.BitwiseXor(value, IR::U32(IR::Value(0x8000u)));
 	}
 	return value;
@@ -895,8 +866,14 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_ADD_F64:
+		case Decoder::Opcode::V_MIN_F64:
+		case Decoder::Opcode::V_MAX_F64:
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
-		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
+		case Decoder::Opcode::V_RCP_F64:
+		case Decoder::Opcode::V_TRUNC_F64:
+		case Decoder::Opcode::V_CEIL_F64:
+		case Decoder::Opcode::V_FLOOR_F64:
+		case Decoder::Opcode::V_FRACT_F64: include_vector(inst.dst, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
 		case Decoder::Opcode::V_CMP_EQ_F64:
 		case Decoder::Opcode::V_CMP_LE_F64:
@@ -905,6 +882,7 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CMP_EQ_I64:
 		case Decoder::Opcode::V_CMP_LT_I64:
 		case Decoder::Opcode::V_CMP_LE_I64:
+		case Decoder::Opcode::V_CMP_NE_I64:
 		case Decoder::Opcode::V_CMP_LT_U64:
 		case Decoder::Opcode::V_CMP_EQ_U64:
 		case Decoder::Opcode::V_CMP_LE_U64:
@@ -1104,6 +1082,16 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				                                  IR::U32(IR::Value(total_threads)));
 			}
 		}
+		if (options.stage == ShaderType::Compute &&
+		    options.input_info.compute->dispatch_thread_dimensions) {
+			for (uint32_t axis = 0; axis < 3u; axis++) {
+				const auto extent = IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::GetDispatchThreadExtent, {IR::Value(axis)}));
+				initial_exec = entry_ir.LogicalAnd(
+				    initial_exec,
+				    entry_ir.ULessThan(builtin(IR::StageInputKind::GlobalInvocationId, axis), extent));
+			}
+		}
 		entry_ir.SetExec(initial_exec);
 		const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec});
 		entry_ir.SetExecLo(entry_ir.CompositeExtract(initial_mask, 0));

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -40,10 +40,9 @@ private:
 	IR::U32                ReadU32(const Decoder::Operand& operand);
 	std::array<IR::U32, 2> ReadU32Pair(const Decoder::Operand& operand);
 	IR::U64                ReadU64(const Decoder::Operand& operand);
-	IR::F32 ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane, bool packed = false);
+	IR::F32 ReadF16LaneAsF32(const Decoder::Operand& operand, bool high_lane);
 	IR::F32 ReadF16AsF32(const Decoder::Operand& operand);
 	IR::F32 ReadMixF32(const Decoder::Operand& operand);
-	IR::U32 ReadU16LaneRaw(const Decoder::Operand& operand, bool high_lane);
 	IR::U32 ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane, bool sign_extend);
 	IR::U32 ReadU16AsU32(const Decoder::Operand& operand, bool sign_extend);
 	IR::U32 Read16LaneBits(const Decoder::Operand& operand, bool high_lane);
@@ -86,7 +85,8 @@ private:
 	void          BUFFER_LOAD(const Decoder::Instruction& inst);
 	void          BUFFER_STORE(const Decoder::Instruction& inst);
 	void          BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode32,
+	                           IR::ValueOpcode opcode64 = IR::ValueOpcode::Count);
 	void DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
 	void FLAT_LOAD(const Decoder::Instruction& inst);
 	void FLAT_STORE(const Decoder::Instruction& inst);
@@ -105,11 +105,10 @@ private:
 	void DS_READ2(const Decoder::Instruction& inst);
 	void DS_WRITE(const Decoder::Instruction& inst);
 	void DS_WRITE2(const Decoder::Instruction& inst);
-	void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void DS_ADDTID(const Decoder::Instruction& inst, bool write);
 	void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
-	void DS_BPERMUTE_B32(const Decoder::Instruction& inst);
+	void DS_PERMUTE(const Decoder::Instruction& inst, bool backward);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -127,7 +126,7 @@ private:
 	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
 	                      bool cmpx);
 	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx);
-	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx);
+	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half = false);
 	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
 	void V_CVT_F32_U32(const Decoder::Instruction& inst);
 	void V_CVT_F32_I32(const Decoder::Instruction& inst);
@@ -208,8 +207,7 @@ private:
 	void    V_BFI_B32(const Decoder::Instruction& inst);
 	void    S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
 	void    S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
-	void    V_ALIGNBIT_B32(const Decoder::Instruction& inst);
-	void    V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
+	void    V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset);
 	void    V_LSHL_ADD_U32(const Decoder::Instruction& inst);
 	void    V_ADD_LSHL_U32(const Decoder::Instruction& inst);
 	void    V_XAD_U32(const Decoder::Instruction& inst);
@@ -229,7 +227,7 @@ private:
 	void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode,
 	                    IR::ValueOpcode compare_opcode);
 	void EmitControlNop();
-	void EmitWaitcnt();
+	void S_WAITCNT_VSCNT(const Decoder::Instruction& inst);
 	void S_BARRIER();
 	void S_SENDMSG(const Decoder::Instruction& inst);
 	void S_TTRACEDATA();

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -106,6 +106,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_GT_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMP_NE_I64:
 		case O::V_CMP_NE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
 			return;
@@ -142,6 +143,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_NE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
 			return;
+		case O::V_CMPX_NE_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, true);
+			return;
 		case O::V_CMP_NE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
 			return;
@@ -342,6 +346,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true, true); return;
 		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return;
 		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return;
+		case O::V_CMPX_CLASS_F16: EmitFloatClassCompare(inst, true, true); return;
 
 		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return;
 		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return;
@@ -453,10 +458,14 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_RCP_F64: return FloatUnary(inst, IR::ValueOpcode::FPRecip64);
 		case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32);
 		case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32);
+		case O::V_FRACT_F64: return FloatUnary(inst, IR::ValueOpcode::FPFract64);
 		case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32);
+		case O::V_TRUNC_F64: return FloatUnary(inst, IR::ValueOpcode::FPTrunc64);
 		case O::V_CEIL_F32: return FloatUnary(inst, IR::ValueOpcode::FPCeil32);
+		case O::V_CEIL_F64: return FloatUnary(inst, IR::ValueOpcode::FPCeil64);
 		case O::V_RNDNE_F32: return FloatUnary(inst, IR::ValueOpcode::FPRoundEven32);
 		case O::V_FLOOR_F32: return FloatUnary(inst, IR::ValueOpcode::FPFloor32);
+		case O::V_FLOOR_F64: return FloatUnary(inst, IR::ValueOpcode::FPFloor64);
 		case O::V_EXP_F32: return FloatUnary(inst, IR::ValueOpcode::FPExp2);
 		case O::V_LOG_F32: return FloatUnary(inst, IR::ValueOpcode::FPLog2);
 		case O::V_RSQ_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipSqrt32);
@@ -469,13 +478,18 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
 		case O::V_ADD_F64: return FloatBinary(inst, IR::ValueOpcode::FPAdd64, false);
 		case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false);
+		case O::V_MIN_F64: return FloatBinary(inst, IR::ValueOpcode::FPMin64, false);
+		case O::V_MAX_F64: return FloatBinary(inst, IR::ValueOpcode::FPMax64, false);
 		case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
 		case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);
 		case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false);
-		case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true);
+		case O::V_MAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPMad32, true, true);
 		case O::V_MADMK_F32:
 		case O::V_MADAK_F32:
-		case O::V_MAD_F32:
+		case O::V_MAD_F32: return FloatTernary(inst, IR::ValueOpcode::FPMad32, false, true);
+		case O::V_FMAC_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, true, true);
+		case O::V_FMAMK_F32:
+		case O::V_FMAAK_F32:
 		case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true);
 		case O::V_FMA_F64: return FloatTernary(inst, IR::ValueOpcode::FPFma64, false, false);
 		case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false);
@@ -591,8 +605,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_BFE_U32: return V_BFE_U32(inst, false);
 		case O::V_BFE_I32: return V_BFE_U32(inst, true);
 		case O::V_BFI_B32: return V_BFI_B32(inst);
-		case O::V_ALIGNBIT_B32: return V_ALIGNBIT_B32(inst);
-		case O::V_ALIGNBYTE_B32: return V_ALIGNBYTE_B32(inst);
+		case O::V_ALIGNBIT_B32: return V_ALIGN_B32(inst, false);
+		case O::V_ALIGNBYTE_B32: return V_ALIGN_B32(inst, true);
 		case O::V_LSHL_ADD_U32: return V_LSHL_ADD_U32(inst);
 		case O::V_ADD_LSHL_U32: return V_ADD_LSHL_U32(inst);
 		case O::V_XAD_U32: return V_XAD_U32(inst);

src/graphics/shader/recompiler/ir/Program.cpp

@@ -190,6 +190,18 @@ Value ResolveInvariantPhi(const ResourcePlan& program, Value value) {
 	return invariant;
 }
 
+Value ResolveActiveU32(Value value, Value active) {
+	for (uint32_t depth = 0; depth <= 32; ++depth) {
+		value = value.Resolve();
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 ||
+		    inst->Arg(0).Resolve() != active)
+			return value;
+		value = inst->Arg(1);
+	}
+	return {};
+}
+
 bool HasShaderMemoryWrites(const Program& program) {
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
@@ -399,7 +411,8 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
 				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind != ResourceKind::ScalarBuffer) {
+				if (memory.kind != ResourceKind::ScalarBuffer &&
+				    memory.kind != ResourceKind::IndirectBuffer) {
 					return Fail(fmt::format("{} has an invalid scalar-memory resource kind",
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
@@ -457,7 +470,7 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 				}
 				if (memory.kind == ResourceKind::IndirectBuffer &&
 				    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) {
-					return Fail("indirect buffer requires a raw DWORD x2/x3/x4 load");
+					return Fail("indirect buffer requires a scalar, raw DWORD x1/x2/x3/x4, or formatted X load");
 				}
 				if (buffer_components > 1u &&
 				    (!vector_buffer || memory.data_bits != 32u ||

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -73,9 +73,13 @@ struct MemoryInfo {
 	bool                    planning_only                                         = false;
 
 	[[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const {
-		return !formatted && !typed && data_bits == 32u &&
-		       (opcode == ValueOpcode::LoadBufferU32x2 || opcode == ValueOpcode::LoadBufferU32x3 ||
-		        opcode == ValueOpcode::LoadBufferU32x4);
+		return !typed && data_bits == 32u &&
+		       (formatted ? opcode == ValueOpcode::LoadBufferU32
+		                  : opcode == ValueOpcode::ReadConstBuffer ||
+		                        opcode == ValueOpcode::LoadBufferU32 ||
+		                        opcode == ValueOpcode::LoadBufferU32x2 ||
+		                        opcode == ValueOpcode::LoadBufferU32x3 ||
+		                        opcode == ValueOpcode::LoadBufferU32x4);
 	}
 
 	bool operator==(const MemoryInfo& other) const = default;
@@ -97,6 +101,7 @@ struct ExportInfo {
 
 struct BufferResource {
 	static constexpr uint32_t NoImageAlias = UINT32_MAX;
+	static constexpr uint32_t NoIndirectBuffer = UINT32_MAX;
 
 	uint32_t               source             = 0;
 	uint32_t               first_use_pc       = 0;
@@ -110,6 +115,10 @@ struct BufferResource {
 	bool                   atomic             = false;
 	bool                   formatted          = false;
 	bool                   scalar             = false;
+	uint32_t               indirect_root              = NoIndirectBuffer;
+	uint32_t               indirect_mapping_offset    = 0;
+	uint32_t               indirect_search_iterations = 0;
+	std::vector<uint32_t>  indirect_resources;
 
 	bool operator==(const BufferResource& other) const = default;
 };
@@ -190,6 +199,7 @@ enum class StageInputKind {
 	BaryCoordSmoothCentroid,
 	BaryCoordNoPerspective,
 	WorkgroupId,
+	NumWorkgroups,
 	LocalInvocationId,
 	LocalInvocationIndex,
 	GlobalInvocationId,
@@ -292,11 +302,12 @@ enum class DescriptorBindingKind : uint32_t {
 	FaultBuffer,
 	FlattenedSrt,
 	ShaderData,
+	SharedMemory,
 	Count,
 };
 
 static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u);
-static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 55u);
+static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u);
 
 struct PushData {
 	static constexpr uint32_t DwordCount = 32;
@@ -421,6 +432,7 @@ struct DescriptorBinding {
 
 struct BindingLayout {
 	uint32_t                       push_data_start_dword = PushData::NoStart;
+	uint32_t                       dispatch_thread_dword = PushData::NoStart;
 	uint32_t                       memory_offset_dword = 0;
 	uint32_t                       memory_offset_count = 0;
 	std::vector<uint32_t>          user_data_registers;
@@ -472,22 +484,33 @@ struct BlockInfo {
 };
 
 struct DescriptorSource {
-	struct IndirectImage {
-		uint32_t material_source = UINT32_MAX;
+	struct IndirectDescriptor {
+		struct SelectorRead {
+			uint32_t source = UINT32_MAX;
+			uint32_t stride = 0;
+			uint32_t offset = 0;
+
+			bool operator==(const SelectorRead& other) const = default;
+		};
+		std::optional<SelectorRead> selector;
 		uint32_t table_source    = 0;
-		uint32_t selector_stride = 0;
-		uint32_t selector_offset = 0;
 		uint32_t table_offset    = 0;
+		uint32_t table_immediate = 0;
+		uint32_t table_stride    = 0;
+		uint32_t table_record_bytes = 0;
+		bool     table_scalar = false;
+		uint32_t workgroup_axis  = UINT32_MAX;
 		Value    key_count;
+		Value                 selector_first;
 		Value    selector_mask;
 		std::vector<uint32_t> sources;
 
-		bool operator==(const IndirectImage& other) const = default;
+		bool operator==(const IndirectDescriptor& other) const = default;
 	};
 
 	std::array<Value, 8>         dwords {};
 	uint32_t                     dword_count = 0;
-	std::optional<IndirectImage> indirect_image;
+	std::optional<IndirectDescriptor> indirect_descriptor;
 
 	bool operator==(const DescriptorSource& other) const = default;
 };
@@ -610,6 +633,7 @@ void  ValidateProgram(const Program& program, bool require_ssa);
 void  ResolveControlFlowIdentities(Program& program);
 bool  EquivalentValue(const ResourcePlan& program, Value left, Value right);
 Value ResolveInvariantPhi(const ResourcePlan& program, Value value);
+Value ResolveActiveU32(Value value, Value active);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -93,6 +93,7 @@ bool HasSideEffects(ValueOpcode opcode) {
 		case ValueOpcode::SetAttribute:
 		case ValueOpcode::SetTessellationAttribute:
 		case ValueOpcode::MeshAllocate:
+		case ValueOpcode::StoreCompletion:
 		case ValueOpcode::Barrier: return true;
 		default: return false;
 	}
@@ -177,7 +178,8 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicAnd32:
 		case ValueOpcode::SharedAtomicOr32:
 		case ValueOpcode::SharedAtomicOr64:
-		case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic;
+		case ValueOpcode::SharedAtomicXor32:
+		case ValueOpcode::SharedAtomicMaskedOr32: return SharedAccess::Atomic;
 		case ValueOpcode::DataAppend: return SharedAccess::Append;
 		case ValueOpcode::DataConsume: return SharedAccess::Consume;
 		default: return SharedAccess::None;
@@ -222,15 +224,21 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
 			return {ImageAccess::Write, ImageResourceClass::Storage, false};
 		case ValueOpcode::ImageAtomicCompareSwap32:
 		case ValueOpcode::ImageAtomicSwap32:
+		case ValueOpcode::ImageAtomicSwap64:
 		case ValueOpcode::ImageAtomicIAdd32:
+		case ValueOpcode::ImageAtomicIAdd64:
 		case ValueOpcode::ImageAtomicSMin32:
 		case ValueOpcode::ImageAtomicUMin32:
+		case ValueOpcode::ImageAtomicUMin64:
 		case ValueOpcode::ImageAtomicSMax32:
 		case ValueOpcode::ImageAtomicUMax32:
 		case ValueOpcode::ImageAtomicUMax64:
 		case ValueOpcode::ImageAtomicAnd32:
+		case ValueOpcode::ImageAtomicAnd64:
 		case ValueOpcode::ImageAtomicOr32:
+		case ValueOpcode::ImageAtomicOr64:
 		case ValueOpcode::ImageAtomicXor32:
+		case ValueOpcode::ImageAtomicXor64:
 		case ValueOpcode::ImageAtomicFMin32:
 		case ValueOpcode::ImageAtomicFMax32:
 			return {ImageAccess::Atomic, ImageResourceClass::Storage, false};

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -8,6 +8,7 @@ VALUE_OPCODE(ConditionRef, U1, U1)
 
 // Translation and register-state pseudo operations.
 VALUE_OPCODE(GetUserData, U32, ScalarReg)
+VALUE_OPCODE(GetDispatchThreadExtent, U32, U32)
 VALUE_OPCODE(GetShaderBase, U64)
 VALUE_OPCODE(MeshDrawParameter, U32, U32)
 VALUE_OPCODE(MeshAllocate, Void, U32)
@@ -174,10 +175,12 @@ VALUE_OPCODE(FPOrdGreaterThanEqual32, U1, F32, F32)
 VALUE_OPCODE(FPUnordGreaterThanEqual32, U1, F32, F32)
 VALUE_OPCODE(FPIsNan32, U1, F32)
 VALUE_OPCODE(FPCmpClass32, U1, F32, U32)
+VALUE_OPCODE(FPCmpClass16, U1, U32, U32)
 
 VALUE_OPCODE(FPAdd32, F32, F32, F32)
 VALUE_OPCODE(FPSub32, F32, F32, F32)
 VALUE_OPCODE(FPFma32, F32, F32, F32, F32)
+VALUE_OPCODE(FPMad32, F32, F32, F32, F32)
 VALUE_OPCODE(FPMul32, F32, F32, F32)
 VALUE_OPCODE(FPAdd64, F64, F64, F64)
 VALUE_OPCODE(FPMul64, F64, F64, F64)
@@ -185,6 +188,8 @@ VALUE_OPCODE(FPFma64, F64, F64, F64, F64)
 VALUE_OPCODE(FPRecip64, F64, F64)
 VALUE_OPCODE(FPMin32, F32, F32, F32)
 VALUE_OPCODE(FPMax32, F32, F32, F32)
+VALUE_OPCODE(FPMin64, F64, F64, F64)
+VALUE_OPCODE(FPMax64, F64, F64, F64)
 VALUE_OPCODE(FPMinTri32, F32, F32, F32, F32)
 VALUE_OPCODE(FPMaxTri32, F32, F32, F32, F32)
 VALUE_OPCODE(FPMedTri32, F32, F32, F32, F32)
@@ -201,7 +206,11 @@ VALUE_OPCODE(FPRoundEven32, F32, F32)
 VALUE_OPCODE(FPFloor32, F32, F32)
 VALUE_OPCODE(FPCeil32, F32, F32)
 VALUE_OPCODE(FPTrunc32, F32, F32)
+VALUE_OPCODE(FPFloor64, F64, F64)
+VALUE_OPCODE(FPCeil64, F64, F64)
+VALUE_OPCODE(FPTrunc64, F64, F64)
 VALUE_OPCODE(FPFract32, F32, F32)
+VALUE_OPCODE(FPFract64, F64, F64)
 
 VALUE_OPCODE(LaneId, U32)
 VALUE_OPCODE(Ballot, U32x4, U1)
@@ -209,6 +218,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
 VALUE_OPCODE(ReadLane, U32, U32, U32)
 VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
 VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1)
+VALUE_OPCODE(PermuteU32, U32, U32, U32, U1)
 VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1)
 
 // Typed shader resources and memory operations. Descriptor state is represented by SSA values;
@@ -271,8 +281,8 @@ VALUE_OPCODE(WriteSharedU32, Void, U32, U32, U1)
 VALUE_OPCODE(WriteSharedU32x2, Void, U32, U32, U32, U1)
 VALUE_OPCODE(WriteSharedU32x3, Void, U32, U32, U32, U32, U1)
 VALUE_OPCODE(WriteSharedU32x4, Void, U32, U32, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
-VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1)
@@ -287,6 +297,7 @@ VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1)
 VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicMaskedOr32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(DataAppend, U32, U32, U1)
 VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
@@ -300,15 +311,21 @@ VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress
 VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress, U32, U32, U32, U32)
 VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1)
 VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSwap64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicIAdd64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicUMin64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicAnd64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicOr64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicXor64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicFMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicFMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(GetAttribute, U32, U32, U32)
@@ -316,7 +333,7 @@ VALUE_OPCODE(GetInterpolationParameter, U32, U32, U32, U32)
 VALUE_OPCODE(SetAttribute, Void, U32x4, U1)
 
 VALUE_OPCODE(ControlNop, Void)
-VALUE_OPCODE(Waitcnt, Void)
+VALUE_OPCODE(StoreCompletion, Void)
 VALUE_OPCODE(Barrier, Void)
 VALUE_OPCODE(Sendmsg, Void)
 VALUE_OPCODE(TtraceData, Void)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -15,11 +15,16 @@ namespace {
 	std::abort();
 }
 
-std::vector<uint32_t> CollectUserData(const Program& program) {
+void CollectShaderData(const Program& program, BindingLayout& layout) {
 	std::array<bool, NumScalarRegs> registers {};
+	bool uses_dispatch_threads = false;
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) {
+			if (!inst.HasUses()) {
+				continue;
+			}
+			uses_dispatch_threads |= inst.GetOpcode() == ValueOpcode::GetDispatchThreadExtent;
+			if (inst.GetOpcode() != ValueOpcode::GetUserData) {
 				continue;
 			}
 			if (inst.Arg(0).GetType() != Type::ScalarReg) {
@@ -32,13 +37,16 @@ std::vector<uint32_t> CollectUserData(const Program& program) {
 			registers[index] = true;
 		}
 	}
-	std::vector<uint32_t> result;
 	for (uint32_t index = 0; index < registers.size(); index++) {
 		if (registers[index]) {
-			result.push_back(index);
+			layout.user_data_registers.push_back(index);
 		}
 	}
-	return result;
+	layout.memory_offset_dword = static_cast<uint32_t>(layout.user_data_registers.size());
+	if (uses_dispatch_threads) {
+		layout.dispatch_thread_dword = layout.memory_offset_dword;
+		layout.memory_offset_dword += 3u;
+	}
 }
 
 void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
@@ -48,14 +56,15 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
 
 } // namespace
 
-bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers) {
+SharedMemoryResources CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers) {
 	std::array<bool, ShaderInfo::MaxBuffers> live_buffers {};
-	bool uses_gds = false;
+	SharedMemoryResources shared;
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto op = inst.GetOpcode();
 			if (BufferAccessOf(op) == BufferAccess::None &&
-			    SharedAccessOf(op) == SharedAccess::None) {
+			    SharedAccessOf(op) == SharedAccess::None &&
+			    AddressOpcodeInfoOf(op).access == AddressAccess::None) {
 				continue;
 			}
 			const auto index = inst.Flags<MemoryFlags>().index;
@@ -66,14 +75,19 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
 			if (memory.planning_only) {
 				continue;
 			}
+			shared.lds |= memory.kind == ResourceKind::FlatLocal;
 			if (SharedAccessOf(op) != SharedAccess::None) {
 				if (memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) {
 					BindingFail("typed shader contains invalid shared-memory metadata");
 				}
-				uses_gds |= memory.kind == ResourceKind::Gds;
+				shared.gds |= memory.kind == ResourceKind::Gds;
+				shared.lds |= memory.kind == ResourceKind::Lds;
 			} else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) {
 				EXIT_IF(memory.resource >= program.info.buffers.size());
 				live_buffers.at(memory.resource) = true;
+				for (const auto child: program.info.buffers[memory.resource].indirect_resources) {
+					live_buffers.at(child) = true;
+				}
 			}
 		}
 	}
@@ -82,20 +96,22 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
 			buffers.push_back(i);
 		}
 	}
-	return uses_gds;
+	return shared;
 }
 
 bool UsesFlattenedSrt(const Program& program) {
+	const auto uses_mapping = [](const auto& resource) {
+		return resource.indirect_root != UINT32_MAX;
+	};
 	return std::ranges::any_of(program.blocks, [](const Block* block) {
 		return std::ranges::any_of(*block, [](const Inst& inst) {
 			return inst.GetOpcode() == ValueOpcode::ReadConst;
 		});
-	}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) {
-		return image.indirect_search_iterations != 0u;
-	});
+	}) || std::ranges::any_of(program.info.buffers, uses_mapping) ||
+	       std::ranges::any_of(program.info.images, uses_mapping);
 }
 
-void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
+void AllocateBindings(Program& program, uint32_t push_data_start_dword, bool lds_storage) {
 	if (!program.shader_info_complete || program.binding_layout_complete) {
 		EXIT("shader binding layout failed: %s", !program.shader_info_complete
 		                                             ? "shader info is not ready"
@@ -103,9 +119,8 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 	}
 	BindingLayout next;
 	std::vector<uint32_t> buffers;
-	const bool            uses_gds = CollectMemoryResources(program, buffers);
-	next.user_data_registers = CollectUserData(program);
-	next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
+	const auto shared = CollectMemoryResources(program, buffers);
+	CollectShaderData(program, next);
 	next.memory_offset_count       = static_cast<uint32_t>(buffers.size());
 	next.push_data_start_dword =
 	    PushData::StartFor(push_data_start_dword, next.ShaderDataDwords());
@@ -148,9 +163,12 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 		}
 		AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
 	}
-	if (uses_gds) {
+	if (shared.gds) {
 		AddBinding(next, DescriptorBindingKind::Gds);
 	}
+	if (shared.lds && lds_storage) {
+		AddBinding(next, DescriptorBindingKind::SharedMemory);
+	}
 	if (program.info.uses_dma) {
 		AddBinding(next, DescriptorBindingKind::BdaPagetable);
 		AddBinding(next, DescriptorBindingKind::FaultBuffer);

src/graphics/shader/recompiler/ir/passes/BindingLayout.h

@@ -5,10 +5,15 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0);
+void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0,
+                      bool lds_storage = false);
 
-// Collect live direct buffer indices and report whether the shader accesses GDS.
-bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers);
+struct SharedMemoryResources {
+	bool lds = false;
+	bool gds = false;
+};
+
+SharedMemoryResources CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers);
 bool UsesFlattenedSrt(const Program& program);
 
 const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -199,6 +199,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
                       std::unordered_set<Inst*>& lowered_ancillary) {
 	auto& inst = *instruction;
 	switch (inst.GetOpcode()) {
+		case ValueOpcode::StoreBufferU32: {
+			const auto data = ResolveActiveU32(Arg(inst, 4), Arg(inst, 5));
+			if (!data.IsEmpty() && data != inst.Arg(4)) inst.SetArg(4, data);
+			return;
+		}
 		case ValueOpcode::Phi: FoldPhi(inst); return;
 		case ValueOpcode::SelectU1:
 			if (!FoldSelect(inst) && IsImmediate(Arg(inst, 2), Type::U1) &&
@@ -342,6 +347,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) {
 				Replace(inst, Value(static_cast<uint64_t>(low.U32()) |
 				                    (static_cast<uint64_t>(high.U32()) << 32u)));
+			} else {
+				const auto* low_extract  = low.TryInstruction();
+				const auto* high_extract = high.TryInstruction();
+				if (low_extract != nullptr && high_extract != nullptr &&
+				    low_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 &&
+				    high_extract->GetOpcode() == ValueOpcode::CompositeExtractU64 &&
+				    Arg(*low_extract, 1) == Value(0u) && Arg(*high_extract, 1) == Value(1u) &&
+				    Arg(*low_extract, 0) == Arg(*high_extract, 0)) {
+					// Guest register pairs need not unpack and repack between wide operations.
+					Replace(inst, Arg(*low_extract, 0));
+				}
 			}
 			return;
 		}
@@ -503,6 +519,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			}
 			return;
 		}
+		case ValueOpcode::BitReverse32: {
+			const auto value = Arg(inst, 0);
+			if (IsImmediate(value, Type::U32)) {
+				uint32_t source = value.U32(), reversed = 0;
+				for (uint32_t bit = 0; bit < 32u; ++bit, source >>= 1u) {
+					reversed = (reversed << 1u) | (source & 1u);
+				}
+				Replace(inst, Value(reversed));
+			}
+			return;
+		}
 		case ValueOpcode::BitCount32: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U32)) {

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -20,7 +20,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
 namespace {
 
 constexpr uint64_t AddressMask            = 0x0000ffffffffffffull;
-constexpr uint64_t MaxIndirectImageProbes = 65536u;
+constexpr uint64_t MaxIndirectDescriptorProbes = 65536u;
 
 bool SpecializationFail(std::string_view message) {
 	std::fprintf(stderr, "shader resource specialization failed: %.*s\n",
@@ -111,7 +111,8 @@ enum class SamplerClass : uint8_t { Float, Integer, PointInteger };
 
 SamplerClass ClassifySampler(const ImageResource& image) {
 	if (image.numeric_class == Prospero::TextureNumericClass::Sint ||
-	    image.conversion_format != Prospero::BufferFormat::kInvalid) {
+	    (image.numeric_class == Prospero::TextureNumericClass::Uint &&
+	     image.conversion_format != Prospero::BufferFormat::kInvalid)) {
 		return SamplerClass::PointInteger;
 	}
 	return image.numeric_class == Prospero::TextureNumericClass::Uint ? SamplerClass::Integer
@@ -224,22 +225,76 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
 	       runtime.read_specialization_memory(runtime.userdata, address, prefix);
 }
 
-bool MaterializeIndirectImage(const ResourcePlan& program,
-                              const DescriptorSource::IndirectImage& indirect,
-                              uint32_t image_index,
-                              const SrtRuntime& runtime, SrtWalker& clean,
-                              ResourceSnapshot& snapshot,
-                              ResourceSpecialization& specialization) {
+bool NormalizeIndirectStoreBuffer(DescriptorValue& value) {
+	ShaderBufferResource descriptor;
+	if (!DecodeBufferDescriptor(value, descriptor)) return false;
+	if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid ||
+	    descriptor.Base48() == 0u || descriptor.GetSize() == 0u) {
+		value.dwords.fill(0u);
+	} else if (descriptor.Stride() != 1u || descriptor.SwizzleEnabled() || descriptor.AddTid() ||
+	           descriptor.OutOfBounds() != 3u) {
+		return SpecializationFail("indirect buffer stores require linear byte bounds");
+	}
+	return true;
+}
+
+bool IsBoundedDescriptorTable(const ResourcePlan& program,
+                              const DescriptorSource::IndirectDescriptor& indirect) {
+	const auto* table = Source(program, indirect.table_source);
+	return indirect.sources.empty() && !indirect.selector &&
+	       indirect.workgroup_axis == UINT32_MAX && table != nullptr && table->dword_count == 4u;
+}
+
+template <typename Specialization, typename Normalize>
+bool MaterializeIndirectDescriptor(const ResourcePlan&                         program,
+                                   const DescriptorSource::IndirectDescriptor& indirect,
+                                   uint32_t resource_index, uint32_t dword_count,
+                                   const SrtRuntime& runtime, SrtWalker& clean,
+                                   ResourceSnapshot&             snapshot,
+                                   std::vector<DescriptorValue>& descriptors,
+                                   std::vector<Specialization>&  specializations,
+                                   uint32_t maximum_resources, Normalize&& normalize) {
+	const auto  descriptor_bytes = dword_count * sizeof(uint32_t);
 	const auto& sources = indirect.sources;
+	const auto* selector = indirect.selector ? &*indirect.selector : nullptr;
 	auto& keys = program.material_keys;
+	const auto children_begin = descriptors.size();
+	const auto root_resource = specializations[resource_index];
+	const auto intern_candidate = [&](const DescriptorValue& candidate, uint32_t& ordinal) {
+		ordinal = 0u;
+		if (candidate == descriptors[resource_index]) return true;
+		const auto found = std::find(descriptors.begin() + children_begin, descriptors.end(), candidate);
+		ordinal = static_cast<uint32_t>(found - descriptors.begin() - children_begin + 1u);
+		if (found == descriptors.end()) {
+			if (descriptors.size() >= maximum_resources) return false;
+			descriptors.push_back(candidate);
+			auto child = root_resource;
+			child.indirect_root = resource_index;
+			specializations.push_back(child);
+		}
+		return true;
+	};
+	const auto read_keys = [&](const ShaderBufferResource& material, uint64_t first,
+	                           uint64_t step, uint64_t count) {
+		keys.resize(count);
+		if (step == 4u) {
+			return ReadScalarTable(material.Base48(), material.GetSize(), first, runtime, keys);
+		}
+		for (auto& key: keys) {
+			if (!ReadScalarTable(material.Base48(), material.GetSize(), first, runtime, {&key, 1}))
+				return false;
+			first += step;
+		}
+		return true;
+	};
 	uint64_t table_base = 0;
 	uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
 	if (sources.empty()) {
 		keys.clear();
 		DescriptorValue material_value;
 		DescriptorValue table_value;
-		if ((indirect.material_source != UINT32_MAX &&
-		     !clean.EvaluateDescriptor(indirect.material_source, material_value)) ||
+		if ((selector != nullptr &&
+		     !clean.EvaluateDescriptor(selector->source, material_value)) ||
 		    !clean.EvaluateDescriptor(indirect.table_source, table_value)) {
 			return false;
 		}
@@ -252,17 +307,83 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 		} else {
 			return false;
 		}
-		if (indirect.material_source == UINT32_MAX) {
+		if (IsBoundedDescriptorTable(program, indirect)) {
+			if (!indirect.table_scalar && table.Format() == Prospero::BufferFormat::kInvalid) {
+				descriptors[resource_index] = {.dword_count = dword_count};
+				return true;
+			}
+			if (table_size > UINT32_MAX ||
+			    (indirect.table_record_bytes != 0u &&
+			     (table.Type() != 0u || table.SwizzleEnabled() || table.AddTid() ||
+			      table.OutOfBounds() != 0u || (table_base & 3u) != 0u ||
+			      (table.Stride() & 3u) != 0u || table.Stride() < indirect.table_record_bytes)))
+				return false;
+			// Descriptor bytes are refreshed together. The GPU selects the DWORD address;
+			// no material selector or unused table field is interpreted on the CPU.
+			keys.resize(table_size / 4u);
+			if (!ReadScalarTable(table_base, table_size, 0u, runtime, keys)) return false;
+			const auto mapping_offset = snapshot.flattened_srt.size();
+			descriptors[resource_index] = {.dword_count = dword_count};
+			snapshot.flattened_srt.resize(mapping_offset + 2u, 0u);
+			for (uint32_t word = 0; word < keys.size(); ++word) {
+				DescriptorValue candidate {.dword_count = dword_count};
+				std::copy_n(keys.begin() + word, std::min<size_t>(dword_count, keys.size() - word),
+				            candidate.dwords.begin());
+				if (!normalize(candidate)) return false;
+				uint32_t ordinal;
+				if (!intern_candidate(candidate, ordinal)) return false;
+				if (ordinal == 0u) continue;
+				snapshot.flattened_srt.resize(mapping_offset + 2u + word, 0u);
+				snapshot.flattened_srt[mapping_offset + 1u + word] = ordinal;
+			}
+			if (descriptors.size() == children_begin) {
+				snapshot.flattened_srt.resize(mapping_offset);
+			} else {
+				snapshot.flattened_srt[mapping_offset] =
+				    static_cast<uint32_t>(snapshot.flattened_srt.size() - mapping_offset - 1u);
+				auto& root = specializations[resource_index];
+				root.indirect_root = resource_index;
+				root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
+				root.indirect_search_iterations = 0u;
+			}
+			return true;
+		}
+		if (selector == nullptr) {
 			uint32_t key_count = 0;
-			const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
-			if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
-			if (table_value.dword_count != 2u || !evaluated ||
-			    key_count > MaxIndirectImageProbes ||
-			    uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
+			if (indirect.workgroup_axis != UINT32_MAX) {
+				if (indirect.workgroup_axis >= runtime.workgroup_counts.size()) return false;
+				key_count = runtime.workgroup_counts[indirect.workgroup_axis];
+				if (key_count == 0u) return false;
+			} else {
+				if (table_value.dword_count != 2u || !clean.Evaluate(indirect.key_count, key_count))
+					return false;
+				if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
+			}
+			if (indirect.table_stride == 0u || key_count > MaxIndirectDescriptorProbes ||
+			    (key_count != 0u && uint64_t {indirect.table_offset} + indirect.table_immediate +
+			         uint64_t {key_count - 1u} * indirect.table_stride + descriptor_bytes >
+			             UINT32_MAX + 1ull)) {
 				return false;
 			}
 			keys.resize(key_count);
 			std::iota(keys.begin(), keys.end(), 0u);
+		} else if (!indirect.selector_first.IsEmpty()) {
+			ShaderBufferResource material;
+			uint32_t             first = 0, count = 0;
+			if (!DecodeBufferDescriptor(material_value, material) || material.Type() != 0u ||
+			    material.SwizzleEnabled() ||
+			    material.AddTid() || material.OutOfBounds() != 0u ||
+			    uint64_t {selector->offset} + 4u > material.Stride() ||
+			    !clean.Evaluate(indirect.selector_first, first) ||
+			    !clean.Evaluate(indirect.key_count, count) || count > MaxIndirectDescriptorProbes ||
+			    uint64_t {first} + count > material.NumRecords())
+				return false;
+			if (count != 0u && (uint64_t {first} + count - 1u) * material.Stride() +
+			                           selector->offset + 4u >
+			                       uint64_t {UINT32_MAX} + 1u)
+				return false;
+			if (!read_keys(material, uint64_t {first} * material.Stride() + selector->offset,
+			               material.Stride(), count)) return false;
 		} else if (!indirect.selector_mask.IsEmpty()) {
 			uint32_t mask = 0;
 			uint32_t count = 0;
@@ -277,8 +398,8 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 			keys.reserve(std::popcount(mask));
 			while (mask != 0u) {
 				const auto index = std::countr_zero(mask);
-				const auto offset = static_cast<uint64_t>(indirect.selector_offset) +
-				                    static_cast<uint64_t>(index) * indirect.selector_stride;
+				const auto offset = static_cast<uint64_t>(selector->offset) +
+				                    static_cast<uint64_t>(index) * selector->stride;
 				if (offset > UINT32_MAX) return false;
 				uint32_t key = 0;
 				if (!ReadScalarTable(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
@@ -286,85 +407,47 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 				keys.push_back(key);
 				mask &= mask - 1u;
 			}
-			std::ranges::sort(keys);
-			keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
-		} else {
-			ShaderBufferResource material;
-			if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
-			    material.Stride() != indirect.selector_stride) {
-				return false;
-			}
-			// The first aligned offset includes the immediate added after shader U32 arithmetic.
-			const auto step = std::max<uint64_t>(4u,
-			    std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
-			const uint64_t first = indirect.selector_offset;
-			const auto size = material.GetSize();
-			const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
-			const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
-			if (probe_count > MaxIndirectImageProbes) {
-				return false;
-			}
-			keys.reserve(static_cast<size_t>(probe_count) + 1u);
-			keys.push_back(0u);
-			for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
-				uint32_t key = 0;
-				if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
-					return false;
-				}
-				keys.push_back(key);
-			}
+		} else return false;
+		if (selector != nullptr) {
 			std::ranges::sort(keys);
 			keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 		}
 	}
 
-	const auto children_begin = snapshot.images.size();
 	const auto mapping_offset = snapshot.flattened_srt.size();
-	const auto root_image = specialization.images[image_index];
 	const auto key_count = sources.empty() ? keys.size() : sources.size();
 	snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u);
 	snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(key_count);
 	for (uint32_t entry = 0; entry < key_count; ++entry) {
 		const auto key = sources.empty() ? keys[entry] : entry;
 		DescriptorValue candidate;
-		candidate.dword_count = 8u;
+		candidate.dword_count = dword_count;
 		if (sources.empty()) {
-			const auto table_offset = (key << 5u) + indirect.table_offset;
-			if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
+			const uint64_t table_offset =
+			    uint64_t {(key * indirect.table_stride + indirect.table_offset) & ~3u} +
+			    (indirect.table_immediate & ~3u);
+			if (!ReadScalarTable(table_base, table_size, table_offset, runtime,
+			                     std::span(candidate.dwords).first(dword_count))) {
 				return false;
 			}
 		} else if (!clean.EvaluateDescriptor(sources[entry], candidate)) {
 			return false;
 		}
-		if (NullImageDescriptor(candidate) ||
-		    !ValidImageDescriptor(candidate, program.info.images[image_index].r128)) {
-			candidate.dwords.fill(0);
-		}
+		if (!normalize(candidate)) return false;
 		uint32_t ordinal = 0;
 		if (entry == 0) {
-			snapshot.images[image_index] = candidate;
-		} else if (snapshot.images[image_index] != candidate) {
-			const auto found = std::find(snapshot.images.begin() + children_begin,
-			                             snapshot.images.end(), candidate);
-			ordinal = static_cast<uint32_t>(found - snapshot.images.begin() - children_begin + 1u);
-			if (found == snapshot.images.end()) {
-				if (snapshot.images.size() >= ShaderInfo::MaxImages) {
-					return false;
-				}
-				snapshot.images.push_back(candidate);
-				auto child = root_image;
-				child.indirect_root = image_index;
-				specialization.images.push_back(child);
-			}
+			descriptors[resource_index] = candidate;
+		} else if (!intern_candidate(candidate, ordinal)) {
+			return false;
 		}
 		snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key;
 		snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal;
 	}
-	if (snapshot.images.size() == children_begin) {
+	if (descriptors.size() == children_begin) {
 		snapshot.flattened_srt.resize(mapping_offset);
 	} else {
-		auto& root = specialization.images[image_index];
-		root.indirect_root = image_index;
+		auto& root                      = specializations[resource_index];
+		root.indirect_root              = resource_index;
 		root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
 		root.indirect_search_iterations = std::bit_width(key_count);
 	}
@@ -383,10 +466,21 @@ struct SamplerPlan {
 	uint32_t                                                  sampler_count = 0;
 };
 
+template <typename T, typename Keep>
+void CompactImages(std::vector<T>& images, Keep&& keep) {
+	size_t count = 0;
+	for (size_t index = 0; index < images.size(); ++index) {
+		if (!keep(index)) continue;
+		if (count != index) images[count] = std::move(images[index]);
+		++count;
+	}
+	images.resize(count);
+}
+
 struct ImageRemap {
 	explicit ImageRemap(const ResourceSpecialization& specialization)
-	    : source_count(static_cast<uint32_t>(specialization.images.size())) {
-		EXIT_IF(specialization.images.size() > indices.size());
+	    : indices(specialization.images.size()),
+	      source_count(static_cast<uint32_t>(specialization.images.size())) {
 		for (uint32_t index = 0; index < source_count; index++) {
 			indices[index] = specialization.images[index].fmask ? UINT32_MAX : count++;
 		}
@@ -403,16 +497,11 @@ struct ImageRemap {
 		if (count == source_count) {
 			return;
 		}
-		for (uint32_t index = 0; index < source_count; index++) {
-			if (indices[index] != UINT32_MAX && indices[index] != index) {
-				images[indices[index]] = std::move(images[index]);
-			}
-		}
-		images.resize(count);
+		CompactImages(images, [&](size_t index) { return indices[index] != UINT32_MAX; });
 	}
 
 private:
-	std::array<uint32_t, ShaderInfo::MaxImages> indices;
+	std::vector<uint32_t> indices;
 	uint32_t                                    source_count;
 	uint32_t                                    count = 0;
 };
@@ -507,9 +596,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 		const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size()
 		                           ? snapshot.flattened_srt[root.indirect_mapping_offset]
 		                           : 0u;
-		if (root.indirect_search_iterations == 0u || key_count < 2u ||
+		if (key_count == 0u ||
 		    static_cast<size_t>(root.indirect_mapping_offset) + 1u +
-		            static_cast<size_t>(key_count) * 2u >
+		            static_cast<size_t>(key_count) * (root.indirect_search_iterations == 0u ? 1u : 2u) >
 		        snapshot.flattened_srt.size()) {
 			return SpecializationFail("indirect image specialization has an invalid key mapping");
 		}
@@ -559,7 +648,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			}
 		}
 	}
-	ImageRemap(specialization).Apply(snapshot.images);
+	CompactImages(snapshot.images, [&](size_t index) { return !specialization.images[index].fmask; });
 	return true;
 }
 
@@ -665,11 +754,33 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 	return blocks;
 }
 
+// ResolveComputeBufferFill proves complete workgroups and matching requested extents, so every
+// dispatch-bound predicate accepted here is true for every invocation covered by the fill.
+static bool IsFullDispatchPredicate(Value value, uint32_t depth = 0) {
+	value = value.Resolve();
+	if (value == Value(true)) return true;
+	const auto* inst = value.TryInstruction();
+	if (inst == nullptr || depth > 32) return false;
+	if (inst->GetOpcode() == ValueOpcode::LogicalAnd) {
+		return IsFullDispatchPredicate(inst->Arg(0), depth + 1) &&
+		       IsFullDispatchPredicate(inst->Arg(1), depth + 1);
+	}
+	if (inst->GetOpcode() != ValueOpcode::ULessThan32) return false;
+	const auto* global = inst->Arg(0).Resolve().TryInstruction();
+	const auto* extent = inst->Arg(1).Resolve().TryInstruction();
+	if (global == nullptr || global->GetOpcode() != ValueOpcode::GetBuiltin ||
+	    global->Arg(0).Resolve() != Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)) ||
+	    extent == nullptr || extent->GetOpcode() != ValueOpcode::GetDispatchThreadExtent)
+		return false;
+	const auto axis = global->Arg(1).Resolve();
+	return axis.IsImmediate() && axis.U32() < 3 && extent->Arg(0).Resolve() == axis;
+}
+
 // Nonnegative affine coefficients for constant, local and workgroup coordinates. Reject modular
 // arithmetic that could wrap; runtime coverage also bounds the largest invocation index.
-static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t axis,
+static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t axis, Value guard,
                                                       uint32_t depth = 0) {
-	value = value.Resolve();
+	value = ResolveActiveU32(value, guard);
 	if (depth > 32 || value.GetType() != Type::U32) {
 		return {};
 	}
@@ -693,8 +804,8 @@ static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t ax
 	    op != ValueOpcode::ShiftLeftLogical32) {
 		return {};
 	}
-	auto left  = FillIndex(inst->Arg(0), axis, depth + 1);
-	auto right = FillIndex(inst->Arg(1), axis, depth + 1);
+	auto left  = FillIndex(inst->Arg(0), axis, guard, depth + 1);
+	auto right = FillIndex(inst->Arg(1), axis, guard, depth + 1);
 	if (!left || !right) {
 		return {};
 	}
@@ -750,6 +861,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 	UniformFillPlan result;
 	result.fill.resource = memory.resource;
 	Value data;
+	Value guard(true);
 	if (store->GetOpcode() == ValueOpcode::ImageWrite) {
 		if (program.info.images.size() != 1 || memory.dmask != 1 || memory.data_bits != 32 ||
 		    memory.image_has_mip || memory.image_sample_flags != 0 || memory.image_r128 ||
@@ -760,7 +872,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 		const auto* address = store->Arg(1).ResolveInstruction();
 		if (address == nullptr || address->GetOpcode() != ValueOpcode::MakeImageAddress) return {};
 		for (uint32_t axis = 0; axis < 3; ++axis) {
-			const auto index = FillIndex(address->Arg(axis), axis);
+			const auto index = FillIndex(address->Arg(axis), axis, guard);
 			if (!index || (*index)[0] != 0) return {};
 			if (axis < 2) {
 				if ((*index)[1] != 1 || (*index)[2] == 0) return {};
@@ -782,20 +894,22 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 		                             ValueOpcode::StoreBufferU32x3, ValueOpcode::StoreBufferU32x4};
 		const auto           store_op = std::ranges::find(stores, op);
 		if (store_op == stores.end() || store->Arg(2).Resolve() != Value(0u) ||
-		    store->Arg(3).Resolve() != Value(0u) || store->Arg(5).Resolve() != Value(true))
+		    store->Arg(3).Resolve() != Value(0u))
 			return {};
+		guard = store->Arg(5).Resolve();
+		if (!IsFullDispatchPredicate(guard)) return {};
 		if (!memory.formatted || memory.typed || !memory.idxen || memory.offen || memory.offset != 0 ||
 		    memory.data_bits != 32 ||
 		    memory.data_dwords != static_cast<uint32_t>(store_op - stores.begin() + 1))
 			return {};
-		const auto address = FillIndex(store->Arg(1), 0);
+		const auto address = FillIndex(store->Arg(1), 0, guard);
 		if (!address || (*address)[0] != 0 || (*address)[1] != 1 || (*address)[2] == 0) return {};
 		result.fill.kind = UniformFillKind::Buffer;
 		result.fill.group_stride[0] = static_cast<uint32_t>((*address)[2]);
 		result.fill.words = memory.data_dwords;
 		data = store->Arg(4);
 	}
-	data = data.Resolve();
+	data = ResolveActiveU32(data, guard);
 	const auto*          vector = data.TryInstruction();
 	constexpr std::array composites {ValueOpcode::CompositeConstructU32x2,
 	                                 ValueOpcode::CompositeConstructU32x3,
@@ -804,7 +918,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 	    (vector == nullptr || vector->GetOpcode() != composites[result.fill.words - 2]))
 		return {};
 	for (uint32_t i = 0; i < result.fill.words; ++i) {
-		const auto word = result.fill.words == 1 ? data : vector->Arg(i);
+		const auto word = ResolveActiveU32(result.fill.words == 1 ? data : vector->Arg(i), guard);
 		if (word.GetType() != Type::U32 ||
 		    !ValidateRuntimeValue(program, word, RuntimeValueType::Integer))
 			return {};
@@ -831,6 +945,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		if (source == nullptr) {
 			return value;
 		}
+		if (source->GetOpcode() == ValueOpcode::ReadFirstLane &&
+		    ValidateRuntimeValue(program, source->Arg(0))) {
+			// Uniform values need no new EXEC context; preserve their shared evaluation memo.
+			return Clone(source->Arg(0));
+		}
 		if (source->GetOpcode() == ValueOpcode::Phi) {
 			const auto invariant = ResolveInvariantPhi(program, value);
 			if (!invariant.IsEmpty() && invariant != value) {
@@ -840,6 +959,10 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		if (const auto found = cloned.find(source); found != cloned.end()) {
 			return Value(found->second);
 		}
+		if (source->GetOpcode() == ValueOpcode::LoadBufferU32) {
+			plan.requires_specialization_memory = true;
+			plan.capture_specialization_reads   = true;
+		}
 		auto& target =
 		    plan.value_storage.emplace_back(source->GetOpcode(), source->Flags<uint64_t>());
 		cloned.emplace(source, &target);
@@ -859,10 +982,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	for (const auto& source: program.descriptor_sources) {
 		auto& target          = plan.descriptor_sources.emplace_back();
 		target.dword_count    = source.dword_count;
-		target.indirect_image = source.indirect_image;
-		if (target.indirect_image.has_value()) {
-			target.indirect_image->key_count = Clone(target.indirect_image->key_count);
-			target.indirect_image->selector_mask = Clone(target.indirect_image->selector_mask);
+		target.indirect_descriptor = source.indirect_descriptor;
+		if (target.indirect_descriptor.has_value()) {
+			target.indirect_descriptor->key_count = Clone(target.indirect_descriptor->key_count);
+			target.indirect_descriptor->selector_first =
+			    Clone(target.indirect_descriptor->selector_first);
+			target.indirect_descriptor->selector_mask =
+			    Clone(target.indirect_descriptor->selector_mask);
 		}
 		for (uint32_t dword = 0; dword < source.dword_count; dword++) {
 			target.dwords[dword] = Clone(source.dwords[dword]);
@@ -909,35 +1035,36 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
 	}
 	plan.clean_flat_slots.resize(plan.srt_reads.size());
-	bool capture_image_reads = false;
-	for (const auto& image: plan.info.images) {
-		const auto* source = Source(plan, image.source);
-		if (source == nullptr || !source->indirect_image.has_value()) {
-			continue;
-		}
+	bool capture_indirect_reads = false;
+	for (const auto& source: plan.descriptor_sources) {
+		if (!source.indirect_descriptor.has_value()) continue;
+		const auto& indirect                = *source.indirect_descriptor;
 		plan.requires_specialization_memory = true;
-		capture_image_reads |= !source->indirect_image->selector_mask.IsEmpty() ||
-		                       !source->indirect_image->sources.empty();
-		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
-		                   plan.clean_flat_slots, source->indirect_image->selector_mask);
-		if (source->indirect_image->sources.empty()) {
-			MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
-			                   plan.clean_flat_slots);
+		capture_indirect_reads = true;
+		if (indirect.selector)
+			MarkCleanFlatSlots(plan, Source(plan, indirect.selector->source), plan.clean_flat_slots);
+		MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_mask);
+		MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_first);
+		MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.key_count);
+		if (indirect.sources.empty()) {
+			MarkCleanFlatSlots(plan, Source(plan, indirect.table_source), plan.clean_flat_slots);
 		} else {
-			for (const auto candidate: source->indirect_image->sources) {
+			for (const auto candidate: indirect.sources) {
 				MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots);
 			}
 		}
 	}
-	plan.capture_specialization_reads = capture_image_reads || !plan.control_flow.empty();
-	// Writable descriptor addresses must come from clean backing for the alias proof.
-	for (const auto& buffer: plan.info.buffers) {
-		if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
+	plan.capture_specialization_reads |= capture_indirect_reads || !plan.control_flow.empty();
+	if (plan.capture_specialization_reads) {
+		// Clean writable addresses prove that shader writes cannot overlap captured reads.
+		for (const auto& buffer: plan.info.buffers) {
+			if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
+		}
+		for (const auto& image: plan.info.images) {
+			if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
+		}
 	}
-	for (const auto& image: plan.info.images) {
-		if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
-	}
-	if (capture_image_reads) plan.resource_tracking_complete &= !program.has_address_writes;
+	if (capture_indirect_reads) plan.resource_tracking_complete &= !program.has_address_writes;
 	return plan;
 }
 
@@ -959,7 +1086,8 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		observed.read_memory = CaptureOrdinaryRead;
 	}
 	SrtWalker clean(program, CleanRuntime(observed));
-	SrtWalker walker(program, observed, program.clean_flat_slots, &clean);
+	SrtWalker walker(program, observed, program.clean_flat_slots,
+	                 capture_reads || program.requires_specialization_memory ? &clean : nullptr);
 	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
 		return false;
 	}
@@ -981,19 +1109,34 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			return false;
 		}
 		if (active.empty() || active[source]) {
-			return (written ? clean : walker).EvaluateDescriptor(source, value);
+			return (capture_reads && written ? clean : walker).EvaluateDescriptor(source, value);
 		}
 		value = {};
 		value.dword_count = program.descriptor_sources[source].dword_count;
 		return true;
 	};
 	snapshot.buffers.resize(program.info.buffers.size());
-	specialization.buffers.clear();
-	specialization.buffers.reserve(program.info.buffers.size());
-	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i],
-		              program.info.buffers[i].written)) {
-			return false;
+	specialization.buffers.assign(program.info.buffers.size(), {});
+	for (uint32_t i = 0; i < snapshot.buffers.size(); ++i) {
+		const auto base_index =
+		    i < program.info.buffers.size() ? i : specialization.buffers[i].indirect_root;
+		if (base_index >= program.info.buffers.size()) return false;
+		const auto& base = program.info.buffers[base_index];
+		if (i < program.info.buffers.size()) {
+			const auto* source = Source(program, base.source);
+			if (source == nullptr) return false;
+			if (source->indirect_descriptor.has_value()) {
+				snapshot.buffers[i] = {.dword_count = 4u};
+				if (active.empty() || active[base.source]) {
+					if (!MaterializeIndirectDescriptor(
+					        program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot,
+					        snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers,
+					        NormalizeIndirectStoreBuffer))
+						return false;
+				}
+			} else if (!evaluate(base.source, snapshot.buffers[i], base.written)) {
+				return false;
+			}
 		}
 		auto&                descriptor_value = snapshot.buffers[i];
 		ShaderBufferResource descriptor;
@@ -1012,15 +1155,12 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		} else if (!swizzle) {
 			packed_stride &= ~(3u << 16u);
 		}
-		specialization.buffers.push_back({
-		    .packed_stride     = packed_stride,
-		    .descriptor_format = program.info.buffers[i].formatted
-		                             ? descriptor.Format()
-		                             : Prospero::BufferFormat::kInvalid,
-		    .descriptor_swizzle =
-		        program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
-		    .zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u,
-		});
+		auto& buffer         = specialization.buffers[i];
+		buffer.packed_stride = packed_stride;
+		buffer.descriptor_format =
+		    base.formatted ? descriptor.Format() : Prospero::BufferFormat::kInvalid;
+		buffer.descriptor_swizzle = base.formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7);
+		buffer.zero_stride_oob    = descriptor.OutOfBounds() == 0u && stride == 0u;
 	}
 	snapshot.images.resize(program.info.images.size());
 	specialization.images.resize(program.info.images.size());
@@ -1041,13 +1181,22 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		if (source == nullptr) {
 			return false;
 		}
-		if (source->indirect_image.has_value()) {
+		if (source->indirect_descriptor.has_value()) {
 			snapshot.images[i] = {.dword_count = 8u};
 			if (!active.empty() && !active[image.source]) {
 				continue;
 			}
-			const auto& indirect = *source->indirect_image;
-			if (!MaterializeIndirectImage(program, indirect, i, observed, clean, snapshot, specialization)) {
+			const auto& indirect = *source->indirect_descriptor;
+			const bool bounded_table = IsBoundedDescriptorTable(program, indirect);
+			if (!MaterializeIndirectDescriptor(
+			        program, indirect, i, 8u, observed, clean, snapshot, snapshot.images,
+			        specialization.images, UINT32_MAX, [&](DescriptorValue& value) {
+				        // A broad heap also contains resources for other typed image operations.
+				        if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128) ||
+				            (bounded_table && DescriptorDimension(value, image.dimension) != image.dimension))
+					        value.dwords.fill(0u);
+				        return true;
+			        })) {
 				return false;
 			}
 		} else {
@@ -1082,14 +1231,33 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
 	EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete ||
 	        program.binding_layout_complete);
-	EXIT_IF(program.info.buffers.size() != specialization.buffers.size() ||
+	EXIT_IF(program.info.buffers.size() > specialization.buffers.size() ||
 	        program.info.images.size() > specialization.images.size());
 
 	auto& buffers = program.info.buffers;
-	for (size_t index = 0; index < buffers.size(); index++) {
-		buffers[index].packed_stride      = specialization.buffers[index].packed_stride;
-		buffers[index].descriptor_format  = specialization.buffers[index].descriptor_format;
-		buffers[index].descriptor_swizzle = specialization.buffers[index].descriptor_swizzle;
+	const auto original_buffer_count = buffers.size();
+	buffers.reserve(specialization.buffers.size());
+	for (uint32_t index = 0; index < specialization.buffers.size(); ++index) {
+		const auto& source = specialization.buffers[index];
+		if (index >= buffers.size()) {
+			EXIT_IF(source.indirect_root >= original_buffer_count);
+			buffers.push_back(buffers[source.indirect_root]);
+		}
+		auto& buffer                      = buffers[index];
+		buffer.packed_stride              = source.packed_stride;
+		buffer.descriptor_format          = source.descriptor_format;
+		buffer.descriptor_swizzle         = source.descriptor_swizzle;
+		buffer.indirect_root              = source.indirect_root;
+		buffer.indirect_mapping_offset    = source.indirect_mapping_offset;
+		buffer.indirect_search_iterations = source.indirect_search_iterations;
+		buffer.indirect_resources.clear();
+	}
+	for (uint32_t index = 0; index < buffers.size(); ++index) {
+		const auto root = buffers[index].indirect_root;
+		if (root != BufferResource::NoIndirectBuffer) {
+			EXIT_IF(root >= buffers.size());
+			buffers[root].indirect_resources.push_back(index);
+		}
 	}
 	auto& images = program.info.images;
 	const auto original_image_count = images.size();

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h

@@ -13,6 +13,9 @@ struct ResourceSpecialization {
 		Prospero::BufferFormat descriptor_format               = Prospero::BufferFormat::kInvalid;
 		uint32_t               descriptor_swizzle              = DstSel(4, 5, 6, 7);
 		bool                   zero_stride_oob                 = false;
+		uint32_t               indirect_root                   = BufferResource::NoIndirectBuffer;
+		uint32_t               indirect_mapping_offset         = 0;
+		uint32_t               indirect_search_iterations      = 0;
 		bool                   operator==(const Buffer&) const = default;
 	};
 

src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp

@@ -327,7 +327,7 @@ void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
 
 void CollectOutputs(const Program& program, ShaderStageInputInfo input_info, ShaderInfo& info) {
 	const bool alpha_remap = program.stage == ShaderType::Pixel && input_info.pixel != nullptr &&
-	                         input_info.pixel->alpha_blend_source_remap;
+	                         input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None;
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() != ValueOpcode::SetAttribute) {

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -44,16 +44,29 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
 
 bool IsRawRead(const ResourcePlan& values, const Inst& inst) {
 	const auto op = inst.GetOpcode();
-	if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) {
+	if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer &&
+	    op != ValueOpcode::LoadBufferU32) {
 		return false;
 	}
 	const auto index = inst.Flags<MemoryFlags>().index;
 	if (index >= values.memory_info.size()) {
 		return false;
 	}
-	const auto kind = values.memory_info[index].kind;
-	return (op == ValueOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) ||
-	       (op == ValueOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer);
+	const auto& memory = values.memory_info[index];
+	if (op != ValueOpcode::LoadBufferU32) {
+		return (op == ValueOpcode::LoadAddressU32 && memory.kind == ResourceKind::ScalarAddress) ||
+		       (op == ValueOpcode::ReadConstBuffer && memory.kind == ResourceKind::ScalarBuffer);
+	}
+	if (inst.NumArgs() != 5u || memory.kind != ResourceKind::Buffer || memory.typed ||
+	    memory.formatted || memory.coherent || memory.data_bits != 32u ||
+	    memory.data_dwords != 1u || memory.idxen || memory.offen || memory.offset != 0u ||
+	    inst.Arg(4).GetType() != Type::U1) {
+		return false;
+	}
+	for (uint32_t arg = 1; arg <= 3; ++arg) {
+		if (inst.Arg(arg).Resolve() != Value(0u)) return false;
+	}
+	return true;
 }
 
 bool IsDescriptorHandle(ValueOpcode opcode) {
@@ -92,6 +105,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::ISub64:
 		case ValueOpcode::IMul32:
 		case ValueOpcode::IMul64:
+		case ValueOpcode::UMulHi:
 		case ValueOpcode::UMin32:
 		case ValueOpcode::ShiftLeftLogical32:
 		case ValueOpcode::ShiftLeftLogical64:
@@ -108,6 +122,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::SelectU32:
 		case ValueOpcode::SelectF32:
 		case ValueOpcode::ULessThan32:
+		case ValueOpcode::ULessThanEqual32:
 		case ValueOpcode::IEqual32:
 		case ValueOpcode::UGreaterThan32:
 		case ValueOpcode::SGreaterThanEqual32:
@@ -120,6 +135,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::FPOrdGreaterThanEqual32:
 		case ValueOpcode::FPIsNan32:
 		case ValueOpcode::FPMul32:
+		case ValueOpcode::FPRecipIFlag32:
 		case ValueOpcode::FPTrunc32: return true;
 		default: return false;
 	}
@@ -160,6 +176,7 @@ private:
 				default: return false;
 			}
 		}
+		if (require_uniform && !m_active_mask.IsEmpty() && value == m_active_mask) return true;
 		// Integer-only dependency checks do not depend on the active EXEC mask.
 		if (!require_uniform && m_validated_dependencies.contains(inst)) return true;
 		if (!m_visiting.insert(inst).second) {
@@ -249,7 +266,8 @@ private:
 			}
 			return finish(true);
 		}
-		if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
+		if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer ||
+		    op == ValueOpcode::LoadBufferU32) {
 			const auto  expected = op == ValueOpcode::LoadAddressU32
 			                           ? ValueOpcode::GetAddressResource
 			                           : ValueOpcode::GetBufferResource;
@@ -258,6 +276,11 @@ private:
 			    handle->GetOpcode() != expected) {
 				return finish(false);
 			}
+			if (op == ValueOpcode::LoadBufferU32) {
+				const auto guard = inst->Arg(4).Resolve();
+				return finish(Validate(inst->Arg(0)) &&
+				              ((!m_active_mask.IsEmpty() && guard == m_active_mask) || Validate(guard)));
+			}
 		} else if (op == ValueOpcode::CompositeExtractU64) {
 			const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
 			if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) {
@@ -344,6 +367,10 @@ bool SrtWalker::EvaluateWide(Value value, uint64_t& result) {
 			default: return false;
 		}
 	}
+	if (!m_active_mask.IsEmpty() && value == m_active_mask) {
+		result = 1u;
+		return true;
+	}
 	auto* inst = value.TryInstruction();
 	if (inst == nullptr) {
 		return false;
@@ -433,6 +460,16 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 		return false;
 	}
 	const auto& mem    = m_program.memory_info[flags.index];
+	const bool vector = inst.GetOpcode() == ValueOpcode::LoadBufferU32;
+	const auto guard = vector ? inst.Arg(4).Resolve() : Value {};
+	if (vector && (guard.IsImmediate() || guard != m_active_mask)) {
+		uint64_t enabled = 0;
+		if (!Arg(inst, 4, enabled)) return false;
+		if (enabled == 0u) {
+			result = 0u;
+			return true;
+		}
+	}
 	const auto* handle = inst.Arg(0).ResolveInstruction();
 	if (handle == nullptr) {
 		return false;
@@ -446,7 +483,7 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 	const auto base      = ((high << 32u) | static_cast<uint32_t>(low)) & AddressMask;
 	const auto immediate = static_cast<int64_t>(static_cast<int32_t>(mem.offset));
 	uint64_t   address   = 0;
-	if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
+	if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer || vector) {
 		uint64_t records = 0;
 		uint64_t word3   = 0;
 		if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) {
@@ -458,6 +495,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 		const auto byte_offset =
 		    (static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
 		const auto stride  = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
+		if (vector) {
+			// Only the uniform, unswizzled structured DWORD address is evaluated on the host.
+			if ((high & (1u << 31u)) != 0u || (word3 & ((1u << 23u) | 0xf0000000u)) != 0u)
+				return false;
+			if (stride == 0u || records == 0u || ((word3 >> 12u) & 0x7fu) == 0u) {
+				result = 0u;
+				return true;
+			}
+		}
 		const auto size = stride == 0u
 		                      ? static_cast<uint64_t>(static_cast<uint32_t>(records))
 		                      : static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
@@ -473,11 +519,13 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 		}
 	}
 	uint32_t word = 0;
-	if (m_runtime.read_memory != nullptr) {
-		if (!m_runtime.read_memory(m_runtime.userdata, address, {&word, 1})) {
+	const auto reader = vector ? m_runtime.read_specialization_memory : m_runtime.read_memory;
+	if (reader != nullptr) {
+		if (!reader(m_runtime.userdata, address, {&word, 1})) {
 			return false;
 		}
 	} else {
+		if (vector) return false;
 		std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
 	}
 	result = word;
@@ -505,6 +553,9 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 		case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
 		case ValueOpcode::Phi: return EvaluatePhi(inst, result);
 		case ValueOpcode::ReadFirstLane: {
+			if (!m_active_mask.IsEmpty() && inst.Arg(1).Resolve() == m_active_mask) {
+				return EvaluateWide(inst.Arg(0), result);
+			}
 			const auto clean_runtime = CleanRuntime(m_runtime);
 			SrtWalker  clean_active(m_program, clean_runtime, {}, nullptr, inst.Arg(1));
 			SrtWalker  active(m_program, m_runtime, m_clean_flat_slots, &clean_active,
@@ -537,7 +588,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 		}
 		case ValueOpcode::LoadAddressU32:
 		case ValueOpcode::ReadConstBuffer:
+		case ValueOpcode::LoadBufferU32:
 			if (IsRawRead(m_program, inst)) {
+				if (inst.GetOpcode() == ValueOpcode::LoadBufferU32 && m_clean_evaluator != nullptr &&
+				    m_clean_evaluator->m_active_mask == m_active_mask) {
+					return m_clean_evaluator->EvaluateWide(Value(const_cast<Inst*>(&inst)), result);
+				}
 				return EvaluateRawRead(inst, result);
 			}
 			break;
@@ -577,6 +633,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::UMulHi:
+			if (binary()) {
+				result = (static_cast<uint64_t>(static_cast<uint32_t>(a)) * static_cast<uint32_t>(b)) >> 32u;
+				return true;
+			}
+			return false;
 		case ValueOpcode::UMin32:
 			if (binary()) {
 				result = std::min(static_cast<uint32_t>(a), static_cast<uint32_t>(b));
@@ -612,6 +674,15 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::FPRecipIFlag32:
+			if (Arg(inst, 0, a)) {
+				const auto exponent = (a >> 23u) & 0xffu;
+				// Normal positive powers of two have exact normal reciprocals in every FP mode.
+				if ((a & 0x807fffffu) != 0u || exponent == 0u || exponent >= 254u) return false;
+				result = (254u - exponent) << 23u;
+				return true;
+			}
+			return false;
 		case ValueOpcode::FPIsNan32:
 			if (Arg(inst, 0, a)) {
 				result = std::isnan(Float32(a));
@@ -782,6 +853,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::ULessThanEqual32:
+			if (binary()) {
+				result = static_cast<uint32_t>(a) <= static_cast<uint32_t>(b);
+				return true;
+			}
+			return false;
 		case ValueOpcode::UGreaterThan32:
 			if (binary()) {
 				result = static_cast<uint32_t>(a) > static_cast<uint32_t>(b);

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -17,6 +17,7 @@ struct SrtRuntime {
 	SrtMemoryReader           read_memory                = nullptr;
 	void*                     userdata                   = nullptr;
 	SrtMemoryReader           read_specialization_memory = nullptr;
+	std::span<const uint32_t> workgroup_counts;
 };
 
 enum class RuntimeValueType { Any, Integer };

src/graphics/shader/shader.cpp

@@ -571,9 +571,11 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
                                        const ShaderMappedData& data, ShaderComputeInputInfo& info) {
 	const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions;
 	const auto host_subgroup_size         = info.host_subgroup_size;
+	const auto workgroup_counts           = std::to_array(info.workgroup_counts);
 	info                                  = {};
 	info.dispatch_thread_dimensions       = dispatch_thread_dimensions;
 	info.host_subgroup_size               = host_subgroup_size;
+	std::ranges::copy(workgroup_counts, info.workgroup_counts);
 	info.threads_num[0]                   = regs.cs_regs.num_thread_x;
 	info.threads_num[1]                   = regs.cs_regs.num_thread_y;
 	info.threads_num[2]                   = regs.cs_regs.num_thread_z;
@@ -660,7 +662,7 @@ void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector<uint32_t>
 	key.push_back(static_cast<uint32_t>(info.ps_sample_mask_export_enable));
 	key.push_back(static_cast<uint32_t>(info.ps_early_z));
 	key.push_back(static_cast<uint32_t>(info.dual_source_blending));
-	key.push_back(static_cast<uint32_t>(info.alpha_blend_source_remap));
+	key.push_back(static_cast<uint32_t>(info.alpha_blend_source));
 	key.insert(key.end(), std::begin(info.target_output_mode), std::end(info.target_output_mode));
 	for (uint32_t base = 0; base < info.target_export_mapping.size(); base += 4u) {
 		uint32_t packed = 0;

src/graphics/shader/shader.h

@@ -148,14 +148,18 @@ struct ShaderVertexInputInfo {
 struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo {
 	uint8_t            float_mode                 = 0xc0;
 	uint32_t           dispatch_threads_num[3]    = {0, 0, 0};
+	uint32_t           workgroup_counts[3]        = {0, 0, 0};
 	bool               group_id[3]                = {false, false, false};
 	bool               dispatch_thread_dimensions = false;
+	bool               lds_storage                = false;
 	int                thread_ids_num             = 0;
 	int                workgroup_register         = 0;
 	bool               tg_size_en                 = false;
 	ShaderStageRuntime stage;
 };
 
+enum class ShaderAlphaBlendSource : uint8_t { None, SourceAlpha, SourceAlphaOne, SourceAlphaZero };
+
 struct ShaderPixelInputInfo {
 	uint32_t                                       interpolator_settings[32]    = {0};
 	uint32_t                                       input_num                    = 0;
@@ -179,8 +183,8 @@ struct ShaderPixelInputInfo {
 	bool                                           ps_sample_mask_export_enable = false;
 	bool                                           ps_sample_shading            = false;
 	bool                                           dual_source_blending         = false;
-	// Export logical alpha through MRT1 for blending after channel swizzling.
-	bool                                           alpha_blend_source_remap     = false;
+	// Export logical alpha or per-channel source factors through MRT1 after channel swizzling.
+	ShaderAlphaBlendSource                         alpha_blend_source = ShaderAlphaBlendSource::None;
 	bool                                           ps_early_z                   = false;
 	bool                                           ps_execute_on_noop           = false;
 	ShaderStageRuntime                             stage;

src/kernel/eventQueue.cpp

@@ -121,6 +121,9 @@ int KernelEqueuePrivate::GetTriggeredEvents(KernelEvent* ev, int num) {
 				event.event = event.pending_events.front();
 				event.pending_events.pop_front();
 				event.triggered = true;
+			} else {
+				// A persistent level is delivered again by the next wait, not this batch.
+				break;
 			}
 
 			if (ret >= num) {
@@ -564,7 +567,7 @@ int KYTY_SYSV_ABI KernelAddAmprEvent(KernelEqueue eq, int id, void* udata) {
 	if (eq != KERNEL_EQUEUE_INVALID) {
 		KernelEqueueEvent event {};
 		event.event.ident         = static_cast<uintptr_t>(id);
-		event.event.filter        = KERNEL_EVFILT_USER;
+		event.event.filter        = KERNEL_EVFILT_AMPR;
 		event.event.flags         = EV_ADD | EV_CLEAR;
 		event.event.fflags        = 0;
 		event.event.data          = 0;
@@ -592,7 +595,7 @@ int KYTY_SYSV_ABI KernelDeleteAmprEvent(KernelEqueue eq, int id) {
 	LOGF("\t AMPR event delete: eq = 0x%016" PRIx64 ", id = %d\n", static_cast<uint64_t>(eq), id);
 
 	if (eq != KERNEL_EQUEUE_INVALID) {
-		(void)KernelDeleteEvent(eq, static_cast<uintptr_t>(id), KERNEL_EVFILT_USER);
+		(void)KernelDeleteEvent(eq, static_cast<uintptr_t>(id), KERNEL_EVFILT_AMPR);
 	}
 
 	return OK;

src/kernel/eventQueue.h

@@ -18,6 +18,7 @@ constexpr int16_t KERNEL_EVFILT_FILE      = -4;
 constexpr int16_t KERNEL_EVFILT_GRAPHICS  = -14;
 constexpr int16_t KERNEL_EVFILT_VIDEO_OUT = -13;
 constexpr int16_t KERNEL_EVFILT_HRTIMER   = -15;
+constexpr int16_t KERNEL_EVFILT_AMPR      = -25;
 
 class KernelEqueuePrivate;
 struct KernelEqueueEvent;

src/kernel/memory.cpp

@@ -166,7 +166,8 @@ static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64
 }
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
-static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX;
+static uint32_t        g_test_backing_store_unmaps_before_failure = UINT32_MAX;
+static callback_func_t g_test_before_backing_map                   = nullptr;
 #endif
 
 #include "memoryAddressSpace.inc"
@@ -880,13 +881,15 @@ bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) {
 	       g_guest_address_space->TryReadBacking(vaddr, data, size);
 }
 
-bool TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size) {
+bool TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size) {
 	if (g_gpu_resources != nullptr && IsGpuAddressRange(vaddr, size)) {
-		if (!Graphics::GuestGpu::IsGpuThread() ||
-		    GetGpuResources().GetBufferCache().HasGpuDirtyBytes(vaddr, size) ||
-		    GetGpuResources().GetTextureCache().IsRegionGpuModified(vaddr, size)) {
+		if (!Graphics::GuestGpu::IsGpuThread()) {
 			return false;
 		}
+		auto& buffers = GetGpuResources().GetBufferCache();
+		if (buffers.HasGpuDirtyBytes(vaddr, size)) {
+			buffers.ReadMemory(vaddr, size);
+		}
 	}
 	return TryReadBacking(vaddr, data, size);
 }
@@ -3024,6 +3027,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 		return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode,
 		                                         &shared_failure);
 	};
+	std::vector<PhysicalMemory::AllocatedBlock> replaced_mappings;
+	auto restore_mappings = [&](size_t count, bool restore_views) {
+		for (size_t index = 0; index < count; ++index) {
+			const auto& old = replaced_mappings[index];
+			if (restore_views) {
+				EXIT_IF(!g_guest_address_space->MapBacking(old.map_vaddr, old.map_size,
+				                                           old.start_addr, old.mode));
+			}
+			EXIT_IF(!g_physical_memory->Map(old.map_vaddr, old.start_addr, old.map_size,
+			                               old.prot, old.mode, old.gpu_mode));
+			g_physical_memory->SetVirtualRangeName(old.map_vaddr, old.map_size, old.name);
+			g_physical_memory->SetVirtualRangeMemoryType(old.map_vaddr, old.map_size,
+			                                          old.memory_type);
+		}
+		MapGpuRange(in_addr, len);
+	};
 	if (fixed) {
 		if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1u)) != 0 ||
 		    (alignment != 0 && in_addr % alignment != 0)) {
@@ -3034,22 +3053,68 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 		}
 
 		std::vector<VirtualRanges::Range> reserved_ranges;
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 		if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) &&
 		    std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) {
-			    return range.type == VirtualRangeType::Reserved;
+			    return range.type == VirtualRangeType::Direct;
 		    })) {
+			uint64_t covered = 0;
+			{
+				Common::LockGuard lock(g_physical_memory->GetMutex());
+				for (const auto& mapping: g_physical_memory->GetMappings()) {
+					const auto start = std::max(in_addr, mapping.map_vaddr);
+					const auto end = std::min(in_addr + len, mapping.map_vaddr + mapping.map_size);
+					if (start >= end) {
+						continue;
+					}
+					auto old = mapping;
+					old.start_addr += start - old.map_vaddr;
+					old.map_vaddr = start;
+					old.size = old.map_size = end - start;
+					covered += old.map_size;
+					replaced_mappings.push_back(old);
+				}
+			}
+			if (covered != len) {
+				return KERNEL_ERROR_EBUSY;
+			}
 			UnmapGpuRange(in_addr, len);
-			reserved_target = true;
-		}
-		if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
-			reserved_target = true;
-		}
-		if (!reserved_target) {
-			return KERNEL_ERROR_ENOMEM;
-		}
-		if (map_shared_fixed(in_addr)) {
+			for (size_t index = 0; index < replaced_mappings.size(); ++index) {
+				const auto& old = replaced_mappings[index];
+				GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess;
+				if (!g_physical_memory->Unmap(old.map_vaddr, old.map_size, &old_gpu_mode)) {
+					restore_mappings(index, false);
+					return KERNEL_ERROR_EBUSY;
+				}
+			}
+			// Keep the old view accessible until mmap atomically replaces it. In particular,
+			// do not publish a temporary reserved/PROT_NONE range between two direct views.
+			if (!map_shared_fixed(in_addr)) {
+				restore_mappings(replaced_mappings.size(), true);
+				return KERNEL_ERROR_ENOMEM;
+			}
 			out_addr       = in_addr;
 			shared_backing = true;
+		} else
+#endif
+		{
+			if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) &&
+			    std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) {
+				    return range.type == VirtualRangeType::Reserved;
+			    })) {
+				UnmapGpuRange(in_addr, len);
+				reserved_target = true;
+			}
+			if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
+				reserved_target = true;
+			}
+			if (!reserved_target) {
+				return KERNEL_ERROR_ENOMEM;
+			}
+			if (map_shared_fixed(in_addr)) {
+				out_addr       = in_addr;
+				shared_backing = true;
+			}
 		}
 	} else {
 		constexpr size_t DEFAULT_ALIGNMENT = 0x4000;
@@ -3105,15 +3170,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 
 	if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) {
 		LOGF_COLOR(Log::Color::Red, "\t [Fail]\n");
-		EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		if (replaced_mappings.empty()) {
+			EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		} else {
+			restore_mappings(replaced_mappings.size(), true);
+		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
 	PhysicalMemory::AllocatedBlock mapped_block {};
 	g_physical_memory->Find(direct_memory_start, false, &mapped_block);
-	const bool published = reserved_target
+	const bool published = !replaced_mappings.empty() || reserved_target
 	                           ? g_virtual_ranges->ReplaceSpan(
-	                                 out_addr, len, VirtualRangeType::Reserved, direct_memory_start,
+	                                 out_addr, len, replaced_mappings.empty()
+	                                                    ? VirtualRangeType::Reserved
+	                                                    : VirtualRangeType::Direct,
+	                                 direct_memory_start,
 	                                 prot, mapped_block.memory_type, VirtualRangeType::Direct, "")
 	                           : g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot,
 	                                                   mapped_block.memory_type,
@@ -3121,7 +3193,11 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 	if (!published) {
 		GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess;
 		EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode));
-		EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		if (replaced_mappings.empty()) {
+			EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		} else {
+			restore_mappings(replaced_mappings.size(), true);
+		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
@@ -3504,6 +3580,10 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags,
 }
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+void TestBeforeNextBackingMap(callback_func_t callback) {
+	g_test_before_backing_map = callback;
+}
+
 void TestFailNextPhysicalMemoryUnmap() {
 	TestFailPhysicalMemoryUnmapAfter(0);
 }

src/kernel/memory.h

@@ -114,7 +114,7 @@ int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size,
 int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
 bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
 bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
-bool                   TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size);
+bool                   TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size);
 bool                   TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size);
 [[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size);
 void                   WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
@@ -191,6 +191,7 @@ bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory
 bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+void     TestBeforeNextBackingMap(callback_func_t callback);
 void     TestFailNextPhysicalMemoryUnmap();
 void     TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
 void     TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);

src/kernel/memoryAddressSpace.inc

@@ -231,9 +231,8 @@ public:
 		return true;
 	}
 
-	bool MapExistingPlaceholderFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
-	                                 VirtualMemory::Mode mode,
-	                                 FailureReason*      failure_reason = nullptr) {
+	bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
+	              VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) {
 		if (!IsAvailable() || !RangeOk(backing_offset, size)) {
 			SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable
 			                                          : FailureReason::PhysicalRangeOutOfBounds);
@@ -257,14 +256,13 @@ public:
 		SetFailure(failure_reason, FailureReason::None);
 		return true;
 #else
+		std::lock_guard lock(m_mutex);
 		void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason);
 		if (mapped == nullptr || reinterpret_cast<uint64_t>(mapped) != vaddr) {
 			return false;
 		}
-		{
-			std::lock_guard lock(m_mutex);
-			EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second);
-		}
+		RemoveMappingsUnlocked(vaddr, size);
+		EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second);
 		SetFailure(failure_reason, FailureReason::None);
 		return true;
 #endif
@@ -302,8 +300,7 @@ public:
 			bool part_placeholder_preserved = false;
 			if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) {
 				for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
-					EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
-					                                     it->mode));
+					EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
 				}
 				if (placeholder_preserved != nullptr) {
 					*placeholder_preserved = false;
@@ -320,8 +317,7 @@ public:
 		// views into one placeholder before publishing it there.
 		if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) {
 			for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
-				EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
-				                                     it->mode));
+				EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
 			}
 			return false;
 		}
@@ -363,12 +359,10 @@ public:
 			return false;
 		}
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		const uint64_t left_size  = vaddr - old.vaddr;
 		const uint64_t right_addr = vaddr + size;
-		const uint64_t old_end    = old.vaddr + old.size;
-		const uint64_t right_size = old_end - right_addr;
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		const uint64_t right_size = old.vaddr + old.size - right_addr;
 		const bool exact_unmap = (vaddr == old.vaddr && size == old.size);
 		if (exact_unmap) {
 			const bool unmapped =
@@ -393,14 +387,11 @@ public:
 				ok = SplitPlaceholder(vaddr, size) && ok;
 			}
 			if (left_size != 0) {
-				ok = MapExistingPlaceholderFixed(old.vaddr, left_size, old.backing_offset,
-				                                 old.mode) &&
-				     ok;
+				ok = MapFixed(old.vaddr, left_size, old.backing_offset, old.mode) && ok;
 			}
 			if (right_size != 0) {
-				ok = MapExistingPlaceholderFixed(right_addr, right_size,
-				                                 old.backing_offset + (right_addr - old.vaddr),
-				                                 old.mode) &&
+				ok = MapFixed(right_addr, right_size,
+				              old.backing_offset + (right_addr - old.vaddr), old.mode) &&
 				     ok;
 			}
 			if (ok) {
@@ -418,8 +409,7 @@ public:
 			}
 			EXIT_IF(VirtualFree(reinterpret_cast<void*>(old.vaddr), old.size,
 			                    MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) == 0);
-			EXIT_IF(
-			    !MapExistingPlaceholderFixed(old.vaddr, old.size, old.backing_offset, old.mode));
+			EXIT_IF(!MapFixed(old.vaddr, old.size, old.backing_offset, old.mode));
 			return false;
 		}
 		std::lock_guard lock(m_mutex);
@@ -433,16 +423,7 @@ public:
 		}
 
 		std::lock_guard lock(m_mutex);
-		m_maps.erase(old.vaddr);
-		if (left_size != 0) {
-			m_maps.emplace(old.vaddr,
-			               MapEntry {old.vaddr, left_size, old.backing_offset, old.mode});
-		}
-		if (right_size != 0) {
-			m_maps.emplace(right_addr,
-			               MapEntry {right_addr, right_size,
-			                         old.backing_offset + right_addr - old.vaddr, old.mode});
-		}
+		RemoveMappingsUnlocked(vaddr, size);
 
 		if (placeholder_preserved != nullptr) {
 			*placeholder_preserved = true;
@@ -516,6 +497,33 @@ private:
 		VirtualMemory::Mode mode           = VirtualMemory::Mode::NoAccess;
 	};
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	void RemoveMappingsUnlocked(uint64_t vaddr, uint64_t size) {
+		const auto end = vaddr + size;
+		auto       it = m_maps.upper_bound(vaddr);
+		if (it != m_maps.begin()) {
+			--it;
+		}
+		while (it != m_maps.end() && it->first < end) {
+			const auto old     = it->second;
+			const auto old_end = old.vaddr + old.size;
+			if (old_end <= vaddr) {
+				++it;
+				continue;
+			}
+			it = m_maps.erase(it);
+			if (old.vaddr < vaddr) {
+				m_maps.emplace(old.vaddr,
+				               MapEntry {old.vaddr, vaddr - old.vaddr, old.backing_offset, old.mode});
+			}
+			if (end < old_end) {
+				m_maps.emplace(end, MapEntry {end, old_end - end,
+				                             old.backing_offset + end - old.vaddr, old.mode});
+			}
+		}
+	}
+#endif
+
 	bool FindContainingUnlocked(uint64_t vaddr, uint64_t size, MapEntry* out) const {
 		EXIT_IF(out == nullptr);
 
@@ -621,6 +629,12 @@ private:
 
 	void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode,
 	              FailureReason* failure_reason) {
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+		if (const auto callback = g_test_before_backing_map) {
+			g_test_before_backing_map = nullptr;
+			callback(vaddr, size);
+		}
+#endif
 		void* ptr = mmap(reinterpret_cast<void*>(vaddr), size, GetProtect(mode),
 		                 MAP_SHARED | MAP_FIXED, m_fd, static_cast<off_t>(backing_offset));
 		if (ptr == MAP_FAILED) {
@@ -778,7 +792,12 @@ public:
 	                VirtualMemory::Mode               mode,
 	                GuestBackingStore::FailureReason* failure_reason = nullptr) {
 		std::lock_guard lock(m_mutex);
-		if (!ConsumeUnlocked(vaddr, size)) {
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+		const bool replacing = ContainsMappedUnlocked(vaddr, size, MappingType::Backing);
+#else
+		constexpr bool replacing = false;
+#endif
+		if (!replacing && !ConsumeUnlocked(vaddr, size)) {
 			if (failure_reason != nullptr) {
 				*failure_reason = GuestBackingStore::FailureReason::AddressRangeUnavailable;
 			}
@@ -787,12 +806,15 @@ public:
 		const auto host_mode = VirtualMemory::IsExecute(mode)
 		                           ? VirtualMemory::Mode::ExecuteReadWrite
 		                           : VirtualMemory::Mode::ReadWrite;
-		if (m_backing->MapExistingPlaceholderFixed(vaddr, size, backing_offset, host_mode,
-		                                           failure_reason)) {
-			EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing));
+		if (m_backing->MapFixed(vaddr, size, backing_offset, host_mode, failure_reason)) {
+			if (!replacing) {
+				EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing));
+			}
 			return true;
 		}
-		AddFreeUnlocked(vaddr, size);
+		if (!replacing) {
+			AddFreeUnlocked(vaddr, size);
+		}
 		return false;
 	}
 

src/kernel/pthread.cpp

@@ -3200,6 +3200,11 @@ static void* RunThread(void* arg) {
 	os_thread_id = GetHostThreadId();
 #endif
 	thread->host_thread_id = os_thread_id;
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	if (!thread->name.empty()) {
+		pthread_setname_np(pthread_self(), thread->name.substr(0, 15).c_str());
+	}
+#endif
 
 	LOGF("\tPthread run begin: %s, id = %d, os_thread_id = %" PRIu64 ", entry = 0x%016" PRIx64
 	     ", arg = 0x%016" PRIx64 ", stack_addr = 0x%016" PRIx64 ", stack_size = %" PRIu64 "\n",