window: auto-hide the cursor over the game window (#958)

* window: auto-hide the cursor over the game window

Hide the cursor when the game window is created and whenever the mouse
has been idle for two seconds; any mouse motion, button or wheel event
shows it again. Games that draw their own cursor (e.g. Blue Prince) no
longer show the host pointer on top of it.

Only real mouse input counts as activity: synthetic touch/pen mouse
events and zero-delta motion (sent by some platforms on focus or cursor
visibility changes) are ignored. The event loop only waits with a
timeout while the cursor is visible, so it still blocks indefinitely
once the cursor is hidden. The cursor is restored on shutdown.

Uses SDL_HideCursor/SDL_ShowCursor, which SDL implements for X11,
Wayland, Cocoa and Win32 and which only affect the cursor over the
emulator window.

Closes #282

* window: make cursor auto-hide optional in CLI and launcher

Co-authored-by: Anatolii Kucheruk <anatolii.kucheruk01@gmail.com>

---------

Co-authored-by: nmzik <Nmzik@mail.ru>
Co-authored-by: Anatolii Kucheruk <anatolii.kucheruk01@gmail.com>

.github/workflows/build.yml

@@ -113,7 +113,7 @@ jobs:
         shell: cmd
         run: |
           cmake --build _Build/windows --target launcher ^
-            audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
+            audio_out2_port_tests pad_haptics_tests controller_settings_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
             shader_recompiler_compute_tests ^
             resource_tracking_tests scalar_provenance_tests ^
             resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests --parallel
@@ -121,7 +121,7 @@ jobs:
       - name: Test
         shell: cmd
         run: |
-          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
+          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|controller_settings|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
 
       - name: Install
         shell: cmd
@@ -202,7 +202,7 @@ jobs:
         shell: bash
         run: |
           cmake --build _Build/macos \
-            --target launcher audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
+            --target launcher audio_out2_port_tests pad_haptics_tests controller_settings_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
               memory_tracker_tests page_manager_tests shader_recompiler_compute_tests \
               resource_tracking_tests scalar_provenance_tests \
               resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
@@ -212,7 +212,7 @@ jobs:
         shell: bash
         run: |
           ctest --test-dir _Build/macos --output-on-failure \
-            -R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|pad_haptics|controller_settings|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash
@@ -386,7 +386,7 @@ jobs:
         run: |
           cmake --build _Build/linux \
             --target launcher page_manager_tests memory_tracker_tests \
-              audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
+              audio_out2_port_tests pad_haptics_tests controller_settings_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
               resource_tracking_tests scalar_provenance_tests \
               resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
             --parallel
@@ -395,7 +395,7 @@ jobs:
         shell: bash
         run: |
           ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|pad_haptics|controller_settings|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash

3rdparty/CMakeLists.txt

@@ -2,10 +2,31 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
 
 add_subdirectory(cpuinfo)
 
-# The Windows guest red-zone patcher needs a decoder/encoder and a small
-# trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 include(FetchContent)
 
+# The Bluetooth DualSense transports speaker audio as fixed-size Opus packets.
+set(kyty_build_testing_saved "${BUILD_TESTING}")
+set(BUILD_TESTING OFF)
+set(OPUS_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE)
+set(OPUS_INSTALL_PKG_CONFIG_MODULE OFF CACHE BOOL "" FORCE)
+set(OPUS_INSTALL_CMAKE_CONFIG_MODULE OFF CACHE BOOL "" FORCE)
+if(MSVC AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
+	# Opus assumes MSVC accepts SSE4.1 intrinsics without per-file flags, but
+	# clang-cl rejects them in its baseline x64 files. Its AVX2 dispatch also
+	# depends on the SSE4.1 declarations, so retain the x64 SSE2 baseline only.
+	set(OPUS_X86_MAY_HAVE_SSE4_1 OFF CACHE BOOL "" FORCE)
+	set(OPUS_X86_MAY_HAVE_AVX2 OFF CACHE BOOL "" FORCE)
+endif()
+FetchContent_Declare(opus
+	GIT_REPOSITORY https://github.com/xiph/opus.git
+	GIT_TAG ddbe48383984d56acd9e1ab6a090c54ca6b735a6
+	GIT_SHALLOW FALSE)
+FetchContent_MakeAvailable(opus)
+set(BUILD_TESTING "${kyty_build_testing_saved}")
+install(FILES "${opus_SOURCE_DIR}/COPYING" DESTINATION "licenses/opus")
+
+# The Windows guest red-zone patcher needs a decoder/encoder and a small
+# trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
 FetchContent_Declare(
 	xbyak
@@ -167,6 +188,7 @@ endif()
 
 set(TRACY_STATIC ON CACHE BOOL "" FORCE)
 set(TRACY_ENABLE ON CACHE BOOL "" FORCE)
+set(TRACY_NO_CRASH_HANDLER ON CACHE BOOL "" FORCE)
 set(TRACY_ON_DEMAND ON CACHE BOOL "" FORCE)
 set(TRACY_DELAYED_INIT ON CACHE BOOL "" FORCE)
 set(TRACY_MANUAL_LIFETIME ON CACHE BOOL "" FORCE)

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.18...4.3)
 
 if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
-	set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies")
+	set(CMAKE_POLICY_VERSION_MINIMUM 3.10 CACHE STRING "Minimum policy version for bundled dependencies")
 endif()
 
 get_property(KYTY_GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
@@ -332,7 +332,7 @@ if (KYTY_CLANG_CL)
 	set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED")
 endif()
 
-set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak)
+set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus)
 
 # Linux system libraries.
 if(LINUX)
@@ -481,6 +481,10 @@ add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
 target_link_libraries(audio_out2_port_tests common fmt::fmt)
 target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
 
+add_executable(audio_out_timing_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AudioOutTimingTests.cpp")
+target_link_libraries(audio_out_timing_tests common fmt::fmt SDL3::SDL3-static)
+target_include_directories(audio_out_timing_tests PRIVATE ${inc_headers})
+
 add_executable(ngs2_sampler_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/Ngs2SamplerTests.cpp"
 	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
@@ -491,15 +495,25 @@ target_include_directories(ngs2_sampler_tests PRIVATE ${inc_headers})
 add_executable(pad_haptics_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/PadHapticsTests.cpp"
 )
-target_link_libraries(pad_haptics_tests common fmt::fmt SDL3::SDL3-static)
+target_link_libraries(pad_haptics_tests common fmt::fmt SDL3::SDL3-static opus)
 target_include_directories(pad_haptics_tests PRIVATE ${inc_headers})
 
+add_executable(controller_settings_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/ControllerSettingsTests.cpp"
+)
+target_link_libraries(controller_settings_tests common fmt::fmt SDL3::SDL3-static)
+target_include_directories(controller_settings_tests PRIVATE ${inc_headers})
+
 add_executable(save_data_memory_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/SaveDataMemoryTests.cpp"
 )
 target_link_libraries(save_data_memory_tests common fmt::fmt)
 target_include_directories(save_data_memory_tests PRIVATE ${inc_headers})
 
+add_executable(png_enc_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/PngEncTests.cpp")
+target_link_libraries(png_enc_tests common fmt::fmt)
+target_include_directories(png_enc_tests PRIVATE ${inc_headers})
+
 add_executable(ces_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/CesTests.cpp")
 target_link_libraries(ces_tests common fmt::fmt SDL3::SDL3-static)
 target_include_directories(ces_tests PRIVATE ${inc_headers})
@@ -602,9 +616,12 @@ if(BUILD_TESTING)
 	add_test(NAME kernel_file_system COMMAND $<TARGET_FILE:kernel_file_system_tests>)
 	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
+	add_test(NAME audio_out_timing COMMAND $<TARGET_FILE:audio_out_timing_tests>)
 	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
 	add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
+	add_test(NAME controller_settings COMMAND $<TARGET_FILE:controller_settings_tests>)
 	add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
+	add_test(NAME png_enc COMMAND $<TARGET_FILE:png_enc_tests>)
 	add_test(NAME ces COMMAND $<TARGET_FILE:ces_tests>)
 	add_test(NAME save_data_dialog COMMAND $<TARGET_FILE:save_data_dialog_tests>)
 	add_test(NAME http_uri_parse COMMAND $<TARGET_FILE:http_uri_parse_tests>)
@@ -677,9 +694,12 @@ if(BUILD_TESTING)
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
 		audio_out2_port_tests
+		audio_out_timing_tests
 		ngs2_sampler_tests
 		pad_haptics_tests
+		controller_settings_tests
 		save_data_memory_tests
+		png_enc_tests
 		ces_tests
 		save_data_dialog_tests
 		http_uri_parse_tests
@@ -767,6 +787,10 @@ endif()
 install(TARGETS kyty_emulator DESTINATION .)
 if(KYTY_BUILD_LAUNCHER)
 	add_subdirectory("${KYTY_SOURCE_DIR}/launcher" "${CMAKE_CURRENT_BINARY_DIR}/launcher")
+
+	if(LINUX)
+		target_link_options(launcher PRIVATE $<$<CONFIG:Release>:-Wl,--strip-all>)
+	endif()
 endif()
 if(KYTY_CLANG_CL)
 	install(FILES "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" DESTINATION .)

src/common/emulatorConfig.cpp

@@ -23,6 +23,7 @@ void Load(const ConfigOptions& cfg) {
 	EXIT_IF(g_config == nullptr);
 	EXIT_IF(cfg.user_name.empty() || cfg.user_name.size() > MAX_USER_NAME_LENGTH);
 	EXIT_IF(!IsConfiguredUserIdValid(cfg.user_id));
+	EXIT_IF(cfg.controller_speaker_volume > 100 || cfg.controller_vibration_intensity > 100);
 
 	*g_config = cfg;
 }
@@ -47,6 +48,18 @@ const std::string& GetAudioInputDevice() {
 	return g_config->audio_input_device;
 }
 
+const std::optional<ControllerColor>& GetControllerColor() {
+	return g_config->controller_color;
+}
+
+uint32_t GetControllerSpeakerVolume() {
+	return g_config->controller_speaker_volume;
+}
+
+uint32_t GetControllerVibrationIntensity() {
+	return g_config->controller_vibration_intensity;
+}
+
 PresentMode GetPresentMode() {
 	return g_config->present_mode;
 }
@@ -59,6 +72,10 @@ bool FullscreenEnabled() {
 	return g_config->fullscreen_enabled;
 }
 
+bool HideCursorEnabled() {
+	return g_config->hide_cursor_enabled;
+}
+
 bool VrEnabled() {
 	return g_config->vr_enabled;
 }

src/common/emulatorConfig.h

@@ -4,7 +4,9 @@
 #include "common/common.h"
 
 #include <cstddef>
+#include <array>
 #include <filesystem>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -26,6 +28,7 @@ enum class LogDirection { Silent, Console, File };
 enum class PresentMode { Fifo, Mailbox, Immediate };
 
 using Keymap = std::vector<std::string>;
+using ControllerColor = std::array<uint8_t, 3>;
 
 constexpr uint32_t DEFAULT_CONSOLE_LANGUAGE = 1;
 constexpr uint32_t MAX_CONSOLE_LANGUAGE     = 29;
@@ -44,9 +47,13 @@ struct ConfigOptions {
 	std::string            user_name                   = "Kyty";
 	int32_t                user_id                     = DEFAULT_USER_ID;
 	std::string            audio_input_device;
+	std::optional<ControllerColor> controller_color;
+	uint32_t               controller_speaker_volume      = 100;
+	uint32_t               controller_vibration_intensity = 100;
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
+	bool                   hide_cursor_enabled         = false;
 	bool                   vr_enabled                  = false;
 	bool                   amd_cpu_enabled             = false;
 	uint32_t               vblank_frequency            = 60;
@@ -81,9 +88,13 @@ uint32_t GetScreenHeight();
 const std::string& GetUserName();
 int32_t  GetUserId();
 const std::string& GetAudioInputDevice();
+const std::optional<ControllerColor>& GetControllerColor();
+uint32_t GetControllerSpeakerVolume();
+uint32_t GetControllerVibrationIntensity();
 PresentMode GetPresentMode();
 int32_t GetGpuIndex();
 bool     FullscreenEnabled();
+bool     HideCursorEnabled();
 bool     VrEnabled();
 bool     AmdCpuEnabled();
 uint32_t GetVblankFrequency();

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -62,7 +62,6 @@ public:
 
 	void            BufferInit();
 	void            BufferFlush();
-	void            BufferFlushAndWait();
 	void            BufferWait();
 	HW::Context&    GetCtx() { return m_ctx; }
 	HW::UserConfig& GetUcfg() { return m_ucfg; }

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -140,6 +140,8 @@ constexpr auto MakeShaderDispatchTable() {
 	g_hw_sh_func[Pm4::COMPUTE_PGM_RSRC3]               = HwShSetCsRegisters;
 	g_hw_sh_func[Pm4::COMPUTE_PACE_ID]                 = HwShSetCsRegisters;
 	g_hw_sh_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_PS]  = HwShSetRegisters;
+	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] = HwShSetRegisters;
+	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] = HwShSetRegisters;
 	g_hw_sh_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_GS]  = HwShSetRegisters;
 	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_GS] = HwShSetRegisters;
 	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_GS] = HwShSetRegisters;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -3622,6 +3622,18 @@ void GraphicsInitJmpTablesShIndirect() {
 	g_hw_sh_indirect_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
+	for (uint32_t offset = Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS;
+	     offset <= Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS; offset++) {
+		g_hw_sh_indirect_func[offset] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+			cp.GetShCtx().SetPsUserDataAddress(cmd_offset - Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS,
+			                                   value);
+		};
+	}
+	for (uint32_t offset = 4; offset <= 5; offset++) {
+		g_hw_sh_indirect_func[offset] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+			cp.GetShCtx().SetPsAuxiliaryTableAddress(cmd_offset - 4, value);
+		};
+	}
 	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};

src/graphics/guest_gpu/gpu_format.cpp

@@ -61,8 +61,8 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
 	{BufferFormat::k32_32_32_32SInt, 16, 0, 16, true, false, true},
 	{BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
-	{BufferFormat::k8Srgb, 1, 0, 0, true, false},
-	{BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
+	{BufferFormat::k8Srgb, 1, 0, 1, true, false},
+	{BufferFormat::k8_8Srgb, 2, 0, 2, true, false},
 	{BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
 	{BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
 	{BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},

src/graphics/guest_gpu/graphicsRun.cpp

@@ -35,7 +35,6 @@ namespace Libs::Graphics {
 
 static thread_local CommandProcessor* g_current_processor = nullptr;
 static thread_local Pm4Execution*     g_current_execution = nullptr;
-static thread_local bool              g_gpu_mutex_owned   = false;
 static thread_local bool              g_gpu_thread        = false;
 static thread_local GuestGpu*         g_gpu_state         = nullptr;
 
@@ -54,27 +53,6 @@ struct DrawIndexedIndirectArgs {
 	uint32_t start_instance_location;
 };
 
-class GpuMutexLock final {
-public:
-	explicit GpuMutexLock(Common::Mutex& mutex): m_mutex(mutex) {
-		if (g_gpu_mutex_owned) {
-			EXIT("recursive GPU mutex acquisition\n");
-		}
-		g_gpu_mutex_owned = true;
-		m_mutex.Lock();
-	}
-	~GpuMutexLock() {
-		if (!g_gpu_mutex_owned) {
-			EXIT("invalid GPU mutex release\n");
-		}
-		m_mutex.Unlock();
-		g_gpu_mutex_owned = false;
-	}
-
-private:
-	Common::Mutex& m_mutex;
-};
-
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
 	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
@@ -84,6 +62,7 @@ GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	GraphicsInitJmpTables();
 	m_gfx_cp = std::make_unique<CommandProcessor>(renderer, 0);
+	m_gfx_cp->Reset();
 	m_thread = std::jthread(ThreadRun, this);
 }
 
@@ -160,20 +139,16 @@ void GuestGpu::Submit(std::span<const uint32_t> draw_commands,
 	if (draw_commands.empty()) {
 		return;
 	}
-	GpuMutexLock lock(m_submission_mutex);
-	Submission   submission;
+	Submission submission;
 	submission.type              = SubmissionType::Graphics;
 	submission.queue_id          = 0;
 	submission.commands          = draw_commands;
 	submission.constant_commands = constant_commands;
-	submission.reset_processor   = m_graphics_done;
-	m_graphics_done              = false;
 	Enqueue(std::move(submission));
 }
 
 void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands) {
 	EXIT_IF(commands.empty());
-	GpuMutexLock lock(m_submission_mutex);
 
 	EXIT_NOT_IMPLEMENTED(queue < ComputeQueueBase || queue >= ComputeQueueBase + ComputeQueueCount);
 
@@ -186,22 +161,21 @@ void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands)
 }
 
 void GuestGpu::SubmitFlipPreparation(uint64_t request_id) {
-	GpuMutexLock lock(m_submission_mutex);
-	Submission   submission;
+	Submission submission;
 	submission.type            = SubmissionType::FlipPreparation;
 	submission.queue_id        = 0;
-	submission.reset_processor = m_graphics_done;
 	submission.flip_request_id = request_id;
-	m_graphics_done            = false;
 	Enqueue(std::move(submission));
 }
 
-void GuestGpu::Done() {
-	GpuMutexLock lock(m_submission_mutex);
-	if (!IsGpuThread()) {
-		WaitForIdle();
-	}
-	m_graphics_done = true;
+void GuestGpu::SuspendPoint() {
+	EXIT_IF(IsGpuThread() || CommandScheduler::InDeferredOperation());
+	// Do not hold a queue lock while waiting: asynchronous work may be needed to
+	// finish the preceding graphics frame. The first point returns immediately.
+	m_suspend_point_ready->acquire();
+	Submission submission;
+	submission.type = SubmissionType::SuspendPoint;
+	Enqueue(std::move(submission));
 	m_done_num++;
 }
 
@@ -229,6 +203,9 @@ void CommandProcessor::Reset() {
 	m_context_state_pushed             = false;
 	m_index_type_and_size              = 0;
 	m_index_buffer_size                = 0;
+	m_index_base_addr                  = 0;
+	m_num_instances                    = 1;
+	m_predicate_skip                   = false;
 	m_user_data_marker                 = HW::UserSgprType::Unknown;
 	m_draw_indirect_args_base_addr     = 0;
 	m_dispatch_indirect_args_base_addr = 0;
@@ -268,10 +245,6 @@ void CommandProcessor::BufferFlush() {
 	GetScheduler().Flush();
 }
 
-void CommandProcessor::BufferFlushAndWait() {
-	GetScheduler().FlushAndWait();
-}
-
 void CommandProcessor::BufferWait() {
 	BufferInit();
 	GetScheduler().Finish();
@@ -459,6 +432,7 @@ void GuestGpu::Enqueue(Submission submission) {
 }
 
 void GuestGpu::WaitForIdle() {
+	EXIT_IF(IsGpuThread() || CommandScheduler::InDeferredOperation());
 	Common::LockGuard lock(m_queue_mutex);
 	while (m_processing || !m_commands.empty() || m_submission_count != 0) {
 		m_idle.Wait(&m_queue_mutex);
@@ -564,11 +538,7 @@ void GuestGpu::ThreadRun(void* data) {
 
 bool GuestGpu::Process(Submission& submission) {
 	const bool first_slice = !submission.started;
-	auto& cp = GetProcessor(submission.queue_id);
-
-	if (first_slice && submission.reset_processor) {
-		cp.Reset();
-	}
+	auto&      cp          = GetProcessor(submission.queue_id);
 
 	if (first_slice) {
 		submission.started = true;
@@ -645,6 +615,13 @@ bool GuestGpu::Process(Submission& submission) {
 			m_renderer.RunGarbageCollector();
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
+		case SubmissionType::SuspendPoint:
+			cp.EmitGlobalBarrier();
+			m_renderer.GetCommandScheduler().DeferPriorityOperation(
+			    [ready = m_suspend_point_ready] { ready->release(); });
+			cp.BufferFlush();
+			cp.Reset();
+			break;
 	}
 
 	return complete;
@@ -849,9 +826,7 @@ void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t
 			}
 		} break;
 		case 0x03:
-			if (wait_op != 0) {
-				BufferFlushAndWait();
-			}
+			// The wait selector applies only to Z-pass query readiness.
 			EXIT_NOT_IMPLEMENTED(address == nullptr);
 			value = *reinterpret_cast<const volatile uint64_t*>(address);
 			break;
@@ -1034,9 +1009,6 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
 
 	uint32_t frame_num = 0;
-	// uint32_t local_x   = 1;
-	// uint32_t local_y   = 1;
-	// uint32_t local_z   = 1;
 
 	{
 		frame_num = m_renderer.GetGpu().GetFrameNum();
@@ -1058,35 +1030,9 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 			}
 		}
 
-		const auto& cs = m_sh_ctx.GetCs().cs_regs;
-		// local_x        = std::max(cs.num_thread_x, 1u);
-		// local_y        = std::max(cs.num_thread_y, 1u);
-		// local_z        = std::max(cs.num_thread_z, 1u);
 		m_renderer.GetRenderExecutor().DispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x,
 		                                              thread_group_y, thread_group_z, mode);
 	}
-
-	/*constexpr uint32_t DispatchInitiatorUseThreadDimensions = 1u << 5u;
-	auto               group_count = [](uint32_t threads, uint32_t group_size) {
-	    return (threads == 0
-	                ? 0u
-	                : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u));
-	};
-
-	auto groups_x = thread_group_x;
-	auto groups_y = thread_group_y;
-	auto groups_z = thread_group_z;
-	if ((mode & DispatchInitiatorUseThreadDimensions) != 0) {
-	    groups_x = group_count(thread_group_x, local_x);
-	    groups_y = group_count(thread_group_y, local_y);
-	    groups_z = group_count(thread_group_z, local_z);
-	}
-
-	const uint64_t invocations =
-	    static_cast<uint64_t>(groups_x) * groups_y * groups_z * local_x * local_y * local_z;
-	if (invocations != 0) {
-	    BufferFlushAndWait();
-	}*/
 }
 
 void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {

src/graphics/guest_gpu/graphicsRun.h

@@ -13,6 +13,7 @@
 #include <deque>
 #include <memory>
 #include <mutex>
+#include <semaphore>
 #include <span>
 #include <thread>
 
@@ -32,11 +33,14 @@ public:
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
 
 	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
-	void              Submit(std::span<const uint32_t> draw_commands,
-	                         std::span<const uint32_t> constant_commands);
-	void              SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
-	void              SubmitFlipPreparation(uint64_t request_id);
-	void              Done();
+	void Submit(std::span<const uint32_t> draw_commands,
+	            std::span<const uint32_t> constant_commands);
+	void SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
+	void SubmitFlipPreparation(uint64_t request_id);
+	// Insert an ordered graphics drain; only a previous suspend point can block the caller.
+	void SuspendPoint();
+	// Wait for guest command processing, including all compute queues (not native GPU completion).
+	void              WaitForIdle();
 	[[nodiscard]] int GetFrameNum() const;
 
 	[[nodiscard]] static bool IsGpuThread() noexcept;
@@ -48,7 +52,7 @@ private:
 	static constexpr uint32_t ComputeQueueBase     = 0x20;
 	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
 
-	enum class SubmissionType { Graphics, Compute, FlipPreparation };
+	enum class SubmissionType { Graphics, Compute, FlipPreparation, SuspendPoint };
 
 	struct Submission {
 		SubmissionType            type     = SubmissionType::Graphics;
@@ -57,7 +61,6 @@ private:
 		std::span<const uint32_t> constant_commands;
 		Pm4Execution              command_execution;
 		Pm4Execution              constant_execution;
-		bool                      reset_processor   = false;
 		bool                      started           = false;
 		bool                      command_complete  = false;
 		bool                      constant_complete = false;
@@ -66,14 +69,12 @@ private:
 	};
 
 	void              Enqueue(Submission submission);
-	void              WaitForIdle();
 	void              ProcessCommands();
 	bool              Process(Submission& submission);
 	static void       ThreadRun(void* data);
 	CommandProcessor& GetProcessor(uint32_t queue_id);
 
 	RenderContext&                                 m_renderer;
-	Common::Mutex                                  m_submission_mutex;
 	Common::Mutex                                  m_queue_mutex;
 	std::mutex                                     m_shutdown_mutex;
 	Common::CondVar                                m_work_available;
@@ -84,10 +85,12 @@ private:
 	uint32_t                                       m_next_queue        = 0;
 	uint32_t                                       m_submission_count  = 0;
 	bool                                           m_processing        = false;
-	bool                                           m_graphics_done     = true;
 	bool                                           m_accepting         = true;
 	bool                                           m_stopping          = false;
 	bool                                           m_shutdown_complete = false;
+	// Completion callbacks can outlive GuestGpu during renderer shutdown.
+	std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =
+	    std::make_shared<std::binary_semaphore>(1);
 
 	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
 	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;

src/graphics/guest_gpu/hardwareContext.h

@@ -458,6 +458,9 @@ struct PsShaderResource2 {
 
 struct PsStageRegisters {
 	uint64_t          data_addr = 0;
+	uint64_t          user_data_addr = 0;
+	// Hades II uses PS offsets 4:5 for a byte-addressed descriptor table.
+	uint64_t          auxiliary_table_addr = 0;
 	PsShaderResource1 rsrc1;
 	PsShaderResource2 rsrc2;
 };
@@ -1017,6 +1020,12 @@ public:
 	void SetGsShaderResource2(const GsShaderResource2& rsrc2) { m_vs.gs_regs.rsrc2 = rsrc2; }
 
 	void SetPsShaderBase(uint64_t addr) { m_ps.ps_regs.data_addr = addr; }
+	void SetPsUserDataAddress(uint32_t word, uint32_t value) {
+		SetUserDataAddressWord(m_ps.ps_regs.user_data_addr, word, value);
+	}
+	void SetPsAuxiliaryTableAddress(uint32_t word, uint32_t value) {
+		SetUserDataAddressWord(m_ps.ps_regs.auxiliary_table_addr, word, value);
+	}
 	void SetPsShaderResource1(const PsShaderResource1& rsrc1) { m_ps.ps_regs.rsrc1 = rsrc1; }
 	void SetPsShaderResource2(const PsShaderResource2& rsrc2) { m_ps.ps_regs.rsrc2 = rsrc2; }
 

src/graphics/guest_gpu/pm4.h

@@ -697,6 +697,8 @@ constexpr uint32_t CX_NUM = 0x3FF + 1;
 
 /* Shader registers */
 
+constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_LO_PS = 0x2;
+constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_HI_PS = 0x3;
 constexpr uint32_t SPI_SHADER_PACE_ID_PS            = 0x6;
 constexpr uint32_t SPI_GRAPHICS_SHADER_CONTROL_PS    = 0x7;
 constexpr uint32_t SPI_SHADER_PGM_LO_PS              = 0x8;

src/graphics/host_gpu/graphicContext.h

@@ -29,6 +29,7 @@ struct GraphicContext {
 	bool                               memory_budget_ext_enabled             = false;
 	bool                               compute_subgroup_size_control_enabled = false;
 	bool                               sample_rate_shading_enabled           = false;
+	bool                               shader_image_int64_atomics_enabled    = false;
 	// bool fp64_denorm_preserve = false; // Temporarily disabled.
 	bool                               attachment_feedback_loop_enabled      = false;
 	bool                               provoking_vertex_last_enabled         = false;

src/graphics/host_gpu/regionManager.h

@@ -30,21 +30,14 @@ class TrackingSpinLock final {
 public:
 	void lock() noexcept {
 		const auto thread = CurrentThread();
-		if (m_owner.load(std::memory_order_relaxed) == thread) {
-			EXIT("recursive region tracking lock\n");
-		}
 		while (m_lock.test_and_set(std::memory_order_acquire)) {
-			if (m_owner.load(std::memory_order_relaxed) == thread) {
-				EXIT("recursive region tracking lock while contended\n");
-			}
+			EXIT_NOT_IMPLEMENTED(m_owner.load(std::memory_order_relaxed) == thread);
 			std::atomic_signal_fence(std::memory_order_seq_cst);
 		}
 		m_owner.store(thread, std::memory_order_relaxed);
 	}
 	void unlock() noexcept {
-		if (m_owner.load(std::memory_order_relaxed) != CurrentThread()) {
-			EXIT("region tracking lock released by non-owner\n");
-		}
+		EXIT_NOT_IMPLEMENTED(m_owner.load(std::memory_order_relaxed) != CurrentThread());
 		m_owner.store(0, std::memory_order_relaxed);
 		m_lock.clear(std::memory_order_release);
 	}
@@ -88,8 +81,7 @@ public:
 	template <DirtySource source>
 	[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
 		const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
-		const auto& bits        = GetBits<source>();
-		return RegionBits(bits, start, end).Any();
+		return GetBits<source>().FirstRangeFrom(start).first < end;
 	}
 
 	template <DirtySource source, bool enable>
@@ -122,6 +114,9 @@ public:
 	void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
 		const auto [start, end] = GetPageRange(vaddr, size);
 		auto&      bits         = GetBits<source>();
+		if (bits.FirstRangeFrom(start).first >= end) {
+			return;
+		}
 		RegionBits mask(bits, start, end);
 		if constexpr (clear) {
 			bits.UnsetRange(start, end);

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -110,6 +110,7 @@ void BufferCache::DeleteBuffer(BufferId id) {
 	}
 }
 
+template <bool async>
 bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size) {
 	std::vector<vk::BufferCopy> copies;
 	uint64_t                    total_size     = 0;
@@ -129,11 +130,16 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
 		return false;
 	}
 
-	const auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
+	auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
+	std::unique_ptr<Buffer> temporary;
 	if (mapped == nullptr) {
-		EXIT("BufferCache: download exceeds 64 MiB staging buffer capacity\n");
+		temporary = std::make_unique<Buffer>(m_graphics, m_scheduler, MemoryUsage::Download, 0,
+		                                     vk::BufferUsageFlagBits::eTransferDst, total_size);
+		mapped = temporary->Mapped().data();
+	} else {
+		m_download_buffer.Commit();
 	}
-	m_download_buffer.Commit();
+	const auto& download = temporary ? *temporary : m_download_buffer;
 	for (auto& copy: copies) {
 		copy.dstOffset += offset;
 	}
@@ -152,27 +158,35 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
 	native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
 	                       vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &before, 0,
 	                       nullptr);
-	native.copyBuffer(buffer.Handle(), m_download_buffer.Handle(),
+	native.copyBuffer(buffer.Handle(), download.Handle(),
 	                  static_cast<uint32_t>(copies.size()), copies.data());
 
 	auto after          = before;
 	after.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
 	after.dstAccessMask = vk::AccessFlagBits::eHostRead;
-	after.buffer        = m_download_buffer.Handle();
+	after.buffer        = download.Handle();
 	after.offset        = offset;
 	after.size          = total_size;
 	native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
 	                       vk::PipelineStageFlagBits::eAllCommands |
 	                           vk::PipelineStageFlagBits::eHost,
 	                       {}, 0, nullptr, 1, &after, 0, nullptr);
-	m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address,
-	                                    copies = std::move(copies)] {
-		m_download_buffer.Invalidate(offset, total_size);
+	auto publish = [this, mapped, offset, total_size, buffer_address,
+	                copies = std::move(copies), owner = std::move(temporary)] {
+		(owner ? *owner : m_download_buffer).Invalidate(offset, total_size);
 		for (const auto& copy: copies) {
 			Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset,
 			                                      mapped + (copy.dstOffset - offset), copy.size);
 		}
-	});
+	};
+	if constexpr (async) {
+		m_scheduler.DeferPriorityOperation(std::move(publish));
+	} else {
+		const auto tick = m_scheduler.CurrentTick();
+		m_scheduler.Wait(tick);
+		m_scheduler.WaitPriorityOperations(tick);
+		publish();
+	}
 	return true;
 }
 
@@ -251,10 +265,7 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
 		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
 
-		if (DownloadBufferMemory(buffer, window_begin, window_end - window_begin)) {
-			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.Wait(tick);
-			m_scheduler.WaitPriorityOperations(tick);
+		if (DownloadBufferMemory<false>(buffer, window_begin, window_end - window_begin)) {
 			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
 		}
 		if (is_write) {
@@ -491,8 +502,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
 	}
 
 	auto [staging, stage_offset] = m_staging_buffer.Map(size, 16);
-	if (staging == nullptr || (!Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size) &&
-	                           !Libs::LibKernel::Memory::TryReadPrtBacking(vaddr, staging, size))) {
+	if (staging == nullptr || !Libs::LibKernel::Memory::TryReadSparseBacking(vaddr, staging, size)) {
 		EXIT("BufferCache: failed to read mapped guest image backing\n");
 	}
 	m_staging_buffer.Commit();
@@ -610,7 +620,7 @@ void BufferCache::RunGarbageCollector() {
 			return false;
 		}
 		if (dirty) {
-			EXIT_IF(!DownloadBufferMemory(buffer, buffer.CpuAddress(), buffer.Size()));
+			EXIT_NOT_IMPLEMENTED(!DownloadBufferMemory<true>(buffer, buffer.CpuAddress(), buffer.Size()));
 			dirty_buffers.push_back(id);
 		} else {
 			m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -117,7 +117,8 @@ private:
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	// Queues backing publication; callers wait before clearing dirty pages or reusing their data.
+	// Synchronous downloads publish before returning; asynchronous callers wait before reuse.
+	template <bool async>
 	[[nodiscard]] bool DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size);
 
 	GraphicContext&                                   m_graphics;

src/graphics/host_gpu/renderer/cache/samplerCache.cpp

@@ -123,8 +123,11 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r, bool intege
 	sampler_info.addressModeU = to_vk_address_mode(r.ClampX());
 	sampler_info.addressModeV = to_vk_address_mode(r.ClampY());
 	sampler_info.addressModeW = to_vk_address_mode(r.ClampZ());
-	sampler_info.mipLodBias =
-	    static_cast<float>(static_cast<int16_t>((r.LodBias() ^ 0x2000u) - 0x2000u)) / 256.0f;
+    const auto max_lod_bias = m_graphics.GetPhysicalDeviceProperties().limits.maxSamplerLodBias;
+    sampler_info.mipLodBias = std::clamp(
+        static_cast<float>(static_cast<int16_t>((r.LodBias() ^ 0x2000u) - 0x2000u)) / 256.0f,
+        -max_lod_bias, max_lod_bias
+    );
 	sampler_info.anisotropyEnable        = (aniso ? VK_TRUE : VK_FALSE);
 	sampler_info.maxAnisotropy           = aniso_ratio;
 	sampler_info.compareEnable           = (r.DepthCompareFunc() != 0 ? VK_TRUE : VK_FALSE);

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -583,7 +583,7 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {
 	}
 
 	auto                         depth_buffer = m_tiler.GetScratchBuffer(depth_size);
-	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size);
+	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer);
 	const TileManager::D16Layout promote_layout {
 	    .width               = depth.info.extent.width,
 	    .height              = depth.info.extent.height,
@@ -791,13 +791,6 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		if (const auto depth_id = ResolveDepthOverlap(requested, binding, cached_id)) {
 			return {depth_id};
 		}
-		if (requested.IsBlock() && !cached.info.IsBlock()) {
-			return {ExpandImage(requested, cached_id)};
-		}
-		if (requested.data.size == cached.info.data.size &&
-		    (requested.IsVolume() || cached.info.IsVolume())) {
-			return {ExpandImage(requested, cached_id)};
-		}
 		// Equal pitch does not imply equal mip placement: a changed extent can move
 		// a level into or out of the mip tail. These are separate guest layouts.
 		if (requested.tile_mode != cached.info.tile_mode ||
@@ -808,6 +801,19 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 			}
 			return {merged_id};
 		}
+		if (requested.IsBlock() && !cached.info.IsBlock()) {
+			return {ExpandImage(requested, cached_id)};
+		}
+		// Volume depth is not an array-layer count. A larger depth can retain the
+		// same block-slice layout while requiring a larger native image.
+		if ((requested.IsVolume() || cached.info.IsVolume()) &&
+		    (requested.data.size == cached.info.data.size ||
+		     (requested.type == cached.info.type && requested.resources == cached.info.resources &&
+		      requested.extent.width == cached.info.extent.width &&
+		      requested.extent.height == cached.info.extent.height &&
+		      requested.extent.depth > cached.info.extent.depth))) {
+			return {ExpandImage(requested, cached_id)};
+		}
 		// PPSA08394
 		// A view cannot change the native image type or grow its extent.
 		if (requested.data.size == cached.info.data.size &&
@@ -878,9 +884,7 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) {
 	RefreshCopySource(source_id);
 	const auto expanded_id = InsertImage(info);
-	auto&      expanded    = m_slot_images[expanded_id];
 	auto&      source      = m_slot_images[source_id];
-	expanded.usage         = source.usage;
 	if (source.binding.is_bound || source.binding.is_target) {
 		source.binding.needs_rebind = true;
 	}
@@ -926,22 +930,12 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 	const bool  upload           = direction == TransferDirection::Upload;
 	const bool  render_target    = binding == BindingType::RenderTarget;
 	const bool  video_out        = binding == BindingType::VideoOut;
-	auto        format           = info.guest_format;
 	uint32_t    layers           = info.TransferLayers();
 	bool        volume           = info.IsVolume();
-	bool        allow_depth_tile = upload;
 	const char* owner            = "TextureCache readback";
 
 	TextureTransfer transfer;
-	transfer.swap_bgra16 = info.bgra16 && (!upload || render_target || video_out);
-	if (render_target) {
-		format = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
-	}
-	if (video_out) {
-		allow_depth_tile = false;
-	} else if (render_target || binding == BindingType::Storage) {
-		allow_depth_tile = true;
-	}
+	transfer.swap_bgra16 = info.bgra16;
 	if (upload) {
 		if ((render_target || video_out) &&
 		    (info.resources.layers == 0 || info.data.size % info.resources.layers != 0 ||
@@ -963,9 +957,9 @@ TextureCache::BuildTextureTransfer(const Image& image, BindingType binding,
 		}
 	}
 
-	transfer.layout  = TextureCalcUploadLayout(format, info.extent.width, info.extent.height,
-	                                       info.resources.levels, layers, info.tile_mode,
-	                                       info.data.size, allow_depth_tile, volume, owner);
+	transfer.layout = TextureCalcUploadLayout(info.guest_format, info.extent.width,
+	                                         info.extent.height, info.resources.levels, layers,
+	                                         info.tile_mode, info.data.size, volume, owner);
 	transfer.regions = TextureBuildImageCopies(transfer.layout);
 	if (info.IsDepth()) {
 		for (auto& region: transfer.regions) {
@@ -1071,7 +1065,7 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 		                     texels_per_slice > UINT64_MAX / transfer_bytes);
 		const uint64_t transfer_slice = texels_per_slice * transfer_bytes;
 		EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers);
-		auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers);
+		auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers, linear.buffer);
 		m_tiler.ConvertD16(
 		    linear, promoted, TileManager::D16Direction::Promote,
 		    info.pixel_format == vk::Format::eD32SfloatS8Uint,
@@ -1133,7 +1127,7 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 		image.info.metadata = desc.info.metadata;
 		// Native color metadata must not retain a reused HTile/CMask/FMask clear flag.
 		m_surface_metas.erase(range.address);
-		if (range.size == 0 || desc.info.resources.levels != 1 || image.info.resources.levels != 1) {
+		if (range.size == 0 || desc.info.resources.levels != 1) {
 			return;
 		}
 	}
@@ -1181,10 +1175,12 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 			           {vk::ImageAspectFlagBits::eColor, view.base_level, view.level_count,
 			            image_first + slice, 1}, clear);
 		}
-		// Native expanded keys own consumption. Existing buffer tracking publishes this CPU
-		// write to future GPU readers; FillBuffer can fault and must run outside the texture lock.
+		// Publish the conversion's expanded keys without treating them as guest writes
+		// to overlapping image data. Invalidate the buffer before updating its backing.
 		if (desc.type != BindingType::VideoOut) {
-			m_buffer_cache.FillBuffer(address, slice_size, UINT32_MAX, false);
+			std::fill(bytes.begin(), bytes.end(), uint8_t {0xff});
+			m_buffer_cache.InvalidateMemory(address, slice_size);
+			LibKernel::Memory::WriteBacking(address, bytes.data(), bytes.size());
 		}
 	}
 }
@@ -1533,7 +1529,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	auto&          image = m_slot_images[selected];
 	vk::ClearValue clear {};
 	if (aspect == vk::ImageAspectFlagBits::eColor) {
-		if (!DecodePackedColorClear(image.info.pixel_format, packed_clear, clear.color)) {
+		if (!DecodeColorDwordFill(image.info.pixel_format, packed_clear, clear.color)) {
 			return false;
 		}
 	} else {
@@ -1679,7 +1675,7 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	auto host_linear = m_tiler.GetScratchBuffer(transfer_size);
 	image.Download(copies, host_linear.buffer, 0, host_linear.size);
 	const bool tiled        = info.IsTiled();
-	auto       guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size)
+	auto       guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size, host_linear.buffer)
 	                                : TileManager::Result {destination.Handle(), destination_offset,
 	                                                       destination.Size() - destination_offset};
 	m_tiler.ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote,
@@ -1850,7 +1846,7 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) {
 	std::scoped_lock lock {m_lock};
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto& image = m_slot_images[id];
-		if (!image.Overlaps(address, size)) {
+		if (image.info.data.address != address) {
 			continue;
 		}
 		if (image.IsGpuModified()) {

src/graphics/host_gpu/renderer/image/image.cpp

@@ -340,7 +340,9 @@ void Image::CopyImage(Image& source) {
 	EXIT_IF(source.backing.samples != backing.samples);
 	m_scheduler.EndRendering();
 	const uint32_t levels     = std::min(source.backing.mip_levels, backing.mip_levels);
-	const uint32_t base_depth = backing.image_type == vk::ImageType::e3D
+	const uint32_t base_depth = source.backing.image_type == backing.image_type
+	                                ? std::min(source.backing.extent.depth, backing.extent.depth)
+	                            : backing.image_type == vk::ImageType::e3D
 	                                ? backing.extent.depth
 	                                : source.backing.extent.depth;
 	const auto     source_aspect =

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -428,6 +428,20 @@ inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {
 	return true;
 }
 
+// Unlike a register clear, a DWORD fill repeats the same word across the entire pixel.
+// Keep this separate: the first register word alone cannot describe a 64-bit color.
+[[nodiscard]] inline bool DecodeColorDwordFill(vk::Format format, uint32_t packed,
+                                               vk::ClearColorValue& clear) {
+	if (format == vk::Format::eR16G16B16A16Sfloat) {
+		if (packed != 0) {
+			return false;
+		}
+		clear = vk::ClearColorValue {};
+		return true;
+	}
+	return DecodePackedColorClear(format, packed, clear);
+}
+
 [[nodiscard]] inline bool DecodePackedStencilClear(uint32_t packed, uint8_t& clear) {
 	const auto value = static_cast<uint8_t>(packed);
 	if (packed != static_cast<uint32_t>(value) * 0x01010101u) {

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -358,10 +358,8 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 	}
 
 	vk::ImageViewUsageCreateInfo usage {};
-	usage.usage = image.usage;
-	if (!is_storage) {
-		usage.usage &= ~vk::ImageUsageFlagBits::eStorage;
-	}
+	usage.usage = is_storage ? vk::ImageUsageFlagBits::eStorage
+	                         : image.usage & ~vk::ImageUsageFlagBits::eStorage;
 	vk::ImageViewMinLodCreateInfoEXT min_lod {};
 	if (normalized.min_lod != 0) {
 		min_lod.minLod = static_cast<float>(normalized.base_level) +

src/graphics/host_gpu/renderer/image/imageView.h

@@ -31,20 +31,14 @@ namespace ImageViewOps {
 	       ImageViewOps::IsFormatDepthCompatible(view_format);
 }
 
-[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle) noexcept {
+[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle,
+                                             uint32_t component_count = 4) noexcept {
 	if ((swizzle & ~0xfffu) != 0) {
 		return false;
 	}
 	for (uint32_t channel = 0; channel < 4; channel++) {
-		switch (GetDstSel(swizzle, channel)) {
-			case 0:
-			case 1:
-			case 4:
-			case 5:
-			case 6:
-			case 7: break;
-			default: return false;
-		}
+		const auto selector = GetDstSel(swizzle, channel);
+		if (selector > 1 && (selector < 4 || selector >= 4 + component_count)) return false;
 	}
 	return true;
 }
@@ -81,15 +75,8 @@ SelectSampledColorView(vk::Format image_format, vk::Format view_format, uint32_t
 [[nodiscard]] inline bool IsSupportedSampledDepthView(vk::Format image_format,
                                                       vk::Format view_format,
                                                       uint32_t   swizzle) noexcept {
-	if (!IsSupportedSampledDepthFormat(image_format, view_format)) {
-		return false;
-	}
-	switch (swizzle) {
-		case DstSel(4, 4, 4, 4):
-		case DstSel(4, 0, 0, 0):
-		case DstSel(4, 0, 0, 1): return true;
-		default: return false;
-	}
+	return IsSupportedSampledDepthFormat(image_format, view_format) &&
+	       IsValidImageSwizzle(swizzle, 1);
 }
 
 [[nodiscard]] inline bool
@@ -103,8 +90,11 @@ IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resou
 	if (resource.numeric_class != Prospero::TextureNumericClass::Float && resource.depth_compare) {
 		return false;
 	}
-	return resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read &&
-	       !resource.written && !resource.atomic;
+	return ((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
+	         resource.mip_count == 1u) ||
+	        (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic &&
+	         resource.mip_count != 0u)) &&
+	       resource.read && !resource.written && !resource.atomic;
 }
 
 inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
@@ -127,7 +117,7 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
 	       ((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
 	         resource.mip_count == 1u) ||
-	        (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage &&
+	        (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic &&
 	         resource.mip_count != 0u)) &&
 	       resource.written &&
 	       (!resource.atomic ||

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -175,8 +175,7 @@ SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
 TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                             uint32_t height, uint32_t levels, uint32_t depth,
                                             Prospero::TileMode tile_mode, uint64_t upload_size,
-                                            bool allow_depth_tile, bool volume_texture,
-                                            const char* owner) {
+                                            bool volume_texture, const char* owner) {
 	TextureUploadLayout layout {};
 	layout.surface.description = {
 	    format,
@@ -204,8 +203,7 @@ TextureUploadLayout TextureCalcUploadLayout(Prospero::BufferFormat format, uint3
 			     static_cast<uint32_t>(format));
 		}
 	} else {
-		if ((tile_mode == Prospero::TileMode::kDepth && !allow_depth_tile) ||
-		    !TileGetTiledTextureLayout(description, layout.surface)) {
+		if (!TileGetTiledTextureLayout(description, layout.surface)) {
 			EXIT("%s: unsupported typed tiled upload: fmt=%u tile=%u "
 			     "size=%" PRIu64 " extent=%ux%u levels=%u\n",
 			     owner, static_cast<uint32_t>(format), static_cast<uint32_t>(tile_mode),

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -47,8 +47,7 @@ RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layo
 TextureUploadLayout    TextureCalcUploadLayout(Prospero::BufferFormat format, uint32_t width,
                                                uint32_t height, uint32_t levels, uint32_t depth,
                                                Prospero::TileMode tile_mode, uint64_t upload_size,
-                                               bool allow_depth_tile, bool volume_texture,
-                                               const char* owner);
+                                               bool volume_texture, const char* owner);
 std::vector<vk::BufferImageCopy> TextureBuildImageCopies(const TextureUploadLayout& layout);
 bool TextureBuildGpuTileInfos(uint64_t tiled_size, const std::vector<vk::BufferImageCopy>& regions,
                               const TextureUploadLayout& layout, uint32_t levels,

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -88,30 +88,6 @@ TileManager::~TileManager() {
 	}
 }
 
-TileManager::Scratch TileManager::AllocateScratch(uint64_t size) {
-	EXIT_IF(size == 0);
-	vk::BufferCreateInfo create {};
-	create.size  = size;
-	create.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc |
-	               vk::BufferUsageFlagBits::eTransferDst;
-
-	VmaAllocationCreateInfo allocate {};
-	allocate.usage       = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
-	VkBuffer      buffer = VK_NULL_HANDLE;
-	VmaAllocation memory = nullptr;
-	const auto    raw    = static_cast<VkBufferCreateInfo>(create);
-	RequireVulkanSuccess(static_cast<vk::Result>(vmaCreateBuffer(
-	                         m_graphics.allocator, &raw, &allocate, &buffer, &memory, nullptr)),
-	                     "allocate TileManager scratch buffer");
-	return {buffer, memory, size};
-}
-
-void TileManager::DeferDestroy(Scratch scratch) {
-	auto allocator = m_graphics.allocator;
-	m_scheduler.DeferOperation(
-	    [allocator, scratch] { vmaDestroyBuffer(allocator, scratch.buffer, scratch.allocation); });
-}
-
 void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
                           std::span<const GpuTileInfo> infos, uint64_t source_base,
                           uint64_t target_base, std::vector<Dispatch>& dispatches) {
@@ -369,8 +345,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
 	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
-	auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4));
-	DeferDestroy(scratch);
+	auto scratch = GetScratchBuffer(linear_capacity, tiled);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
 	return {scratch.buffer, 0, linear_capacity};
@@ -400,11 +375,9 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        target_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	// Reserve all stream parameters before creating a scheduler-lived scratch dependency:
-	// StreamBuffer::Map is allowed to submit the current tick when it wraps.
+	// Reserve stream parameters before recording the image download and conversion.
 	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
-	auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4));
-	DeferDestroy(linear);
+	auto linear = GetScratchBuffer(linear_capacity, tiled);
 	image.Download(regions, linear.buffer, 0, linear.size);
 	Result source {linear.buffer, 0, linear.size};
 	if (transform == ColorTransform::SwapBgra16) {
@@ -414,10 +387,19 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	       dispatches, false);
 }
 
-TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
-	auto scratch = AllocateScratch(Common::AlignUp(size, 4));
-	DeferDestroy(scratch);
-	return {scratch.buffer, 0, scratch.size};
+TileManager::Result TileManager::GetScratchBuffer(uint64_t size, vk::Buffer input) {
+	EXIT_IF(size == 0);
+	size = Common::AlignUp(size, 4);
+	auto& buffer = m_scratch[m_scratch[0] && m_scratch[0]->Handle() == input ? 1 : 0];
+	if (!buffer || buffer->Size() < size) {
+		if (buffer) {
+			m_scheduler.DeferOperation([old = std::move(buffer)]() mutable { old.reset(); });
+		}
+		buffer = std::make_unique<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0,
+		    vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc |
+		        vk::BufferUsageFlagBits::eTransferDst, size);
+	}
+	return {buffer->Handle(), 0, size};
 }
 
 TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const {
@@ -668,9 +650,7 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 
 TileManager::Result TileManager::SwapBgra16(Result input) {
 	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX);
-	auto output = AllocateScratch(input.size);
-	DeferDestroy(output);
-	Result result {output.buffer, 0, output.size};
+	auto result = GetScratchBuffer(input.size, input.buffer);
 	SwapBgra16(input, result, static_cast<uint32_t>(input.size / 8u));
 	return result;
 }

src/graphics/host_gpu/renderer/image/tiler.h

@@ -6,13 +6,14 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
+#include <memory>
 #include <span>
 #include <vector>
-#include <vk_mem_alloc.h>
 
 namespace Libs::Graphics {
 
 class CommandScheduler;
+class Buffer;
 class Image;
 class StreamBuffer;
 struct GraphicContext;
@@ -62,7 +63,7 @@ public:
 	~TileManager();
 	KYTY_CLASS_NO_COPY(TileManager);
 
-	// The returned device-local buffer remains alive through the current scheduler tick.
+	// Consume scratch results before the next acquisition, or pass their buffer as input.
 	[[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity,
 	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos);
 	void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled,
@@ -71,7 +72,7 @@ public:
 	               uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity,
 	               std::span<const GpuTileInfo> infos,
 	               ColorTransform               transform = ColorTransform::None);
-	[[nodiscard]] Result GetScratchBuffer(uint64_t size);
+	[[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr);
 	void                 ConvertD16(Result source, Result target, D16Direction direction, bool d32,
 	                                const D16Layout& layout);
 	[[nodiscard]] Result SwapBgra16(Result input);
@@ -105,23 +106,16 @@ private:
 		uint32_t pipeline_slot = 0;
 		uint64_t params_offset = 0;
 	};
-	struct Scratch {
-		vk::Buffer    buffer     = nullptr;
-		VmaAllocation allocation = nullptr;
-		uint64_t      size       = 0;
-	};
 	struct StorageBinding {
 		vk::DescriptorBufferInfo info;
 		uint32_t                 base = 0;
 	};
 
-	[[nodiscard]] Scratch         AllocateScratch(uint64_t size);
 	[[nodiscard]] StorageBinding  BindStorage(Result buffer, uint64_t size) const;
 	[[nodiscard]] static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride,
 	                                             uint64_t active, uint32_t remaining,
 	                                             uint64_t alignment, uint64_t max_range,
 	                                             uint32_t max_groups) noexcept;
-	void                          DeferDestroy(Scratch scratch);
 	void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
 	             std::span<const GpuTileInfo> infos, uint64_t source_base, uint64_t target_base,
 	             std::vector<Dispatch>& dispatches);
@@ -134,6 +128,7 @@ private:
 	GraphicContext&                         m_graphics;
 	CommandScheduler&                       m_scheduler;
 	StreamBuffer&                           m_stream_buffer;
+	std::array<std::unique_ptr<Buffer>, 2>   m_scratch;
 	vk::DescriptorSetLayout                 m_descriptor_layout = nullptr;
 	vk::PipelineLayout                      m_pipeline_layout   = nullptr;
 	std::array<vk::Pipeline, PipelineCount> m_pipelines {};

src/graphics/host_gpu/renderer/pipeline/blendMapping.cpp

@@ -0,0 +1,58 @@
+#include "graphics/host_gpu/renderer/pipeline/blendMapping.h"
+
+#include "graphics/guest_gpu/hardwareContext.h"
+
+#include <initializer_list>
+
+namespace Libs::Graphics {
+
+bool BlendFactorIsDualSource(uint8_t factor) {
+	return factor >= static_cast<uint8_t>(Prospero::BlendFactor::kSrc1Color) &&
+	       factor <= static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrc1Alpha);
+}
+
+namespace {
+
+bool BlendFactorIsConstantColor(uint8_t factor) {
+	return factor == static_cast<uint8_t>(Prospero::BlendFactor::kConstantColor) ||
+	       factor == static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusConstantColor);
+}
+
+} // namespace
+
+BlendMappingSupport ClassifyBlendMapping(const HW::BlendControl&                blend,
+                                         const Prospero::ColorComponentMapping& mapping) {
+	// Color constants are not swizzled with the exports; scalar constant alpha is unaffected.
+	if (!mapping.IsIdentity() && (BlendFactorIsConstantColor(blend.color_srcblend) ||
+	                              BlendFactorIsConstantColor(blend.color_destblend))) {
+		return BlendMappingSupport::Unsupported;
+	}
+	if (mapping.Map(3) == 3u) {
+		return BlendMappingSupport::Direct;
+	}
+	// Moving alpha requires the same equation for all channels.
+	if (blend.separate_alpha_blend && (blend.alpha_srcblend != blend.color_srcblend ||
+	                                   blend.alpha_destblend != blend.color_destblend ||
+	                                   blend.alpha_comb_fcn != blend.color_comb_fcn)) {
+		return BlendMappingSupport::Unsupported;
+	}
+	auto support = BlendMappingSupport::Direct;
+	for (const auto factor: {blend.color_srcblend, blend.color_destblend}) {
+		if (BlendFactorIsDualSource(factor)) {
+			return BlendMappingSupport::Unsupported;
+		}
+		switch (static_cast<Prospero::BlendFactor>(factor)) {
+			case Prospero::BlendFactor::kSrcAlpha:
+			case Prospero::BlendFactor::kOneMinusSrcAlpha:
+				support = BlendMappingSupport::SourceAlpha;
+				break;
+			case Prospero::BlendFactor::kDstAlpha:
+			case Prospero::BlendFactor::kOneMinusDstAlpha:
+			case Prospero::BlendFactor::kSrcAlphaSaturate: return BlendMappingSupport::Unsupported;
+			default: break;
+		}
+	}
+	return support;
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/blendMapping.h

@@ -0,0 +1,27 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_PIPELINE_BLENDMAPPING_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_PIPELINE_BLENDMAPPING_H_
+
+#include <cstdint>
+
+namespace Libs::Graphics {
+
+namespace HW {
+struct BlendControl;
+}
+namespace Prospero {
+struct ColorComponentMapping;
+}
+
+enum class BlendMappingSupport {
+	Direct,
+	SourceAlpha, // Requires logical alpha in the second blend source.
+	Unsupported,
+};
+
+bool                BlendFactorIsDualSource(uint8_t factor);
+BlendMappingSupport ClassifyBlendMapping(const HW::BlendControl&                blend,
+                                         const Prospero::ColorComponentMapping& mapping);
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_PIPELINE_BLENDMAPPING_H_

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -312,13 +312,14 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	                              resource.written && !resource.read && !resource.atomic;
 	const auto numeric_class = Prospero::SampledTextureNumericClass(format);
 	const bool raw_float_atomic = format == Prospero::BufferFormat::k32Float && uint_resource &&
-	                              resource.atomic;
+	                              resource.atomic && !resource.atomic64;
 	const bool format_ok =
 	    raw_sint_storage || raw_float_atomic ||
 	    (numeric_class != Prospero::TextureNumericClass::Unsupported &&
 	     numeric_class != Prospero::TextureNumericClass::Sint &&
 	     uint_resource == (numeric_class == Prospero::TextureNumericClass::Uint) &&
-	     (!resource.atomic || format == Prospero::BufferFormat::k32UInt));
+	     (!resource.atomic || format == (resource.atomic64 ? Prospero::BufferFormat::k32_32UInt
+	                                                       : Prospero::BufferFormat::k32UInt)));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
 		return;
 	}
@@ -354,7 +355,8 @@ static TextureCache::ImageDesc NullTextureDesc(const ShaderRecompiler::IR::Image
 			desc.info.guest_format = Prospero::BufferFormat::k32Float;
 			break;
 		case Prospero::TextureNumericClass::Uint:
-			desc.info.guest_format = Prospero::BufferFormat::k32UInt;
+			desc.info.guest_format = resource.atomic64 ? Prospero::BufferFormat::k32_32UInt
+			                                         : Prospero::BufferFormat::k32UInt;
 			break;
 		case Prospero::TextureNumericClass::Sint:
 			desc.info.guest_format = Prospero::BufferFormat::k32SInt;
@@ -365,10 +367,10 @@ static TextureCache::ImageDesc NullTextureDesc(const ShaderRecompiler::IR::Image
 	desc.info.type            = Prospero::ImageType::kColor2D;
 	desc.info.extent          = {1, 1, 1};
 	desc.info.resources       = {1, 1};
-	desc.info.bytes_per_block = 4;
+	desc.info.bytes_per_block = Prospero::NumBytesPerElement(desc.info.guest_format);
 	desc.info.samples         = 1;
 	desc.info.mip_layout[0]   = {0, 0, 1, 1};
-	desc.view_info.format     = desc.info.pixel_format;
+	desc.view_info.format     = resource.atomic64 ? vk::Format::eR64Uint : desc.info.pixel_format;
 	desc.view_info.type       = vk::ImageViewType::e2D;
 	desc.view_info.aspect     = vk::ImageAspectFlagBits::eColor;
 	desc.view_info.usage      = binding == TextureCache::BindingType::Storage
@@ -538,6 +540,9 @@ static bool ResolveTextureMipView(const TileSurfaceDescription& description, boo
 
 TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
                                               const ShaderRecompiler::IR::DescriptorValue& value) {
+	if (resource.atomic64 && !m_context.GetGraphics().shader_image_int64_atomics_enabled) {
+		EXIT("64-bit image atomics require shaderImageInt64Atomics\n");
+	}
 	auto descriptor = DecodeNativeDescriptor<ShaderTextureResource>(value);
 	const bool storage = resource.written;
 	if (storage) {
@@ -563,7 +568,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	const auto physical_levels = multisampled ? 1u : static_cast<uint32_t>(max_mip) + 1u;
 	// IMAGE_STORE addresses BASE_LEVEL; only IMAGE_STORE_MIP selects other view mips.
 	const bool single_storage_mip =
-	    storage && resource.mip_mode != ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
+	    storage && resource.mip_mode != ShaderRecompiler::IR::ImageMipMode::Dynamic;
 	const auto view_levels = multisampled || single_storage_mip
 	                             ? 1u
 	                             : static_cast<uint32_t>(last_level - base_level) + 1u;
@@ -650,8 +655,9 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 			pixel_format = depth_format->depth_attachment_format;
 		}
 	}
-	const auto storage_view_format = storage && (resource.atomic ||
-	                                            format == Prospero::BufferFormat::k32SInt)
+	const auto storage_view_format = resource.atomic64 ? vk::Format::eR64Uint
+	                                 : storage && (resource.atomic ||
+	                                               format == Prospero::BufferFormat::k32SInt)
 	                                     ? vk::Format::eR32Uint
 	                                     : SrgbStorageViewFormat(pixel_format);
 	const auto view_format         = storage && storage_view_format != vk::Format::eUndefined
@@ -773,8 +779,6 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 	prepared.gds = {nullptr, 0, VK_WHOLE_SIZE};
 	prepared.flattened_srt = {};
 	prepared.shader_data_buffer = {};
-	prepared.buffer_sources.clear();
-	prepared.buffers.clear();
 	prepared.images.resize(program.info.images.size());
 	prepared.samplers.clear();
 	prepared.shader_data.clear();
@@ -787,7 +791,8 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
 	}
 	prepared.samplers.reserve(program.info.samplers.size());
 	for (uint32_t i = 0; i < program.info.samplers.size(); i++) {
-		prepared.samplers.push_back(NativeSampler(m_context, program, i, snapshot.samplers[i]));
+		prepared.samplers.push_back(NativeSampler(
+		    m_context, program, i, snapshot.samplers[program.info.samplers[i].snapshot_index]));
 	}
 	prepared.shader_data.reserve(program.bindings.ShaderDataDwords());
 	for (const auto reg: program.bindings.user_data_registers) {
@@ -887,7 +892,7 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 		auto& binding = images[i];
 		binding.mip_views.clear();
 		const auto& resource = program.info.images[i];
-		if (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage) {
+		if (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic) {
 			EXIT_IF(resource.mip_count == 0u ||
 			        resource.mip_count != binding.desc.view_info.level_count);
 			binding.mip_views.reserve(resource.mip_count);
@@ -895,6 +900,8 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 				auto desc = binding.desc;
 				desc.view_info.base_level += mip;
 				desc.view_info.level_count = 1;
+				// The shader selects the mip after applying the guest minimum LOD.
+				desc.view_info.min_lod = 0;
 				binding.mip_views.push_back(texture_cache.FindTexture(binding.image_id, desc));
 			}
 			binding.image_view = binding.mip_views.front();
@@ -974,6 +981,41 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
 		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
+	for (const auto* reader: prepared_bindings) {
+		const auto& reads = reader->runtime->resources->specialization_reads;
+		if (reads.empty()) continue;
+		for (const auto* writer: prepared_bindings) {
+			if (writer->runtime->program->has_address_writes) {
+				EXIT("scalar resource reads cannot be proven disjoint from shader address writes\n");
+			}
+		}
+		for (const auto [address, size]: reads) {
+			for (const auto id: m_bound_images) {
+				const auto* image = m_context.GetTextureCache().m_slot_images.try_get(id);
+				if (image == nullptr ||
+				    (!image->binding.shader_write && !image->binding.is_target)) continue;
+				for (const auto written: {image->info.data, image->info.stencil,
+				                          image->info.metadata.range}) {
+					if (written.size != 0 && ImageRangeOverlaps(address, size,
+					                                          written.address, written.size)) {
+						EXIT("scalar resource reads overlap an image or attachment write\n");
+					}
+				}
+			}
+			for (const auto* writer: prepared_bindings) {
+				const auto& program = *writer->runtime->program;
+				for (uint32_t i = 0; i < writer->buffer_sources.size(); ++i) {
+					const auto resource = program.bindings.descriptors.front().resources[i];
+					if (!program.info.buffers[resource].written) continue;
+					const auto& written = writer->buffer_sources[i];
+					if (written.size != 0 && ImageRangeOverlaps(address, size,
+					                                          written.address, written.size)) {
+						EXIT("scalar resource reads overlap a shader buffer write\n");
+					}
+				}
+			}
+		}
+	}
 	m_descriptor_buffers.clear();
 	m_descriptor_images.clear();
 	m_descriptor_writes.clear();

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -11,6 +11,7 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/pipeline/blendMapping.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/shader/recompiler/ShaderRecompiler.h"
@@ -219,7 +220,6 @@ struct PipelineCache::ProgramCache {
 		ShaderRecompiler::IR::ResourceSnapshot       resources;
 		ShaderRecompiler::IR::ResourceSpecialization specialization;
 		std::vector<Permutation>                    permutations;
-		bool                                        skip_dispatch = false;
 	};
 
 	struct ProgramKeyHash {
@@ -287,9 +287,6 @@ struct PipelineCache::ProgramCache {
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
-		if (entry != programs.end() && entry->second.skip_dispatch) {
-			return {};
-		}
 		const ShaderRecompiler::IR::SrtRuntime       runtime {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
@@ -357,11 +354,6 @@ struct PipelineCache::ProgramCache {
 		}
 		DumpShaderOriginal(stage_name, options.shader_hash, params.code);
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
-		if (translated.skip_dispatch) {
-			entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
-			entry->second.skip_dispatch = true;
-			return {};
-		}
 		if (entry == programs.end()) {
 			entry = programs.try_emplace(lookup_key,
 			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
@@ -609,21 +601,30 @@ PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms(
 	}
 	ShaderParams pixel_params;
 	if (pixel_active) {
-		pixel_params = PrepareProgram(pixel_regs, sh, target_export_mapping, pixel_info);
-		const auto& blend          = context.GetBlendControl(0);
-		const auto  is_dual_source = [](uint8_t factor) {
-			return factor >= static_cast<uint8_t>(Prospero::BlendFactor::kSrc1Color) &&
-			       factor <= static_cast<uint8_t>(Prospero::BlendFactor::kOneMinusSrc1Alpha);
-		};
+		pixel_params      = PrepareProgram(pixel_regs, sh, target_export_mapping, pixel_info);
+		const auto& blend = context.GetBlendControl(0);
 		pixel_info.dual_source_blending =
 		    blend.enable && !context.GetRenderTarget(0).info.blend_bypass &&
-		    (is_dual_source(blend.color_srcblend) || is_dual_source(blend.color_destblend) ||
-		     (blend.separate_alpha_blend &&
-		      (is_dual_source(blend.alpha_srcblend) || is_dual_source(blend.alpha_destblend))));
+		    (BlendFactorIsDualSource(blend.color_srcblend) ||
+		     BlendFactorIsDualSource(blend.color_destblend) ||
+		     (blend.separate_alpha_blend && (BlendFactorIsDualSource(blend.alpha_srcblend) ||
+		                                     BlendFactorIsDualSource(blend.alpha_destblend))));
 		if (pixel_info.dual_source_blending) {
-			// MRT1 supplies a second blend source for the same render target as MRT0.
+			// MRT1 supplies the second blend source for target 0.
 			pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
 			pixel_info.target_export_mapping[1] = pixel_info.target_export_mapping[0];
+		} else if (blend.enable && !context.GetRenderTarget(0).info.blend_bypass &&
+		           pixel_info.target_output_mode[0] != 0 && pixel_info.target_output_mode[0] != 7 &&
+		           std::all_of(std::begin(pixel_info.target_output_mode) + 1,
+		                       std::end(pixel_info.target_output_mode),
+		                       [](uint8_t mode) { return mode == 0; }) &&
+		           ClassifyBlendMapping(blend, pixel_info.target_export_mapping[0]) ==
+		               BlendMappingSupport::SourceAlpha) {
+			// Preserve logical alpha when the export mapping moves it.
+			pixel_info.alpha_blend_source_remap = true;
+			pixel_info.dual_source_blending     = true;
+			pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
+			pixel_info.target_export_mapping[1] = {};
 		}
 	}
 	if (context.GetClipControl().clip_disable) {
@@ -657,6 +658,19 @@ ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs
                                                ShaderComputeInputInfo&      input_info) {
 	input_info.host_subgroup_size = m_graphics.SupportsComputeWave64() ? 64u : 32u;
 	const auto        params      = PrepareProgram(regs, sh, input_info);
+	// Use one effective size for the cache key, LDS declaration, and access bounds.
+	const auto max_lds_dwords =
+	    m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize / 4u;
+	if (input_info.lds_size_dwords > max_lds_dwords) {
+		static std::atomic_bool warned = false;
+		if (!warned.exchange(true, std::memory_order_relaxed)) {
+			PipelineCacheLog("GPU warning: game compute shader requests {} bytes of LDS, but "
+			                 "the Vulkan device limit is {} bytes. Clamping LDS; rendering may "
+			                 "be incorrect.",
+			                 input_info.lds_size_dwords * 4u, max_lds_dwords * 4u);
+		}
+	}
+	input_info.lds_size_dwords = std::min(input_info.lds_size_dwords, max_lds_dwords);
 	uint32_t          push_data_cursor = 0;
 	return m_program_cache->Get(params, input_info, push_data_cursor);
 }
@@ -713,14 +727,35 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		}
 		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
 		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		static_params.color_srcblend[slot]       = bc.color_srcblend;
-		static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
-		static_params.color_destblend[slot]      = bc.color_destblend;
-		static_params.alpha_srcblend[slot]       = bc.alpha_srcblend;
-		static_params.alpha_comb_fcn[slot]       = bc.alpha_comb_fcn;
-		static_params.alpha_destblend[slot]      = bc.alpha_destblend;
-		static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
-		static_params.blend_enable[slot]         = bc.enable && !rt.info.blend_bypass;
+		const bool alpha_remap =
+		    slot == 0 && ps_input_info != nullptr && ps_input_info->alpha_blend_source_remap;
+		static_params.blend_enable[slot] = bc.enable && !rt.info.blend_bypass;
+		if (static_params.blend_enable[slot] && !alpha_remap &&
+		    ClassifyBlendMapping(bc, colors[i].export_mapping) != BlendMappingSupport::Direct) {
+			static_params.blend_enable[slot] = false;
+			static std::atomic_bool warned = false;
+			if (!warned.exchange(true, std::memory_order_relaxed)) {
+				Log::WriteToConsoleAndLog(fmt::format(
+				    "Warning: blending disabled for unsupported color mapping "
+				    "(slot={} mapping=0x{:02x} color={}/{} alpha={}/{} separate={}).\n",
+				    slot, colors[i].export_mapping.packed, bc.color_srcblend, bc.color_destblend,
+				    bc.alpha_srcblend, bc.alpha_destblend, bc.separate_alpha_blend ? 1 : 0));
+			}
+		}
+		if (alpha_remap) {
+			static_params.blend_alpha_source_remap = true;
+		}
+		if (static_params.blend_enable[slot]) {
+			static_params.color_srcblend[slot]       = bc.color_srcblend;
+			static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
+			static_params.color_destblend[slot]      = bc.color_destblend;
+			static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
+			if (bc.separate_alpha_blend) {
+				static_params.alpha_srcblend[slot]  = bc.alpha_srcblend;
+				static_params.alpha_comb_fcn[slot]  = bc.alpha_comb_fcn;
+				static_params.alpha_destblend[slot] = bc.alpha_destblend;
+			}
+		}
 	}
 	const bool with_depth =
 	    depth.desc.view_info.format != vk::Format::eUndefined && static_cast<bool>(depth.image_id);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -55,6 +55,7 @@ struct PipelineStaticParameters {
 	uint8_t                    alpha_destblend[RENDER_COLOR_ATTACHMENTS_MAX]      = {};
 	bool                       separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	bool                       blend_enable[RENDER_COLOR_ATTACHMENTS_MAX]         = {};
+	bool                       blend_alpha_source_remap                           = false;
 
 	bool operator==(const PipelineStaticParameters& other) const noexcept;
 };
@@ -64,7 +65,7 @@ struct PipelineStaticParameters {
 static_assert(std::is_trivially_copyable_v<PipelineStaticParameters>);
 static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
-static_assert(sizeof(PipelineStaticParameters) == 125);
+static_assert(sizeof(PipelineStaticParameters) == 126);
 
 struct PipelineRenderingState {
 	std::array<vk::Format, RENDER_COLOR_ATTACHMENTS_MAX> color_formats {};

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -137,14 +137,17 @@ static void GetInputFormat(const ShaderBufferResource& res, vk::Format& format,
 	}
 }
 
-static vk::BlendFactor GetBlendFactor(uint32_t factor) {
+static vk::BlendFactor GetBlendFactor(uint32_t factor, bool remap_source_alpha) {
 	switch (static_cast<Prospero::BlendFactor>(factor)) {
 		case Prospero::BlendFactor::kZero: return vk::BlendFactor::eZero;
 		case Prospero::BlendFactor::kOne: return vk::BlendFactor::eOne;
 		case Prospero::BlendFactor::kSrcColor: return vk::BlendFactor::eSrcColor;
 		case Prospero::BlendFactor::kOneMinusSrcColor: return vk::BlendFactor::eOneMinusSrcColor;
-		case Prospero::BlendFactor::kSrcAlpha: return vk::BlendFactor::eSrcAlpha;
-		case Prospero::BlendFactor::kOneMinusSrcAlpha: return vk::BlendFactor::eOneMinusSrcAlpha;
+		case Prospero::BlendFactor::kSrcAlpha:
+			return remap_source_alpha ? vk::BlendFactor::eSrc1Color : vk::BlendFactor::eSrcAlpha;
+		case Prospero::BlendFactor::kOneMinusSrcAlpha:
+			return remap_source_alpha ? vk::BlendFactor::eOneMinusSrc1Color
+			                          : vk::BlendFactor::eOneMinusSrcAlpha;
 		case Prospero::BlendFactor::kDstAlpha: return vk::BlendFactor::eDstAlpha;
 		case Prospero::BlendFactor::kOneMinusDstAlpha: return vk::BlendFactor::eOneMinusDstAlpha;
 		case Prospero::BlendFactor::kDstColor: return vk::BlendFactor::eDstColor;
@@ -327,9 +330,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 			}
 		}
 
-		EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].AddTid());
-		EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].SwizzleEnabled());
-
 		EXIT_IF(registers_num < 1 || registers_num > 4);
 	}
 
@@ -397,17 +397,20 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		color_blend_attachment[i].colorWriteMask =
 		    vk::ColorComponentFlags {static_params.color_mask[i]};
 		color_blend_attachment[i].blendEnable = static_params.blend_enable[i] ? VK_TRUE : VK_FALSE;
+		// Only target 0 can use the second blend source carrying logical alpha.
+		const bool remap_source_alpha         = i == 0 && static_params.blend_alpha_source_remap;
 		color_blend_attachment[i].srcColorBlendFactor =
-		    GetBlendFactor(static_params.color_srcblend[i]);
+		    GetBlendFactor(static_params.color_srcblend[i], remap_source_alpha);
 		color_blend_attachment[i].dstColorBlendFactor =
-		    GetBlendFactor(static_params.color_destblend[i]);
+		    GetBlendFactor(static_params.color_destblend[i], remap_source_alpha);
 		color_blend_attachment[i].colorBlendOp = GetBlendOp(static_params.color_comb_fcn[i]);
 		color_blend_attachment[i].srcAlphaBlendFactor =
-		    (static_params.separate_alpha_blend[i] ? GetBlendFactor(static_params.alpha_srcblend[i])
-		                                           : color_blend_attachment[i].srcColorBlendFactor);
+		    (static_params.separate_alpha_blend[i]
+		         ? GetBlendFactor(static_params.alpha_srcblend[i], remap_source_alpha)
+		         : color_blend_attachment[i].srcColorBlendFactor);
 		color_blend_attachment[i].dstAlphaBlendFactor =
 		    (static_params.separate_alpha_blend[i]
-		         ? GetBlendFactor(static_params.alpha_destblend[i])
+		         ? GetBlendFactor(static_params.alpha_destblend[i], remap_source_alpha)
 		         : color_blend_attachment[i].dstColorBlendFactor);
 		color_blend_attachment[i].alphaBlendOp =
 		    (static_params.separate_alpha_blend[i] ? GetBlendOp(static_params.alpha_comb_fcn[i])

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -256,10 +256,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	input_info.dispatch_thread_dimensions = use_thread_dimensions;
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
-	if (!compute_program) {
-		// Temporary until RT is implemented.
-		return;
-	}
 	if (use_thread_dimensions) {
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
@@ -268,15 +264,14 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 
 	const auto& program   = *input_info.stage.program;
 	const auto& resources = *input_info.stage.resources;
-	if (TryConsumeComputeMetaClear(input_info, buffer)) {
-		ResetBindings();
-		return;
-	}
-	if (TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y,
-	                                thread_group_z, mode)) {
+	if (resources.specialization_reads.empty() &&
+	    (TryConsumeComputeMetaClear(input_info, buffer) ||
+	     TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y,
+	                                 thread_group_z, mode))) {
 		ResetBindings();
 		return;
 	}
+
 	const bool large_workgroup =
 	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const bool                   has_sampler = !program.info.samplers.empty();
@@ -326,7 +321,8 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 			     static_cast<uint32_t>(r.TileMode()));
 		}
 		for (uint32_t i = 0; i < program.info.samplers.size(); i++) {
-			const auto r = DecodeNativeDescriptor<ShaderSamplerResource>(resources.samplers[i]);
+			const auto r = DecodeNativeDescriptor<ShaderSamplerResource>(
+			    resources.samplers[program.info.samplers[i].snapshot_index]);
 			LOGF("  CS sampler[%u]: source=%u clamp=%u/%u/%u filter=%u/%u/%u mip=%u "
 			     "lod=%u-%u bias=%d\n",
 			     i, program.info.samplers[i].source, static_cast<uint32_t>(r.ClampX()),
@@ -417,10 +413,6 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	ShaderComputeInputInfo input_info {};
 	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
 	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
-	if (!compute_program) {
-		// Temporary until RT is implemented.
-		return;
-	}
 	buffer.EndRendering();
 	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto& bindings = m_compute_bindings;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -119,6 +119,10 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void RenderContext::PrepareBda() {
+	if (!m_bda_logged) {
+		Log::WriteToConsoleAndLog("GPU: using buffer device address (BDA) shader memory access.\n");
+		m_bda_logged = true;
+	}
 	std::shared_lock lock(m_mapped_ranges_mutex);
 	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
 		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);

src/graphics/host_gpu/renderer/renderContext.h

@@ -81,6 +81,7 @@ private:
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 	bool                      m_fault_process_pending = false;
+	bool                      m_bda_logged = false;
 
 	Common::Mutex                        m_interrupt_mutex;
 	std::vector<InterruptEqRegistration> m_interrupt_eqs;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -1032,6 +1032,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	uint32_t   mesh_groups = 0;
 	if (mesh_active) {
 		const auto& mesh = state.vertex_info[0].mesh;
+		EXIT_NOT_IMPLEMENTED(mesh.fast_launch && (draw.IsIndexed() || primitive_restart_enable));
 		static std::atomic_bool restart_warned = false;
 		if (primitive_restart_enable && !restart_warned.exchange(true, std::memory_order_relaxed)) {
 			std::printf("Warning: primitive restart is not implemented for mesh shaders; "

src/graphics/presentation/systemOverlay.cpp

@@ -29,9 +29,9 @@ namespace Libs::Graphics {
 
 namespace {
 
-namespace CoreIme   = Libs::Ime;
-namespace DialogIme = Libs::Dialog::ImeDialog;
-namespace ErrorDialog = Libs::Dialog::ErrorDialog;
+namespace CoreIme      = Libs::Ime;
+namespace DialogIme    = Libs::Dialog::ImeDialog;
+namespace SystemDialog = Libs::Dialog::SystemDialog;
 
 namespace Ime {
 
@@ -104,7 +104,7 @@ bool HostQueueExternalInput(uint64_t generation, ExternalInput input) {
 
 } // namespace Ime
 
-enum class OverlayKind : uint8_t { None, Ime, Error };
+enum class OverlayKind : uint8_t { None, Ime, Dialog };
 
 struct OverlaySession {
 	OverlayKind kind       = OverlayKind::None;
@@ -114,14 +114,14 @@ struct OverlaySession {
 };
 
 struct OverlaySnapshot {
-	OverlaySession            session;
-	Ime::HostSnapshot         ime;
-	ErrorDialog::HostSnapshot error;
+	OverlaySession             session;
+	Ime::HostSnapshot          ime;
+	SystemDialog::HostSnapshot dialog;
 };
 
 bool GetOverlaySnapshot(OverlaySnapshot* snapshot) {
-	if (ErrorDialog::GetHostSnapshot(&snapshot->error)) {
-		snapshot->session = {OverlayKind::Error, snapshot->error.generation};
+	if (SystemDialog::GetHostSnapshot(&snapshot->dialog)) {
+		snapshot->session = {OverlayKind::Dialog, snapshot->dialog.generation};
 		return true;
 	}
 	if (Ime::GetHostSnapshot(&snapshot->ime)) {
@@ -234,7 +234,7 @@ void RefreshVisibility() {
 	bool capture_keyboard   = false;
 	bool text_input         = false;
 	bool multiline          = false;
-	if (snapshot.session.kind == OverlayKind::Error) {
+	if (snapshot.session.kind == OverlayKind::Dialog) {
 		capture_controller = true;
 		capture_keyboard   = true;
 	} else if (snapshot.session.kind == OverlayKind::Ime) {
@@ -429,14 +429,14 @@ void InitializeSystemOverlayInput(SDL_Window* window) {
 	}
 	CoreIme::SetVisibilityCallback(OnCoreVisibilityChanged);
 	DialogIme::SetVisibilityCallback(OnDialogVisibilityChanged);
-	ErrorDialog::SetVisibilityCallback(RefreshVisibility);
+	SystemDialog::SetVisibilityCallback(RefreshVisibility);
 	RefreshVisibility();
 }
 
 void ShutdownSystemOverlayInput() {
 	CoreIme::SetVisibilityCallback(nullptr);
 	DialogIme::SetVisibilityCallback(nullptr);
-	ErrorDialog::SetVisibilityCallback(nullptr);
+	SystemDialog::SetVisibilityCallback(nullptr);
 	{
 		std::scoped_lock lock(g_visibility_mutex);
 		g_input_lifecycle_active = false;
@@ -466,9 +466,9 @@ void ShutdownSystemOverlayInput() {
 SystemOverlayVisualState GetSystemOverlayVisualState() noexcept {
 	const auto core   = CoreIme::GetVisualState();
 	const auto dialog = DialogIme::GetVisualState();
-	const auto error  = ErrorDialog::GetVisualState();
-	return {core.active || dialog.active || error.active,
-	        core.revision + dialog.revision + error.revision};
+	const auto system = SystemDialog::GetVisualState();
+	return {core.active || dialog.active || system.active,
+	        core.revision + dialog.revision + system.revision};
 }
 
 bool ProcessSystemOverlayInput(const SDL_Event& event) {
@@ -521,10 +521,11 @@ bool ProcessSystemOverlayInput(const SDL_Event& event) {
 	if (keyboard_event && !g_input_keyboard) {
 		return false;
 	}
-	if (keyboard_event && session.kind == OverlayKind::Error) {
+	if (keyboard_event && session.kind == OverlayKind::Dialog) {
 		if (event.type == SDL_EVENT_KEY_DOWN && !event.key.repeat &&
-		    (event.key.key == SDLK_RETURN || event.key.key == SDLK_KP_ENTER)) {
-			ErrorDialog::HostAccept(generation);
+		    (event.key.key == SDLK_RETURN || event.key.key == SDLK_KP_ENTER ||
+		     event.key.key == SDLK_ESCAPE)) {
+			SystemDialog::HostClose(generation);
 		}
 		return true;
 	}
@@ -904,7 +905,7 @@ struct SystemOverlay::Impl {
 		ImGui::End();
 	}
 
-	void DrawError(const ErrorDialog::HostSnapshot& snapshot, vk::Extent2D extent) {
+	void DrawDialog(const SystemDialog::HostSnapshot& snapshot, vk::Extent2D extent) {
 		const ImVec2 display(static_cast<float>(extent.width), static_cast<float>(extent.height));
 		const float  scale = std::max(std::min(display.x / 1280.0f, display.y / 720.0f), 0.5f);
 		ImGui::GetBackgroundDrawList()->AddRectFilled({0.0f, 0.0f}, display,
@@ -922,17 +923,23 @@ struct SystemOverlay::Impl {
 		constexpr ImGuiWindowFlags flags = ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove |
 		                                   ImGuiWindowFlags_NoSavedSettings |
 		                                   ImGuiWindowFlags_AlwaysAutoResize;
-		ImGui::Begin("##SystemError", nullptr, flags);
-		ImGui::TextUnformatted("Error");
+		const bool                 signin = snapshot.kind == SystemDialog::Kind::Signin;
+		ImGui::Begin("##SystemDialog", nullptr, flags);
+		ImGui::TextUnformatted(signin ? "Sign in to PlayStation Network" : "Error");
 		ImGui::Separator();
 		const auto error_code = static_cast<uint32_t>(snapshot.error_code);
-		ImGui::TextWrapped("%s", error_code == 0x80550006u
+		ImGui::TextWrapped("%s", signin ? "PlayStation Network sign-in is unavailable."
+		                         : error_code == 0x80550006u
 		                             ? "You are not signed in to PlayStation Network."
 		                             : "An error has occurred.");
-		ImGui::TextDisabled("Error code: 0x%08X", error_code);
+		if (!signin) {
+			ImGui::TextDisabled("Error code: 0x%08X", error_code);
+		}
 		const float button_width = std::min(140.0f * scale, ImGui::GetContentRegionAvail().x);
 		ImGui::SetCursorPosX((ImGui::GetWindowSize().x - button_width) * 0.5f);
-		const bool accepted = ImGui::Button("OK", {button_width, 44.0f * scale});
+		const bool accepted =
+		    ImGui::Button(signin ? "Cancel" : "OK", {button_width, 44.0f * scale}) ||
+		    (signin && ImGui::IsKeyPressed(ImGuiKey_GamepadFaceRight, false));
 		if (focus_pending) {
 			ImGui::SetItemDefaultFocus();
 			focus_pending = false;
@@ -941,7 +948,7 @@ struct SystemOverlay::Impl {
 		ImGui::PopFont();
 		ImGui::PopStyleVar(2);
 		if (accepted) {
-			ErrorDialog::HostAccept(snapshot.generation);
+			SystemDialog::HostClose(snapshot.generation);
 		}
 	}
 
@@ -955,7 +962,8 @@ struct SystemOverlay::Impl {
 		if (session != snapshot.session) {
 			session       = snapshot.session;
 			focus_pending = true;
-			shift         = (snapshot.ime.option & Ime::OPTION_NO_AUTO_CAPITALIZE) == 0;
+			shift         = session.kind == OverlayKind::Ime &&
+			                (snapshot.ime.option & Ime::OPTION_NO_AUTO_CAPITALIZE) == 0;
 			symbol_mode   = false;
 			panel_offset  = {};
 			right_stick   = {};
@@ -981,8 +989,8 @@ struct SystemOverlay::Impl {
 			ImGui::EndFrame();
 			return false;
 		}
-		if (snapshot.session.kind == OverlayKind::Error) {
-			DrawError(snapshot.error, frame_extent);
+		if (snapshot.session.kind == OverlayKind::Dialog) {
+			DrawDialog(snapshot.dialog, frame_extent);
 		} else {
 			DrawIme(snapshot.ime, frame_extent);
 		}

src/graphics/presentation/videoOut.cpp

@@ -1814,6 +1814,38 @@ KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* stat
 	return OK;
 }
 
+KYTY_SYSV_ABI int VideoOutAllowOutputResolutionWqhdDetection(int handle) {
+	if (!DriverState().IsOpened(handle)) {
+		return VIDEO_OUT_ERROR_INVALID_HANDLE;
+	}
+	return OK;
+}
+
+KYTY_SYSV_ABI int VideoOutVrrPegToFixedRate(int handle, uint64_t arg1, uint64_t arg2) {
+	PRINT_NAME();
+
+	static std::atomic_bool logged {false};
+	if (!logged.exchange(true, std::memory_order_relaxed)) {
+		LOGF("\t handle = %d\n"
+		     "\t arg1   = 0x%016" PRIx64 "\n"
+		     "\t arg2   = 0x%016" PRIx64 "\n",
+		     handle, arg1, arg2);
+	}
+
+	return OK;
+}
+
+KYTY_SYSV_ABI int VideoOutVrrUnpegFromFixedRate(int handle) {
+	PRINT_NAME();
+
+	static std::atomic_bool logged {false};
+	if (!logged.exchange(true, std::memory_order_relaxed)) {
+		LOGF("\t handle = %d\n", handle);
+	}
+
+	return OK;
+}
+
 static int ValidateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options,
                                 void* reserved_ptr, uint64_t reserved) {
 	if (!DriverState().IsOpened(handle)) {

src/graphics/presentation/videoOut.h

@@ -90,6 +90,9 @@ KYTY_SYSV_ABI int VideoOutGetEventData(const LibKernel::EventQueue::KernelEvent*
 KYTY_SYSV_ABI int VideoOutGetEventCount(const LibKernel::EventQueue::KernelEvent* ev);
 KYTY_SYSV_ABI int VideoOutWaitVblank(int handle);
 KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status);
+KYTY_SYSV_ABI int VideoOutAllowOutputResolutionWqhdDetection(int handle);
+KYTY_SYSV_ABI int VideoOutVrrPegToFixedRate(int handle, uint64_t arg1, uint64_t arg2);
+KYTY_SYSV_ABI int VideoOutVrrUnpegFromFixedRate(int handle);
 KYTY_SYSV_ABI int VideoOutInitializeOutputOptions(VideoOutOutputOptions* options);
 KYTY_SYSV_ABI int VideoOutIsOutputSupported(int handle, uint64_t mode,
                                             const VideoOutOutputOptions* options,

src/graphics/presentation/window/hostInput.cpp

@@ -25,6 +25,8 @@ struct ControlInfo {
 	Controller::Axis axis     = Controller::Axis::AxisMax;
 	bool             positive = false;
 	float            touch_x  = 0.0f;
+
+	Controller::Setting setting {};
 };
 
 static constexpr std::array CONTROL_INFO = {
@@ -55,6 +57,10 @@ static constexpr std::array CONTROL_INFO = {
     ControlInfo {"RightStickRight", 0, Controller::Axis::RightX, true},
     ControlInfo {"RightStickUp", 0, Controller::Axis::RightY, false},
     ControlInfo {"RightStickDown", 0, Controller::Axis::RightY, true},
+    ControlInfo {.name = "SpeakerVolume", .setting = Controller::Setting::SpeakerVolume},
+    ControlInfo {.name = "VibrationIntensity", .setting = Controller::Setting::VibrationIntensity},
+    ControlInfo {.name    = "TriggerEffectIntensity",
+                 .setting = Controller::Setting::TriggerEffectIntensity},
 };
 
 constexpr std::size_t INVALID_CONTROL = CONTROL_INFO.size();
@@ -66,6 +72,7 @@ struct Binding {
 };
 
 constexpr int              MOUSE_POLL_INTERVAL_MS = 33;
+constexpr uint64_t         CURSOR_IDLE_HIDE_MS    = 2000;
 constexpr std::string_view MOUSE_SENSITIVITY      = "MouseSensitivity=";
 
 struct MouseJoystickState {
@@ -75,7 +82,8 @@ struct MouseJoystickState {
 };
 
 MouseJoystickState g_mouse;
-SDL_Window*        g_mouse_window = nullptr;
+SDL_Window*        g_mouse_window   = nullptr;
+uint64_t           g_cursor_hide_at = 0;
 
 std::size_t ControlFromName(std::string_view name) {
 	const auto info = std::find_if(CONTROL_INFO.begin(), CONTROL_INFO.end(),
@@ -242,6 +250,12 @@ void SetStickAxis(Controller::Axis axis, bool negative, bool positive) {
 	Controller::SetAxis(Controller::HOST_INPUT_CONTROLLER_ID, axis, value);
 }
 
+void CycleSettingOnPress(Controller::Setting setting, bool down) {
+	if (down) {
+		Controller::CycleSetting(setting);
+	}
+}
+
 void SetControl(std::size_t control, bool down) {
 	if (control == INVALID_CONTROL) {
 		return;
@@ -256,6 +270,10 @@ void SetControl(std::size_t control, bool down) {
 		SetButton(info.button, down);
 		return;
 	}
+	if (info.axis == Controller::Axis::AxisMax) {
+		CycleSettingOnPress(info.setting, down);
+		return;
+	}
 
 	struct AxisKeys {
 		bool negative = false;
@@ -280,6 +298,9 @@ void DefaultKeyboardInput(int key_code, bool down) {
 	switch (NormalizeKey(static_cast<SDL_Keycode>(key_code))) {
 		case SDLK_BACKSPACE: SetTouchPad(0.25f, down); return;
 		case SDLK_TAB: SetTouchPad(0.75f, down); return;
+		case SDLK_1: CycleSettingOnPress(Controller::Setting::SpeakerVolume, down); return;
+		case SDLK_2: CycleSettingOnPress(Controller::Setting::VibrationIntensity, down); return;
+		case SDLK_3: CycleSettingOnPress(Controller::Setting::TriggerEffectIntensity, down); return;
 		case SDLK_A:
 			left.left = down;
 			SetStickAxis(Controller::Axis::LeftX, left.left, left.right);
@@ -361,11 +382,31 @@ int PollMouse(uint64_t now_ms) {
 	return MOUSE_POLL_INTERVAL_MS;
 }
 
+bool IsCursorActivity(const SDL_Event& event) {
+	SDL_MouseID which;
+	switch (event.type) {
+		case SDL_EVENT_MOUSE_MOTION:
+			if (event.motion.xrel == 0.0f && event.motion.yrel == 0.0f) {
+				return false;
+			}
+			which = event.motion.which;
+			break;
+		case SDL_EVENT_MOUSE_BUTTON_DOWN:
+		case SDL_EVENT_MOUSE_BUTTON_UP: which = event.button.which; break;
+		case SDL_EVENT_MOUSE_WHEEL: which = event.wheel.which; break;
+		default: return false;
+	}
+	return which != SDL_TOUCH_MOUSEID && which != SDL_PEN_MOUSEID;
+}
+
 } // namespace
 
 void HostInputInit(SDL_Window* window) {
 	GetInputMap();
 	g_mouse_window = window;
+	if (Config::HideCursorEnabled()) {
+		g_cursor_hide_at = SDL_GetTicks() + CURSOR_IDLE_HIDE_MS;
+	}
 }
 
 void HostInputShutdown() {
@@ -374,7 +415,11 @@ void HostInputShutdown() {
 		CenterMouseStick();
 		g_mouse = {};
 	}
-	g_mouse_window = nullptr;
+	g_mouse_window   = nullptr;
+	g_cursor_hide_at = 0;
+	if (Config::HideCursorEnabled()) {
+		SDL_ShowCursor();
+	}
 }
 
 void HostInputKey(int key_code, bool down) {
@@ -410,20 +455,36 @@ void HostInputToggleMouseToJoystick() {
 }
 
 bool HostInputWaitEvent(SDL_Event* event) {
-	bool has_event;
+	int timeout = -1;
 	if (!g_mouse.enabled || SDL_GetKeyboardFocus() != g_mouse_window) {
 		g_mouse.next_poll = 0;
 		CenterMouseStick();
-		has_event = SDL_WaitEvent(event);
-		if (!has_event) {
-			EXIT("%s\n", SDL_GetError());
-		}
 	} else {
 		if (g_mouse.next_poll == 0) {
 			SDL_GetRelativeMouseState(nullptr, nullptr);
 			g_mouse.next_poll = SDL_GetTicks() + MOUSE_POLL_INTERVAL_MS;
 		}
-		has_event = SDL_WaitEventTimeout(event, PollMouse(SDL_GetTicks()));
+		timeout = PollMouse(SDL_GetTicks());
+	}
+
+	if (g_cursor_hide_at != 0) {
+		const auto now_ms = SDL_GetTicks();
+		const int  cursor_timeout =
+		    now_ms < g_cursor_hide_at ? static_cast<int>(g_cursor_hide_at - now_ms) : 0;
+		timeout = timeout < 0 ? cursor_timeout : std::min(timeout, cursor_timeout);
+	}
+	const bool has_event = SDL_WaitEventTimeout(event, timeout);
+
+	if (Config::HideCursorEnabled()) {
+		const auto now_ms = SDL_GetTicks();
+		if (has_event && !g_mouse.enabled && IsCursorActivity(*event) &&
+		    SDL_GetWindowFromEvent(event) == g_mouse_window) {
+			SDL_ShowCursor();
+			g_cursor_hide_at = now_ms + CURSOR_IDLE_HIDE_MS;
+		} else if (g_cursor_hide_at != 0 && now_ms >= g_cursor_hide_at) {
+			SDL_HideCursor();
+			g_cursor_hide_at = 0;
+		}
 	}
 
 	if (has_event && event->type == SDL_EVENT_WINDOW_FOCUS_LOST &&

src/graphics/presentation/window/vulkanWindow.cpp

@@ -137,11 +137,13 @@ static uint32_t VulkanFindQueueFamily(vk::PhysicalDevice device, vk::SurfaceKHR
 	return static_cast<uint32_t>(-1);
 }
 
+// On failure out_device is null and out_rejections says why each device was skipped.
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surface,
                                      const std::vector<const char*>& device_extensions,
                                      SurfaceCapabilities&            out_capabilities,
-                                     vk::PhysicalDevice& out_device, uint32_t& out_queue_family) {
+                                     vk::PhysicalDevice& out_device, uint32_t& out_queue_family,
+                                     std::string& out_rejections) {
 	EXIT_IF(instance == nullptr);
 	EXIT_IF(surface == nullptr);
 
@@ -170,14 +172,22 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		vk::PhysicalDeviceProperties device_properties {};
 		device.getProperties(&device_properties);
 
+		const auto reject = [&](const std::string& reason) {
+			LOGF("%s\n", reason.c_str());
+			out_rejections +=
+			    skip_device ? "; " : fmt::format("\n  {}: ", device_properties.deviceName.data());
+			out_rejections += reason;
+			skip_device = true;
+		};
+
 		LOGF("Vulkan device: %s\n", device_properties.deviceName.data());
 		if (device_properties.apiVersion < VULKAN_TARGET_API_VERSION) {
-			LOGF("Vulkan %u.%u is required, but device supports only %u.%u.%u\n",
-			     VK_VERSION_MAJOR(VULKAN_TARGET_API_VERSION),
-			     VK_VERSION_MINOR(VULKAN_TARGET_API_VERSION),
-			     VK_VERSION_MAJOR(device_properties.apiVersion),
-			     VK_VERSION_MINOR(device_properties.apiVersion),
-			     VK_VERSION_PATCH(device_properties.apiVersion));
+			reject(fmt::format("Vulkan {}.{} is required, but device supports only {}.{}.{}",
+			                   VK_VERSION_MAJOR(VULKAN_TARGET_API_VERSION),
+			                   VK_VERSION_MINOR(VULKAN_TARGET_API_VERSION),
+			                   VK_VERSION_MAJOR(device_properties.apiVersion),
+			                   VK_VERSION_MINOR(device_properties.apiVersion),
+			                   VK_VERSION_PATCH(device_properties.apiVersion)));
 			continue;
 		}
 
@@ -203,7 +213,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
-		features13.pNext = &features12;
+		features13.pNext       = &features12;
 		device_features2.pNext = &features13;
 
 		device.getFeatures2(&device_features2);
@@ -212,151 +222,82 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		const auto queue_family = VulkanFindQueueFamily(device, surface);
 		if (queue_family == static_cast<uint32_t>(-1)) {
-			LOGF("No universal graphics, compute, and presentation queue\n");
-			skip_device = true;
+			reject("No universal graphics, compute, and presentation queue");
 		}
 
+		const auto check_feature = [&](vk::Bool32 supported, const char* name,
+		                               vk::Bool32 required = VK_TRUE) {
+			if (required == VK_TRUE && supported != VK_TRUE) {
+				reject(fmt::format("{} is not supported", name));
+			}
+		};
+
+#if defined(__APPLE__)
 		if (color_write_ext.colorWriteEnable != VK_TRUE) {
 			LOGF("colorWriteEnable is not supported\n");
-#if !defined(__APPLE__)
-			skip_device = true;
+		}
+#else
+		check_feature(color_write_ext.colorWriteEnable, "colorWriteEnable");
 #endif
-		}
-		if (image_view_min_lod.minLod != VK_TRUE) {
-			LOGF("image view minLod is not supported\n");
-			skip_device = true;
-		}
+		check_feature(image_view_min_lod.minLod, "image view minLod");
 
-		if (depth_clip_control.depthClipControl != VK_TRUE) {
-			LOGF("depthClipControl is not supported\n");
-			skip_device = true;
-		}
+		check_feature(depth_clip_control.depthClipControl, "depthClipControl");
+#if defined(__APPLE__)
 		if (depth_clip_enable.depthClipEnable != VK_TRUE) {
 			LOGF("depthClipEnable is not supported\n");
-#if !defined(__APPLE__)
-			skip_device = true;
+		}
+#else
+		check_feature(depth_clip_enable.depthClipEnable, "depthClipEnable");
 #endif
-		}
 #if !defined(__APPLE__)
-		if (device_features2.features.depthClamp != VK_TRUE) {
-			LOGF("depthClamp is not supported\n");
-			skip_device = true;
-		}
-		if (fragment_barycentric.fragmentShaderBarycentric != VK_TRUE) {
-			LOGF("fragmentShaderBarycentric is not supported\n");
-			skip_device = true;
-		}
+		check_feature(device_features2.features.depthClamp, "depthClamp");
+		check_feature(fragment_barycentric.fragmentShaderBarycentric, "fragmentShaderBarycentric");
 #endif
 
-		if (required_features12.samplerMirrorClampToEdge == VK_TRUE &&
-		    features12.samplerMirrorClampToEdge != VK_TRUE) {
-			LOGF("samplerMirrorClampToEdge is not supported\n");
-			skip_device = true;
-		}
-		if (required_features12.timelineSemaphore == VK_TRUE &&
-		    features12.timelineSemaphore != VK_TRUE) {
-			LOGF("timelineSemaphore is not supported\n");
-			skip_device = true;
-		}
-		if (required_features12.shaderOutputLayer == VK_TRUE &&
-		    features12.shaderOutputLayer != VK_TRUE) {
-			LOGF("shaderOutputLayer is not supported\n");
-			skip_device = true;
-		}
-		if (required_features12.shaderOutputViewportIndex == VK_TRUE &&
-		    features12.shaderOutputViewportIndex != VK_TRUE) {
-			LOGF("shaderOutputViewportIndex is not supported\n");
-			skip_device = true;
-		}
-		if (required_features12.bufferDeviceAddress == VK_TRUE &&
-		    features12.bufferDeviceAddress != VK_TRUE) {
-			LOGF("bufferDeviceAddress is not supported\n");
-			skip_device = true;
-		}
-		if (required_features12.shaderBufferInt64Atomics == VK_TRUE &&
-		    features12.shaderBufferInt64Atomics != VK_TRUE) {
-			LOGF("shaderBufferInt64Atomics is not supported\n");
-			skip_device = true;
-		}
-		if (features13.robustImageAccess != VK_TRUE) {
-			LOGF("robustImageAccess is not supported\n");
-			skip_device = true;
-		}
-		if (required_features13.dynamicRendering == VK_TRUE &&
-		    features13.dynamicRendering != VK_TRUE) {
-			LOGF("dynamicRendering is not supported\n");
-			skip_device = true;
-		}
-		if (required_features13.synchronization2 == VK_TRUE &&
-		    features13.synchronization2 != VK_TRUE) {
-			LOGF("synchronization2 is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.sampleRateShading != VK_TRUE) {
-			LOGF("sampleRateShading is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.depthBiasClamp != VK_TRUE) {
-			LOGF("depthBiasClamp is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.shaderClipDistance != VK_TRUE) {
-			LOGF("shaderClipDistance is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.shaderCullDistance != VK_TRUE) {
-			LOGF("shaderCullDistance is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.largePoints != VK_TRUE) {
-			LOGF("largePoints is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.multiViewport != VK_TRUE) {
-			LOGF("multiViewport is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.fillModeNonSolid != VK_TRUE) {
-			LOGF("fillModeNonSolid is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.fragmentStoresAndAtomics != VK_TRUE) {
-			LOGF("fragmentStoresAndAtomics is not supported\n");
-			skip_device = true;
-		}
+		check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge",
+		              required_features12.samplerMirrorClampToEdge);
+		check_feature(features12.timelineSemaphore, "timelineSemaphore",
+		              required_features12.timelineSemaphore);
+		check_feature(features12.shaderOutputLayer, "shaderOutputLayer",
+		              required_features12.shaderOutputLayer);
+		check_feature(features12.shaderOutputViewportIndex, "shaderOutputViewportIndex",
+		              required_features12.shaderOutputViewportIndex);
+		check_feature(features12.bufferDeviceAddress, "bufferDeviceAddress",
+		              required_features12.bufferDeviceAddress);
+		check_feature(features12.shaderBufferInt64Atomics, "shaderBufferInt64Atomics",
+		              required_features12.shaderBufferInt64Atomics);
+		check_feature(features13.robustImageAccess, "robustImageAccess");
+		check_feature(features13.dynamicRendering, "dynamicRendering",
+		              required_features13.dynamicRendering);
+		check_feature(features13.synchronization2, "synchronization2",
+		              required_features13.synchronization2);
+		check_feature(device_features2.features.sampleRateShading, "sampleRateShading");
+		check_feature(device_features2.features.depthBiasClamp, "depthBiasClamp");
+		check_feature(device_features2.features.shaderClipDistance, "shaderClipDistance");
+		check_feature(device_features2.features.shaderCullDistance, "shaderCullDistance");
+		check_feature(device_features2.features.largePoints, "largePoints");
+		check_feature(device_features2.features.multiViewport, "multiViewport");
+		check_feature(device_features2.features.fillModeNonSolid, "fillModeNonSolid");
+		check_feature(device_features2.features.fragmentStoresAndAtomics,
+		              "fragmentStoresAndAtomics");
 
-		if (device_features2.features.samplerAnisotropy != VK_TRUE) {
-			LOGF("samplerAnisotropy is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.robustBufferAccess != VK_TRUE) {
-			LOGF("robustBufferAccess is not supported\n");
-			skip_device = true;
-		}
+		check_feature(device_features2.features.samplerAnisotropy, "samplerAnisotropy");
+		check_feature(device_features2.features.robustBufferAccess, "robustBufferAccess");
+#if defined(__APPLE__)
 		if (device_features2.features.depthBounds != VK_TRUE) {
 			LOGF("depthBounds is not supported\n");
-#if !defined(__APPLE__)
-			skip_device = true;
+		}
+#else
+		check_feature(device_features2.features.depthBounds, "depthBounds");
 #endif
-		}
-		if (device_features2.features.shaderStorageImageWriteWithoutFormat != VK_TRUE) {
-			LOGF("shaderStorageImageWriteWithoutFormat is not supported\n");
-			skip_device = true;
-		}
+		check_feature(device_features2.features.shaderStorageImageWriteWithoutFormat,
+		              "shaderStorageImageWriteWithoutFormat");
 
-		if (device_features2.features.shaderImageGatherExtended != VK_TRUE) {
-			LOGF("shaderImageGatherExtended is not supported\n");
-			skip_device = true;
-		}
+		check_feature(device_features2.features.shaderImageGatherExtended,
+		              "shaderImageGatherExtended");
 
-		if (device_features2.features.independentBlend != VK_TRUE) {
-			LOGF("independentBlend is not supported\n");
-			skip_device = true;
-		}
-		if (device_features2.features.tessellationShader != VK_TRUE) {
-			LOGF("tessellationShader is not supported\n");
-			skip_device = true;
-		}
+		check_feature(device_features2.features.independentBlend, "independentBlend");
+		check_feature(device_features2.features.tessellationShader, "tessellationShader");
 
 		if (!skip_device) {
 			auto available_extensions = EnumerateVulkan<vk::ExtensionProperties>(
@@ -368,8 +309,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 			for (const char* ext: device_extensions) {
 				if (!HasExtension(available_extensions, ext)) {
-					skip_device = true;
-					break;
+					reject(fmt::format("{} is not supported", ext));
 				}
 			}
 
@@ -387,86 +327,46 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 			if (!(candidate_capabilities.capabilities.supportedUsageFlags &
 			      vk::ImageUsageFlagBits::eTransferDst)) {
-				LOGF("Surface cannot be destination of blit\n");
-				skip_device = true;
+				reject("Surface cannot be destination of blit");
 			}
 		}
 
-		if (!skip_device && !CheckFormat(device, vk::Format::eD32Sfloat, true,
-		                                 vk::FormatFeatureFlagBits::eDepthStencilAttachment)) {
-			LOGF("Format vk::Format::eD32Sfloat cannot be used as depth buffer\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eD32SfloatS8Uint, true,
-		                                 vk::FormatFeatureFlagBits::eDepthStencilAttachment)) {
-			LOGF("Format vk::Format::eD32SfloatS8Uint cannot be used as depth buffer\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eD16Unorm, true,
-		                                 vk::FormatFeatureFlagBits::eDepthStencilAttachment)) {
-			LOGF("Format vk::Format::eD16Unorm cannot be used as depth buffer\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eBc3SrgbBlock, true,
-		                                 vk::FormatFeatureFlagBits::eSampledImage |
-		                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-			LOGF("Format vk::Format::eBc3SrgbBlock cannot be used as texture\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eR8G8B8A8Srgb, true,
-		                                 vk::FormatFeatureFlagBits::eSampledImage |
-		                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-			LOGF("Format vk::Format::eR8G8B8A8Srgb cannot be used as texture\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eR8Unorm, true,
-		                                 vk::FormatFeatureFlagBits::eSampledImage |
-		                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-			LOGF("Format vk::Format::eR8Unorm cannot be used as texture\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eR8G8Unorm, true,
-		                                 vk::FormatFeatureFlagBits::eSampledImage |
-		                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-			LOGF("Format vk::Format::eR8G8Unorm cannot be used as texture\n");
-			skip_device = true;
-		}
-
-		if (!skip_device && !CheckFormat(device, vk::Format::eR8G8B8A8Srgb, true,
-		                                 vk::FormatFeatureFlagBits::eStorageImage |
-		                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-			LOGF("Format vk::Format::eR8G8B8A8Srgb cannot be used as texture\n");
-
-			if (!skip_device && !CheckFormat(device, vk::Format::eR8G8B8A8Unorm, true,
-			                                 vk::FormatFeatureFlagBits::eStorageImage |
-			                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-				LOGF("Format vk::Format::eR8G8B8A8Unorm cannot be used as texture\n");
-				skip_device = true;
+		const auto check_format = [&](vk::Format format, vk::FormatFeatureFlags features,
+		                              const char* usage) {
+			if (!skip_device && !CheckFormat(device, format, true, features)) {
+				reject(fmt::format("Format vk::Format::e{} cannot be used as {}",
+				                   vk::to_string(format), usage));
 			}
+		};
+
+		const auto depth_features = vk::FormatFeatureFlagBits::eDepthStencilAttachment;
+		const auto texture_features =
+		    vk::FormatFeatureFlagBits::eSampledImage | vk::FormatFeatureFlagBits::eTransferDst;
+		const auto storage_features =
+		    vk::FormatFeatureFlagBits::eStorageImage | vk::FormatFeatureFlagBits::eTransferDst;
+
+		check_format(vk::Format::eD32Sfloat, depth_features, "depth buffer");
+		check_format(vk::Format::eD32SfloatS8Uint, depth_features, "depth buffer");
+		check_format(vk::Format::eD16Unorm, depth_features, "depth buffer");
+		check_format(vk::Format::eBc3SrgbBlock, texture_features, "texture");
+		check_format(vk::Format::eR8G8B8A8Srgb, texture_features, "texture");
+		check_format(vk::Format::eR8Unorm, texture_features, "texture");
+		check_format(vk::Format::eR8G8Unorm, texture_features, "texture");
+
+		if (!skip_device &&
+		    !CheckFormat(device, vk::Format::eR8G8B8A8Srgb, true, storage_features)) {
+			LOGF("Format vk::Format::eR8G8B8A8Srgb cannot be used as storage image\n");
+			check_format(vk::Format::eR8G8B8A8Unorm, storage_features, "storage image");
 		}
 
-		if (!skip_device && !CheckFormat(device, vk::Format::eB8G8R8A8Srgb, true,
-		                                 vk::FormatFeatureFlagBits::eStorageImage |
-		                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-			LOGF("Format vk::Format::eB8G8R8A8Srgb cannot be used as texture\n");
-
-			if (!skip_device && !CheckFormat(device, vk::Format::eB8G8R8A8Unorm, true,
-			                                 vk::FormatFeatureFlagBits::eStorageImage |
-			                                     vk::FormatFeatureFlagBits::eTransferDst)) {
-				LOGF("Format vk::Format::eB8G8R8A8Unorm cannot be used as texture\n");
-				skip_device = true;
-			}
+		if (!skip_device &&
+		    !CheckFormat(device, vk::Format::eB8G8R8A8Srgb, true, storage_features)) {
+			LOGF("Format vk::Format::eB8G8R8A8Srgb cannot be used as storage image\n");
+			check_format(vk::Format::eB8G8R8A8Unorm, storage_features, "storage image");
 		}
 
 		if (!skip_device && device_properties.limits.maxSamplerAnisotropy < 16.0f) {
-			LOGF("maxSamplerAnisotropy < 16.0f");
-			skip_device = true;
+			reject("maxSamplerAnisotropy < 16.0f");
 		}
 
 		if (skip_device) {
@@ -560,7 +460,15 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		provoking_vertex.pNext = supported_features2.pNext;
 		supported_features2.pNext = &provoking_vertex;
 	}
+	const bool image_atomic_int64_extension =
+	    HasExtension(device_extensions, VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME);
+	vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic_int64 {};
+	if (image_atomic_int64_extension) {
+		image_atomic_int64.pNext = supported_features2.pNext;
+		supported_features2.pNext = &image_atomic_int64;
+	}
 	physical_device.getFeatures2(&supported_features2);
+	graphics.shader_image_int64_atomics_enabled = image_atomic_int64.shaderImageInt64Atomics;
 
 	auto features12 = WindowContext::RequiredVulkan12Features();
 	features12.shaderSharedInt64Atomics = supported_features12.shaderSharedInt64Atomics;
@@ -705,6 +613,11 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		provoking_vertex.transformFeedbackPreservesProvokingVertex = VK_FALSE;
 		create_info.pNext = &provoking_vertex;
 	}
+	if (graphics.shader_image_int64_atomics_enabled) {
+		image_atomic_int64.pNext = const_cast<void*>(create_info.pNext);
+		image_atomic_int64.sparseImageInt64Atomics = VK_FALSE;
+		create_info.pNext = &image_atomic_int64;
+	}
 	create_info.pQueueCreateInfos       = &queue_create_info;
 	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
@@ -1024,11 +937,13 @@ void WindowContext::CreateVulkan() {
 	}
 #endif
 
+	std::string rejected_devices;
 	VulkanFindPhysicalDevice(graphic_ctx.instance, surface, device_extensions, surface_capabilities,
-	                         graphic_ctx.physical_device, graphic_ctx.queue_family);
+	                         graphic_ctx.physical_device, graphic_ctx.queue_family,
+	                         rejected_devices);
 
 	if (graphic_ctx.physical_device == nullptr) {
-		EXIT("Could not find suitable device");
+		EXIT("Could not find suitable device:%s", rejected_devices.c_str());
 	}
 
 	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
@@ -1067,6 +982,7 @@ void WindowContext::CreateVulkan() {
 			graphic_ctx.memory_budget_ext_enabled = true;
 		}
 		for (const auto* extension: {VK_EXT_ROBUSTNESS_2_EXTENSION_NAME,
+		                             VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME,
 		                             VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME,
 		                             VK_EXT_MESH_SHADER_EXTENSION_NAME,
 		                             VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME,

src/graphics/presentation/window/window.cpp

@@ -306,14 +306,20 @@ static void GameEventController([[maybe_unused]] const EventController& f) {
 
 	if (f.added) {
 		auto* pad = SDL_OpenGamepad(f.id);
-		EXIT_NOT_IMPLEMENTED(pad == nullptr);
+		if (pad == nullptr) {
+			LOGF("Controller: ignoring gamepad %d that could not be opened: %s\n", f.id,
+			     SDL_GetError());
+			return;
+		}
 		int id = SDL_GetJoystickID(SDL_GetGamepadJoystick(pad));
 		Controller::Connect(id);
 	}
 
 	if (f.removed) {
-		Controller::Disconnect(f.id);
-		SDL_CloseGamepad(SDL_GetGamepadFromID(f.id));
+		if (auto* pad = SDL_GetGamepadFromID(f.id); pad != nullptr) {
+			Controller::Disconnect(f.id);
+			SDL_CloseGamepad(pad);
+		}
 	}
 
 	if (f.down || f.up) {

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -19,7 +19,6 @@
 
 #include <algorithm>
 #include <array>
-#include <atomic>
 #include <chrono>
 #include <fmt/format.h>
 #include <map>
@@ -523,20 +522,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
 
-	// Temporary workaround for games that compile ray-tracing shaders before
-	// the player can select a mode without ray tracing.
-	if (options.stage == ShaderType::Compute && decoded.has_bvh) {
-		static std::atomic_flag warned = ATOMIC_FLAG_INIT;
-		if (!warned.test_and_set(std::memory_order_relaxed)) {
-			const auto& bvh = decoded.instructions.back();
-			Log::WriteToConsoleAndLog(fmt::format(
-			    "Warning: ray tracing is not implemented; skipping compute dispatches containing "
-			    "BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n",
-			    options.shader_hash, bvh.pc, bvh.opcode_id));
-		}
-		return {.skip_dispatch = true};
-	}
-
 	std::string decoded_dump;
 	if (options.dump_ir) {
 		decoded_dump = Decoder::ProgramToString(decoded);
@@ -609,7 +594,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
 	IR::RewriteToSsa(ir.blocks);
-	IR::ConstantPropagationPass(ir.blocks);
+	IR::ConstantPropagationPass(ir.blocks, ir.wave_size);
 	IR::ResolveControlFlowIdentities(ir);
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
@@ -617,7 +602,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	if (read_lane_stats.rewritten_reads != 0) {
 		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
 		     read_lane_stats.rewritten_reads);
-		IR::ConstantPropagationPass(ir.blocks);
+		IR::ConstantPropagationPass(ir.blocks, ir.wave_size);
 		IR::ResolveControlFlowIdentities(ir);
 		IR::RemoveIdentities(ir.blocks);
 		IR::EliminateDeadCode(ir.blocks);
@@ -632,7 +617,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 		}
 	}
 	IR::TrackResources(ir, decoded, native_cfg);
-	IR::EliminateDeadCode(ir.blocks);
 	TranslateResult result;
 	result.program = std::move(ir);
 	if (options.dump_ir) {
@@ -645,7 +629,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options,
                              const IR::ResourceSpecialization& specialization,
                              uint32_t push_data_start_dword) {
-	EXIT_IF(translated.skip_dispatch);
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
 	IR::ApplyResourceSpecialization(ir, specialization);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -28,7 +28,6 @@ struct TranslateResult {
 	IR::Program program;
 	std::string decoded_dump;
 	std::string cfg_dump;
-	bool        skip_dispatch = false;
 };
 
 struct CompileResult {

src/graphics/shader/recompiler/Tessellation.cpp

@@ -217,7 +217,7 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 			    2, ActiveAddress(*block, it, it->Arg(2), it->Arg(it->NumArgs() - 1u), resolved));
 		}
 	}
-	ConstantPropagationPass(program.blocks);
+	ConstantPropagationPass(program.blocks, program.wave_size);
 	uint32_t reads = 0, writes = 0, factors = 0;
 	for (auto* block: program.blocks) {
 		for (auto it = block->begin(); it != block->end(); ++it) {
@@ -317,7 +317,7 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 			inst.Invalidate();
 		}
 	}
-	ConstantPropagationPass(program.blocks);
+	ConstantPropagationPass(program.blocks, program.wave_size);
 	RemoveIdentities(program.blocks);
 	EliminateDeadCode(program.blocks);
 	LOGF("%s tessellation lowering: reads=%u writes=%u factors=%u\n",

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -41,7 +41,7 @@ void ValidateNativeProgram(const IR::Program& program) {
 			Fail(program, "native shader plan has an invalid image class");
 		}
 		present[static_cast<size_t>(*kind)] = true;
-		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage;
+		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::Dynamic;
 		const auto count   = dynamic ? program.info.images[i].mip_count : 1u;
 		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
 			Fail(program, "native shader plan has an invalid image mip descriptor count");
@@ -122,6 +122,16 @@ void ValidateNativeProgram(const IR::Program& program) {
 			              program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer;
 		       });
 	};
+	const auto local_flat_handle = [&](const IR::Inst& handle) {
+		return !handle.Uses().empty() &&
+		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
+			       if (IR::AddressOpcodeInfoOf(use.user->GetOpcode()).access == IR::AddressAccess::None)
+				       return false;
+			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].kind == IR::ResourceKind::FlatLocal;
+		       });
+	};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto dense = inst.Flags<uint32_t>();
@@ -138,7 +148,7 @@ void ValidateNativeProgram(const IR::Program& program) {
 					if (planning_only_handle(inst)) {
 						break;
 					}
-					if (inst.NumArgs() != 2 || !program.info.uses_dma) {
+					if (inst.NumArgs() != 2 || (!program.info.uses_dma && !local_flat_handle(inst))) {
 						Fail(program, "typed address handle has invalid DMA metadata");
 					}
 					break;
@@ -188,11 +198,16 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (memory_index >= program.memory_info.size()) {
 					Fail(program, "address operation has invalid memory metadata");
 				}
-				if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+				const auto kind = program.memory_info[memory_index].kind;
+				if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
 					if (program.scratch_dwords == 0) {
 						Fail(program, "scratch operation has no per-thread storage");
 					}
 					requirements.function_scratch = true;
+					if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute &&
+					    program.stage != ShaderType::Mesh) {
+						requirements.function_lds = true;
+					}
 				} else if (address_access == IR::AddressAccess::Write) {
 					Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
 				}
@@ -229,7 +244,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
 					Fail(program, "shared operation has invalid resource kind");
 				}
-				if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicOr64) {
+				if (shared_access == IR::SharedAccess::Atomic &&
+				    IR::SharedComponentCount(inst.GetOpcode()) == 2u) {
 					if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) {
 						Fail(program, "64-bit shared atomics require compute LDS");
 					}
@@ -247,6 +263,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				}
 			}
 			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
 				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
 				case IR::ValueOpcode::ReadFirstLane:

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -444,6 +444,18 @@ uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
 }
 
+uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
+}
+
+uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
+}
+
+uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
+}
+
 uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
 	return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -124,13 +124,21 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
 		sampled = 2;
 		if (image.atomic) {
 			EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint);
-			format = spv::ImageFormatR32ui;
+			format = image.atomic64 ? spv::ImageFormatR64ui : spv::ImageFormatR32ui;
+			if (image.atomic64) {
+				state.builder.RequireExtension("SPV_EXT_shader_image_int64");
+				state.builder.RequireCapability(spv::CapabilityInt64);
+				state.builder.RequireCapability(spv::CapabilityInt64Atomics);
+				state.builder.RequireCapability(spv::CapabilityInt64ImageEXT);
+			}
 		}
 	} else {
 		EXIT("invalid image resource class");
 	}
 	const auto& info = ImageDimensionInfoFor(image.dimension);
-	return state.builder.Type(spv::OpTypeImage, ImageScalarType(state, image.numeric_class),
+	const auto scalar_type = image.atomic64 ? TypeScalarU64(state)
+	                                        : ImageScalarType(state, image.numeric_class);
+	return state.builder.Type(spv::OpTypeImage, scalar_type,
 	                          info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed,
 	                          info.multisampled, sampled, format);
 }
@@ -144,20 +152,11 @@ uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
 	}
 }
 
-uint32_t LoadSampledImageDescriptor(EmitterState& state, uint32_t resource) {
-	const auto& image_resource = state.program.info.images.at(resource);
-	EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
-	const auto kind = IR::DescriptorBindingForImage(image_resource);
-	EXIT_IF(!kind.has_value());
-	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
-	const auto variable     = state.image_variables[IR::ImageBindingIndex(*kind)];
-	const auto pointer_type = state.builder.Type(
-	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image_resource));
-	const auto pointer =
-	    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
-	                             "sampled image descriptor array was not emitted");
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip) {
+	const auto pointer = ImageDescriptorPointer(state, resource, mip);
 	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpLoad, ImageType(state, image_resource), image, pointer);
+	state.builder.AddFunction(spv::OpLoad, ImageType(state, state.program.info.images.at(resource)),
+	                          image, pointer);
 	return image;
 }
 
@@ -175,10 +174,10 @@ uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
 	return sampler_id;
 }
 
-uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler) {
+uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip) {
 	const auto& image_resource = state.program.info.images.at(resource);
-	const auto  image          = LoadSampledImageDescriptor(state, resource);
-	const auto  sampler_id     = LoadSamplerDescriptor(state, sampler);
+	EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
+	const auto  image          = LoadImageDescriptor(state, resource, mip);
 	const auto  sampled_image = state.builder.AllocateId();
 	const auto  sampled_type =
 	    state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource));
@@ -186,17 +185,17 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
 	return sampled_image;
 }
 
-uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource) {
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip) {
 	const auto& image = state.program.info.images.at(resource);
-	EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
+	EXIT_IF(mip >= image.mip_count);
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
-	const auto array_index  = ResourceForDescriptor(state, *kind, resource);
+	const auto array_index  = ResourceForDescriptor(state, *kind, resource) + mip;
 	const auto pointer_type = state.builder.Type(
 	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
 	return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
-	                                "storage image descriptor array was not emitted");
+	                                "image descriptor array was not emitted");
 }
 
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
@@ -206,46 +205,15 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 	if (!image.atomic) {
 		state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
 	}
-	const auto kind = IR::DescriptorBindingForImage(image);
-	EXIT_IF(!kind.has_value());
-	const auto array_index = ResourceForDescriptor(state, *kind, resource);
-	const auto image_type  = ImageType(state, image);
-	const auto pointer_type =
-	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, image_type);
-	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
-	const auto LoadAt   = [&](uint32_t array_index) {
-		const auto pointer =
-		    DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
-		                             "storage image descriptor array was not emitted");
-		const auto descriptor = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpLoad, image_type, descriptor, pointer);
-		return descriptor;
+	const auto EmitWrite = [&](uint32_t mip) {
+		state.builder.AddFunction(spv::OpImageWrite, LoadImageDescriptor(state, resource, mip),
+		                          coord, texel);
 	};
-	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
-		state.builder.AddFunction(spv::OpImageWrite, LoadAt(array_index), coord, texel);
+	if (image.mip_mode != IR::ImageMipMode::Dynamic) {
+		EmitWrite(0);
 		return;
 	}
-	if (image.mip_count == 0u) {
-		ExitDescriptorBindingFailure(state, *kind, resource,
-		                             "dynamic storage image has no mip descriptors");
-	}
-
-	const auto            merge_label = state.builder.AllocateId();
-	std::vector<uint32_t> labels(image.mip_count);
-	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
-	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
-		labels[mip] = state.builder.AllocateId();
-		words.push_back(mip);
-		words.push_back(labels[mip]);
-	}
-	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
-	state.builder.AddFunction(words);
-	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
-		EmitLabel(state, labels[mip]);
-		state.builder.AddFunction(spv::OpImageWrite, LoadAt(array_index + mip), coord, texel);
-		state.builder.AddFunction(spv::OpBranch, merge_label);
-	}
-	EmitLabel(state, merge_label);
+	EmitImageMipSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
 }
 
 spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -0,0 +1,348 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+using Vec3 = std::array<uint32_t, 3>;
+
+uint32_t Extract(EmitterState& s, uint32_t type, uint32_t vector, uint32_t index) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeExtract, type, result, vector, index);
+	return result;
+}
+
+uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> components) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, type, result, components);
+	return result;
+}
+
+uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
+	const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
+	const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
+	                            Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
+	const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
+	return result;
+}
+
+uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
+                  Vec3 origin, Vec3 direction) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
+		const auto value = Binary(s, code, f, x, y);
+		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
+		return value;
+	};
+	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
+	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
+	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
+	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
+	const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); };
+	// The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0).
+	const std::array vertices {
+		EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, kind, ConstantU32(s, 2)),
+		Select(s, u, is_kind(0), ConstantU32(s, 1),
+		       Select(s, u, is_kind(3), ConstantU32(s, 4), ConstantU32(s, 3))),
+		Select(s, u, is_kind(2), ConstantU32(s, 4),
+		       Select(s, u, is_kind(3), ConstantU32(s, 0), ConstantU32(s, 2))) };
+	std::array<Vec3, 3> positions;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto first = Binary(s, spv::OpIMul, u, vertices[i], ConstantU32(s, 3));
+		for (uint32_t axis = 0; axis < 3; ++axis)
+			positions[i][axis] = Unary(s, spv::OpBitcast, f,
+			    NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis))));
+	}
+	const auto flag = NodeWord(s, block, ConstantU32(s, 15));
+	const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1]));
+	const auto z_largest = cmp(spv::OpFOrdLessThan,
+	    Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2]));
+	const auto rotate = [&](Vec3 value) {
+		Vec3 result;
+		for (uint32_t i = 0; i < 3; ++i)
+			result[i] = Select(s, f, z_largest, value[i],
+			    Select(s, f, y_largest, value[(i + 2) % 3], value[(i + 1) % 3]));
+		return result;
+	};
+	origin = rotate(origin);
+	direction = rotate(direction);
+	std::array<Vec3, 3> projected;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto position = rotate(positions[i]);
+		Vec3 relative;
+		for (uint32_t axis = 0; axis < 3; ++axis)
+			relative[axis] = op(spv::OpFSub, position[axis], origin[axis]);
+		projected[i] = {op(spv::OpFSub, op(spv::OpFMul, relative[0], direction[2]),
+		                              op(spv::OpFMul, direction[0], relative[2])),
+		                op(spv::OpFSub, op(spv::OpFMul, relative[1], direction[2]),
+		                              op(spv::OpFMul, direction[1], relative[2])), relative[2]};
+	}
+	Vec3 edge, weighted;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto& first = projected[(i + 1) % 3];
+		const auto& second = projected[(i + 2) % 3];
+		edge[i] = op(spv::OpFSub, op(spv::OpFMul, second[0], first[1]),
+		                        op(spv::OpFMul, second[1], first[0]));
+		weighted[i] = op(spv::OpFMul, edge[i], direction[2]);
+	}
+	auto numerator = op(spv::OpFAdd, op(spv::OpFAdd,
+	    op(spv::OpFMul, edge[0], projected[0][2]), op(spv::OpFMul, edge[1], projected[1][2])),
+	    op(spv::OpFMul, edge[2], projected[2][2]));
+	auto denominator = op(spv::OpFAdd, op(spv::OpFAdd, weighted[0], weighted[1]), weighted[2]);
+	uint32_t negative = ConstantBool(s, false), positive = negative;
+	for (auto value: edge) {
+		negative = either(negative, cmp(spv::OpFOrdLessThan, value, zero));
+		positive = either(positive, cmp(spv::OpFOrdGreaterThan, value, zero));
+	}
+	const auto winding = cmp(spv::OpFOrdGreaterThan, denominator, zero);
+	auto missed = either(both(negative, positive), cmp(spv::OpFOrdEqual, denominator, zero));
+	missed = either(missed, Unary(s, spv::OpIsNan, b, numerator));
+	missed = either(missed, cmp(spv::OpFOrdLessThan,
+	    Select(s, f, winding, numerator, Unary(s, spv::OpFNegate, f, numerator)), zero));
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto first = projected[(i + 1) % 3][1], second = projected[(i + 2) % 3][1];
+		const auto first_zero = cmp(spv::OpFOrdEqual, first, zero);
+		const auto horizontal = both(first_zero, cmp(spv::OpFOrdEqual, second, zero));
+		const auto below = either(cmp(spv::OpFOrdLessThan, first, zero),
+		    both(first_zero, cmp(spv::OpFOrdGreaterThan, second, zero)));
+		const auto right = Binary(s, spv::OpLogicalNotEqual, b, below, winding);
+		const auto excluded = Select(s, b, horizontal,
+		    cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right);
+		missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded));
+	}
+	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
+	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
+	const auto coordinate = [&](uint32_t shift) {
+		const auto index = Binary(s, spv::OpBitwiseAnd, u,
+		    Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3));
+		return Unary(s, spv::OpBitcast, u, Select(s, f,
+		    cmp(spv::OpIEqual, index, ConstantU32(s, 1)), weighted[1],
+		    Select(s, f, cmp(spv::OpIEqual, index, ConstantU32(s, 2)), weighted[2], weighted[0])));
+	};
+	const std::array result {
+		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)),
+		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)),
+		Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)),
+		Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) };
+	return Vector(s, TypeU32Vector(s, 4), result);
+}
+
+std::array<uint32_t, 6> Box(EmitterState& s, uint32_t first, uint32_t second,
+                           uint32_t half, uint32_t child) {
+	const auto f = TypeF32(s);
+	const auto half_label = s.builder.AllocateId(), full_label = s.builder.AllocateId();
+	const auto merge = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+	s.builder.AddFunction(spv::OpBranchConditional, half, half_label, full_label);
+	EmitLabel(s, half_label);
+	std::array<uint32_t, 6> packed, full, result;
+	for (uint32_t pair = 0; pair < 3; ++pair) {
+		const auto word = NodeWord(s, first, ConstantU32(s, 4 + child * 3 + pair));
+		const auto unpacked = EmitGlsl<GLSLstd450UnpackHalf2x16, IR::Type::F32x2>(s, word);
+		packed[pair * 2] = Extract(s, f, unpacked, 0);
+		packed[pair * 2 + 1] = Extract(s, f, unpacked, 1);
+	}
+	s.builder.AddFunction(spv::OpBranch, merge);
+	EmitLabel(s, full_label);
+	for (uint32_t component = 0; component < 6; ++component) {
+		const auto word = 4 + child * 6 + component;
+		full[component] = Unary(s, spv::OpBitcast, f,
+		    NodeWord(s, word < 16 ? first : second, ConstantU32(s, word % 16)));
+	}
+	s.builder.AddFunction(spv::OpBranch, merge);
+	EmitLabel(s, merge);
+	for (uint32_t component = 0; component < 6; ++component) {
+		result[component] = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpPhi, f, result[component], packed[component], half_label,
+		                      full[component], full_label);
+	}
+	return result;
+}
+
+uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
+               uint32_t grow, uint32_t sort, uint32_t extent, const Vec3& origin, const Vec3& inverse) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	std::array<uint32_t, 4> depths, indices;
+	Vec3 forward;
+	for (uint32_t axis = 0; axis < 3; ++axis)
+		forward[axis] = Binary(s, spv::OpFOrdGreaterThanEqual, b, inverse[axis], zero);
+	for (uint32_t child = 0; child < 4; ++child) {
+		const auto box = Box(s, first, second, half, child);
+		Vec3 near, far;
+		for (uint32_t axis = 0; axis < 3; ++axis) {
+			const auto lo = Binary(s, spv::OpFMul, f,
+			    Binary(s, spv::OpFSub, f, box[axis], origin[axis]), inverse[axis]);
+			const auto hi = Binary(s, spv::OpFMul, f,
+			    Binary(s, spv::OpFSub, f, box[axis + 3], origin[axis]), inverse[axis]);
+			near[axis] = Select(s, f, forward[axis], lo, hi);
+			far[axis] = Select(s, f, forward[axis], hi, lo);
+		}
+		const auto entry = EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s,
+		    EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s, near[0], near[1]), near[2]);
+		const auto exit = EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s,
+		    EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s, far[0], far[1]), far[2]);
+		// Valid hits have nonnegative exit distance. Saturate ULP growth at +infinity.
+		const auto bits = Unary(s, spv::OpBitcast, u, Select(s, f,
+		    Binary(s, spv::OpFOrdEqual, b, exit, zero), zero, exit));
+		const auto grown_bits = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+		    Binary(s, spv::OpIAdd, u, bits, grow), ConstantU32(s, 0x7f800000));
+		const auto grown = Unary(s, spv::OpBitcast, f, grown_bits);
+		auto hit = Binary(s, spv::OpLogicalAnd, b,
+		    Binary(s, spv::OpFOrdGreaterThanEqual, b, exit, zero),
+		    Binary(s, spv::OpFOrdLessThan, b, entry, extent));
+		hit = Binary(s, spv::OpLogicalAnd, b, hit,
+		    Binary(s, spv::OpFOrdLessThanEqual, b, entry, grown));
+		depths[child] = Unary(s, spv::OpBitcast, u, Select(s, f, hit,
+		    Select(s, f, Binary(s, spv::OpFOrdGreaterThan, b, entry, zero), entry, zero),
+		    ConstantF32(s, 0x7f800000)));
+		indices[child] = Select(s, u, hit, NodeWord(s, first, ConstantU32(s, child)), ConstantU32(s, ~0u));
+	}
+	// A select network keeps the original child order when sorting is disabled.
+	constexpr std::array<std::pair<uint32_t, uint32_t>, 5> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2}}};
+	for (size_t step = 0; step < network.size(); ++step) {
+		const auto [a, c] = network[step];
+		const auto swap = Binary(s, spv::OpLogicalAnd, b, sort,
+		    Binary(s, spv::OpUGreaterThan, b, depths[a], depths[c]));
+		const auto depth = depths[a], index = indices[a];
+		const auto later = [&](uint32_t position) {
+			return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) {
+				return pair.first == position || pair.second == position;
+			});
+		};
+		if (later(a)) depths[a] = Select(s, u, swap, depths[c], depth);
+		if (later(c)) depths[c] = Select(s, u, swap, depth, depths[c]);
+		indices[a] = Select(s, u, swap, indices[c], index);
+		indices[c] = Select(s, u, swap, index, indices[c]);
+	}
+	return Vector(s, TypeU32Vector(s, 4), indices);
+}
+
+} // namespace
+
+void DefineBvhIntersect(EmitterState& s) {
+	if (!s.requirements.bvh) return;
+	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
+	const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
+	s.bvh_intersect_function = s.builder.AllocateId();
+	s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
+	s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
+	                      spv::FunctionControlMaskNone, signature);
+	std::array<uint32_t, 6> args;
+	const std::array types {vec4, u, f, vec3, vec3, vec3};
+	for (uint32_t i = 0; i < args.size(); ++i) {
+		args[i] = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
+	}
+	EmitLabel(s, s.builder.AllocateId());
+	const auto descriptor = args[0], node = args[1], extent = args[2];
+	Vec3 origin, direction, inverse;
+	for (uint32_t i = 0; i < 3; ++i) {
+		origin[i] = Extract(s, f, args[3], i);
+		direction[i] = Extract(s, f, args[4], i);
+		inverse[i] = Extract(s, f, args[5], i);
+	}
+	std::array<uint32_t, 4> words;
+	for (uint32_t i = 0; i < 4; ++i) words[i] = Extract(s, u, descriptor, i);
+	const auto and_bits = [&](uint32_t value, uint32_t mask) {
+		return Binary(s, spv::OpBitwiseAnd, u, value, ConstantU32(s, mask));
+	};
+	const auto shr = [&](uint32_t value, uint32_t shift) {
+		return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
+	};
+	const auto kind = and_bits(node, 7);
+	const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
+	const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
+	const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
+	const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
+	const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
+	const auto grow = and_bits(shr(words[1], 23), 0xff);
+	const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
+	    DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
+	const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
+	const auto last_index = Binary(s, spv::OpIAdd, wide, index,
+	    Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
+	const auto address = Binary(s, spv::OpIAdd, wide, base,
+	    Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
+	const auto last_address = Binary(s, spv::OpIAdd, wide, base,
+	    Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
+	auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
+	const auto require = [&](uint32_t condition) {
+		valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
+	};
+	require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
+	require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
+	require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
+	require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
+	for (uint32_t i = 0; i < 3; ++i) {
+		for (const auto value: {origin[i], direction[i], inverse[i]})
+			require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, value)));
+		require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsInf, b, origin[i])));
+	}
+	const std::array triangle_invalid {ConstantU32(s, 0x7f800000), ConstantU32(s, 0x3f800000),
+	                                  ConstantU32(s, ~0u), ConstantU32(s, 0)};
+	const std::array box_invalid {ConstantU32(s, ~0u), ConstantU32(s, ~0u),
+	                             ConstantU32(s, ~0u), ConstantU32(s, ~0u)};
+	std::array<uint32_t, 4> invalid_words;
+	for (uint32_t i = 0; i < 4; ++i)
+		invalid_words[i] = Select(s, u, triangle, triangle_invalid[i], box_invalid[i]);
+	const auto invalid = Vector(s, vec4, invalid_words);
+	const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
+		const auto first = GetBdaPointer(s, address);
+		const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
+		});
+		const auto present = Binary(s, spv::OpLogicalAnd, b,
+		    Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
+		    Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
+		        Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
+		return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
+			const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
+			const auto merge = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+			s.builder.AddFunction(spv::OpBranchConditional, triangle, tri_label, box_label);
+			EmitLabel(s, tri_label);
+			const auto tri_value = Triangle(s, first, kind, bary, origin, direction);
+			const auto tri_exit = s.current_label;
+			s.builder.AddFunction(spv::OpBranch, merge);
+			EmitLabel(s, box_label);
+			const auto box_value = Boxes(s, first, second, half, grow, sort, extent, origin, inverse);
+			const auto box_exit = s.current_label;
+			s.builder.AddFunction(spv::OpBranch, merge);
+			EmitLabel(s, merge);
+			const auto value = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpPhi, vec4, value, tri_value, tri_exit, box_value, box_exit);
+			return value;
+		});
+	});
+	s.builder.AddFunction(spv::OpReturnValue, result);
+	s.builder.AddFunction(spv::OpFunctionEnd);
+}
+
+uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto* ray = ctx.ImageAddress(inst.Arg(1));
+	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
+		ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
+	const auto vector = [&](uint32_t first) {
+		Vec3 values;
+		for (uint32_t i = 0; i < 3; ++i)
+			values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
+		return Vector(s, TypeF32Vector(s, 3), values);
+	};
+	const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
+	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
+	const auto origin = vector(2), direction = vector(5), inverse = vector(8);
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
+	    ConstantU32CompositeZero(s, 4), [&] {
+		const auto result = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpFunctionCall, TypeU32Vector(s, 4), result,
+		    s.bvh_intersect_function, descriptor, node, extent, origin, direction, inverse);
+		return result;
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -452,6 +452,11 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
 		return;
 	}
+	// Skip dormant color exports after their valid mask; MRT1 is reserved for logical alpha.
+	if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+	    exp.index != 0 && state.input_info.pixel->alpha_blend_source_remap) {
+		return;
+	}
 	EmitIfCondition(state, exec, [&]() {
 		const auto data = ctx.Arg(inst, 0);
 		if (exp.kind == IR::ExportTargetKind::Primitive) {
@@ -492,6 +497,18 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const bool uint_output = MrtOutputMode(state, exp) == 7u;
 		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
+		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+		    exp.index == 0 && !uint_output && state.input_info.pixel->alpha_blend_source_remap) {
+			// Broadcast logical alpha before swizzling the primary output.
+			const auto blend_output =
+			    OutputVariableForExport(state, {.kind = IR::ExportTargetKind::Mrt, .index = 1});
+			if (blend_output != 0) {
+				const auto alpha = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpVectorShuffle, vector_type, alpha, value, value,
+				                          3u, 3u, 3u, 3u);
+				state.builder.AddFunction(spv::OpStore, blend_output, alpha);
+			}
+		}
 		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
 		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
 			const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -3,9 +3,7 @@
 #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
 
 #include <algorithm>
-#include <atomic>
 #include <bit>
-#include <cstdio>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -86,6 +84,46 @@ uint32_t ZeroF32(EmitterState& state) {
 	return ConstantF32(state, 0);
 }
 
+uint32_t GatherMip(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                   const IR::Inst& address, const ImageSampleLayout& layout, uint32_t mip_count) {
+	auto& state = ctx.state;
+	const auto Extract = [&](uint32_t word, uint32_t first, uint32_t count) {
+		return EmitBitFieldUExtract(state, word, ConstantU32(state, first),
+		                           ConstantU32(state, count));
+	};
+	const auto LodFixed = [&](uint32_t word, uint32_t first) {
+		return Binary(state, spv::OpFMul, TypeF32(state),
+		              Unary(state, spv::OpConvertUToF, TypeF32(state), Extract(word, first, 12)),
+		              ConstantF32Value(state, 1.0f / 256.0f));
+	};
+	const auto sampler_control = ctx.Arg(inst, 6);
+	const auto preclamp = Binary(state, spv::OpINotEqual, TypeBool(state),
+	                            Extract(sampler_control, 28, 1), ConstantU32(state, 0));
+	const auto half = ConstantF32Value(state, 0.5f);
+	const auto zero = ZeroF32(state);
+	const auto base = Unary(state, spv::OpConvertUToF, TypeF32(state),
+	                        Extract(ctx.Arg(inst, 4), 12, 4));
+	const auto minimum = Binary(state, spv::OpFSub, TypeF32(state),
+	                            LodFixed(ctx.Arg(inst, 3), 8), base);
+	const auto last = ConstantF32Value(state, static_cast<float>(mip_count - 1u));
+	auto lod = Binary(state, spv::OpFAdd, TypeF32(state),
+	                  AddressF32(ctx, mem, address, layout.lod),
+	                  Select(state, TypeF32(state), preclamp, half, zero));
+	lod = EmitGlsl<GLSLstd450FClamp, IR::Type::F32>(
+	    state, lod, LodFixed(ctx.Arg(inst, 5), 0), LodFixed(ctx.Arg(inst, 5), 12));
+	// T# minimum LOD is in physical mip space and applies after the S# view-relative clamp.
+	lod = EmitGlsl<GLSLstd450FMax, IR::Type::F32>(state, lod, minimum);
+	lod = EmitGlsl<GLSLstd450FClamp, IR::Type::F32>(state, lod, zero, last);
+	lod = Binary(state, spv::OpFAdd, TypeF32(state), lod,
+	             Select(state, TypeF32(state), preclamp, zero, half));
+	// The clamped value is nonnegative, so conversion performs the integer mip rounding.
+	const auto mip = EmitUMin32(state, Unary(state, spv::OpConvertFToU, TypeU32(state), lod),
+	                           ConstantU32(state, mip_count - 1u));
+	const auto mip_none = Binary(state, spv::OpIEqual, TypeBool(state),
+	                            Extract(sampler_control, 26, 2), ConstantU32(state, 0));
+	return Select(state, TypeU32(state), mip_none, ConstantU32(state, 0), mip);
+}
+
 uint32_t CubeAxis(EmitterState& state, uint32_t value) {
 	return Binary(state, spv::OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
 }
@@ -252,7 +290,7 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                          const IR::Inst& address) {
 	const auto  dimension = ctx.state.program.info.images.at(mem.resource).dimension;
 	const auto& info      = ImageDimensionInfoFor(dimension);
-	const auto  image     = LoadSampledImageDescriptor(ctx.state, mem.resource);
+	const auto  image     = LoadImageDescriptor(ctx.state, mem.resource);
 	const auto  size      = ctx.state.builder.AllocateId();
 	if (info.multisampled != 0u) {
 		ctx.state.builder.AddFunction(spv::OpImageQuerySize,
@@ -433,7 +471,7 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
                                     uint32_t coord, Prospero::TextureNumericClass numeric_class) {
 	auto& state = ctx.state;
 	state.builder.RequireCapability(spv::CapabilityImageQuery);
-	const auto image = LoadSampledImageDescriptor(state, mem.resource);
+	const auto image = LoadImageDescriptor(state, mem.resource);
 	const auto width = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpImageQuerySizeLod, TypeU32(state), width, image,
 	                          ConstantU32(state, 0));
@@ -446,7 +484,8 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
 	                                 Binary(state, spv::OpFMul, TypeF32(state), coord, width_f32),
 	                                 ConstantF32(state, 0x3f000000u)));
 
-	const auto sampled     = MakeSampledImage(state, mem.resource, mem.sampler);
+	const auto sampled = MakeSampledImage(state, mem.resource,
+	                                     LoadSamplerDescriptor(state, mem.sampler));
 	const auto vector_type = ImageVectorType(state, numeric_class, 4);
 	const auto scalar_type = ImageScalarType(state, numeric_class);
 	const auto component =
@@ -539,19 +578,6 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 	return PackImageTexel(ctx, mem, texel);
 }
 
-spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) {
-	switch (opcode) {
-		case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
-		case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
-		case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
-		case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
-		case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
-		case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
-		case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
-		default: return spv::OpNop;
-	}
-}
-
 } // namespace
 
 void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -571,7 +597,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (op == IR::ValueOpcode::ImageQueryLod) {
 		state.builder.RequireCapability(spv::CapabilityImageQuery);
 		const auto dimension = image.dimension;
-		const auto sampled   = MakeSampledImage(state, mem.resource, mem.sampler);
+		const auto sampled = MakeSampledImage(state, mem.resource,
+		                                     LoadSamplerDescriptor(state, mem.sampler));
 		const auto lod       = state.builder.AllocateId();
 		state.builder.AddFunction(
 		    spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
@@ -601,7 +628,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		    EmitValueOrDefaultIfCondition(
 		        state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4),
 		        [&]() {
-			        const auto descriptor = LoadSampledImageDescriptor(state, mem.resource);
+			        const auto descriptor = LoadImageDescriptor(state, mem.resource);
 			        const auto color      = state.builder.AllocateId();
 			        const auto coord      = CoordU32(ctx, mem, *address, dimension);
 			        if (dimension_info.multisampled != 0u) {
@@ -625,7 +652,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto dimension  = image.dimension;
 		EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
 			const auto mip_lod =
-			    state.program.info.images[mem.resource].mip_mode == IR::ImageMipMode::DynamicStorage
+			    state.program.info.images[mem.resource].mip_mode == IR::ImageMipMode::Dynamic
 			        ? LodU32(ctx, mem, *address, dimension)
 			        : 0u;
 			const auto coord = CoordU32(ctx, mem, *address, dimension);
@@ -648,13 +675,6 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (op == IR::ValueOpcode::ImageGatherRaw) {
 			const auto coord = CoordF32(ctx, mem, *address, layout.coord,
 			                            dimension_info.coordinate_components, image.cube);
-			if (HasFlag(mem, Decoder::ImageSampleFlagLod)) {
-				static std::atomic_flag warned = ATOMIC_FLAG_INIT;
-				if (!warned.test_and_set(std::memory_order_relaxed)) {
-					std::fputs("Warning: approximating IMAGE_GATHER4_L at mip level 0; explicit LOD is ignored.\n",
-					           stderr);
-				}
-			}
 			if (dimension == ImageDimension::Dim1D) {
 				if (dref || !HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
 				    HasFlag(mem, Decoder::ImageSampleFlagOffset) ||
@@ -671,44 +691,48 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				ctx.Fail(inst, "has an unsupported 1D-array gather");
 				return;
 			}
-			const auto            sampled = MakeSampledImage(state, mem.resource, mem.sampler);
-			const auto            sample  = state.builder.AllocateId();
-			std::vector<uint32_t> words;
+			const auto result_numeric_class = dref ? Prospero::TextureNumericClass::Float
+			                                       : numeric_class;
+			const auto result_type = ImageVectorType(state, result_numeric_class, 4);
+			uint32_t component_or_dref;
 			if (dref) {
-				auto dref_value = ZeroF32(state);
-				if (layout.dref != NoImageComponent) {
-					dref_value = AddressF32(ctx, mem, *address, layout.dref);
-				}
-				words = {spv::OpImageDrefGather,
-				         TypeF32Vector(state, 4),
-				         sample,
-				         sampled,
-				         coord,
-				         dref_value};
+				component_or_dref = layout.dref != NoImageComponent
+				                        ? AddressF32(ctx, mem, *address, layout.dref)
+				                        : ZeroF32(state);
 			} else {
-				uint32_t component = 0;
-				if (ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid) {
-					component = ImageGatherComponent(mem.dmask);
-				}
-				words = {spv::OpImageGather,
-				         ImageVectorType(state, numeric_class, 4),
-				         sample,
-				         sampled,
-				         coord,
-				         ConstantU32(state, component)};
+				component_or_dref = ConstantU32(
+				    state, ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid
+				               ? ImageGatherComponent(mem.dmask) : 0u);
 			}
+			uint32_t operand_mask = 0;
+			uint32_t offset = 0;
 			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
-				words.push_back(spv::ImageOperandsConstOffsetsMask);
-				words.push_back(HorizontalOffsets(state, dimension));
+				operand_mask = spv::ImageOperandsConstOffsetsMask;
+				offset = HorizontalOffsets(state, dimension);
 			} else if (layout.offset != NoImageComponent) {
-				words.push_back(spv::ImageOperandsOffsetMask);
-				words.push_back(PackedOffset(ctx, mem, *address, layout, dimension));
-			}
-			state.builder.AddFunction(words);
-			auto result_numeric_class = numeric_class;
-			if (dref) {
-				result_numeric_class = Prospero::TextureNumericClass::Float;
+				operand_mask = spv::ImageOperandsOffsetMask;
+				offset = PackedOffset(ctx, mem, *address, layout, dimension);
 			}
+			const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler);
+			const auto EmitGather = [&](uint32_t mip) {
+				const auto sampled = MakeSampledImage(state, mem.resource, sampler_id, mip);
+				const auto sample = state.builder.AllocateId();
+				std::vector<uint32_t> words {
+				    static_cast<uint32_t>(dref ? spv::OpImageDrefGather : spv::OpImageGather), result_type,
+				    sample, sampled, coord, component_or_dref};
+				if (operand_mask != 0u) {
+					words.push_back(operand_mask);
+					words.push_back(offset);
+				}
+				state.builder.AddFunction(words);
+				return sample;
+			};
+			const auto sample = image.mip_mode == IR::ImageMipMode::Dynamic && image.mip_count > 1u
+			                        ? EmitImageMipSwitch(
+			                              state, GatherMip(ctx, inst, mem, *address, layout,
+			                                               image.mip_count),
+			                              image.mip_count, result_type, EmitGather)
+			                        : EmitGather(0);
 			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
 			                              result_numeric_class, false, mem, true));
 			return;
@@ -751,13 +775,14 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			operand_mask |= spv::ImageOperandsBiasMask;
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
+		const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler);
 		const auto EmitSample = [&](uint32_t resource) {
 			const auto& candidate = state.program.info.images[resource];
 			const auto coord =
 			    CoordF32(ctx, mem, *address, layout.coord,
 			             ImageDimensionInfoFor(candidate.dimension).coordinate_components,
 			             candidate.cube);
-			const auto            sampled = MakeSampledImage(state, resource, mem.sampler);
+			const auto            sampled = MakeSampledImage(state, resource, sampler_id);
 			const auto            sample  = state.builder.AllocateId();
 			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};
 			if (dref) {
@@ -876,15 +901,18 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
 		return;
 	}
-	const auto atomic_opcode = ImageAtomicOpcode(op);
 	if (image_info.access == IR::ImageAccess::Atomic) {
 		const auto dimension = image.dimension;
-		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
+		const auto result_type = TypeId(state, inst.GetType());
+		const auto zero = image.atomic64 ? ConstantU64(state, 0) : ConstantU32(state, 0);
+		ctx.Define(inst, EmitValueOrDefaultIfCondition(
+		                     state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() {
 			           const auto pointer      = state.builder.AllocateId();
 			           const auto pointer_type = state.builder.Type(
-			               spv::OpTypePointer, spv::StorageClassImage, TypeU32(state));
+			               spv::OpTypePointer, spv::StorageClassImage,
+			               image.atomic64 ? TypeScalarU64(state) : TypeU32(state));
 			           state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer,
-			                                     StorageImageDescriptorPointer(state, mem.resource),
+			                                     ImageDescriptorPointer(state, mem.resource),
 			                                     CoordU32(ctx, mem, *address, dimension),
 			                                     ConstantU32(state, 0));
 			           if (op == IR::ValueOpcode::ImageAtomicFMin32 ||
@@ -896,12 +924,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 					                                   op == IR::ValueOpcode::ImageAtomicFMax32);
 				                               });
 			           }
-			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer,
-			                                     ConstantU32(state, spv::ScopeDevice),
-			                                     ConstantU32(state, spv::MemorySemanticsMaskNone),
-			                                     ctx.Arg(inst, 2));
-			           EmitDeviceAtomicMemoryBarrier(state);
+			           const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
+			           EmitAtomicMemoryBarrier(state, IR::ResourceKind::Image);
 			           return old;
 		           }));
 		return;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -111,6 +111,9 @@ uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
@@ -133,6 +136,9 @@ uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
 }
 inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
+EMIT_NATIVE(FPOrdEqual64, OpFOrdEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThanEqual64, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThanEqual64, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
 inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
 inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
 inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
@@ -149,6 +155,7 @@ inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPAdd64, OpFAdd, F64, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t)
 inline constexpr auto EmitFPFma64 =
     EmitGlsl<GLSLstd450Fma, IR::Type::F64, uint32_t, uint32_t, uint32_t>;
@@ -222,7 +229,7 @@ void                  EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst
 void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
-void                  EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -258,6 +265,7 @@ inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicAnd64     = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
@@ -279,6 +287,7 @@ inline constexpr auto EmitSharedAtomicFMin32    = EmitSharedFloatAtomic;
 inline constexpr auto EmitSharedAtomicFMax32    = EmitSharedFloatAtomic;
 inline constexpr auto EmitSharedAtomicSwap32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicIAdd64    = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicISub32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicInc32     = EmitSharedIncDec;
 inline constexpr auto EmitSharedAtomicDec32     = EmitSharedIncDec;
@@ -288,21 +297,27 @@ inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicOr64      = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicXor32     = EmitAtomic32;
 inline constexpr auto EmitDataAppend            = EmitAppendConsume;
 inline constexpr auto EmitDataConsume           = EmitAppendConsume;
 uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
 inline constexpr auto EmitImageQueryDimensions = EmitImage;
 inline constexpr auto EmitImageQueryLod        = EmitImage;
 inline constexpr auto EmitImageRead            = EmitImage;
 inline constexpr auto EmitImageWrite           = EmitImage;
 inline constexpr auto EmitImageSampleRaw       = EmitImage;
 inline constexpr auto EmitImageGatherRaw       = EmitImage;
+inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage;
 inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
 inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicSMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicSMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
+inline constexpr auto EmitImageAtomicUMax64    = EmitImage;
 inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
 inline constexpr auto EmitImageAtomicOr32      = EmitImage;
 inline constexpr auto EmitImageAtomicXor32     = EmitImage;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -19,6 +19,7 @@
 #include <map>
 #include <spirv/unified1/GLSL.std.450.h>
 #include <string>
+#include <type_traits>
 #include <unordered_map>
 #include <utility>
 #include <vector>
@@ -58,6 +59,7 @@ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
 const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 
 struct SpirvRequirements {
+	bool bvh                          = false;
 	bool subgroup_ballot              = false;
 	bool subgroup_shuffle             = false;
 	bool subgroup_local_invocation_id = false;
@@ -83,6 +85,20 @@ struct EmitterState {
 	Builder                                          builder;
 	const IR::Program&                               program;
 	ShaderStageInputInfo                             input_info;
+	uint32_t                                        void_type = 0;
+	uint32_t                                        bool_type = 0;
+	uint32_t                                        u32_type = 0;
+	uint32_t                                        native_u64_type = 0;
+	uint32_t                                        i32_type = 0;
+	uint32_t                                        f32_type = 0;
+	uint32_t                                        f64_type = 0;
+	uint32_t                                        u32_pair_type = 0;
+	uint32_t                                        i32_pair_type = 0;
+	uint32_t                                        function_type = 0;
+	std::array<uint32_t, 3>                          bool_vector_types {};
+	std::array<uint32_t, 3>                          u32_vector_types {};
+	std::array<uint32_t, 3>                          i32_vector_types {};
+	std::array<uint32_t, 3>                          f32_vector_types {};
 	std::array<uint32_t, 6>                          tess_variables {};
 	uint32_t                                         tess_inner_variable = 0;
 	uint32_t                                         tess_patch_base     = 0;
@@ -96,6 +112,7 @@ struct EmitterState {
 	uint32_t                                         bda_pagetable_variable  = 0;
 	uint32_t                                         fault_buffer_variable   = 0;
 	uint32_t                                         bda_pointer_function    = 0;
+	uint32_t                                         bvh_intersect_function  = 0;
 	uint32_t                                         gds_variable            = 0;
 	uint32_t                                         gds_length              = 0;
 	uint32_t                                         push_constant_variable  = 0;
@@ -298,13 +315,14 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image);
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension);
 
-uint32_t LoadSampledImageDescriptor(EmitterState& state, uint32_t resource);
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0);
 
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler);
 
-uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler);
-
-uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource);
+uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id,
+                          uint32_t mip = 0);
 
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
                            uint32_t texel);
@@ -420,7 +438,12 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits);
 uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
                                   uint32_t component, uint32_t raw);
 
-void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
+spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode);
+
+uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
+                             uint32_t scope);
+
+void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind);
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value);
@@ -487,8 +510,51 @@ uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 void EmitProgram(EmitterState& state);
 
 void DefineGetBdaPointer(EmitterState& state);
+void DefineBvhIntersect(EmitterState& state);
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
+uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
+uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
 
 // These templates accept local lambdas from several emitter translation units.
+template <typename Fn>
+auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_count,
+                        uint32_t result_type, Fn&& emit) {
+	constexpr bool has_result = !std::is_void_v<std::invoke_result_t<Fn, uint32_t>>;
+	EXIT_IF(mip_count == 0u);
+	const auto merge_label = state.builder.AllocateId();
+	std::vector<uint32_t> labels(mip_count);
+	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
+	for (uint32_t mip = 0; mip < mip_count; mip++) {
+		labels[mip] = state.builder.AllocateId();
+		words.push_back(mip);
+		words.push_back(labels[mip]);
+	}
+	if constexpr (has_result) {
+		words[2] = labels.front();
+	}
+	state.builder.AddFunction(spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone);
+	state.builder.AddFunction(words);
+	std::vector<uint32_t> phi;
+	if constexpr (has_result) {
+		phi = {spv::OpPhi, result_type, state.builder.AllocateId()};
+	}
+	for (uint32_t mip = 0; mip < mip_count; mip++) {
+		EmitLabel(state, labels[mip]);
+		if constexpr (has_result) {
+			phi.push_back(emit(mip));
+			phi.push_back(state.current_label);
+		} else {
+			emit(mip);
+		}
+		state.builder.AddFunction(spv::OpBranch, merge_label);
+	}
+	EmitLabel(state, merge_label);
+	if constexpr (has_result) {
+		state.builder.AddFunction(phi);
+		return phi[2];
+	}
+}
+
 template <typename Fn>
 void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
 	const auto then_label  = state.builder.AllocateId();
@@ -528,14 +594,7 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
 
 template <typename Fn>
 uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
-	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
-	const auto memory = [&] {
-		switch (kind) {
-			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
-			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
-			default: return spv::MemorySemanticsUniformMemoryMask;
-		}
-	}();
+	const auto scope     = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto preheader = state.builder.AllocateId();
 	const auto header    = state.builder.AllocateId();
 	const auto cont      = state.builder.AllocateId();
@@ -564,8 +623,7 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
 	EmitLabel(state, cont);
 	state.builder.AddFunction(spv::OpBranch, header);
 	EmitLabel(state, merge);
-	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
-	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
+	EmitAtomicMemoryBarrier(state, kind);
 	return observed;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -1,6 +1,7 @@
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <algorithm>
 
@@ -105,31 +106,23 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad
 	return result;
 }
 
-uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
-                            uint32_t high) {
-	auto& state     = ctx.state;
-	auto  immediate = static_cast<int32_t>(mem.offset);
+uint32_t AddAddressOffset(EmitterState& state, const IR::MemoryInfo& mem, uint32_t low,
+                          uint32_t& high) {
+	const auto immediate = static_cast<int32_t>(mem.offset);
+	if (immediate == 0) return low;
 	const auto immediate_low  = ConstantU32(state, static_cast<uint32_t>(immediate));
 	const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
-	low                      = AddU64Low(state, low, high, immediate_low, immediate_high, high);
+	return AddU64Low(state, low, high, immediate_low, immediate_high, high);
+}
+
+uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
+                            uint32_t high) {
+	auto& state = ctx.state;
+	low = AddAddressOffset(state, mem, low, high);
 	const auto valid = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
 	return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
 }
 
-uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
-	return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
-	                              static_cast<uint32_t>(value),
-	                              static_cast<uint32_t>(value >> 32u));
-}
-
-uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
-	const auto low64  = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
-	const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
-	                           Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
-	                           ConstantDeviceAddress(state, 32));
-	return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
-}
-
 uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
 	auto& state = ctx.state;
 	auto  low   = ctx.Arg(inst, 1);
@@ -186,17 +179,9 @@ void RecordBdaFault(EmitterState& state, uint32_t page) {
 	                          Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit));
 }
 
-uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
-	auto&      state  = ctx.state;
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
-	                          state.bda_pointer_function, address);
-	return result;
-}
-
 uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
 	auto&      state   = ctx.state;
-	const auto bda     = GetBdaPointer(ctx, address);
+	const auto bda     = GetBdaPointer(state, address);
 	const auto present =
 	    Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
 	return EmitValueOrZeroIfCondition(state, present, [&]() {
@@ -511,6 +496,50 @@ void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resour
 	                              resource.memory_access);
 }
 
+template <typename Fn>
+void ForEachLocalFlatAccess(ValueEmitContext& ctx, const IR::Inst& inst, Fn&& emit) {
+	auto& state = ctx.state;
+	auto mem = ctx.Memory(inst);
+	auto high = ctx.Arg(inst, 2);
+	const auto low = AddAddressOffset(state, mem, ctx.Arg(inst, 1), high);
+	const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), low, ConstantU32(state, 2));
+	const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), ConstantU32(state, 0),
+	    Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, 3)));
+	for (const auto kind: {IR::ResourceKind::Scratch, IR::ResourceKind::Lds}) {
+		mem.kind = kind;
+		const auto resource = PrepareMemoryResourceAccess(state, mem);
+		const auto aperture = kind == IR::ResourceKind::Scratch ? Decoder::PrivateApertureHigh
+		                                                       : Decoder::SharedApertureHigh;
+		const auto selected = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, aperture));
+		const auto valid = AndCondition(state, aligned,
+		    AndCondition(state, selected, EmitMemoryElementInBounds(state, resource, index)));
+		emit(resource, index, valid);
+	}
+}
+
+uint32_t LoadLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		uint32_t result = 0;
+		ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
+			const auto value = EmitValueOrZeroIfCondition(ctx.state, valid, [&]() {
+				return LoadWordInBounds(ctx, resource, index);
+			});
+			result = result == 0 ? value : Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), result, value);
+		});
+		return result;
+	});
+}
+
+void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
+	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
+			EmitIfCondition(ctx.state, valid, [&]() {
+				StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, inst.NumArgs() - 2));
+			});
+		});
+	});
+}
+
 void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                        const MemoryResourceAccess& resource, uint32_t data) {
 	const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
@@ -526,55 +555,6 @@ void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem)
 	});
 }
 
-spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
-	switch (opcode) {
-		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
-		case IR::ValueOpcode::BufferAtomicSwap32:
-		case IR::ValueOpcode::BufferAtomicSwap64:
-		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
-		case IR::ValueOpcode::BufferAtomicIAdd32:
-		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
-		case IR::ValueOpcode::BufferAtomicISub32:
-		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
-		case IR::ValueOpcode::BufferAtomicSMin32:
-		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
-		case IR::ValueOpcode::BufferAtomicUMin32:
-		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
-		case IR::ValueOpcode::BufferAtomicSMax32:
-		case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
-		case IR::ValueOpcode::BufferAtomicUMax32:
-		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
-		case IR::ValueOpcode::BufferAtomicAnd32:
-		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
-		case IR::ValueOpcode::BufferAtomicOr32:
-		case IR::ValueOpcode::BufferAtomicOr64:
-		case IR::ValueOpcode::SharedAtomicOr64:
-		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
-		case IR::ValueOpcode::BufferAtomicXor32:
-		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
-		default: return spv::OpNop;
-	}
-}
-
-uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
-                             uint32_t scope) {
-	const auto old = ctx.state.builder.AllocateId();
-	if (inst.GetOpcode() == IR::ValueOpcode::BufferAtomicCmpSwap32) {
-		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
-		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
-		ctx.state.builder.AddFunction(
-		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
-		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
-		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
-	} else {
-		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
-		ctx.state.builder.AddFunction(SpirvAtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old,
-		                              pointer, ConstantU32(ctx.state, scope),
-		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
-	}
-	return old;
-}
-
 template <typename Fn>
 uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
                           const IR::MemoryInfo& mem, Fn&& operation) {
@@ -961,6 +941,27 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 
 } // namespace
 
+uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
+	return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
+	                              static_cast<uint32_t>(value),
+	                              static_cast<uint32_t>(value >> 32u));
+}
+
+uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto low64  = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
+	const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
+	                           Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
+	                           ConstantDeviceAddress(state, 32));
+	return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
+}
+
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
+	                          state.bda_pointer_function, address);
+	return result;
+}
+
 void DefineGetBdaPointer(EmitterState& state) {
 	if (!state.program.info.uses_dma) {
 		return;
@@ -1024,14 +1025,7 @@ uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto scope =
 		    mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 		const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
-		if (mem.kind == IR::ResourceKind::Lds) {
-			const auto semantics =
-			    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
-			ctx.state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
-			                              ConstantU32(ctx.state, semantics));
-		} else {
-			EmitDeviceAtomicMemoryBarrier(ctx.state);
-		}
+		EmitAtomicMemoryBarrier(ctx.state, mem.kind);
 		return old;
 	});
 }
@@ -1050,22 +1044,16 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 		    return EmitValueOrDefaultIfCondition(
 		        state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
 		        ConstantU64(state, 0), [&]() {
-			        const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state),
-			                                 ctx.Arg(inst, inst.NumArgs() - 2));
-			        const auto old   = state.builder.AllocateId();
-			        state.builder.AddFunction(
-			            SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old,
-			            EmitStorageBufferElementPointer(state, resource, index,
-			                                            TypeStorageBufferU64ElementPointer(state)),
-			            ConstantU32(state, spv::ScopeDevice),
-			            ConstantU32(state, spv::MemorySemanticsMaskNone), value);
-			        EmitDeviceAtomicMemoryBarrier(state);
-			        return Unary(state, spv::OpBitcast, TypeU64(state), old);
+			        const auto pointer = EmitStorageBufferElementPointer(
+			            state, resource, index, TypeStorageBufferU64ElementPointer(state));
+			        const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
+			        EmitAtomicMemoryBarrier(state, mem.kind);
+			        return old;
 		        });
 	    });
 }
 
-void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst) {
+void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	const auto& mem = ctx.Memory(inst);
 	EnsureLdsStorage(state);
@@ -1243,7 +1231,9 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto shared_components = IR::SharedComponentCount(op);
 	const auto address_info      = IR::AddressOpcodeInfoOf(op);
 	uint32_t   value;
-	if (buffer_components > 1u)
+	if (mem.kind == IR::ResourceKind::FlatLocal)
+		value = LoadLocalFlat(ctx, inst);
+	else if (buffer_components > 1u)
 		value = LoadWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		value = LoadWideShared(ctx, inst, shared_components);
@@ -1270,7 +1260,9 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto  buffer_components = IR::BufferComponentCount(op);
 	const auto  shared_components = IR::SharedComponentCount(op);
 	const auto  type              = inst.Arg(inst.NumArgs() - 2).GetType();
-	if (buffer_components > 1u)
+	if (mem.kind == IR::ResourceKind::FlatLocal)
+		StoreLocalFlat(ctx, inst);
+	else if (buffer_components > 1u)
 		StoreWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		StoreWideShared(ctx, inst, shared_components);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -306,11 +306,83 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 	}
 }
 
-void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
-	const auto semantics =
-	    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask;
-	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice),
-	                          ConstantU32(state, semantics));
+spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
+	switch (opcode) {
+		case IR::ValueOpcode::ImageAtomicCompareSwap32:
+		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
+		case IR::ValueOpcode::ImageAtomicSwap32:
+		case IR::ValueOpcode::BufferAtomicSwap32:
+		case IR::ValueOpcode::BufferAtomicSwap64:
+		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
+		case IR::ValueOpcode::ImageAtomicIAdd32:
+		case IR::ValueOpcode::BufferAtomicIAdd32:
+		case IR::ValueOpcode::SharedAtomicIAdd64:
+		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
+		case IR::ValueOpcode::BufferAtomicISub32:
+		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
+		case IR::ValueOpcode::ImageAtomicSMin32:
+		case IR::ValueOpcode::BufferAtomicSMin32:
+		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
+		case IR::ValueOpcode::ImageAtomicUMin32:
+		case IR::ValueOpcode::BufferAtomicUMin32:
+		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicSMax32:
+		case IR::ValueOpcode::BufferAtomicSMax32:
+		case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
+		case IR::ValueOpcode::ImageAtomicUMax32:
+		case IR::ValueOpcode::ImageAtomicUMax64:
+		case IR::ValueOpcode::BufferAtomicUMax32:
+		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32:
+		case IR::ValueOpcode::BufferAtomicAnd32:
+		case IR::ValueOpcode::BufferAtomicAnd64:
+		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32:
+		case IR::ValueOpcode::BufferAtomicOr32:
+		case IR::ValueOpcode::BufferAtomicOr64:
+		case IR::ValueOpcode::SharedAtomicOr64:
+		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32:
+		case IR::ValueOpcode::BufferAtomicXor32:
+		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
+		default: return spv::OpNop;
+	}
+}
+
+uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
+                             uint32_t scope) {
+	const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
+	const auto old    = ctx.state.builder.AllocateId();
+	const bool wide   = inst.GetType() == IR::Type::U64;
+	const auto type   = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state);
+	if (opcode == spv::OpAtomicCompareExchange) {
+		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
+		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		ctx.state.builder.AddFunction(
+		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
+		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
+		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
+	} else {
+		auto value = ctx.Arg(inst, inst.NumArgs() - 2);
+		if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value);
+		ctx.state.builder.AddFunction(opcode, type, old, pointer,
+		                              ConstantU32(ctx.state, scope),
+		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
+	}
+	return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old;
+}
+
+void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
+	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
+	const auto memory = [&] {
+		switch (kind) {
+			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
+			default: return spv::MemorySemanticsUniformMemoryMask;
+		}
+	}();
+	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
+	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
 }
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -6,19 +6,32 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t TypeVoid(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeVoid);
+	if (state.void_type == 0) {
+		state.void_type = state.builder.Type(spv::OpTypeVoid);
+	}
+	return state.void_type;
 }
 
 uint32_t TypeBool(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeBool);
+	if (state.bool_type == 0) {
+		state.bool_type = state.builder.Type(spv::OpTypeBool);
+	}
+	return state.bool_type;
 }
 
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeBool(state), components);
+	auto& type = state.bool_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeBool(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeU32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 32, 0);
+	if (state.u32_type == 0) {
+		state.u32_type = state.builder.Type(spv::OpTypeInt, 32, 0);
+	}
+	return state.u32_type;
 }
 
 uint32_t TypeU64(EmitterState& state) {
@@ -26,33 +39,55 @@ uint32_t TypeU64(EmitterState& state) {
 }
 
 uint32_t TypeScalarU64(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 64, 0);
+	if (state.native_u64_type == 0) {
+		state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
+	}
+	return state.native_u64_type;
 }
 
 uint32_t TypeU32Pair(EmitterState& state) {
-	const auto element = TypeU32(state);
-	return state.builder.Type(spv::OpTypeStruct, element, element);
+	if (state.u32_pair_type == 0) {
+		const auto element = TypeU32(state);
+		state.u32_pair_type = state.builder.Type(spv::OpTypeStruct, element, element);
+	}
+	return state.u32_pair_type;
 }
 
 uint32_t TypeI32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 32, 1);
+	if (state.i32_type == 0) {
+		state.i32_type = state.builder.Type(spv::OpTypeInt, 32, 1);
+	}
+	return state.i32_type;
 }
 
 uint32_t TypeI32Pair(EmitterState& state) {
-	const auto element = TypeI32(state);
-	return state.builder.Type(spv::OpTypeStruct, element, element);
+	if (state.i32_pair_type == 0) {
+		const auto element = TypeI32(state);
+		state.i32_pair_type = state.builder.Type(spv::OpTypeStruct, element, element);
+	}
+	return state.i32_pair_type;
 }
 
 uint32_t TypeF32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFloat, 32);
+	if (state.f32_type == 0) {
+		state.f32_type = state.builder.Type(spv::OpTypeFloat, 32);
+	}
+	return state.f32_type;
 }
 
 uint32_t TypeF64(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFloat, 64);
+	if (state.f64_type == 0) {
+		state.f64_type = state.builder.Type(spv::OpTypeFloat, 64);
+	}
+	return state.f64_type;
 }
 
 uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
+	auto& type = state.u32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
@@ -61,11 +96,19 @@ uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
 }
 
 uint32_t TypeI32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeI32(state), components);
+	auto& type = state.i32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeI32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeF32(state), components);
+	auto& type = state.f32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeF32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee) {
@@ -73,7 +116,10 @@ uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint3
 }
 
 uint32_t TypeFunction(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFunction, TypeVoid(state));
+	if (state.function_type == 0) {
+		state.function_type = state.builder.Type(spv::OpTypeFunction, TypeVoid(state));
+	}
+	return state.function_type;
 }
 
 uint32_t StorageRuntimeArrayType(EmitterState& state) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -243,6 +243,12 @@ void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::
 }
 
 void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
+	if (ctx.half != 0 && (inst.GetOpcode() == IR::ValueOpcode::Ballot ||
+	                     inst.GetOpcode() == IR::ValueOpcode::ReadFirstLane)) {
+		// Both operations already combine both emulated halves into one whole-wave result.
+		ctx.Define(inst, ctx.other_half->Result(inst));
+		return;
+	}
 	switch (inst.GetOpcode()) {
 #define VALUE_OPCODE(name, ...)                                                                    \
 	case IR::ValueOpcode::name: return Invoke(Emit##name, ctx, inst);
@@ -679,6 +685,7 @@ void EmitProgram(EmitterState& state) {
 		}
 	}
 	DefineGetBdaPointer(state);
+	DefineBvhIntersect(state);
 	for (const auto* block: program.blocks) {
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
 			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -192,8 +192,11 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
 
 constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
     {0x0fu, Opcode::IMAGE_ATOMIC_SWAP},
+    {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP},
     {0x11u, Opcode::IMAGE_ATOMIC_ADD},
+    {0x14u, Opcode::IMAGE_ATOMIC_SMIN},
     {0x15u, Opcode::IMAGE_ATOMIC_UMIN},
+    {0x16u, Opcode::IMAGE_ATOMIC_SMAX},
     {0x17u, Opcode::IMAGE_ATOMIC_UMAX},
     {0x18u, Opcode::IMAGE_ATOMIC_AND},
     {0x19u, Opcode::IMAGE_ATOMIC_OR},
@@ -225,6 +228,7 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
+		case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
@@ -253,6 +257,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
 	}
 
 	switch (opcode) {
+		case 0xe6u: return 11u;
 		case 0x0eu: return 1u;
 		case 0x01u:
 		case 0x09u: return ImageCoordComponents(dimension) + 1u;
@@ -355,11 +360,30 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		SetUnsupported(inst, Family::MIMG, opcode,
 		               "MIMG image gather requires exactly one dmask bit");
 	}
+	if (atomic != nullptr) {
+		const bool compare_swap = inst.opcode == Opcode::IMAGE_ATOMIC_CMPSWAP;
+		const auto mask32       = compare_swap ? 0x3u : 0x1u;
+		const auto mask64       = compare_swap ? 0xfu : 0x3u;
+		if (inst.dmask != mask32 && inst.dmask != mask64) {
+			SetUnsupported(inst, Family::MIMG, opcode, "MIMG image atomic has invalid DMASK");
+		} else if (inst.dmask == mask64) {
+			inst.data_bits = 64u;
+			if (inst.opcode != Opcode::IMAGE_ATOMIC_UMAX) {
+				SetUnsupported(inst, Family::MIMG, opcode,
+				               "MIMG 64-bit image atomic opcode is not implemented");
+			}
+		}
+	}
 	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
 	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
+	if (opcode == 0xe6u &&
+	    (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
+		SetUnsupported(inst, Family::MIMG, opcode,
+		               "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
+	}
 
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -61,6 +61,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
     {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
     {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
+    {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
     {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
 };
 
@@ -106,7 +107,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
-    {0x4au, Opcode::DS_OR_B64, 2, 32},
+    {0x40u, Opcode::DS_ADD_U64, 2, 32},         {0x4au, Opcode::DS_OR_B64, 2, 32},
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
     {0x77u, Opcode::DS_READ2_B64, 4, 32},        {0x78u, Opcode::DS_READ2ST64_B64, 4, 32},
@@ -168,6 +169,7 @@ bool IsDsWriteOpcode(Opcode opcode) {
 bool IsDsAtomicOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_U64:
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -46,6 +46,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x09u, Opcode::S_WQM_B32},
     {0x0au, Opcode::S_WQM_B64},
     {0x0bu, Opcode::S_BREV_B32},
+    {0x0cu, Opcode::S_BREV_B64},
     {0x0fu, Opcode::S_BCNT1_I32_B32},
     {0x10u, Opcode::S_BCNT1_I32_B64},
     {0x13u, Opcode::S_FF1_I32_B32},
@@ -108,6 +109,7 @@ constexpr OpcodeMap SOPP_OPCODE_LIST[] = {
     {0x12u, Opcode::S_TRAP},
     {0x16u, Opcode::S_TTRACEDATA},
     {0x17u, Opcode::S_CBRANCH_CDBGSYS},
+    {0x19u, Opcode::S_CBRANCH_CDBGSYS_OR_USER},
     {0x20u, Opcode::S_INST_PREFETCH},
     {0x23u, Opcode::S_WAITCNT_DEPCTR},
 };

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -184,8 +184,9 @@ std::string FormatExp(const Instruction& inst) {
 
 bool IsConditionalBranch(Opcode opcode) {
 	switch (opcode) {
-		// DevKit NGG should be disabled.
-		case Opcode::S_CBRANCH_CDBGSYS: return false;
+		// Conditional shader debugging is disabled.
+		case Opcode::S_CBRANCH_CDBGSYS:
+		case Opcode::S_CBRANCH_CDBGSYS_OR_USER: return false;
 		case Opcode::S_CBRANCH_SCC0:
 		case Opcode::S_CBRANCH_SCC1:
 		case Opcode::S_CBRANCH_VCCZ:
@@ -253,6 +254,8 @@ void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) {
 		case 125u: operand.kind = OperandKind::Null; return;
 		case 126u: operand.kind = OperandKind::ExecLo; return;
 		case 127u: operand.kind = OperandKind::ExecHi; return;
+		case 235u: operand.kind = OperandKind::SharedBase; return;
+		case 237u: operand.kind = OperandKind::PrivateBase; return;
 		case 239u: operand.kind = OperandKind::PopsExitingWaveId; return;
 		case 248u:
 			operand.kind      = OperandKind::FloatInlineConstant;
@@ -403,7 +406,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 	program.instructions.clear();
 	program.instructions.reserve(code.size());
 	program.code = code;
-	program.has_bvh = false;
 
 	std::vector<bool> branch_targets;
 	for (uint32_t word_index = 0; word_index < code.size();) {
@@ -412,10 +414,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 
 		const auto& inst = program.instructions.back();
 		word_index += inst.word_count;
-		if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) {
-			program.has_bvh = true;
-			return;
-		}
 
 		if (IsDirectBranch(inst.opcode)) {
 			const auto target_index = inst.branch_target / sizeof(uint32_t);
@@ -454,6 +452,8 @@ std::string OperandToString(const Operand& operand) {
 		case OperandKind::Scc: text = "scc"; break;
 		case OperandKind::M0: text = "m0"; break;
 		case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break;
+		case OperandKind::SharedBase: text = "shared_base"; break;
+		case OperandKind::PrivateBase: text = "private_base"; break;
 		case OperandKind::Null: text = "null"; break;
 		default: text = "unknown"; break;
 	}
@@ -504,6 +504,7 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_ABS_I32:
 		case Opcode::S_BREV_B32:
+		case Opcode::S_BREV_B64:
 		case Opcode::S_BCNT1_I32_B32:
 		case Opcode::S_FLBIT_I32_B32:
 		case Opcode::S_FF1_I32_B32:
@@ -560,6 +561,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_CBRANCH_EXECZ:
 		case Opcode::S_CBRANCH_EXECNZ:
 		case Opcode::S_CBRANCH_CDBGSYS:
+		case Opcode::S_CBRANCH_CDBGSYS_OR_USER:
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc,
 			                                               magic_enum::enum_name(inst.opcode),
 			                                               inst.branch_target));
@@ -572,9 +574,12 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_STORE:
 		case Opcode::IMAGE_STORE_MIP:
+		case Opcode::IMAGE_ATOMIC_CMPSWAP:
 		case Opcode::IMAGE_ATOMIC_SWAP:
 		case Opcode::IMAGE_ATOMIC_ADD:
+		case Opcode::IMAGE_ATOMIC_SMIN:
 		case Opcode::IMAGE_ATOMIC_UMIN:
+		case Opcode::IMAGE_ATOMIC_SMAX:
 		case Opcode::IMAGE_ATOMIC_UMAX:
 		case Opcode::IMAGE_ATOMIC_AND:
 		case Opcode::IMAGE_ATOMIC_OR:
@@ -641,6 +646,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_SMAX:
 		case Opcode::BUFFER_ATOMIC_UMAX:
 		case Opcode::BUFFER_ATOMIC_AND:
+		case Opcode::BUFFER_ATOMIC_AND_X2:
 		case Opcode::BUFFER_ATOMIC_OR:
 		case Opcode::BUFFER_ATOMIC_OR_X2:
 		case Opcode::BUFFER_ATOMIC_XOR:
@@ -663,6 +669,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_STORE_DWORDX3:
 		case Opcode::FLAT_STORE_DWORDX4:
 		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_U64:
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -10,6 +10,10 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
+// PS5 flat stack addresses use these SH_MEM_BASES aperture tags in VA[63:32].
+constexpr uint32_t PrivateApertureHigh = 0x70000000u;
+constexpr uint32_t SharedApertureHigh = 0x80000000u;
+
 enum class Family {
 	Unknown,
 	SOP1,
@@ -43,6 +47,7 @@ enum class Opcode {
 	S_ABS_I32,
 	S_ABSDIFF_I32,
 	S_BREV_B32,
+	S_BREV_B64,
 	S_BCNT1_I32_B32,
 	S_BCNT1_I32_B64,
 	S_FF1_I32_B32,
@@ -240,6 +245,7 @@ enum class Opcode {
 	V_MAD_U32_U24,
 	V_MAD_U64_U32,
 	V_FMA_F64,
+	V_ADD_F64,
 	V_MUL_F64,
 	V_FMA_F32,
 	V_FMA_F16,
@@ -355,6 +361,10 @@ enum class Opcode {
 	V_CMP_NEQ_F32,
 	V_CMP_NLT_F32,
 	V_CMP_TRU_F32,
+	V_CMP_EQ_F64,
+	V_CMP_LE_F64,
+	V_CMPX_LE_F64,
+	V_CMPX_GE_F64,
 	V_CMPX_LT_F32,
 	V_CMPX_EQ_F32,
 	V_CMPX_LE_F32,
@@ -384,12 +394,19 @@ enum class Opcode {
 	V_CMP_GT_I16,
 	V_CMP_NE_I16,
 	V_CMP_GE_I16,
+	V_CMPX_LT_I16,
+	V_CMPX_EQ_I16,
+	V_CMPX_LE_I16,
+	V_CMPX_GT_I16,
+	V_CMPX_NE_I16,
+	V_CMPX_GE_I16,
 	V_CMP_LT_F16,
 	V_CMP_EQ_F16,
 	V_CMP_LE_F16,
 	V_CMP_GT_F16,
 	V_CMP_LG_F16,
 	V_CMP_GE_F16,
+	V_CMP_NGE_F16,
 	V_CMP_NGT_F16,
 	V_CMP_NEQ_F16,
 	V_CMP_NLT_F16,
@@ -426,11 +443,16 @@ enum class Opcode {
 	V_CMP_GE_U32,
 	V_CMP_T_U32,
 	V_CMP_EQ_I64,
+	V_CMP_LT_I64,
+	V_CMP_LE_I64,
 	V_CMP_LT_U64,
 	V_CMP_EQ_U64,
+	V_CMP_LE_U64,
 	V_CMP_GT_U64,
 	V_CMP_NE_U64,
+	V_CMP_GE_U64,
 	V_CMPX_NE_I64,
+	V_CMPX_LE_U64,
 	V_CMPX_NE_U64,
 	V_CMPX_LT_U32,
 	V_CMPX_EQ_U32,
@@ -490,6 +512,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_SMAX,
 	BUFFER_ATOMIC_UMAX,
 	BUFFER_ATOMIC_AND,
+	BUFFER_ATOMIC_AND_X2,
 	BUFFER_ATOMIC_OR,
 	BUFFER_ATOMIC_OR_X2,
 	BUFFER_ATOMIC_XOR,
@@ -510,6 +533,7 @@ enum class Opcode {
 	FLAT_STORE_DWORDX3,
 	FLAT_STORE_DWORDX4,
 	DS_ADD_U32,
+	DS_ADD_U64,
 	DS_ADD_RTN_U32,
 	DS_SUB_U32,
 	DS_SUB_RTN_U32,
@@ -567,6 +591,7 @@ enum class Opcode {
 	DS_WRITE_B128,
 	DS_WRITE_ADDTID_B32,
 	DS_READ_ADDTID_B32,
+	IMAGE_BVH_INTERSECT_RAY,
 	IMAGE_GET_RESINFO,
 	IMAGE_GET_LOD,
 	IMAGE_LOAD,
@@ -574,8 +599,11 @@ enum class Opcode {
 	IMAGE_STORE,
 	IMAGE_STORE_MIP,
 	IMAGE_ATOMIC_SWAP,
+	IMAGE_ATOMIC_CMPSWAP,
 	IMAGE_ATOMIC_ADD,
+	IMAGE_ATOMIC_SMIN,
 	IMAGE_ATOMIC_UMIN,
+	IMAGE_ATOMIC_SMAX,
 	IMAGE_ATOMIC_UMAX,
 	IMAGE_ATOMIC_AND,
 	IMAGE_ATOMIC_OR,
@@ -607,6 +635,7 @@ enum class Opcode {
 	S_CBRANCH_EXECZ,
 	S_CBRANCH_EXECNZ,
 	S_CBRANCH_CDBGSYS,
+	S_CBRANCH_CDBGSYS_OR_USER,
 	S_SENDMSG,
 	S_SETREG_B32,
 	S_SLEEP,
@@ -633,6 +662,8 @@ enum class OperandKind {
 	Scc,
 	M0,
 	PopsExitingWaveId,
+	SharedBase,
+	PrivateBase,
 	Null,
 	Vgpr,
 };
@@ -744,7 +775,6 @@ struct Instruction {
 struct Program {
 	std::span<const uint32_t> code;
 	std::vector<Instruction>  instructions;
-	bool                     has_bvh = false;
 };
 
 // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -228,6 +228,8 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x19u, Opcode::V_CMPX_NGE_F32},       {0x1au, Opcode::V_CMPX_NLG_F32},
     {0x1bu, Opcode::V_CMPX_NGT_F32},       {0x1cu, Opcode::V_CMPX_NLE_F32},
     {0x1du, Opcode::V_CMPX_NEQ_F32},       {0x1eu, Opcode::V_CMPX_NLT_F32},
+    {0x22u, Opcode::V_CMP_EQ_F64, false},  {0x23u, Opcode::V_CMP_LE_F64, false},
+    {0x33u, Opcode::V_CMPX_LE_F64, false}, {0x36u, Opcode::V_CMPX_GE_F64, false},
     {0x80u, Opcode::V_CMP_F_I32},          {0x81u, Opcode::V_CMP_LT_I32},
     {0x82u, Opcode::V_CMP_EQ_I32},         {0x83u, Opcode::V_CMP_LE_I32},
     {0x84u, Opcode::V_CMP_GT_I32},         {0x85u, Opcode::V_CMP_NE_I32},
@@ -239,6 +241,9 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x92u, Opcode::V_CMPX_EQ_I32},        {0x93u, Opcode::V_CMPX_LE_I32},
     {0x94u, Opcode::V_CMPX_GT_I32},        {0x95u, Opcode::V_CMPX_NE_I32},
     {0x96u, Opcode::V_CMPX_GE_I32},        {0x98u, Opcode::V_CMPX_CLASS_F32},
+    {0x99u, Opcode::V_CMPX_LT_I16, false}, {0x9au, Opcode::V_CMPX_EQ_I16, false},
+    {0x9bu, Opcode::V_CMPX_LE_I16, false}, {0x9cu, Opcode::V_CMPX_GT_I16, false},
+    {0x9du, Opcode::V_CMPX_NE_I16, false}, {0x9eu, Opcode::V_CMPX_GE_I16, false},
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
@@ -248,16 +253,19 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
     {0xc5u, Opcode::V_CMP_NE_U32},         {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},          {0xa2u, Opcode::V_CMP_EQ_I64, false},
+    {0xa1u, Opcode::V_CMP_LT_I64, false},  {0xa3u, Opcode::V_CMP_LE_I64, false},
     {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
     {0xd2u, Opcode::V_CMPX_EQ_U32},        {0xd3u, Opcode::V_CMPX_LE_U32},
     {0xd4u, Opcode::V_CMPX_GT_U32},        {0xd5u, Opcode::V_CMPX_NE_U32},
     {0xd6u, Opcode::V_CMPX_GE_U32},        {0xe1u, Opcode::V_CMP_LT_U64, false},
-    {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe4u, Opcode::V_CMP_GT_U64, false},
-    {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe3u, Opcode::V_CMP_LE_U64, false},
+    {0xe4u, Opcode::V_CMP_GT_U64, false},  {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xe6u, Opcode::V_CMP_GE_U64, false},  {0xf3u, Opcode::V_CMPX_LE_U64, false},
     {0xf5u, Opcode::V_CMPX_NE_U64, false}, {0xc9u, Opcode::V_CMP_LT_F16},
     {0xcau, Opcode::V_CMP_EQ_F16},         {0xcbu, Opcode::V_CMP_LE_F16},
     {0xccu, Opcode::V_CMP_GT_F16},         {0xcdu, Opcode::V_CMP_LG_F16},
-    {0xceu, Opcode::V_CMP_GE_F16},         {0xebu, Opcode::V_CMP_NGT_F16},
+    {0xceu, Opcode::V_CMP_GE_F16},         {0xe9u, Opcode::V_CMP_NGE_F16},
+    {0xebu, Opcode::V_CMP_NGT_F16},
     {0xedu, Opcode::V_CMP_NEQ_F16},        {0xeeu, Opcode::V_CMP_NLT_F16},
     {0xd9u, Opcode::V_CMPX_LT_F16},        {0xdau, Opcode::V_CMPX_EQ_F16},
     {0xdbu, Opcode::V_CMPX_LE_F16},        {0xdcu, Opcode::V_CMPX_GT_F16},
@@ -279,6 +287,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x147u, Opcode::V_CUBEMA_F32},
     {0x14bu, Opcode::V_FMA_F32},
     {0x14cu, Opcode::V_FMA_F64},
+    {0x164u, Opcode::V_ADD_F64},
     {0x165u, Opcode::V_MUL_F64},
     {0x148u, Opcode::V_BFE_U32},
     {0x149u, Opcode::V_BFE_I32},
@@ -828,6 +837,10 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMP_F_F32:
 		case Opcode::V_CMP_LT_F32:
 		case Opcode::V_CMP_EQ_F32:
+		case Opcode::V_CMP_EQ_F64:
+		case Opcode::V_CMP_LE_F64:
+		case Opcode::V_CMPX_LE_F64:
+		case Opcode::V_CMPX_GE_F64:
 		case Opcode::V_CMP_LE_F32:
 		case Opcode::V_CMP_GT_F32:
 		case Opcode::V_CMP_LG_F32:
@@ -860,6 +873,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMP_GT_F16:
 		case Opcode::V_CMP_LG_F16:
 		case Opcode::V_CMP_GE_F16:
+		case Opcode::V_CMP_NGE_F16:
 		case Opcode::V_CMP_NGT_F16:
 		case Opcode::V_CMP_NEQ_F16:
 		case Opcode::V_CMP_NLT_F16:
@@ -1138,7 +1152,9 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) {
 }
 
 bool SupportsVopcSdwa(Opcode opcode) {
-	return opcode != Opcode::UNSUPPORTED;
+	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 &&
+	       opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 &&
+	       opcode != Opcode::V_CMPX_GE_F64;
 }
 
 void DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
@@ -1203,6 +1219,7 @@ void DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 
 uint32_t NativeVop3SourceCount(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
 		case Opcode::V_MUL_LO_U32:
 		case Opcode::V_MUL_HI_U32:
@@ -1354,6 +1371,7 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MAD_F32:
+		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
 		case Opcode::V_FMA_F64:
 		case Opcode::V_FMA_F32:
@@ -1459,6 +1477,8 @@ void ApplyNativeVop3SourceModifiers(Instruction& inst, uint32_t abs, uint32_t ne
 
 bool IsVopcCompareExec(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::V_CMPX_LE_F64:
+		case Opcode::V_CMPX_GE_F64:
 		case Opcode::V_CMPX_LT_F32:
 		case Opcode::V_CMPX_EQ_F32:
 		case Opcode::V_CMPX_LE_F32:
@@ -1486,7 +1506,14 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_U32:
 		case Opcode::V_CMPX_GE_U32:
 		case Opcode::V_CMPX_NE_I64:
+		case Opcode::V_CMPX_LE_U64:
 		case Opcode::V_CMPX_NE_U64:
+		case Opcode::V_CMPX_LT_I16:
+		case Opcode::V_CMPX_EQ_I16:
+		case Opcode::V_CMPX_LE_I16:
+		case Opcode::V_CMPX_GT_I16:
+		case Opcode::V_CMPX_NE_I16:
+		case Opcode::V_CMPX_GE_I16:
 		case Opcode::V_CMPX_LT_U16:
 		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -1,5 +1,7 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
+#include <bit>
+
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar,
@@ -26,9 +28,20 @@ void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool valu
 
 void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                     IR::Type type, bool scalar, bool cmpx) {
-	const auto lhs = ReadOperand(inst.src0, type);
-	const auto rhs = ReadOperand(inst.src1, type);
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx);
+	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
+	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMPX_NE_I64;
+	const auto read = [&](const Decoder::Operand& operand) {
+		// RDNA2 expands signed 64-bit integer literals by sign extension.
+		if (signed_64 && operand.kind == Decoder::OperandKind::LiteralConstant) {
+			return IR::Value(static_cast<uint64_t>(
+			    static_cast<int64_t>(std::bit_cast<int32_t>(operand.value))));
+		}
+		return ReadOperand(operand, type);
+	};
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {read(inst.src0), read(inst.src1)})), scalar,
+	                  cmpx);
 }
 
 void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -40,11 +53,13 @@ void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::Valu
 
 void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                   bool half, bool cmpx) {
+	const auto type = IR::ArgTypeOf(opcode, 0);
 	const auto lhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
+	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, type);
 	const auto rhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
-	const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
+	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, type);
+	const IR::FPCompareFlags flags{
+	    .flush_input_denorms = !half && type == IR::Type::F32 && flush_f32_inputs};
 	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
 }
 

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -558,9 +558,20 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
-	const auto result =
-	    ir.Emit(opcode, {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
-	            AddMemoryInfo(memory, inst.pc));
+	const auto data     = MemorySourceAt(inst, 0);
+	const auto flags    = AddMemoryInfo(memory, inst.pc);
+	IR::Value  result;
+	if (opcode == IR::ValueOpcode::ImageAtomicCompareSwap32) {
+		// VDATA supplies the replacement; VDATA+1 supplies the comparison.
+		result = ir.Emit(opcode,
+		                 {resource, address, ReadU32(data), ReadU32(OffsetOperand(data, 1u)),
+		                  ir.GetExec()},
+		                 flags);
+	} else {
+		const IR::Value value =
+		    memory.data_bits == 64u ? IR::Value(ReadU64(data)) : IR::Value(ReadU32(data));
+		result = ir.Emit(opcode, {resource, address, value, ir.GetExec()}, flags);
+	}
 	if (inst.glc) {
 		WriteOperand(inst.dst, result);
 	}
@@ -697,8 +708,16 @@ void Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
 	const auto resource = GetImageResource(memory);
 	const auto sampler  = GetSamplerResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 0));
-	const auto result   = ir.Emit(IR::ValueOpcode::ImageGatherRaw, {resource, sampler, address},
-	                              AddMemoryInfo(memory, inst.pc));
+	const bool has_lod = (memory.image_sample_flags & Decoder::ImageSampleFlagLod) != 0u;
+	// LOD selection consumes these words on the GPU after descriptor handles are stripped.
+	const auto result = ir.Emit(
+	    IR::ValueOpcode::ImageGatherRaw,
+	    {resource, sampler, address,
+	     has_lod ? resource.Instruction()->Arg(1) : IR::Value(0u),
+	     has_lod ? resource.Instruction()->Arg(3) : IR::Value(0u),
+	     has_lod ? sampler.Instruction()->Arg(1) : IR::Value(0u),
+	     has_lod ? sampler.Instruction()->Arg(2) : IR::Value(0u)},
+	    AddMemoryInfo(memory, inst.pc));
 	for (uint32_t index = 0; index < memory.data_dwords; index++) {
 		WriteOperand(OffsetOperand(inst.dst, index),
 		             ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(index)}));
@@ -883,6 +902,15 @@ void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
 
 void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
+		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
+			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
+			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+			for (uint32_t component = 0; component < 4; ++component) {
+				WriteOperand(OffsetOperand(inst.dst, component),
+				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
+			}
+			return;
+		}
 		case Decoder::Opcode::S_LOAD_DWORD:
 		case Decoder::Opcode::S_LOAD_DWORDX2:
 		case Decoder::Opcode::S_LOAD_DWORDX4:
@@ -957,6 +985,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMax32);
 		case Decoder::Opcode::BUFFER_ATOMIC_AND:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd32);
+		case Decoder::Opcode::BUFFER_ATOMIC_AND_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd64);
 		case Decoder::Opcode::BUFFER_ATOMIC_OR:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr32);
 		case Decoder::Opcode::BUFFER_ATOMIC_OR_X2:
@@ -970,6 +1000,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 
 		case Decoder::Opcode::DS_ADD_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false);
+		case Decoder::Opcode::DS_ADD_U64:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd64, false);
 		case Decoder::Opcode::DS_ADD_RTN_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, true);
 		case Decoder::Opcode::DS_SUB_U32:
@@ -1017,14 +1049,21 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
 
+		case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_SWAP:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32);
+		case Decoder::Opcode::IMAGE_ATOMIC_SMIN:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32);
+		case Decoder::Opcode::IMAGE_ATOMIC_SMAX:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
-			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32);
+			return IMAGE_ATOMIC(inst, inst.data_bits == 64u ? IR::ValueOpcode::ImageAtomicUMax64
+			                                                : IR::ValueOpcode::ImageAtomicUMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_AND:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32);
 		case Decoder::Opcode::IMAGE_ATOMIC_OR:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -155,6 +155,12 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BREV_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false,
 			                     false);
+		case O::S_BREV_B64: {
+			const auto value = ReadU32Pair(inst.src0);
+			WriteU32Pair(inst.dst, {IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[1]})),
+			                        IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[0]}))});
+			return;
+		}
 		case O::S_BCNT1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
 			                     true);
@@ -231,6 +237,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_CBRANCH_EXECZ:
 		case O::S_CBRANCH_EXECNZ:
 		case O::S_CBRANCH_CDBGSYS:
+		case O::S_CBRANCH_CDBGSYS_OR_USER:
 		case O::S_ENDPGM: return;
 		default: return FailMissingTranslation(inst);
 	}

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -135,6 +135,8 @@ IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
 		case Decoder::OperandKind::IntegerInlineConstant:
 		case Decoder::OperandKind::FloatInlineConstant: return IR::U32(IR::Value(operand.value));
 		case Decoder::OperandKind::Null:
+		case Decoder::OperandKind::SharedBase:
+		case Decoder::OperandKind::PrivateBase:
 		case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u));
 		case Decoder::OperandKind::Sgpr:
 			return ir.GetScalarReg(static_cast<IR::ScalarReg>(operand.reg));
@@ -474,6 +476,12 @@ IR::U32 Translator::ReadU32(const Decoder::Operand& operand) {
 }
 
 std::array<IR::U32, 2> Translator::ReadU32Pair(const Decoder::Operand& operand) {
+	if (operand.kind == Decoder::OperandKind::PrivateBase ||
+	    operand.kind == Decoder::OperandKind::SharedBase) {
+		return {IR::U32(IR::Value(0u)), IR::U32(IR::Value(
+		    operand.kind == Decoder::OperandKind::PrivateBase ? Decoder::PrivateApertureHigh
+		                                                      : Decoder::SharedApertureHigh))};
+	}
 	if (operand.kind == Decoder::OperandKind::ExecLo) {
 		return {ir.GetExecLo(), ir.GetExecHi()};
 	}
@@ -560,6 +568,12 @@ IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_la
 	} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
 		offset = operand.sdwa_sel == 5u ? 16u : 0u;
 	}
+	if (width == 8u && operand.sdwa_sext) {
+		// Extend the selected byte into the 16-bit operand before halfword modifiers.
+		const auto value = IR::U32(
+		    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {bits, IR::Value(offset), IR::Value(width)}));
+		return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
+	}
 	return IR::U32(
 	    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)}));
 }
@@ -567,7 +581,7 @@ IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_la
 IR::U32 Translator::ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane,
                                      bool sign_extend) {
 	auto value = Read16LaneBits(operand, high_lane);
-	if (sign_extend || operand.sdwa_sext) {
+	if (sign_extend) {
 		value = IR::U32(
 		    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {value, IR::Value(0u), IR::Value(16u)}));
 	}
@@ -880,13 +894,35 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CVT_F64_F32:
 		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
+		case Decoder::Opcode::V_ADD_F64:
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
+		case Decoder::Opcode::V_CMP_EQ_F64:
+		case Decoder::Opcode::V_CMP_LE_F64:
+		case Decoder::Opcode::V_CMPX_LE_F64:
+		case Decoder::Opcode::V_CMPX_GE_F64:
+		case Decoder::Opcode::V_CMP_EQ_I64:
+		case Decoder::Opcode::V_CMP_LT_I64:
+		case Decoder::Opcode::V_CMP_LE_I64:
+		case Decoder::Opcode::V_CMP_LT_U64:
+		case Decoder::Opcode::V_CMP_EQ_U64:
+		case Decoder::Opcode::V_CMP_LE_U64:
+		case Decoder::Opcode::V_CMP_GT_U64:
+		case Decoder::Opcode::V_CMP_NE_U64:
+		case Decoder::Opcode::V_CMP_GE_U64:
+		case Decoder::Opcode::V_CMPX_NE_I64:
+		case Decoder::Opcode::V_CMPX_LE_U64:
+		case Decoder::Opcode::V_CMPX_NE_U64:
+			include_vector(inst.src0, 2u);
+			include_vector(inst.src1, 2u);
+			break;
 		default: break;
 	}
 	if (inst.family == Decoder::Family::DS) {
 		switch (inst.opcode) {
+			case Decoder::Opcode::DS_ADD_U64:
+			case Decoder::Opcode::DS_OR_B64:
 			case Decoder::Opcode::DS_WRITE_B64:
 			case Decoder::Opcode::DS_WRITE_B96:
 			case Decoder::Opcode::DS_WRITE_B128: include_vector(inst.src1, inst.data_dwords); break;
@@ -1120,12 +1156,13 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				return entry_ir.ISub(lhs, minimum(lhs, rhs));
 			};
 			const auto local = builtin(IR::StageInputKind::LocalInvocationIndex);
-			const auto primitive_chunk = entry_ir.IMul(builtin(IR::StageInputKind::WorkgroupId, 0),
-			                                           u32(mesh.primitives_per_group));
+			const auto group = builtin(IR::StageInputKind::WorkgroupId, 0);
+			const auto primitive_chunk = mesh.fast_launch ? group :
+			    entry_ir.IMul(group, u32(mesh.primitives_per_group));
 			const auto step  = u32(mesh.InputPrimitiveStep());
 			const auto size  = u32(mesh.InputPrimitiveSize());
-			const auto chunk = entry_ir.IMul(primitive_chunk, step);
-			const auto vertices =
+			const auto chunk = mesh.fast_launch ? group : entry_ir.IMul(primitive_chunk, step);
+			const auto vertices = mesh.fast_launch ? u32(mesh.vertices_per_group) :
 			    minimum(subtract_saturate(draw(0), chunk), u32(mesh.vertices_per_group));
 			const auto primitives = entry_ir.Select(
 			    entry_ir.ULessThan(vertices, size), u32(0),
@@ -1146,56 +1183,64 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			    entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(
 			                                                         primitive_count, u32(8)),
 			                                                     vertex_count)));
-			// GS adjacency addresses local ES records in LDS. Fans retain the draw's
-			// center in every subgroup; strip winding follows the global primitive.
-			const auto vertex = entry_ir.IMul(local, step);
-			auto       first  = vertex;
-			auto       second = u32(0);
-			auto       third  = u32(0);
-			if (mesh.InputPrimitiveSize() >= 2u) {
-				second = entry_ir.IAdd(vertex, u32(1));
+			if (mesh.fast_launch) {
+				// Fast launch broadcasts the subgroup's base vertex and instance to every lane.
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5), entry_ir.IAdd(draw(1), group));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(6),
+				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
+			} else {
+				// GS adjacency addresses local ES records in LDS. Fans retain the draw's
+				// center in every subgroup; strip winding follows the global primitive.
+				const auto vertex = entry_ir.IMul(local, step);
+				auto       first  = vertex;
+				auto       second = u32(0);
+				auto       third  = u32(0);
+				if (mesh.InputPrimitiveSize() >= 2u) {
+					second = entry_ir.IAdd(vertex, u32(1));
+				}
+				if (mesh.InputPrimitiveSize() == 3u) {
+					third = entry_ir.IAdd(vertex, u32(2));
+				}
+				auto input_vertex = entry_ir.IAdd(chunk, local);
+				if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriFan)) {
+					first = u32(0);
+					input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex);
+				} else if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriStrip)) {
+					const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1));
+					first = entry_ir.IAdd(first, parity);
+					second = entry_ir.ISub(second, parity);
+				}
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
+				                      entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
+				                                         entry_ir.ShiftLeftLogical(second, u32(18))));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
+				                      entry_ir.ShiftLeftLogical(third, u32(2)));
+				const auto index_bytes  = draw(3);
+				const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
+				const auto index_low    = draw(4);
+				const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
+				                                           entry_ir.IMul(input_vertex, index_bytes));
+				const auto index_resource = entry_ir.Emit(
+				    IR::ValueOpcode::GetAddressResource,
+				    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
+				const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
+				result.memory_info.push_back({.kind = IR::ResourceKind::Global});
+				const auto packed_index = entry_ir.Emit(
+				    IR::ValueOpcode::LoadAddressU32,
+				    {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0),
+				     entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))},
+				    IR::MemoryFlags {.index = memory_index});
+				const auto index = IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::BitFieldUExtract,
+				    {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)),
+				     entry_ir.IMul(index_bytes, u32(8))}));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
+				                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(8),
+				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
 			}
-			if (mesh.InputPrimitiveSize() == 3u) {
-				third = entry_ir.IAdd(vertex, u32(2));
-			}
-			auto input_vertex = entry_ir.IAdd(chunk, local);
-			if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriFan)) {
-				first = u32(0);
-				input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex);
-			} else if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriStrip)) {
-				const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1));
-				first = entry_ir.IAdd(first, parity);
-				second = entry_ir.ISub(second, parity);
-			}
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
-			                      entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
-			                                         entry_ir.ShiftLeftLogical(second, u32(18))));
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
-			                      entry_ir.ShiftLeftLogical(third, u32(2)));
-			const auto index_bytes  = draw(3);
-			const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
-			const auto index_low    = draw(4);
-			const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
-			                                           entry_ir.IMul(input_vertex, index_bytes));
-			const auto index_resource = entry_ir.Emit(
-			    IR::ValueOpcode::GetAddressResource,
-			    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
-			const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
-			result.memory_info.push_back({.kind = IR::ResourceKind::Global});
-			const auto packed_index = entry_ir.Emit(
-			    IR::ValueOpcode::LoadAddressU32,
-			    {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0),
-			     entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))},
-			    IR::MemoryFlags {.index = memory_index});
-			const auto index = IR::U32(entry_ir.Emit(
-			    IR::ValueOpcode::BitFieldUExtract,
-			    {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)),
-			     entry_ir.IMul(index_bytes, u32(8))}));
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
-			                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
-			entry_ir.SetVectorReg(
-			    static_cast<IR::VectorReg>(8),
-			    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
 		} else if (options.stage == ShaderType::Local) {
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(2),

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -114,18 +114,40 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
 			return;
 
+		case O::V_CMP_LT_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMP_LE_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMP_LE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMPX_LE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, true);
+			return;
+		case O::V_CMP_GE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false);
+			return;
+
 		case O::V_CMP_EQ_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
 			return;
 		case O::V_CMP_EQ_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
 			return;
+		case O::V_CMPX_EQ_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, true);
+			return;
 		case O::V_CMP_NE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
 			return;
 		case O::V_CMP_NE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
 			return;
+		case O::V_CMPX_NE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, true);
+			return;
 		case O::V_CMP_GT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
 			return;
@@ -150,21 +172,45 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_GT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
 			return;
+		case O::V_CMPX_GT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, true);
+			return;
 		case O::V_CMP_GE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
 			return;
+		case O::V_CMPX_GE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, true);
+			return;
 		case O::V_CMP_LT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
 			return;
+		case O::V_CMPX_LT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, true);
+			return;
 		case O::V_CMP_LE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
 			return;
+		case O::V_CMPX_LE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, true);
+			return;
 
 		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return;
 		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return;
 		case O::V_CMP_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
 			return;
+		case O::V_CMP_EQ_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false);
+			return;
+		case O::V_CMP_LE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, false);
+			return;
+		case O::V_CMPX_LE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, true);
+			return;
+		case O::V_CMPX_GE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, true);
+			return;
 		case O::V_CMPX_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
 			return;
@@ -267,6 +313,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMPX_LE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
 			return;
+		case O::V_CMP_NGE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true, false);
+			return;
 		case O::V_CMP_NGT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false);
 			return;
@@ -418,6 +467,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false);
 		case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true);
 		case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
+		case O::V_ADD_F64: return FloatBinary(inst, IR::ValueOpcode::FPAdd64, false);
 		case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false);
 		case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
 		case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);

src/graphics/shader/recompiler/ir/Program.cpp

@@ -142,6 +142,7 @@ CompiledShaderInfo Program::TakeCompiledInfo() && {
 	    .user_data_base  = user_data_base,
 	    .user_data_count = user_data_count,
 	    .scratch_dwords  = scratch_dwords,
+	    .has_address_writes = has_address_writes,
 	    .info            = std::move(info),
 	    .bindings        = std::move(bindings),
 	};

src/graphics/shader/recompiler/ir/ResourceSnapshot.h

@@ -3,6 +3,7 @@
 
 #include <array>
 #include <cstdint>
+#include <utility>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -34,6 +35,7 @@ struct ResourceSnapshot {
 	std::vector<DescriptorValue> samplers;
 	std::vector<uint32_t>        flattened_srt;
 	std::vector<uint32_t>        user_data;
+	std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 	UniformFill                 uniform_fill;
 };
 

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -30,6 +30,7 @@ enum class ResourceKind {
 	Buffer,
 	IndirectBuffer,
 	Flat,
+	FlatLocal,
 	Global,
 	Scratch,
 	Lds,
@@ -40,7 +41,8 @@ enum class ResourceKind {
 
 [[nodiscard]] constexpr bool IsAddressResourceKind(ResourceKind kind) {
 	return kind == ResourceKind::ScalarAddress || kind == ResourceKind::Flat ||
-	       kind == ResourceKind::Global || kind == ResourceKind::Scratch;
+	       kind == ResourceKind::FlatLocal || kind == ResourceKind::Global ||
+	       kind == ResourceKind::Scratch;
 }
 
 struct MemoryInfo {
@@ -112,7 +114,7 @@ struct BufferResource {
 	bool operator==(const BufferResource& other) const = default;
 };
 
-enum class ImageMipMode { None, DynamicStorage };
+enum class ImageMipMode { None, Dynamic };
 
 constexpr uint32_t ShaderImageIdentitySwizzle = 0x00000facu;
 
@@ -131,6 +133,7 @@ struct ImageResource {
 	bool                          read              = false;
 	bool                          written           = false;
 	bool                          atomic            = false;
+	bool                          atomic64          = false;
 	bool                          depth_compare     = false;
 	bool                          cube              = false;
 	bool                          r128              = false;
@@ -145,9 +148,12 @@ struct ImageResource {
 struct SamplerResource {
 	uint32_t source                = 0;
 	uint32_t first_use_pc          = 0;
+	// Native filtering/border variants share the original sampler's runtime descriptor.
+	uint32_t snapshot_index        = 0;
 	bool     force_point_filtering = false;
 	bool     depth_compare         = false;
 	bool     integer_border        = false;
+	bool     gather_lod            = false;
 
 	bool operator==(const SamplerResource& other) const = default;
 };
@@ -276,7 +282,7 @@ struct StageOutput {
 inline constexpr uint32_t FirstImageBinding           = 1u;
 inline constexpr uint32_t FirstComparisonImageBinding = 22u;
 inline constexpr uint32_t FirstStorageImageBinding    = 29u;
-inline constexpr uint32_t ImageBindingCount           = 43u;
+inline constexpr uint32_t ImageBindingCount           = 48u;
 
 enum class DescriptorBindingKind : uint32_t {
 	Buffers  = 0u,
@@ -289,8 +295,8 @@ enum class DescriptorBindingKind : uint32_t {
 	Count,
 };
 
-static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 44u);
-static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 50u);
+static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u);
+static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 55u);
 
 struct PushData {
 	static constexpr uint32_t DwordCount = 32;
@@ -367,7 +373,7 @@ DescriptorBindingForImage(const ImageResource& image) {
 			if (image.numeric_class != Prospero::TextureNumericClass::Uint) {
 				return std::nullopt;
 			}
-			base = AtomicUintBinding;
+			base = AtomicUintBinding + (image.atomic64 ? 5u : 0u);
 		} else {
 			switch (image.numeric_class) {
 				case Prospero::TextureNumericClass::Float: base = StorageFloatBinding; break;
@@ -474,6 +480,7 @@ struct DescriptorSource {
 		uint32_t table_offset    = 0;
 		Value    key_count;
 		Value    selector_mask;
+		std::vector<uint32_t> sources;
 
 		bool operator==(const IndirectImage& other) const = default;
 	};
@@ -497,6 +504,7 @@ struct ResourceBlock {
 	Value                 condition;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> sources;
+	std::vector<uint32_t> srt_reads;
 };
 
 // Stable shader metadata consumed by the renderer after native IR has been discarded.
@@ -508,6 +516,7 @@ struct CompiledShaderInfo {
 	uint32_t                      user_data_count     = 64;
 	uint32_t                      scratch_dwords      = 0;
 	uint32_t                      param_export_mask   = 0;
+	bool                          has_address_writes  = false;
 	ShaderInfo                    info;
 	BindingLayout                 bindings;
 };
@@ -562,7 +571,6 @@ struct ResourcePlan {
 	mutable std::vector<uint8_t>            visited_blocks;
 	mutable std::vector<uint32_t>           pending_blocks;
 	mutable std::vector<uint32_t>           material_keys;
-	mutable std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 };
 
 struct Program: ResourcePlan {

src/graphics/shader/recompiler/ir/Value.cpp

@@ -2,7 +2,7 @@
 
 #include <algorithm>
 #include <cstring>
-#include <limits>
+#include <memory>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
@@ -135,10 +135,16 @@ bool Value::operator==(const Value& other) const {
 }
 
 Inst::Inst(ValueOpcode value_opcode, uint64_t value_flags)
-    : opcode(value_opcode), flags(value_flags) {
-	const auto count = NumArgsOf(opcode);
-	if (count != std::numeric_limits<size_t>::max()) {
-		args.resize(count);
+    : opcode(value_opcode),
+      num_args(value_opcode == ValueOpcode::Phi ? PhiArity
+                                               : static_cast<uint8_t>(NumArgsOf(value_opcode))),
+      flags(value_flags) {
+	if (num_args == PhiArity) {
+		std::destroy_at(&fixed_args);
+		std::construct_at(&phi_args);
+	} else if (num_args > InlineArity) {
+		std::destroy_at(&fixed_args);
+		std::construct_at(&large_args, num_args);
 	}
 }
 
@@ -154,8 +160,8 @@ Type Inst::GetType() const {
 	if (opcode == ValueOpcode::Phi) {
 		return static_cast<Type>(flags);
 	}
-	if (opcode == ValueOpcode::Identity && !args.empty()) {
-		return args.front().GetType();
+	if (opcode == ValueOpcode::Identity && num_args != 0) {
+		return Arg(0).GetType();
 	}
 	return TypeOf(opcode);
 }
@@ -173,21 +179,24 @@ size_t Inst::UseCount() const {
 }
 
 size_t Inst::NumArgs() const {
-	return args.size();
+	return num_args == PhiArity ? phi_args.size() : num_args;
 }
 
 size_t Inst::NumPhiBlocks() const {
-	return phi_blocks.size();
+	return num_args == PhiArity ? phi_args.size() : 0;
 }
 
 Value Inst::Arg(size_t index) const {
-	EXIT_IF(index >= args.size());
-	return args[index];
+	EXIT_IF(index >= NumArgs());
+	if (num_args <= InlineArity) {
+		return fixed_args[index];
+	}
+	return num_args == PhiArity ? phi_args[index].second : large_args[index];
 }
 
 Block* Inst::PhiBlock(size_t index) const {
-	EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_blocks.size());
-	return phi_blocks[index];
+	EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_args.size());
+	return phi_args[index].first;
 }
 
 Block* Inst::Parent() const {
@@ -203,15 +212,17 @@ void Inst::SetParent(Block* block) {
 }
 
 void Inst::SetArg(size_t index, Value value) {
-	if (index >= args.size()) {
-		EXIT_IF(NumArgsOf(opcode) != std::numeric_limits<size_t>::max());
-		args.resize(index + 1);
-	}
-	const auto old = args[index];
+	const auto old = Arg(index);
 	if (auto* old_inst = old.TryInstruction(); old_inst != nullptr) {
 		RemoveUse(old_inst, index);
 	}
-	args[index] = value;
+	if (num_args <= InlineArity) {
+		fixed_args[index] = value;
+	} else if (num_args == PhiArity) {
+		phi_args[index].second = value;
+	} else {
+		large_args[index] = value;
+	}
 	if (auto* new_inst = value.TryInstruction(); new_inst != nullptr) {
 		AddUse(new_inst, index);
 	}
@@ -219,9 +230,8 @@ void Inst::SetArg(size_t index, Value value) {
 
 void Inst::AddPhiOperand(Block* predecessor, Value value) {
 	EXIT_IF(opcode != ValueOpcode::Phi);
-	const auto index = args.size();
-	args.push_back(value);
-	phi_blocks.push_back(predecessor);
+	const auto index = phi_args.size();
+	phi_args.emplace_back(predecessor, value);
 	if (auto* value_inst = value.TryInstruction(); value_inst != nullptr) {
 		AddUse(value_inst, index);
 	}
@@ -234,21 +244,12 @@ void Inst::ReplaceUsesWith(Value replacement, bool preserve) {
 	}
 	Invalidate();
 	if (preserve) {
-		ReplaceOpcode(ValueOpcode::Identity);
-		args.resize(1);
+		opcode = ValueOpcode::Identity;
+		num_args = 1;
 		SetArg(0, replacement);
 	}
 }
 
-void Inst::ReplaceOpcode(ValueOpcode value_opcode) {
-	opcode           = value_opcode;
-	const auto count = NumArgsOf(opcode);
-	if (count != std::numeric_limits<size_t>::max()) {
-		EXIT_IF(!args.empty() && args.size() != count);
-		args.resize(count);
-	}
-}
-
 void Inst::Invalidate() {
 	ClearArgs();
 	opcode = ValueOpcode::Void;
@@ -269,13 +270,21 @@ void Inst::RemoveUse(Inst* used, size_t operand) {
 }
 
 void Inst::ClearArgs() {
-	for (size_t index = 0; index < args.size(); index++) {
-		if (auto* value_inst = args[index].TryInstruction(); value_inst != nullptr) {
+	for (size_t index = 0; index < NumArgs(); index++) {
+		if (auto* value_inst = Arg(index).TryInstruction(); value_inst != nullptr) {
 			RemoveUse(value_inst, index);
 		}
 	}
-	args.clear();
-	phi_blocks.clear();
+	if (num_args == PhiArity) {
+		std::destroy_at(&phi_args);
+		std::construct_at(&fixed_args);
+	} else if (num_args > InlineArity) {
+		std::destroy_at(&large_args);
+		std::construct_at(&fixed_args);
+	} else {
+		fixed_args.fill(Value {});
+	}
+	num_args = 0;
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/Value.h

@@ -4,6 +4,7 @@
 #include "graphics/shader/recompiler/ir/Reg.h"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h"
 
+#include <array>
 #include <bit>
 #include <cstdint>
 #include <cstring>
@@ -133,7 +134,6 @@ public:
 	void SetArg(size_t index, Value value);
 	void AddPhiOperand(Block* predecessor, Value value);
 	void ReplaceUsesWith(Value replacement, bool preserve = true);
-	void ReplaceOpcode(ValueOpcode opcode);
 	void Invalidate();
 
 	template <typename T>
@@ -152,17 +152,29 @@ public:
 	}
 
 private:
+	friend void EliminateDeadCode(const std::vector<Block*>& blocks);
+
 	void AddUse(Inst* used, size_t operand);
 	void RemoveUse(Inst* used, size_t operand);
 	void ClearArgs();
 
+	static constexpr uint8_t InlineArity = 4;
+	static constexpr uint8_t PhiArity = UINT8_MAX;
+
 	ValueOpcode         opcode;
+	uint8_t             num_args;
+	bool                live = false;
+	mutable uint32_t    evaluation_index = UINT32_MAX;
 	uint64_t            flags;
 	Block*              parent = nullptr;
-	std::vector<Value>  args;
-	std::vector<Block*> phi_blocks;
+	union {
+		std::array<Value, InlineArity> fixed_args {};
+		std::vector<Value> large_args;
+		std::vector<std::pair<Block*, Value>> phi_args;
+	};
 	std::vector<Use>    uses;
-	mutable uint32_t    evaluation_index = UINT32_MAX;
 };
 
+static_assert(sizeof(Inst) <= 112, "Inst operand storage unintentionally increased");
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -123,6 +123,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicSMax32:
 		case ValueOpcode::BufferAtomicUMax32:
 		case ValueOpcode::BufferAtomicAnd32:
+		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr32:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::BufferAtomicXor32:
@@ -135,6 +136,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
+		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
 		case ValueOpcode::StoreBufferU32x2: return 2u;
@@ -164,6 +166,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicFMax32:
 		case ValueOpcode::SharedAtomicSwap32:
 		case ValueOpcode::SharedAtomicIAdd32:
+		case ValueOpcode::SharedAtomicIAdd64:
 		case ValueOpcode::SharedAtomicISub32:
 		case ValueOpcode::SharedAtomicInc32:
 		case ValueOpcode::SharedAtomicDec32:
@@ -183,6 +186,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 
 uint32_t SharedComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
+		case ValueOpcode::SharedAtomicIAdd64:
 		case ValueOpcode::SharedAtomicOr64:
 		case ValueOpcode::LoadSharedU32x2:
 		case ValueOpcode::WriteSharedU32x2: return 2u;
@@ -216,10 +220,14 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
 			return {ImageAccess::Read, ImageResourceClass::Sampled, true};
 		case ValueOpcode::ImageWrite:
 			return {ImageAccess::Write, ImageResourceClass::Storage, false};
+		case ValueOpcode::ImageAtomicCompareSwap32:
 		case ValueOpcode::ImageAtomicSwap32:
 		case ValueOpcode::ImageAtomicIAdd32:
+		case ValueOpcode::ImageAtomicSMin32:
 		case ValueOpcode::ImageAtomicUMin32:
+		case ValueOpcode::ImageAtomicSMax32:
 		case ValueOpcode::ImageAtomicUMax32:
+		case ValueOpcode::ImageAtomicUMax64:
 		case ValueOpcode::ImageAtomicAnd32:
 		case ValueOpcode::ImageAtomicOr32:
 		case ValueOpcode::ImageAtomicXor32:

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -9,7 +9,7 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-enum class ValueOpcode {
+enum class ValueOpcode : uint16_t {
 #define VALUE_OPCODE(name, ...) name,
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc"
 #undef VALUE_OPCODE

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -137,6 +137,9 @@ VALUE_OPCODE(SMedTri32, U32, U32, U32, U32)
 VALUE_OPCODE(UMedTri32, U32, U32, U32, U32)
 VALUE_OPCODE(SLessThan32, U1, U32, U32)
 VALUE_OPCODE(SLessThan64, U1, U64, U64)
+VALUE_OPCODE(SLessThanEqual64, U1, U64, U64)
+VALUE_OPCODE(ULessThanEqual64, U1, U64, U64)
+VALUE_OPCODE(UGreaterThanEqual64, U1, U64, U64)
 VALUE_OPCODE(ULessThan32, U1, U32, U32)
 VALUE_OPCODE(ULessThan64, U1, U64, U64)
 VALUE_OPCODE(IEqual32, U1, U32, U32)
@@ -155,6 +158,9 @@ VALUE_OPCODE(LogicalAnd, U1, U1, U1)
 VALUE_OPCODE(LogicalXor, U1, U1, U1)
 VALUE_OPCODE(LogicalNot, U1, U1)
 VALUE_OPCODE(FPOrdEqual32, U1, F32, F32)
+VALUE_OPCODE(FPOrdEqual64, U1, F64, F64)
+VALUE_OPCODE(FPOrdLessThanEqual64, U1, F64, F64)
+VALUE_OPCODE(FPOrdGreaterThanEqual64, U1, F64, F64)
 VALUE_OPCODE(FPUnordEqual32, U1, F32, F32)
 VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32)
 VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32)
@@ -173,6 +179,7 @@ VALUE_OPCODE(FPAdd32, F32, F32, F32)
 VALUE_OPCODE(FPSub32, F32, F32, F32)
 VALUE_OPCODE(FPFma32, F32, F32, F32, F32)
 VALUE_OPCODE(FPMul32, F32, F32, F32)
+VALUE_OPCODE(FPAdd64, F64, F64, F64)
 VALUE_OPCODE(FPMul64, F64, F64, F64)
 VALUE_OPCODE(FPFma64, F64, F64, F64, F64)
 VALUE_OPCODE(FPRecip64, F64, F64)
@@ -245,6 +252,7 @@ VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicAnd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicOr64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -267,6 +275,7 @@ VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1)
 VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicInc32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicDec32, U32, U32, U32, U1)
@@ -283,15 +292,20 @@ VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
+VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
 VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
 VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
 VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
-VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
+VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress, U32, U32, U32, U32)
+VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1)
 VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicUMax64, U64, ImageResource, ImageAddress, U64, U1)
 VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -126,7 +126,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 			EXIT("shader binding layout failed: image %u has an unmapped binding class", i);
 		}
 		auto&      resources = image_groups[group];
-		const auto dynamic   = program.info.images[i].mip_mode == ImageMipMode::DynamicStorage;
+		const auto dynamic   = program.info.images[i].mip_mode == ImageMipMode::Dynamic;
 		const auto count     = dynamic ? program.info.images[i].mip_count : 1u;
 		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
 			EXIT("shader binding layout failed: image %u has invalid specialized mip count %u", i,

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <bit>
 #include <cstdint>
+#include <unordered_map>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -575,6 +576,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 		case ValueOpcode::UGreaterThan64:
 			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; });
 			return;
+		case ValueOpcode::ULessThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; });
+			return;
+		case ValueOpcode::UGreaterThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; });
+			return;
+		case ValueOpcode::SLessThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) {
+				return std::bit_cast<int64_t>(a) <= std::bit_cast<int64_t>(b);
+			});
+			return;
 		case ValueOpcode::SLessThan64:
 			FoldU64Compare(inst, [](uint64_t a, uint64_t b) {
 				return std::bit_cast<int64_t>(a) < std::bit_cast<int64_t>(b);
@@ -649,12 +661,110 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 	}
 }
 
+// In wave32, extracting this invocation's bit from a ballot recovers its predicate.
+// Keep that identity through scalar EXEC operations and their loop-carried mask Phis.
+class LaneMaskProjection {
+public:
+	explicit LaneMaskProjection(std::unordered_set<Inst*>& lowered_ancillary)
+	    : m_lowered_ancillary(lowered_ancillary) {}
+
+	void Fold(Inst& inst) {
+		if (inst.GetOpcode() != ValueOpcode::INotEqual32 || !Immediate(Arg(inst, 1), 0u)) return;
+		const auto* bit = Arg(inst, 0).TryInstruction();
+		if (bit == nullptr || bit->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
+		    !Immediate(Arg(*bit, 1), 1u)) return;
+		const auto* shift = Arg(*bit, 0).TryInstruction();
+		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftRightLogical32) return;
+		const auto* index = Arg(*shift, 1).TryInstruction();
+		if (index == nullptr || index->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
+		    !Immediate(Arg(*index, 1), 31u)) return;
+		const auto* lane = Arg(*index, 0).TryInstruction();
+		if (lane == nullptr || lane->GetOpcode() != ValueOpcode::LaneId) return;
+		m_visited.clear();
+		m_grounded = false;
+		if (CanProject(Arg(*shift, 0)) && m_grounded) Replace(inst, Project(Arg(*shift, 0)));
+	}
+
+private:
+	static bool Immediate(Value value, uint32_t expected) {
+		return IsImmediate(value, Type::U32) && value.U32() == expected;
+	}
+
+	static Value BallotPredicate(const Inst& inst) {
+		if (inst.GetOpcode() != ValueOpcode::CompositeExtractU32x4 ||
+		    !Immediate(Arg(inst, 1), 0u)) return {};
+		const auto* source = Arg(inst, 0).TryInstruction();
+		return source != nullptr && source->GetOpcode() == ValueOpcode::Ballot
+		           ? Arg(*source, 0) : Value {};
+	}
+
+	bool CanProject(Value value) {
+		if (value.IsImmediate()) {
+			const bool supported = Immediate(value, 0u) || Immediate(value, UINT32_MAX);
+			m_grounded |= supported;
+			return supported;
+		}
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return false;
+		if (m_values.contains(inst) || !BallotPredicate(*inst).IsEmpty()) {
+			m_grounded = true;
+			return true;
+		}
+		if (!m_visited.insert(inst).second) return true;
+		const auto op = inst->GetOpcode();
+		if (op != ValueOpcode::BitwiseAnd32 && op != ValueOpcode::BitwiseOr32 &&
+		    op != ValueOpcode::BitwiseNot32 && op != ValueOpcode::SelectU32 && op != ValueOpcode::Phi)
+			return false;
+		for (size_t arg = op == ValueOpcode::SelectU32 ? 1u : 0u; arg < inst->NumArgs(); ++arg) {
+			if (!CanProject(Arg(*inst, arg))) return false;
+		}
+		return inst->NumArgs() != 0;
+	}
+
+	Value Project(Value value) {
+		if (value.IsImmediate()) return Value(value.U32() != 0u);
+		auto* source = value.TryInstruction();
+		if (const auto found = m_values.find(source); found != m_values.end()) return found->second.Resolve();
+		if (const auto predicate = BallotPredicate(*source); !predicate.IsEmpty()) return predicate;
+		auto* block = source->Parent();
+		const auto where = std::find_if(block->begin(), block->end(),
+		                               [source](const Inst& candidate) { return &candidate == source; });
+		const auto op = source->GetOpcode() == ValueOpcode::BitwiseAnd32 ? ValueOpcode::LogicalAnd
+		              : source->GetOpcode() == ValueOpcode::BitwiseOr32 ? ValueOpcode::LogicalOr
+		              : source->GetOpcode() == ValueOpcode::BitwiseNot32 ? ValueOpcode::LogicalNot
+		              : source->GetOpcode() == ValueOpcode::SelectU32 ? ValueOpcode::SelectU1
+		                                                               : ValueOpcode::Phi;
+		auto result = op == ValueOpcode::Phi ? block->PrependNewInst(where, op)
+		            : op == ValueOpcode::LogicalNot
+		                ? block->PrependNewInst(where, op, {Value(false)})
+		            : op == ValueOpcode::SelectU1
+		                ? block->PrependNewInst(where, op, {Arg(*source, 0), Value(false), Value(false)})
+		                : block->PrependNewInst(where, op, {Value(false), Value(false)});
+		m_values.emplace(source, Value(&*result));
+		if (op == ValueOpcode::Phi) result->SetFlags(Type::U1);
+		for (size_t arg = op == ValueOpcode::SelectU1 ? 1u : 0u; arg < source->NumArgs(); ++arg) {
+			const auto projected = Project(Arg(*source, arg));
+			if (op == ValueOpcode::Phi) result->AddPhiOperand(source->PhiBlock(arg), projected);
+			else result->SetArg(arg, projected);
+		}
+		FoldInstruction(*block, result, m_lowered_ancillary);
+		return Value(&*result).Resolve();
+	}
+
+	std::unordered_set<Inst*>& m_lowered_ancillary;
+	std::unordered_set<const Inst*> m_visited;
+	std::unordered_map<const Inst*, Value> m_values;
+	bool m_grounded = false;
+};
+
 } // namespace
 
-void ConstantPropagationPass(const BlockList& blocks) {
+void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size) {
 	std::unordered_set<Inst*> lowered_ancillary;
+	LaneMaskProjection mask_projection(lowered_ancillary);
 	for (auto* block: blocks) {
 		for (auto inst = block->begin(); inst != block->end(); ++inst) {
+			if (wave_size == 32u) mask_projection.Fold(*inst);
 			FoldInstruction(*block, inst, lowered_ancillary);
 		}
 	}

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h

@@ -4,6 +4,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-void ConstantPropagationPass(const BlockList& blocks);
+void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size = 64);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp

@@ -1,5 +1,7 @@
 #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
 
+#include <algorithm>
+
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
 void RemoveIdentities(const BlockList& blocks) {
@@ -18,23 +20,32 @@ void RemoveIdentities(const BlockList& blocks) {
 }
 
 void EliminateDeadCode(const BlockList& blocks) {
-	bool changed;
-	do {
-		changed = false;
-		for (auto block = blocks.rbegin(); block != blocks.rend(); block++) {
-			auto& instructions = (*block)->Instructions();
-			auto  inst         = instructions.end();
-			while (inst != instructions.begin()) {
-				--inst;
-				if (inst->HasUses() || inst->MayHaveSideEffects()) {
-					continue;
-				}
-				inst->Invalidate();
-				inst    = instructions.erase(inst);
-				changed = true;
+	std::vector<Inst*> pending;
+	for (auto* block: blocks) {
+		for (auto& inst: *block) {
+			// Retained resource-planning instructions live outside the block list.
+			inst.live = inst.MayHaveSideEffects() || std::ranges::any_of(inst.Uses(),
+			    [](const Use& use) { return use.user->Parent() == nullptr; });
+			if (inst.live) pending.push_back(&inst);
+		}
+	}
+	while (!pending.empty()) {
+		const auto* inst = pending.back();
+		pending.pop_back();
+		for (size_t arg = 0; arg < inst->NumArgs(); ++arg) {
+			auto* input = inst->Arg(arg).TryInstruction();
+			if (input != nullptr && input->Parent() != nullptr && !input->live) {
+				input->live = true;
+				pending.push_back(input);
 			}
 		}
-	} while (changed);
+	}
+	// Unlink every dead operand before freeing nodes in a dead Phi cycle.
+	for (auto* block: blocks)
+		for (auto& inst: *block)
+			if (!inst.live) inst.Invalidate();
+	for (auto* block: blocks)
+		std::erase_if(block->Instructions(), [](const Inst& inst) { return !inst.live; });
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -109,8 +109,7 @@ Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) {
 
 enum class SamplerClass : uint8_t { Float, Integer, PointInteger };
 
-template <typename Image>
-SamplerClass ClassifySampler(const Image& image) {
+SamplerClass ClassifySampler(const ImageResource& image) {
 	if (image.numeric_class == Prospero::TextureNumericClass::Sint ||
 	    image.conversion_format != Prospero::BufferFormat::kInvalid) {
 		return SamplerClass::PointInteger;
@@ -124,8 +123,8 @@ bool DescriptorIsCube(const DescriptorValue& descriptor) {
 	       Prospero::ImageType::kCube;
 }
 
-uint32_t StorageMipCount(const ImageResource& image, const DescriptorValue& descriptor) {
-	if (image.mip_mode != ImageMipMode::DynamicStorage || NullImageDescriptor(descriptor)) {
+uint32_t ImageMipCount(const ImageResource& image, const DescriptorValue& descriptor) {
+	if (image.mip_mode != ImageMipMode::Dynamic || NullImageDescriptor(descriptor)) {
 		return 1;
 	}
 	const auto base = (descriptor.dwords[3] >> 12u) & 0xfu;
@@ -157,34 +156,15 @@ bool CaptureStrictRead(void* userdata, uint64_t address, std::span<uint32_t> val
 
 bool CaptureOrdinaryRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
 	auto& capture = *static_cast<ReadCapture*>(userdata);
-	if (!capture.source.read_memory(capture.source.userdata, address, values)) {
-		return false;
+	if (capture.source.read_memory != nullptr) {
+		if (!capture.source.read_memory(capture.source.userdata, address, values)) return false;
+	} else {
+		std::memcpy(values.data(), reinterpret_cast<const void*>(address), values.size_bytes());
 	}
 	capture.ranges.emplace_back(address, values.size_bytes());
 	return true;
 }
 
-bool WrittenBuffersDisjoint(const ResourcePlan& program, const ResourceSnapshot& snapshot,
-                            std::span<const std::pair<uint64_t, uint64_t>> reads) {
-	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!program.info.buffers[i].written) continue;
-		ShaderBufferResource buffer;
-		if (!DecodeBufferDescriptor(snapshot.buffers[i], buffer)) return false;
-		const auto base = buffer.Base48();
-		const auto size = buffer.GetSize();
-		if (size == 0u) continue;
-		if (size - 1u > AddressMask - base) return false;
-		for (const auto [address, bytes]: reads) {
-			if (bytes == 0u) continue;
-			if (address > AddressMask || bytes - 1u > AddressMask - address ||
-			    (address <= base ? base - address < bytes : address - base < size)) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
 const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
 	if (source >= program.descriptor_sources.size()) {
 		return nullptr;
@@ -246,100 +226,114 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
 
 bool MaterializeIndirectImage(const ResourcePlan& program,
                               const DescriptorSource::IndirectImage& indirect,
-                              const DescriptorValue& material_value,
-                              const DescriptorValue& table_value, uint32_t image_index,
+                              uint32_t image_index,
                               const SrtRuntime& runtime, SrtWalker& clean,
                               ResourceSnapshot& snapshot,
                               ResourceSpecialization& specialization) {
+	const auto& sources = indirect.sources;
+	auto& keys = program.material_keys;
 	uint64_t table_base = 0;
 	uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
-	ShaderBufferResource table;
-	if (table_value.dword_count == 2u) {
-		table_base = (static_cast<uint64_t>(table_value.dwords[1]) << 32u) | table_value.dwords[0];
-	} else if (DecodeBufferDescriptor(table_value, table)) {
-		table_base = table.Base48();
-		table_size = table.GetSize();
-	} else {
-		return false;
-	}
-	auto& keys = program.material_keys;
-	keys.clear();
-	if (indirect.material_source == UINT32_MAX) {
-		uint32_t key_count = 0;
-		const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
-		if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
-		if (table_value.dword_count != 2u || !evaluated ||
-		    key_count > MaxIndirectImageProbes ||
-		    uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
+	if (sources.empty()) {
+		keys.clear();
+		DescriptorValue material_value;
+		DescriptorValue table_value;
+		if ((indirect.material_source != UINT32_MAX &&
+		     !clean.EvaluateDescriptor(indirect.material_source, material_value)) ||
+		    !clean.EvaluateDescriptor(indirect.table_source, table_value)) {
 			return false;
 		}
-		keys.resize(key_count);
-		std::iota(keys.begin(), keys.end(), 0u);
-	} else if (!indirect.selector_mask.IsEmpty()) {
-		uint32_t mask = 0;
-		uint32_t count = 0;
-		if (material_value.dword_count != 2u || table_value.dword_count != 2u ||
-		    !clean.Evaluate(indirect.selector_mask, mask) ||
-		    !clean.Evaluate(indirect.key_count, count) || count == 0u || count > 32u) {
+		ShaderBufferResource table;
+		if (table_value.dword_count == 2u) {
+			table_base = (static_cast<uint64_t>(table_value.dwords[1]) << 32u) | table_value.dwords[0];
+		} else if (DecodeBufferDescriptor(table_value, table)) {
+			table_base = table.Base48();
+			table_size = table.GetSize();
+		} else {
 			return false;
 		}
-		if (count < 32u) mask &= (1u << count) - 1u;
-		const auto material_base =
-		    (static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
-		keys.reserve(std::popcount(mask));
-		while (mask != 0u) {
-			const auto index = std::countr_zero(mask);
-			const auto offset = static_cast<uint64_t>(indirect.selector_offset) +
-			                    static_cast<uint64_t>(index) * indirect.selector_stride;
-			if (offset > UINT32_MAX) return false;
-			uint32_t key = 0;
-			if (!ReadScalarTable(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
-			                     runtime, {&key, 1})) return false;
-			keys.push_back(key);
-			mask &= mask - 1u;
-		}
-		std::ranges::sort(keys);
-		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
-	} else {
-		ShaderBufferResource material;
-		if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
-		    material.Stride() != indirect.selector_stride) {
-			return false;
-		}
-		// The first aligned offset includes the immediate added after shader U32 arithmetic.
-		const auto step = std::max<uint64_t>(4u,
-		    std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
-		const uint64_t first = indirect.selector_offset;
-		const auto size = material.GetSize();
-		const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
-		const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
-		if (probe_count > MaxIndirectImageProbes) {
-			return false;
-		}
-		keys.reserve(static_cast<size_t>(probe_count) + 1u);
-		keys.push_back(0u);
-		for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
-			uint32_t key = 0;
-			if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
+		if (indirect.material_source == UINT32_MAX) {
+			uint32_t key_count = 0;
+			const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
+			if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
+			if (table_value.dword_count != 2u || !evaluated ||
+			    key_count > MaxIndirectImageProbes ||
+			    uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
 				return false;
 			}
-			keys.push_back(key);
+			keys.resize(key_count);
+			std::iota(keys.begin(), keys.end(), 0u);
+		} else if (!indirect.selector_mask.IsEmpty()) {
+			uint32_t mask = 0;
+			uint32_t count = 0;
+			if (material_value.dword_count != 2u || table_value.dword_count != 2u ||
+			    !clean.Evaluate(indirect.selector_mask, mask) ||
+			    !clean.Evaluate(indirect.key_count, count) || count == 0u || count > 32u) {
+				return false;
+			}
+			if (count < 32u) mask &= (1u << count) - 1u;
+			const auto material_base =
+			    (static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
+			keys.reserve(std::popcount(mask));
+			while (mask != 0u) {
+				const auto index = std::countr_zero(mask);
+				const auto offset = static_cast<uint64_t>(indirect.selector_offset) +
+				                    static_cast<uint64_t>(index) * indirect.selector_stride;
+				if (offset > UINT32_MAX) return false;
+				uint32_t key = 0;
+				if (!ReadScalarTable(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
+				                     runtime, {&key, 1})) return false;
+				keys.push_back(key);
+				mask &= mask - 1u;
+			}
+			std::ranges::sort(keys);
+			keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
+		} else {
+			ShaderBufferResource material;
+			if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
+			    material.Stride() != indirect.selector_stride) {
+				return false;
+			}
+			// The first aligned offset includes the immediate added after shader U32 arithmetic.
+			const auto step = std::max<uint64_t>(4u,
+			    std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
+			const uint64_t first = indirect.selector_offset;
+			const auto size = material.GetSize();
+			const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
+			const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
+			if (probe_count > MaxIndirectImageProbes) {
+				return false;
+			}
+			keys.reserve(static_cast<size_t>(probe_count) + 1u);
+			keys.push_back(0u);
+			for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
+				uint32_t key = 0;
+				if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
+					return false;
+				}
+				keys.push_back(key);
+			}
+			std::ranges::sort(keys);
+			keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 		}
-		std::ranges::sort(keys);
-		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 	}
 
 	const auto children_begin = snapshot.images.size();
 	const auto mapping_offset = snapshot.flattened_srt.size();
 	const auto root_image = specialization.images[image_index];
-	snapshot.flattened_srt.resize(mapping_offset + 1u + keys.size() * 2u);
-	snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(keys.size());
-	for (uint32_t entry = 0; entry < keys.size(); ++entry) {
-		const auto key = keys[entry];
+	const auto key_count = sources.empty() ? keys.size() : sources.size();
+	snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u);
+	snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(key_count);
+	for (uint32_t entry = 0; entry < key_count; ++entry) {
+		const auto key = sources.empty() ? keys[entry] : entry;
 		DescriptorValue candidate;
 		candidate.dword_count = 8u;
-		const auto table_offset = (key << 5u) + indirect.table_offset;
-		if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
+		if (sources.empty()) {
+			const auto table_offset = (key << 5u) + indirect.table_offset;
+			if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
+				return false;
+			}
+		} else if (!clean.EvaluateDescriptor(sources[entry], candidate)) {
 			return false;
 		}
 		if (NullImageDescriptor(candidate) ||
@@ -372,7 +366,7 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 		auto& root = specialization.images[image_index];
 		root.indirect_root = image_index;
 		root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
-		root.indirect_search_iterations = std::bit_width(keys.size());
+		root.indirect_search_iterations = std::bit_width(key_count);
 	}
 	return true;
 }
@@ -423,41 +417,8 @@ private:
 	uint32_t                                    count = 0;
 };
 
-template <typename Images>
-bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan);
-
 static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSnapshot& snapshot,
                                         ResourceSpecialization& specialization) {
-	specialization.buffers.clear();
-	specialization.buffers.reserve(program.info.buffers.size());
-	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
-		auto&                descriptor_value = snapshot.buffers[i];
-		ShaderBufferResource descriptor;
-		if (!DecodeBufferDescriptor(descriptor_value, descriptor)) {
-			return SpecializationFail(fmt::format("buffer descriptor {} has invalid width", i));
-		}
-		if (descriptor.Type() != 0) {
-			descriptor_value.dwords.fill(0);
-			descriptor = {};
-		}
-		auto       packed_stride = descriptor.PackedStride();
-		const auto stride        = packed_stride & 0x3fffu;
-		const bool swizzle       = stride != 0u && ((packed_stride >> 14u) & 1u) != 0u;
-		if (stride == 0u) {
-			packed_stride &= ~((1u << 14u) | (3u << 16u));
-		} else if (!swizzle) {
-			packed_stride &= ~(3u << 16u);
-		}
-		specialization.buffers.push_back({
-		    .packed_stride     = packed_stride,
-		    .descriptor_format = program.info.buffers[i].formatted
-		                             ? descriptor.Format()
-		                             : Prospero::BufferFormat::kInvalid,
-		    .descriptor_swizzle =
-		        program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
-		    .zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u,
-		});
-	}
 	for (uint32_t i = 0; i < specialization.images.size(); i++) {
 		const auto& descriptor = snapshot.images[i];
 		auto&       image      = specialization.images[i];
@@ -470,10 +431,10 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 		    (base.atomic && base.resource_class != ImageResourceClass::Storage)) {
 			return SpecializationFail(fmt::format("image resource {} has an invalid class", i));
 		}
-		image.mip_count = StorageMipCount(base, descriptor);
+		image.mip_count = ImageMipCount(base, descriptor);
 		if (image.mip_count == 0u) {
 			return SpecializationFail(
-			    fmt::format("storage image descriptor {} has an invalid mip range", i));
+			    fmt::format("image descriptor {} has an invalid mip range", i));
 		}
 		if (NullImageDescriptor(descriptor)) {
 			image.numeric_class = base.atomic ? Prospero::TextureNumericClass::Uint
@@ -495,8 +456,10 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 		image.cube      = DescriptorIsCube(descriptor);
 		const auto format =
 		    static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
-		if (base.atomic && format != Prospero::BufferFormat::k32UInt &&
-		    format != Prospero::BufferFormat::k32Float) {
+		if (base.atomic &&
+		    (base.atomic64 ? format != Prospero::BufferFormat::k32_32UInt
+		                   : format != Prospero::BufferFormat::k32UInt &&
+		                         format != Prospero::BufferFormat::k32Float)) {
 			return SpecializationFail(
 			    fmt::format("atomic image descriptor {} uses unsupported format {}", i,
 			                static_cast<uint32_t>(format)));
@@ -596,30 +559,21 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			}
 		}
 	}
-	SamplerPlan sampler_plan;
-	if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) {
-		return SpecializationFail("specialized sampler layout exceeds its resource limit");
-	}
-	for (uint32_t index = static_cast<uint32_t>(program.info.samplers.size());
-	     index < sampler_plan.sampler_count; index++) {
-		snapshot.samplers.push_back(snapshot.samplers[sampler_plan.bindings[index].source]);
-	}
 	ImageRemap(specialization).Apply(snapshot.images);
 	return true;
 }
 
-template <typename Images>
-bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) {
+bool BuildSamplerPlan(const ShaderInfo& base, SamplerPlan& plan) {
 	if (base.samplers.size() > plan.mapping.size()) {
 		return false;
 	}
 	std::array<uint8_t, ShaderInfo::MaxSamplers> usage {};
 	plan.sampler_count = static_cast<uint32_t>(base.samplers.size());
 	for (const auto& pair: base.sampled_pairs) {
-		if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) {
+		if (pair.image >= base.images.size() || pair.sampler >= base.samplers.size()) {
 			return false;
 		}
-		usage[pair.sampler] |= 1u << static_cast<uint32_t>(ClassifySampler(images[pair.image]));
+		usage[pair.sampler] |= 1u << static_cast<uint32_t>(ClassifySampler(base.images[pair.image]));
 	}
 	for (uint32_t index = 0; index < base.samplers.size(); index++) {
 		auto& mapping = plan.mapping[index];
@@ -638,9 +592,10 @@ bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan&
 	return true;
 }
 
-static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
-	// Any shader write may alias a scalar predicate read, including on a later loop visit.
-	if (program.blocks.size() != program.block_info.size() || HasShaderMemoryWrites(program)) {
+template <typename Predicate>
+static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, const Predicate& predicate) {
+	// The renderer checks captured scalar reads against final buffer and image write ranges.
+	if (program.blocks.size() != program.block_info.size() || program.has_address_writes) {
 		return {};
 	}
 	std::unordered_map<uint32_t, uint32_t> indices;
@@ -659,9 +614,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 			case CFG::TerminatorKind::Branch: successors.push_back(terminator.true_block); break;
 			case CFG::TerminatorKind::ConditionalBranch:
 				successors = {terminator.true_block, terminator.false_block};
-				if (ValidateRuntimeValue(program, info.condition, RuntimeValueType::Integer)) {
-					block.condition = info.condition;
-				}
+				block.condition = info.condition;
 				break;
 			case CFG::TerminatorKind::IndirectBranch:
 				successors = terminator.indirect_targets;
@@ -678,13 +631,17 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 		}
 		for (const auto& inst: *program.blocks[i]) {
 			const auto op     = inst.GetOpcode();
+			if (op == ValueOpcode::ReadConst) {
+				block.srt_reads.push_back(inst.Arg(1).Resolve().U32());
+				continue;
+			}
 			const auto buffer = BufferAccessOf(op);
 			const auto image  = ImageOpcodeInfoOf(op);
 			if (buffer == BufferAccess::None && image.access == ImageAccess::None) {
 				continue;
 			}
 			const auto& memory = program.memory_info.at(inst.Flags<MemoryFlags>().index);
-			if (memory.planning_only) {
+			if (memory.planning_only || memory.kind == ResourceKind::IndirectBuffer) {
 				continue;
 			}
 			if (buffer != BufferAccess::None) {
@@ -700,6 +657,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 		block.sources.erase(std::unique(block.sources.begin(), block.sources.end()),
 		                    block.sources.end());
 	}
+	for (auto& block: blocks) block.condition = predicate(block.condition);
 	if (std::ranges::none_of(
 	        blocks, [](const ResourceBlock& block) { return !block.condition.IsEmpty(); })) {
 		return {};
@@ -914,34 +872,72 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	for (const auto& read: program.srt_reads) {
 		plan.srt_reads.push_back({Clone(read.value), read.flat_offset});
 	}
-	plan.control_flow = ResourceControlFlow(program);
-	for (auto& block: plan.control_flow) {
-		block.condition = Clone(block.condition);
-	}
+	// A proven uniform factor can decide a branch even when its other lanes are unknown.
+	// Keep only that Boolean structure, never the varying shader dependency graph.
+	Value unknown;
+	std::function<Value(Value)> ClonePredicate = [&](Value value) -> Value {
+		value = value.Resolve();
+		if (value.IsEmpty()) return {};
+		if (ValidateRuntimeValue(program, value, RuntimeValueType::Integer)) return Clone(value);
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return {};
+		const auto op = inst->GetOpcode();
+		if (op == ValueOpcode::ConditionRef || op == ValueOpcode::LogicalNot) {
+			const auto operand = ClonePredicate(inst->Arg(0));
+			if (operand.IsEmpty() || op == ValueOpcode::ConditionRef) return operand;
+			auto& node = plan.value_storage.emplace_back(op);
+			node.SetArg(0, operand);
+			return Value(&node);
+		}
+		if (op != ValueOpcode::LogicalAnd && op != ValueOpcode::LogicalOr) return {};
+		auto left = ClonePredicate(inst->Arg(0));
+		auto right = ClonePredicate(inst->Arg(1));
+		if (left.IsEmpty() && right.IsEmpty()) return {};
+		if (left.IsEmpty() || right.IsEmpty()) {
+			if (unknown.IsEmpty()) unknown = Value(&plan.value_storage.emplace_back(ValueOpcode::UndefU1));
+			if (left.IsEmpty()) left = unknown;
+			if (right.IsEmpty()) right = unknown;
+		}
+		auto& node = plan.value_storage.emplace_back(op);
+		node.SetArg(0, left);
+		node.SetArg(1, right);
+		return Value(&node);
+	};
+	plan.control_flow = ResourceControlFlow(program, ClonePredicate);
 	plan.uniform_fill = AnalyzeUniformFill(program);
 	for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
 	}
 	plan.clean_flat_slots.resize(plan.srt_reads.size());
-	bool masked_image = false;
+	bool capture_image_reads = false;
 	for (const auto& image: plan.info.images) {
 		const auto* source = Source(plan, image.source);
 		if (source == nullptr || !source->indirect_image.has_value()) {
 			continue;
 		}
 		plan.requires_specialization_memory = true;
-		masked_image |= !source->indirect_image->selector_mask.IsEmpty();
+		capture_image_reads |= !source->indirect_image->selector_mask.IsEmpty() ||
+		                       !source->indirect_image->sources.empty();
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
 		                   plan.clean_flat_slots, source->indirect_image->selector_mask);
-		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
-		                   plan.clean_flat_slots);
+		if (source->indirect_image->sources.empty()) {
+			MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
+			                   plan.clean_flat_slots);
+		} else {
+			for (const auto candidate: source->indirect_image->sources) {
+				MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots);
+			}
+		}
 	}
-	if (masked_image) {
-		plan.resource_tracking_complete &= !program.has_address_writes &&
-		    !std::ranges::any_of(plan.info.images, &ImageResource::written);
-		plan.capture_specialization_reads =
-		    std::ranges::any_of(plan.info.buffers, &BufferResource::written);
+	plan.capture_specialization_reads = capture_image_reads || !plan.control_flow.empty();
+	// Writable descriptor addresses must come from clean backing for the alias proof.
+	for (const auto& buffer: plan.info.buffers) {
+		if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
 	}
+	for (const auto& image: plan.info.images) {
+		if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
+	}
+	if (capture_image_reads) plan.resource_tracking_complete &= !program.has_address_writes;
 	return plan;
 }
 
@@ -952,21 +948,22 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		return false;
 	}
 	const bool capture_reads = program.capture_specialization_reads;
-	auto& reads = program.specialization_reads;
+	auto& reads = snapshot.specialization_reads;
+	reads.clear();
 	ReadCapture capture {runtime, reads};
 	SrtRuntime observed = runtime;
 	if (capture_reads) {
-		reads.clear();
 		observed.userdata = &capture;
-		observed.read_specialization_memory = CaptureStrictRead;
-		if (observed.read_memory != nullptr) observed.read_memory = CaptureOrdinaryRead;
+		observed.read_specialization_memory = runtime.read_specialization_memory != nullptr
+		                                         ? CaptureStrictRead : nullptr;
+		observed.read_memory = CaptureOrdinaryRead;
 	}
 	SrtWalker clean(program, CleanRuntime(observed));
 	SrtWalker walker(program, observed, program.clean_flat_slots, &clean);
-	const auto active = clean.FindActiveSources();
 	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
 		return false;
 	}
+	const auto active = std::span<const uint8_t>(program.active_sources);
 	snapshot.uniform_fill = {};
 	const auto& fill = program.uniform_fill;
 	const auto words = fill.fill.words;
@@ -979,31 +976,51 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		snapshot.uniform_fill = fill.fill;
 		snapshot.uniform_fill.value = stored[0];
 	}
-	const auto evaluate = [&](uint32_t source, DescriptorValue& value) {
+	const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
 		if (source >= program.descriptor_sources.size()) {
 			return false;
 		}
 		if (active.empty() || active[source]) {
-			return walker.EvaluateDescriptor(source, value);
+			return (written ? clean : walker).EvaluateDescriptor(source, value);
 		}
 		value = {};
 		value.dword_count = program.descriptor_sources[source].dword_count;
 		return true;
 	};
 	snapshot.buffers.resize(program.info.buffers.size());
+	specialization.buffers.clear();
+	specialization.buffers.reserve(program.info.buffers.size());
 	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i])) {
+		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i],
+		              program.info.buffers[i].written)) {
 			return false;
 		}
-	}
-	if (capture_reads) {
-		for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-			const auto& buffer = program.info.buffers[i];
-			if (!buffer.written || (!active.empty() && !active[buffer.source])) continue;
-			DescriptorValue strict;
-			if (!clean.EvaluateDescriptor(buffer.source, strict) ||
-			    strict != snapshot.buffers[i]) return false;
+		auto&                descriptor_value = snapshot.buffers[i];
+		ShaderBufferResource descriptor;
+		if (!DecodeBufferDescriptor(descriptor_value, descriptor)) {
+			return SpecializationFail(fmt::format("buffer descriptor {} has invalid width", i));
 		}
+		if (descriptor.Type() != 0) {
+			descriptor_value.dwords.fill(0);
+			descriptor = {};
+		}
+		auto       packed_stride = descriptor.PackedStride();
+		const auto stride        = packed_stride & 0x3fffu;
+		const bool swizzle       = stride != 0u && ((packed_stride >> 14u) & 1u) != 0u;
+		if (stride == 0u) {
+			packed_stride &= ~((1u << 14u) | (3u << 16u));
+		} else if (!swizzle) {
+			packed_stride &= ~(3u << 16u);
+		}
+		specialization.buffers.push_back({
+		    .packed_stride     = packed_stride,
+		    .descriptor_format = program.info.buffers[i].formatted
+		                             ? descriptor.Format()
+		                             : Prospero::BufferFormat::kInvalid,
+		    .descriptor_swizzle =
+		        program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
+		    .zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u,
+		});
 	}
 	snapshot.images.resize(program.info.images.size());
 	specialization.images.resize(program.info.images.size());
@@ -1030,17 +1047,11 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 				continue;
 			}
 			const auto& indirect = *source->indirect_image;
-			DescriptorValue material;
-			DescriptorValue table;
-			if ((indirect.material_source != UINT32_MAX &&
-			     !clean.EvaluateDescriptor(indirect.material_source, material)) ||
-			    !clean.EvaluateDescriptor(indirect.table_source, table) ||
-			    !MaterializeIndirectImage(program, indirect, material, table, i, observed, clean, snapshot,
-			                              specialization)) {
+			if (!MaterializeIndirectImage(program, indirect, i, observed, clean, snapshot, specialization)) {
 				return false;
 			}
 		} else {
-			if (!evaluate(image.source, snapshot.images[i])) {
+			if (!evaluate(image.source, snapshot.images[i], image.written)) {
 				return false;
 			}
 			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
@@ -1053,8 +1064,17 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		if (!evaluate(program.info.samplers[i].source, snapshot.samplers[i])) {
 			return false;
 		}
+		if (program.info.samplers[i].gather_lod) {
+			const auto control = snapshot.samplers[i].dwords[2];
+			const auto filter = (control >> 26u) & 3u;
+			// MipNone always selects the base level. Explicit point gathers currently require
+			// encoded-zero primary and secondary bias; linear primary-mip selection is unsupported.
+			if (filter > 1u || (filter == 1u && (control & 0xfffffu) != 0u)) {
+				return SpecializationFail(
+				    "explicit-LOD gather requires mip filtering None or Point with zero LOD biases");
+			}
+		}
 	}
-	if (capture_reads && !WrittenBuffersDisjoint(program, snapshot, reads)) return false;
 	snapshot.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
 	return BuildResourceSpecialization(program, snapshot, specialization);
 }
@@ -1065,19 +1085,20 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 	EXIT_IF(program.info.buffers.size() != specialization.buffers.size() ||
 	        program.info.images.size() > specialization.images.size());
 
-	auto buffers = program.info.buffers;
+	auto& buffers = program.info.buffers;
 	for (size_t index = 0; index < buffers.size(); index++) {
 		buffers[index].packed_stride      = specialization.buffers[index].packed_stride;
 		buffers[index].descriptor_format  = specialization.buffers[index].descriptor_format;
 		buffers[index].descriptor_swizzle = specialization.buffers[index].descriptor_swizzle;
 	}
-	auto images = program.info.images;
+	auto& images = program.info.images;
+	const auto original_image_count = images.size();
 	images.reserve(specialization.images.size());
 	for (uint32_t index = 0; index < specialization.images.size(); index++) {
 		const auto& source = specialization.images[index];
 		if (index >= images.size()) {
-			EXIT_IF(source.indirect_root >= program.info.images.size());
-			images.push_back(program.info.images[source.indirect_root]);
+			EXIT_IF(source.indirect_root >= original_image_count);
+			images.push_back(images[source.indirect_root]);
 		}
 		auto& image                      = images[index];
 		image.numeric_class              = source.numeric_class;
@@ -1100,15 +1121,17 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 	}
 
 	SamplerPlan sampler_plan;
-	EXIT_IF(!BuildSamplerPlan(program.info, images, sampler_plan));
-	auto samplers      = program.info.samplers;
-	auto sampled_pairs = program.info.sampled_pairs;
+	EXIT_IF(!BuildSamplerPlan(program.info, sampler_plan));
+	auto& samplers      = program.info.samplers;
+	auto& sampled_pairs = program.info.sampled_pairs;
+	const auto original_sampler_count = samplers.size();
 	samplers.reserve(sampler_plan.sampler_count);
 	for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) {
 		const auto& binding = sampler_plan.bindings[index];
-		if (index >= program.info.samplers.size()) {
-			samplers.push_back(program.info.samplers[binding.source]);
+		if (index >= original_sampler_count) {
+			samplers.push_back(samplers[binding.source]);
 		}
+		samplers[index].snapshot_index = binding.source;
 		samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger;
 		samplers[index].integer_border        = binding.type != SamplerClass::Float;
 	}
@@ -1119,11 +1142,15 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 		samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare;
 	}
 
-	auto memory_info = program.memory_info;
+	auto& memory_info = program.memory_info;
 	const ImageRemap image_remap(specialization);
 	for (auto* block: program.blocks) {
 		for (auto it = block->begin(); it != block->end(); ++it) {
 			auto& inst = *it;
+			if (inst.GetOpcode() == ValueOpcode::GetImageResource) {
+				inst.SetFlags(image_remap[inst.Flags<uint32_t>()]);
+				continue;
+			}
 			if (BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read) {
 				const auto& memory = memory_info[inst.Flags<MemoryFlags>().index];
 				if (memory.kind == ResourceKind::Buffer &&
@@ -1194,7 +1221,7 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 				continue;
 			}
 			if (image_opcode.needs_sampler &&
-			    memory.sampler < program.info.samplers.size()) {
+			    memory.sampler < original_sampler_count) {
 				const auto type = static_cast<uint32_t>(ClassifySampler(image));
 				memory.sampler = sampler_plan.mapping[memory.sampler][type];
 				EXIT_IF(memory.sampler == UINT32_MAX);
@@ -1203,13 +1230,6 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 			        inst.GetOpcode() != ValueOpcode::ImageSampleRaw);
 		}
 	}
-	for (auto* block: program.blocks) {
-		for (auto& inst: *block) {
-			if (inst.GetOpcode() == ValueOpcode::GetImageResource) {
-				inst.SetFlags(image_remap[inst.Flags<uint32_t>()]);
-			}
-		}
-	}
 	for (auto& memory: memory_info) {
 		if (memory.kind == ResourceKind::Image && !memory.planning_only) {
 			memory.resource = image_remap[memory.resource];
@@ -1232,11 +1252,6 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
 		}
 	}
 	image_remap.Apply(images);
-	program.info.buffers       = std::move(buffers);
-	program.info.images        = std::move(images);
-	program.info.samplers      = std::move(samplers);
-	program.info.sampled_pairs = std::move(sampled_pairs);
-	program.memory_info        = std::move(memory_info);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -44,6 +44,34 @@ uint32_t PossibleU32Bits(Value value) {
 	}
 }
 
+bool IsNoncanonicalFlatAddress(Value high, Value active) {
+	high = high.Resolve();
+	active = active.Resolve();
+	// A predicated VGPR write supplies its new value on exactly the lanes which
+	// execute this access. Do not discard a selection under a different mask.
+	while (const auto* select = high.TryInstruction()) {
+		if (select->GetOpcode() != ValueOpcode::SelectU32 || select->Arg(0).Resolve() != active) break;
+		high = select->Arg(1).Resolve();
+	}
+	uint32_t lower = 0, upper = UINT32_MAX;
+	if (high.IsImmediate() && high.GetType() == Type::U32) {
+		lower = upper = high.U32();
+	} else if (const auto* inst = high.TryInstruction();
+	           inst != nullptr && inst->GetOpcode() == ValueOpcode::UMedTri32) {
+		// The median lies between any two constant inputs, regardless of the third.
+		uint32_t count = 0;
+		for (size_t index = 0; index < inst->NumArgs(); ++index) {
+			const auto value = inst->Arg(index).Resolve();
+			if (!value.IsImmediate() || value.GetType() != Type::U32) continue;
+			if (count++ == 0) lower = upper = value.U32();
+			else { lower = std::min(lower, value.U32()); upper = std::max(upper, value.U32()); }
+		}
+		if (count < 2) return false;
+	}
+	// Leave one high DWORD on each side for the signed instruction offset/carry.
+	return lower > 0x00008000u && upper < 0xffff7fffu;
+}
+
 Value CanonicalizeSampleAdjustDword3(Value value) {
 	for (;;) {
 		value            = value.Resolve();
@@ -293,8 +321,9 @@ public:
 			for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
 				plan.handle->SetArg(dword, plan.key);
 			}
-			for (const auto index: plan.memory) {
-				m_program.memory_info[index].planning_only = true;
+			if (plan.reads[0] != nullptr) {
+				for (const auto index: plan.memory)
+					m_program.memory_info[index].planning_only = true;
 			}
 		}
 		m_program.descriptor_sources         = std::move(m_sources);
@@ -734,7 +763,7 @@ private:
 				const auto& b = *descriptor.indirect_image;
 				if (a.material_source != b.material_source || a.table_source != b.table_source ||
 				    a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset ||
-				    a.table_offset != b.table_offset ||
+				    a.table_offset != b.table_offset || a.sources != b.sources ||
 				    !EquivalentValue(m_program, a.key_count, b.key_count) ||
 				    a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() ||
 				    (!a.selector_mask.IsEmpty() &&
@@ -1446,6 +1475,152 @@ private:
 		return true;
 	}
 
+	uint32_t SelectorMaximum(Value value) const {
+		value = value.Resolve();
+		if (value.IsImmediate() && value.GetType() == Type::U32) return value.U32();
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return UINT32_MAX;
+		if (inst->GetOpcode() == ValueOpcode::FindILsb32)
+			return NonzeroOnEntry(inst->Arg(0), inst->Parent()) ? 31u : UINT32_MAX;
+		if (inst->GetOpcode() == ValueOpcode::UMin32)
+			return std::min(SelectorMaximum(inst->Arg(0)), SelectorMaximum(inst->Arg(1)));
+		uint32_t shift;
+		if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
+		    ImmediateU32(inst->Arg(1), shift) && shift < 32u) {
+			const auto maximum = uint64_t {SelectorMaximum(inst->Arg(0))} << shift;
+			if (maximum <= UINT32_MAX) return static_cast<uint32_t>(maximum);
+		}
+		return UINT32_MAX;
+	}
+
+	bool ImpossibleSwitchEdge(const Block* from, const Block* to) const {
+		const auto from_it = std::ranges::find(m_program.blocks, from);
+		const auto to_it = std::ranges::find(m_program.blocks, to);
+		if (from_it == m_program.blocks.end() || to_it == m_program.blocks.end() ||
+		    m_program.block_info.size() != m_program.blocks.size()) return false;
+		const auto& info = m_program.block_info[from_it - m_program.blocks.begin()];
+		const auto& term = info.terminator;
+		if (term.kind != CFG::TerminatorKind::IndirectBranch ||
+		    term.indirect_selector_code == UINT32_MAX ||
+		    term.indirect_selector_values.empty() ||
+		    term.indirect_selector_values.size() != term.indirect_selector_targets.size()) return false;
+		const auto target = m_program.block_info[to_it - m_program.blocks.begin()].id;
+		const auto maximum = SelectorMaximum(info.indirect_target);
+		bool found = false;
+		for (size_t i = 0; i < term.indirect_selector_values.size(); ++i) {
+			if (term.indirect_selector_targets[i] != target) continue;
+			if (term.indirect_selector_values[i] <= maximum) return false;
+			found = true;
+		}
+		return found;
+	}
+
+	bool TryMakeFiniteImage(Inst& handle, IndirectImagePlan& plan) {
+		if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u)
+			return false;
+		if (std::ranges::any_of(handle.Uses(), [](const Use& use) {
+			const auto op = use.user->GetOpcode();
+			return op != ValueOpcode::ImageSampleRaw && op != ValueOpcode::ImageGatherRaw;
+		})) return false;
+		bool has_phi = false;
+		for (size_t word = 0; word < handle.NumArgs(); ++word) has_phi |= handle.Arg(word).Resolve().IsPhi();
+		if (!has_phi) return false;
+		// Normalize the eight synchronized descriptor words to one GPU ordinal.
+		// Build the complete graph before changing IR so an unsupported leaf is transactional.
+		struct Choice {
+			DescriptorSource descriptor;
+			const Inst* branch = nullptr;
+			std::vector<uint32_t> children;
+			Value key;
+		};
+		std::vector<Choice> choices;
+		const auto visit = [&](const auto& self, DescriptorSource descriptor) -> uint32_t {
+			for (auto& word: descriptor.dwords) word = word.Resolve();
+			for (uint32_t i = 0; i < choices.size(); ++i)
+				if (choices[i].descriptor.dwords == descriptor.dwords) return i;
+			const auto index = static_cast<uint32_t>(choices.size());
+			choices.push_back({descriptor});
+			uint32_t bad = 0;
+			if (ValidateSource(descriptor, bad)) return index;
+			const auto* branch = descriptor.dwords[bad].TryInstruction();
+			if (branch == nullptr || branch->Parent() == nullptr ||
+			    branch->GetOpcode() != ValueOpcode::Phi ||
+			    branch->NumPhiBlocks() != branch->NumArgs()) return UINT32_MAX;
+			choices[index].branch = branch;
+			const auto count = branch->NumArgs();
+			for (uint32_t arm = 0; arm < count; ++arm) {
+				// A nonzero bit scan cannot choose its zero-input switch sentinel.
+				// Preserve the Phi edge; its ordinal is never observed at runtime.
+				if (ImpossibleSwitchEdge(branch->PhiBlock(arm), branch->Parent())) {
+					choices[index].children.push_back(UINT32_MAX);
+					continue;
+				}
+				auto child = descriptor;
+				for (uint32_t word = 0; word < child.dword_count; ++word) {
+					const auto* selected = descriptor.dwords[word].TryInstruction();
+					if (selected == nullptr || selected->GetOpcode() != branch->GetOpcode() ||
+					    selected->Parent() != branch->Parent()) {
+						if (!ValidateRuntimeValue(m_program, descriptor.dwords[word])) return UINT32_MAX;
+						continue;
+					}
+					if (selected->NumArgs() != count || selected->NumPhiBlocks() != count)
+						return UINT32_MAX;
+					uint32_t incoming = 0;
+					while (incoming < count && selected->PhiBlock(incoming) != branch->PhiBlock(arm))
+						++incoming;
+					if (incoming == count) return UINT32_MAX;
+					child.dwords[word] = selected->Arg(incoming);
+				}
+				const auto next = self(self, child);
+				if (next == UINT32_MAX) return UINT32_MAX;
+				choices[index].children.push_back(next);
+			}
+			return index;
+		};
+		DescriptorSource root;
+		root.dword_count = 8u;
+		for (uint32_t word = 0; word < root.dword_count; ++word) root.dwords[word] = handle.Arg(word);
+		if (visit(visit, root) == UINT32_MAX || choices.front().branch == nullptr) return false;
+		DescriptorSource image_source;
+		image_source.dword_count = 8u;
+		image_source.dwords.fill(Value(0u));
+		image_source.indirect_image.emplace(DescriptorSource::IndirectImage{});
+		auto& sources = image_source.indirect_image->sources;
+		for (auto& choice: choices) {
+			if (choice.branch != nullptr) continue;
+			const auto source = InternSource(choice.descriptor);
+			auto found = std::ranges::find(sources, source);
+			choice.key = Value(static_cast<uint32_t>(found - sources.begin()));
+			if (found == sources.end()) sources.push_back(source);
+		}
+		if (sources.empty()) return false;
+		for (auto& choice: choices) {
+			if (choice.branch == nullptr) continue;
+			auto* block = choice.branch->Parent();
+			const auto where = std::ranges::find_if(block->Instructions(), [&](const Inst& inst) {
+				return &inst == choice.branch;
+			});
+			auto& key = *block->PrependNewInst(where, choice.branch->GetOpcode());
+			key.SetFlags(Type::U32);
+			choice.key = Value(&key);
+		}
+		for (const auto& choice: choices) {
+			if (choice.branch == nullptr) continue;
+			auto* key = choice.key.Instruction();
+			for (uint32_t arm = 0; arm < choice.children.size(); ++arm) {
+				const auto child = choice.children[arm];
+				const auto value = child == UINT32_MAX ? Value(0u) : choices[child].key;
+				key->AddPhiOperand(choice.branch->PhiBlock(arm), value);
+			}
+		}
+		plan.handle = &handle;
+		plan.source = InternSource(image_source);
+		plan.key = choices.front().key;
+		plan.roots = image_source.dwords;
+		plan.reads.fill(nullptr);
+		return true;
+	}
+
 	const IndirectImagePlan* FindIndirectImage(const Inst& handle) const {
 		const auto found =
 		    std::find_if(m_indirect_images.begin(), m_indirect_images.end(),
@@ -1458,7 +1633,8 @@ private:
 	bool IsIndirectPlanningMemory(uint32_t index) const {
 		return std::any_of(m_indirect_images.begin(), m_indirect_images.end(),
 		                   [&](const IndirectImagePlan& plan) {
-			return std::ranges::find(plan.memory, index) != plan.memory.end();
+			return plan.reads[0] != nullptr &&
+			       std::ranges::find(plan.memory, index) != plan.memory.end();
 		});
 	}
 
@@ -1474,7 +1650,7 @@ private:
 					continue;
 				}
 				IndirectImagePlan plan;
-				if (TryMakeIndirectImage(*handle, plan)) {
+				if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan)) {
 					m_indirect_images.push_back(std::move(plan));
 				}
 			}
@@ -1549,15 +1725,20 @@ private:
 
 	uint32_t AddImage(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
 		const auto resource_class = ImageOpcodeInfoOf(op).resource_class;
-		const auto mip   = resource_class == ImageResourceClass::Storage && memory.image_has_mip
-		                       ? ImageMipMode::DynamicStorage
-		                       : ImageMipMode::None;
+		const bool atomic64 = ImageOpcodeInfoOf(op).access == ImageAccess::Atomic &&
+		                      memory.data_bits == 64u;
+		const bool dynamic_mip =
+		    (resource_class == ImageResourceClass::Storage && memory.image_has_mip) ||
+		    (op == ValueOpcode::ImageGatherRaw &&
+		     (memory.image_sample_flags & Decoder::ImageSampleFlagLod) != 0u);
+		const auto mip = dynamic_mip ? ImageMipMode::Dynamic : ImageMipMode::None;
 		const bool depth = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0;
 		for (uint32_t i = 0; i < m_info.images.size(); i++) {
 			auto& image = m_info.images[i];
 			if (image.source == source && image.resource_class == resource_class &&
 			    image.dimension == memory.image_dimension && image.mip_mode == mip &&
-			    image.depth_compare == depth && image.r128 == memory.image_r128) {
+			    image.depth_compare == depth && image.r128 == memory.image_r128 &&
+			    image.atomic64 == atomic64) {
 				Merge(image, op, pc);
 				return i;
 			}
@@ -1573,6 +1754,7 @@ private:
 		image.mip_mode       = mip;
 		image.depth_compare  = depth;
 		image.r128           = memory.image_r128;
+		image.atomic64       = atomic64;
 		Merge(image, op, pc);
 		m_info.images.push_back(image);
 		return static_cast<uint32_t>(m_info.images.size() - 1);
@@ -1649,6 +1831,10 @@ private:
 
 	void Collect(Inst& inst) {
 		const auto op           = inst.GetOpcode();
+		if (op == ValueOpcode::BvhIntersect) {
+			m_info.uses_dma = true;
+			return;
+		}
 		const auto buffer       = BufferAccessOf(op);
 		const auto address_info = AddressOpcodeInfoOf(op);
 		const auto image_info   = ImageOpcodeInfoOf(op);
@@ -1695,6 +1881,15 @@ private:
 			if (!IsAddressResourceKind(memory.kind)) {
 				Fail(flags.pc, "address operation has invalid resource kind");
 			}
+			if (memory.kind == ResourceKind::Flat && memory.address_is_full &&
+			    IsNoncanonicalFlatAddress(inst.Arg(2), inst.Arg(inst.NumArgs() - 1))) {
+				ValidateAddressHandle(inst.Arg(0), flags.pc);
+				if (memory.data_bits != 32u || m_program.scratch_dwords == 0) {
+					Fail(flags.pc, "local FLAT access requires DWORD data and per-thread scratch storage");
+				}
+				m_program.memory_info[flags.index].kind = ResourceKind::FlatLocal;
+				return;
+			}
 			if (memory.kind == ResourceKind::Scratch) {
 				handle = inst.Arg(0).Resolve().TryInstruction();
 				if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource ||
@@ -1746,6 +1941,9 @@ private:
 			if (sampler == UINT32_MAX) {
 				Fail(flags.pc, "sampler resource limit exceeded");
 			}
+			m_info.samplers[sampler].gather_lod |=
+			    op == ValueOpcode::ImageGatherRaw &&
+			    (memory.image_sample_flags & Decoder::ImageSampleFlagLod) != 0u;
 			AddHandlePatch(sampler_handle, sampler, flags.pc);
 			AddSampledPair(resource, sampler, flags.pc);
 		}

src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp

@@ -326,6 +326,8 @@ void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
 }
 
 void CollectOutputs(const Program& program, ShaderStageInputInfo input_info, ShaderInfo& info) {
+	const bool alpha_remap = program.stage == ShaderType::Pixel && input_info.pixel != nullptr &&
+	                         input_info.pixel->alpha_blend_source_remap;
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			if (inst.GetOpcode() != ValueOpcode::SetAttribute) {
@@ -385,8 +387,14 @@ void CollectOutputs(const Program& program, ShaderStageInputInfo input_info, Sha
 					          export_info.index, fmt::format("out_param_{}", export_info.index));
 					break;
 				case ExportTargetKind::Mrt:
+					if (alpha_remap && export_info.index != 0) {
+						break;
+					}
 					AddOutput(info, StageOutputKind::Mrt, export_info.index, export_info.index,
 					          fmt::format("out_mrt_{}", export_info.index));
+					if (alpha_remap) {
+						AddOutput(info, StageOutputKind::Mrt, 1, 1, "out_mrt_1");
+					}
 					break;
 				default: break;
 			}
@@ -404,7 +412,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
 	ValidateOptions(program, input_info);
 	ValidateValueReferences(program, input_info);
 
-	auto next = program.info;
+	auto& next = program.info;
 	next.inputs.clear();
 	next.outputs.clear();
 	next.has_bitwise_xor =
@@ -425,7 +433,6 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
 	}
 	CollectBuiltinInputs(program, next);
 	CollectOutputs(program, input_info, next);
-	program.info                 = std::move(next);
 	program.shader_info_complete = true;
 }
 

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -795,18 +795,26 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
-		case ValueOpcode::LogicalAnd:
-			if (binary()) {
-				result = (a != 0u) && (b != 0u);
+		case ValueOpcode::LogicalAnd: {
+			const bool left = Arg(inst, 0, a);
+			if (left && a == 0u) {
+				result = 0u;
 				return true;
 			}
-			return false;
-		case ValueOpcode::LogicalOr:
-			if (binary()) {
-				result = (a != 0u) || (b != 0u);
+			if (!Arg(inst, 1, b) || (b != 0u && !left)) return false;
+			result = b != 0u;
+			return true;
+		}
+		case ValueOpcode::LogicalOr: {
+			const bool left = Arg(inst, 0, a);
+			if (left && a != 0u) {
+				result = 1u;
 				return true;
 			}
-			return false;
+			if (!Arg(inst, 1, b) || (b == 0u && !left)) return false;
+			result = b != 0u;
+			return true;
+		}
 		case ValueOpcode::LogicalXor:
 			if (binary()) {
 				result = (a != 0u) != (b != 0u);
@@ -844,16 +852,31 @@ bool SrtWalker::EvaluateDescriptor(uint32_t source, DescriptorValue& result) {
 	return true;
 }
 
-std::span<const uint8_t> SrtWalker::FindActiveSources() {
-	if (m_program.control_flow.empty()) {
-		return {};
-	}
+bool SrtWalker::RefreshFlatBuffer(std::vector<uint32_t>& flat) {
+	if (!m_program.srt_plan_complete) return false;
+	const auto refresh = [&](uint32_t slot) {
+		if (slot >= m_program.srt_reads.size()) return false;
+		const auto& read = m_program.srt_reads[slot];
+		const bool clean = read.flat_offset < m_clean_flat_slots.size() &&
+		                   m_clean_flat_slots[read.flat_offset] != 0u;
+		if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr))
+			return false;
+		auto& evaluator = clean ? *m_clean_evaluator : *this;
+		return read.flat_offset < flat.size() && evaluator.Evaluate(read.value, flat[read.flat_offset]);
+	};
 	auto& active = m_program.active_sources;
+	if (m_program.control_flow.empty()) {
+		active.clear();
+		flat.resize(m_program.srt_reads.size());
+		for (uint32_t slot = 0; slot < m_program.srt_reads.size(); ++slot) {
+			if (!refresh(slot)) return false;
+		}
+		return true;
+	}
+	flat.assign(m_program.srt_reads.size(), 0u);
 	active.assign(m_program.descriptor_sources.size(), 1u);
 	for (const auto& block: m_program.control_flow) {
-		for (const auto source: block.sources) {
-			active.at(source) = 0u;
-		}
+		for (const auto source: block.sources) active.at(source) = 0u;
 	}
 	auto& visited = m_program.visited_blocks;
 	auto& pending = m_program.pending_blocks;
@@ -863,41 +886,22 @@ std::span<const uint8_t> SrtWalker::FindActiveSources() {
 	while (!pending.empty()) {
 		const auto index = pending.back();
 		pending.pop_back();
-		if (visited.at(index)) {
-			continue;
-		}
+		if (visited.at(index)) continue;
 		visited[index] = 1u;
 		const auto& block = m_program.control_flow[index];
-		for (const auto source: block.sources) {
-			active[source] = 1u;
+		for (const auto source: block.sources) active[source] = 1u;
+		for (const auto slot: block.srt_reads) {
+			if (!refresh(slot)) return false;
 		}
 		uint32_t condition = 0;
+		auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this;
 		if (!block.condition.IsEmpty() && m_runtime.read_specialization_memory != nullptr &&
-		    Evaluate(block.condition, condition)) {
+		    predicate.Evaluate(block.condition, condition)) {
 			pending.push_back(block.successors[condition != 0u ? 0u : 1u]);
 		} else {
 			pending.insert(pending.end(), block.successors.begin(), block.successors.end());
 		}
 	}
-	return active;
-}
-
-bool SrtWalker::RefreshFlatBuffer(std::vector<uint32_t>& flat) {
-	if (!m_program.srt_plan_complete) {
-		return false;
-	}
-	flat.resize(m_program.srt_reads.size());
-	for (const auto& read: m_program.srt_reads) {
-		const bool clean = read.flat_offset < m_clean_flat_slots.size() &&
-		                   m_clean_flat_slots[read.flat_offset] != 0u;
-		if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr)) {
-			return false;
-		}
-		auto& evaluator = clean ? *m_clean_evaluator : *this;
-		if (read.flat_offset >= flat.size() || !evaluator.Evaluate(read.value, flat[read.flat_offset])) {
-			return false;
-		}
-	}
 	return true;
 }
 

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -38,8 +38,7 @@ public:
 
 	bool Evaluate(Value value, uint32_t& result);
 	bool EvaluateDescriptor(uint32_t source, DescriptorValue& result);
-	// An empty span means that all sources are active.
-	std::span<const uint8_t> FindActiveSources();
+	// Refreshes reachable scalar reads and active descriptor sources in one walk.
 	bool RefreshFlatBuffer(std::vector<uint32_t>& flat);
 
 private:

src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp

@@ -165,7 +165,6 @@ Value InitialValue(M0Tag) {
 
 enum class ReadStep { Start, SetValue, PushPhiArgument };
 
-template <typename T>
 struct ReadState {
 	Block*   block = nullptr;
 	Value    result;
@@ -183,8 +182,8 @@ public:
 
 	template <typename T>
 	Value Read(T variable, Block* root) {
-		std::vector<ReadState<T>> stack {{}, {.block = root}};
-		const auto                prepare_phi = [&]() {
+		stack.assign({{}, {.block = root}});
+		const auto prepare_phi = [&]() {
 			auto&      state        = stack.back();
 			const auto predecessors = state.block->ImmPredecessors();
 			if (state.pred == predecessors.size()) {
@@ -290,6 +289,7 @@ private:
 
 	std::unordered_map<Block*, std::map<Variable, Inst*>> incomplete_phis;
 	DefTable                                              definitions;
+	std::vector<ReadState>                               stack;
 };
 
 void VisitInstruction(Pass& pass, Block* block, Inst& inst) {

src/graphics/shader/shader.cpp

@@ -41,44 +41,33 @@
 
 namespace Libs::Graphics {
 
-struct ShaderBinaryInfo {
-	uint8_t  signature[7];
-	uint8_t  version;
-	uint32_t pssl_or_cg  : 1;
-	uint32_t cached      : 1;
-	uint32_t type        : 4;
-	uint32_t source_type : 2;
-	uint32_t length      : 24;
-	uint8_t  chunk_usage_base_offset_dw;
-	uint8_t  num_input_usage_slots;
-	uint8_t  is_srt                 : 1;
-	uint8_t  is_srt_used_info_valid : 1;
-	uint8_t  is_extended_usage_info : 1;
-	uint8_t  reserved2              : 5;
-	uint8_t  reserved3;
-	uint32_t hash0;
-	uint32_t hash1;
-	uint32_t crc32;
+struct ShaderMapEntry {
+	ShaderMappedData data;
+	uint64_t hash;
 };
 
-static std::unique_ptr<std::unordered_map<uint64_t, ShaderMappedData>> g_shader_map;
+static std::unique_ptr<std::unordered_map<uint64_t, ShaderMapEntry>> g_shader_map;
 static std::mutex                                                      g_shader_map_mutex;
 
 void ShaderInit() {
 	EXIT_IF(g_shader_map != nullptr);
 
-	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMappedData>>();
+	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMapEntry>>();
 }
 
 void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 	EXIT_IF(g_shader_map == nullptr);
 
+	if (addr == 0 || data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
+		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n",
+		     addr, data.code_size_bytes);
+	}
+	const auto hash = XXH3_64bits(reinterpret_cast<const void*>(addr), data.code_size_bytes);
 	std::scoped_lock lock(g_shader_map_mutex);
-
-	(*g_shader_map)[addr] = data;
+	(*g_shader_map)[addr] = {data, hash};
 }
 
-static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
+static ShaderMapEntry ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT_IF(g_shader_map == nullptr);
 
 	std::scoped_lock lock(g_shader_map_mutex);
@@ -90,37 +79,15 @@ static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT("%s shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, addr);
 }
 
-static const ShaderBinaryInfo* GetBinaryInfo(const uint32_t* code) {
-	EXIT_IF(code == nullptr);
-
-	if (code[0] == 0xBEEB03FF) {
-		return reinterpret_cast<const ShaderBinaryInfo*>(code +
-		                                                 static_cast<size_t>(code[1] + 1) * 2);
-	}
-
-	return nullptr;
-}
-
-static uint64_t GetDeclaredShaderHash(uint64_t shader_addr) {
-	const auto* header = GetBinaryInfo(reinterpret_cast<const uint32_t*>(shader_addr));
-	return header != nullptr ? (static_cast<uint64_t>(header->hash1) << 32u) | header->hash0 : 0;
-}
-
-static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash,
+static ShaderParams GetShaderParams(uint64_t shader_addr, uint64_t hash,
 	                                std::span<const uint32_t> user_data,
 	                                const ShaderMappedData& data, uint32_t user_data_base = 0) {
-	if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
-		EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64
-		     " has invalid AGC shader_size=0x%08" PRIx32 "\n",
-		     label, declared_hash, shader_addr, data.code_size_bytes);
-	}
 	const auto code_words = data.code_size_bytes / sizeof(uint32_t);
 	const auto code = std::span {reinterpret_cast<const uint32_t*>(shader_addr), code_words};
 	ShaderParams params {
 	    .code            = code,
 	    .user_data_count = static_cast<uint32_t>(user_data.size()) + user_data_base,
-	    .hash            = declared_hash != 0 ? declared_hash
-	                                          : XXH3_64bits(code.data(), code.size_bytes()),
+	    .hash            = hash,
 	};
 	EXIT_IF(user_data.size() > HW::UserSgprInfo::SGPRS_MAX ||
 	        params.user_data_count > params.user_data.size());
@@ -414,6 +381,8 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 		r.fields[1]       = sharp[1];
 		r.fields[2]       = sharp[2];
 		r.fields[3]       = sharp[3];
+		EXIT_NOT_IMPLEMENTED(r.AddTid());
+		EXIT_NOT_IMPLEMENTED(r.SwizzleEnabled());
 		if (format != Prospero::VertexAttribFormat::kInvalid) {
 			const auto                   format_raw    = static_cast<uint32_t>(format);
 			const auto                   buffer_format = format_raw >> 2u;
@@ -650,7 +619,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 		key.insert(key.end(), {mesh.wave_size, mesh.host_subgroup_size, mesh.lds_size_dwords,
 		                       mesh.scratch_size_dwords, mesh.input_primitive,
 		                       mesh.primitives_per_group, mesh.vertices_per_group,
-		                       mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex});
+		                       mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex,
+		                       static_cast<uint32_t>(mesh.fast_launch)});
 	}
 	key.push_back(info.tess.input_control_points);
 	if (info.tess.input_control_points != 0) {
@@ -662,12 +632,9 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 	for (int i = 0; i < info.resources_num; i++) {
 		const auto& resource    = info.resources[i];
 		const auto& destination = info.resources_dst[i];
-		key.push_back(destination.register_start);
 		key.push_back(destination.registers_num);
-		key.push_back(destination.fetch_index);
 		key.push_back(static_cast<uint32_t>(destination.attr_id));
-		key.push_back(resource.fields[1] & 0xbfff0000u); // Stride and swizzle enable.
-		key.push_back(resource.fields[3] & 0x3087ffffu); // Channels, format, OOB, and add TID.
+		key.push_back(resource.fields[3] & 0x7ffffu); // Channels and format used by embedded fetch.
 	}
 }
 
@@ -693,6 +660,7 @@ void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector<uint32_t>
 	key.push_back(static_cast<uint32_t>(info.ps_sample_mask_export_enable));
 	key.push_back(static_cast<uint32_t>(info.ps_early_z));
 	key.push_back(static_cast<uint32_t>(info.dual_source_blending));
+	key.push_back(static_cast<uint32_t>(info.alpha_blend_source_remap));
 	key.insert(key.end(), std::begin(info.target_output_mode), std::end(info.target_output_mode));
 	for (uint32_t base = 0; base < info.target_export_mapping.size(); base += 4u) {
 		uint32_t packed = 0;
@@ -725,11 +693,10 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector<uint32_
 ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             const HW::UserConfig& user_config, ShaderVertexInputInfo& info) {
 	const auto& sh     = context.GetShaderRegisters();
-	const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
 	const bool merged = (context.GetShaderStages() & 0x20u) != 0;
 	auto        params = GetShaderParams(
-	    regs.es_regs.data_addr, "ShaderRecompiler VS",
-	    GetDeclaredShaderHash(regs.es_regs.data_addr),
+	    regs.es_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data,
 	    merged ? 8u : 0u);
 	if (!merged) {
@@ -752,12 +719,14 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 	mesh.provoking_vertex    = context.GetModeControl().provoking_vtx_last ? 2u : 0u;
 	mesh.lds_size_dwords     = static_cast<uint32_t>(regs.gs_regs.rsrc2.lds_size) * 128u;
 	mesh.scratch_size_dwords = data.scratch_size_dwords;
+	const auto fast_launch = (context.GetShaderStages() >> 19u) & 3u;
+	EXIT_NOT_IMPLEMENTED(fast_launch > 1u);
+	mesh.fast_launch = fast_launch != 0;
 	if (data.type == Prospero::ShaderBinaryType::kGsFront) {
 		EXIT_IF(regs.gs_regs.data_addr == 0);
-		const auto back = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
+		const auto [back, back_hash] = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
 		const auto back_params =
-		    GetShaderParams(regs.gs_regs.data_addr, "ShaderRecompiler GS",
-		                    GetDeclaredShaderHash(regs.gs_regs.data_addr), {}, back);
+		    GetShaderParams(regs.gs_regs.data_addr, back_hash, {}, back);
 		params.back_code = back_params.code;
 		params.user_data[0] = static_cast<uint32_t>(regs.gs_regs.user_data_addr);
 		params.user_data[1] = static_cast<uint32_t>(regs.gs_regs.user_data_addr >> 32u);
@@ -765,9 +734,20 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 		params.hash = XXH3_64bits(hashes, sizeof(hashes));
 		mesh.scratch_size_dwords = std::max(mesh.scratch_size_dwords, back.scratch_size_dwords);
 	}
-	EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
-	                     regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
 	const auto& group = user_config.GetGeControl();
+	if (mesh.fast_launch) {
+		const auto& user_vgpr = user_config.GetGeUserVgprEn();
+		EXIT_NOT_IMPLEMENTED(data.type != Prospero::ShaderBinaryType::kGs ||
+		                     regs.gs_regs.rsrc1.gs_vgpr_component_count != 0u ||
+		                     regs.gs_regs.rsrc2.es_vgpr_component_count != 1u ||
+		                     user_config.GetPrimType() != Prospero::PrimitiveType::kPointList ||
+		                     group.primitive_group_size != 1u || group.vertex_group_size != 1u ||
+		                     user_vgpr.vgpr1 || user_vgpr.vgpr2 || user_vgpr.vgpr3 ||
+		                     mesh.max_vertices != sh.m_vgtGsMaxVertOut);
+	} else {
+		EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
+		                     regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
+	}
 	if ((user_config.GetPrimType() != Prospero::PrimitiveType::kPointList &&
 	     user_config.GetPrimType() != Prospero::PrimitiveType::kLineList &&
 	     user_config.GetPrimType() != Prospero::PrimitiveType::kTriFan &&
@@ -780,7 +760,8 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 		     mesh.input_primitive, sh.m_vgtGsOutPrimType, sh.m_vgtGsMaxVertOut,
 		     group.primitive_group_size, group.vertex_group_size, mesh.max_vertices);
 	}
-	mesh.max_primitives       = group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
+	mesh.max_primitives = mesh.fast_launch ? mesh.max_vertices :
+	                      group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
 	mesh.primitives_per_group = std::min({static_cast<uint32_t>(group.primitive_group_size),
 	                                      mesh.InputPrimitiveCount(group.vertex_group_size),
 	                                      mesh.max_vertices / sh.m_vgtGsMaxVertOut});
@@ -796,9 +777,9 @@ std::array<ShaderParams, 3>
 PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             std::array<ShaderVertexInputInfo, 3>& input_info) {
 	const auto& sh        = context.GetShaderRegisters();
-	const auto  local     = ShaderGetMappedData(regs.ls_regs.data_addr, "ShaderGetInputInfoLS():");
-	const auto  control   = ShaderGetMappedData(regs.hs_regs.data_addr, "ShaderGetInputInfoHS():");
-	const auto evaluation = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoTES():");
+	const auto [local, local_hash] = ShaderGetMappedData(regs.ls_regs.data_addr, "ShaderGetInputInfoLS():");
+	const auto [control, control_hash] = ShaderGetMappedData(regs.hs_regs.data_addr, "ShaderGetInputInfoHS():");
+	const auto [evaluation, evaluation_hash] = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoTES():");
 	EXIT_NOT_IMPLEMENTED(local.type != Prospero::ShaderBinaryType::kHsFront ||
 	                     control.type != Prospero::ShaderBinaryType::kHsBack ||
 	                     evaluation.type != Prospero::ShaderBinaryType::kGs);
@@ -809,12 +790,9 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
 	const auto local_users      = std::span(regs.hs_user_sgpr.value, regs.hs_regs.rsrc2.user_sgpr);
 	const auto evaluation_users = std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr);
 	std::array<ShaderParams, 3> params {
-	    GetShaderParams(regs.ls_regs.data_addr, "ShaderRecompiler LS",
-	                    GetDeclaredShaderHash(regs.ls_regs.data_addr), local_users, local),
-	    GetShaderParams(regs.hs_regs.data_addr, "ShaderRecompiler HS",
-	                    GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control, 8u),
-	    GetShaderParams(regs.es_regs.data_addr, "ShaderRecompiler TES",
-	                    GetDeclaredShaderHash(regs.es_regs.data_addr), evaluation_users,
+	    GetShaderParams(regs.ls_regs.data_addr, local_hash, local_users, local),
+	    GetShaderParams(regs.hs_regs.data_addr, control_hash, local_users, control, 8u),
+	    GetShaderParams(regs.es_regs.data_addr, evaluation_hash, evaluation_users,
 	                    evaluation),
 	};
 	// The fused HS back half receives its separate user-data address in s0:s1.
@@ -854,19 +832,19 @@ ShaderParams PrepareProgram(
     const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
     std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
     ShaderPixelInputInfo&                               ps_info) {
-	const auto data = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():");
 	ShaderGetStaticInputInfoPS(regs, sh, target_export_mapping, data, ps_info);
 	return GetShaderParams(
-	    regs.ps_regs.data_addr, "ShaderRecompiler PS", GetDeclaredShaderHash(regs.ps_regs.data_addr),
+	    regs.ps_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), data);
 }
 
 ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
                             ShaderComputeInputInfo& info) {
-	const auto data = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
 	ShaderGetStaticInputInfoCS(regs, sh, data, info);
 	return GetShaderParams(
-	    regs.cs_regs.data_addr, "ShaderRecompiler CS", GetDeclaredShaderHash(regs.cs_regs.data_addr),
+	    regs.cs_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), data);
 }
 

src/graphics/shader/shader.h

@@ -88,6 +88,7 @@ struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo {
 	uint32_t max_vertices         = 0;
 	uint32_t max_primitives       = 0;
 	uint32_t provoking_vertex     = 0;
+	bool     fast_launch          = false;
 
 	[[nodiscard]] constexpr uint32_t InputPrimitiveSize() const {
 		switch (static_cast<Prospero::PrimitiveType>(input_primitive)) {
@@ -178,6 +179,8 @@ struct ShaderPixelInputInfo {
 	bool                                           ps_sample_mask_export_enable = false;
 	bool                                           ps_sample_shading            = false;
 	bool                                           dual_source_blending         = false;
+	// Export logical alpha through MRT1 for blending after channel swizzling.
+	bool                                           alpha_blend_source_remap     = false;
 	bool                                           ps_early_z                   = false;
 	bool                                           ps_execute_on_noop           = false;
 	ShaderStageRuntime                             stage;

src/kernel/eventQueue.cpp

@@ -138,7 +138,13 @@ int KernelEqueuePrivate::GetTriggeredEvents(KernelEvent* ev, int num) {
 
 void KernelEqueuePrivate::TriggerExpiredTimers(uint64_t now_ns) {
 	for (auto& event: m_events) {
-		if (!event.triggered && event.deadline_ns != 0 && event.deadline_ns <= now_ns) {
+		if (event.deadline_ns != 0 && event.deadline_ns <= now_ns) {
+			if (event.event.filter == KERNEL_EVFILT_TIMER) {
+				const auto count = event.interval_ns == 0 ? !event.triggered :
+				    1 + (now_ns - event.deadline_ns) / event.interval_ns;
+				event.event.data += static_cast<intptr_t>(count);
+				event.deadline_ns += count * event.interval_ns;
+			}
 			event.triggered = true;
 		}
 	}
@@ -209,7 +215,9 @@ int KernelEqueuePrivate::AddEvent(const KernelEqueueEvent& event) {
 		                       return e.event.ident == ident && e.event.filter == filter;
 	                       });
 	if (it != m_events.end()) {
+		TriggerExpiredTimers(MonotonicTimeNs());
 		it->deadline_ns = event.deadline_ns;
+		it->interval_ns = event.interval_ns;
 		it->event.udata = event.event.udata;
 		for (auto& pending: it->pending_events) {
 			pending.udata = event.event.udata;
@@ -511,6 +519,22 @@ int KYTY_SYSV_ABI KernelDeleteUserEvent(KernelEqueue eq, int id) {
 	return KernelDeleteEvent(eq, static_cast<uintptr_t>(id), KERNEL_EVFILT_USER);
 }
 
+static int AddTimerEvent(KernelEqueue eq, int id, uint64_t delay_ns, bool periodic, void* udata) {
+	const auto now_ns = MonotonicTimeNs();
+	KernelEqueueEvent event {};
+	event.deadline_ns  = delay_ns <= UINT64_MAX - now_ns ? now_ns + delay_ns : UINT64_MAX;
+	event.interval_ns  = periodic ? delay_ns : 0;
+	event.event.ident  = static_cast<uintptr_t>(id);
+	event.event.filter = periodic ? KERNEL_EVFILT_TIMER : KERNEL_EVFILT_HRTIMER;
+	event.event.flags  = EV_ADD | (periodic ? EV_CLEAR : EV_ONESHOT);
+	event.event.udata  = udata;
+	return KernelAddEvent(eq, event);
+}
+
+int KYTY_SYSV_ABI KernelAddTimerEvent(KernelEqueue eq, int id, KernelUseconds usec, void* udata) {
+	return AddTimerEvent(eq, id, static_cast<uint64_t>(usec) * 1000, true, udata);
+}
+
 int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTimespec* ts,
                                         void* udata) {
 	if (ts == nullptr) {
@@ -524,17 +548,7 @@ int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTim
 
 	const auto delay_ns =
 	    static_cast<uint64_t>(ts->tv_sec) * 1000000000ull + static_cast<uint64_t>(ts->tv_nsec);
-	const auto now_ns = MonotonicTimeNs();
-
-	KernelEqueueEvent event {};
-	event.deadline_ns  = delay_ns <= UINT64_MAX - now_ns ? now_ns + delay_ns : UINT64_MAX;
-	event.event.ident  = static_cast<uintptr_t>(id);
-	event.event.filter = KERNEL_EVFILT_HRTIMER;
-	event.event.flags  = EV_ADD | EV_ONESHOT;
-	event.event.fflags = 0;
-	event.event.data   = 0;
-	event.event.udata  = udata;
-	return KernelAddEvent(eq, event);
+	return AddTimerEvent(eq, id, delay_ns, false, udata);
 }
 
 int KYTY_SYSV_ABI KernelDeleteHRTimerEvent(KernelEqueue eq, int id) {

src/kernel/eventQueue.h

@@ -51,6 +51,7 @@ struct KernelFilter {
 struct KernelEqueueEvent {
 	bool                    triggered   = false;
 	uint64_t                deadline_ns = 0;
+	uint64_t                interval_ns = 0;
 	KernelEvent             event;
 	KernelFilter            filter;
 	std::deque<KernelEvent> pending_events;
@@ -72,6 +73,8 @@ int KYTY_SYSV_ABI KernelAddUserEventEdge(KernelEqueue eq, int id);
 int KYTY_SYSV_ABI KernelTriggerUserEvent(KernelEqueue eq, int id, void* udata);
 int KYTY_SYSV_ABI KernelTriggerUserEventForAll(int id, void* udata);
 int KYTY_SYSV_ABI KernelDeleteUserEvent(KernelEqueue eq, int id);
+int KYTY_SYSV_ABI KernelAddTimerEvent(KernelEqueue eq, int id, KernelUseconds usec,
+                                      void* udata);
 int KYTY_SYSV_ABI KernelAddHRTimerEvent(KernelEqueue eq, int id, const KernelTimespec* ts,
                                         void* udata);
 int KYTY_SYSV_ABI KernelDeleteHRTimerEvent(KernelEqueue eq, int id);

src/kernel/memory.cpp

@@ -963,18 +963,7 @@ static bool IsInPrtAperture(uint64_t address, uint64_t size = 1) {
 	return false;
 }
 
-bool TryReadPrtBacking(uint64_t vaddr, void* data, uint64_t size) {
-	std::vector<VirtualRanges::Range> ranges;
-	if (g_guest_address_space == nullptr || g_virtual_ranges == nullptr ||
-	    !IsInPrtAperture(vaddr, size) || !g_virtual_ranges->QuerySpan(vaddr, size, &ranges)) {
-		return false;
-	}
-	if (std::any_of(ranges.begin(), ranges.end(), [](const auto& range) {
-		    return !IsReservedRangeType(range.type) &&
-		           !g_guest_address_space->BackingContains(range.start, range.size);
-	    })) {
-		return false;
-	}
+bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) {
 	return g_guest_address_space->TryReadSparseBacking(vaddr, data, size);
 }
 

src/kernel/memory.h

@@ -115,7 +115,7 @@ int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
 bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
 bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
 bool                   TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size);
-bool                   TryReadPrtBacking(uint64_t vaddr, void* data, uint64_t size);
+bool                   TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size);
 [[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size);
 void                   WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
 void                   InvalidateMemory(uint64_t vaddr, uint64_t size);

src/kernel/memoryAddressSpace.inc

@@ -212,14 +212,21 @@ public:
 		}
 
 		auto* destination = static_cast<uint8_t*>(data);
-		std::memset(destination, 0, size);
+		auto  cursor      = vaddr;
 		for (auto mapping = first; mapping != m_maps.end() && mapping->second.vaddr < end;
 		     ++mapping) {
 			const auto& entry  = mapping->second;
 			const auto  start  = std::max(vaddr, entry.vaddr);
 			const auto  bytes  = std::min(end, entry.vaddr + entry.size) - start;
 			const auto  offset = entry.backing_offset + start - entry.vaddr;
-			std::memcpy(destination + start - vaddr, m_backing_base + offset, bytes);
+			if (cursor < start) {
+				std::memset(destination + (cursor - vaddr), 0, start - cursor);
+			}
+			std::memcpy(destination + (start - vaddr), m_backing_base + offset, bytes);
+			cursor = start + bytes;
+		}
+		if (cursor < end) {
+			std::memset(destination + (cursor - vaddr), 0, end - cursor);
 		}
 		return true;
 	}
@@ -731,6 +738,20 @@ public:
 	}
 
 	bool TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size) {
+		if (!Owns(vaddr, size)) {
+			return false;
+		}
+		std::lock_guard lock(m_mutex);
+		auto            mapping = m_mapped.upper_bound(vaddr);
+		if (mapping != m_mapped.begin()) {
+			--mapping;
+		}
+		for (; mapping != m_mapped.end() && mapping->second.vaddr < vaddr + size; ++mapping) {
+			const auto& region = mapping->second;
+			if (region.type == MappingType::Private && region.vaddr + region.size > vaddr) {
+				return false;
+			}
+		}
 		return m_backing->TryReadSparseBacking(vaddr, data, size);
 	}
 

src/kernel/pthread.cpp

@@ -4208,6 +4208,10 @@ int KYTY_SYSV_ABI pthread_attr_setschedpolicy(LibKernel::PthreadAttr* attr, int
 	return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetschedpolicy(attr, policy));
 }
 
+int KYTY_SYSV_ABI pthread_attr_setstack(LibKernel::PthreadAttr* attr, void* addr, size_t size) {
+	return POSIX_PTHREAD_CALL(LibKernel::PthreadAttrSetstack(attr, addr, size));
+}
+
 int KYTY_SYSV_ABI pthread_attr_setstacksize(LibKernel::PthreadAttr* attr, size_t stack_size) {
 	// PRINT_NAME();
 

src/kernel/pthread.h

@@ -249,6 +249,7 @@ int KYTY_SYSV_ABI pthread_attr_setinheritsched(LibKernel::PthreadAttr* attr, int
 int KYTY_SYSV_ABI pthread_attr_setschedparam(LibKernel::PthreadAttr*            attr,
                                              const LibKernel::KernelSchedParam* param);
 int KYTY_SYSV_ABI pthread_attr_setschedpolicy(LibKernel::PthreadAttr* attr, int policy);
+int KYTY_SYSV_ABI pthread_attr_setstack(LibKernel::PthreadAttr* attr, void* addr, size_t size);
 int KYTY_SYSV_ABI pthread_attr_setstacksize(LibKernel::PthreadAttr* attr, size_t stack_size);
 int KYTY_SYSV_ABI pthread_cond_broadcast(LibKernel::PthreadCond* cond);
 int KYTY_SYSV_ABI pthread_cond_signal(LibKernel::PthreadCond* cond);

src/launcher/forms/configuration_edit_dialog.ui

@@ -23,7 +23,7 @@
    <property name="verticalSpacing">
     <number>8</number>
    </property>
-   <item row="0" column="0">
+   <item row="0" column="0" colspan="2">
     <layout class="QGridLayout" name="sectionsLayout" columnstretch="1,1">
      <property name="horizontalSpacing">
       <number>8</number>
@@ -173,6 +173,12 @@
           <property name="text"><string>Enable tessellation support (experimental)</string></property>
          </widget>
         </item>
+        <item row="8" column="1">
+         <widget class="QCheckBox" name="checkBox_hide_cursor">
+          <property name="toolTip"><string>Hide the host cursor after 2 seconds without mouse activity</string></property>
+          <property name="text"><string>Auto-hide cursor</string></property>
+         </widget>
+        </item>
        </layout>
       </widget>
      </item>
@@ -295,7 +301,82 @@
      </item>
     </layout>
    </item>
-   <item row="2" column="0">
+   <item row="1" column="1">
+    <widget class="QGroupBox" name="controller_group">
+     <property name="title"><string>Controller</string></property>
+     <layout class="QFormLayout" name="controllerLayout">
+      <property name="fieldGrowthPolicy"><enum>QFormLayout::AllNonFixedFieldsGrow</enum></property>
+      <item row="0" column="0">
+       <widget class="QLabel" name="label_controller_color">
+        <property name="text"><string>DualSense lightbar:</string></property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <layout class="QHBoxLayout" name="controllerColorLayout">
+        <item>
+         <widget class="QPushButton" name="button_controller_color">
+          <property name="toolTip"><string>Choose and preview a fixed LED color. Bluetooth DualSense may take several seconds to update.</string></property>
+          <property name="text"><string>Custom</string></property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QPushButton" name="button_controller_color_reset">
+          <property name="toolTip"><string>Let the game control the lightbar</string></property>
+          <property name="text"><string>Default</string></property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_controller_vibration">
+        <property name="text"><string>Vibration intensity:</string></property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <layout class="QHBoxLayout" name="controllerVibrationLayout">
+        <item>
+         <widget class="QSlider" name="slider_controller_vibration">
+          <property name="toolTip"><string>Scale DualSense vibration over USB and Bluetooth</string></property>
+          <property name="maximum"><number>100</number></property>
+          <property name="value"><number>100</number></property>
+          <property name="orientation"><enum>Qt::Horizontal</enum></property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="label_controller_vibration_value">
+          <property name="minimumSize"><size><width>40</width><height>0</height></size></property>
+          <property name="text"><string>100%</string></property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="label_controller_volume">
+        <property name="text"><string>Speaker volume:</string></property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <layout class="QHBoxLayout" name="controllerVolumeLayout">
+        <item>
+         <widget class="QSlider" name="slider_controller_volume">
+          <property name="toolTip"><string>Scale the DualSense controller speaker over USB and Bluetooth</string></property>
+          <property name="maximum"><number>100</number></property>
+          <property name="value"><number>100</number></property>
+          <property name="orientation"><enum>Qt::Horizontal</enum></property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="label_controller_volume_value">
+          <property name="minimumSize"><size><width>40</width><height>0</height></size></property>
+          <property name="text"><string>100%</string></property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="2" column="0" colspan="2">
     <layout class="QHBoxLayout" name="buttonLayout">
      <item>
      <widget class="QPushButton" name="clear_button">

src/launcher/include/configuration.h

@@ -5,6 +5,7 @@
 
 #include <QByteArray>
 #include <QChar>
+#include <QColor>
 #include <QMetaEnum>
 #include <QMetaType>
 #include <QObject>
@@ -45,6 +46,30 @@ QString EnumToText(T value) {
 	                                                                     : key);
 }
 
+struct ControllerSettings {
+	QString color;
+	int     speaker_volume      = 100;
+	int     vibration_intensity = 100;
+
+	void WriteSettings(QSettings* s) const {
+		s->setValue("controller_color", color);
+		s->setValue("controller_speaker_volume", speaker_volume);
+		s->setValue("controller_vibration_intensity", vibration_intensity);
+	}
+
+	void ReadSettings(QSettings* s) {
+		const QColor saved_color(s->value("controller_color").toString());
+		color = saved_color.isValid() ? saved_color.name(QColor::HexRgb) : QString {};
+		const auto read_percent = [s](const char* key) {
+			bool      ok    = false;
+			const int value = s->value(key, 100).toInt(&ok);
+			return ok ? qBound(0, value, 100) : 100;
+		};
+		speaker_volume      = read_percent("controller_speaker_volume");
+		vibration_intensity = read_percent("controller_vibration_intensity");
+	}
+};
+
 class Configuration: public QObject {
 	Q_OBJECT
 
@@ -84,6 +109,9 @@ public:
 	GameStatus game_status     = GameStatus::Unknown;
 	QString    game_comment;
 
+	// Controller preferences always come from the global configuration.
+	ControllerSettings controller;
+
 	Resolution             screen_resolution           = Resolution::R1280X720;
 	QString                user_name                   = "Kyty";
 	int                    user_id                     = Config::DEFAULT_USER_ID;
@@ -91,6 +119,7 @@ public:
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int                    gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
+	bool                   hide_cursor_enabled         = false;
 	bool                   readback_linear_images      = false;
 	bool                   tessellation_enabled        = false;
 	int                    vblank_frequency            = 60;
@@ -122,6 +151,7 @@ public:
 		present_mode                = other.present_mode;
 		gpu_index                   = other.gpu_index;
 		fullscreen_enabled          = other.fullscreen_enabled;
+		hide_cursor_enabled         = other.hide_cursor_enabled;
 		readback_linear_images      = other.readback_linear_images;
 		tessellation_enabled        = other.tessellation_enabled;
 		vblank_frequency            = other.vblank_frequency;
@@ -168,6 +198,7 @@ public:
 		KYTY_CFG_SET(present_mode);
 		KYTY_CFG_SET(gpu_index);
 		KYTY_CFG_SET(fullscreen_enabled);
+		KYTY_CFG_SET(hide_cursor_enabled);
 		KYTY_CFG_SET(readback_linear_images);
 		KYTY_CFG_SET(tessellation_enabled);
 		KYTY_CFG_SET(vblank_frequency);
@@ -210,6 +241,7 @@ public:
 			present_mode = PresentMode::Mailbox;
 		}
 		KYTY_CFG_GET(fullscreen_enabled);
+		KYTY_CFG_GET(hide_cursor_enabled);
 		KYTY_CFG_GET(readback_linear_images);
 		KYTY_CFG_GET(tessellation_enabled);
 		vblank_frequency = s->value("vblank_frequency", vblank_frequency).toInt();

src/launcher/include/configurationEditDialog.h

@@ -28,16 +28,19 @@ public:
 	static void WriteSettings(QSettings& s);
 	static void ReadSettings(QSettings& s);
 
-	void                      SetGameDirectories(const QStringList& dirs);
+	void                      SetGlobalSettings(const QStringList& dirs);
 	[[nodiscard]] QStringList GetGameDirectories() const;
 
+signals:
+	void PreviewControllerColor(const QString& color);
+
 private:
 	Ui::ConfigurationEditDialog* m_ui = nullptr;
 	Configuration&               m_info;
 	QGroupBox*                   m_game_dirs_group        = nullptr;
 	QListWidget*                 m_game_dirs_list         = nullptr;
 	QToolButton*                 m_remove_game_dir_button = nullptr;
-	bool                         m_show_game_dirs         = false;
+	bool                         m_global_settings        = false;
 
 protected:
 	void Init(const Configuration& info);