shader: implement 32-bit RDNA2 IMAGE_ATOMIC_CMPSWAP (#952)

* shader: implement 32-bit RDNA2 image atomic compare-and-swap

* shader: share atomic emission and correct image barriers

---------

Co-authored-by: nmzik <Nmzik@mail.ru>

CMakeLists.txt

@@ -500,6 +500,10 @@ add_executable(save_data_memory_tests EXCLUDE_FROM_ALL
 target_link_libraries(save_data_memory_tests common fmt::fmt)
 target_include_directories(save_data_memory_tests PRIVATE ${inc_headers})
 
+add_executable(png_enc_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/PngEncTests.cpp")
+target_link_libraries(png_enc_tests common fmt::fmt)
+target_include_directories(png_enc_tests PRIVATE ${inc_headers})
+
 add_executable(ces_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/CesTests.cpp")
 target_link_libraries(ces_tests common fmt::fmt SDL3::SDL3-static)
 target_include_directories(ces_tests PRIVATE ${inc_headers})
@@ -605,6 +609,7 @@ if(BUILD_TESTING)
 	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
 	add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
 	add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
+	add_test(NAME png_enc COMMAND $<TARGET_FILE:png_enc_tests>)
 	add_test(NAME ces COMMAND $<TARGET_FILE:ces_tests>)
 	add_test(NAME save_data_dialog COMMAND $<TARGET_FILE:save_data_dialog_tests>)
 	add_test(NAME http_uri_parse COMMAND $<TARGET_FILE:http_uri_parse_tests>)
@@ -680,6 +685,7 @@ if(BUILD_TESTING)
 		ngs2_sampler_tests
 		pad_haptics_tests
 		save_data_memory_tests
+		png_enc_tests
 		ces_tests
 		save_data_dialog_tests
 		http_uri_parse_tests

src/common/emulatorConfig.cpp

@@ -47,6 +47,10 @@ const std::string& GetAudioInputDevice() {
 	return g_config->audio_input_device;
 }
 
+const std::optional<ControllerColor>& GetControllerColor() {
+	return g_config->controller_color;
+}
+
 PresentMode GetPresentMode() {
 	return g_config->present_mode;
 }

src/common/emulatorConfig.h

@@ -4,7 +4,9 @@
 #include "common/common.h"
 
 #include <cstddef>
+#include <array>
 #include <filesystem>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -26,6 +28,7 @@ enum class LogDirection { Silent, Console, File };
 enum class PresentMode { Fifo, Mailbox, Immediate };
 
 using Keymap = std::vector<std::string>;
+using ControllerColor = std::array<uint8_t, 3>;
 
 constexpr uint32_t DEFAULT_CONSOLE_LANGUAGE = 1;
 constexpr uint32_t MAX_CONSOLE_LANGUAGE     = 29;
@@ -44,6 +47,7 @@ struct ConfigOptions {
 	std::string            user_name                   = "Kyty";
 	int32_t                user_id                     = DEFAULT_USER_ID;
 	std::string            audio_input_device;
+	std::optional<ControllerColor> controller_color;
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
@@ -81,6 +85,7 @@ uint32_t GetScreenHeight();
 const std::string& GetUserName();
 int32_t  GetUserId();
 const std::string& GetAudioInputDevice();
+const std::optional<ControllerColor>& GetControllerColor();
 PresentMode GetPresentMode();
 int32_t GetGpuIndex();
 bool     FullscreenEnabled();

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -62,7 +62,6 @@ public:
 
 	void            BufferInit();
 	void            BufferFlush();
-	void            BufferFlushAndWait();
 	void            BufferWait();
 	HW::Context&    GetCtx() { return m_ctx; }
 	HW::UserConfig& GetUcfg() { return m_ucfg; }

src/graphics/guest_gpu/graphicsRun.cpp

@@ -35,7 +35,6 @@ namespace Libs::Graphics {
 
 static thread_local CommandProcessor* g_current_processor = nullptr;
 static thread_local Pm4Execution*     g_current_execution = nullptr;
-static thread_local bool              g_gpu_mutex_owned   = false;
 static thread_local bool              g_gpu_thread        = false;
 static thread_local GuestGpu*         g_gpu_state         = nullptr;
 
@@ -54,27 +53,6 @@ struct DrawIndexedIndirectArgs {
 	uint32_t start_instance_location;
 };
 
-class GpuMutexLock final {
-public:
-	explicit GpuMutexLock(Common::Mutex& mutex): m_mutex(mutex) {
-		if (g_gpu_mutex_owned) {
-			EXIT("recursive GPU mutex acquisition\n");
-		}
-		g_gpu_mutex_owned = true;
-		m_mutex.Lock();
-	}
-	~GpuMutexLock() {
-		if (!g_gpu_mutex_owned) {
-			EXIT("invalid GPU mutex release\n");
-		}
-		m_mutex.Unlock();
-		g_gpu_mutex_owned = false;
-	}
-
-private:
-	Common::Mutex& m_mutex;
-};
-
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
 	       Config::GetPrintfDirection() != Config::LogDirection::Silent;
@@ -84,6 +62,7 @@ GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	GraphicsInitJmpTables();
 	m_gfx_cp = std::make_unique<CommandProcessor>(renderer, 0);
+	m_gfx_cp->Reset();
 	m_thread = std::jthread(ThreadRun, this);
 }
 
@@ -160,20 +139,16 @@ void GuestGpu::Submit(std::span<const uint32_t> draw_commands,
 	if (draw_commands.empty()) {
 		return;
 	}
-	GpuMutexLock lock(m_submission_mutex);
-	Submission   submission;
+	Submission submission;
 	submission.type              = SubmissionType::Graphics;
 	submission.queue_id          = 0;
 	submission.commands          = draw_commands;
 	submission.constant_commands = constant_commands;
-	submission.reset_processor   = m_graphics_done;
-	m_graphics_done              = false;
 	Enqueue(std::move(submission));
 }
 
 void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands) {
 	EXIT_IF(commands.empty());
-	GpuMutexLock lock(m_submission_mutex);
 
 	EXIT_NOT_IMPLEMENTED(queue < ComputeQueueBase || queue >= ComputeQueueBase + ComputeQueueCount);
 
@@ -186,22 +161,21 @@ void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands)
 }
 
 void GuestGpu::SubmitFlipPreparation(uint64_t request_id) {
-	GpuMutexLock lock(m_submission_mutex);
-	Submission   submission;
+	Submission submission;
 	submission.type            = SubmissionType::FlipPreparation;
 	submission.queue_id        = 0;
-	submission.reset_processor = m_graphics_done;
 	submission.flip_request_id = request_id;
-	m_graphics_done            = false;
 	Enqueue(std::move(submission));
 }
 
-void GuestGpu::Done() {
-	GpuMutexLock lock(m_submission_mutex);
-	if (!IsGpuThread()) {
-		WaitForIdle();
-	}
-	m_graphics_done = true;
+void GuestGpu::SuspendPoint() {
+	EXIT_IF(IsGpuThread() || CommandScheduler::InDeferredOperation());
+	// Do not hold a queue lock while waiting: asynchronous work may be needed to
+	// finish the preceding graphics frame. The first point returns immediately.
+	m_suspend_point_ready->acquire();
+	Submission submission;
+	submission.type = SubmissionType::SuspendPoint;
+	Enqueue(std::move(submission));
 	m_done_num++;
 }
 
@@ -229,6 +203,9 @@ void CommandProcessor::Reset() {
 	m_context_state_pushed             = false;
 	m_index_type_and_size              = 0;
 	m_index_buffer_size                = 0;
+	m_index_base_addr                  = 0;
+	m_num_instances                    = 1;
+	m_predicate_skip                   = false;
 	m_user_data_marker                 = HW::UserSgprType::Unknown;
 	m_draw_indirect_args_base_addr     = 0;
 	m_dispatch_indirect_args_base_addr = 0;
@@ -268,10 +245,6 @@ void CommandProcessor::BufferFlush() {
 	GetScheduler().Flush();
 }
 
-void CommandProcessor::BufferFlushAndWait() {
-	GetScheduler().FlushAndWait();
-}
-
 void CommandProcessor::BufferWait() {
 	BufferInit();
 	GetScheduler().Finish();
@@ -459,6 +432,7 @@ void GuestGpu::Enqueue(Submission submission) {
 }
 
 void GuestGpu::WaitForIdle() {
+	EXIT_IF(IsGpuThread() || CommandScheduler::InDeferredOperation());
 	Common::LockGuard lock(m_queue_mutex);
 	while (m_processing || !m_commands.empty() || m_submission_count != 0) {
 		m_idle.Wait(&m_queue_mutex);
@@ -564,11 +538,7 @@ void GuestGpu::ThreadRun(void* data) {
 
 bool GuestGpu::Process(Submission& submission) {
 	const bool first_slice = !submission.started;
-	auto& cp = GetProcessor(submission.queue_id);
-
-	if (first_slice && submission.reset_processor) {
-		cp.Reset();
-	}
+	auto&      cp          = GetProcessor(submission.queue_id);
 
 	if (first_slice) {
 		submission.started = true;
@@ -645,6 +615,13 @@ bool GuestGpu::Process(Submission& submission) {
 			m_renderer.RunGarbageCollector();
 			cp.PrepareCpuFlip(submission.flip_request_id);
 			break;
+		case SubmissionType::SuspendPoint:
+			cp.EmitGlobalBarrier();
+			m_renderer.GetCommandScheduler().DeferPriorityOperation(
+			    [ready = m_suspend_point_ready] { ready->release(); });
+			cp.BufferFlush();
+			cp.Reset();
+			break;
 	}
 
 	return complete;
@@ -849,9 +826,7 @@ void CommandProcessor::SetPredication(uint32_t condition, uint32_t op, uint32_t
 			}
 		} break;
 		case 0x03:
-			if (wait_op != 0) {
-				BufferFlushAndWait();
-			}
+			// The wait selector applies only to Z-pass query readiness.
 			EXIT_NOT_IMPLEMENTED(address == nullptr);
 			value = *reinterpret_cast<const volatile uint64_t*>(address);
 			break;
@@ -1034,9 +1009,6 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
 
 	uint32_t frame_num = 0;
-	// uint32_t local_x   = 1;
-	// uint32_t local_y   = 1;
-	// uint32_t local_z   = 1;
 
 	{
 		frame_num = m_renderer.GetGpu().GetFrameNum();
@@ -1058,35 +1030,9 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 			}
 		}
 
-		const auto& cs = m_sh_ctx.GetCs().cs_regs;
-		// local_x        = std::max(cs.num_thread_x, 1u);
-		// local_y        = std::max(cs.num_thread_y, 1u);
-		// local_z        = std::max(cs.num_thread_z, 1u);
 		m_renderer.GetRenderExecutor().DispatchDirect(m_submit_id, CurrentBuffer(), thread_group_x,
 		                                              thread_group_y, thread_group_z, mode);
 	}
-
-	/*constexpr uint32_t DispatchInitiatorUseThreadDimensions = 1u << 5u;
-	auto               group_count = [](uint32_t threads, uint32_t group_size) {
-	    return (threads == 0
-	                ? 0u
-	                : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u));
-	};
-
-	auto groups_x = thread_group_x;
-	auto groups_y = thread_group_y;
-	auto groups_z = thread_group_z;
-	if ((mode & DispatchInitiatorUseThreadDimensions) != 0) {
-	    groups_x = group_count(thread_group_x, local_x);
-	    groups_y = group_count(thread_group_y, local_y);
-	    groups_z = group_count(thread_group_z, local_z);
-	}
-
-	const uint64_t invocations =
-	    static_cast<uint64_t>(groups_x) * groups_y * groups_z * local_x * local_y * local_z;
-	if (invocations != 0) {
-	    BufferFlushAndWait();
-	}*/
 }
 
 void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {

src/graphics/guest_gpu/graphicsRun.h

@@ -13,6 +13,7 @@
 #include <deque>
 #include <memory>
 #include <mutex>
+#include <semaphore>
 #include <span>
 #include <thread>
 
@@ -32,11 +33,14 @@ public:
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
 
 	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
-	void              Submit(std::span<const uint32_t> draw_commands,
-	                         std::span<const uint32_t> constant_commands);
-	void              SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
-	void              SubmitFlipPreparation(uint64_t request_id);
-	void              Done();
+	void Submit(std::span<const uint32_t> draw_commands,
+	            std::span<const uint32_t> constant_commands);
+	void SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
+	void SubmitFlipPreparation(uint64_t request_id);
+	// Insert an ordered graphics drain; only a previous suspend point can block the caller.
+	void SuspendPoint();
+	// Wait for guest command processing, including all compute queues (not native GPU completion).
+	void              WaitForIdle();
 	[[nodiscard]] int GetFrameNum() const;
 
 	[[nodiscard]] static bool IsGpuThread() noexcept;
@@ -48,7 +52,7 @@ private:
 	static constexpr uint32_t ComputeQueueBase     = 0x20;
 	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
 
-	enum class SubmissionType { Graphics, Compute, FlipPreparation };
+	enum class SubmissionType { Graphics, Compute, FlipPreparation, SuspendPoint };
 
 	struct Submission {
 		SubmissionType            type     = SubmissionType::Graphics;
@@ -57,7 +61,6 @@ private:
 		std::span<const uint32_t> constant_commands;
 		Pm4Execution              command_execution;
 		Pm4Execution              constant_execution;
-		bool                      reset_processor   = false;
 		bool                      started           = false;
 		bool                      command_complete  = false;
 		bool                      constant_complete = false;
@@ -66,14 +69,12 @@ private:
 	};
 
 	void              Enqueue(Submission submission);
-	void              WaitForIdle();
 	void              ProcessCommands();
 	bool              Process(Submission& submission);
 	static void       ThreadRun(void* data);
 	CommandProcessor& GetProcessor(uint32_t queue_id);
 
 	RenderContext&                                 m_renderer;
-	Common::Mutex                                  m_submission_mutex;
 	Common::Mutex                                  m_queue_mutex;
 	std::mutex                                     m_shutdown_mutex;
 	Common::CondVar                                m_work_available;
@@ -84,10 +85,12 @@ private:
 	uint32_t                                       m_next_queue        = 0;
 	uint32_t                                       m_submission_count  = 0;
 	bool                                           m_processing        = false;
-	bool                                           m_graphics_done     = true;
 	bool                                           m_accepting         = true;
 	bool                                           m_stopping          = false;
 	bool                                           m_shutdown_complete = false;
+	// Completion callbacks can outlive GuestGpu during renderer shutdown.
+	std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =
+	    std::make_shared<std::binary_semaphore>(1);
 
 	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
 	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -583,7 +583,7 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {
 	}
 
 	auto                         depth_buffer = m_tiler.GetScratchBuffer(depth_size);
-	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size);
+	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer);
 	const TileManager::D16Layout promote_layout {
 	    .width               = depth.info.extent.width,
 	    .height              = depth.info.extent.height,
@@ -1071,7 +1071,7 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 		                     texels_per_slice > UINT64_MAX / transfer_bytes);
 		const uint64_t transfer_slice = texels_per_slice * transfer_bytes;
 		EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers);
-		auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers);
+		auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers, linear.buffer);
 		m_tiler.ConvertD16(
 		    linear, promoted, TileManager::D16Direction::Promote,
 		    info.pixel_format == vk::Format::eD32SfloatS8Uint,
@@ -1679,7 +1679,7 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	auto host_linear = m_tiler.GetScratchBuffer(transfer_size);
 	image.Download(copies, host_linear.buffer, 0, host_linear.size);
 	const bool tiled        = info.IsTiled();
-	auto       guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size)
+	auto       guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size, host_linear.buffer)
 	                                : TileManager::Result {destination.Handle(), destination_offset,
 	                                                       destination.Size() - destination_offset};
 	m_tiler.ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote,

src/graphics/host_gpu/renderer/image/imageView.h

@@ -31,20 +31,14 @@ namespace ImageViewOps {
 	       ImageViewOps::IsFormatDepthCompatible(view_format);
 }
 
-[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle) noexcept {
+[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle,
+                                             uint32_t component_count = 4) noexcept {
 	if ((swizzle & ~0xfffu) != 0) {
 		return false;
 	}
 	for (uint32_t channel = 0; channel < 4; channel++) {
-		switch (GetDstSel(swizzle, channel)) {
-			case 0:
-			case 1:
-			case 4:
-			case 5:
-			case 6:
-			case 7: break;
-			default: return false;
-		}
+		const auto selector = GetDstSel(swizzle, channel);
+		if (selector > 1 && (selector < 4 || selector >= 4 + component_count)) return false;
 	}
 	return true;
 }
@@ -81,15 +75,8 @@ SelectSampledColorView(vk::Format image_format, vk::Format view_format, uint32_t
 [[nodiscard]] inline bool IsSupportedSampledDepthView(vk::Format image_format,
                                                       vk::Format view_format,
                                                       uint32_t   swizzle) noexcept {
-	if (!IsSupportedSampledDepthFormat(image_format, view_format)) {
-		return false;
-	}
-	switch (swizzle) {
-		case DstSel(4, 4, 4, 4):
-		case DstSel(4, 0, 0, 0):
-		case DstSel(4, 0, 0, 1): return true;
-		default: return false;
-	}
+	return IsSupportedSampledDepthFormat(image_format, view_format) &&
+	       IsValidImageSwizzle(swizzle, 1);
 }
 
 [[nodiscard]] inline bool

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -88,30 +88,6 @@ TileManager::~TileManager() {
 	}
 }
 
-TileManager::Scratch TileManager::AllocateScratch(uint64_t size) {
-	EXIT_IF(size == 0);
-	vk::BufferCreateInfo create {};
-	create.size  = size;
-	create.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc |
-	               vk::BufferUsageFlagBits::eTransferDst;
-
-	VmaAllocationCreateInfo allocate {};
-	allocate.usage       = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
-	VkBuffer      buffer = VK_NULL_HANDLE;
-	VmaAllocation memory = nullptr;
-	const auto    raw    = static_cast<VkBufferCreateInfo>(create);
-	RequireVulkanSuccess(static_cast<vk::Result>(vmaCreateBuffer(
-	                         m_graphics.allocator, &raw, &allocate, &buffer, &memory, nullptr)),
-	                     "allocate TileManager scratch buffer");
-	return {buffer, memory, size};
-}
-
-void TileManager::DeferDestroy(Scratch scratch) {
-	auto allocator = m_graphics.allocator;
-	m_scheduler.DeferOperation(
-	    [allocator, scratch] { vmaDestroyBuffer(allocator, scratch.buffer, scratch.allocation); });
-}
-
 void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
                           std::span<const GpuTileInfo> infos, uint64_t source_base,
                           uint64_t target_base, std::vector<Dispatch>& dispatches) {
@@ -369,8 +345,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
 	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
-	auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4));
-	DeferDestroy(scratch);
+	auto scratch = GetScratchBuffer(linear_capacity, tiled);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
 	return {scratch.buffer, 0, linear_capacity};
@@ -400,11 +375,9 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        target_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	// Reserve all stream parameters before creating a scheduler-lived scratch dependency:
-	// StreamBuffer::Map is allowed to submit the current tick when it wraps.
+	// Reserve stream parameters before recording the image download and conversion.
 	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
-	auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4));
-	DeferDestroy(linear);
+	auto linear = GetScratchBuffer(linear_capacity, tiled);
 	image.Download(regions, linear.buffer, 0, linear.size);
 	Result source {linear.buffer, 0, linear.size};
 	if (transform == ColorTransform::SwapBgra16) {
@@ -414,10 +387,19 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	       dispatches, false);
 }
 
-TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
-	auto scratch = AllocateScratch(Common::AlignUp(size, 4));
-	DeferDestroy(scratch);
-	return {scratch.buffer, 0, scratch.size};
+TileManager::Result TileManager::GetScratchBuffer(uint64_t size, vk::Buffer input) {
+	EXIT_IF(size == 0);
+	size = Common::AlignUp(size, 4);
+	auto& buffer = m_scratch[m_scratch[0] && m_scratch[0]->Handle() == input ? 1 : 0];
+	if (!buffer || buffer->Size() < size) {
+		if (buffer) {
+			m_scheduler.DeferOperation([old = std::move(buffer)]() mutable { old.reset(); });
+		}
+		buffer = std::make_unique<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0,
+		    vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc |
+		        vk::BufferUsageFlagBits::eTransferDst, size);
+	}
+	return {buffer->Handle(), 0, size};
 }
 
 TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const {
@@ -668,9 +650,7 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 
 TileManager::Result TileManager::SwapBgra16(Result input) {
 	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX);
-	auto output = AllocateScratch(input.size);
-	DeferDestroy(output);
-	Result result {output.buffer, 0, output.size};
+	auto result = GetScratchBuffer(input.size, input.buffer);
 	SwapBgra16(input, result, static_cast<uint32_t>(input.size / 8u));
 	return result;
 }

src/graphics/host_gpu/renderer/image/tiler.h

@@ -6,13 +6,14 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
+#include <memory>
 #include <span>
 #include <vector>
-#include <vk_mem_alloc.h>
 
 namespace Libs::Graphics {
 
 class CommandScheduler;
+class Buffer;
 class Image;
 class StreamBuffer;
 struct GraphicContext;
@@ -62,7 +63,7 @@ public:
 	~TileManager();
 	KYTY_CLASS_NO_COPY(TileManager);
 
-	// The returned device-local buffer remains alive through the current scheduler tick.
+	// Consume scratch results before the next acquisition, or pass their buffer as input.
 	[[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity,
 	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos);
 	void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled,
@@ -71,7 +72,7 @@ public:
 	               uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity,
 	               std::span<const GpuTileInfo> infos,
 	               ColorTransform               transform = ColorTransform::None);
-	[[nodiscard]] Result GetScratchBuffer(uint64_t size);
+	[[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr);
 	void                 ConvertD16(Result source, Result target, D16Direction direction, bool d32,
 	                                const D16Layout& layout);
 	[[nodiscard]] Result SwapBgra16(Result input);
@@ -105,23 +106,16 @@ private:
 		uint32_t pipeline_slot = 0;
 		uint64_t params_offset = 0;
 	};
-	struct Scratch {
-		vk::Buffer    buffer     = nullptr;
-		VmaAllocation allocation = nullptr;
-		uint64_t      size       = 0;
-	};
 	struct StorageBinding {
 		vk::DescriptorBufferInfo info;
 		uint32_t                 base = 0;
 	};
 
-	[[nodiscard]] Scratch         AllocateScratch(uint64_t size);
 	[[nodiscard]] StorageBinding  BindStorage(Result buffer, uint64_t size) const;
 	[[nodiscard]] static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride,
 	                                             uint64_t active, uint32_t remaining,
 	                                             uint64_t alignment, uint64_t max_range,
 	                                             uint32_t max_groups) noexcept;
-	void                          DeferDestroy(Scratch scratch);
 	void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
 	             std::span<const GpuTileInfo> infos, uint64_t source_base, uint64_t target_base,
 	             std::vector<Dispatch>& dispatches);
@@ -134,6 +128,7 @@ private:
 	GraphicContext&                         m_graphics;
 	CommandScheduler&                       m_scheduler;
 	StreamBuffer&                           m_stream_buffer;
+	std::array<std::unique_ptr<Buffer>, 2>   m_scratch;
 	vk::DescriptorSetLayout                 m_descriptor_layout = nullptr;
 	vk::PipelineLayout                      m_pipeline_layout   = nullptr;
 	std::array<vk::Pipeline, PipelineCount> m_pipelines {};

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -974,6 +974,41 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
 		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
+	for (const auto* reader: prepared_bindings) {
+		const auto& reads = reader->runtime->resources->specialization_reads;
+		if (reads.empty()) continue;
+		for (const auto* writer: prepared_bindings) {
+			if (writer->runtime->program->has_address_writes) {
+				EXIT("scalar resource reads cannot be proven disjoint from shader address writes\n");
+			}
+		}
+		for (const auto [address, size]: reads) {
+			for (const auto id: m_bound_images) {
+				const auto* image = m_context.GetTextureCache().m_slot_images.try_get(id);
+				if (image == nullptr ||
+				    (!image->binding.shader_write && !image->binding.is_target)) continue;
+				for (const auto written: {image->info.data, image->info.stencil,
+				                          image->info.metadata.range}) {
+					if (written.size != 0 && ImageRangeOverlaps(address, size,
+					                                          written.address, written.size)) {
+						EXIT("scalar resource reads overlap an image or attachment write\n");
+					}
+				}
+			}
+			for (const auto* writer: prepared_bindings) {
+				const auto& program = *writer->runtime->program;
+				for (uint32_t i = 0; i < writer->buffer_sources.size(); ++i) {
+					const auto resource = program.bindings.descriptors.front().resources[i];
+					if (!program.info.buffers[resource].written) continue;
+					const auto& written = writer->buffer_sources[i];
+					if (written.size != 0 && ImageRangeOverlaps(address, size,
+					                                          written.address, written.size)) {
+						EXIT("scalar resource reads overlap a shader buffer write\n");
+					}
+				}
+			}
+		}
+	}
 	m_descriptor_buffers.clear();
 	m_descriptor_images.clear();
 	m_descriptor_writes.clear();

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -220,7 +220,6 @@ struct PipelineCache::ProgramCache {
 		ShaderRecompiler::IR::ResourceSnapshot       resources;
 		ShaderRecompiler::IR::ResourceSpecialization specialization;
 		std::vector<Permutation>                    permutations;
-		bool                                        skip_dispatch = false;
 	};
 
 	struct ProgramKeyHash {
@@ -288,9 +287,6 @@ struct PipelineCache::ProgramCache {
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
-		if (entry != programs.end() && entry->second.skip_dispatch) {
-			return {};
-		}
 		const ShaderRecompiler::IR::SrtRuntime       runtime {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
@@ -358,11 +354,6 @@ struct PipelineCache::ProgramCache {
 		}
 		DumpShaderOriginal(stage_name, options.shader_hash, params.code);
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
-		if (translated.skip_dispatch) {
-			entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
-			entry->second.skip_dispatch = true;
-			return {};
-		}
 		if (entry == programs.end()) {
 			entry = programs.try_emplace(lookup_key,
 			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
@@ -723,13 +714,6 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		}
 		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
 		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		static_params.color_srcblend[slot]       = bc.color_srcblend;
-		static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
-		static_params.color_destblend[slot]      = bc.color_destblend;
-		static_params.alpha_srcblend[slot]       = bc.alpha_srcblend;
-		static_params.alpha_comb_fcn[slot]       = bc.alpha_comb_fcn;
-		static_params.alpha_destblend[slot]      = bc.alpha_destblend;
-		static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
 		const bool alpha_remap =
 		    slot == 0 && ps_input_info != nullptr && ps_input_info->alpha_blend_source_remap;
 		static_params.blend_enable[slot] = bc.enable && !rt.info.blend_bypass;
@@ -748,6 +732,17 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		if (alpha_remap) {
 			static_params.blend_alpha_source_remap = true;
 		}
+		if (static_params.blend_enable[slot]) {
+			static_params.color_srcblend[slot]       = bc.color_srcblend;
+			static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
+			static_params.color_destblend[slot]      = bc.color_destblend;
+			static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
+			if (bc.separate_alpha_blend) {
+				static_params.alpha_srcblend[slot]  = bc.alpha_srcblend;
+				static_params.alpha_comb_fcn[slot]  = bc.alpha_comb_fcn;
+				static_params.alpha_destblend[slot] = bc.alpha_destblend;
+			}
+		}
 	}
 	const bool with_depth =
 	    depth.desc.view_info.format != vk::Format::eUndefined && static_cast<bool>(depth.image_id);

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -330,9 +330,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 			}
 		}
 
-		EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].AddTid());
-		EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].SwizzleEnabled());
-
 		EXIT_IF(registers_num < 1 || registers_num > 4);
 	}
 

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -256,10 +256,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	input_info.dispatch_thread_dimensions = use_thread_dimensions;
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
-	if (!compute_program) {
-		// Temporary until RT is implemented.
-		return;
-	}
 	if (use_thread_dimensions) {
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
@@ -268,15 +264,14 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 
 	const auto& program   = *input_info.stage.program;
 	const auto& resources = *input_info.stage.resources;
-	if (TryConsumeComputeMetaClear(input_info, buffer)) {
-		ResetBindings();
-		return;
-	}
-	if (TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y,
-	                                thread_group_z, mode)) {
+	if (resources.specialization_reads.empty() &&
+	    (TryConsumeComputeMetaClear(input_info, buffer) ||
+	     TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y,
+	                                 thread_group_z, mode))) {
 		ResetBindings();
 		return;
 	}
+
 	const bool large_workgroup =
 	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const bool                   has_sampler = !program.info.samplers.empty();
@@ -417,10 +412,6 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	ShaderComputeInputInfo input_info {};
 	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
 	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
-	if (!compute_program) {
-		// Temporary until RT is implemented.
-		return;
-	}
 	buffer.EndRendering();
 	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto& bindings = m_compute_bindings;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -119,6 +119,10 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void RenderContext::PrepareBda() {
+	if (!m_bda_logged) {
+		Log::WriteToConsoleAndLog("GPU: using buffer device address (BDA) shader memory access.\n");
+		m_bda_logged = true;
+	}
 	std::shared_lock lock(m_mapped_ranges_mutex);
 	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
 		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);

src/graphics/host_gpu/renderer/renderContext.h

@@ -81,6 +81,7 @@ private:
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 	bool                      m_fault_process_pending = false;
+	bool                      m_bda_logged = false;
 
 	Common::Mutex                        m_interrupt_mutex;
 	std::vector<InterruptEqRegistration> m_interrupt_eqs;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -1032,6 +1032,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	uint32_t   mesh_groups = 0;
 	if (mesh_active) {
 		const auto& mesh = state.vertex_info[0].mesh;
+		EXIT_NOT_IMPLEMENTED(mesh.fast_launch && (draw.IsIndexed() || primitive_restart_enable));
 		static std::atomic_bool restart_warned = false;
 		if (primitive_restart_enable && !restart_warned.exchange(true, std::memory_order_relaxed)) {
 			std::printf("Warning: primitive restart is not implemented for mesh shaders; "

src/graphics/presentation/videoOut.cpp

@@ -1814,6 +1814,38 @@ KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* stat
 	return OK;
 }
 
+KYTY_SYSV_ABI int VideoOutAllowOutputResolutionWqhdDetection(int handle) {
+	if (!DriverState().IsOpened(handle)) {
+		return VIDEO_OUT_ERROR_INVALID_HANDLE;
+	}
+	return OK;
+}
+
+KYTY_SYSV_ABI int VideoOutVrrPegToFixedRate(int handle, uint64_t arg1, uint64_t arg2) {
+	PRINT_NAME();
+
+	static std::atomic_bool logged {false};
+	if (!logged.exchange(true, std::memory_order_relaxed)) {
+		LOGF("\t handle = %d\n"
+		     "\t arg1   = 0x%016" PRIx64 "\n"
+		     "\t arg2   = 0x%016" PRIx64 "\n",
+		     handle, arg1, arg2);
+	}
+
+	return OK;
+}
+
+KYTY_SYSV_ABI int VideoOutVrrUnpegFromFixedRate(int handle) {
+	PRINT_NAME();
+
+	static std::atomic_bool logged {false};
+	if (!logged.exchange(true, std::memory_order_relaxed)) {
+		LOGF("\t handle = %d\n", handle);
+	}
+
+	return OK;
+}
+
 static int ValidateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options,
                                 void* reserved_ptr, uint64_t reserved) {
 	if (!DriverState().IsOpened(handle)) {

src/graphics/presentation/videoOut.h

@@ -90,6 +90,9 @@ KYTY_SYSV_ABI int VideoOutGetEventData(const LibKernel::EventQueue::KernelEvent*
 KYTY_SYSV_ABI int VideoOutGetEventCount(const LibKernel::EventQueue::KernelEvent* ev);
 KYTY_SYSV_ABI int VideoOutWaitVblank(int handle);
 KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status);
+KYTY_SYSV_ABI int VideoOutAllowOutputResolutionWqhdDetection(int handle);
+KYTY_SYSV_ABI int VideoOutVrrPegToFixedRate(int handle, uint64_t arg1, uint64_t arg2);
+KYTY_SYSV_ABI int VideoOutVrrUnpegFromFixedRate(int handle);
 KYTY_SYSV_ABI int VideoOutInitializeOutputOptions(VideoOutOutputOptions* options);
 KYTY_SYSV_ABI int VideoOutIsOutputSupported(int handle, uint64_t mode,
                                             const VideoOutOutputOptions* options,

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -19,7 +19,6 @@
 
 #include <algorithm>
 #include <array>
-#include <atomic>
 #include <chrono>
 #include <fmt/format.h>
 #include <map>
@@ -523,20 +522,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
 
-	// Temporary workaround for games that compile ray-tracing shaders before
-	// the player can select a mode without ray tracing.
-	if (options.stage == ShaderType::Compute && decoded.has_bvh) {
-		static std::atomic_flag warned = ATOMIC_FLAG_INIT;
-		if (!warned.test_and_set(std::memory_order_relaxed)) {
-			const auto& bvh = decoded.instructions.back();
-			Log::WriteToConsoleAndLog(fmt::format(
-			    "Warning: ray tracing is not implemented; skipping compute dispatches containing "
-			    "BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n",
-			    options.shader_hash, bvh.pc, bvh.opcode_id));
-		}
-		return {.skip_dispatch = true};
-	}
-
 	std::string decoded_dump;
 	if (options.dump_ir) {
 		decoded_dump = Decoder::ProgramToString(decoded);
@@ -609,7 +594,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
 	IR::RewriteToSsa(ir.blocks);
-	IR::ConstantPropagationPass(ir.blocks);
+	IR::ConstantPropagationPass(ir.blocks, ir.wave_size);
 	IR::ResolveControlFlowIdentities(ir);
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
@@ -617,7 +602,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	if (read_lane_stats.rewritten_reads != 0) {
 		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
 		     read_lane_stats.rewritten_reads);
-		IR::ConstantPropagationPass(ir.blocks);
+		IR::ConstantPropagationPass(ir.blocks, ir.wave_size);
 		IR::ResolveControlFlowIdentities(ir);
 		IR::RemoveIdentities(ir.blocks);
 		IR::EliminateDeadCode(ir.blocks);
@@ -645,7 +630,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options,
                              const IR::ResourceSpecialization& specialization,
                              uint32_t push_data_start_dword) {
-	EXIT_IF(translated.skip_dispatch);
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
 	IR::ApplyResourceSpecialization(ir, specialization);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -28,7 +28,6 @@ struct TranslateResult {
 	IR::Program program;
 	std::string decoded_dump;
 	std::string cfg_dump;
-	bool        skip_dispatch = false;
 };
 
 struct CompileResult {

src/graphics/shader/recompiler/Tessellation.cpp

@@ -217,7 +217,7 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 			    2, ActiveAddress(*block, it, it->Arg(2), it->Arg(it->NumArgs() - 1u), resolved));
 		}
 	}
-	ConstantPropagationPass(program.blocks);
+	ConstantPropagationPass(program.blocks, program.wave_size);
 	uint32_t reads = 0, writes = 0, factors = 0;
 	for (auto* block: program.blocks) {
 		for (auto it = block->begin(); it != block->end(); ++it) {
@@ -317,7 +317,7 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 			inst.Invalidate();
 		}
 	}
-	ConstantPropagationPass(program.blocks);
+	ConstantPropagationPass(program.blocks, program.wave_size);
 	RemoveIdentities(program.blocks);
 	EliminateDeadCode(program.blocks);
 	LOGF("%s tessellation lowering: reads=%u writes=%u factors=%u\n",

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -122,6 +122,16 @@ void ValidateNativeProgram(const IR::Program& program) {
 			              program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer;
 		       });
 	};
+	const auto local_flat_handle = [&](const IR::Inst& handle) {
+		return !handle.Uses().empty() &&
+		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
+			       if (IR::AddressOpcodeInfoOf(use.user->GetOpcode()).access == IR::AddressAccess::None)
+				       return false;
+			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].kind == IR::ResourceKind::FlatLocal;
+		       });
+	};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto dense = inst.Flags<uint32_t>();
@@ -138,7 +148,7 @@ void ValidateNativeProgram(const IR::Program& program) {
 					if (planning_only_handle(inst)) {
 						break;
 					}
-					if (inst.NumArgs() != 2 || !program.info.uses_dma) {
+					if (inst.NumArgs() != 2 || (!program.info.uses_dma && !local_flat_handle(inst))) {
 						Fail(program, "typed address handle has invalid DMA metadata");
 					}
 					break;
@@ -188,11 +198,16 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (memory_index >= program.memory_info.size()) {
 					Fail(program, "address operation has invalid memory metadata");
 				}
-				if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+				const auto kind = program.memory_info[memory_index].kind;
+				if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
 					if (program.scratch_dwords == 0) {
 						Fail(program, "scratch operation has no per-thread storage");
 					}
 					requirements.function_scratch = true;
+					if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute &&
+					    program.stage != ShaderType::Mesh) {
+						requirements.function_lds = true;
+					}
 				} else if (address_access == IR::AddressAccess::Write) {
 					Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
 				}
@@ -229,7 +244,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
 					Fail(program, "shared operation has invalid resource kind");
 				}
-				if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicOr64) {
+				if (shared_access == IR::SharedAccess::Atomic &&
+				    IR::SharedComponentCount(inst.GetOpcode()) == 2u) {
 					if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) {
 						Fail(program, "64-bit shared atomics require compute LDS");
 					}
@@ -247,6 +263,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				}
 			}
 			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
 				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
 				case IR::ValueOpcode::ReadFirstLane:

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -444,6 +444,18 @@ uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
 }
 
+uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
+}
+
+uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
+}
+
+uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
+}
+
 uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
 	return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -0,0 +1,348 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+using Vec3 = std::array<uint32_t, 3>;
+
+uint32_t Extract(EmitterState& s, uint32_t type, uint32_t vector, uint32_t index) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeExtract, type, result, vector, index);
+	return result;
+}
+
+uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> components) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, type, result, components);
+	return result;
+}
+
+uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
+	const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
+	const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
+	                            Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
+	const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
+	return result;
+}
+
+uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
+                  Vec3 origin, Vec3 direction) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
+		const auto value = Binary(s, code, f, x, y);
+		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
+		return value;
+	};
+	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
+	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
+	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
+	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
+	const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); };
+	// The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0).
+	const std::array vertices {
+		EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, kind, ConstantU32(s, 2)),
+		Select(s, u, is_kind(0), ConstantU32(s, 1),
+		       Select(s, u, is_kind(3), ConstantU32(s, 4), ConstantU32(s, 3))),
+		Select(s, u, is_kind(2), ConstantU32(s, 4),
+		       Select(s, u, is_kind(3), ConstantU32(s, 0), ConstantU32(s, 2))) };
+	std::array<Vec3, 3> positions;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto first = Binary(s, spv::OpIMul, u, vertices[i], ConstantU32(s, 3));
+		for (uint32_t axis = 0; axis < 3; ++axis)
+			positions[i][axis] = Unary(s, spv::OpBitcast, f,
+			    NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis))));
+	}
+	const auto flag = NodeWord(s, block, ConstantU32(s, 15));
+	const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1]));
+	const auto z_largest = cmp(spv::OpFOrdLessThan,
+	    Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2]));
+	const auto rotate = [&](Vec3 value) {
+		Vec3 result;
+		for (uint32_t i = 0; i < 3; ++i)
+			result[i] = Select(s, f, z_largest, value[i],
+			    Select(s, f, y_largest, value[(i + 2) % 3], value[(i + 1) % 3]));
+		return result;
+	};
+	origin = rotate(origin);
+	direction = rotate(direction);
+	std::array<Vec3, 3> projected;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto position = rotate(positions[i]);
+		Vec3 relative;
+		for (uint32_t axis = 0; axis < 3; ++axis)
+			relative[axis] = op(spv::OpFSub, position[axis], origin[axis]);
+		projected[i] = {op(spv::OpFSub, op(spv::OpFMul, relative[0], direction[2]),
+		                              op(spv::OpFMul, direction[0], relative[2])),
+		                op(spv::OpFSub, op(spv::OpFMul, relative[1], direction[2]),
+		                              op(spv::OpFMul, direction[1], relative[2])), relative[2]};
+	}
+	Vec3 edge, weighted;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto& first = projected[(i + 1) % 3];
+		const auto& second = projected[(i + 2) % 3];
+		edge[i] = op(spv::OpFSub, op(spv::OpFMul, second[0], first[1]),
+		                        op(spv::OpFMul, second[1], first[0]));
+		weighted[i] = op(spv::OpFMul, edge[i], direction[2]);
+	}
+	auto numerator = op(spv::OpFAdd, op(spv::OpFAdd,
+	    op(spv::OpFMul, edge[0], projected[0][2]), op(spv::OpFMul, edge[1], projected[1][2])),
+	    op(spv::OpFMul, edge[2], projected[2][2]));
+	auto denominator = op(spv::OpFAdd, op(spv::OpFAdd, weighted[0], weighted[1]), weighted[2]);
+	uint32_t negative = ConstantBool(s, false), positive = negative;
+	for (auto value: edge) {
+		negative = either(negative, cmp(spv::OpFOrdLessThan, value, zero));
+		positive = either(positive, cmp(spv::OpFOrdGreaterThan, value, zero));
+	}
+	const auto winding = cmp(spv::OpFOrdGreaterThan, denominator, zero);
+	auto missed = either(both(negative, positive), cmp(spv::OpFOrdEqual, denominator, zero));
+	missed = either(missed, Unary(s, spv::OpIsNan, b, numerator));
+	missed = either(missed, cmp(spv::OpFOrdLessThan,
+	    Select(s, f, winding, numerator, Unary(s, spv::OpFNegate, f, numerator)), zero));
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto first = projected[(i + 1) % 3][1], second = projected[(i + 2) % 3][1];
+		const auto first_zero = cmp(spv::OpFOrdEqual, first, zero);
+		const auto horizontal = both(first_zero, cmp(spv::OpFOrdEqual, second, zero));
+		const auto below = either(cmp(spv::OpFOrdLessThan, first, zero),
+		    both(first_zero, cmp(spv::OpFOrdGreaterThan, second, zero)));
+		const auto right = Binary(s, spv::OpLogicalNotEqual, b, below, winding);
+		const auto excluded = Select(s, b, horizontal,
+		    cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right);
+		missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded));
+	}
+	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
+	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
+	const auto coordinate = [&](uint32_t shift) {
+		const auto index = Binary(s, spv::OpBitwiseAnd, u,
+		    Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3));
+		return Unary(s, spv::OpBitcast, u, Select(s, f,
+		    cmp(spv::OpIEqual, index, ConstantU32(s, 1)), weighted[1],
+		    Select(s, f, cmp(spv::OpIEqual, index, ConstantU32(s, 2)), weighted[2], weighted[0])));
+	};
+	const std::array result {
+		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)),
+		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)),
+		Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)),
+		Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) };
+	return Vector(s, TypeU32Vector(s, 4), result);
+}
+
+std::array<uint32_t, 6> Box(EmitterState& s, uint32_t first, uint32_t second,
+                           uint32_t half, uint32_t child) {
+	const auto f = TypeF32(s);
+	const auto half_label = s.builder.AllocateId(), full_label = s.builder.AllocateId();
+	const auto merge = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+	s.builder.AddFunction(spv::OpBranchConditional, half, half_label, full_label);
+	EmitLabel(s, half_label);
+	std::array<uint32_t, 6> packed, full, result;
+	for (uint32_t pair = 0; pair < 3; ++pair) {
+		const auto word = NodeWord(s, first, ConstantU32(s, 4 + child * 3 + pair));
+		const auto unpacked = EmitGlsl<GLSLstd450UnpackHalf2x16, IR::Type::F32x2>(s, word);
+		packed[pair * 2] = Extract(s, f, unpacked, 0);
+		packed[pair * 2 + 1] = Extract(s, f, unpacked, 1);
+	}
+	s.builder.AddFunction(spv::OpBranch, merge);
+	EmitLabel(s, full_label);
+	for (uint32_t component = 0; component < 6; ++component) {
+		const auto word = 4 + child * 6 + component;
+		full[component] = Unary(s, spv::OpBitcast, f,
+		    NodeWord(s, word < 16 ? first : second, ConstantU32(s, word % 16)));
+	}
+	s.builder.AddFunction(spv::OpBranch, merge);
+	EmitLabel(s, merge);
+	for (uint32_t component = 0; component < 6; ++component) {
+		result[component] = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpPhi, f, result[component], packed[component], half_label,
+		                      full[component], full_label);
+	}
+	return result;
+}
+
+uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
+               uint32_t grow, uint32_t sort, uint32_t extent, const Vec3& origin, const Vec3& inverse) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	std::array<uint32_t, 4> depths, indices;
+	Vec3 forward;
+	for (uint32_t axis = 0; axis < 3; ++axis)
+		forward[axis] = Binary(s, spv::OpFOrdGreaterThanEqual, b, inverse[axis], zero);
+	for (uint32_t child = 0; child < 4; ++child) {
+		const auto box = Box(s, first, second, half, child);
+		Vec3 near, far;
+		for (uint32_t axis = 0; axis < 3; ++axis) {
+			const auto lo = Binary(s, spv::OpFMul, f,
+			    Binary(s, spv::OpFSub, f, box[axis], origin[axis]), inverse[axis]);
+			const auto hi = Binary(s, spv::OpFMul, f,
+			    Binary(s, spv::OpFSub, f, box[axis + 3], origin[axis]), inverse[axis]);
+			near[axis] = Select(s, f, forward[axis], lo, hi);
+			far[axis] = Select(s, f, forward[axis], hi, lo);
+		}
+		const auto entry = EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s,
+		    EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s, near[0], near[1]), near[2]);
+		const auto exit = EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s,
+		    EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s, far[0], far[1]), far[2]);
+		// Valid hits have nonnegative exit distance. Saturate ULP growth at +infinity.
+		const auto bits = Unary(s, spv::OpBitcast, u, Select(s, f,
+		    Binary(s, spv::OpFOrdEqual, b, exit, zero), zero, exit));
+		const auto grown_bits = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+		    Binary(s, spv::OpIAdd, u, bits, grow), ConstantU32(s, 0x7f800000));
+		const auto grown = Unary(s, spv::OpBitcast, f, grown_bits);
+		auto hit = Binary(s, spv::OpLogicalAnd, b,
+		    Binary(s, spv::OpFOrdGreaterThanEqual, b, exit, zero),
+		    Binary(s, spv::OpFOrdLessThan, b, entry, extent));
+		hit = Binary(s, spv::OpLogicalAnd, b, hit,
+		    Binary(s, spv::OpFOrdLessThanEqual, b, entry, grown));
+		depths[child] = Unary(s, spv::OpBitcast, u, Select(s, f, hit,
+		    Select(s, f, Binary(s, spv::OpFOrdGreaterThan, b, entry, zero), entry, zero),
+		    ConstantF32(s, 0x7f800000)));
+		indices[child] = Select(s, u, hit, NodeWord(s, first, ConstantU32(s, child)), ConstantU32(s, ~0u));
+	}
+	// A select network keeps the original child order when sorting is disabled.
+	constexpr std::array<std::pair<uint32_t, uint32_t>, 5> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2}}};
+	for (size_t step = 0; step < network.size(); ++step) {
+		const auto [a, c] = network[step];
+		const auto swap = Binary(s, spv::OpLogicalAnd, b, sort,
+		    Binary(s, spv::OpUGreaterThan, b, depths[a], depths[c]));
+		const auto depth = depths[a], index = indices[a];
+		const auto later = [&](uint32_t position) {
+			return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) {
+				return pair.first == position || pair.second == position;
+			});
+		};
+		if (later(a)) depths[a] = Select(s, u, swap, depths[c], depth);
+		if (later(c)) depths[c] = Select(s, u, swap, depth, depths[c]);
+		indices[a] = Select(s, u, swap, indices[c], index);
+		indices[c] = Select(s, u, swap, index, indices[c]);
+	}
+	return Vector(s, TypeU32Vector(s, 4), indices);
+}
+
+} // namespace
+
+void DefineBvhIntersect(EmitterState& s) {
+	if (!s.requirements.bvh) return;
+	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
+	const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
+	s.bvh_intersect_function = s.builder.AllocateId();
+	s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
+	s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
+	                      spv::FunctionControlMaskNone, signature);
+	std::array<uint32_t, 6> args;
+	const std::array types {vec4, u, f, vec3, vec3, vec3};
+	for (uint32_t i = 0; i < args.size(); ++i) {
+		args[i] = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
+	}
+	EmitLabel(s, s.builder.AllocateId());
+	const auto descriptor = args[0], node = args[1], extent = args[2];
+	Vec3 origin, direction, inverse;
+	for (uint32_t i = 0; i < 3; ++i) {
+		origin[i] = Extract(s, f, args[3], i);
+		direction[i] = Extract(s, f, args[4], i);
+		inverse[i] = Extract(s, f, args[5], i);
+	}
+	std::array<uint32_t, 4> words;
+	for (uint32_t i = 0; i < 4; ++i) words[i] = Extract(s, u, descriptor, i);
+	const auto and_bits = [&](uint32_t value, uint32_t mask) {
+		return Binary(s, spv::OpBitwiseAnd, u, value, ConstantU32(s, mask));
+	};
+	const auto shr = [&](uint32_t value, uint32_t shift) {
+		return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
+	};
+	const auto kind = and_bits(node, 7);
+	const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
+	const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
+	const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
+	const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
+	const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
+	const auto grow = and_bits(shr(words[1], 23), 0xff);
+	const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
+	    DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
+	const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
+	const auto last_index = Binary(s, spv::OpIAdd, wide, index,
+	    Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
+	const auto address = Binary(s, spv::OpIAdd, wide, base,
+	    Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
+	const auto last_address = Binary(s, spv::OpIAdd, wide, base,
+	    Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
+	auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
+	const auto require = [&](uint32_t condition) {
+		valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
+	};
+	require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
+	require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
+	require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
+	require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
+	for (uint32_t i = 0; i < 3; ++i) {
+		for (const auto value: {origin[i], direction[i], inverse[i]})
+			require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, value)));
+		require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsInf, b, origin[i])));
+	}
+	const std::array triangle_invalid {ConstantU32(s, 0x7f800000), ConstantU32(s, 0x3f800000),
+	                                  ConstantU32(s, ~0u), ConstantU32(s, 0)};
+	const std::array box_invalid {ConstantU32(s, ~0u), ConstantU32(s, ~0u),
+	                             ConstantU32(s, ~0u), ConstantU32(s, ~0u)};
+	std::array<uint32_t, 4> invalid_words;
+	for (uint32_t i = 0; i < 4; ++i)
+		invalid_words[i] = Select(s, u, triangle, triangle_invalid[i], box_invalid[i]);
+	const auto invalid = Vector(s, vec4, invalid_words);
+	const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
+		const auto first = GetBdaPointer(s, address);
+		const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
+		});
+		const auto present = Binary(s, spv::OpLogicalAnd, b,
+		    Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
+		    Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
+		        Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
+		return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
+			const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
+			const auto merge = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+			s.builder.AddFunction(spv::OpBranchConditional, triangle, tri_label, box_label);
+			EmitLabel(s, tri_label);
+			const auto tri_value = Triangle(s, first, kind, bary, origin, direction);
+			const auto tri_exit = s.current_label;
+			s.builder.AddFunction(spv::OpBranch, merge);
+			EmitLabel(s, box_label);
+			const auto box_value = Boxes(s, first, second, half, grow, sort, extent, origin, inverse);
+			const auto box_exit = s.current_label;
+			s.builder.AddFunction(spv::OpBranch, merge);
+			EmitLabel(s, merge);
+			const auto value = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpPhi, vec4, value, tri_value, tri_exit, box_value, box_exit);
+			return value;
+		});
+	});
+	s.builder.AddFunction(spv::OpReturnValue, result);
+	s.builder.AddFunction(spv::OpFunctionEnd);
+}
+
+uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto* ray = ctx.ImageAddress(inst.Arg(1));
+	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
+		ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
+	const auto vector = [&](uint32_t first) {
+		Vec3 values;
+		for (uint32_t i = 0; i < 3; ++i)
+			values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
+		return Vector(s, TypeF32Vector(s, 3), values);
+	};
+	const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
+	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
+	const auto origin = vector(2), direction = vector(5), inverse = vector(8);
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
+	    ConstantU32CompositeZero(s, 4), [&] {
+		const auto result = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpFunctionCall, TypeU32Vector(s, 4), result,
+		    s.bvh_intersect_function, descriptor, node, extent, origin, direction, inverse);
+		return result;
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -539,19 +539,6 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 	return PackImageTexel(ctx, mem, texel);
 }
 
-spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) {
-	switch (opcode) {
-		case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
-		case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
-		case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
-		case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
-		case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
-		case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
-		case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
-		default: return spv::OpNop;
-	}
-}
-
 } // namespace
 
 void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -876,10 +863,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
 		return;
 	}
-	const auto atomic_opcode = ImageAtomicOpcode(op);
 	if (image_info.access == IR::ImageAccess::Atomic) {
 		const auto dimension = image.dimension;
-		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
+		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
 			           const auto pointer      = state.builder.AllocateId();
 			           const auto pointer_type = state.builder.Type(
 			               spv::OpTypePointer, spv::StorageClassImage, TypeU32(state));
@@ -896,12 +882,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 					                                   op == IR::ValueOpcode::ImageAtomicFMax32);
 				                               });
 			           }
-			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer,
-			                                     ConstantU32(state, spv::ScopeDevice),
-			                                     ConstantU32(state, spv::MemorySemanticsMaskNone),
-			                                     ctx.Arg(inst, 2));
-			           EmitDeviceAtomicMemoryBarrier(state);
+			           const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
+			           EmitAtomicMemoryBarrier(state, IR::ResourceKind::Image);
 			           return old;
 		           }));
 		return;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -111,6 +111,9 @@ uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
@@ -133,6 +136,9 @@ uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
 }
 inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
+EMIT_NATIVE(FPOrdEqual64, OpFOrdEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThanEqual64, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThanEqual64, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
 inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
 inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
 inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
@@ -149,6 +155,7 @@ inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPAdd64, OpFAdd, F64, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t)
 inline constexpr auto EmitFPFma64 =
     EmitGlsl<GLSLstd450Fma, IR::Type::F64, uint32_t, uint32_t, uint32_t>;
@@ -222,7 +229,7 @@ void                  EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst
 void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
-void                  EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -258,6 +265,7 @@ inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicAnd64     = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
@@ -279,6 +287,7 @@ inline constexpr auto EmitSharedAtomicFMin32    = EmitSharedFloatAtomic;
 inline constexpr auto EmitSharedAtomicFMax32    = EmitSharedFloatAtomic;
 inline constexpr auto EmitSharedAtomicSwap32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicIAdd64    = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicISub32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicInc32     = EmitSharedIncDec;
 inline constexpr auto EmitSharedAtomicDec32     = EmitSharedIncDec;
@@ -288,20 +297,25 @@ inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicOr64      = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicXor32     = EmitAtomic32;
 inline constexpr auto EmitDataAppend            = EmitAppendConsume;
 inline constexpr auto EmitDataConsume           = EmitAppendConsume;
 uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
 inline constexpr auto EmitImageQueryDimensions = EmitImage;
 inline constexpr auto EmitImageQueryLod        = EmitImage;
 inline constexpr auto EmitImageRead            = EmitImage;
 inline constexpr auto EmitImageWrite           = EmitImage;
 inline constexpr auto EmitImageSampleRaw       = EmitImage;
 inline constexpr auto EmitImageGatherRaw       = EmitImage;
+inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage;
 inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
 inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicSMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicSMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
 inline constexpr auto EmitImageAtomicOr32      = EmitImage;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -58,6 +58,7 @@ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
 const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 
 struct SpirvRequirements {
+	bool bvh                          = false;
 	bool subgroup_ballot              = false;
 	bool subgroup_shuffle             = false;
 	bool subgroup_local_invocation_id = false;
@@ -83,6 +84,20 @@ struct EmitterState {
 	Builder                                          builder;
 	const IR::Program&                               program;
 	ShaderStageInputInfo                             input_info;
+	uint32_t                                        void_type = 0;
+	uint32_t                                        bool_type = 0;
+	uint32_t                                        u32_type = 0;
+	uint32_t                                        native_u64_type = 0;
+	uint32_t                                        i32_type = 0;
+	uint32_t                                        f32_type = 0;
+	uint32_t                                        f64_type = 0;
+	uint32_t                                        u32_pair_type = 0;
+	uint32_t                                        i32_pair_type = 0;
+	uint32_t                                        function_type = 0;
+	std::array<uint32_t, 3>                          bool_vector_types {};
+	std::array<uint32_t, 3>                          u32_vector_types {};
+	std::array<uint32_t, 3>                          i32_vector_types {};
+	std::array<uint32_t, 3>                          f32_vector_types {};
 	std::array<uint32_t, 6>                          tess_variables {};
 	uint32_t                                         tess_inner_variable = 0;
 	uint32_t                                         tess_patch_base     = 0;
@@ -96,6 +111,7 @@ struct EmitterState {
 	uint32_t                                         bda_pagetable_variable  = 0;
 	uint32_t                                         fault_buffer_variable   = 0;
 	uint32_t                                         bda_pointer_function    = 0;
+	uint32_t                                         bvh_intersect_function  = 0;
 	uint32_t                                         gds_variable            = 0;
 	uint32_t                                         gds_length              = 0;
 	uint32_t                                         push_constant_variable  = 0;
@@ -420,7 +436,12 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits);
 uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
                                   uint32_t component, uint32_t raw);
 
-void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
+spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode);
+
+uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
+                             uint32_t scope);
+
+void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind);
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value);
@@ -487,6 +508,10 @@ uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 void EmitProgram(EmitterState& state);
 
 void DefineGetBdaPointer(EmitterState& state);
+void DefineBvhIntersect(EmitterState& state);
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
+uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
+uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
@@ -528,14 +553,7 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
 
 template <typename Fn>
 uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
-	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
-	const auto memory = [&] {
-		switch (kind) {
-			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
-			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
-			default: return spv::MemorySemanticsUniformMemoryMask;
-		}
-	}();
+	const auto scope     = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto preheader = state.builder.AllocateId();
 	const auto header    = state.builder.AllocateId();
 	const auto cont      = state.builder.AllocateId();
@@ -564,8 +582,7 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
 	EmitLabel(state, cont);
 	state.builder.AddFunction(spv::OpBranch, header);
 	EmitLabel(state, merge);
-	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
-	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
+	EmitAtomicMemoryBarrier(state, kind);
 	return observed;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -1,6 +1,7 @@
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <algorithm>
 
@@ -105,31 +106,23 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad
 	return result;
 }
 
-uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
-                            uint32_t high) {
-	auto& state     = ctx.state;
-	auto  immediate = static_cast<int32_t>(mem.offset);
+uint32_t AddAddressOffset(EmitterState& state, const IR::MemoryInfo& mem, uint32_t low,
+                          uint32_t& high) {
+	const auto immediate = static_cast<int32_t>(mem.offset);
+	if (immediate == 0) return low;
 	const auto immediate_low  = ConstantU32(state, static_cast<uint32_t>(immediate));
 	const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
-	low                      = AddU64Low(state, low, high, immediate_low, immediate_high, high);
+	return AddU64Low(state, low, high, immediate_low, immediate_high, high);
+}
+
+uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
+                            uint32_t high) {
+	auto& state = ctx.state;
+	low = AddAddressOffset(state, mem, low, high);
 	const auto valid = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
 	return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
 }
 
-uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
-	return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
-	                              static_cast<uint32_t>(value),
-	                              static_cast<uint32_t>(value >> 32u));
-}
-
-uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
-	const auto low64  = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
-	const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
-	                           Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
-	                           ConstantDeviceAddress(state, 32));
-	return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
-}
-
 uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
 	auto& state = ctx.state;
 	auto  low   = ctx.Arg(inst, 1);
@@ -186,17 +179,9 @@ void RecordBdaFault(EmitterState& state, uint32_t page) {
 	                          Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit));
 }
 
-uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
-	auto&      state  = ctx.state;
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
-	                          state.bda_pointer_function, address);
-	return result;
-}
-
 uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
 	auto&      state   = ctx.state;
-	const auto bda     = GetBdaPointer(ctx, address);
+	const auto bda     = GetBdaPointer(state, address);
 	const auto present =
 	    Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
 	return EmitValueOrZeroIfCondition(state, present, [&]() {
@@ -511,6 +496,50 @@ void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resour
 	                              resource.memory_access);
 }
 
+template <typename Fn>
+void ForEachLocalFlatAccess(ValueEmitContext& ctx, const IR::Inst& inst, Fn&& emit) {
+	auto& state = ctx.state;
+	auto mem = ctx.Memory(inst);
+	auto high = ctx.Arg(inst, 2);
+	const auto low = AddAddressOffset(state, mem, ctx.Arg(inst, 1), high);
+	const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), low, ConstantU32(state, 2));
+	const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), ConstantU32(state, 0),
+	    Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, 3)));
+	for (const auto kind: {IR::ResourceKind::Scratch, IR::ResourceKind::Lds}) {
+		mem.kind = kind;
+		const auto resource = PrepareMemoryResourceAccess(state, mem);
+		const auto aperture = kind == IR::ResourceKind::Scratch ? Decoder::PrivateApertureHigh
+		                                                       : Decoder::SharedApertureHigh;
+		const auto selected = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, aperture));
+		const auto valid = AndCondition(state, aligned,
+		    AndCondition(state, selected, EmitMemoryElementInBounds(state, resource, index)));
+		emit(resource, index, valid);
+	}
+}
+
+uint32_t LoadLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		uint32_t result = 0;
+		ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
+			const auto value = EmitValueOrZeroIfCondition(ctx.state, valid, [&]() {
+				return LoadWordInBounds(ctx, resource, index);
+			});
+			result = result == 0 ? value : Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), result, value);
+		});
+		return result;
+	});
+}
+
+void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
+	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
+			EmitIfCondition(ctx.state, valid, [&]() {
+				StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, inst.NumArgs() - 2));
+			});
+		});
+	});
+}
+
 void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                        const MemoryResourceAccess& resource, uint32_t data) {
 	const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
@@ -526,55 +555,6 @@ void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem)
 	});
 }
 
-spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
-	switch (opcode) {
-		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
-		case IR::ValueOpcode::BufferAtomicSwap32:
-		case IR::ValueOpcode::BufferAtomicSwap64:
-		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
-		case IR::ValueOpcode::BufferAtomicIAdd32:
-		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
-		case IR::ValueOpcode::BufferAtomicISub32:
-		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
-		case IR::ValueOpcode::BufferAtomicSMin32:
-		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
-		case IR::ValueOpcode::BufferAtomicUMin32:
-		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
-		case IR::ValueOpcode::BufferAtomicSMax32:
-		case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
-		case IR::ValueOpcode::BufferAtomicUMax32:
-		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
-		case IR::ValueOpcode::BufferAtomicAnd32:
-		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
-		case IR::ValueOpcode::BufferAtomicOr32:
-		case IR::ValueOpcode::BufferAtomicOr64:
-		case IR::ValueOpcode::SharedAtomicOr64:
-		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
-		case IR::ValueOpcode::BufferAtomicXor32:
-		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
-		default: return spv::OpNop;
-	}
-}
-
-uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
-                             uint32_t scope) {
-	const auto old = ctx.state.builder.AllocateId();
-	if (inst.GetOpcode() == IR::ValueOpcode::BufferAtomicCmpSwap32) {
-		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
-		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
-		ctx.state.builder.AddFunction(
-		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
-		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
-		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
-	} else {
-		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
-		ctx.state.builder.AddFunction(SpirvAtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old,
-		                              pointer, ConstantU32(ctx.state, scope),
-		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
-	}
-	return old;
-}
-
 template <typename Fn>
 uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
                           const IR::MemoryInfo& mem, Fn&& operation) {
@@ -961,6 +941,27 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 
 } // namespace
 
+uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
+	return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
+	                              static_cast<uint32_t>(value),
+	                              static_cast<uint32_t>(value >> 32u));
+}
+
+uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto low64  = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
+	const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
+	                           Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
+	                           ConstantDeviceAddress(state, 32));
+	return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
+}
+
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
+	                          state.bda_pointer_function, address);
+	return result;
+}
+
 void DefineGetBdaPointer(EmitterState& state) {
 	if (!state.program.info.uses_dma) {
 		return;
@@ -1024,14 +1025,7 @@ uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto scope =
 		    mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 		const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
-		if (mem.kind == IR::ResourceKind::Lds) {
-			const auto semantics =
-			    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
-			ctx.state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
-			                              ConstantU32(ctx.state, semantics));
-		} else {
-			EmitDeviceAtomicMemoryBarrier(ctx.state);
-		}
+		EmitAtomicMemoryBarrier(ctx.state, mem.kind);
 		return old;
 	});
 }
@@ -1059,13 +1053,13 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 			                                            TypeStorageBufferU64ElementPointer(state)),
 			            ConstantU32(state, spv::ScopeDevice),
 			            ConstantU32(state, spv::MemorySemanticsMaskNone), value);
-			        EmitDeviceAtomicMemoryBarrier(state);
+			        EmitAtomicMemoryBarrier(state, mem.kind);
 			        return Unary(state, spv::OpBitcast, TypeU64(state), old);
 		        });
 	    });
 }
 
-void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst) {
+void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	const auto& mem = ctx.Memory(inst);
 	EnsureLdsStorage(state);
@@ -1243,7 +1237,9 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto shared_components = IR::SharedComponentCount(op);
 	const auto address_info      = IR::AddressOpcodeInfoOf(op);
 	uint32_t   value;
-	if (buffer_components > 1u)
+	if (mem.kind == IR::ResourceKind::FlatLocal)
+		value = LoadLocalFlat(ctx, inst);
+	else if (buffer_components > 1u)
 		value = LoadWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		value = LoadWideShared(ctx, inst, shared_components);
@@ -1270,7 +1266,9 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto  buffer_components = IR::BufferComponentCount(op);
 	const auto  shared_components = IR::SharedComponentCount(op);
 	const auto  type              = inst.Arg(inst.NumArgs() - 2).GetType();
-	if (buffer_components > 1u)
+	if (mem.kind == IR::ResourceKind::FlatLocal)
+		StoreLocalFlat(ctx, inst);
+	else if (buffer_components > 1u)
 		StoreWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		StoreWideShared(ctx, inst, shared_components);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -306,11 +306,79 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 	}
 }
 
-void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
-	const auto semantics =
-	    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask;
-	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice),
-	                          ConstantU32(state, semantics));
+spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
+	switch (opcode) {
+		case IR::ValueOpcode::ImageAtomicCompareSwap32:
+		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
+		case IR::ValueOpcode::ImageAtomicSwap32:
+		case IR::ValueOpcode::BufferAtomicSwap32:
+		case IR::ValueOpcode::BufferAtomicSwap64:
+		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
+		case IR::ValueOpcode::ImageAtomicIAdd32:
+		case IR::ValueOpcode::BufferAtomicIAdd32:
+		case IR::ValueOpcode::SharedAtomicIAdd64:
+		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
+		case IR::ValueOpcode::BufferAtomicISub32:
+		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
+		case IR::ValueOpcode::ImageAtomicSMin32:
+		case IR::ValueOpcode::BufferAtomicSMin32:
+		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
+		case IR::ValueOpcode::ImageAtomicUMin32:
+		case IR::ValueOpcode::BufferAtomicUMin32:
+		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicSMax32:
+		case IR::ValueOpcode::BufferAtomicSMax32:
+		case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
+		case IR::ValueOpcode::ImageAtomicUMax32:
+		case IR::ValueOpcode::BufferAtomicUMax32:
+		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32:
+		case IR::ValueOpcode::BufferAtomicAnd32:
+		case IR::ValueOpcode::BufferAtomicAnd64:
+		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32:
+		case IR::ValueOpcode::BufferAtomicOr32:
+		case IR::ValueOpcode::BufferAtomicOr64:
+		case IR::ValueOpcode::SharedAtomicOr64:
+		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32:
+		case IR::ValueOpcode::BufferAtomicXor32:
+		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
+		default: return spv::OpNop;
+	}
+}
+
+uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
+                             uint32_t scope) {
+	const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
+	const auto old    = ctx.state.builder.AllocateId();
+	if (opcode == spv::OpAtomicCompareExchange) {
+		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
+		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		ctx.state.builder.AddFunction(
+		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
+		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
+		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
+	} else {
+		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
+		ctx.state.builder.AddFunction(opcode, TypeU32(ctx.state), old, pointer,
+		                              ConstantU32(ctx.state, scope),
+		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
+	}
+	return old;
+}
+
+void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
+	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
+	const auto memory = [&] {
+		switch (kind) {
+			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
+			default: return spv::MemorySemanticsUniformMemoryMask;
+		}
+	}();
+	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
+	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
 }
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -6,19 +6,32 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t TypeVoid(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeVoid);
+	if (state.void_type == 0) {
+		state.void_type = state.builder.Type(spv::OpTypeVoid);
+	}
+	return state.void_type;
 }
 
 uint32_t TypeBool(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeBool);
+	if (state.bool_type == 0) {
+		state.bool_type = state.builder.Type(spv::OpTypeBool);
+	}
+	return state.bool_type;
 }
 
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeBool(state), components);
+	auto& type = state.bool_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeBool(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeU32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 32, 0);
+	if (state.u32_type == 0) {
+		state.u32_type = state.builder.Type(spv::OpTypeInt, 32, 0);
+	}
+	return state.u32_type;
 }
 
 uint32_t TypeU64(EmitterState& state) {
@@ -26,33 +39,55 @@ uint32_t TypeU64(EmitterState& state) {
 }
 
 uint32_t TypeScalarU64(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 64, 0);
+	if (state.native_u64_type == 0) {
+		state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
+	}
+	return state.native_u64_type;
 }
 
 uint32_t TypeU32Pair(EmitterState& state) {
-	const auto element = TypeU32(state);
-	return state.builder.Type(spv::OpTypeStruct, element, element);
+	if (state.u32_pair_type == 0) {
+		const auto element = TypeU32(state);
+		state.u32_pair_type = state.builder.Type(spv::OpTypeStruct, element, element);
+	}
+	return state.u32_pair_type;
 }
 
 uint32_t TypeI32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 32, 1);
+	if (state.i32_type == 0) {
+		state.i32_type = state.builder.Type(spv::OpTypeInt, 32, 1);
+	}
+	return state.i32_type;
 }
 
 uint32_t TypeI32Pair(EmitterState& state) {
-	const auto element = TypeI32(state);
-	return state.builder.Type(spv::OpTypeStruct, element, element);
+	if (state.i32_pair_type == 0) {
+		const auto element = TypeI32(state);
+		state.i32_pair_type = state.builder.Type(spv::OpTypeStruct, element, element);
+	}
+	return state.i32_pair_type;
 }
 
 uint32_t TypeF32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFloat, 32);
+	if (state.f32_type == 0) {
+		state.f32_type = state.builder.Type(spv::OpTypeFloat, 32);
+	}
+	return state.f32_type;
 }
 
 uint32_t TypeF64(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFloat, 64);
+	if (state.f64_type == 0) {
+		state.f64_type = state.builder.Type(spv::OpTypeFloat, 64);
+	}
+	return state.f64_type;
 }
 
 uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
+	auto& type = state.u32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
@@ -61,11 +96,19 @@ uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
 }
 
 uint32_t TypeI32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeI32(state), components);
+	auto& type = state.i32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeI32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeF32(state), components);
+	auto& type = state.f32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeF32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee) {
@@ -73,7 +116,10 @@ uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint3
 }
 
 uint32_t TypeFunction(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFunction, TypeVoid(state));
+	if (state.function_type == 0) {
+		state.function_type = state.builder.Type(spv::OpTypeFunction, TypeVoid(state));
+	}
+	return state.function_type;
 }
 
 uint32_t StorageRuntimeArrayType(EmitterState& state) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -243,6 +243,12 @@ void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::
 }
 
 void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
+	if (ctx.half != 0 && (inst.GetOpcode() == IR::ValueOpcode::Ballot ||
+	                     inst.GetOpcode() == IR::ValueOpcode::ReadFirstLane)) {
+		// Both operations already combine both emulated halves into one whole-wave result.
+		ctx.Define(inst, ctx.other_half->Result(inst));
+		return;
+	}
 	switch (inst.GetOpcode()) {
 #define VALUE_OPCODE(name, ...)                                                                    \
 	case IR::ValueOpcode::name: return Invoke(Emit##name, ctx, inst);
@@ -679,6 +685,7 @@ void EmitProgram(EmitterState& state) {
 		}
 	}
 	DefineGetBdaPointer(state);
+	DefineBvhIntersect(state);
 	for (const auto* block: program.blocks) {
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
 			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -192,8 +192,11 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
 
 constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
     {0x0fu, Opcode::IMAGE_ATOMIC_SWAP},
+    {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP},
     {0x11u, Opcode::IMAGE_ATOMIC_ADD},
+    {0x14u, Opcode::IMAGE_ATOMIC_SMIN},
     {0x15u, Opcode::IMAGE_ATOMIC_UMIN},
+    {0x16u, Opcode::IMAGE_ATOMIC_SMAX},
     {0x17u, Opcode::IMAGE_ATOMIC_UMAX},
     {0x18u, Opcode::IMAGE_ATOMIC_AND},
     {0x19u, Opcode::IMAGE_ATOMIC_OR},
@@ -225,6 +228,7 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
+		case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
@@ -253,6 +257,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
 	}
 
 	switch (opcode) {
+		case 0xe6u: return 11u;
 		case 0x0eu: return 1u;
 		case 0x01u:
 		case 0x09u: return ImageCoordComponents(dimension) + 1u;
@@ -355,11 +360,20 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		SetUnsupported(inst, Family::MIMG, opcode,
 		               "MIMG image gather requires exactly one dmask bit");
 	}
+	if (inst.opcode == Opcode::IMAGE_ATOMIC_CMPSWAP && inst.dmask != 0x3u) {
+		SetUnsupported(inst, Family::MIMG, opcode,
+		               "MIMG image compare-and-swap requires 32-bit DMASK 0x3");
+	}
 	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
 	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
+	if (opcode == 0xe6u &&
+	    (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
+		SetUnsupported(inst, Family::MIMG, opcode,
+		               "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
+	}
 
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -61,6 +61,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
     {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
     {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
+    {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
     {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
 };
 
@@ -106,7 +107,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
-    {0x4au, Opcode::DS_OR_B64, 2, 32},
+    {0x40u, Opcode::DS_ADD_U64, 2, 32},         {0x4au, Opcode::DS_OR_B64, 2, 32},
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
     {0x77u, Opcode::DS_READ2_B64, 4, 32},        {0x78u, Opcode::DS_READ2ST64_B64, 4, 32},
@@ -168,6 +169,7 @@ bool IsDsWriteOpcode(Opcode opcode) {
 bool IsDsAtomicOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_U64:
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -46,6 +46,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x09u, Opcode::S_WQM_B32},
     {0x0au, Opcode::S_WQM_B64},
     {0x0bu, Opcode::S_BREV_B32},
+    {0x0cu, Opcode::S_BREV_B64},
     {0x0fu, Opcode::S_BCNT1_I32_B32},
     {0x10u, Opcode::S_BCNT1_I32_B64},
     {0x13u, Opcode::S_FF1_I32_B32},
@@ -108,6 +109,7 @@ constexpr OpcodeMap SOPP_OPCODE_LIST[] = {
     {0x12u, Opcode::S_TRAP},
     {0x16u, Opcode::S_TTRACEDATA},
     {0x17u, Opcode::S_CBRANCH_CDBGSYS},
+    {0x19u, Opcode::S_CBRANCH_CDBGSYS_OR_USER},
     {0x20u, Opcode::S_INST_PREFETCH},
     {0x23u, Opcode::S_WAITCNT_DEPCTR},
 };

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -184,8 +184,9 @@ std::string FormatExp(const Instruction& inst) {
 
 bool IsConditionalBranch(Opcode opcode) {
 	switch (opcode) {
-		// DevKit NGG should be disabled.
-		case Opcode::S_CBRANCH_CDBGSYS: return false;
+		// Conditional shader debugging is disabled.
+		case Opcode::S_CBRANCH_CDBGSYS:
+		case Opcode::S_CBRANCH_CDBGSYS_OR_USER: return false;
 		case Opcode::S_CBRANCH_SCC0:
 		case Opcode::S_CBRANCH_SCC1:
 		case Opcode::S_CBRANCH_VCCZ:
@@ -253,6 +254,8 @@ void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) {
 		case 125u: operand.kind = OperandKind::Null; return;
 		case 126u: operand.kind = OperandKind::ExecLo; return;
 		case 127u: operand.kind = OperandKind::ExecHi; return;
+		case 235u: operand.kind = OperandKind::SharedBase; return;
+		case 237u: operand.kind = OperandKind::PrivateBase; return;
 		case 239u: operand.kind = OperandKind::PopsExitingWaveId; return;
 		case 248u:
 			operand.kind      = OperandKind::FloatInlineConstant;
@@ -403,7 +406,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 	program.instructions.clear();
 	program.instructions.reserve(code.size());
 	program.code = code;
-	program.has_bvh = false;
 
 	std::vector<bool> branch_targets;
 	for (uint32_t word_index = 0; word_index < code.size();) {
@@ -412,10 +414,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 
 		const auto& inst = program.instructions.back();
 		word_index += inst.word_count;
-		if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) {
-			program.has_bvh = true;
-			return;
-		}
 
 		if (IsDirectBranch(inst.opcode)) {
 			const auto target_index = inst.branch_target / sizeof(uint32_t);
@@ -454,6 +452,8 @@ std::string OperandToString(const Operand& operand) {
 		case OperandKind::Scc: text = "scc"; break;
 		case OperandKind::M0: text = "m0"; break;
 		case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break;
+		case OperandKind::SharedBase: text = "shared_base"; break;
+		case OperandKind::PrivateBase: text = "private_base"; break;
 		case OperandKind::Null: text = "null"; break;
 		default: text = "unknown"; break;
 	}
@@ -504,6 +504,7 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_ABS_I32:
 		case Opcode::S_BREV_B32:
+		case Opcode::S_BREV_B64:
 		case Opcode::S_BCNT1_I32_B32:
 		case Opcode::S_FLBIT_I32_B32:
 		case Opcode::S_FF1_I32_B32:
@@ -560,6 +561,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_CBRANCH_EXECZ:
 		case Opcode::S_CBRANCH_EXECNZ:
 		case Opcode::S_CBRANCH_CDBGSYS:
+		case Opcode::S_CBRANCH_CDBGSYS_OR_USER:
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc,
 			                                               magic_enum::enum_name(inst.opcode),
 			                                               inst.branch_target));
@@ -572,9 +574,12 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_STORE:
 		case Opcode::IMAGE_STORE_MIP:
+		case Opcode::IMAGE_ATOMIC_CMPSWAP:
 		case Opcode::IMAGE_ATOMIC_SWAP:
 		case Opcode::IMAGE_ATOMIC_ADD:
+		case Opcode::IMAGE_ATOMIC_SMIN:
 		case Opcode::IMAGE_ATOMIC_UMIN:
+		case Opcode::IMAGE_ATOMIC_SMAX:
 		case Opcode::IMAGE_ATOMIC_UMAX:
 		case Opcode::IMAGE_ATOMIC_AND:
 		case Opcode::IMAGE_ATOMIC_OR:
@@ -641,6 +646,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_SMAX:
 		case Opcode::BUFFER_ATOMIC_UMAX:
 		case Opcode::BUFFER_ATOMIC_AND:
+		case Opcode::BUFFER_ATOMIC_AND_X2:
 		case Opcode::BUFFER_ATOMIC_OR:
 		case Opcode::BUFFER_ATOMIC_OR_X2:
 		case Opcode::BUFFER_ATOMIC_XOR:
@@ -663,6 +669,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_STORE_DWORDX3:
 		case Opcode::FLAT_STORE_DWORDX4:
 		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_U64:
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -10,6 +10,10 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
+// PS5 flat stack addresses use these SH_MEM_BASES aperture tags in VA[63:32].
+constexpr uint32_t PrivateApertureHigh = 0x70000000u;
+constexpr uint32_t SharedApertureHigh = 0x80000000u;
+
 enum class Family {
 	Unknown,
 	SOP1,
@@ -43,6 +47,7 @@ enum class Opcode {
 	S_ABS_I32,
 	S_ABSDIFF_I32,
 	S_BREV_B32,
+	S_BREV_B64,
 	S_BCNT1_I32_B32,
 	S_BCNT1_I32_B64,
 	S_FF1_I32_B32,
@@ -240,6 +245,7 @@ enum class Opcode {
 	V_MAD_U32_U24,
 	V_MAD_U64_U32,
 	V_FMA_F64,
+	V_ADD_F64,
 	V_MUL_F64,
 	V_FMA_F32,
 	V_FMA_F16,
@@ -355,6 +361,10 @@ enum class Opcode {
 	V_CMP_NEQ_F32,
 	V_CMP_NLT_F32,
 	V_CMP_TRU_F32,
+	V_CMP_EQ_F64,
+	V_CMP_LE_F64,
+	V_CMPX_LE_F64,
+	V_CMPX_GE_F64,
 	V_CMPX_LT_F32,
 	V_CMPX_EQ_F32,
 	V_CMPX_LE_F32,
@@ -384,12 +394,19 @@ enum class Opcode {
 	V_CMP_GT_I16,
 	V_CMP_NE_I16,
 	V_CMP_GE_I16,
+	V_CMPX_LT_I16,
+	V_CMPX_EQ_I16,
+	V_CMPX_LE_I16,
+	V_CMPX_GT_I16,
+	V_CMPX_NE_I16,
+	V_CMPX_GE_I16,
 	V_CMP_LT_F16,
 	V_CMP_EQ_F16,
 	V_CMP_LE_F16,
 	V_CMP_GT_F16,
 	V_CMP_LG_F16,
 	V_CMP_GE_F16,
+	V_CMP_NGE_F16,
 	V_CMP_NGT_F16,
 	V_CMP_NEQ_F16,
 	V_CMP_NLT_F16,
@@ -426,11 +443,16 @@ enum class Opcode {
 	V_CMP_GE_U32,
 	V_CMP_T_U32,
 	V_CMP_EQ_I64,
+	V_CMP_LT_I64,
+	V_CMP_LE_I64,
 	V_CMP_LT_U64,
 	V_CMP_EQ_U64,
+	V_CMP_LE_U64,
 	V_CMP_GT_U64,
 	V_CMP_NE_U64,
+	V_CMP_GE_U64,
 	V_CMPX_NE_I64,
+	V_CMPX_LE_U64,
 	V_CMPX_NE_U64,
 	V_CMPX_LT_U32,
 	V_CMPX_EQ_U32,
@@ -490,6 +512,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_SMAX,
 	BUFFER_ATOMIC_UMAX,
 	BUFFER_ATOMIC_AND,
+	BUFFER_ATOMIC_AND_X2,
 	BUFFER_ATOMIC_OR,
 	BUFFER_ATOMIC_OR_X2,
 	BUFFER_ATOMIC_XOR,
@@ -510,6 +533,7 @@ enum class Opcode {
 	FLAT_STORE_DWORDX3,
 	FLAT_STORE_DWORDX4,
 	DS_ADD_U32,
+	DS_ADD_U64,
 	DS_ADD_RTN_U32,
 	DS_SUB_U32,
 	DS_SUB_RTN_U32,
@@ -567,6 +591,7 @@ enum class Opcode {
 	DS_WRITE_B128,
 	DS_WRITE_ADDTID_B32,
 	DS_READ_ADDTID_B32,
+	IMAGE_BVH_INTERSECT_RAY,
 	IMAGE_GET_RESINFO,
 	IMAGE_GET_LOD,
 	IMAGE_LOAD,
@@ -574,8 +599,11 @@ enum class Opcode {
 	IMAGE_STORE,
 	IMAGE_STORE_MIP,
 	IMAGE_ATOMIC_SWAP,
+	IMAGE_ATOMIC_CMPSWAP,
 	IMAGE_ATOMIC_ADD,
+	IMAGE_ATOMIC_SMIN,
 	IMAGE_ATOMIC_UMIN,
+	IMAGE_ATOMIC_SMAX,
 	IMAGE_ATOMIC_UMAX,
 	IMAGE_ATOMIC_AND,
 	IMAGE_ATOMIC_OR,
@@ -607,6 +635,7 @@ enum class Opcode {
 	S_CBRANCH_EXECZ,
 	S_CBRANCH_EXECNZ,
 	S_CBRANCH_CDBGSYS,
+	S_CBRANCH_CDBGSYS_OR_USER,
 	S_SENDMSG,
 	S_SETREG_B32,
 	S_SLEEP,
@@ -633,6 +662,8 @@ enum class OperandKind {
 	Scc,
 	M0,
 	PopsExitingWaveId,
+	SharedBase,
+	PrivateBase,
 	Null,
 	Vgpr,
 };
@@ -744,7 +775,6 @@ struct Instruction {
 struct Program {
 	std::span<const uint32_t> code;
 	std::vector<Instruction>  instructions;
-	bool                     has_bvh = false;
 };
 
 // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -228,6 +228,8 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x19u, Opcode::V_CMPX_NGE_F32},       {0x1au, Opcode::V_CMPX_NLG_F32},
     {0x1bu, Opcode::V_CMPX_NGT_F32},       {0x1cu, Opcode::V_CMPX_NLE_F32},
     {0x1du, Opcode::V_CMPX_NEQ_F32},       {0x1eu, Opcode::V_CMPX_NLT_F32},
+    {0x22u, Opcode::V_CMP_EQ_F64, false},  {0x23u, Opcode::V_CMP_LE_F64, false},
+    {0x33u, Opcode::V_CMPX_LE_F64, false}, {0x36u, Opcode::V_CMPX_GE_F64, false},
     {0x80u, Opcode::V_CMP_F_I32},          {0x81u, Opcode::V_CMP_LT_I32},
     {0x82u, Opcode::V_CMP_EQ_I32},         {0x83u, Opcode::V_CMP_LE_I32},
     {0x84u, Opcode::V_CMP_GT_I32},         {0x85u, Opcode::V_CMP_NE_I32},
@@ -239,6 +241,9 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x92u, Opcode::V_CMPX_EQ_I32},        {0x93u, Opcode::V_CMPX_LE_I32},
     {0x94u, Opcode::V_CMPX_GT_I32},        {0x95u, Opcode::V_CMPX_NE_I32},
     {0x96u, Opcode::V_CMPX_GE_I32},        {0x98u, Opcode::V_CMPX_CLASS_F32},
+    {0x99u, Opcode::V_CMPX_LT_I16, false}, {0x9au, Opcode::V_CMPX_EQ_I16, false},
+    {0x9bu, Opcode::V_CMPX_LE_I16, false}, {0x9cu, Opcode::V_CMPX_GT_I16, false},
+    {0x9du, Opcode::V_CMPX_NE_I16, false}, {0x9eu, Opcode::V_CMPX_GE_I16, false},
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
@@ -248,16 +253,19 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
     {0xc5u, Opcode::V_CMP_NE_U32},         {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},          {0xa2u, Opcode::V_CMP_EQ_I64, false},
+    {0xa1u, Opcode::V_CMP_LT_I64, false},  {0xa3u, Opcode::V_CMP_LE_I64, false},
     {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
     {0xd2u, Opcode::V_CMPX_EQ_U32},        {0xd3u, Opcode::V_CMPX_LE_U32},
     {0xd4u, Opcode::V_CMPX_GT_U32},        {0xd5u, Opcode::V_CMPX_NE_U32},
     {0xd6u, Opcode::V_CMPX_GE_U32},        {0xe1u, Opcode::V_CMP_LT_U64, false},
-    {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe4u, Opcode::V_CMP_GT_U64, false},
-    {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe3u, Opcode::V_CMP_LE_U64, false},
+    {0xe4u, Opcode::V_CMP_GT_U64, false},  {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xe6u, Opcode::V_CMP_GE_U64, false},  {0xf3u, Opcode::V_CMPX_LE_U64, false},
     {0xf5u, Opcode::V_CMPX_NE_U64, false}, {0xc9u, Opcode::V_CMP_LT_F16},
     {0xcau, Opcode::V_CMP_EQ_F16},         {0xcbu, Opcode::V_CMP_LE_F16},
     {0xccu, Opcode::V_CMP_GT_F16},         {0xcdu, Opcode::V_CMP_LG_F16},
-    {0xceu, Opcode::V_CMP_GE_F16},         {0xebu, Opcode::V_CMP_NGT_F16},
+    {0xceu, Opcode::V_CMP_GE_F16},         {0xe9u, Opcode::V_CMP_NGE_F16},
+    {0xebu, Opcode::V_CMP_NGT_F16},
     {0xedu, Opcode::V_CMP_NEQ_F16},        {0xeeu, Opcode::V_CMP_NLT_F16},
     {0xd9u, Opcode::V_CMPX_LT_F16},        {0xdau, Opcode::V_CMPX_EQ_F16},
     {0xdbu, Opcode::V_CMPX_LE_F16},        {0xdcu, Opcode::V_CMPX_GT_F16},
@@ -279,6 +287,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x147u, Opcode::V_CUBEMA_F32},
     {0x14bu, Opcode::V_FMA_F32},
     {0x14cu, Opcode::V_FMA_F64},
+    {0x164u, Opcode::V_ADD_F64},
     {0x165u, Opcode::V_MUL_F64},
     {0x148u, Opcode::V_BFE_U32},
     {0x149u, Opcode::V_BFE_I32},
@@ -828,6 +837,10 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMP_F_F32:
 		case Opcode::V_CMP_LT_F32:
 		case Opcode::V_CMP_EQ_F32:
+		case Opcode::V_CMP_EQ_F64:
+		case Opcode::V_CMP_LE_F64:
+		case Opcode::V_CMPX_LE_F64:
+		case Opcode::V_CMPX_GE_F64:
 		case Opcode::V_CMP_LE_F32:
 		case Opcode::V_CMP_GT_F32:
 		case Opcode::V_CMP_LG_F32:
@@ -860,6 +873,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMP_GT_F16:
 		case Opcode::V_CMP_LG_F16:
 		case Opcode::V_CMP_GE_F16:
+		case Opcode::V_CMP_NGE_F16:
 		case Opcode::V_CMP_NGT_F16:
 		case Opcode::V_CMP_NEQ_F16:
 		case Opcode::V_CMP_NLT_F16:
@@ -1138,7 +1152,9 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) {
 }
 
 bool SupportsVopcSdwa(Opcode opcode) {
-	return opcode != Opcode::UNSUPPORTED;
+	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 &&
+	       opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 &&
+	       opcode != Opcode::V_CMPX_GE_F64;
 }
 
 void DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
@@ -1203,6 +1219,7 @@ void DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 
 uint32_t NativeVop3SourceCount(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
 		case Opcode::V_MUL_LO_U32:
 		case Opcode::V_MUL_HI_U32:
@@ -1354,6 +1371,7 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MAD_F32:
+		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
 		case Opcode::V_FMA_F64:
 		case Opcode::V_FMA_F32:
@@ -1459,6 +1477,8 @@ void ApplyNativeVop3SourceModifiers(Instruction& inst, uint32_t abs, uint32_t ne
 
 bool IsVopcCompareExec(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::V_CMPX_LE_F64:
+		case Opcode::V_CMPX_GE_F64:
 		case Opcode::V_CMPX_LT_F32:
 		case Opcode::V_CMPX_EQ_F32:
 		case Opcode::V_CMPX_LE_F32:
@@ -1486,7 +1506,14 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_U32:
 		case Opcode::V_CMPX_GE_U32:
 		case Opcode::V_CMPX_NE_I64:
+		case Opcode::V_CMPX_LE_U64:
 		case Opcode::V_CMPX_NE_U64:
+		case Opcode::V_CMPX_LT_I16:
+		case Opcode::V_CMPX_EQ_I16:
+		case Opcode::V_CMPX_LE_I16:
+		case Opcode::V_CMPX_GT_I16:
+		case Opcode::V_CMPX_NE_I16:
+		case Opcode::V_CMPX_GE_I16:
 		case Opcode::V_CMPX_LT_U16:
 		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -1,5 +1,7 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
+#include <bit>
+
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar,
@@ -26,9 +28,20 @@ void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool valu
 
 void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                     IR::Type type, bool scalar, bool cmpx) {
-	const auto lhs = ReadOperand(inst.src0, type);
-	const auto rhs = ReadOperand(inst.src1, type);
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx);
+	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
+	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMPX_NE_I64;
+	const auto read = [&](const Decoder::Operand& operand) {
+		// RDNA2 expands signed 64-bit integer literals by sign extension.
+		if (signed_64 && operand.kind == Decoder::OperandKind::LiteralConstant) {
+			return IR::Value(static_cast<uint64_t>(
+			    static_cast<int64_t>(std::bit_cast<int32_t>(operand.value))));
+		}
+		return ReadOperand(operand, type);
+	};
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {read(inst.src0), read(inst.src1)})), scalar,
+	                  cmpx);
 }
 
 void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -40,11 +53,13 @@ void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::Valu
 
 void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                   bool half, bool cmpx) {
+	const auto type = IR::ArgTypeOf(opcode, 0);
 	const auto lhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
+	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, type);
 	const auto rhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
-	const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
+	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, type);
+	const IR::FPCompareFlags flags{
+	    .flush_input_denorms = !half && type == IR::Type::F32 && flush_f32_inputs};
 	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
 }
 

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -558,9 +558,18 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
-	const auto result =
-	    ir.Emit(opcode, {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
-	            AddMemoryInfo(memory, inst.pc));
+	const auto data     = MemorySourceAt(inst, 0);
+	const auto flags    = AddMemoryInfo(memory, inst.pc);
+	IR::Value  result;
+	if (opcode == IR::ValueOpcode::ImageAtomicCompareSwap32) {
+		// VDATA supplies the replacement; VDATA+1 supplies the comparison.
+		result = ir.Emit(opcode,
+		                 {resource, address, ReadU32(data), ReadU32(OffsetOperand(data, 1u)),
+		                  ir.GetExec()},
+		                 flags);
+	} else {
+		result = ir.Emit(opcode, {resource, address, ReadU32(data), ir.GetExec()}, flags);
+	}
 	if (inst.glc) {
 		WriteOperand(inst.dst, result);
 	}
@@ -883,6 +892,15 @@ void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
 
 void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
+		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
+			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
+			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+			for (uint32_t component = 0; component < 4; ++component) {
+				WriteOperand(OffsetOperand(inst.dst, component),
+				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
+			}
+			return;
+		}
 		case Decoder::Opcode::S_LOAD_DWORD:
 		case Decoder::Opcode::S_LOAD_DWORDX2:
 		case Decoder::Opcode::S_LOAD_DWORDX4:
@@ -957,6 +975,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMax32);
 		case Decoder::Opcode::BUFFER_ATOMIC_AND:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd32);
+		case Decoder::Opcode::BUFFER_ATOMIC_AND_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd64);
 		case Decoder::Opcode::BUFFER_ATOMIC_OR:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr32);
 		case Decoder::Opcode::BUFFER_ATOMIC_OR_X2:
@@ -970,6 +990,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 
 		case Decoder::Opcode::DS_ADD_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false);
+		case Decoder::Opcode::DS_ADD_U64:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd64, false);
 		case Decoder::Opcode::DS_ADD_RTN_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, true);
 		case Decoder::Opcode::DS_SUB_U32:
@@ -1017,12 +1039,18 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
 
+		case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_SWAP:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32);
+		case Decoder::Opcode::IMAGE_ATOMIC_SMIN:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32);
+		case Decoder::Opcode::IMAGE_ATOMIC_SMAX:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_AND:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -155,6 +155,12 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BREV_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false,
 			                     false);
+		case O::S_BREV_B64: {
+			const auto value = ReadU32Pair(inst.src0);
+			WriteU32Pair(inst.dst, {IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[1]})),
+			                        IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[0]}))});
+			return;
+		}
 		case O::S_BCNT1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
 			                     true);
@@ -231,6 +237,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_CBRANCH_EXECZ:
 		case O::S_CBRANCH_EXECNZ:
 		case O::S_CBRANCH_CDBGSYS:
+		case O::S_CBRANCH_CDBGSYS_OR_USER:
 		case O::S_ENDPGM: return;
 		default: return FailMissingTranslation(inst);
 	}

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -135,6 +135,8 @@ IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
 		case Decoder::OperandKind::IntegerInlineConstant:
 		case Decoder::OperandKind::FloatInlineConstant: return IR::U32(IR::Value(operand.value));
 		case Decoder::OperandKind::Null:
+		case Decoder::OperandKind::SharedBase:
+		case Decoder::OperandKind::PrivateBase:
 		case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u));
 		case Decoder::OperandKind::Sgpr:
 			return ir.GetScalarReg(static_cast<IR::ScalarReg>(operand.reg));
@@ -474,6 +476,12 @@ IR::U32 Translator::ReadU32(const Decoder::Operand& operand) {
 }
 
 std::array<IR::U32, 2> Translator::ReadU32Pair(const Decoder::Operand& operand) {
+	if (operand.kind == Decoder::OperandKind::PrivateBase ||
+	    operand.kind == Decoder::OperandKind::SharedBase) {
+		return {IR::U32(IR::Value(0u)), IR::U32(IR::Value(
+		    operand.kind == Decoder::OperandKind::PrivateBase ? Decoder::PrivateApertureHigh
+		                                                      : Decoder::SharedApertureHigh))};
+	}
 	if (operand.kind == Decoder::OperandKind::ExecLo) {
 		return {ir.GetExecLo(), ir.GetExecHi()};
 	}
@@ -560,6 +568,12 @@ IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_la
 	} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
 		offset = operand.sdwa_sel == 5u ? 16u : 0u;
 	}
+	if (width == 8u && operand.sdwa_sext) {
+		// Extend the selected byte into the 16-bit operand before halfword modifiers.
+		const auto value = IR::U32(
+		    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {bits, IR::Value(offset), IR::Value(width)}));
+		return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
+	}
 	return IR::U32(
 	    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)}));
 }
@@ -567,7 +581,7 @@ IR::U32 Translator::ReadU16LaneRaw(const Decoder::Operand& operand, bool high_la
 IR::U32 Translator::ReadU16LaneAsU32(const Decoder::Operand& operand, bool high_lane,
                                      bool sign_extend) {
 	auto value = Read16LaneBits(operand, high_lane);
-	if (sign_extend || operand.sdwa_sext) {
+	if (sign_extend) {
 		value = IR::U32(
 		    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {value, IR::Value(0u), IR::Value(16u)}));
 	}
@@ -880,13 +894,35 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CVT_F64_F32:
 		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
+		case Decoder::Opcode::V_ADD_F64:
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
+		case Decoder::Opcode::V_CMP_EQ_F64:
+		case Decoder::Opcode::V_CMP_LE_F64:
+		case Decoder::Opcode::V_CMPX_LE_F64:
+		case Decoder::Opcode::V_CMPX_GE_F64:
+		case Decoder::Opcode::V_CMP_EQ_I64:
+		case Decoder::Opcode::V_CMP_LT_I64:
+		case Decoder::Opcode::V_CMP_LE_I64:
+		case Decoder::Opcode::V_CMP_LT_U64:
+		case Decoder::Opcode::V_CMP_EQ_U64:
+		case Decoder::Opcode::V_CMP_LE_U64:
+		case Decoder::Opcode::V_CMP_GT_U64:
+		case Decoder::Opcode::V_CMP_NE_U64:
+		case Decoder::Opcode::V_CMP_GE_U64:
+		case Decoder::Opcode::V_CMPX_NE_I64:
+		case Decoder::Opcode::V_CMPX_LE_U64:
+		case Decoder::Opcode::V_CMPX_NE_U64:
+			include_vector(inst.src0, 2u);
+			include_vector(inst.src1, 2u);
+			break;
 		default: break;
 	}
 	if (inst.family == Decoder::Family::DS) {
 		switch (inst.opcode) {
+			case Decoder::Opcode::DS_ADD_U64:
+			case Decoder::Opcode::DS_OR_B64:
 			case Decoder::Opcode::DS_WRITE_B64:
 			case Decoder::Opcode::DS_WRITE_B96:
 			case Decoder::Opcode::DS_WRITE_B128: include_vector(inst.src1, inst.data_dwords); break;
@@ -1120,12 +1156,13 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				return entry_ir.ISub(lhs, minimum(lhs, rhs));
 			};
 			const auto local = builtin(IR::StageInputKind::LocalInvocationIndex);
-			const auto primitive_chunk = entry_ir.IMul(builtin(IR::StageInputKind::WorkgroupId, 0),
-			                                           u32(mesh.primitives_per_group));
+			const auto group = builtin(IR::StageInputKind::WorkgroupId, 0);
+			const auto primitive_chunk = mesh.fast_launch ? group :
+			    entry_ir.IMul(group, u32(mesh.primitives_per_group));
 			const auto step  = u32(mesh.InputPrimitiveStep());
 			const auto size  = u32(mesh.InputPrimitiveSize());
-			const auto chunk = entry_ir.IMul(primitive_chunk, step);
-			const auto vertices =
+			const auto chunk = mesh.fast_launch ? group : entry_ir.IMul(primitive_chunk, step);
+			const auto vertices = mesh.fast_launch ? u32(mesh.vertices_per_group) :
 			    minimum(subtract_saturate(draw(0), chunk), u32(mesh.vertices_per_group));
 			const auto primitives = entry_ir.Select(
 			    entry_ir.ULessThan(vertices, size), u32(0),
@@ -1146,56 +1183,64 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			    entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(
 			                                                         primitive_count, u32(8)),
 			                                                     vertex_count)));
-			// GS adjacency addresses local ES records in LDS. Fans retain the draw's
-			// center in every subgroup; strip winding follows the global primitive.
-			const auto vertex = entry_ir.IMul(local, step);
-			auto       first  = vertex;
-			auto       second = u32(0);
-			auto       third  = u32(0);
-			if (mesh.InputPrimitiveSize() >= 2u) {
-				second = entry_ir.IAdd(vertex, u32(1));
+			if (mesh.fast_launch) {
+				// Fast launch broadcasts the subgroup's base vertex and instance to every lane.
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5), entry_ir.IAdd(draw(1), group));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(6),
+				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
+			} else {
+				// GS adjacency addresses local ES records in LDS. Fans retain the draw's
+				// center in every subgroup; strip winding follows the global primitive.
+				const auto vertex = entry_ir.IMul(local, step);
+				auto       first  = vertex;
+				auto       second = u32(0);
+				auto       third  = u32(0);
+				if (mesh.InputPrimitiveSize() >= 2u) {
+					second = entry_ir.IAdd(vertex, u32(1));
+				}
+				if (mesh.InputPrimitiveSize() == 3u) {
+					third = entry_ir.IAdd(vertex, u32(2));
+				}
+				auto input_vertex = entry_ir.IAdd(chunk, local);
+				if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriFan)) {
+					first = u32(0);
+					input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex);
+				} else if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriStrip)) {
+					const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1));
+					first = entry_ir.IAdd(first, parity);
+					second = entry_ir.ISub(second, parity);
+				}
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
+				                      entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
+				                                         entry_ir.ShiftLeftLogical(second, u32(18))));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
+				                      entry_ir.ShiftLeftLogical(third, u32(2)));
+				const auto index_bytes  = draw(3);
+				const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
+				const auto index_low    = draw(4);
+				const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
+				                                           entry_ir.IMul(input_vertex, index_bytes));
+				const auto index_resource = entry_ir.Emit(
+				    IR::ValueOpcode::GetAddressResource,
+				    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
+				const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
+				result.memory_info.push_back({.kind = IR::ResourceKind::Global});
+				const auto packed_index = entry_ir.Emit(
+				    IR::ValueOpcode::LoadAddressU32,
+				    {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0),
+				     entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))},
+				    IR::MemoryFlags {.index = memory_index});
+				const auto index = IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::BitFieldUExtract,
+				    {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)),
+				     entry_ir.IMul(index_bytes, u32(8))}));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
+				                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(8),
+				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
 			}
-			if (mesh.InputPrimitiveSize() == 3u) {
-				third = entry_ir.IAdd(vertex, u32(2));
-			}
-			auto input_vertex = entry_ir.IAdd(chunk, local);
-			if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriFan)) {
-				first = u32(0);
-				input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex);
-			} else if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriStrip)) {
-				const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1));
-				first = entry_ir.IAdd(first, parity);
-				second = entry_ir.ISub(second, parity);
-			}
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
-			                      entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
-			                                         entry_ir.ShiftLeftLogical(second, u32(18))));
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
-			                      entry_ir.ShiftLeftLogical(third, u32(2)));
-			const auto index_bytes  = draw(3);
-			const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
-			const auto index_low    = draw(4);
-			const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
-			                                           entry_ir.IMul(input_vertex, index_bytes));
-			const auto index_resource = entry_ir.Emit(
-			    IR::ValueOpcode::GetAddressResource,
-			    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
-			const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
-			result.memory_info.push_back({.kind = IR::ResourceKind::Global});
-			const auto packed_index = entry_ir.Emit(
-			    IR::ValueOpcode::LoadAddressU32,
-			    {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0),
-			     entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))},
-			    IR::MemoryFlags {.index = memory_index});
-			const auto index = IR::U32(entry_ir.Emit(
-			    IR::ValueOpcode::BitFieldUExtract,
-			    {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)),
-			     entry_ir.IMul(index_bytes, u32(8))}));
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
-			                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
-			entry_ir.SetVectorReg(
-			    static_cast<IR::VectorReg>(8),
-			    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
 		} else if (options.stage == ShaderType::Local) {
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(2),

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -114,18 +114,40 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
 			return;
 
+		case O::V_CMP_LT_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMP_LE_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMP_LE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMPX_LE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, true);
+			return;
+		case O::V_CMP_GE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false);
+			return;
+
 		case O::V_CMP_EQ_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
 			return;
 		case O::V_CMP_EQ_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
 			return;
+		case O::V_CMPX_EQ_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, true);
+			return;
 		case O::V_CMP_NE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
 			return;
 		case O::V_CMP_NE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
 			return;
+		case O::V_CMPX_NE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, true);
+			return;
 		case O::V_CMP_GT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
 			return;
@@ -150,21 +172,45 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_GT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
 			return;
+		case O::V_CMPX_GT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, true);
+			return;
 		case O::V_CMP_GE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
 			return;
+		case O::V_CMPX_GE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, true);
+			return;
 		case O::V_CMP_LT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
 			return;
+		case O::V_CMPX_LT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, true);
+			return;
 		case O::V_CMP_LE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
 			return;
+		case O::V_CMPX_LE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, true);
+			return;
 
 		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return;
 		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return;
 		case O::V_CMP_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
 			return;
+		case O::V_CMP_EQ_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false);
+			return;
+		case O::V_CMP_LE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, false);
+			return;
+		case O::V_CMPX_LE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, true);
+			return;
+		case O::V_CMPX_GE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, true);
+			return;
 		case O::V_CMPX_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
 			return;
@@ -267,6 +313,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMPX_LE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
 			return;
+		case O::V_CMP_NGE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true, false);
+			return;
 		case O::V_CMP_NGT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false);
 			return;
@@ -418,6 +467,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false);
 		case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true);
 		case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
+		case O::V_ADD_F64: return FloatBinary(inst, IR::ValueOpcode::FPAdd64, false);
 		case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false);
 		case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
 		case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);

src/graphics/shader/recompiler/ir/Program.cpp

@@ -142,6 +142,7 @@ CompiledShaderInfo Program::TakeCompiledInfo() && {
 	    .user_data_base  = user_data_base,
 	    .user_data_count = user_data_count,
 	    .scratch_dwords  = scratch_dwords,
+	    .has_address_writes = has_address_writes,
 	    .info            = std::move(info),
 	    .bindings        = std::move(bindings),
 	};

src/graphics/shader/recompiler/ir/ResourceSnapshot.h

@@ -3,6 +3,7 @@
 
 #include <array>
 #include <cstdint>
+#include <utility>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -34,6 +35,7 @@ struct ResourceSnapshot {
 	std::vector<DescriptorValue> samplers;
 	std::vector<uint32_t>        flattened_srt;
 	std::vector<uint32_t>        user_data;
+	std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 	UniformFill                 uniform_fill;
 };
 

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -30,6 +30,7 @@ enum class ResourceKind {
 	Buffer,
 	IndirectBuffer,
 	Flat,
+	FlatLocal,
 	Global,
 	Scratch,
 	Lds,
@@ -40,7 +41,8 @@ enum class ResourceKind {
 
 [[nodiscard]] constexpr bool IsAddressResourceKind(ResourceKind kind) {
 	return kind == ResourceKind::ScalarAddress || kind == ResourceKind::Flat ||
-	       kind == ResourceKind::Global || kind == ResourceKind::Scratch;
+	       kind == ResourceKind::FlatLocal || kind == ResourceKind::Global ||
+	       kind == ResourceKind::Scratch;
 }
 
 struct MemoryInfo {
@@ -497,6 +499,7 @@ struct ResourceBlock {
 	Value                 condition;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> sources;
+	std::vector<uint32_t> srt_reads;
 };
 
 // Stable shader metadata consumed by the renderer after native IR has been discarded.
@@ -508,6 +511,7 @@ struct CompiledShaderInfo {
 	uint32_t                      user_data_count     = 64;
 	uint32_t                      scratch_dwords      = 0;
 	uint32_t                      param_export_mask   = 0;
+	bool                          has_address_writes  = false;
 	ShaderInfo                    info;
 	BindingLayout                 bindings;
 };
@@ -562,7 +566,6 @@ struct ResourcePlan {
 	mutable std::vector<uint8_t>            visited_blocks;
 	mutable std::vector<uint32_t>           pending_blocks;
 	mutable std::vector<uint32_t>           material_keys;
-	mutable std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 };
 
 struct Program: ResourcePlan {

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -123,6 +123,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicSMax32:
 		case ValueOpcode::BufferAtomicUMax32:
 		case ValueOpcode::BufferAtomicAnd32:
+		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr32:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::BufferAtomicXor32:
@@ -135,6 +136,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
+		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
 		case ValueOpcode::StoreBufferU32x2: return 2u;
@@ -164,6 +166,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicFMax32:
 		case ValueOpcode::SharedAtomicSwap32:
 		case ValueOpcode::SharedAtomicIAdd32:
+		case ValueOpcode::SharedAtomicIAdd64:
 		case ValueOpcode::SharedAtomicISub32:
 		case ValueOpcode::SharedAtomicInc32:
 		case ValueOpcode::SharedAtomicDec32:
@@ -183,6 +186,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 
 uint32_t SharedComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
+		case ValueOpcode::SharedAtomicIAdd64:
 		case ValueOpcode::SharedAtomicOr64:
 		case ValueOpcode::LoadSharedU32x2:
 		case ValueOpcode::WriteSharedU32x2: return 2u;
@@ -216,9 +220,12 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
 			return {ImageAccess::Read, ImageResourceClass::Sampled, true};
 		case ValueOpcode::ImageWrite:
 			return {ImageAccess::Write, ImageResourceClass::Storage, false};
+		case ValueOpcode::ImageAtomicCompareSwap32:
 		case ValueOpcode::ImageAtomicSwap32:
 		case ValueOpcode::ImageAtomicIAdd32:
+		case ValueOpcode::ImageAtomicSMin32:
 		case ValueOpcode::ImageAtomicUMin32:
+		case ValueOpcode::ImageAtomicSMax32:
 		case ValueOpcode::ImageAtomicUMax32:
 		case ValueOpcode::ImageAtomicAnd32:
 		case ValueOpcode::ImageAtomicOr32:

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -137,6 +137,9 @@ VALUE_OPCODE(SMedTri32, U32, U32, U32, U32)
 VALUE_OPCODE(UMedTri32, U32, U32, U32, U32)
 VALUE_OPCODE(SLessThan32, U1, U32, U32)
 VALUE_OPCODE(SLessThan64, U1, U64, U64)
+VALUE_OPCODE(SLessThanEqual64, U1, U64, U64)
+VALUE_OPCODE(ULessThanEqual64, U1, U64, U64)
+VALUE_OPCODE(UGreaterThanEqual64, U1, U64, U64)
 VALUE_OPCODE(ULessThan32, U1, U32, U32)
 VALUE_OPCODE(ULessThan64, U1, U64, U64)
 VALUE_OPCODE(IEqual32, U1, U32, U32)
@@ -155,6 +158,9 @@ VALUE_OPCODE(LogicalAnd, U1, U1, U1)
 VALUE_OPCODE(LogicalXor, U1, U1, U1)
 VALUE_OPCODE(LogicalNot, U1, U1)
 VALUE_OPCODE(FPOrdEqual32, U1, F32, F32)
+VALUE_OPCODE(FPOrdEqual64, U1, F64, F64)
+VALUE_OPCODE(FPOrdLessThanEqual64, U1, F64, F64)
+VALUE_OPCODE(FPOrdGreaterThanEqual64, U1, F64, F64)
 VALUE_OPCODE(FPUnordEqual32, U1, F32, F32)
 VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32)
 VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32)
@@ -173,6 +179,7 @@ VALUE_OPCODE(FPAdd32, F32, F32, F32)
 VALUE_OPCODE(FPSub32, F32, F32, F32)
 VALUE_OPCODE(FPFma32, F32, F32, F32, F32)
 VALUE_OPCODE(FPMul32, F32, F32, F32)
+VALUE_OPCODE(FPAdd64, F64, F64, F64)
 VALUE_OPCODE(FPMul64, F64, F64, F64)
 VALUE_OPCODE(FPFma64, F64, F64, F64, F64)
 VALUE_OPCODE(FPRecip64, F64, F64)
@@ -245,6 +252,7 @@ VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicAnd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicOr64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -267,6 +275,7 @@ VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1)
 VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicInc32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicDec32, U32, U32, U32, U1)
@@ -283,14 +292,18 @@ VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
+VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
 VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
 VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
 VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
+VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1)
 VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <bit>
 #include <cstdint>
+#include <unordered_map>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -575,6 +576,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 		case ValueOpcode::UGreaterThan64:
 			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; });
 			return;
+		case ValueOpcode::ULessThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; });
+			return;
+		case ValueOpcode::UGreaterThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; });
+			return;
+		case ValueOpcode::SLessThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) {
+				return std::bit_cast<int64_t>(a) <= std::bit_cast<int64_t>(b);
+			});
+			return;
 		case ValueOpcode::SLessThan64:
 			FoldU64Compare(inst, [](uint64_t a, uint64_t b) {
 				return std::bit_cast<int64_t>(a) < std::bit_cast<int64_t>(b);
@@ -649,12 +661,110 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 	}
 }
 
+// In wave32, extracting this invocation's bit from a ballot recovers its predicate.
+// Keep that identity through scalar EXEC operations and their loop-carried mask Phis.
+class LaneMaskProjection {
+public:
+	explicit LaneMaskProjection(std::unordered_set<Inst*>& lowered_ancillary)
+	    : m_lowered_ancillary(lowered_ancillary) {}
+
+	void Fold(Inst& inst) {
+		if (inst.GetOpcode() != ValueOpcode::INotEqual32 || !Immediate(Arg(inst, 1), 0u)) return;
+		const auto* bit = Arg(inst, 0).TryInstruction();
+		if (bit == nullptr || bit->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
+		    !Immediate(Arg(*bit, 1), 1u)) return;
+		const auto* shift = Arg(*bit, 0).TryInstruction();
+		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftRightLogical32) return;
+		const auto* index = Arg(*shift, 1).TryInstruction();
+		if (index == nullptr || index->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
+		    !Immediate(Arg(*index, 1), 31u)) return;
+		const auto* lane = Arg(*index, 0).TryInstruction();
+		if (lane == nullptr || lane->GetOpcode() != ValueOpcode::LaneId) return;
+		m_visited.clear();
+		m_grounded = false;
+		if (CanProject(Arg(*shift, 0)) && m_grounded) Replace(inst, Project(Arg(*shift, 0)));
+	}
+
+private:
+	static bool Immediate(Value value, uint32_t expected) {
+		return IsImmediate(value, Type::U32) && value.U32() == expected;
+	}
+
+	static Value BallotPredicate(const Inst& inst) {
+		if (inst.GetOpcode() != ValueOpcode::CompositeExtractU32x4 ||
+		    !Immediate(Arg(inst, 1), 0u)) return {};
+		const auto* source = Arg(inst, 0).TryInstruction();
+		return source != nullptr && source->GetOpcode() == ValueOpcode::Ballot
+		           ? Arg(*source, 0) : Value {};
+	}
+
+	bool CanProject(Value value) {
+		if (value.IsImmediate()) {
+			const bool supported = Immediate(value, 0u) || Immediate(value, UINT32_MAX);
+			m_grounded |= supported;
+			return supported;
+		}
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return false;
+		if (m_values.contains(inst) || !BallotPredicate(*inst).IsEmpty()) {
+			m_grounded = true;
+			return true;
+		}
+		if (!m_visited.insert(inst).second) return true;
+		const auto op = inst->GetOpcode();
+		if (op != ValueOpcode::BitwiseAnd32 && op != ValueOpcode::BitwiseOr32 &&
+		    op != ValueOpcode::BitwiseNot32 && op != ValueOpcode::SelectU32 && op != ValueOpcode::Phi)
+			return false;
+		for (size_t arg = op == ValueOpcode::SelectU32 ? 1u : 0u; arg < inst->NumArgs(); ++arg) {
+			if (!CanProject(Arg(*inst, arg))) return false;
+		}
+		return inst->NumArgs() != 0;
+	}
+
+	Value Project(Value value) {
+		if (value.IsImmediate()) return Value(value.U32() != 0u);
+		auto* source = value.TryInstruction();
+		if (const auto found = m_values.find(source); found != m_values.end()) return found->second.Resolve();
+		if (const auto predicate = BallotPredicate(*source); !predicate.IsEmpty()) return predicate;
+		auto* block = source->Parent();
+		const auto where = std::find_if(block->begin(), block->end(),
+		                               [source](const Inst& candidate) { return &candidate == source; });
+		const auto op = source->GetOpcode() == ValueOpcode::BitwiseAnd32 ? ValueOpcode::LogicalAnd
+		              : source->GetOpcode() == ValueOpcode::BitwiseOr32 ? ValueOpcode::LogicalOr
+		              : source->GetOpcode() == ValueOpcode::BitwiseNot32 ? ValueOpcode::LogicalNot
+		              : source->GetOpcode() == ValueOpcode::SelectU32 ? ValueOpcode::SelectU1
+		                                                               : ValueOpcode::Phi;
+		auto result = op == ValueOpcode::Phi ? block->PrependNewInst(where, op)
+		            : op == ValueOpcode::LogicalNot
+		                ? block->PrependNewInst(where, op, {Value(false)})
+		            : op == ValueOpcode::SelectU1
+		                ? block->PrependNewInst(where, op, {Arg(*source, 0), Value(false), Value(false)})
+		                : block->PrependNewInst(where, op, {Value(false), Value(false)});
+		m_values.emplace(source, Value(&*result));
+		if (op == ValueOpcode::Phi) result->SetFlags(Type::U1);
+		for (size_t arg = op == ValueOpcode::SelectU1 ? 1u : 0u; arg < source->NumArgs(); ++arg) {
+			const auto projected = Project(Arg(*source, arg));
+			if (op == ValueOpcode::Phi) result->AddPhiOperand(source->PhiBlock(arg), projected);
+			else result->SetArg(arg, projected);
+		}
+		FoldInstruction(*block, result, m_lowered_ancillary);
+		return Value(&*result).Resolve();
+	}
+
+	std::unordered_set<Inst*>& m_lowered_ancillary;
+	std::unordered_set<const Inst*> m_visited;
+	std::unordered_map<const Inst*, Value> m_values;
+	bool m_grounded = false;
+};
+
 } // namespace
 
-void ConstantPropagationPass(const BlockList& blocks) {
+void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size) {
 	std::unordered_set<Inst*> lowered_ancillary;
+	LaneMaskProjection mask_projection(lowered_ancillary);
 	for (auto* block: blocks) {
 		for (auto inst = block->begin(); inst != block->end(); ++inst) {
+			if (wave_size == 32u) mask_projection.Fold(*inst);
 			FoldInstruction(*block, inst, lowered_ancillary);
 		}
 	}

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h

@@ -4,6 +4,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-void ConstantPropagationPass(const BlockList& blocks);
+void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size = 64);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -157,34 +157,15 @@ bool CaptureStrictRead(void* userdata, uint64_t address, std::span<uint32_t> val
 
 bool CaptureOrdinaryRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
 	auto& capture = *static_cast<ReadCapture*>(userdata);
-	if (!capture.source.read_memory(capture.source.userdata, address, values)) {
-		return false;
+	if (capture.source.read_memory != nullptr) {
+		if (!capture.source.read_memory(capture.source.userdata, address, values)) return false;
+	} else {
+		std::memcpy(values.data(), reinterpret_cast<const void*>(address), values.size_bytes());
 	}
 	capture.ranges.emplace_back(address, values.size_bytes());
 	return true;
 }
 
-bool WrittenBuffersDisjoint(const ResourcePlan& program, const ResourceSnapshot& snapshot,
-                            std::span<const std::pair<uint64_t, uint64_t>> reads) {
-	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!program.info.buffers[i].written) continue;
-		ShaderBufferResource buffer;
-		if (!DecodeBufferDescriptor(snapshot.buffers[i], buffer)) return false;
-		const auto base = buffer.Base48();
-		const auto size = buffer.GetSize();
-		if (size == 0u) continue;
-		if (size - 1u > AddressMask - base) return false;
-		for (const auto [address, bytes]: reads) {
-			if (bytes == 0u) continue;
-			if (address > AddressMask || bytes - 1u > AddressMask - address ||
-			    (address <= base ? base - address < bytes : address - base < size)) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
 const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
 	if (source >= program.descriptor_sources.size()) {
 		return nullptr;
@@ -638,9 +619,10 @@ bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan&
 	return true;
 }
 
-static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
-	// Any shader write may alias a scalar predicate read, including on a later loop visit.
-	if (program.blocks.size() != program.block_info.size() || HasShaderMemoryWrites(program)) {
+template <typename Predicate>
+static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, const Predicate& predicate) {
+	// The renderer checks captured scalar reads against final buffer and image write ranges.
+	if (program.blocks.size() != program.block_info.size() || program.has_address_writes) {
 		return {};
 	}
 	std::unordered_map<uint32_t, uint32_t> indices;
@@ -659,9 +641,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 			case CFG::TerminatorKind::Branch: successors.push_back(terminator.true_block); break;
 			case CFG::TerminatorKind::ConditionalBranch:
 				successors = {terminator.true_block, terminator.false_block};
-				if (ValidateRuntimeValue(program, info.condition, RuntimeValueType::Integer)) {
-					block.condition = info.condition;
-				}
+				block.condition = info.condition;
 				break;
 			case CFG::TerminatorKind::IndirectBranch:
 				successors = terminator.indirect_targets;
@@ -678,13 +658,17 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 		}
 		for (const auto& inst: *program.blocks[i]) {
 			const auto op     = inst.GetOpcode();
+			if (op == ValueOpcode::ReadConst) {
+				block.srt_reads.push_back(inst.Arg(1).Resolve().U32());
+				continue;
+			}
 			const auto buffer = BufferAccessOf(op);
 			const auto image  = ImageOpcodeInfoOf(op);
 			if (buffer == BufferAccess::None && image.access == ImageAccess::None) {
 				continue;
 			}
 			const auto& memory = program.memory_info.at(inst.Flags<MemoryFlags>().index);
-			if (memory.planning_only) {
+			if (memory.planning_only || memory.kind == ResourceKind::IndirectBuffer) {
 				continue;
 			}
 			if (buffer != BufferAccess::None) {
@@ -700,6 +684,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 		block.sources.erase(std::unique(block.sources.begin(), block.sources.end()),
 		                    block.sources.end());
 	}
+	for (auto& block: blocks) block.condition = predicate(block.condition);
 	if (std::ranges::none_of(
 	        blocks, [](const ResourceBlock& block) { return !block.condition.IsEmpty(); })) {
 		return {};
@@ -914,10 +899,38 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	for (const auto& read: program.srt_reads) {
 		plan.srt_reads.push_back({Clone(read.value), read.flat_offset});
 	}
-	plan.control_flow = ResourceControlFlow(program);
-	for (auto& block: plan.control_flow) {
-		block.condition = Clone(block.condition);
-	}
+	// A proven uniform factor can decide a branch even when its other lanes are unknown.
+	// Keep only that Boolean structure, never the varying shader dependency graph.
+	Value unknown;
+	std::function<Value(Value)> ClonePredicate = [&](Value value) -> Value {
+		value = value.Resolve();
+		if (value.IsEmpty()) return {};
+		if (ValidateRuntimeValue(program, value, RuntimeValueType::Integer)) return Clone(value);
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return {};
+		const auto op = inst->GetOpcode();
+		if (op == ValueOpcode::ConditionRef || op == ValueOpcode::LogicalNot) {
+			const auto operand = ClonePredicate(inst->Arg(0));
+			if (operand.IsEmpty() || op == ValueOpcode::ConditionRef) return operand;
+			auto& node = plan.value_storage.emplace_back(op);
+			node.SetArg(0, operand);
+			return Value(&node);
+		}
+		if (op != ValueOpcode::LogicalAnd && op != ValueOpcode::LogicalOr) return {};
+		auto left = ClonePredicate(inst->Arg(0));
+		auto right = ClonePredicate(inst->Arg(1));
+		if (left.IsEmpty() && right.IsEmpty()) return {};
+		if (left.IsEmpty() || right.IsEmpty()) {
+			if (unknown.IsEmpty()) unknown = Value(&plan.value_storage.emplace_back(ValueOpcode::UndefU1));
+			if (left.IsEmpty()) left = unknown;
+			if (right.IsEmpty()) right = unknown;
+		}
+		auto& node = plan.value_storage.emplace_back(op);
+		node.SetArg(0, left);
+		node.SetArg(1, right);
+		return Value(&node);
+	};
+	plan.control_flow = ResourceControlFlow(program, ClonePredicate);
 	plan.uniform_fill = AnalyzeUniformFill(program);
 	for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
@@ -936,12 +949,15 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
 		                   plan.clean_flat_slots);
 	}
-	if (masked_image) {
-		plan.resource_tracking_complete &= !program.has_address_writes &&
-		    !std::ranges::any_of(plan.info.images, &ImageResource::written);
-		plan.capture_specialization_reads =
-		    std::ranges::any_of(plan.info.buffers, &BufferResource::written);
+	plan.capture_specialization_reads = masked_image || !plan.control_flow.empty();
+	// Writable descriptor addresses must come from clean backing for the alias proof.
+	for (const auto& buffer: plan.info.buffers) {
+		if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
 	}
+	for (const auto& image: plan.info.images) {
+		if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
+	}
+	if (masked_image) plan.resource_tracking_complete &= !program.has_address_writes;
 	return plan;
 }
 
@@ -952,21 +968,22 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		return false;
 	}
 	const bool capture_reads = program.capture_specialization_reads;
-	auto& reads = program.specialization_reads;
+	auto& reads = snapshot.specialization_reads;
+	reads.clear();
 	ReadCapture capture {runtime, reads};
 	SrtRuntime observed = runtime;
 	if (capture_reads) {
-		reads.clear();
 		observed.userdata = &capture;
-		observed.read_specialization_memory = CaptureStrictRead;
-		if (observed.read_memory != nullptr) observed.read_memory = CaptureOrdinaryRead;
+		observed.read_specialization_memory = runtime.read_specialization_memory != nullptr
+		                                         ? CaptureStrictRead : nullptr;
+		observed.read_memory = CaptureOrdinaryRead;
 	}
 	SrtWalker clean(program, CleanRuntime(observed));
 	SrtWalker walker(program, observed, program.clean_flat_slots, &clean);
-	const auto active = clean.FindActiveSources();
 	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
 		return false;
 	}
+	const auto active = std::span<const uint8_t>(program.active_sources);
 	snapshot.uniform_fill = {};
 	const auto& fill = program.uniform_fill;
 	const auto words = fill.fill.words;
@@ -979,12 +996,12 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		snapshot.uniform_fill = fill.fill;
 		snapshot.uniform_fill.value = stored[0];
 	}
-	const auto evaluate = [&](uint32_t source, DescriptorValue& value) {
+	const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
 		if (source >= program.descriptor_sources.size()) {
 			return false;
 		}
 		if (active.empty() || active[source]) {
-			return walker.EvaluateDescriptor(source, value);
+			return (written ? clean : walker).EvaluateDescriptor(source, value);
 		}
 		value = {};
 		value.dword_count = program.descriptor_sources[source].dword_count;
@@ -992,19 +1009,11 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	};
 	snapshot.buffers.resize(program.info.buffers.size());
 	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i])) {
+		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i],
+		              program.info.buffers[i].written)) {
 			return false;
 		}
 	}
-	if (capture_reads) {
-		for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-			const auto& buffer = program.info.buffers[i];
-			if (!buffer.written || (!active.empty() && !active[buffer.source])) continue;
-			DescriptorValue strict;
-			if (!clean.EvaluateDescriptor(buffer.source, strict) ||
-			    strict != snapshot.buffers[i]) return false;
-		}
-	}
 	snapshot.images.resize(program.info.images.size());
 	specialization.images.resize(program.info.images.size());
 	for (uint32_t i = 0; i < program.info.images.size(); ++i) {
@@ -1040,7 +1049,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 				return false;
 			}
 		} else {
-			if (!evaluate(image.source, snapshot.images[i])) {
+			if (!evaluate(image.source, snapshot.images[i], image.written)) {
 				return false;
 			}
 			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
@@ -1054,7 +1063,6 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			return false;
 		}
 	}
-	if (capture_reads && !WrittenBuffersDisjoint(program, snapshot, reads)) return false;
 	snapshot.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
 	return BuildResourceSpecialization(program, snapshot, specialization);
 }

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -44,6 +44,34 @@ uint32_t PossibleU32Bits(Value value) {
 	}
 }
 
+bool IsNoncanonicalFlatAddress(Value high, Value active) {
+	high = high.Resolve();
+	active = active.Resolve();
+	// A predicated VGPR write supplies its new value on exactly the lanes which
+	// execute this access. Do not discard a selection under a different mask.
+	while (const auto* select = high.TryInstruction()) {
+		if (select->GetOpcode() != ValueOpcode::SelectU32 || select->Arg(0).Resolve() != active) break;
+		high = select->Arg(1).Resolve();
+	}
+	uint32_t lower = 0, upper = UINT32_MAX;
+	if (high.IsImmediate() && high.GetType() == Type::U32) {
+		lower = upper = high.U32();
+	} else if (const auto* inst = high.TryInstruction();
+	           inst != nullptr && inst->GetOpcode() == ValueOpcode::UMedTri32) {
+		// The median lies between any two constant inputs, regardless of the third.
+		uint32_t count = 0;
+		for (size_t index = 0; index < inst->NumArgs(); ++index) {
+			const auto value = inst->Arg(index).Resolve();
+			if (!value.IsImmediate() || value.GetType() != Type::U32) continue;
+			if (count++ == 0) lower = upper = value.U32();
+			else { lower = std::min(lower, value.U32()); upper = std::max(upper, value.U32()); }
+		}
+		if (count < 2) return false;
+	}
+	// Leave one high DWORD on each side for the signed instruction offset/carry.
+	return lower > 0x00008000u && upper < 0xffff7fffu;
+}
+
 Value CanonicalizeSampleAdjustDword3(Value value) {
 	for (;;) {
 		value            = value.Resolve();
@@ -1649,6 +1677,10 @@ private:
 
 	void Collect(Inst& inst) {
 		const auto op           = inst.GetOpcode();
+		if (op == ValueOpcode::BvhIntersect) {
+			m_info.uses_dma = true;
+			return;
+		}
 		const auto buffer       = BufferAccessOf(op);
 		const auto address_info = AddressOpcodeInfoOf(op);
 		const auto image_info   = ImageOpcodeInfoOf(op);
@@ -1695,6 +1727,15 @@ private:
 			if (!IsAddressResourceKind(memory.kind)) {
 				Fail(flags.pc, "address operation has invalid resource kind");
 			}
+			if (memory.kind == ResourceKind::Flat && memory.address_is_full &&
+			    IsNoncanonicalFlatAddress(inst.Arg(2), inst.Arg(inst.NumArgs() - 1))) {
+				ValidateAddressHandle(inst.Arg(0), flags.pc);
+				if (memory.data_bits != 32u || m_program.scratch_dwords == 0) {
+					Fail(flags.pc, "local FLAT access requires DWORD data and per-thread scratch storage");
+				}
+				m_program.memory_info[flags.index].kind = ResourceKind::FlatLocal;
+				return;
+			}
 			if (memory.kind == ResourceKind::Scratch) {
 				handle = inst.Arg(0).Resolve().TryInstruction();
 				if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource ||

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -795,18 +795,26 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
-		case ValueOpcode::LogicalAnd:
-			if (binary()) {
-				result = (a != 0u) && (b != 0u);
+		case ValueOpcode::LogicalAnd: {
+			const bool left = Arg(inst, 0, a);
+			if (left && a == 0u) {
+				result = 0u;
 				return true;
 			}
-			return false;
-		case ValueOpcode::LogicalOr:
-			if (binary()) {
-				result = (a != 0u) || (b != 0u);
+			if (!Arg(inst, 1, b) || (b != 0u && !left)) return false;
+			result = b != 0u;
+			return true;
+		}
+		case ValueOpcode::LogicalOr: {
+			const bool left = Arg(inst, 0, a);
+			if (left && a != 0u) {
+				result = 1u;
 				return true;
 			}
-			return false;
+			if (!Arg(inst, 1, b) || (b == 0u && !left)) return false;
+			result = b != 0u;
+			return true;
+		}
 		case ValueOpcode::LogicalXor:
 			if (binary()) {
 				result = (a != 0u) != (b != 0u);
@@ -844,16 +852,31 @@ bool SrtWalker::EvaluateDescriptor(uint32_t source, DescriptorValue& result) {
 	return true;
 }
 
-std::span<const uint8_t> SrtWalker::FindActiveSources() {
-	if (m_program.control_flow.empty()) {
-		return {};
-	}
+bool SrtWalker::RefreshFlatBuffer(std::vector<uint32_t>& flat) {
+	if (!m_program.srt_plan_complete) return false;
+	const auto refresh = [&](uint32_t slot) {
+		if (slot >= m_program.srt_reads.size()) return false;
+		const auto& read = m_program.srt_reads[slot];
+		const bool clean = read.flat_offset < m_clean_flat_slots.size() &&
+		                   m_clean_flat_slots[read.flat_offset] != 0u;
+		if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr))
+			return false;
+		auto& evaluator = clean ? *m_clean_evaluator : *this;
+		return read.flat_offset < flat.size() && evaluator.Evaluate(read.value, flat[read.flat_offset]);
+	};
 	auto& active = m_program.active_sources;
+	if (m_program.control_flow.empty()) {
+		active.clear();
+		flat.resize(m_program.srt_reads.size());
+		for (uint32_t slot = 0; slot < m_program.srt_reads.size(); ++slot) {
+			if (!refresh(slot)) return false;
+		}
+		return true;
+	}
+	flat.assign(m_program.srt_reads.size(), 0u);
 	active.assign(m_program.descriptor_sources.size(), 1u);
 	for (const auto& block: m_program.control_flow) {
-		for (const auto source: block.sources) {
-			active.at(source) = 0u;
-		}
+		for (const auto source: block.sources) active.at(source) = 0u;
 	}
 	auto& visited = m_program.visited_blocks;
 	auto& pending = m_program.pending_blocks;
@@ -863,41 +886,22 @@ std::span<const uint8_t> SrtWalker::FindActiveSources() {
 	while (!pending.empty()) {
 		const auto index = pending.back();
 		pending.pop_back();
-		if (visited.at(index)) {
-			continue;
-		}
+		if (visited.at(index)) continue;
 		visited[index] = 1u;
 		const auto& block = m_program.control_flow[index];
-		for (const auto source: block.sources) {
-			active[source] = 1u;
+		for (const auto source: block.sources) active[source] = 1u;
+		for (const auto slot: block.srt_reads) {
+			if (!refresh(slot)) return false;
 		}
 		uint32_t condition = 0;
+		auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this;
 		if (!block.condition.IsEmpty() && m_runtime.read_specialization_memory != nullptr &&
-		    Evaluate(block.condition, condition)) {
+		    predicate.Evaluate(block.condition, condition)) {
 			pending.push_back(block.successors[condition != 0u ? 0u : 1u]);
 		} else {
 			pending.insert(pending.end(), block.successors.begin(), block.successors.end());
 		}
 	}
-	return active;
-}
-
-bool SrtWalker::RefreshFlatBuffer(std::vector<uint32_t>& flat) {
-	if (!m_program.srt_plan_complete) {
-		return false;
-	}
-	flat.resize(m_program.srt_reads.size());
-	for (const auto& read: m_program.srt_reads) {
-		const bool clean = read.flat_offset < m_clean_flat_slots.size() &&
-		                   m_clean_flat_slots[read.flat_offset] != 0u;
-		if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr)) {
-			return false;
-		}
-		auto& evaluator = clean ? *m_clean_evaluator : *this;
-		if (read.flat_offset >= flat.size() || !evaluator.Evaluate(read.value, flat[read.flat_offset])) {
-			return false;
-		}
-	}
 	return true;
 }
 

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -38,8 +38,7 @@ public:
 
 	bool Evaluate(Value value, uint32_t& result);
 	bool EvaluateDescriptor(uint32_t source, DescriptorValue& result);
-	// An empty span means that all sources are active.
-	std::span<const uint8_t> FindActiveSources();
+	// Refreshes reachable scalar reads and active descriptor sources in one walk.
 	bool RefreshFlatBuffer(std::vector<uint32_t>& flat);
 
 private:

src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp

@@ -165,7 +165,6 @@ Value InitialValue(M0Tag) {
 
 enum class ReadStep { Start, SetValue, PushPhiArgument };
 
-template <typename T>
 struct ReadState {
 	Block*   block = nullptr;
 	Value    result;
@@ -183,8 +182,8 @@ public:
 
 	template <typename T>
 	Value Read(T variable, Block* root) {
-		std::vector<ReadState<T>> stack {{}, {.block = root}};
-		const auto                prepare_phi = [&]() {
+		stack.assign({{}, {.block = root}});
+		const auto prepare_phi = [&]() {
 			auto&      state        = stack.back();
 			const auto predecessors = state.block->ImmPredecessors();
 			if (state.pred == predecessors.size()) {
@@ -290,6 +289,7 @@ private:
 
 	std::unordered_map<Block*, std::map<Variable, Inst*>> incomplete_phis;
 	DefTable                                              definitions;
+	std::vector<ReadState>                               stack;
 };
 
 void VisitInstruction(Pass& pass, Block* block, Inst& inst) {

src/graphics/shader/shader.cpp

@@ -41,44 +41,33 @@
 
 namespace Libs::Graphics {
 
-struct ShaderBinaryInfo {
-	uint8_t  signature[7];
-	uint8_t  version;
-	uint32_t pssl_or_cg  : 1;
-	uint32_t cached      : 1;
-	uint32_t type        : 4;
-	uint32_t source_type : 2;
-	uint32_t length      : 24;
-	uint8_t  chunk_usage_base_offset_dw;
-	uint8_t  num_input_usage_slots;
-	uint8_t  is_srt                 : 1;
-	uint8_t  is_srt_used_info_valid : 1;
-	uint8_t  is_extended_usage_info : 1;
-	uint8_t  reserved2              : 5;
-	uint8_t  reserved3;
-	uint32_t hash0;
-	uint32_t hash1;
-	uint32_t crc32;
+struct ShaderMapEntry {
+	ShaderMappedData data;
+	uint64_t hash;
 };
 
-static std::unique_ptr<std::unordered_map<uint64_t, ShaderMappedData>> g_shader_map;
+static std::unique_ptr<std::unordered_map<uint64_t, ShaderMapEntry>> g_shader_map;
 static std::mutex                                                      g_shader_map_mutex;
 
 void ShaderInit() {
 	EXIT_IF(g_shader_map != nullptr);
 
-	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMappedData>>();
+	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMapEntry>>();
 }
 
 void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 	EXIT_IF(g_shader_map == nullptr);
 
+	if (addr == 0 || data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
+		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n",
+		     addr, data.code_size_bytes);
+	}
+	const auto hash = XXH3_64bits(reinterpret_cast<const void*>(addr), data.code_size_bytes);
 	std::scoped_lock lock(g_shader_map_mutex);
-
-	(*g_shader_map)[addr] = data;
+	(*g_shader_map)[addr] = {data, hash};
 }
 
-static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
+static ShaderMapEntry ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT_IF(g_shader_map == nullptr);
 
 	std::scoped_lock lock(g_shader_map_mutex);
@@ -90,37 +79,15 @@ static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT("%s shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, addr);
 }
 
-static const ShaderBinaryInfo* GetBinaryInfo(const uint32_t* code) {
-	EXIT_IF(code == nullptr);
-
-	if (code[0] == 0xBEEB03FF) {
-		return reinterpret_cast<const ShaderBinaryInfo*>(code +
-		                                                 static_cast<size_t>(code[1] + 1) * 2);
-	}
-
-	return nullptr;
-}
-
-static uint64_t GetDeclaredShaderHash(uint64_t shader_addr) {
-	const auto* header = GetBinaryInfo(reinterpret_cast<const uint32_t*>(shader_addr));
-	return header != nullptr ? (static_cast<uint64_t>(header->hash1) << 32u) | header->hash0 : 0;
-}
-
-static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash,
+static ShaderParams GetShaderParams(uint64_t shader_addr, uint64_t hash,
 	                                std::span<const uint32_t> user_data,
 	                                const ShaderMappedData& data, uint32_t user_data_base = 0) {
-	if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
-		EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64
-		     " has invalid AGC shader_size=0x%08" PRIx32 "\n",
-		     label, declared_hash, shader_addr, data.code_size_bytes);
-	}
 	const auto code_words = data.code_size_bytes / sizeof(uint32_t);
 	const auto code = std::span {reinterpret_cast<const uint32_t*>(shader_addr), code_words};
 	ShaderParams params {
 	    .code            = code,
 	    .user_data_count = static_cast<uint32_t>(user_data.size()) + user_data_base,
-	    .hash            = declared_hash != 0 ? declared_hash
-	                                          : XXH3_64bits(code.data(), code.size_bytes()),
+	    .hash            = hash,
 	};
 	EXIT_IF(user_data.size() > HW::UserSgprInfo::SGPRS_MAX ||
 	        params.user_data_count > params.user_data.size());
@@ -414,6 +381,8 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 		r.fields[1]       = sharp[1];
 		r.fields[2]       = sharp[2];
 		r.fields[3]       = sharp[3];
+		EXIT_NOT_IMPLEMENTED(r.AddTid());
+		EXIT_NOT_IMPLEMENTED(r.SwizzleEnabled());
 		if (format != Prospero::VertexAttribFormat::kInvalid) {
 			const auto                   format_raw    = static_cast<uint32_t>(format);
 			const auto                   buffer_format = format_raw >> 2u;
@@ -650,7 +619,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 		key.insert(key.end(), {mesh.wave_size, mesh.host_subgroup_size, mesh.lds_size_dwords,
 		                       mesh.scratch_size_dwords, mesh.input_primitive,
 		                       mesh.primitives_per_group, mesh.vertices_per_group,
-		                       mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex});
+		                       mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex,
+		                       static_cast<uint32_t>(mesh.fast_launch)});
 	}
 	key.push_back(info.tess.input_control_points);
 	if (info.tess.input_control_points != 0) {
@@ -662,12 +632,9 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 	for (int i = 0; i < info.resources_num; i++) {
 		const auto& resource    = info.resources[i];
 		const auto& destination = info.resources_dst[i];
-		key.push_back(destination.register_start);
 		key.push_back(destination.registers_num);
-		key.push_back(destination.fetch_index);
 		key.push_back(static_cast<uint32_t>(destination.attr_id));
-		key.push_back(resource.fields[1] & 0xbfff0000u); // Stride and swizzle enable.
-		key.push_back(resource.fields[3] & 0x3087ffffu); // Channels, format, OOB, and add TID.
+		key.push_back(resource.fields[3] & 0x7ffffu); // Channels and format used by embedded fetch.
 	}
 }
 
@@ -726,11 +693,10 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector<uint32_
 ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             const HW::UserConfig& user_config, ShaderVertexInputInfo& info) {
 	const auto& sh     = context.GetShaderRegisters();
-	const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
 	const bool merged = (context.GetShaderStages() & 0x20u) != 0;
 	auto        params = GetShaderParams(
-	    regs.es_regs.data_addr, "ShaderRecompiler VS",
-	    GetDeclaredShaderHash(regs.es_regs.data_addr),
+	    regs.es_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data,
 	    merged ? 8u : 0u);
 	if (!merged) {
@@ -753,12 +719,14 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 	mesh.provoking_vertex    = context.GetModeControl().provoking_vtx_last ? 2u : 0u;
 	mesh.lds_size_dwords     = static_cast<uint32_t>(regs.gs_regs.rsrc2.lds_size) * 128u;
 	mesh.scratch_size_dwords = data.scratch_size_dwords;
+	const auto fast_launch = (context.GetShaderStages() >> 19u) & 3u;
+	EXIT_NOT_IMPLEMENTED(fast_launch > 1u);
+	mesh.fast_launch = fast_launch != 0;
 	if (data.type == Prospero::ShaderBinaryType::kGsFront) {
 		EXIT_IF(regs.gs_regs.data_addr == 0);
-		const auto back = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
+		const auto [back, back_hash] = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
 		const auto back_params =
-		    GetShaderParams(regs.gs_regs.data_addr, "ShaderRecompiler GS",
-		                    GetDeclaredShaderHash(regs.gs_regs.data_addr), {}, back);
+		    GetShaderParams(regs.gs_regs.data_addr, back_hash, {}, back);
 		params.back_code = back_params.code;
 		params.user_data[0] = static_cast<uint32_t>(regs.gs_regs.user_data_addr);
 		params.user_data[1] = static_cast<uint32_t>(regs.gs_regs.user_data_addr >> 32u);
@@ -766,9 +734,20 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 		params.hash = XXH3_64bits(hashes, sizeof(hashes));
 		mesh.scratch_size_dwords = std::max(mesh.scratch_size_dwords, back.scratch_size_dwords);
 	}
-	EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
-	                     regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
 	const auto& group = user_config.GetGeControl();
+	if (mesh.fast_launch) {
+		const auto& user_vgpr = user_config.GetGeUserVgprEn();
+		EXIT_NOT_IMPLEMENTED(data.type != Prospero::ShaderBinaryType::kGs ||
+		                     regs.gs_regs.rsrc1.gs_vgpr_component_count != 0u ||
+		                     regs.gs_regs.rsrc2.es_vgpr_component_count != 1u ||
+		                     user_config.GetPrimType() != Prospero::PrimitiveType::kPointList ||
+		                     group.primitive_group_size != 1u || group.vertex_group_size != 1u ||
+		                     user_vgpr.vgpr1 || user_vgpr.vgpr2 || user_vgpr.vgpr3 ||
+		                     mesh.max_vertices != sh.m_vgtGsMaxVertOut);
+	} else {
+		EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
+		                     regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
+	}
 	if ((user_config.GetPrimType() != Prospero::PrimitiveType::kPointList &&
 	     user_config.GetPrimType() != Prospero::PrimitiveType::kLineList &&
 	     user_config.GetPrimType() != Prospero::PrimitiveType::kTriFan &&
@@ -781,7 +760,8 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 		     mesh.input_primitive, sh.m_vgtGsOutPrimType, sh.m_vgtGsMaxVertOut,
 		     group.primitive_group_size, group.vertex_group_size, mesh.max_vertices);
 	}
-	mesh.max_primitives       = group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
+	mesh.max_primitives = mesh.fast_launch ? mesh.max_vertices :
+	                      group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
 	mesh.primitives_per_group = std::min({static_cast<uint32_t>(group.primitive_group_size),
 	                                      mesh.InputPrimitiveCount(group.vertex_group_size),
 	                                      mesh.max_vertices / sh.m_vgtGsMaxVertOut});
@@ -797,9 +777,9 @@ std::array<ShaderParams, 3>
 PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             std::array<ShaderVertexInputInfo, 3>& input_info) {
 	const auto& sh        = context.GetShaderRegisters();
-	const auto  local     = ShaderGetMappedData(regs.ls_regs.data_addr, "ShaderGetInputInfoLS():");
-	const auto  control   = ShaderGetMappedData(regs.hs_regs.data_addr, "ShaderGetInputInfoHS():");
-	const auto evaluation = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoTES():");
+	const auto [local, local_hash] = ShaderGetMappedData(regs.ls_regs.data_addr, "ShaderGetInputInfoLS():");
+	const auto [control, control_hash] = ShaderGetMappedData(regs.hs_regs.data_addr, "ShaderGetInputInfoHS():");
+	const auto [evaluation, evaluation_hash] = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoTES():");
 	EXIT_NOT_IMPLEMENTED(local.type != Prospero::ShaderBinaryType::kHsFront ||
 	                     control.type != Prospero::ShaderBinaryType::kHsBack ||
 	                     evaluation.type != Prospero::ShaderBinaryType::kGs);
@@ -810,12 +790,9 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
 	const auto local_users      = std::span(regs.hs_user_sgpr.value, regs.hs_regs.rsrc2.user_sgpr);
 	const auto evaluation_users = std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr);
 	std::array<ShaderParams, 3> params {
-	    GetShaderParams(regs.ls_regs.data_addr, "ShaderRecompiler LS",
-	                    GetDeclaredShaderHash(regs.ls_regs.data_addr), local_users, local),
-	    GetShaderParams(regs.hs_regs.data_addr, "ShaderRecompiler HS",
-	                    GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control, 8u),
-	    GetShaderParams(regs.es_regs.data_addr, "ShaderRecompiler TES",
-	                    GetDeclaredShaderHash(regs.es_regs.data_addr), evaluation_users,
+	    GetShaderParams(regs.ls_regs.data_addr, local_hash, local_users, local),
+	    GetShaderParams(regs.hs_regs.data_addr, control_hash, local_users, control, 8u),
+	    GetShaderParams(regs.es_regs.data_addr, evaluation_hash, evaluation_users,
 	                    evaluation),
 	};
 	// The fused HS back half receives its separate user-data address in s0:s1.
@@ -855,19 +832,19 @@ ShaderParams PrepareProgram(
     const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
     std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
     ShaderPixelInputInfo&                               ps_info) {
-	const auto data = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():");
 	ShaderGetStaticInputInfoPS(regs, sh, target_export_mapping, data, ps_info);
 	return GetShaderParams(
-	    regs.ps_regs.data_addr, "ShaderRecompiler PS", GetDeclaredShaderHash(regs.ps_regs.data_addr),
+	    regs.ps_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), data);
 }
 
 ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
                             ShaderComputeInputInfo& info) {
-	const auto data = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
 	ShaderGetStaticInputInfoCS(regs, sh, data, info);
 	return GetShaderParams(
-	    regs.cs_regs.data_addr, "ShaderRecompiler CS", GetDeclaredShaderHash(regs.cs_regs.data_addr),
+	    regs.cs_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), data);
 }
 

src/graphics/shader/shader.h

@@ -88,6 +88,7 @@ struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo {
 	uint32_t max_vertices         = 0;
 	uint32_t max_primitives       = 0;
 	uint32_t provoking_vertex     = 0;
+	bool     fast_launch          = false;
 
 	[[nodiscard]] constexpr uint32_t InputPrimitiveSize() const {
 		switch (static_cast<Prospero::PrimitiveType>(input_primitive)) {

src/launcher/forms/configuration_edit_dialog.ui

@@ -72,6 +72,27 @@
           <property name="toolTip"><string>Language</string></property>
          </widget>
         </item>
+        <item row="3" column="0">
+         <widget class="QLabel" name="label_controller_color">
+          <property name="text"><string>DualSense lightbar:</string></property>
+         </widget>
+        </item>
+        <item row="3" column="1">
+         <layout class="QHBoxLayout" name="controllerColorLayout">
+          <item>
+           <widget class="QPushButton" name="button_controller_color">
+            <property name="toolTip"><string>Choose and preview a fixed LED color. Bluetooth DualSense may take several seconds to update.</string></property>
+            <property name="text"><string>Custom</string></property>
+           </widget>
+          </item>
+          <item>
+           <widget class="QPushButton" name="button_controller_color_reset">
+            <property name="toolTip"><string>Let the game control the lightbar</string></property>
+            <property name="text"><string>Default</string></property>
+           </widget>
+          </item>
+         </layout>
+        </item>
        </layout>
       </widget>
      </item>

src/launcher/include/configuration.h

@@ -5,6 +5,7 @@
 
 #include <QByteArray>
 #include <QChar>
+#include <QColor>
 #include <QMetaEnum>
 #include <QMetaType>
 #include <QObject>
@@ -88,6 +89,7 @@ public:
 	QString                user_name                   = "Kyty";
 	int                    user_id                     = Config::DEFAULT_USER_ID;
 	QString                audio_input_device;
+	QString                controller_color;
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int                    gpu_index                   = -1;
 	bool                   fullscreen_enabled          = false;
@@ -119,6 +121,7 @@ public:
 		user_name                   = other.user_name;
 		user_id                     = other.user_id;
 		audio_input_device          = other.audio_input_device;
+		controller_color            = other.controller_color;
 		present_mode                = other.present_mode;
 		gpu_index                   = other.gpu_index;
 		fullscreen_enabled          = other.fullscreen_enabled;
@@ -165,6 +168,7 @@ public:
 		KYTY_CFG_SET(user_name);
 		KYTY_CFG_SET(user_id);
 		KYTY_CFG_SET(audio_input_device);
+		KYTY_CFG_SET(controller_color);
 		KYTY_CFG_SET(present_mode);
 		KYTY_CFG_SET(gpu_index);
 		KYTY_CFG_SET(fullscreen_enabled);
@@ -204,6 +208,8 @@ public:
 		                         ? saved_user_id
 		                         : Config::DEFAULT_USER_ID;
 		audio_input_device = s->value("audio_input_device", audio_input_device).toString();
+		const QColor color(s->value("controller_color", controller_color).toString());
+		controller_color = color.isValid() ? color.name(QColor::HexRgb) : QString {};
 		KYTY_CFG_GET(present_mode);
 		gpu_index = s->value("gpu_index", -1).toInt();
 		if (EnumToText(present_mode).isEmpty()) {

src/launcher/include/configurationEditDialog.h

@@ -31,6 +31,9 @@ public:
 	void                      SetGameDirectories(const QStringList& dirs);
 	[[nodiscard]] QStringList GetGameDirectories() const;
 
+signals:
+	void PreviewControllerColor(const QString& color);
+
 private:
 	Ui::ConfigurationEditDialog* m_ui = nullptr;
 	Configuration&               m_info;

src/launcher/include/configurationListWidget.h

@@ -33,6 +33,9 @@ public:
 	ConfigurationItem*                     GetSelectedItem() { return m_selected_item; }
 
 	[[nodiscard]] const QString& GetSettingsFile() const { return m_settings_file; }
+	[[nodiscard]] const QString& GetGlobalControllerColor() const {
+		return m_global_info.controller_color;
+	}
 	[[nodiscard]] std::unique_ptr<Configuration>
 	CreateConfiguration(const ConfigurationItem& item) const;
 
@@ -44,6 +47,7 @@ signals:
 
 	void Run();
 	void Select();
+	void PreviewControllerColor(const QString& color);
 
 protected:
 	void changeEvent(QEvent* event) override;

src/launcher/include/controllerLightbar.h

@@ -0,0 +1,29 @@
+#ifndef CONTROLLER_LIGHTBAR_H
+#define CONTROLLER_LIGHTBAR_H
+
+#include <QColor>
+#include <QHash>
+#include <QObject>
+#include <QString>
+#include <QTimer>
+
+#include <SDL3/SDL_gamepad.h>
+
+class ControllerLightbar final: public QObject {
+public:
+	explicit ControllerLightbar(QObject* parent = nullptr);
+	~ControllerLightbar() override;
+
+	void SetColor(const QString& color);
+	void Stop();
+
+private:
+	void Refresh();
+	void ApplyColor(SDL_Gamepad* gamepad) const;
+
+	QTimer                              m_timer;
+	QHash<SDL_JoystickID, SDL_Gamepad*> m_gamepads;
+	QColor                              m_color;
+};
+
+#endif // CONTROLLER_LIGHTBAR_H

src/launcher/src/configurationEditDialog.cpp

@@ -7,6 +7,7 @@
 
 #include <QAbstractItemView>
 #include <QCheckBox>
+#include <QColorDialog>
 #include <QComboBox>
 #include <QCoreApplication>
 #include <QDir>
@@ -40,6 +41,22 @@ constexpr char SETTINGS_CFG_DIALOG[]               = "ConfigurationEditDialog";
 constexpr char SETTINGS_CFG_LAST_GEOMETRY[]        = "geometry";
 constexpr int  GLOBAL_SETTINGS_GAME_DIRS_MIN_WIDTH = 560;
 
+static void UpdateControllerColorButton(QPushButton* button, const QString& hex) {
+	const QColor color(hex);
+	if (!color.isValid()) {
+		button->setProperty("controllerColor", QString {});
+		button->setText(QObject::tr("Custom"));
+		button->setStyleSheet({});
+		return;
+	}
+	const auto normalized = color.name(QColor::HexRgb);
+	button->setProperty("controllerColor", normalized);
+	button->setText(normalized.toUpper());
+	button->setStyleSheet(QStringLiteral("background-color: %1; color: %2;")
+	                          .arg(normalized, color.lightness() < 128 ? QStringLiteral("white")
+	                                                                   : QStringLiteral("black")));
+}
+
 const QStringList CONSOLE_LANGUAGE_NAMES = {
     "Japanese",
     "English (United States)",
@@ -110,6 +127,21 @@ ConfigurationEditDialog::ConfigurationEditDialog(Configuration& info, QWidget* p
 	connect(m_ui->ok_button, &QPushButton::clicked, this, &ConfigurationEditDialog::save);
 	connect(m_ui->cancel_button, &QPushButton::clicked, this, &QDialog::reject);
 	connect(m_ui->clear_button, &QPushButton::clicked, this, &ConfigurationEditDialog::clear);
+	connect(m_ui->button_controller_color, &QPushButton::clicked, this, [this]() {
+		const auto current = m_ui->button_controller_color->property("controllerColor").toString();
+		const auto initial =
+		    current.isEmpty() ? QColor(QStringLiteral("#0070d1")) : QColor(current);
+		const auto color = QColorDialog::getColor(initial, this, tr("DualSense lightbar color"));
+		if (color.isValid()) {
+			const auto hex = color.name(QColor::HexRgb);
+			UpdateControllerColorButton(m_ui->button_controller_color, hex);
+			emit PreviewControllerColor(hex);
+		}
+	});
+	connect(m_ui->button_controller_color_reset, &QPushButton::clicked, this, [this]() {
+		UpdateControllerColorButton(m_ui->button_controller_color, {});
+		emit PreviewControllerColor({});
+	});
 	connect(m_ui->comboBox_shader_log_direction, &QComboBox::currentTextChanged, this,
 	        [this](const QString& text) {
 		        auto log = TextToEnum<Configuration::LogDirection>(text);
@@ -169,6 +201,7 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
 	m_ui->lineEdit_user_name->setMaxLength(static_cast<int>(Config::MAX_USER_NAME_LENGTH));
 	m_ui->lineEdit_user_name->setText(info.user_name);
 	m_ui->spinBox_user_id->setValue(info.user_id);
+	UpdateControllerColorButton(m_ui->button_controller_color, info.controller_color);
 	auto* microphone = m_ui->comboBox_audio_input_device;
 	microphone->clear();
 	microphone->addItem(tr("None"), QString {});
@@ -370,6 +403,7 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui) {
 	info.user_name = ui.lineEdit_user_name->text().trimmed();
 	info.user_id   = ui.spinBox_user_id->value();
 	info.audio_input_device = ui.comboBox_audio_input_device->currentData().toString();
+	info.controller_color   = ui.button_controller_color->property("controllerColor").toString();
 	info.screen_resolution =
 	    TextToEnum<Configuration::Resolution>(ui.comboBox_screen_resolution->currentText());
 	info.present_mode =
@@ -427,6 +461,7 @@ void ConfigurationEditDialog::save() {
 void ConfigurationEditDialog::clear() {
 	Configuration default_info;
 	Init(default_info);
+	emit PreviewControllerColor({});
 
 	if (m_show_game_dirs) {
 		m_game_dirs_list->clear();

src/launcher/src/configurationListWidget.cpp

@@ -751,6 +751,8 @@ void ConfigurationListWidget::edit_configuration() {
 	auto                    info = CreateConfiguration(*item);
 	ConfigurationEditDialog dlg(*info, this);
 	dlg.setWindowTitle(tr("Edit game settings"));
+	connect(&dlg, &ConfigurationEditDialog::PreviewControllerColor, this,
+	        &ConfigurationListWidget::PreviewControllerColor);
 
 	if (dlg.exec() == QDialog::Accepted) {
 		info->custom_settings           = true;
@@ -761,6 +763,8 @@ void ConfigurationListWidget::edit_configuration() {
 		WriteSettings();
 		item->Update();
 		SelectItem(item);
+	} else {
+		emit Select();
 	}
 }
 
@@ -788,6 +792,8 @@ void ConfigurationListWidget::edit_global_settings() {
 	ConfigurationEditDialog dlg(info, this);
 	dlg.setWindowTitle(tr("Global settings"));
 	dlg.SetGameDirectories(m_game_dirs);
+	connect(&dlg, &ConfigurationEditDialog::PreviewControllerColor, this,
+	        &ConfigurationListWidget::PreviewControllerColor);
 
 	if (dlg.exec() == QDialog::Accepted) {
 		m_global_info.CopyEmulatorSettingsFrom(info);
@@ -799,6 +805,7 @@ void ConfigurationListWidget::edit_global_settings() {
 			ScanGameDirectory();
 		}
 	}
+	emit Select();
 }
 
 void ConfigurationListWidget::edit_input_mapping() {

src/launcher/src/controllerLightbar.cpp

@@ -0,0 +1,84 @@
+#include "controllerLightbar.h"
+
+#include <QDebug>
+
+#include <SDL3/SDL.h>
+
+ControllerLightbar::ControllerLightbar(QObject* parent): QObject(parent) {
+	m_timer.setInterval(100);
+	connect(&m_timer, &QTimer::timeout, this, &ControllerLightbar::Refresh);
+}
+
+ControllerLightbar::~ControllerLightbar() {
+	Stop();
+}
+
+void ControllerLightbar::SetColor(const QString& color) {
+	const QColor parsed(color);
+	if (!parsed.isValid()) {
+		Stop();
+		return;
+	}
+	if (m_color == parsed) {
+		return;
+	}
+	if (!m_timer.isActive()) {
+		if (!SDL_InitSubSystem(SDL_INIT_GAMEPAD)) {
+			qWarning() << "Cannot initialize controller color preview:" << SDL_GetError();
+			return;
+		}
+		m_timer.start();
+	}
+	m_color = parsed;
+	for (auto* gamepad: m_gamepads) {
+		ApplyColor(gamepad);
+	}
+	Refresh();
+}
+
+void ControllerLightbar::Stop() {
+	if (!m_timer.isActive()) {
+		return;
+	}
+	m_timer.stop();
+	for (auto* gamepad: m_gamepads) {
+		SDL_CloseGamepad(gamepad);
+	}
+	m_gamepads.clear();
+	SDL_QuitSubSystem(SDL_INIT_GAMEPAD);
+	m_color = {};
+}
+
+void ControllerLightbar::Refresh() {
+	// Bluetooth DualSense may defer an accepted LED command until its connection
+	// animation ends. Polling events also delivers those deferred commands.
+	SDL_Event event;
+	while (SDL_PollEvent(&event)) {
+		if (event.type == SDL_EVENT_GAMEPAD_ADDED && !m_gamepads.contains(event.gdevice.which)) {
+			auto* gamepad = SDL_OpenGamepad(event.gdevice.which);
+			if (gamepad == nullptr) {
+				qWarning() << "Cannot open controller color preview:" << SDL_GetError();
+				continue;
+			}
+			if (SDL_GetBooleanProperty(SDL_GetGamepadProperties(gamepad),
+			                           SDL_PROP_GAMEPAD_CAP_RGB_LED_BOOLEAN, false)) {
+				m_gamepads.insert(event.gdevice.which, gamepad);
+				ApplyColor(gamepad);
+			} else {
+				SDL_CloseGamepad(gamepad);
+			}
+		} else if (event.type == SDL_EVENT_GAMEPAD_REMOVED) {
+			if (auto* gamepad = m_gamepads.take(event.gdevice.which); gamepad != nullptr) {
+				SDL_CloseGamepad(gamepad);
+			}
+		}
+	}
+}
+
+void ControllerLightbar::ApplyColor(SDL_Gamepad* gamepad) const {
+	if (!SDL_SetGamepadLED(gamepad, static_cast<Uint8>(m_color.red()),
+	                       static_cast<Uint8>(m_color.green()),
+	                       static_cast<Uint8>(m_color.blue()))) {
+		qWarning() << "Cannot set controller LED:" << SDL_GetError();
+	}
+}

src/launcher/src/mainDialog.cpp

@@ -3,6 +3,7 @@
 #include "configuration.h"
 #include "configurationItem.h"
 #include "configurationListWidget.h"
+#include "controllerLightbar.h"
 #include "gameContent.h"
 #include "patchesDialog.h"
 #include "updateChecker.h"
@@ -88,6 +89,7 @@ private:
 	Ui::MainDialog* m_ui             = {nullptr};
 	MainDialog*     m_main_dialog    = nullptr;
 	UpdateChecker*  m_update_checker = nullptr;
+	ControllerLightbar m_lightbar;
 	QString         m_interpreter;
 
 	QProcess m_process;
@@ -103,6 +105,7 @@ MainDialog::MainDialog(QWidget* parent): QDialog(parent), m_p(new MainDialogPriv
 }
 
 MainDialogPrivate::~MainDialogPrivate() {
+	m_process.disconnect(this);
 	delete m_ui;
 }
 
@@ -121,6 +124,12 @@ void MainDialogPrivate::Setup(MainDialog* main_dialog) {
 	connect(main_dialog, &MainDialog::Start, this, &MainDialogPrivate::FindInterpreter,
 	        Qt::QueuedConnection);
 	connect(m_ui->widget, &ConfigurationListWidget::Select, this, &MainDialogPrivate::Update);
+	connect(m_ui->widget, &ConfigurationListWidget::PreviewControllerColor, this,
+	        [this](const QString& color) {
+		        if (m_process.state() == QProcess::NotRunning) {
+			        m_lightbar.SetColor(color);
+		        }
+	        });
 	connect(m_ui->widget, &ConfigurationListWidget::Run, this, &MainDialogPrivate::Run);
 	connect(m_ui->check_updates_link, &QLabel::linkActivated, this,
 	        [this](const QString&) { m_update_checker->Check(true); });
@@ -135,14 +144,14 @@ void MainDialogPrivate::Setup(MainDialog* main_dialog) {
 		m_ui->widget->WriteSettings();
 	});
 
-	connect(&m_process,
-	        static_cast<void (QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
-	        [this](int /*exitCode*/, QProcess::ExitStatus /*exitStatus*/) {
-		        if (m_running_item != nullptr) {
-			        m_running_item->SetRunning(false);
-		        }
-		        Update();
-	        });
+	connect(&m_process, &QProcess::stateChanged, this, [this](QProcess::ProcessState state) {
+		if (state == QProcess::NotRunning) {
+			if (m_running_item != nullptr) {
+				m_running_item->SetRunning(false);
+			}
+			Update();
+		}
+	});
 
 	m_ui->label_settings_file->setText(tr("Settings file: ") + m_ui->widget->GetSettingsFile());
 
@@ -219,6 +228,9 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	if (!info.audio_input_device.isEmpty()) {
 		args << "--mic" << info.audio_input_device;
 	}
+	if (!info.controller_color.isEmpty()) {
+		args << "--controller-color" << info.controller_color;
+	}
 	args << "--present-mode" << EnumToText(info.present_mode);
 	if (info.gpu_index >= 0) {
 		args << "--gpu" << QString::number(info.gpu_index);
@@ -493,6 +505,7 @@ void MainDialogPrivate::Run() {
 	}
 
 	m_running_item->SetRunning(true);
+	m_lightbar.Stop();
 
 	auto info = m_ui->widget->CreateConfiguration(*m_running_item);
 	m_main_dialog->RunInterpreter(&m_process, *info);
@@ -512,6 +525,12 @@ void MainDialogPrivate::Update() {
 	}
 
 	m_ui->widget->SetRunEnabled(run_enabled);
+	if (m_process.state() != QProcess::NotRunning) {
+		m_lightbar.Stop();
+		return;
+	}
+	m_lightbar.SetColor(item != nullptr ? m_ui->widget->CreateConfiguration(*item)->controller_color
+	                                    : m_ui->widget->GetGlobalControllerColor());
 }
 
 #include "mainDialog.moc"

src/libs/agc.cpp

@@ -1552,7 +1552,7 @@ int KYTY_SYSV_ABI AgcSuspendPoint() {
 	PRINT_NAME();
 
 	EXIT_IF(g_renderer == nullptr);
-	g_renderer->GetGpu().Done();
+	g_renderer->GetGpu().SuspendPoint();
 
 	return OK;
 }

src/libs/controller.cpp

@@ -133,6 +133,9 @@ private:
 	int              m_connected_count = 0;
 	bool             m_motion_enabled  = true;
 	uint64_t         m_gyro_time       = 0;
+	// Accelerometer gravity direction in the reset frame.
+	std::array<float, 3> m_up_reference {0.0f, 1.0f, 0.0f};
+	bool                 m_up_reference_valid = false;
 	ControllerState  m_state;
 	ControllerState  m_states[STATES_MAX];
 	bool             m_obtained[STATES_MAX] {};
@@ -297,6 +300,9 @@ void GameController::Connect(int id) {
 	if (id != HOST_INPUT_CONTROLLER_ID) {
 		if (auto* pad = SDL_GetGamepadFromID(static_cast<SDL_JoystickID>(id));
 		    pad != nullptr) {
+			if (const auto& color = Config::GetControllerColor()) {
+				(void)SDL_SetGamepadLED(pad, (*color)[0], (*color)[1], (*color)[2]);
+			}
 			for (auto sensor: {SDL_SENSOR_ACCEL, SDL_SENSOR_GYRO}) {
 				if (SDL_GamepadHasSensor(pad, sensor) &&
 				    !SDL_SetGamepadSensorEnabled(pad, sensor, true)) {
@@ -345,6 +351,7 @@ void GameController::CheckActive() {
 	m_connected     = new_connected;
 	m_state         = {};
 	m_gyro_time     = 0;
+	m_up_reference_valid = false;
 	m_states_num    = 0;
 	m_first_state   = 0;
 	m_next_touch_id = 1;
@@ -436,6 +443,29 @@ void GameController::TouchPad(int id, int finger, bool down, float x, float y) {
 	}
 }
 
+namespace {
+
+using Vec3 = std::array<float, 3>;
+using Quat = std::array<float, 4>; // x, y, z, w
+
+Vec3 QuatRotate(const Quat& q, const Vec3& v) {
+	const Vec3 t {2.0f * (q[1] * v[2] - q[2] * v[1]), 2.0f * (q[2] * v[0] - q[0] * v[2]),
+	              2.0f * (q[0] * v[1] - q[1] * v[0])};
+	return {v[0] + q[3] * t[0] + (q[1] * t[2] - q[2] * t[1]),
+	        v[1] + q[3] * t[1] + (q[2] * t[0] - q[0] * t[2]),
+	        v[2] + q[3] * t[2] + (q[0] * t[1] - q[1] * t[0])};
+}
+
+Quat QuatConjugate(const Quat& q) {
+	return {-q[0], -q[1], -q[2], q[3]};
+}
+
+float Length(const Vec3& v) {
+	return std::sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
+}
+
+} // namespace
+
 void GameController::Motion(int id, Sensor sensor, const float* data, uint64_t time_us) {
 	Common::LockGuard lock(m_mutex);
 	if (id != m_active_id || !m_motion_enabled) {
@@ -447,20 +477,38 @@ void GameController::Motion(int id, Sensor sensor, const float* data, uint64_t t
 		for (int i = 0; i < 3; i++) {
 			m_state.accel[i] = data[i] / SDL_STANDARD_GRAVITY;
 		}
+		// Capture gravity near 1 G to limit interference from linear acceleration.
+		const float magnitude = Length(m_state.accel);
+		if (!m_up_reference_valid && magnitude > 0.9f && magnitude < 1.1f) {
+			const Vec3 up {m_state.accel[0] / magnitude, m_state.accel[1] / magnitude,
+			               m_state.accel[2] / magnitude};
+			m_up_reference       = QuatRotate(m_state.orientation, up);
+			m_up_reference_valid = true;
+		}
 	} else {
 		std::copy_n(data, 3, m_state.gyro.begin());
+		// Mahony tilt correction; heading is unobservable and reported gyro values stay raw.
+		auto        rate      = m_state.gyro;
+		const float magnitude = Length(m_state.accel);
+		if (m_up_reference_valid && magnitude > 0.8f && magnitude < 1.2f) {
+			const Vec3 measured {m_state.accel[0] / magnitude, m_state.accel[1] / magnitude,
+			                     m_state.accel[2] / magnitude};
+			const Vec3 predicted = QuatRotate(QuatConjugate(m_state.orientation), m_up_reference);
+			rate[0] += measured[1] * predicted[2] - measured[2] * predicted[1];
+			rate[1] += measured[2] * predicted[0] - measured[0] * predicted[2];
+			rate[2] += measured[0] * predicted[1] - measured[1] * predicted[0];
+		}
 		// Do not extrapolate a single sample across lost reports (e.g. loss of window focus).
 		constexpr uint64_t max_gyro_interval_us = 100000;
 		if (m_gyro_time != 0 && time_us > m_gyro_time &&
 		    time_us - m_gyro_time <= max_gyro_interval_us) {
-			const float dt = static_cast<float>(time_us - m_gyro_time) * 0.000001f;
-			const float speed =
-			    std::sqrt(data[0] * data[0] + data[1] * data[1] + data[2] * data[2]);
+			const float dt         = static_cast<float>(time_us - m_gyro_time) * 0.000001f;
+			const float speed      = Length(rate);
 			const float half_angle = speed * dt * 0.5f;
 			const float scale      = speed > 0.0f ? std::sin(half_angle) / speed : 0.0f;
-			const float x          = data[0] * scale;
-			const float y          = data[1] * scale;
-			const float z          = data[2] * scale;
+			const float x          = rate[0] * scale;
+			const float y          = rate[1] * scale;
+			const float z          = rate[2] * scale;
 			const float w          = std::cos(half_angle);
 			const auto  q          = m_state.orientation;
 			// Accumulate body-local rotation relative to connection / orientation reset.
@@ -486,19 +534,21 @@ void GameController::Motion(int id, Sensor sensor, const float* data, uint64_t t
 void GameController::SetMotionSensorState(bool enable) {
 	Common::LockGuard lock(m_mutex);
 	if (m_motion_enabled != enable) {
-		m_motion_enabled = enable;
-		m_gyro_time      = 0;
-		m_state.accel    = {0.0f, 1.0f, 0.0f};
-		m_state.gyro     = {};
-		m_state.time     = LibKernel::KernelGetProcessTime();
+		m_motion_enabled     = enable;
+		m_gyro_time          = 0;
+		m_state.accel        = {0.0f, 1.0f, 0.0f};
+		m_state.gyro         = {};
+		m_up_reference_valid = false;
+		m_state.time         = LibKernel::KernelGetProcessTime();
 		AddState();
 	}
 }
 
 void GameController::ResetOrientation() {
 	Common::LockGuard lock(m_mutex);
-	m_state.orientation = {0.0f, 0.0f, 0.0f, 1.0f};
-	m_gyro_time         = 0;
+	m_state.orientation  = {0.0f, 0.0f, 0.0f, 1.0f};
+	m_gyro_time          = 0;
+	m_up_reference_valid = false;
 	m_state.time        = LibKernel::KernelGetProcessTime();
 	AddState();
 }
@@ -581,6 +631,11 @@ int GameController::GetActiveControllerId() {
 
 void GameController::SetLightBar(uint8_t r, uint8_t g, uint8_t b) {
 	Common::LockGuard lock(m_mutex);
+	if (const auto& color = Config::GetControllerColor()) {
+		r = (*color)[0];
+		g = (*color)[1];
+		b = (*color)[2];
+	}
 	if (auto* pad = SDL_GetGamepadFromID(static_cast<SDL_JoystickID>(m_active_id));
 	    pad != nullptr) {
 		(void)SDL_SetGamepadLED(pad, r, g, b);

src/libs/libNet.cpp

@@ -3834,6 +3834,27 @@ namespace LibSharePlay {
 
 LIB_VERSION("SharePlay", 1, "SharePlay", 1, 1);
 
+constexpr int SHARE_PLAY_ERROR_INVALID_ARGS        = -2129788927;
+constexpr int SHARE_PLAY_ERROR_ALREADY_INITIALIZED = -2129788925;
+constexpr int SHARE_PLAY_ERROR_NOT_INITIALIZED     = -2129788924;
+
+struct SharePlayConnectionInfoA {
+	int32_t  status;
+	int32_t  mode;
+	char     host_online_id[20];
+	char     visitor_online_id[20];
+	uint64_t host_account_id;
+	uint64_t visitor_account_id;
+	int32_t  host_user_id;
+	int32_t  visitor_user_id;
+};
+
+static_assert(sizeof(SharePlayConnectionInfoA) == 72);
+static_assert(offsetof(SharePlayConnectionInfoA, host_account_id) == 48);
+static_assert(offsetof(SharePlayConnectionInfoA, host_user_id) == 64);
+
+static bool g_share_play_initialized = false;
+
 static int KYTY_SYSV_ABI SharePlayInitialize(void* heap, size_t heap_size) {
 	PRINT_NAME();
 
@@ -3841,18 +3862,41 @@ static int KYTY_SYSV_ABI SharePlayInitialize(void* heap, size_t heap_size) {
 	     "\t heap_size = %" PRIu64 "\n",
 	     reinterpret_cast<uint64_t>(heap), static_cast<uint64_t>(heap_size));
 
+	if (g_share_play_initialized) {
+		return SHARE_PLAY_ERROR_ALREADY_INITIALIZED;
+	}
+	if (heap != nullptr && heap_size < 6u * 1024u) {
+		return SHARE_PLAY_ERROR_INVALID_ARGS;
+	}
+	g_share_play_initialized = true;
 	return 0;
 }
 
 static int KYTY_SYSV_ABI SharePlayTerminate() {
 	PRINT_NAME();
 
+	if (!g_share_play_initialized) {
+		return SHARE_PLAY_ERROR_NOT_INITIALIZED;
+	}
+	g_share_play_initialized = false;
+	return 0;
+}
+
+static int KYTY_SYSV_ABI SharePlayGetCurrentConnectionInfoA(SharePlayConnectionInfoA* info) {
+	if (!g_share_play_initialized) {
+		return SHARE_PLAY_ERROR_NOT_INITIALIZED;
+	}
+	if (info == nullptr) {
+		return SHARE_PLAY_ERROR_INVALID_ARGS;
+	}
+	*info = {.status = 0, .host_user_id = -1, .visitor_user_id = -1};
 	return 0;
 }
 
 LIB_DEFINE(InitPlatform_1_SharePlay) {
 	LIB_FUNC("isruqthpYcw", LibSharePlay::SharePlayInitialize);
 	LIB_FUNC("UaLjloJinow", LibSharePlay::SharePlayTerminate);
+	LIB_FUNC("+MCXJlWdi+s", LibSharePlay::SharePlayGetCurrentConnectionInfoA);
 }
 
 } // namespace LibSharePlay

src/libs/libPngEnc.cpp

@@ -0,0 +1,349 @@
+#include "common/abi.h"
+#include "common/logging/log.h"
+#include "libs/libs.h"
+#include "loader/symbolDatabase.h"
+
+#include <bit>
+#include <cstddef>
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <mutex>
+#include <new>
+
+#define STB_IMAGE_WRITE_IMPLEMENTATION
+#define STB_IMAGE_WRITE_STATIC
+#define STBIWDEF static inline
+#define STBI_WRITE_NO_STDIO
+#include "stb_image_write.h"
+
+namespace Libs {
+
+LIB_VERSION("PngEnc", 1, "PngEnc", 1, 1);
+
+namespace PngEnc {
+
+constexpr int32_t PNG_ENC_ERROR_INVALID_ADDR   = -2140602111; // 0x80690101
+constexpr int32_t PNG_ENC_ERROR_INVALID_SIZE   = -2140602110; // 0x80690102
+constexpr int32_t PNG_ENC_ERROR_INVALID_PARAM  = -2140602109; // 0x80690103
+constexpr int32_t PNG_ENC_ERROR_INVALID_HANDLE = -2140602108; // 0x80690104
+constexpr int32_t PNG_ENC_ERROR_DATA_OVERFLOW  = -2140602096; // 0x80690110
+constexpr int32_t PNG_ENC_ERROR_FATAL          = -2140602080; // 0x80690120
+
+constexpr uint32_t PNG_ENC_MAX_IMAGE_WIDTH       = 1000000;
+constexpr uint32_t PNG_ENC_MAX_IMAGE_HEIGHT      = 1000000;
+constexpr uint32_t PNG_ENC_MAX_FILTER_NUMBER     = 4;
+constexpr uint16_t PNG_ENC_COLOR_SPACE_RGB       = 3;
+constexpr uint16_t PNG_ENC_COLOR_SPACE_RGBA      = 19;
+constexpr uint16_t PNG_ENC_PIXEL_FORMAT_R8G8B8A8 = 0;
+constexpr uint16_t PNG_ENC_PIXEL_FORMAT_B8G8R8A8 = 1;
+constexpr uint16_t PNG_ENC_FILTER_TYPE_SUB       = 1;
+constexpr uint16_t PNG_ENC_FILTER_TYPE_UP        = 2;
+constexpr uint16_t PNG_ENC_FILTER_TYPE_AVERAGE   = 4;
+constexpr uint16_t PNG_ENC_FILTER_TYPE_PAETH     = 8;
+
+struct PngEncCreateParam {
+	uint32_t this_size;
+	uint32_t attribute;
+	uint32_t max_image_width;
+	uint32_t max_filter_number;
+};
+
+struct PngEncEncodeParam {
+	const uint8_t* image_mem_addr;
+	uint8_t*       png_mem_addr;
+	uint32_t       image_mem_size;
+	uint32_t       png_mem_size;
+	uint32_t       image_width;
+	uint32_t       image_height;
+	uint32_t       image_pitch;
+	uint16_t       pixel_format;
+	uint16_t       color_space;
+	uint16_t       bit_depth;
+	uint16_t       clut_number;
+	uint16_t       filter_type;
+	uint16_t       compression_level;
+};
+
+struct PngEncOutputInfo {
+	uint32_t data_size;
+	uint32_t processed_height;
+};
+
+static_assert(sizeof(PngEncCreateParam) == 16);
+static_assert(sizeof(PngEncEncodeParam) == 48);
+
+struct PngEncContext {
+	uint64_t magic;
+	uint32_t max_image_width;
+	uint32_t max_filter_number;
+};
+
+constexpr uint64_t PNG_ENC_CONTEXT_MAGIC = 0x4b595459504e4745ull; // KYTYPNGE
+
+struct PngEncWriter {
+	uint8_t* dst;
+	uint32_t capacity;
+	uint32_t size;
+	bool     overflow;
+};
+
+// stb_image_write keeps its compression level and filter choice in globals
+static std::mutex g_png_enc_mutex;
+
+static void PngEncWrite(void* context, void* data, int size) {
+	auto* writer = static_cast<PngEncWriter*>(context);
+
+	if (writer->overflow || size < 0 ||
+	    static_cast<uint64_t>(writer->size) + static_cast<uint32_t>(size) > writer->capacity) {
+		writer->overflow = true;
+		return;
+	}
+
+	std::memcpy(writer->dst + writer->size, data, static_cast<size_t>(size));
+	writer->size += static_cast<uint32_t>(size);
+}
+
+static int PngEncForcedFilter(uint16_t filter_type) {
+	if (filter_type == (PNG_ENC_FILTER_TYPE_SUB | PNG_ENC_FILTER_TYPE_UP |
+	                    PNG_ENC_FILTER_TYPE_AVERAGE | PNG_ENC_FILTER_TYPE_PAETH)) {
+		return -1;
+	}
+	// stb's adaptive mode cannot restrict itself to the guest's filter mask.
+	if ((filter_type & PNG_ENC_FILTER_TYPE_SUB) != 0) {
+		return 1;
+	}
+	if ((filter_type & PNG_ENC_FILTER_TYPE_UP) != 0) {
+		return 2;
+	}
+	if ((filter_type & PNG_ENC_FILTER_TYPE_AVERAGE) != 0) {
+		return 3;
+	}
+	if ((filter_type & PNG_ENC_FILTER_TYPE_PAETH) != 0) {
+		return 4;
+	}
+	return 0;
+}
+
+static int32_t ValidateCreateParam(const PngEncCreateParam* param) {
+	if (param == nullptr) {
+		return PNG_ENC_ERROR_INVALID_ADDR;
+	}
+
+	LOGF("\t this_size         = %" PRIu32 "\n", param->this_size);
+	LOGF("\t attribute         = %" PRIu32 "\n", param->attribute);
+	LOGF("\t max_image_width   = %" PRIu32 "\n", param->max_image_width);
+	LOGF("\t max_filter_number = %" PRIu32 "\n", param->max_filter_number);
+
+	if (param->this_size != sizeof(PngEncCreateParam) || param->attribute != 0 ||
+	    param->max_filter_number > PNG_ENC_MAX_FILTER_NUMBER) {
+		return PNG_ENC_ERROR_INVALID_PARAM;
+	}
+
+	if (param->max_image_width == 0 || param->max_image_width > PNG_ENC_MAX_IMAGE_WIDTH) {
+		return PNG_ENC_ERROR_INVALID_SIZE;
+	}
+
+	return 0;
+}
+
+static int32_t KYTY_SYSV_ABI PngEncQueryMemorySize(const PngEncCreateParam* param) {
+	PRINT_NAME();
+
+	if (int32_t result = ValidateCreateParam(param); result != 0) {
+		return result;
+	}
+
+	return sizeof(PngEncContext);
+}
+
+static int32_t KYTY_SYSV_ABI PngEncCreate(const PngEncCreateParam* param, void* memory_address,
+                                          uint32_t memory_size, void** handle) {
+	PRINT_NAME();
+
+	LOGF("\t memory_address    = %p\n", memory_address);
+	LOGF("\t memory_size       = %" PRIu32 "\n", memory_size);
+
+	if (int32_t result = ValidateCreateParam(param); result != 0) {
+		return result;
+	}
+
+	if (memory_address == nullptr || handle == nullptr) {
+		return PNG_ENC_ERROR_INVALID_ADDR;
+	}
+
+	if (memory_size < sizeof(PngEncContext)) {
+		return PNG_ENC_ERROR_INVALID_SIZE;
+	}
+
+	auto* ctx              = static_cast<PngEncContext*>(memory_address);
+	ctx->magic             = PNG_ENC_CONTEXT_MAGIC;
+	ctx->max_image_width   = param->max_image_width;
+	ctx->max_filter_number = param->max_filter_number;
+	*handle                = ctx;
+
+	return 0;
+}
+
+static int32_t KYTY_SYSV_ABI PngEncEncode(void* handle, const PngEncEncodeParam* param,
+                                          PngEncOutputInfo* output_info) {
+	PRINT_NAME();
+
+	if (handle == nullptr) {
+		return PNG_ENC_ERROR_INVALID_HANDLE;
+	}
+
+	const auto* ctx = static_cast<const PngEncContext*>(handle);
+	if (ctx->magic != PNG_ENC_CONTEXT_MAGIC) {
+		return PNG_ENC_ERROR_INVALID_HANDLE;
+	}
+
+	if (param == nullptr) {
+		return PNG_ENC_ERROR_INVALID_PARAM;
+	}
+
+	LOGF("\t image_mem_addr    = %p\n", static_cast<const void*>(param->image_mem_addr));
+	LOGF("\t png_mem_addr      = %p\n", static_cast<const void*>(param->png_mem_addr));
+	LOGF("\t image_mem_size    = %" PRIu32 "\n", param->image_mem_size);
+	LOGF("\t png_mem_size      = %" PRIu32 "\n", param->png_mem_size);
+	LOGF("\t image_width       = %" PRIu32 "\n", param->image_width);
+	LOGF("\t image_height      = %" PRIu32 "\n", param->image_height);
+	LOGF("\t image_pitch       = %" PRIu32 "\n", param->image_pitch);
+	LOGF("\t pixel_format      = %" PRIu16 "\n", param->pixel_format);
+	LOGF("\t color_space       = %" PRIu16 "\n", param->color_space);
+	LOGF("\t bit_depth         = %" PRIu16 "\n", param->bit_depth);
+	LOGF("\t filter_type       = %" PRIu16 "\n", param->filter_type);
+	LOGF("\t compression_level = %" PRIu16 "\n", param->compression_level);
+
+	if (param->image_mem_addr == nullptr || param->png_mem_addr == nullptr ||
+	    (reinterpret_cast<uintptr_t>(param->image_mem_addr) & 3u) != 0) {
+		return PNG_ENC_ERROR_INVALID_ADDR;
+	}
+
+	if (param->pixel_format != PNG_ENC_PIXEL_FORMAT_R8G8B8A8 &&
+	    param->pixel_format != PNG_ENC_PIXEL_FORMAT_B8G8R8A8) {
+		return PNG_ENC_ERROR_INVALID_PARAM;
+	}
+
+	if (param->color_space != PNG_ENC_COLOR_SPACE_RGB &&
+	    param->color_space != PNG_ENC_COLOR_SPACE_RGBA) {
+		return PNG_ENC_ERROR_INVALID_PARAM;
+	}
+
+	if (param->bit_depth != 8 || param->clut_number != 0 || param->compression_level > 9) {
+		return PNG_ENC_ERROR_INVALID_PARAM;
+	}
+
+	constexpr uint16_t filter_mask = PNG_ENC_FILTER_TYPE_SUB | PNG_ENC_FILTER_TYPE_UP |
+	                                 PNG_ENC_FILTER_TYPE_AVERAGE | PNG_ENC_FILTER_TYPE_PAETH;
+	if ((param->filter_type & ~filter_mask) != 0 ||
+	    static_cast<uint32_t>(std::popcount(param->filter_type)) > ctx->max_filter_number) {
+		return PNG_ENC_ERROR_INVALID_PARAM;
+	}
+
+	const uint32_t width  = param->image_width;
+	const uint32_t height = param->image_height;
+	if (width == 0 || height == 0 || width > ctx->max_image_width ||
+	    width > PNG_ENC_MAX_IMAGE_WIDTH || height > PNG_ENC_MAX_IMAGE_HEIGHT) {
+		return PNG_ENC_ERROR_INVALID_SIZE;
+	}
+
+	const int      components = (param->color_space == PNG_ENC_COLOR_SPACE_RGBA ? 4 : 3);
+	const uint32_t row_size   = width * static_cast<uint32_t>(components);
+	const uint32_t min_pitch  = width * 4u;
+	const uint32_t pitch      = param->image_pitch;
+
+	// Leave room for deflate expansion and stb's signed-int buffer-capacity doubling.
+	constexpr uint64_t max_filtered_size = std::numeric_limits<int>::max() / 2u;
+	if (static_cast<uint64_t>(row_size + 1u) * height > max_filtered_size ||
+	    param->png_mem_size == 0 || pitch < min_pitch || (pitch & 3u) != 0 ||
+	    static_cast<uint64_t>(pitch) * height > param->image_mem_size) {
+		return PNG_ENC_ERROR_INVALID_SIZE;
+	}
+
+	const bool                 bgr = (param->pixel_format == PNG_ENC_PIXEL_FORMAT_B8G8R8A8);
+	std::unique_ptr<uint8_t[]> pixels(new (std::nothrow)
+	                                      uint8_t[static_cast<size_t>(row_size) * height]);
+	if (pixels == nullptr) {
+		return PNG_ENC_ERROR_FATAL;
+	}
+
+	for (uint32_t y = 0; y < height; y++) {
+		const auto* src = param->image_mem_addr + static_cast<size_t>(y) * pitch;
+		auto*       dst = pixels.get() + static_cast<size_t>(y) * row_size;
+
+		for (uint32_t x = 0; x < width; x++) {
+			dst[0] = src[bgr ? 2 : 0];
+			dst[1] = src[1];
+			dst[2] = src[bgr ? 0 : 2];
+			if (components == 4) {
+				dst[3] = src[3];
+			}
+
+			src += 4;
+			dst += components;
+		}
+	}
+
+	PngEncWriter writer {param->png_mem_addr, param->png_mem_size, 0, false};
+
+	int ok = 0;
+	{
+		std::scoped_lock lock(g_png_enc_mutex);
+
+		stbi_write_png_compression_level = param->compression_level;
+		stbi_write_force_png_filter      = PngEncForcedFilter(param->filter_type);
+
+		ok = stbi_write_png_to_func(PngEncWrite, &writer, static_cast<int>(width),
+		                            static_cast<int>(height), components, pixels.get(),
+		                            static_cast<int>(row_size));
+	}
+
+	if (output_info != nullptr) {
+		output_info->data_size        = (writer.overflow ? 0 : writer.size);
+		output_info->processed_height = (ok != 0 && !writer.overflow ? height : 0);
+	}
+
+	if (ok == 0) {
+		LOGF("\t encode failed\n");
+		return PNG_ENC_ERROR_FATAL;
+	}
+
+	if (writer.overflow) {
+		LOGF("\t output buffer too small\n");
+		return PNG_ENC_ERROR_DATA_OVERFLOW;
+	}
+
+	LOGF("\t data_size = %" PRIu32 "\n", writer.size);
+
+	return static_cast<int32_t>(writer.size);
+}
+
+static int32_t KYTY_SYSV_ABI PngEncDelete(void* handle) {
+	PRINT_NAME();
+
+	if (handle == nullptr) {
+		return PNG_ENC_ERROR_INVALID_HANDLE;
+	}
+
+	auto* ctx = static_cast<PngEncContext*>(handle);
+	if (ctx->magic != PNG_ENC_CONTEXT_MAGIC) {
+		return PNG_ENC_ERROR_INVALID_HANDLE;
+	}
+
+	ctx->magic = 0;
+	return 0;
+}
+
+} // namespace PngEnc
+
+LIB_DEFINE(InitPngEnc_1) {
+	PRINT_NAME_ENABLE(true);
+
+	LIB_FUNC("9030RnBDoh4", PngEnc::PngEncQueryMemorySize);
+	LIB_FUNC("7aGTPfrqT9s", PngEnc::PngEncCreate);
+	LIB_FUNC("xgDjJKpcyHo", PngEnc::PngEncEncode);
+	LIB_FUNC("RUrWdwTWZy8", PngEnc::PngEncDelete);
+}
+
+} // namespace Libs

src/libs/libVideoOut.cpp

@@ -39,6 +39,8 @@ LIB_DEFINE(InitVideoOut_1) {
 	LIB_FUNC("Mt4QHHkxkOc", VideoOut::VideoOutGetEventCount);
 	LIB_FUNC("j6RaAUlaLv0", VideoOut::VideoOutWaitVblank);
 	LIB_FUNC("utPrVdxio-8", VideoOut::VideoOutGetOutputStatus);
+	LIB_FUNC("5tRaBjtdTzY", VideoOut::VideoOutVrrPegToFixedRate);
+	LIB_FUNC("T4ucGB8CsnM", VideoOut::VideoOutVrrUnpegFromFixedRate);
 	LIB_FUNC("+I4K03i3EL0", VideoOut::VideoOutInitializeOutputOptions);
 	LIB_FUNC("Nv8c-Kb+DUM", VideoOut::VideoOutIsOutputSupported);
 	LIB_FUNC("w0hLuNarQxY", VideoOut::VideoOutConfigureOutput);
@@ -46,6 +48,7 @@ LIB_DEFINE(InitVideoOut_1) {
 	LIB_FUNC("MCJ8SkzsQxY", VideoOut::VideoOutLatencyMeasureSetStartPoint);
 	LIB_FUNC("DYhhWbJSeRg", VideoOut::VideoOutColorSettingsSetGamma);
 	LIB_FUNC("pv9CI5VC+R0", VideoOut::VideoOutAdjustColor);
+	LIB_FUNC("w7Ipp9Xl7hg", VideoOut::VideoOutAllowOutputResolutionWqhdDetection);
 }
 
 } // namespace LibGen5

src/libs/libs.cpp

@@ -91,6 +91,7 @@ LIB_DEFINE(InitNet_1);
 LIB_DEFINE(InitPad_1);
 LIB_DEFINE(InitPlayGo_1);
 LIB_DEFINE(InitPngDec_1);
+LIB_DEFINE(InitPngEnc_1);
 LIB_DEFINE(InitPlatform_1);
 LIB_DEFINE(InitRudp_1);
 LIB_DEFINE(InitSaveData_1);
@@ -129,6 +130,7 @@ void InitAll(Loader::SymbolDatabase* s) {
 	InitPlayGo_1(s);
 	LibPsml::InitPsml_1(s);
 	InitPngDec_1(s);
+	InitPngEnc_1(s);
 	InitPlatform_1(s);
 	InitRudp_1(s);
 	LibRtc::InitRtc_1(s);

src/libs/network.cpp

@@ -997,14 +997,31 @@ static int ConvertHostSocketError(int error) {
 		case EAGAIN: posix_error = Posix::POSIX_EWOULDBLOCK; break;
 		case EBADF: posix_error = Posix::POSIX_EBADF; break;
 		case EFAULT: posix_error = Posix::POSIX_EFAULT; break;
+		case EINTR: posix_error = Posix::POSIX_EINTR; break;
 		case EINVAL: posix_error = Posix::POSIX_EINVAL; break;
+		case EISCONN: posix_error = Posix::POSIX_EISCONN; break;
 		case EMFILE: posix_error = Posix::POSIX_EMFILE; break;
+		case EMSGSIZE: posix_error = Posix::POSIX_EMSGSIZE; break;
 		case ENFILE: posix_error = Posix::POSIX_ENFILE; break;
 		case ENOBUFS: posix_error = Posix::POSIX_ENOBUFS; break;
 		case ENOMEM: posix_error = Posix::POSIX_ENOMEM; break;
+		case ENETDOWN: posix_error = Posix::POSIX_ENETDOWN; break;
+		case ENETRESET: posix_error = Posix::POSIX_ENETRESET; break;
+		case ENETUNREACH: posix_error = Posix::POSIX_ENETUNREACH; break;
+		case ENOTCONN: posix_error = Posix::POSIX_ENOTCONN; break;
 		case ENOTSOCK: posix_error = Posix::POSIX_ENOTSOCK; break;
+		case EOPNOTSUPP: posix_error = Posix::POSIX_EOPNOTSUPP; break;
 		case EPIPE: posix_error = Posix::POSIX_EPIPE; break;
 		case EPROTONOSUPPORT: posix_error = Posix::POSIX_EPROTONOSUPPORT; break;
+		case ESHUTDOWN: posix_error = Posix::POSIX_ESHUTDOWN; break;
+		case ETIMEDOUT: posix_error = Posix::POSIX_ETIMEDOUT; break;
+		case ECONNABORTED: posix_error = Posix::POSIX_ECONNABORTED; break;
+		case ECONNREFUSED: posix_error = Posix::POSIX_ECONNREFUSED; break;
+		case ECONNRESET: posix_error = Posix::POSIX_ECONNRESET; break;
+		case EDESTADDRREQ: posix_error = Posix::POSIX_EDESTADDRREQ; break;
+		case EHOSTUNREACH: posix_error = Posix::POSIX_EHOSTUNREACH; break;
+		case EINPROGRESS: posix_error = Posix::POSIX_EINPROGRESS; break;
+		case EALREADY: posix_error = Posix::POSIX_EALREADY; break;
 		default: break;
 	}
 #endif
@@ -2098,17 +2115,22 @@ int KYTY_SYSV_ABI Setsockopt(int s, int level, int optname, const void* optval,
 		return 0;
 	}
 
-#if defined(_WIN32)
 	constexpr int ORBIS_SO_NBIO = 0x1200;
 	if (ConvertSocketOptionLevel(level) == SOL_SOCKET && optname == ORBIS_SO_NBIO &&
 	    optlen >= sizeof(int)) {
-		u_long enabled = (*static_cast<const int*>(optval) != 0 ? 1 : 0);
-		if (ioctlsocket(socket, FIONBIO, &enabled) == SOCKET_ERROR) {
-			return SetHostSocketError();
-		}
-		return 0;
-	}
+		const bool enabled = *static_cast<const int*>(optval) != 0;
+#if defined(_WIN32)
+		u_long     mode   = enabled ? 1 : 0;
+		const bool failed = ioctlsocket(socket, FIONBIO, &mode) == SOCKET_ERROR;
 #else
+		const int  flags = ::fcntl(socket, F_GETFL, 0);
+		const bool failed =
+		    flags < 0 ||
+		    ::fcntl(socket, F_SETFL, enabled ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK)) != 0;
+#endif
+		return failed ? SetHostSocketError() : 0;
+	}
+#if !defined(_WIN32)
 	// Guest TCP options: IPPROTO_TCP=6, TCP_NODELAY=1.
 	if (level != 6 || optname != 1) {
 		return SetGuestSocketError(Posix::POSIX_ENOPROTOOPT);

src/main.cpp

@@ -53,6 +53,7 @@ static void PrintUsage() {
 	::printf("  --user-id <num>                      Local user ID. Default: %d.\n",
 	         Config::DEFAULT_USER_ID);
 	::printf("  --mic <name>                        Capture from this microphone; omit for silence.\n");
+	::printf("  --controller-color <#RRGGBB>        Override the controller lightbar color.\n");
 	::printf(
 	    "  --present-mode <value>               Fifo, Mailbox, or Immediate. Default: Mailbox.\n");
 	::printf(
@@ -145,6 +146,20 @@ static bool ParseUint32(const std::string& value, uint32_t& out) {
 	return true;
 }
 
+static bool ParseControllerColor(const std::string& value, Config::ControllerColor& out) {
+	if (value.size() != 7 || value[0] != '#') {
+		return false;
+	}
+	uint32_t rgb = 0;
+	auto [end, error] = std::from_chars(value.data() + 1, value.data() + value.size(), rgb, 16);
+	if (error != std::errc {} || end != value.data() + value.size()) {
+		return false;
+	}
+	out = {static_cast<uint8_t>(rgb >> 16), static_cast<uint8_t>(rgb >> 8),
+	       static_cast<uint8_t>(rgb)};
+	return true;
+}
+
 static bool ParseInt32(const std::string& value, int32_t& out) {
 	int32_t number    = 0;
 	auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), number);
@@ -302,6 +317,13 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
 			}
 		} else if (arg == "--mic") {
 			options.config.audio_input_device = value;
+		} else if (arg == "--controller-color") {
+			Config::ControllerColor color {};
+			if (!ParseControllerColor(value, color)) {
+				::printf("invalid controller color (expected #RRGGBB): %s\n", value.c_str());
+				return false;
+			}
+			options.config.controller_color = color;
 		} else if (arg == "--present-mode") {
 			if (!ParseEnum(value, options.config.present_mode)) {
 				::printf("invalid present mode: %s\n", value.c_str());

tests/PngEncTests.cpp

@@ -0,0 +1,247 @@
+#include "libs/libPngEnc.cpp"
+
+#define STB_IMAGE_IMPLEMENTATION
+#define STBI_ONLY_PNG
+#define STBI_NO_STDIO
+#include "stb_image.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+namespace Loader {
+void SymbolDatabase::Add(const SymbolResolve&, uint64_t, const std::string&) {}
+namespace Timer {
+double GetTimeMs() {
+	return 0.0;
+}
+} // namespace Timer
+} // namespace Loader
+
+namespace {
+using namespace Libs::PngEnc;
+
+#define CHECK(condition)                                                                           \
+	do {                                                                                           \
+		if (!(condition)) {                                                                        \
+			std::fprintf(stderr, "PngEncTests:%d: %s\n", __LINE__, #condition);                    \
+			std::abort();                                                                          \
+		}                                                                                          \
+	} while (false)
+
+struct Encoder {
+	PngEncContext context {};
+	void*         handle = nullptr;
+
+	explicit Encoder(uint32_t max_filters = 4) {
+		const PngEncCreateParam create {sizeof(PngEncCreateParam), 0, PNG_ENC_MAX_IMAGE_WIDTH,
+		                                max_filters};
+		CHECK(PngEncQueryMemorySize(&create) == sizeof(context));
+		CHECK(PngEncCreate(&create, &context, sizeof(context), &handle) == 0);
+	}
+
+	~Encoder() {
+		CHECK(PngEncDelete(handle) == 0);
+		CHECK(PngEncDelete(handle) == PNG_ENC_ERROR_INVALID_HANDLE);
+	}
+};
+
+PngEncEncodeParam EncodeParam(const uint8_t* source, uint32_t source_size, uint8_t* destination,
+                              uint32_t destination_size) {
+	PngEncEncodeParam param {};
+	param.image_mem_addr    = source;
+	param.png_mem_addr      = destination;
+	param.image_mem_size    = source_size;
+	param.png_mem_size      = destination_size;
+	param.image_width       = 2;
+	param.image_height      = 2;
+	param.image_pitch       = 8;
+	param.color_space       = PNG_ENC_COLOR_SPACE_RGBA;
+	param.bit_depth         = 8;
+	param.compression_level = 6;
+	return param;
+}
+
+uint32_t ReadBe32(const uint8_t* bytes) {
+	return (uint32_t {bytes[0]} << 24u) | (uint32_t {bytes[1]} << 16u) |
+	       (uint32_t {bytes[2]} << 8u) | bytes[3];
+}
+
+void TestPixels() {
+	Encoder                           encoder;
+	constexpr std::array<uint8_t, 16> rgba {255, 0, 13,  0,  4,   210, 6,   255,
+	                                        7,   8, 220, 64, 111, 12,  113, 128};
+	for (const bool bgr: {false, true}) {
+		for (const bool alpha: {false, true}) {
+			for (const bool padded: {false, true}) {
+				const uint32_t       pitch = padded ? 12 : 8;
+				std::vector<uint8_t> source(pitch * 2, 0xee);
+				std::vector<uint8_t> expected;
+				for (uint32_t y = 0; y < 2; ++y) {
+					for (uint32_t x = 0; x < 2; ++x) {
+						const auto* pixel = rgba.data() + (y * 2 + x) * 4;
+						auto*       input = source.data() + y * pitch + x * 4;
+						input[0]          = pixel[bgr ? 2 : 0];
+						input[1]          = pixel[1];
+						input[2]          = pixel[bgr ? 0 : 2];
+						input[3]          = pixel[3];
+						expected.insert(expected.end(), pixel, pixel + (alpha ? 4 : 3));
+					}
+				}
+				std::array<uint8_t, 1024> png {};
+				auto param = EncodeParam(source.data(), source.size(), png.data(), png.size());
+				param.pixel_format =
+				    bgr ? PNG_ENC_PIXEL_FORMAT_B8G8R8A8 : PNG_ENC_PIXEL_FORMAT_R8G8B8A8;
+				param.color_space = alpha ? PNG_ENC_COLOR_SPACE_RGBA : PNG_ENC_COLOR_SPACE_RGB;
+				param.image_pitch = pitch;
+				PngEncOutputInfo info {};
+				const auto       size = PngEncEncode(encoder.handle, &param, &info);
+				CHECK(size > 0 && info.data_size == static_cast<uint32_t>(size));
+				CHECK(info.processed_height == 2);
+				int   width = 0, height = 0, components = 0;
+				auto* decoded =
+				    stbi_load_from_memory(png.data(), size, &width, &height, &components, 0);
+				CHECK(decoded != nullptr);
+				CHECK(width == 2 && height == 2 && components == (alpha ? 4 : 3));
+				CHECK(std::memcmp(decoded, expected.data(), expected.size()) == 0);
+				stbi_image_free(decoded);
+			}
+		}
+	}
+}
+
+void TestFilterMasks() {
+	Encoder                            encoder;
+	alignas(4) std::array<uint8_t, 16> pixels {};
+	std::array<uint8_t, 1024>          png {};
+	auto param = EncodeParam(pixels.data(), pixels.size(), png.data(), png.size());
+	for (uint16_t mask = 0; mask <= 15; ++mask) {
+		param.filter_type = mask;
+		const auto size   = PngEncEncode(encoder.handle, &param, nullptr);
+		CHECK(size > 0);
+		CHECK(std::memcmp(png.data() + 37, "IDAT", 4) == 0);
+		std::array<char, 18> filtered {};
+		CHECK(stbi_zlib_decode_buffer(filtered.data(), filtered.size(),
+		                              reinterpret_cast<const char*>(png.data() + 41),
+		                              ReadBe32(png.data() + 33)) == filtered.size());
+		for (size_t row = 0; row < 2; ++row) {
+			const auto filter = static_cast<uint8_t>(filtered[row * 9]);
+			CHECK(filter <= 4);
+			CHECK(filter == 0 ? (mask == 0 || mask == 15) : (mask & (1u << (filter - 1u))) != 0);
+		}
+	}
+	param.filter_type = 16;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_PARAM);
+	for (uint32_t count = 0; count <= 4; ++count) {
+		Encoder limited(count);
+		for (uint16_t mask = 0; mask <= 15; ++mask) {
+			param.filter_type = mask;
+			const auto result = PngEncEncode(limited.handle, &param, nullptr);
+			CHECK(std::popcount(mask) <= count ? result > 0
+			                                   : result == PNG_ENC_ERROR_INVALID_PARAM);
+		}
+	}
+}
+
+void TestOutputCapacity() {
+	Encoder                            encoder;
+	alignas(4) std::array<uint8_t, 16> pixels {};
+	std::array<uint8_t, 1024>          png {};
+	auto       param = EncodeParam(pixels.data(), pixels.size(), png.data(), png.size());
+	const auto size  = PngEncEncode(encoder.handle, &param, nullptr);
+	CHECK(size > 0);
+	std::vector<uint8_t> guarded(size + 2, 0xa5);
+	param.png_mem_addr = guarded.data() + 1;
+	param.png_mem_size = size;
+	PngEncOutputInfo info {};
+	CHECK(PngEncEncode(encoder.handle, &param, &info) == size);
+	CHECK(guarded.front() == 0xa5 && guarded.back() == 0xa5);
+	CHECK(std::memcmp(guarded.data() + 1, png.data(), size) == 0);
+	std::fill(guarded.begin(), guarded.end(), 0xa5);
+	param.png_mem_size = size - 1;
+	CHECK(PngEncEncode(encoder.handle, &param, &info) == PNG_ENC_ERROR_DATA_OVERFLOW);
+	CHECK(info.data_size == 0 && info.processed_height == 0);
+	CHECK(std::all_of(guarded.begin(), guarded.end(), [](uint8_t byte) { return byte == 0xa5; }));
+}
+
+void TestValidation() {
+	Encoder           encoder;
+	PngEncCreateParam create {sizeof(PngEncCreateParam), 0, 2, 4};
+	CHECK(PngEncQueryMemorySize(nullptr) == PNG_ENC_ERROR_INVALID_ADDR);
+	CHECK(PngEncCreate(&create, nullptr, sizeof(PngEncContext), &encoder.handle) ==
+	      PNG_ENC_ERROR_INVALID_ADDR);
+	CHECK(PngEncCreate(&create, &encoder.context, 0, &encoder.handle) ==
+	      PNG_ENC_ERROR_INVALID_SIZE);
+	create.max_image_width = 0;
+	CHECK(PngEncQueryMemorySize(&create) == PNG_ENC_ERROR_INVALID_SIZE);
+	create.max_image_width = PNG_ENC_MAX_IMAGE_WIDTH + 1;
+	CHECK(PngEncQueryMemorySize(&create) == PNG_ENC_ERROR_INVALID_SIZE);
+	create.max_image_width = 2;
+	create.this_size       = 0;
+	CHECK(PngEncQueryMemorySize(&create) == PNG_ENC_ERROR_INVALID_PARAM);
+	create.this_size         = sizeof(create);
+	create.max_filter_number = 5;
+	CHECK(PngEncQueryMemorySize(&create) == PNG_ENC_ERROR_INVALID_PARAM);
+	alignas(4) std::array<uint8_t, 16> pixels {};
+	std::array<uint8_t, 1024>          png {};
+	const auto valid = EncodeParam(pixels.data(), pixels.size(), png.data(), png.size());
+	CHECK(PngEncEncode(nullptr, &valid, nullptr) == PNG_ENC_ERROR_INVALID_HANDLE);
+	CHECK(PngEncEncode(encoder.handle, nullptr, nullptr) == PNG_ENC_ERROR_INVALID_PARAM);
+	for (const auto width: {0u, PNG_ENC_MAX_IMAGE_WIDTH + 1, 0x40000000u, 0xffffffffu}) {
+		auto param        = valid;
+		param.image_width = width;
+		CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_SIZE);
+	}
+	for (const auto height: {0u, PNG_ENC_MAX_IMAGE_HEIGHT + 1, 0xffffffffu}) {
+		auto param           = valid;
+		param.image_width    = 1;
+		param.image_height   = height;
+		param.image_mem_size = 0xffffffffu;
+		CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_SIZE);
+	}
+	for (const auto height: {262144u, 400000u, 0x7ffffu}) {
+		auto param           = valid;
+		param.image_width    = 1024;
+		param.image_height   = height;
+		param.image_pitch    = 4096;
+		param.image_mem_size = 0xffffffffu;
+		CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_SIZE);
+	}
+	for (const auto pitch: {0u, 7u, 9u, 0xfffffffcu}) {
+		auto param        = valid;
+		param.image_pitch = pitch;
+		CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_SIZE);
+	}
+	auto param           = valid;
+	param.image_pitch    = 12;
+	param.image_mem_size = 20;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_SIZE);
+	param = valid;
+	param.image_mem_size--;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_SIZE);
+	param                = valid;
+	param.image_mem_addr = nullptr;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_ADDR);
+	param.image_mem_addr = pixels.data() + 1;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_ADDR);
+	param           = valid;
+	param.bit_depth = 16;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_PARAM);
+	param             = valid;
+	param.clut_number = 1;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_PARAM);
+	param                   = valid;
+	param.compression_level = 10;
+	CHECK(PngEncEncode(encoder.handle, &param, nullptr) == PNG_ENC_ERROR_INVALID_PARAM);
+}
+} // namespace
+
+int main() {
+	TestPixels();
+	TestFilterMasks();
+	TestOutputCapacity();
+	TestValidation();
+	std::puts("PNG encoder tests passed");
+}

tests/ResourceMaterializationTests.cpp

@@ -219,6 +219,56 @@ void TestUnbasedFlatCacheHitMaterializes() {
         "unbased FLAT plan produced unexpected descriptors");
 }
 
+void TestWrittenDescriptorUsesStrictReaderOnce() {
+  using namespace Libs::Graphics::ShaderRecompiler::IR;
+  Program program;
+  program.stage = Libs::Graphics::ShaderType::Compute;
+  program.srt_plan_complete = true;
+  program.resource_tracking_complete = true;
+  auto &block = AddValueBlock(program);
+  program.memory_info.push_back({.kind = ResourceKind::ScalarAddress});
+  auto &handle = block.AppendNewInst(ValueOpcode::GetAddressResource,
+                                     {Value(0x1000u), Value(0u)});
+  auto &offset = block.AppendNewInst(ValueOpcode::GetUserData,
+                                     {Value(static_cast<ScalarReg>(0))});
+  auto &read = block.AppendNewInst(ValueOpcode::LoadAddressU32,
+      {Value(&handle), Value(&offset), Value(0u), Value(false)});
+  read.SetFlags(MemoryFlags{.index = 0});
+  DescriptorSource source;
+  source.dwords = {Value(&read), Value(0u), Value(4u), Value(0u)};
+  source.dword_count = 4;
+  program.descriptor_sources.push_back(source);
+  program.info.buffers.push_back({.source = 0, .written = true});
+  auto plan = ExtractResourcePlan(program);
+  struct Reads { uint32_t ordinary = 0; uint32_t strict = 0; bool clean = false; } reads;
+  const std::array<uint32_t, 1> user_data{4u};
+  const SrtRuntime runtime{
+      .user_data = user_data,
+      .read_memory = +[](void *data, uint64_t, std::span<uint32_t> words) {
+        ++static_cast<Reads *>(data)->ordinary;
+        words[0] = 0x8000u;
+        return true;
+      },
+      .userdata = &reads,
+      .read_specialization_memory = +[](void *data, uint64_t address, std::span<uint32_t> words) {
+        auto &reads = *static_cast<Reads *>(data);
+        ++reads.strict;
+        if (!reads.clean || address != 0x1004u) return false;
+        words[0] = 0x8000u;
+        return true;
+      }};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
+            reads.ordinary == 0 && reads.strict == 1,
+        "GPU-dirty dynamic writable descriptor bypassed strict provenance");
+  reads.clean = true;
+  reads.strict = 0;
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            reads.ordinary == 0 && reads.strict == 1 && snapshot.buffers[0].dwords[0] == 0x8000u,
+        "writable descriptor was evaluated twice or scalar EXEC suppressed its read");
+}
+
 void TestFailedMaterializationRejectsStage() {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   auto plan = UserDataBufferPlan();
@@ -262,6 +312,7 @@ int main() {
   TestMappedSrtUsesDirectReaderByDefault();
   TestIntegerRuntimeValueFollowsSrtReads();
   TestUnbasedFlatCacheHitMaterializes();
+  TestWrittenDescriptorUsesStrictReaderOnce();
   TestFailedMaterializationRejectsStage();
   TestMixedSamplerDuplicatesTheCorrectSnapshot();
   std::puts("ResourceMaterializationTests: all cases passed");

tests/ResourceTrackingTests.cpp

@@ -1386,11 +1386,17 @@ void TestUniformizedMaterialImageKeys() {
                 specialization.images[0].indirect_mapping_offset + 3u] == 4096u,
         "material mask did not limit sparse descriptor reads");
   user_data[8] = first_material;
-  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
-        "written buffer alias with a material key was accepted");
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            std::ranges::find(snapshot.specialization_reads,
+                std::pair<uint64_t, uint64_t>{first_material, 4}) !=
+                snapshot.specialization_reads.end(),
+        "material key read was omitted from the renderer write-overlap check");
   user_data[8] = first_table;
-  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
-        "written buffer alias with an image record was accepted");
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            std::ranges::find(snapshot.specialization_reads,
+                std::pair<uint64_t, uint64_t>{first_table, 32}) !=
+                snapshot.specialization_reads.end(),
+        "image descriptor read was omitted from the renderer write-overlap check");
   CheckFatal([&] { make_plan(Variant::WrongUpdate); }, "not a valid runtime value",
              "non-clearing material mask was accepted");
   CheckFatal([&] { make_plan(Variant::WrongEquality); }, "not a valid runtime value",
@@ -2785,6 +2791,108 @@ void TestConditionalBufferMaterialization() {
   CheckActive();
 }
 
+void TestGuardedScalarDescriptorReads() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  for (const bool shared : {false, true}) {
+    for (const bool exec : {false, true}) {
+      Fixture fixture;
+      auto *entry = fixture.block;
+      auto *optional = fixture.AddBlock();
+      auto *done = fixture.AddBlock();
+      entry->AddBranch(optional);
+      entry->AddBranch(done);
+      optional->AddBranch(done);
+      fixture.program.block_info[0].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 1, .false_block = 2};
+      fixture.program.block_info[1].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 2};
+      fixture.program.block_info[2].terminator.kind = CFG::TerminatorKind::Return;
+      const auto control = fixture.Buffer({fixture.UserData(0), fixture.UserData(1),
+                                          fixture.UserData(2), fixture.UserData(3)});
+      MemoryInfo scalar;
+      scalar.kind = ResourceKind::ScalarBuffer;
+      scalar.offset = 28;
+      const auto flag = fixture.Emit(ValueOpcode::ReadConstBuffer, {control, Value(0u)},
+                                     fixture.AddMemory(scalar, 4));
+      const auto lane = fixture.Emit(ValueOpcode::GetBuiltin,
+          {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
+      const auto varying = fixture.Emit(ValueOpcode::INotEqual32, {lane, Value(0u)});
+      const auto enabled = fixture.Emit(ValueOpcode::INotEqual32, {flag, Value(0u)});
+      fixture.program.block_info[0].condition =
+          fixture.Emit(ValueOpcode::LogicalAnd, {varying, enabled});
+      const auto root = fixture.Address(fixture.UserData(4), Value(0u));
+      const auto Load = [&](Block *block) {
+        fixture.block = block;
+        MemoryInfo address;
+        address.kind = ResourceKind::ScalarAddress;
+        const auto pointer = fixture.Emit(ValueOpcode::LoadAddressU32,
+            {root, Value(0u), Value(0u), Value(exec)}, fixture.AddMemory(address, 8));
+        const auto table = fixture.Address(pointer, Value(0u));
+        std::array<Value, 4> words;
+        for (uint32_t word = 0; word < words.size(); ++word) {
+          address.offset = 40 + word * 4;
+          words[word] = fixture.Emit(ValueOpcode::LoadAddressU32,
+              {table, Value(0u), Value(0u), Value(exec)}, fixture.AddMemory(address, 12));
+        }
+        scalar.offset = 0;
+        fixture.Emit(ValueOpcode::ReadConstBuffer, {fixture.Buffer(words), Value(0u)},
+                     fixture.AddMemory(scalar, 16));
+      };
+      Load(optional);
+      if (shared) Load(done);
+      fixture.block = done;
+      MemoryInfo store;
+      store.kind = ResourceKind::Buffer;
+      fixture.Emit(ValueOpcode::StoreBufferU32,
+          {fixture.Buffer({Value(0x9000u), Value(0u), Value(4u), Value(0u)}),
+           Value(0u), Value(0u), Value(0u), Value(1u), Value(true)}, fixture.AddMemory(store, 20));
+      fixture.PlanAndTrack();
+      auto plan = ExtractResourcePlan(fixture.program);
+      Check(!exec || (!plan.srt_reads.empty() && !plan.control_flow[1].srt_reads.empty() &&
+                      (!shared || !plan.control_flow[2].srt_reads.empty())),
+            "guarded shared-read fixture lost its flattened execution sites");
+      struct Memory { LinearTestMemory data; uint32_t null_reads = 0; } memory;
+      const auto Read = +[](void *data, uint64_t address, std::span<uint32_t> words) {
+        auto &memory = *static_cast<Memory *>(data);
+        if (address == 40) ++memory.null_reads;
+        return ReadLinearTestMemory(&memory.data, address, words);
+      };
+      const std::array<uint32_t, 5> user_data{0x1000u, 0u, 64u, 0u, 0x1040u};
+      const SrtRuntime runtime{.user_data = user_data, .read_memory = Read,
+                               .userdata = &memory, .read_specialization_memory = Read};
+      memory.data.words[0xa8 / 4] = 0x2000u;
+      memory.data.words[0xb0 / 4] = 4u;
+      ResourceSnapshot snapshot;
+      ResourceSpecialization specialization;
+      if (!shared) {
+        Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 0 &&
+                  std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }),
+              "disabled feature speculatively dereferenced its null BVH table");
+        memory.data.words[7] = 1;
+        Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 1,
+              "active feature accepted an unreadable BVH table");
+      }
+      memory.data.words[0x40 / 4] = 0x1080u;
+      memory.data.watched_address = 0x10a8u;
+      memory.data.watched_reads = 0;
+      Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+                std::ranges::any_of(snapshot.buffers, [](const auto& descriptor) {
+                  return descriptor.dwords[0] == 0x2000u;
+                }) &&
+                (!exec || (memory.data.watched_reads == 1 &&
+                           std::ranges::find(snapshot.flattened_srt, 0x2000u) != snapshot.flattened_srt.end())),
+            "active or shared descriptor read was omitted after refreshing the cached plan");
+      if (!shared) {
+        memory.data.words[7] = 0;
+        memory.data.words[0x40 / 4] = 0;
+        Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 1 &&
+                  std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }),
+              "cached plan retained reads after the feature was disabled");
+      }
+    }
+  }
+}
+
 void TestConservativeBufferReachability() {
   std::array<uint32_t, 8> user_data{
       0x1000, 16u << 16u, 1, 0x4dfac,
@@ -2797,11 +2905,14 @@ void TestConservativeBufferReachability() {
     auto plan = ConditionalBufferPlan(use);
     ResourceSnapshot snapshot;
     ResourceSpecialization specialization;
+    std::array<uint32_t, 4> expected{};
+    if (use != ConditionalBufferUse::Writable)
+      std::copy_n(user_data.begin() + 4, expected.size(), expected.begin());
     Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
               snapshot.buffers.size() == 2 &&
-              std::equal(user_data.begin() + 4, user_data.end(),
+              std::equal(expected.begin(), expected.end(),
                          snapshot.buffers[1].dwords.begin()),
-          "shared, loop-dependent, or writable-alias resource was pruned");
+          "shared or loop-dependent resource was pruned, or inactive scalar branch retained");
   }
 }
 
@@ -3187,6 +3298,7 @@ int main() {
     Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma);
     Run("buffer swizzle specialization", TestBufferSwizzleSpecialization);
     Run("conditional buffer materialization", TestConditionalBufferMaterialization);
+    Run("guarded scalar descriptor reads", TestGuardedScalarDescriptorReads);
     Run("conservative buffer reachability", TestConservativeBufferReachability);
     Run("conditional indirect image", TestConditionalIndirectImageMaterialization);
     Run("shader info and bindings", TestShaderInfoAndBindingLayout);

tests/ShaderRayTracingGpuTests.inc

@@ -0,0 +1,229 @@
+TestCase FlatStackApertures(u32 wave_size) {
+  TestCase test;
+  test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64";
+  auto &code = test.code;
+  code = {EncodeSop1(0x04, 8, 237), // private base
+          EncodeSop1(0x04, 10, 235), // shared base
+          EncodeVop2(0x1a, 20, InlineU32(2), 0),
+          EncodeVop2(0x1b, 22, InlineU32(1), 0),
+          EncodeVop2(0x1a, 22, InlineU32(31), 22)};
+  AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22)); // Clamp the stack address to the local apertures.
+  AppendVMovU32(&code, 30, 100);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeFlat0(0x1c, 1), EncodeFlat1(0, 0x7d, 30, 20)});
+  AppendVMovU32(&code, 30, 200);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeDs0(0x0d), EncodeDs1(0, 30, 20)});
+  AppendVMovU32(&code, 30, 300);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)});
+  // FLAT writes must alias both explicit memory instructions, per invocation.
+  code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20),
+                           EncodeDs0(0x36), EncodeDs1(25, 0, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 0);
+  AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, wave_size);
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 2 * wave_size);
+  // Inactive lanes must preserve both stores and destination VGPRs.
+  AppendSMovLiteral(&code, 126, 0x55555555);
+  if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x55555555);
+  code.push_back(EncodeVop2(0x25, 22, InlineU32(1), 22));
+  AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22));
+  AppendVMovU32(&code, 30, 400);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)});
+  code.insert(code.end(), {EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  code.push_back(EncodeSop1(0x04, 126, 193)); // -1 sign-extends across EXEC
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 3 * wave_size);
+  // Noncanonical addresses outside either aperture, aperture overflow, and
+  // offsets outside allocated local storage must not wrap into valid storage.
+  AppendVMovLiteral(&code, 21, 0x78000000);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 4 * wave_size);
+  code.push_back(EncodeVop1(0x01, 21, 11));
+  AppendVMovLiteral(&code, 20, 0xfffffffc);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0, 4), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0, 4), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 5 * wave_size);
+  AppendVMovU32(&code, 20, wave_size * 4);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 6 * wave_size);
+  AppendVMovU32(&code, 20, 1);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 9 * wave_size);
+  code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 0));
+  code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20),
+                           EncodeDs0(0x36), EncodeDs1(25, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 7 * wave_size);
+  AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, 8 * wave_size);
+  AppendEnd(&code);
+  test.expected.resize(10 * wave_size);
+  for (u32 lane = 0; lane < wave_size; ++lane) {
+    const bool shared = lane & 1;
+    test.expected[lane] = (shared ? 100 : 300) + lane;
+    test.expected[wave_size + lane] = (shared ? 300 : 200) + lane;
+    test.expected[2 * wave_size + lane] = 300 + lane;
+    test.expected[3 * wave_size + lane] = (shared ? 300 : 400) + lane;
+    test.expected[7 * wave_size + lane] = (shared ? 100 : 400) + lane;
+    test.expected[8 * wave_size + lane] = (shared ? 300 : 200) + lane;
+  }
+  test.compute_info.wave_size = wave_size;
+  test.compute_info.threads_num[0] = wave_size;
+  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
+  test.compute_info.thread_ids_num = 1;
+  test.compute_info.scratch_size_dwords = wave_size;
+  test.compute_info.lds_size_dwords = wave_size;
+  test.has_compute_info = true;
+  test.forbidden_spirv = {"get_bda_pointer"};
+  return test;
+}
+
+TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) {
+  TestCase test;
+  test.name = barycentrics ? (sorted ? "BvhBarycentricSorted" : "BvhBarycentricUnsorted")
+                          : (sorted ? "BvhTriangleIdSorted" : "BvhTriangleIdUnsorted");
+  if (grow != 0) test.name = "BvhBoxExitGrowth";
+  constexpr u32 NodeStart = 1024, OutputStart = 512, LaneCount = 32;
+  constexpr uint64_t GuestBase = 0x10000;
+  const auto bits = [](float value) { return std::bit_cast<u32>(value); };
+  constexpr u32 Infinity = 0x7f800000, Invalid = UINT32_MAX, Flag = 0x04040404;
+  test.initial.resize(NodeStart + 258 * 16);
+  auto &data = test.initial;
+  // Pack five float3 vertices and the triangle flag into a 64-byte node.
+  const std::array<float, 15> vertices{-1, -1, 2, 1, -1, 2, 0, 1, 2, 2, 1, 2, 0, 3, 2};
+  for (u32 i = 0; i < vertices.size(); ++i) data[NodeStart + i] = bits(vertices[i]);
+  data[NodeStart + 15] = Flag;
+  constexpr std::array<u32, 4> children{0x88, 0x90, 0x98, 0xa0};
+  constexpr std::array<float, 4> z{5, 1, 3, 7};
+  constexpr std::array<u32, 4> half_z{0x4500, 0x3c00, 0x4200, 0x4700};
+  constexpr std::array<u32, 4> half_far{0x4600, 0x4000, 0x4400, 0x4800};
+  for (u32 child = 0; child < 4; ++child) {
+    data[NodeStart + 16 + child] = data[NodeStart + 32 + child] = children[child];
+    const auto fp16 = NodeStart + 20 + child * 3;
+    data[fp16] = 0xbc00bc00; // min.xy = -1,-1
+    data[fp16 + 1] = 0x3c000000 | half_z[child];
+    data[fp16 + 2] = (half_far[child] << 16) | 0x3c00;
+    const auto fp32 = NodeStart + 36 + child * 6;
+    const std::array<float, 6> bounds{-1, -1, z[child], 1, 1, z[child] + 1};
+    for (u32 i = 0; i < 6; ++i) data[fp32 + i] = bits(bounds[i]);
+  }
+  // A grazing ray enters one ULP after exiting on another axis.
+  for (u32 child = 0; child < 4; ++child) {
+    data[NodeStart + 64 + child] = children[child];
+    const std::array<float, 6> bounds = child == 0
+        ? std::array<float, 6>{0, 0, std::nextafter(1.0f, 2.0f), 1, 2, 2}
+        : child == 1 ? std::array<float, 6>{-1, 1, -1, -0.0f, 2, 2}
+                     : std::array<float, 6>{1, 1, 1, -1, -1, -1};
+    for (u32 i = 0; i < 6; ++i) data[NodeStart + 68 + child * 6 + i] = bits(bounds[i]);
+  }
+  // An FP32 node straddles two guest pages with a 64-byte gap in physical backing.
+  std::copy_n(data.begin() + NodeStart + 32, 16, data.begin() + NodeStart + 255 * 16);
+  std::copy_n(data.begin() + NodeStart + 48, 16, data.begin() + NodeStart + 257 * 16);
+  const std::array<std::array<float, 2>, 4> origins{{{0, 0}, {1, 0}, {.5f, 1.5f}, {-.25f, 1}}};
+  std::array<std::array<u32, 4>, LaneCount> expected{};
+  for (u32 lane = 0; lane < LaneCount; ++lane) {
+    const auto kind = lane % 8;
+    const auto input = lane * 16;
+    data[input] = kind < 4 ? kind : kind == 4 ? 12 : kind == 5 ? 21 : kind == 6 ? 6 : 0x1000;
+    data[input + 1] = bits(6);
+    data[input + 2] = bits(kind < 4 ? origins[kind][0] : 0);
+    data[input + 3] = bits(kind < 4 ? origins[kind][1] : 0);
+    data[input + 7] = bits(1);
+    data[input + 8] = data[input + 9] = Infinity;
+    data[input + 10] = bits(1);
+    if (kind < 4) {
+      const float denominator = kind == 3 ? -2 : -4;
+      const float i = kind == 0 || kind == 3 ? -1 : -2;
+      const float j = kind == 3 ? -.5f : -1;
+      expected[lane] = {bits(2 * denominator), bits(denominator),
+                        barycentrics ? bits(i) : Flag + kind,
+                        barycentrics ? bits(j) : 1};
+    } else if (kind < 6) {
+      expected[lane] = sorted ? std::array<u32, 4>{children[1], children[2], children[0], Invalid}
+                              : std::array<u32, 4>{children[0], children[1], children[2], Invalid};
+    } else {
+      expected[lane] = kind == 6 ? std::array<u32, 4>{Invalid, Invalid, Invalid, Invalid}
+                                 : std::array<u32, 4>{Infinity, bits(1), Invalid, 0};
+    }
+    if (lane >= 8 && lane < 12) {
+      data[input + 2] = 0x7fc00000; // invalid ray origin
+      expected[lane] = {Infinity, bits(1), Invalid, 0};
+    } else if (lane == 12 || lane == 13) {
+      data[input + 1] = bits(1); // nearest face equals the exclusive extent
+      expected[lane] = {Invalid, Invalid, Invalid, Invalid};
+    } else if (lane >= 16 && lane < 20) {
+      const u32 axis = lane & 1;
+      data[input + 2 + axis] = bits(origins[kind][axis] - 4);
+      data[input + 5 + axis] = bits(2);
+      data[input + 8 + axis] = bits(.5f);
+      // Dominant X/Y projection scales all intersection numerators and the
+      // denominator by four; the distance and barycentrics remain unchanged.
+      for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i)
+        expected[lane][i] = bits(std::bit_cast<float>(expected[lane][i]) * 4);
+    } else if (lane == 22) {
+      data[input] = 37; // node 4, FP32
+      data[input + 1] = bits(10);
+      for (u32 i = 5; i <= 10; ++i) data[input + i] = bits(1);
+      expected[lane] = {grow ? children[0] : Invalid, Invalid, Invalid, Invalid};
+    } else if (lane >= 24 && lane < 28) {
+      data[input + 4] = bits(4);
+      data[input + 7] = data[input + 10] = bits(-1);
+      for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i)
+        expected[lane][i] = bits(-std::bit_cast<float>(expected[lane][i]));
+    } else if (lane == 28) {
+      data[input] = 0;
+      data[input + 3] = bits(-1); // horizontal triangle edge: top-left rule rejects
+      expected[lane] = {Infinity, bits(1), barycentrics ? bits(-2) : Flag,
+                        barycentrics ? bits(-2) : 0};
+    } else if (lane == 29) {
+      data[input] = (255 << 3) | 5;
+    }
+  }
+  test.expected = test.initial;
+  for (u32 lane = 0; lane < LaneCount; ++lane) {
+    std::copy(expected[lane].begin(), expected[lane].end(),
+              test.expected.begin() + OutputStart + lane * 4);
+    auto bounded = expected[lane];
+    if (lane % 8 == 5) bounded = {Invalid, Invalid, Invalid, Invalid};
+    if (lane >= 20) bounded = {0x12345678, 0x12345679, 0x1234567a, 0x1234567b};
+    std::copy(bounded.begin(), bounded.end(),
+              test.expected.begin() + OutputStart + LaneCount * 4 + lane * 4);
+  }
+  auto &code = test.code;
+  // The resource is a GPU value, not a materialized host image descriptor.
+  AppendSMovLiteral(&code, 8, GuestBase >> 8);
+  AppendSMovLiteral(&code, 9, (sorted ? 0x80000000 : 0) | (grow << 23));
+  AppendSMovLiteral(&code, 10, 256);
+  AppendSMovLiteral(&code, 11, barycentrics ? 0x81000000 : 0x80000000);
+  code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0));
+  for (u32 offset : {0u, 4u, 8u}) {
+    code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4),
+                             EncodeMubuf1(4 + offset, 0, 1)});
+  }
+  code.insert(code.end(), {EncodeMimg0(0xe6, 15, 0, false, 0, true),
+                           EncodeMimg1(20, 4, 2)});
+  code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
+  for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 20 + i, 2, OutputStart + i);
+  // NSA addressing, the FP32 second-block bound, and inactive destination lanes.
+  AppendSMovLiteral(&code, 10, 2);
+  for (u32 i = 0; i < 4; ++i) AppendVMovLiteral(&code, 24 + i, 0x12345678 + i);
+  AppendSMovLiteral(&code, 126, 0x000fffff);
+  code.insert(code.end(), {EncodeMimg0(0xe6, 15, 3, false, 0, true),
+                           EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, 0x00000e0d});
+  code.push_back(EncodeSop1(0x04, 126, 193));
+  for (u32 i = 0; i < 4; ++i)
+    AppendStoreVgprAtLaneDwordOffset(&code, 24 + i, 2, OutputStart + LaneCount * 4 + i);
+  AppendEnd(&code);
+  test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)},
+                       {GuestBase + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}};
+  test.compute_info.wave_size = LaneCount;
+  test.compute_info.threads_num[0] = LaneCount;
+  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
+  test.compute_info.thread_ids_num = 1;
+  test.has_compute_info = true;
+  test.required_spirv = {"get_bda_pointer"};
+  return test;
+}

tests/ShaderRayTracingTests.inc

@@ -1,87 +1,50 @@
-void TestRayTracingDispatchDetection() {
+void TestRayTracingInstructions() {
   namespace D = ShaderRecompiler::Decoder;
-  const auto check_decoded = [](std::span<const uint32_t> code, bool expected,
-                                const char *message) {
-    D::Program decoded;
-    D::DecodeProgram(code, decoded);
-    Check(decoded.has_bvh == expected, message);
-    if (expected) {
-      const auto &last = decoded.instructions.back();
-      Check(last.family == D::Family::MIMG &&
-                (last.opcode_id == 0xe6u || last.opcode_id == 0xe7u),
-            "decoder continued past the first BVH instruction");
-      const auto result = ShaderRecompiler::TranslateProgram(
-          code, MakeCompileOptions(ShaderType::Compute));
-      Check(result.skip_dispatch && result.program.blocks.empty() && result.cfg_dump.empty(),
-            "BVH compute shader reached IR translation");
-    }
-  };
-  const auto check = [&](std::initializer_list<uint32_t> code, bool expected,
-                         const char *message) {
-    check_decoded(std::span<const uint32_t>{code.begin(), code.size()}, expected, message);
-  };
-
-  check({0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
-        "captured contiguous BVH instruction did not skip dispatch");
-  check({0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
-         EncodeSopp(0x01)}, true,
-        "captured NSA BVH instruction did not skip dispatch");
-  check({0xf19c9f01u, 0x00081014u, EncodeSopp(0x01)}, true,
-        "RDNA2 BVH64 instruction did not skip dispatch");
-  check({0xbe8e210eu, 0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
-        "unsupported SWAPPC before BVH prevented dispatch detection");
-  check({0xf1989f01u, 0x00081014u, 0xc0000000u, EncodeSopp(0x01)}, true,
-        "unknown instruction after BVH prevented dispatch skipping");
-
-  for (uint32_t opcode : {0xe6u, 0xe7u}) {
-    for (uint32_t nsa = 0; nsa < 4; ++nsa) {
-      for (uint32_t slc = 0; slc < 2; ++slc) {
-        std::array<uint32_t, 6> code{
-            EncodeMimg0(opcode, 0xf, true) | (nsa << 1u) | (slc << 25u),
-            EncodeMimg1(0, 0, 0, 0)};
-        code[2 + nsa] = EncodeSopp(0x01);
-        check_decoded(std::span(code).first(3 + nsa), true,
-                      "BVH cache flags or NSA length prevented dispatch detection");
-      }
-    }
+  const std::array code{
+      0xf1989f01u, 0x00081014u,
+      EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(16, 0, 0),
+      0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
+      EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(16, 0, 0),
+      EncodeSopp(0x01)};
+  D::Program decoded;
+  D::DecodeProgram(code, decoded);
+  Check(decoded.instructions.size() == 5 &&
+            decoded.instructions.back().opcode == D::Opcode::S_ENDPGM,
+        "decoder stopped before instructions following BVH intersections");
+  for (const auto index : {0u, 2u}) {
+    const auto &inst = decoded.instructions[index];
+    Check(inst.opcode == D::Opcode::IMAGE_BVH_INTERSECT_RAY &&
+              inst.image_address_components == 11 && inst.data_dwords == 4 && inst.image_r128,
+          "BVH32 instruction lost its ray or descriptor width");
   }
-  for (uint32_t opcode : {0x66u, 0x67u, 0xe4u, 0xe5u, 0xe8u, 0xe9u}) {
-    check({EncodeMimg0(opcode, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
-          false, "non-BVH image opcode was incorrectly skipped");
+  Check(decoded.instructions[2].image_nsa_addr[9] == 58,
+        "BVH NSA ray components were not decoded");
+  const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute));
+  Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 2 &&
+            result.program.info.uses_dma && result.program.info.images.empty() &&
+            result.program.info.samplers.empty(),
+        "BVH intersections did not use GPU descriptors and raw guest memory");
+  CheckSpirvBinaryValidates(result.spirv);
+
+  const std::array literal{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
+  D::DecodeProgram(literal, decoded);
+  Check(decoded.instructions.size() == 2 && decoded.instructions[0].opcode == D::Opcode::S_MOV_B32,
+        "BVH-shaped scalar literal was decoded as an instruction");
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+  for (const auto words : {std::array{0xf19c9f01u, 0x00081014u}, // BVH64
+                           std::array{0xf1989f01u, 0x40081014u}, // A16
+                           std::array{0xf1981f01u, 0x00081014u}, // not R128
+                           std::array{0xf1989f01u, 0x80081014u}}) { // D16
+    const std::array unsupported{words[0], words[1], EncodeSopp(0x01)};
+    ExpectFatal([&]() { (void)RecompileForTest(unsupported, MakeCompileOptions(ShaderType::Compute)); },
+                "unsupported BVH encoding was silently accepted");
   }
-
-  check({EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}, false,
-        "scalar literal was mistaken for a BVH instruction");
-  check({EncodeMimg0(0x20, 0xf) | 6u, EncodeMimg1(0, 0, 0, 0),
-         0xf1989f01u, 0xf19c9f01u, 0u, EncodeSopp(0x01)}, false,
-        "NSA address words were mistaken for BVH instructions");
-  check({EncodeMimg0(0x20, 0xf), 0xf1989f01u, EncodeSopp(0x01)}, false,
-        "MIMG operand word was mistaken for a BVH instruction");
-  check({EncodeSopp(0x01), 0xf1989f01u, 0x00081014u}, false,
-        "data after the shader end was mistaken for a BVH instruction");
-  check({EncodeMimg0(0x20, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
-        false, "ordinary image shader was incorrectly skipped");
-
-  // An end instruction on one branch does not end another branch's code.
-  check({EncodeSopp(0x04, 1), EncodeSopp(0x01), 0xf1989f01u, 0x00081014u,
-         EncodeSopp(0x01)}, true,
-        "BVH at a branch target after an early end was not detected");
-
-  const std::array bvh_code{0xf1989f01u, 0x00081014u, 0xf19c9f01u, 0x00081014u,
-                            EncodeSopp(0x01)};
-  const std::array ordinary_code{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
-  D::Program reused;
-  D::DecodeProgram(bvh_code, reused);
-  Check(reused.has_bvh && reused.instructions.size() == 1,
-        "decoder did not stop at the first BVH instruction");
-  D::DecodeProgram(ordinary_code, reused);
-  Check(!reused.has_bvh, "decoder retained stale BVH metadata");
-  for (const auto stage : {ShaderType::Compute, ShaderType::Vertex, ShaderType::Pixel}) {
-    const auto options = MakeCompileOptions(stage);
-    Check(!ShaderRecompiler::TranslateProgram(ordinary_code, options).skip_dispatch,
-          "BVH-shaped literal marked an ordinary shader for skipping");
-    const auto compiled = RecompileForTest(ordinary_code, options);
-    Check(!compiled.spirv.empty(), "ordinary shader did not compile");
-    CheckSpirvBinaryValidates(compiled.spirv);
+  for (const auto nsa : {1u, 2u}) {
+    std::array<uint32_t, 5> truncated{0xf1989f01u | (nsa << 1), 0x00081014u};
+    truncated[2 + nsa] = EncodeSopp(0x01);
+    ExpectFatal([&]() {
+      (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute));
+    }, "BVH NSA instruction accepted fewer than eleven ray addresses");
   }
+#endif
 }

tests/shaderCfgTests.cpp

@@ -1884,37 +1884,41 @@ void TestSopkCompareImmediateExtension() {
   }
 }
 
-void TestDisabledSystemDebugBranch() {
+void TestDisabledDebugBranches() {
   using namespace ShaderRecompiler;
-  // Relocate PPSA08709 MS pc 0x530 to zero, retaining its displacement to the
-  // system validation helper immediately after the main shader's S_ENDPGM.
-  std::array<uint32_t, 274> shader;
-  shader.fill(EncodeSopp(0x00));
-  shader[0] = 0xbf97010fu;
-  shader[1] = EncodeVop1(0x01, 1, 129);
-  shader[2] = EncodeMubuf0(0x1c, 0, false);
-  shader[3] = EncodeMubuf1(1, 0, 0);
-  shader[271] = EncodeSopp(0x01);
-  shader[272] = 0xffffffffu; // System-only helper must not be decoded.
-  shader[273] = 0xffffffffu;
-  Decoder::Program decoded;
-  Decoder::DecodeProgram(shader, decoded);
-  Check(decoded.instructions.front().opcode == Decoder::Opcode::S_CBRANCH_CDBGSYS &&
-            decoded.instructions.front().branch_target == 0x440u &&
-            decoded.instructions.back().pc == 0x43cu &&
-            decoded.instructions.back().opcode == Decoder::Opcode::S_ENDPGM,
-        "disabled system debug branch reached its post-ENDPGM validation helper");
-  auto graph = CFG::BuildGraph(decoded);
-  Check(graph.blocks.size() == 1u && graph.FindBlockByPc(0x440u) == nullptr &&
-            graph.blocks.front().terminator.kind == CFG::TerminatorKind::Return,
-        "disabled system debug branch changed normal shader control flow");
-  auto options = MakeCompileOptions(ShaderType::Compute);
-  options.dump_ir = true;
-  auto result = RecompileForTest(shader, options);
-  Check((result.decoded_dump.find("S_CBRANCH_CDBGSYS 0x00000440") != std::string::npos) &&
-            (result.ir_dump.find("StoreBufferU32") != std::string::npos),
-        "disabled system debug branch lost its identity or fallthrough write");
-  CheckSpirvBinaryValidates(result.spirv);
+  // Both debug branches fall through when conditional shader debugging is disabled.
+  // Retain the post-ENDPGM helper displacement from PPSA08709 and exercise the
+  // OR_USER variant encountered in PPSA03671 with the same body.
+  for (const auto opcode : {Decoder::Opcode::S_CBRANCH_CDBGSYS,
+                            Decoder::Opcode::S_CBRANCH_CDBGSYS_OR_USER}) {
+    std::array<uint32_t, 274> shader;
+    shader.fill(EncodeSopp(0x00));
+    shader[0] = EncodeSopp(opcode == Decoder::Opcode::S_CBRANCH_CDBGSYS ? 0x17 : 0x19, 0x10f);
+    shader[1] = EncodeVop1(0x01, 1, 129);
+    shader[2] = EncodeMubuf0(0x1c, 0, false);
+    shader[3] = EncodeMubuf1(1, 0, 0);
+    shader[271] = EncodeSopp(0x01);
+    shader[272] = 0xffffffffu; // Debug-only helper must not be decoded.
+    shader[273] = 0xffffffffu;
+    Decoder::Program decoded;
+    Decoder::DecodeProgram(shader, decoded);
+    Check(decoded.instructions.front().opcode == opcode &&
+              decoded.instructions.front().branch_target == 0x440u &&
+              decoded.instructions.back().pc == 0x43cu &&
+              decoded.instructions.back().opcode == Decoder::Opcode::S_ENDPGM,
+          "disabled debug branch reached its post-ENDPGM validation helper");
+    auto graph = CFG::BuildGraph(decoded);
+    Check(graph.blocks.size() == 1u && graph.FindBlockByPc(0x440u) == nullptr &&
+              graph.blocks.front().terminator.kind == CFG::TerminatorKind::Return,
+          "disabled debug branch changed normal shader control flow");
+    auto options = MakeCompileOptions(ShaderType::Compute);
+    options.dump_ir = true;
+    auto result = RecompileForTest(shader, options);
+    Check((result.decoded_dump.find(" 0x00000440") != std::string::npos) &&
+              (result.ir_dump.find("StoreBufferU32") != std::string::npos),
+          "disabled debug branch lost its identity or fallthrough write");
+    CheckSpirvBinaryValidates(result.spirv);
+  }
 }
 
 void TestNewShaderRecompilerRdna2ScalarOpcodes() {
@@ -8580,11 +8584,17 @@ void TestNestedSelectionPreservesDescriptorSources() {
           "nested descriptor selection lost its structured resource bindings");
     for (size_t i = 0; i < result.program.info.buffers.size(); ++i) {
       const auto pc = result.program.info.buffers[i].first_use_pc;
-      const uint32_t expected = pc == 8u * 4u ? table[0]
-                                : pc == 14u * 4u ? table[4]
-                                : pc == direct_arm * 4u ? user_data[8]
-                                : pc == join * 4u ? user_data[4] : 0u;
-      Check(expected != 0u && result.resources.buffers[i].dwords[0] == expected,
+      Check(pc == 8u * 4u || pc == 14u * 4u || pc == direct_arm * 4u || pc == join * 4u,
+            "nested descriptor selection lost a native source location");
+      const auto selected_pc = (mode == 0u ? 8u : mode == 1u ? 14u : direct_arm) * 4u;
+      const auto *expected = pc == 8u * 4u ? table.data()
+                           : pc == 14u * 4u ? table.data() + 4
+                           : pc == direct_arm * 4u ? user_data.data() + 8 : user_data.data() + 4;
+      const auto &descriptor = result.resources.buffers[i];
+      const bool unused = pc != selected_pc && pc != join * 4u;
+      Check(descriptor.dword_count == 4 &&
+                (std::equal(expected, expected + 4, descriptor.dwords.begin()) ||
+                 (unused && std::ranges::all_of(descriptor.dwords, [](auto word) { return word == 0; }))),
             "nested descriptor selection chose another arm's descriptor");
     }
     CheckSpirvBinaryValidates(result.spirv);
@@ -8692,10 +8702,10 @@ void TestNativeScalarReadDescriptorPlanning() {
     auto options = MakeCompileOptions(ShaderType::Pixel);
     options.user_data = user_data;
     const auto result = RecompileForTest(shader, options, ReadHostTestMemory);
-    Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u &&
-              std::ranges::equal(result.resources.flattened_srt, table),
-          "scalar descriptor planning retained an unrelated native execution mask");
     const auto expected = mode == 2u ? table : std::array<uint32_t, 4>{};
+    Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u &&
+              std::ranges::equal(result.resources.flattened_srt, expected),
+          "scalar descriptor planning retained an unrelated native execution mask");
     Check(std::equal(expected.begin(), expected.end(), result.resources.buffers[0].dwords.begin()),
           "scalar descriptor planning selected the wrong native resource");
     CheckSpirvBinaryValidates(result.spirv);
@@ -10104,6 +10114,44 @@ void TestMergedShaderUserDataSnapshot() {
   Check(fan_input.mesh.primitives_per_group == 30 && fan_input.mesh.vertices_per_group == 32 &&
             fan_input.mesh.max_vertices == 256 && fan_input.mesh.max_primitives == 192,
         "captured triangle-fan GS configuration lost its subgroup assembly limits");
+  context.SetShaderStages(0x00482030u);
+  context.SetMaxOutputPerSubgroup(32);
+  context.SetGsMaxVertOut(32);
+  user_config.SetPrimitiveType(Prospero::PrimitiveType::kPointList);
+  user_config.SetGeControl({1, 1});
+  regs.gs_regs.rsrc1.gs_vgpr_component_count = 0;
+  regs.gs_regs.rsrc2.es_vgpr_component_count = 1;
+  ShaderVertexInputInfo fast_input{};
+  PrepareProgram(regs, context, user_config, fast_input);
+  Check(fast_input.mesh.fast_launch && fast_input.mesh.threads_num[0] == 32 &&
+            fast_input.mesh.primitives_per_group == 1 && fast_input.mesh.vertices_per_group == 1 &&
+            fast_input.mesh.max_vertices == 32 && fast_input.mesh.max_primitives == 32,
+        "captured fast-launch shader lost its per-group launch and output limits");
+  auto ordinary_input = fast_input;
+  ordinary_input.mesh.fast_launch = false;
+  Check(MakeStageStaticKey(fast_input) != MakeStageStaticKey(ordinary_input),
+        "fast launch shared the ordinary input-assembly shader key");
+
+  const uint32_t fast_code[] = {
+      EncodeSMovB32(124, 255), 0x20020u, // allocate the full 32 vertices and primitives
+      EncodeSopp(0x10, 9),
+      EncodeVop1(0x01, 0, 256 + 5),
+      EncodeVop1(0x01, 1, 256 + 6),
+      EncodeExp0(0x0c, 0xf, false), EncodeExp1(0, 1, 0, 1),
+      EncodeExp0(0x14, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01),
+  };
+  options.input_info.vertex = &fast_input;
+  options.wave_size = fast_input.mesh.wave_size;
+  options.back_code = {};
+  const auto fast_result = RecompileForTest(fast_code, options);
+  CheckSpirvBinaryValidates(fast_result.spirv);
+  Check(fast_result.program.memory_info.empty(),
+        "compiled fast-launch entry retained an unused index-buffer resource");
+  const auto fast_source = DisassembleSpirvBinary(fast_result.spirv);
+  Check(SpirvSourceHasInstructionUsing(fast_source, "OpExecutionMode", "OutputVertices 32") &&
+            SpirvSourceHasInstructionUsing(fast_source, "OpExecutionMode", "OutputPrimitivesEXT 32"),
+        "fast-launch SPIR-V truncated the mesh output capacity");
+
 }
 
 void TestEmbeddedFetchPreservesSharedScalarLoad() {
@@ -10152,6 +10200,39 @@ void TestEmbeddedFetchPreservesSharedScalarLoad() {
                        result.resources.buffers[0].dwords.begin()),
         "embedded fetch discarded a scalar-load component used by another buffer read");
   CheckSpirvBinaryValidates(result.spirv);
+
+  const auto key = MakeStageStaticKey(input);
+  const auto compile = [&](const ShaderVertexInputInfo &vertex) {
+    auto variant_options = options;
+    variant_options.input_info.vertex = &vertex;
+    return RecompileForTest(code, variant_options).spirv;
+  };
+  auto layout = input;
+  layout.resources_dst[0].register_start = 32;
+  layout.resources_dst[0].fetch_index = 1;
+  layout.resources[0].UpdateAddress48(0x56780000u);
+  layout.resources[0].fields[1] = 52u << 16u;
+  layout.resources[0].fields[2] = 128;
+  layout.resources[0].fields[3] |= 3u << 28u;
+  layout.buffers_num = 1;
+  layout.buffers[0] = {.addr = 0x56780000u, .stride = 52, .num_records = 128,
+                       .fetch_index = 1};
+  Check(compile(layout) == result.spirv,
+        "pipeline-only vertex layout changed the embedded-fetch shader");
+  Check(MakeStageStaticKey(layout) == key,
+        "pipeline-only vertex layout fragmented the shader module cache key");
+
+  auto formatted = input;
+  formatted.resources[0].fields[3] =
+      (static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32UInt) << 12u) |
+      DstSel(4, 5, 6, 7);
+  Check(compile(formatted) != result.spirv && MakeStageStaticKey(formatted) != key,
+        "vertex input numeric type lost its shader module specialization");
+  auto swizzled = input;
+  swizzled.resources[0].fields[3] =
+      (swizzled.resources[0].fields[3] & ~0xfffu) | DstSel(5, 4, 6, 7);
+  Check(compile(swizzled) != result.spirv && MakeStageStaticKey(swizzled) != key,
+        "vertex input destination selectors lost their shader module specialization");
 }
 
 void TestEmbeddedVertexFormatSwizzle() {
@@ -10203,7 +10284,7 @@ void TestEmbeddedVertexFormatSwizzle() {
     Decoder::Program decoded;
     Decoder::DecodeProgram(code, decoded);
     const uint32_t attribute = static_cast<uint32_t>(test.attribute_format) << 5u;
-    const uint32_t buffer[] = {0x10000000u, 16u << 16u, 1u,
+    uint32_t buffer[] = {0x10000000u, 16u << 16u, 1u,
                                (static_cast<uint32_t>(test.format) << 12u) | test.swizzle};
     HW::VertexShaderInfo regs{};
     regs.es_regs.data_addr = reinterpret_cast<uint64_t>(code);
@@ -10222,6 +10303,16 @@ void TestEmbeddedVertexFormatSwizzle() {
     ShaderMapUserData(regs.es_regs.data_addr, mapped);
     ShaderVertexInputInfo input{};
     PrepareProgram(regs, HW::Context{}, HW::UserConfig{}, input);
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+    if (&test == cases) {
+      for (const auto [word, flag] : {std::pair{1u, 1u << 31u}, std::pair{3u, 1u << 23u}}) {
+        buffer[word] |= flag;
+        ExpectFatal([&] { PrepareProgram(regs, HW::Context{}, HW::UserConfig{}, input); },
+                    "unsupported vertex fetch flags bypassed acquisition validation");
+        buffer[word] &= ~flag;
+      }
+    }
+#endif
     Frontend::EmbeddedFetchPlan fetch;
     fetch.loads.push_back({.pc = 0, .attrib_id = 0, .components = test.components});
     Frontend::TranslateOptions options{};
@@ -10274,8 +10365,10 @@ void TestMeshInputAssembly() {
     uint32_t wave_info, first, second, third, byte_offset, vertex_id;
     bool fetch;
     uint32_t wave_size = 64;
+    bool fast_launch = false;
+    uint32_t threads = 256;
   };
-  const Case cases[] = {
+  std::vector<Case> cases = {
       {Prospero::PrimitiveType::kTriList, 14, 177, 14, 2, 2, 0x1002, UINT32_MAX,
        0x40000309, 6, 7, 8, 340, 0xabcc, true},
       {Prospero::PrimitiveType::kTriList, 14, 177, 14, 9, 2, 0x1002, 0,
@@ -10323,14 +10416,30 @@ void TestMeshInputAssembly() {
       {Prospero::PrimitiveType::kTriStrip, 40, 40, 0, 32, 0, 0, 11,
        0x81000608, 32, 33, 34, 0, 43, false, 32},
   };
+  for (const uint32_t wave_size : {32u, 64u}) {
+    for (const uint32_t threads : {wave_size, 2u * wave_size}) {
+      for (const uint32_t group : {0u, 1u, 11u}) {
+        for (const uint32_t lane : {0u, wave_size - 1u, threads - 1u}) {
+          const uint32_t wave_info = ((threads / wave_size) << 28u) |
+                                     ((lane / wave_size) << 24u) |
+                                     (lane < wave_size ? 0x101u : 0u);
+          const uint32_t base_vertex = group == 11u ? UINT32_MAX : 11u;
+          cases.push_back({Prospero::PrimitiveType::kPointList, 1, 12, group, lane, 0, 0,
+                           base_vertex, wave_info, 0, 0, 0, 0, group + base_vertex, false,
+                           wave_size, true, threads});
+        }
+      }
+    }
+  }
   for (const auto &test : cases) {
     ShaderVertexInputInfo input{};
     auto &mesh = input.mesh;
     mesh.input_primitive = static_cast<uint32_t>(test.topology);
     mesh.wave_size = test.wave_size;
+    mesh.fast_launch = test.fast_launch;
     mesh.primitives_per_group = mesh.InputPrimitiveCount(test.capacity);
     mesh.vertices_per_group = mesh.InputVertexCount(mesh.primitives_per_group);
-    mesh.threads_num[0] = 256;
+    mesh.threads_num[0] = test.threads;
     mesh.threads_num[1] = mesh.threads_num[2] = 1;
     Decoder::Program decoded;
     CFG::Graph graph;
@@ -10363,16 +10472,21 @@ void TestMeshInputAssembly() {
       }
     }
     ConstantPropagationPass(program.blocks);
-    Check(load != nullptr && load->Arg(1).Resolve().U32() == test.byte_offset &&
-              load->Arg(3).Resolve().U1() == test.fetch,
-          "mesh index fetch address or active-lane predicate is wrong");
-    const auto *resource = load->Arg(0).ResolveInstruction();
-    Check(resource != nullptr && resource->Arg(0).Resolve().U32() == (test.address_low & ~3u) &&
-              resource->Arg(1).Resolve().U32() == 0x12 &&
-              program.memory_info[load->Flags<MemoryFlags>().index].kind == ResourceKind::Global,
-          "mesh index fetch lost its aligned guest address resource");
-    load->ReplaceUsesWith(Value(test.fetch ? 0xabcd0123u : 0u));
-    ConstantPropagationPass(program.blocks);
+    if (test.fast_launch) {
+      Check(load == nullptr && program.memory_info.empty(),
+            "fast launch emitted an ordinary input-assembly index resource");
+    } else {
+      Check(load != nullptr && load->Arg(1).Resolve().U32() == test.byte_offset &&
+                load->Arg(3).Resolve().U1() == test.fetch,
+            "mesh index fetch address or active-lane predicate is wrong");
+      const auto *resource = load->Arg(0).ResolveInstruction();
+      Check(resource != nullptr && resource->Arg(0).Resolve().U32() == (test.address_low & ~3u) &&
+                resource->Arg(1).Resolve().U32() == 0x12 &&
+                program.memory_info[load->Flags<MemoryFlags>().index].kind == ResourceKind::Global,
+            "mesh index fetch lost its aligned guest address resource");
+      load->ReplaceUsesWith(Value(test.fetch ? 0xabcd0123u : 0u));
+      ConstantPropagationPass(program.blocks);
+    }
     std::array<uint32_t, 9> vgprs{};
     uint32_t sgpr3 = 0;
     for (const auto &inst : *program.blocks.front()) {
@@ -10383,7 +10497,8 @@ void TestMeshInputAssembly() {
       }
     }
     Check(sgpr3 == test.wave_info && vgprs[0] == ((test.first << 2) | (test.second << 18)) &&
-              vgprs[1] == test.third * 4 && vgprs[5] == test.vertex_id && vgprs[8] == 9,
+              vgprs[1] == test.third * 4 && vgprs[5] == test.vertex_id &&
+              vgprs[test.fast_launch ? 6 : 8] == 9u,
           "mesh prolog changed input assembly, wave counts, vertex ID, or instance ID");
   }
 }
@@ -10623,35 +10738,6 @@ void TestNewShaderRecompilerClipDisabledPosition() {
   shifted.clip_space.offset[0] += 1.0f;
   Check(MakeStageStaticKey(clipped) != MakeStageStaticKey(shifted),
         "clip-disabled viewport transform is absent from the shader cache key");
-
-  ShaderVertexInputInfo layout_a{};
-  layout_a.resources_num = 1;
-  layout_a.buffers_num = 1;
-  layout_a.buffers[0].stride = 16;
-  auto layout_b = layout_a;
-  layout_b.buffers[0].stride = 32;
-  layout_b.buffers[0].fetch_index = 1;
-  layout_b.resources[0].UpdateAddress48(4);
-  Check(MakeStageStaticKey(layout_a) == MakeStageStaticKey(layout_b),
-        "pipeline-only vertex layout fragmented the shader module cache key");
-
-  const auto descriptor_state = [](const ShaderBufferResource &resource) {
-    return std::array<uint32_t, 9>{
-        resource.Stride(), resource.SwizzleEnabled(), resource.DstSelX(),
-        resource.DstSelY(), resource.DstSelZ(), resource.DstSelW(),
-        resource.RawFormat(), resource.OutOfBounds(), resource.AddTid()};
-  };
-  const auto key = MakeStageStaticKey(layout_a);
-  const auto state = descriptor_state(layout_a.resources[0]);
-  for (uint32_t word = 0; word < 4; word++) {
-    for (uint32_t bit = 0; bit < 32; bit++) {
-      auto changed = layout_a;
-      changed.resources[0].fields[word] ^= 1u << bit;
-      Check((MakeStageStaticKey(changed) == key) ==
-                (descriptor_state(changed.resources[0]) == state),
-            "vertex key lost a descriptor field or included runtime-only bits");
-    }
-  }
 }
 
 void TestNewShaderRecompilerAuxPositionExports() {
@@ -11479,6 +11565,124 @@ void TestValuePhiValidation() {
 #endif
 }
 
+void TestWave32MaskProjection() {
+  using namespace ShaderRecompiler::IR;
+  for (const uint32_t wave_size : {32u, 64u}) {
+    Program program;
+    program.stage = ShaderType::Compute;
+    program.wave_size = wave_size;
+    for (uint32_t id = 0; id < 3; ++id) {
+      program.block_storage.push_back(std::make_unique<Block>());
+      program.blocks.push_back(program.block_storage.back().get());
+      program.block_info.push_back({.id = id});
+    }
+    auto *entry = program.blocks[0];
+    auto *loop = program.blocks[1];
+    entry->AddBranch(loop);
+    loop->AddBranch(loop);
+    loop->AddBranch(program.blocks[2]);
+    program.block_info[0].terminator.kind = ShaderRecompiler::CFG::TerminatorKind::Branch;
+    program.block_info[0].terminator.true_block = 1;
+    program.block_info[1].terminator.kind = ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch;
+    program.block_info[1].terminator.true_block = 1;
+    program.block_info[1].terminator.false_block = 2;
+    program.block_info[2].terminator.kind = ShaderRecompiler::CFG::TerminatorKind::Return;
+    IREmitter e(entry);
+    const auto zero = U32(Value(0u));
+    const auto raw = e.GetUserData(static_cast<ScalarReg>(0));
+    const auto active = e.INotEqual(raw, zero);
+    const auto selected = e.IEqual(raw, U32(Value(7u)));
+    const auto ballot = [&](IREmitter &ir, U1 predicate, uint32_t component = 0u) {
+      return ir.CompositeExtract(ir.Emit(ValueOpcode::Ballot, {predicate}), component);
+    };
+    const auto thread_bit = [&](IREmitter &ir, U32 word, bool own_lane = true) {
+      const auto bit = own_lane
+          ? ir.BitwiseAnd(U32(ir.Emit(ValueOpcode::LaneId)), U32(Value(31u)))
+          : U32(Value(3u));
+      return ir.INotEqual(ir.BitwiseAnd(ir.ShiftRightLogical(word, bit), U32(Value(1u))), zero);
+    };
+    const auto active_mask = ballot(e, active);
+    const auto direct = thread_bit(e, active_mask);
+    const auto arbitrary = thread_bit(e, raw);
+    const auto mixed = thread_bit(e, e.BitwiseAnd(active_mask, raw));
+    const auto mixed_or = thread_bit(e, e.BitwiseOr(active_mask, raw));
+    const auto mixed_select = thread_bit(e, e.Select(selected, active_mask, raw));
+    const auto different_lane = thread_bit(e, active_mask, false);
+    const auto high_half = thread_bit(e, ballot(e, active, 1u));
+    const auto initial_mask = e.BitwiseAnd(active_mask, ballot(e, selected));
+    const auto raw_consumer = e.Emit(ValueOpcode::IAdd32, {initial_mask, Value(3u)});
+    const auto initial_predicate = e.LogicalAnd(active, selected);
+    const auto entry_size = entry->Instructions().size();
+    ConstantPropagationPass({entry}, wave_size);
+    Check(entry->Instructions().size() == entry_size,
+          "rejected mixed mask graph left unused projected instructions");
+    const auto combined_mask = e.BitwiseOr(active_mask, ballot(e, selected));
+    const auto combined = thread_bit(e, combined_mask);
+    const auto gated_mask = e.Select(selected, combined_mask, zero);
+    const auto gated = thread_bit(e, gated_mask);
+    const auto rejected = thread_bit(e, e.Select(selected, zero, combined_mask));
+    const auto combined_predicate = e.LogicalOr(active, selected);
+    const auto gated_predicate = e.LogicalAnd(selected, combined_predicate);
+    const auto rejected_predicate = e.Emit(ValueOpcode::SelectU1,
+                                          {selected, Value(false), combined_predicate});
+    const auto gated_raw_consumer = e.Emit(ValueOpcode::IAdd32, {gated_mask, Value(3u)});
+    auto &mask = loop->AppendNewInst(ValueOpcode::Phi);
+    mask.SetFlags(Type::U32);
+    auto &predicate = loop->AppendNewInst(ValueOpcode::Phi);
+    predicate.SetFlags(Type::U1);
+    IREmitter body(loop);
+    const auto matches = body.LogicalAnd(U1(Value(&predicate)), selected);
+    const auto remaining = body.BitwiseAnd(U32(Value(&mask)), body.BitwiseNot(ballot(body, matches)));
+    const auto remaining_predicate = body.LogicalAnd(U1(Value(&predicate)), body.LogicalNot(matches));
+    const auto remaining_live = thread_bit(body, remaining);
+    const auto live = thread_bit(body, U32(Value(&mask)));
+    mask.AddPhiOperand(entry, initial_mask);
+    mask.AddPhiOperand(loop, remaining);
+    predicate.AddPhiOperand(entry, initial_predicate);
+    predicate.AddPhiOperand(loop, remaining_predicate);
+    program.block_info[1].condition = remaining_predicate;
+
+    ConstantPropagationPass(program.blocks, wave_size);
+    if (wave_size == 32u) {
+      Check(direct.Resolve() == active.Resolve(), "wave32 ballot round trip lost its predicate");
+      Check(EquivalentValue(program, combined, combined_predicate),
+            "wave32 OR mask projection changed a lane predicate");
+      Check(EquivalentValue(program, gated, gated_predicate) &&
+                EquivalentValue(program, rejected, rejected_predicate),
+            "wave32 select mask projection changed a scalar gate or selected arm");
+      Check(EquivalentValue(program, live.Resolve(), Value(&predicate)),
+            "projected mask Phi differs from the existing per-lane EXEC Phi");
+      Check(EquivalentValue(program, remaining_live.Resolve(), remaining_predicate) &&
+                std::count_if(loop->begin(), loop->end(), [](const Inst &inst) {
+                  return inst.GetOpcode() == ValueOpcode::LogicalAnd;
+                }) == 3,
+            "backedge-first projection duplicated the loop's logical mask operation");
+      const auto *projected = live.Resolve().TryInstruction();
+      Check(projected != nullptr && projected->GetOpcode() == ValueOpcode::Phi &&
+                projected->GetType() == Type::U1 && projected->PhiBlock(0) == entry &&
+                projected->PhiBlock(1) == loop,
+            "mask projection lost loop Phi type or predecessor order");
+    } else {
+      Check(direct.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                live.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                combined.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                gated.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                rejected.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32,
+            "wave64 low-half mask was projected onto the wrong invocation");
+    }
+    for (const auto value : {arbitrary, mixed, mixed_or, mixed_select, different_lane, high_half}) {
+      Check(value.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32,
+            "mask projection accepted an arbitrary mask, lane or ballot half");
+    }
+    Check(raw_consumer.Resolve().TryInstruction()->Arg(0).Resolve() == initial_mask.Resolve() &&
+              gated_raw_consumer.Resolve().TryInstruction()->Arg(0).Resolve() == gated_mask.Resolve(),
+          "mask projection changed a raw scalar mask consumer");
+    ResolveControlFlowIdentities(program);
+    RemoveIdentities(program.blocks);
+    ValidateProgram(program, true);
+  }
+}
+
 void TestU64ShiftConstantPropagation() {
   using namespace ShaderRecompiler::IR;
 
@@ -13503,11 +13707,11 @@ void TestPixelProgramCacheBindingIdentity() {
   Check(MakeStageStaticKey(no_depth_export) != MakeStageStaticKey(depth_export),
         "pixel shader identity omitted depth-export semantics");
 
-  auto check_program_identity = [&](const uint32_t *shader) {
+  auto check_program_identity = [&](std::span<const uint32_t> shader) {
     HW::PixelShaderInfo regs{};
-    regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(shader);
+    regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(shader.data());
     ShaderMappedData mapped{};
-    mapped.code_size_bytes = sizeof(uint32_t);
+    mapped.code_size_bytes = static_cast<uint32_t>(shader.size_bytes());
     ShaderMapUserData(regs.ps_regs.data_addr, mapped);
 
     const std::array<Prospero::ColorComponentMapping, 8> identity_mappings{};
@@ -13522,7 +13726,8 @@ void TestPixelProgramCacheBindingIdentity() {
                                               second_info);
     CompilePixelRuntime(second_params, second_info);
     Check(
-        first_params.hash == second_params.hash &&
+        first_params.hash == XXH3_64bits(shader.data(), shader.size_bytes()) &&
+            first_params.hash == second_params.hash &&
             first_key == MakeStageStaticKey(second_info) &&
             first_info.stage.program != nullptr &&
             second_info.stage.program != nullptr,
@@ -13535,6 +13740,22 @@ void TestPixelProgramCacheBindingIdentity() {
   Check(request_01 == request_10,
         "relocated identical pixel programs did not share their source identity");
 
+  // Identical metadata after the scalar-literal prefix must not mask changes
+  // to the executable code when computing program identity.
+  std::array<uint32_t, 200> shader{};
+  shader[0] = 0xbeeb03ffu;
+  shader[1] = 0x5fu;
+  shader[2] = EncodeSMovB32(0, 129);
+  shader[3] = EncodeSopp(0x01);
+  shader[192] = 0x65726162u;
+  shader[193] = 0x746f6f66u;
+  shader[196] = 0x12345678u;
+  shader[197] = 0x9abcdef0u;
+  const auto first = check_program_identity(shader);
+  shader[2] = EncodeSMovB32(0, 130);
+  Check(check_program_identity(shader).first != first.first,
+        "re-registering changed code at the same address reused its old identity");
+
 }
 
 void TestGraphicsPushConstantPlacement() {
@@ -13637,6 +13858,30 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() {
   Check((result.ir_dump.find("GetScratchResource") != std::string::npos) &&
             result.program.scratch_dwords == 1,
         "SCRATCH incorrectly entered guest address tracking");
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+  // A local aperture proof must not admit a median which can still produce a
+  // canonical global address. Such stores need the existing ownership proof.
+  const uint32_t mixed_domain_store[] = {
+      EncodeSop1(0x04, 8, 237), // s_mov_b64 s[8:9], private_base
+      EncodeVop3Word0(0x159, 1),
+      EncodeVop3Word1(128, 9, 256 + 2), // median(0, private high, dynamic v2)
+      EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0),
+      EncodeSopp(0x01),
+  };
+  ExpectFatal([&]() { (void)RecompileForTest(mixed_domain_store, options); },
+              "mixed local/global FLAT store bypassed GPU ownership tracking");
+  const uint32_t changed_mask_store[] = {
+      EncodeSop1(0x04, 8, 237), EncodeSop1(0x04, 10, 235),
+      EncodeVop1(0x01, 1, 128), // canonical old address high
+      EncodeSop1(0x03, 126, 255), 0x55555555u,
+      EncodeVop3Word0(0x159, 1), EncodeVop3Word1(9, 11, 256 + 2),
+      EncodeSop1(0x04, 126, 193), // restore all lanes after the median
+      EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0),
+      EncodeSopp(0x01),
+  };
+  ExpectFatal([&]() { (void)RecompileForTest(changed_mask_store, options); },
+              "FLAT proof discarded a differently predicated address value");
+#endif
 }
 
 void TestCompilerStageInputOwnership() {
@@ -14022,7 +14267,7 @@ int main() {
   using namespace Libs::Graphics;
 
   EnsureConfigInitialized();
-  TestRayTracingDispatchDetection();
+  TestRayTracingInstructions();
   TestResourceDescriptorClassification();
   TestShaderBufferResourceSize();
   TestNativeShaderResourceDependencies();
@@ -14125,7 +14370,7 @@ int main() {
   TestNewShaderRecompilerBufferLoadsGuardedByExec();
   TestNewShaderRecompilerBufferAtomicsGuardedByBounds();
   TestCapturedBufferAtomicsX2();
-  TestDisabledSystemDebugBranch();
+  TestDisabledDebugBranches();
   TestNewShaderRecompilerPixelImageSampleLodSelection();
   TestNewShaderRecompilerBranchConditionForms();
   TestNewShaderRecompilerSetpcBranch();
@@ -14143,6 +14388,7 @@ int main() {
   TestFinalSsaRejectsRegisterStatePseudos();
 #endif
   TestValuePhiValidation();
+  TestWave32MaskProjection();
   TestU64ShiftConstantPropagation();
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
   TestNativeWideValueValidation();