DualSense haptics and controller speaker sound bluethooth (#969)

* pad: send DualSense speaker and haptic audio over Bluetooth

* ci: fix Opus clang-cl build and macOS memory tracker test

* docs: remove Bluetooth notes from project README

* pad: keep Bluetooth audio work off emulation timing path

* pad: smooth DualSense speaker audio on USB and Bluetooth

* tests: find available macOS guest range for page manager

* pad: keep haptic waveform when Bluetooth queue overflows

* audio: reject zero output sample frequency

* Simplify DualSense audio routing and fix Bluetooth batching

---------

Co-authored-by: Codex <codex@openai.com>
Co-authored-by: nmzik <Nmzik@mail.ru>

3rdparty/CMakeLists.txt

@@ -2,10 +2,31 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
 
 add_subdirectory(cpuinfo)
 
-# The Windows guest red-zone patcher needs a decoder/encoder and a small
-# trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 include(FetchContent)
 
+# The Bluetooth DualSense transports speaker audio as fixed-size Opus packets.
+set(kyty_build_testing_saved "${BUILD_TESTING}")
+set(BUILD_TESTING OFF)
+set(OPUS_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE)
+set(OPUS_INSTALL_PKG_CONFIG_MODULE OFF CACHE BOOL "" FORCE)
+set(OPUS_INSTALL_CMAKE_CONFIG_MODULE OFF CACHE BOOL "" FORCE)
+if(MSVC AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
+	# Opus assumes MSVC accepts SSE4.1 intrinsics without per-file flags, but
+	# clang-cl rejects them in its baseline x64 files. Its AVX2 dispatch also
+	# depends on the SSE4.1 declarations, so retain the x64 SSE2 baseline only.
+	set(OPUS_X86_MAY_HAVE_SSE4_1 OFF CACHE BOOL "" FORCE)
+	set(OPUS_X86_MAY_HAVE_AVX2 OFF CACHE BOOL "" FORCE)
+endif()
+FetchContent_Declare(opus
+	GIT_REPOSITORY https://github.com/xiph/opus.git
+	GIT_TAG ddbe48383984d56acd9e1ab6a090c54ca6b735a6
+	GIT_SHALLOW FALSE)
+FetchContent_MakeAvailable(opus)
+set(BUILD_TESTING "${kyty_build_testing_saved}")
+install(FILES "${opus_SOURCE_DIR}/COPYING" DESTINATION "licenses/opus")
+
+# The Windows guest red-zone patcher needs a decoder/encoder and a small
+# trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
 FetchContent_Declare(
 	xbyak

CMakeLists.txt

@@ -332,7 +332,7 @@ if (KYTY_CLANG_CL)
 	set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED")
 endif()
 
-set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak)
+set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL3::SDL3-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak opus)
 
 # Linux system libraries.
 if(LINUX)
@@ -481,6 +481,10 @@ add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
 target_link_libraries(audio_out2_port_tests common fmt::fmt)
 target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
 
+add_executable(audio_out_timing_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AudioOutTimingTests.cpp")
+target_link_libraries(audio_out_timing_tests common fmt::fmt SDL3::SDL3-static)
+target_include_directories(audio_out_timing_tests PRIVATE ${inc_headers})
+
 add_executable(ngs2_sampler_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/Ngs2SamplerTests.cpp"
 	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
@@ -491,7 +495,7 @@ target_include_directories(ngs2_sampler_tests PRIVATE ${inc_headers})
 add_executable(pad_haptics_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/PadHapticsTests.cpp"
 )
-target_link_libraries(pad_haptics_tests common fmt::fmt SDL3::SDL3-static)
+target_link_libraries(pad_haptics_tests common fmt::fmt SDL3::SDL3-static opus)
 target_include_directories(pad_haptics_tests PRIVATE ${inc_headers})
 
 add_executable(save_data_memory_tests EXCLUDE_FROM_ALL
@@ -606,6 +610,7 @@ if(BUILD_TESTING)
 	add_test(NAME kernel_file_system COMMAND $<TARGET_FILE:kernel_file_system_tests>)
 	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
+	add_test(NAME audio_out_timing COMMAND $<TARGET_FILE:audio_out_timing_tests>)
 	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
 	add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
 	add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
@@ -682,6 +687,7 @@ if(BUILD_TESTING)
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
 		audio_out2_port_tests
+		audio_out_timing_tests
 		ngs2_sampler_tests
 		pad_haptics_tests
 		save_data_memory_tests

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -140,6 +140,8 @@ constexpr auto MakeShaderDispatchTable() {
 	g_hw_sh_func[Pm4::COMPUTE_PGM_RSRC3]               = HwShSetCsRegisters;
 	g_hw_sh_func[Pm4::COMPUTE_PACE_ID]                 = HwShSetCsRegisters;
 	g_hw_sh_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_PS]  = HwShSetRegisters;
+	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS] = HwShSetRegisters;
+	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS] = HwShSetRegisters;
 	g_hw_sh_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_GS]  = HwShSetRegisters;
 	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_LO_GS] = HwShSetRegisters;
 	g_hw_sh_func[Pm4::SPI_SHADER_USER_DATA_ADDR_HI_GS] = HwShSetRegisters;

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -3622,6 +3622,18 @@ void GraphicsInitJmpTablesShIndirect() {
 	g_hw_sh_indirect_func[Pm4::SPI_GRAPHICS_SHADER_CONTROL_PS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};
+	for (uint32_t offset = Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS;
+	     offset <= Pm4::SPI_SHADER_USER_DATA_ADDR_HI_PS; offset++) {
+		g_hw_sh_indirect_func[offset] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+			cp.GetShCtx().SetPsUserDataAddress(cmd_offset - Pm4::SPI_SHADER_USER_DATA_ADDR_LO_PS,
+			                                   value);
+		};
+	}
+	for (uint32_t offset = 4; offset <= 5; offset++) {
+		g_hw_sh_indirect_func[offset] = [](KYTY_HW_SH_INDIRECT_ARGS) {
+			cp.GetShCtx().SetPsAuxiliaryTableAddress(cmd_offset - 4, value);
+		};
+	}
 	g_hw_sh_indirect_func[Pm4::SPI_SHADER_PACE_ID_GS] = [](KYTY_HW_SH_INDIRECT_ARGS) {
 		HwShIgnoreShaderRegister(cmd_offset, value);
 	};

src/graphics/guest_gpu/hardwareContext.h

@@ -458,6 +458,9 @@ struct PsShaderResource2 {
 
 struct PsStageRegisters {
 	uint64_t          data_addr = 0;
+	uint64_t          user_data_addr = 0;
+	// Hades II uses PS offsets 4:5 for a byte-addressed descriptor table.
+	uint64_t          auxiliary_table_addr = 0;
 	PsShaderResource1 rsrc1;
 	PsShaderResource2 rsrc2;
 };
@@ -1017,6 +1020,12 @@ public:
 	void SetGsShaderResource2(const GsShaderResource2& rsrc2) { m_vs.gs_regs.rsrc2 = rsrc2; }
 
 	void SetPsShaderBase(uint64_t addr) { m_ps.ps_regs.data_addr = addr; }
+	void SetPsUserDataAddress(uint32_t word, uint32_t value) {
+		SetUserDataAddressWord(m_ps.ps_regs.user_data_addr, word, value);
+	}
+	void SetPsAuxiliaryTableAddress(uint32_t word, uint32_t value) {
+		SetUserDataAddressWord(m_ps.ps_regs.auxiliary_table_addr, word, value);
+	}
 	void SetPsShaderResource1(const PsShaderResource1& rsrc1) { m_ps.ps_regs.rsrc1 = rsrc1; }
 	void SetPsShaderResource2(const PsShaderResource2& rsrc2) { m_ps.ps_regs.rsrc2 = rsrc2; }
 

src/graphics/guest_gpu/pm4.h

@@ -697,6 +697,8 @@ constexpr uint32_t CX_NUM = 0x3FF + 1;
 
 /* Shader registers */
 
+constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_LO_PS = 0x2;
+constexpr uint32_t SPI_SHADER_USER_DATA_ADDR_HI_PS = 0x3;
 constexpr uint32_t SPI_SHADER_PACE_ID_PS            = 0x6;
 constexpr uint32_t SPI_GRAPHICS_SHADER_CONTROL_PS    = 0x7;
 constexpr uint32_t SPI_SHADER_PGM_LO_PS              = 0x8;

src/graphics/host_gpu/regionManager.h

@@ -30,21 +30,14 @@ class TrackingSpinLock final {
 public:
 	void lock() noexcept {
 		const auto thread = CurrentThread();
-		if (m_owner.load(std::memory_order_relaxed) == thread) {
-			EXIT("recursive region tracking lock\n");
-		}
 		while (m_lock.test_and_set(std::memory_order_acquire)) {
-			if (m_owner.load(std::memory_order_relaxed) == thread) {
-				EXIT("recursive region tracking lock while contended\n");
-			}
+			EXIT_NOT_IMPLEMENTED(m_owner.load(std::memory_order_relaxed) == thread);
 			std::atomic_signal_fence(std::memory_order_seq_cst);
 		}
 		m_owner.store(thread, std::memory_order_relaxed);
 	}
 	void unlock() noexcept {
-		if (m_owner.load(std::memory_order_relaxed) != CurrentThread()) {
-			EXIT("region tracking lock released by non-owner\n");
-		}
+		EXIT_NOT_IMPLEMENTED(m_owner.load(std::memory_order_relaxed) != CurrentThread());
 		m_owner.store(0, std::memory_order_relaxed);
 		m_lock.clear(std::memory_order_release);
 	}
@@ -88,8 +81,7 @@ public:
 	template <DirtySource source>
 	[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
 		const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
-		const auto& bits        = GetBits<source>();
-		return RegionBits(bits, start, end).Any();
+		return GetBits<source>().FirstRangeFrom(start).first < end;
 	}
 
 	template <DirtySource source, bool enable>

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -110,6 +110,7 @@ void BufferCache::DeleteBuffer(BufferId id) {
 	}
 }
 
+template <bool async>
 bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size) {
 	std::vector<vk::BufferCopy> copies;
 	uint64_t                    total_size     = 0;
@@ -170,14 +171,22 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
 	                       vk::PipelineStageFlagBits::eAllCommands |
 	                           vk::PipelineStageFlagBits::eHost,
 	                       {}, 0, nullptr, 1, &after, 0, nullptr);
-	m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address,
-	                                    copies = std::move(copies), owner = std::move(temporary)] {
+	auto publish = [this, mapped, offset, total_size, buffer_address,
+	                copies = std::move(copies), owner = std::move(temporary)] {
 		(owner ? *owner : m_download_buffer).Invalidate(offset, total_size);
 		for (const auto& copy: copies) {
 			Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset,
 			                                      mapped + (copy.dstOffset - offset), copy.size);
 		}
-	});
+	};
+	if constexpr (async) {
+		m_scheduler.DeferPriorityOperation(std::move(publish));
+	} else {
+		const auto tick = m_scheduler.CurrentTick();
+		m_scheduler.Wait(tick);
+		m_scheduler.WaitPriorityOperations(tick);
+		publish();
+	}
 	return true;
 }
 
@@ -256,10 +265,7 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
 		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
 
-		if (DownloadBufferMemory(buffer, window_begin, window_end - window_begin)) {
-			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.Wait(tick);
-			m_scheduler.WaitPriorityOperations(tick);
+		if (DownloadBufferMemory<false>(buffer, window_begin, window_end - window_begin)) {
 			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
 		}
 		if (is_write) {
@@ -614,7 +620,7 @@ void BufferCache::RunGarbageCollector() {
 			return false;
 		}
 		if (dirty) {
-			EXIT_IF(!DownloadBufferMemory(buffer, buffer.CpuAddress(), buffer.Size()));
+			EXIT_NOT_IMPLEMENTED(!DownloadBufferMemory<true>(buffer, buffer.CpuAddress(), buffer.Size()));
 			dirty_buffers.push_back(id);
 		} else {
 			m_memory_tracker.UntrackMemory(buffer.CpuAddress(), buffer.Size());

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -117,7 +117,8 @@ private:
 	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
 	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	// Queues backing publication; callers wait before clearing dirty pages or reusing their data.
+	// Synchronous downloads publish before returning; asynchronous callers wait before reuse.
+	template <bool async>
 	[[nodiscard]] bool DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t size);
 
 	GraphicContext&                                   m_graphics;

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -583,7 +583,7 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {
 	}
 
 	auto                         depth_buffer = m_tiler.GetScratchBuffer(depth_size);
-	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size);
+	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer);
 	const TileManager::D16Layout promote_layout {
 	    .width               = depth.info.extent.width,
 	    .height              = depth.info.extent.height,
@@ -1071,7 +1071,7 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 		                     texels_per_slice > UINT64_MAX / transfer_bytes);
 		const uint64_t transfer_slice = texels_per_slice * transfer_bytes;
 		EXIT_NOT_IMPLEMENTED(transfer_slice > UINT64_MAX / layers);
-		auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers);
+		auto promoted = m_tiler.GetScratchBuffer(transfer_slice * layers, linear.buffer);
 		m_tiler.ConvertD16(
 		    linear, promoted, TileManager::D16Direction::Promote,
 		    info.pixel_format == vk::Format::eD32SfloatS8Uint,
@@ -1533,7 +1533,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	auto&          image = m_slot_images[selected];
 	vk::ClearValue clear {};
 	if (aspect == vk::ImageAspectFlagBits::eColor) {
-		if (!DecodePackedColorClear(image.info.pixel_format, packed_clear, clear.color)) {
+		if (!DecodeColorDwordFill(image.info.pixel_format, packed_clear, clear.color)) {
 			return false;
 		}
 	} else {
@@ -1679,7 +1679,7 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
 	auto host_linear = m_tiler.GetScratchBuffer(transfer_size);
 	image.Download(copies, host_linear.buffer, 0, host_linear.size);
 	const bool tiled        = info.IsTiled();
-	auto       guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size)
+	auto       guest_linear = tiled ? m_tiler.GetScratchBuffer(info.data.size, host_linear.buffer)
 	                                : TileManager::Result {destination.Handle(), destination_offset,
 	                                                       destination.Size() - destination_offset};
 	m_tiler.ConvertD16(host_linear, guest_linear, TileManager::D16Direction::Demote,

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -428,6 +428,20 @@ inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {
 	return true;
 }
 
+// Unlike a register clear, a DWORD fill repeats the same word across the entire pixel.
+// Keep this separate: the first register word alone cannot describe a 64-bit color.
+[[nodiscard]] inline bool DecodeColorDwordFill(vk::Format format, uint32_t packed,
+                                               vk::ClearColorValue& clear) {
+	if (format == vk::Format::eR16G16B16A16Sfloat) {
+		if (packed != 0) {
+			return false;
+		}
+		clear = vk::ClearColorValue {};
+		return true;
+	}
+	return DecodePackedColorClear(format, packed, clear);
+}
+
 [[nodiscard]] inline bool DecodePackedStencilClear(uint32_t packed, uint8_t& clear) {
 	const auto value = static_cast<uint8_t>(packed);
 	if (packed != static_cast<uint32_t>(value) * 0x01010101u) {

src/graphics/host_gpu/renderer/image/imageView.h

@@ -31,20 +31,14 @@ namespace ImageViewOps {
 	       ImageViewOps::IsFormatDepthCompatible(view_format);
 }
 
-[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle) noexcept {
+[[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle,
+                                             uint32_t component_count = 4) noexcept {
 	if ((swizzle & ~0xfffu) != 0) {
 		return false;
 	}
 	for (uint32_t channel = 0; channel < 4; channel++) {
-		switch (GetDstSel(swizzle, channel)) {
-			case 0:
-			case 1:
-			case 4:
-			case 5:
-			case 6:
-			case 7: break;
-			default: return false;
-		}
+		const auto selector = GetDstSel(swizzle, channel);
+		if (selector > 1 && (selector < 4 || selector >= 4 + component_count)) return false;
 	}
 	return true;
 }
@@ -81,15 +75,8 @@ SelectSampledColorView(vk::Format image_format, vk::Format view_format, uint32_t
 [[nodiscard]] inline bool IsSupportedSampledDepthView(vk::Format image_format,
                                                       vk::Format view_format,
                                                       uint32_t   swizzle) noexcept {
-	if (!IsSupportedSampledDepthFormat(image_format, view_format)) {
-		return false;
-	}
-	switch (swizzle) {
-		case DstSel(4, 4, 4, 4):
-		case DstSel(4, 0, 0, 0):
-		case DstSel(4, 0, 0, 1): return true;
-		default: return false;
-	}
+	return IsSupportedSampledDepthFormat(image_format, view_format) &&
+	       IsValidImageSwizzle(swizzle, 1);
 }
 
 [[nodiscard]] inline bool

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -88,30 +88,6 @@ TileManager::~TileManager() {
 	}
 }
 
-TileManager::Scratch TileManager::AllocateScratch(uint64_t size) {
-	EXIT_IF(size == 0);
-	vk::BufferCreateInfo create {};
-	create.size  = size;
-	create.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc |
-	               vk::BufferUsageFlagBits::eTransferDst;
-
-	VmaAllocationCreateInfo allocate {};
-	allocate.usage       = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
-	VkBuffer      buffer = VK_NULL_HANDLE;
-	VmaAllocation memory = nullptr;
-	const auto    raw    = static_cast<VkBufferCreateInfo>(create);
-	RequireVulkanSuccess(static_cast<vk::Result>(vmaCreateBuffer(
-	                         m_graphics.allocator, &raw, &allocate, &buffer, &memory, nullptr)),
-	                     "allocate TileManager scratch buffer");
-	return {buffer, memory, size};
-}
-
-void TileManager::DeferDestroy(Scratch scratch) {
-	auto allocator = m_graphics.allocator;
-	m_scheduler.DeferOperation(
-	    [allocator, scratch] { vmaDestroyBuffer(allocator, scratch.buffer, scratch.allocation); });
-}
-
 void TileManager::Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
                           std::span<const GpuTileInfo> infos, uint64_t source_base,
                           uint64_t target_base, std::vector<Dispatch>& dispatches) {
@@ -369,8 +345,7 @@ TileManager::Result TileManager::Detile(vk::Buffer tiled, uint64_t tiled_offset,
 	const uint64_t        source_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
 	Prepare(false, tiled_capacity, linear_capacity, infos, source_base, 0, dispatches);
-	auto scratch = AllocateScratch(Common::AlignUp(linear_capacity, 4));
-	DeferDestroy(scratch);
+	auto scratch = GetScratchBuffer(linear_capacity, tiled);
 	Record(tiled, tiled_offset, tiled_capacity, scratch.buffer, 0, scratch.size, dispatches,
 	       true);
 	return {scratch.buffer, 0, linear_capacity};
@@ -400,11 +375,9 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	    std::max<uint64_t>(limits.minStorageBufferOffsetAlignment, 4);
 	const uint64_t        target_base = tiled_offset & (descriptor_alignment - 1);
 	std::vector<Dispatch> dispatches;
-	// Reserve all stream parameters before creating a scheduler-lived scratch dependency:
-	// StreamBuffer::Map is allowed to submit the current tick when it wraps.
+	// Reserve stream parameters before recording the image download and conversion.
 	Prepare(true, tiled_capacity, linear_capacity, infos, 0, target_base, dispatches);
-	auto linear = AllocateScratch(Common::AlignUp(linear_capacity, 4));
-	DeferDestroy(linear);
+	auto linear = GetScratchBuffer(linear_capacity, tiled);
 	image.Download(regions, linear.buffer, 0, linear.size);
 	Result source {linear.buffer, 0, linear.size};
 	if (transform == ColorTransform::SwapBgra16) {
@@ -414,10 +387,19 @@ void TileManager::TileImage(Image& image, std::span<const vk::BufferImageCopy> r
 	       dispatches, false);
 }
 
-TileManager::Result TileManager::GetScratchBuffer(uint64_t size) {
-	auto scratch = AllocateScratch(Common::AlignUp(size, 4));
-	DeferDestroy(scratch);
-	return {scratch.buffer, 0, scratch.size};
+TileManager::Result TileManager::GetScratchBuffer(uint64_t size, vk::Buffer input) {
+	EXIT_IF(size == 0);
+	size = Common::AlignUp(size, 4);
+	auto& buffer = m_scratch[m_scratch[0] && m_scratch[0]->Handle() == input ? 1 : 0];
+	if (!buffer || buffer->Size() < size) {
+		if (buffer) {
+			m_scheduler.DeferOperation([old = std::move(buffer)]() mutable { old.reset(); });
+		}
+		buffer = std::make_unique<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0,
+		    vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc |
+		        vk::BufferUsageFlagBits::eTransferDst, size);
+	}
+	return {buffer->Handle(), 0, size};
 }
 
 TileManager::StorageBinding TileManager::BindStorage(Result buffer, uint64_t size) const {
@@ -668,9 +650,7 @@ void TileManager::SwapBgra16(Result input, Result output, uint32_t pixels) {
 
 TileManager::Result TileManager::SwapBgra16(Result input) {
 	EXIT_NOT_IMPLEMENTED(input.size == 0 || input.size % 8u != 0 || input.size / 8u > UINT32_MAX);
-	auto output = AllocateScratch(input.size);
-	DeferDestroy(output);
-	Result result {output.buffer, 0, output.size};
+	auto result = GetScratchBuffer(input.size, input.buffer);
 	SwapBgra16(input, result, static_cast<uint32_t>(input.size / 8u));
 	return result;
 }

src/graphics/host_gpu/renderer/image/tiler.h

@@ -6,13 +6,14 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
+#include <memory>
 #include <span>
 #include <vector>
-#include <vk_mem_alloc.h>
 
 namespace Libs::Graphics {
 
 class CommandScheduler;
+class Buffer;
 class Image;
 class StreamBuffer;
 struct GraphicContext;
@@ -62,7 +63,7 @@ public:
 	~TileManager();
 	KYTY_CLASS_NO_COPY(TileManager);
 
-	// The returned device-local buffer remains alive through the current scheduler tick.
+	// Consume scratch results before the next acquisition, or pass their buffer as input.
 	[[nodiscard]] Result Detile(vk::Buffer tiled, uint64_t tiled_offset, uint64_t tiled_capacity,
 	                            uint64_t linear_capacity, std::span<const GpuTileInfo> infos);
 	void Tile(vk::Buffer linear, uint64_t linear_offset, uint64_t linear_capacity, vk::Buffer tiled,
@@ -71,7 +72,7 @@ public:
 	               uint64_t tiled_offset, uint64_t tiled_capacity, uint64_t linear_capacity,
 	               std::span<const GpuTileInfo> infos,
 	               ColorTransform               transform = ColorTransform::None);
-	[[nodiscard]] Result GetScratchBuffer(uint64_t size);
+	[[nodiscard]] Result GetScratchBuffer(uint64_t size, vk::Buffer input = nullptr);
 	void                 ConvertD16(Result source, Result target, D16Direction direction, bool d32,
 	                                const D16Layout& layout);
 	[[nodiscard]] Result SwapBgra16(Result input);
@@ -105,23 +106,16 @@ private:
 		uint32_t pipeline_slot = 0;
 		uint64_t params_offset = 0;
 	};
-	struct Scratch {
-		vk::Buffer    buffer     = nullptr;
-		VmaAllocation allocation = nullptr;
-		uint64_t      size       = 0;
-	};
 	struct StorageBinding {
 		vk::DescriptorBufferInfo info;
 		uint32_t                 base = 0;
 	};
 
-	[[nodiscard]] Scratch         AllocateScratch(uint64_t size);
 	[[nodiscard]] StorageBinding  BindStorage(Result buffer, uint64_t size) const;
 	[[nodiscard]] static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride,
 	                                             uint64_t active, uint32_t remaining,
 	                                             uint64_t alignment, uint64_t max_range,
 	                                             uint32_t max_groups) noexcept;
-	void                          DeferDestroy(Scratch scratch);
 	void Prepare(bool tile, uint64_t tiled_capacity, uint64_t linear_capacity,
 	             std::span<const GpuTileInfo> infos, uint64_t source_base, uint64_t target_base,
 	             std::vector<Dispatch>& dispatches);
@@ -134,6 +128,7 @@ private:
 	GraphicContext&                         m_graphics;
 	CommandScheduler&                       m_scheduler;
 	StreamBuffer&                           m_stream_buffer;
+	std::array<std::unique_ptr<Buffer>, 2>   m_scratch;
 	vk::DescriptorSetLayout                 m_descriptor_layout = nullptr;
 	vk::PipelineLayout                      m_pipeline_layout   = nullptr;
 	std::array<vk::Pipeline, PipelineCount> m_pipelines {};

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -974,6 +974,41 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
 		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
+	for (const auto* reader: prepared_bindings) {
+		const auto& reads = reader->runtime->resources->specialization_reads;
+		if (reads.empty()) continue;
+		for (const auto* writer: prepared_bindings) {
+			if (writer->runtime->program->has_address_writes) {
+				EXIT("scalar resource reads cannot be proven disjoint from shader address writes\n");
+			}
+		}
+		for (const auto [address, size]: reads) {
+			for (const auto id: m_bound_images) {
+				const auto* image = m_context.GetTextureCache().m_slot_images.try_get(id);
+				if (image == nullptr ||
+				    (!image->binding.shader_write && !image->binding.is_target)) continue;
+				for (const auto written: {image->info.data, image->info.stencil,
+				                          image->info.metadata.range}) {
+					if (written.size != 0 && ImageRangeOverlaps(address, size,
+					                                          written.address, written.size)) {
+						EXIT("scalar resource reads overlap an image or attachment write\n");
+					}
+				}
+			}
+			for (const auto* writer: prepared_bindings) {
+				const auto& program = *writer->runtime->program;
+				for (uint32_t i = 0; i < writer->buffer_sources.size(); ++i) {
+					const auto resource = program.bindings.descriptors.front().resources[i];
+					if (!program.info.buffers[resource].written) continue;
+					const auto& written = writer->buffer_sources[i];
+					if (written.size != 0 && ImageRangeOverlaps(address, size,
+					                                          written.address, written.size)) {
+						EXIT("scalar resource reads overlap a shader buffer write\n");
+					}
+				}
+			}
+		}
+	}
 	m_descriptor_buffers.clear();
 	m_descriptor_images.clear();
 	m_descriptor_writes.clear();

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -220,7 +220,6 @@ struct PipelineCache::ProgramCache {
 		ShaderRecompiler::IR::ResourceSnapshot       resources;
 		ShaderRecompiler::IR::ResourceSpecialization specialization;
 		std::vector<Permutation>                    permutations;
-		bool                                        skip_dispatch = false;
 	};
 
 	struct ProgramKeyHash {
@@ -288,9 +287,6 @@ struct PipelineCache::ProgramCache {
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
-		if (entry != programs.end() && entry->second.skip_dispatch) {
-			return {};
-		}
 		const ShaderRecompiler::IR::SrtRuntime       runtime {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
@@ -358,11 +354,6 @@ struct PipelineCache::ProgramCache {
 		}
 		DumpShaderOriginal(stage_name, options.shader_hash, params.code);
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
-		if (translated.skip_dispatch) {
-			entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
-			entry->second.skip_dispatch = true;
-			return {};
-		}
 		if (entry == programs.end()) {
 			entry = programs.try_emplace(lookup_key,
 			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
@@ -723,13 +714,6 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		}
 		const auto& rt                        = ctx.GetRenderTarget(colors[i].target_slot);
 		const auto& bc                        = ctx.GetBlendControl(colors[i].target_slot);
-		static_params.color_srcblend[slot]       = bc.color_srcblend;
-		static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
-		static_params.color_destblend[slot]      = bc.color_destblend;
-		static_params.alpha_srcblend[slot]       = bc.alpha_srcblend;
-		static_params.alpha_comb_fcn[slot]       = bc.alpha_comb_fcn;
-		static_params.alpha_destblend[slot]      = bc.alpha_destblend;
-		static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
 		const bool alpha_remap =
 		    slot == 0 && ps_input_info != nullptr && ps_input_info->alpha_blend_source_remap;
 		static_params.blend_enable[slot] = bc.enable && !rt.info.blend_bypass;
@@ -748,6 +732,17 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(
 		if (alpha_remap) {
 			static_params.blend_alpha_source_remap = true;
 		}
+		if (static_params.blend_enable[slot]) {
+			static_params.color_srcblend[slot]       = bc.color_srcblend;
+			static_params.color_comb_fcn[slot]       = bc.color_comb_fcn;
+			static_params.color_destblend[slot]      = bc.color_destblend;
+			static_params.separate_alpha_blend[slot] = bc.separate_alpha_blend;
+			if (bc.separate_alpha_blend) {
+				static_params.alpha_srcblend[slot]  = bc.alpha_srcblend;
+				static_params.alpha_comb_fcn[slot]  = bc.alpha_comb_fcn;
+				static_params.alpha_destblend[slot] = bc.alpha_destblend;
+			}
+		}
 	}
 	const bool with_depth =
 	    depth.desc.view_info.format != vk::Format::eUndefined && static_cast<bool>(depth.image_id);

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -330,9 +330,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 			}
 		}
 
-		EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].AddTid());
-		EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].SwizzleEnabled());
-
 		EXIT_IF(registers_num < 1 || registers_num > 4);
 	}
 

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -256,10 +256,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	input_info.dispatch_thread_dimensions = use_thread_dimensions;
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
-	if (!compute_program) {
-		// Temporary until RT is implemented.
-		return;
-	}
 	if (use_thread_dimensions) {
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
@@ -268,15 +264,14 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 
 	const auto& program   = *input_info.stage.program;
 	const auto& resources = *input_info.stage.resources;
-	if (TryConsumeComputeMetaClear(input_info, buffer)) {
-		ResetBindings();
-		return;
-	}
-	if (TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y,
-	                                thread_group_z, mode)) {
+	if (resources.specialization_reads.empty() &&
+	    (TryConsumeComputeMetaClear(input_info, buffer) ||
+	     TryConsumeComputeImageClear(input_info, buffer, thread_group_x, thread_group_y,
+	                                 thread_group_z, mode))) {
 		ResetBindings();
 		return;
 	}
+
 	const bool large_workgroup =
 	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
 	const bool                   has_sampler = !program.info.samplers.empty();
@@ -417,10 +412,6 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	ShaderComputeInputInfo input_info {};
 	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
 	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
-	if (!compute_program) {
-		// Temporary until RT is implemented.
-		return;
-	}
 	buffer.EndRendering();
 	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
 	auto& bindings = m_compute_bindings;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -119,6 +119,10 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void RenderContext::PrepareBda() {
+	if (!m_bda_logged) {
+		Log::WriteToConsoleAndLog("GPU: using buffer device address (BDA) shader memory access.\n");
+		m_bda_logged = true;
+	}
 	std::shared_lock lock(m_mapped_ranges_mutex);
 	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
 		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);

src/graphics/host_gpu/renderer/renderContext.h

@@ -81,6 +81,7 @@ private:
 	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 	bool                      m_fault_process_pending = false;
+	bool                      m_bda_logged = false;
 
 	Common::Mutex                        m_interrupt_mutex;
 	std::vector<InterruptEqRegistration> m_interrupt_eqs;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -1032,6 +1032,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	uint32_t   mesh_groups = 0;
 	if (mesh_active) {
 		const auto& mesh = state.vertex_info[0].mesh;
+		EXIT_NOT_IMPLEMENTED(mesh.fast_launch && (draw.IsIndexed() || primitive_restart_enable));
 		static std::atomic_bool restart_warned = false;
 		if (primitive_restart_enable && !restart_warned.exchange(true, std::memory_order_relaxed)) {
 			std::printf("Warning: primitive restart is not implemented for mesh shaders; "

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -19,7 +19,6 @@
 
 #include <algorithm>
 #include <array>
-#include <atomic>
 #include <chrono>
 #include <fmt/format.h>
 #include <map>
@@ -523,20 +522,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
 
-	// Temporary workaround for games that compile ray-tracing shaders before
-	// the player can select a mode without ray tracing.
-	if (options.stage == ShaderType::Compute && decoded.has_bvh) {
-		static std::atomic_flag warned = ATOMIC_FLAG_INIT;
-		if (!warned.test_and_set(std::memory_order_relaxed)) {
-			const auto& bvh = decoded.instructions.back();
-			Log::WriteToConsoleAndLog(fmt::format(
-			    "Warning: ray tracing is not implemented; skipping compute dispatches containing "
-			    "BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n",
-			    options.shader_hash, bvh.pc, bvh.opcode_id));
-		}
-		return {.skip_dispatch = true};
-	}
-
 	std::string decoded_dump;
 	if (options.dump_ir) {
 		decoded_dump = Decoder::ProgramToString(decoded);
@@ -609,7 +594,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
 	IR::RewriteToSsa(ir.blocks);
-	IR::ConstantPropagationPass(ir.blocks);
+	IR::ConstantPropagationPass(ir.blocks, ir.wave_size);
 	IR::ResolveControlFlowIdentities(ir);
 	IR::RemoveIdentities(ir.blocks);
 	IR::EliminateDeadCode(ir.blocks);
@@ -617,7 +602,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	if (read_lane_stats.rewritten_reads != 0) {
 		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
 		     read_lane_stats.rewritten_reads);
-		IR::ConstantPropagationPass(ir.blocks);
+		IR::ConstantPropagationPass(ir.blocks, ir.wave_size);
 		IR::ResolveControlFlowIdentities(ir);
 		IR::RemoveIdentities(ir.blocks);
 		IR::EliminateDeadCode(ir.blocks);
@@ -645,7 +630,6 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options,
                              const IR::ResourceSpecialization& specialization,
                              uint32_t push_data_start_dword) {
-	EXIT_IF(translated.skip_dispatch);
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
 	IR::ApplyResourceSpecialization(ir, specialization);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -28,7 +28,6 @@ struct TranslateResult {
 	IR::Program program;
 	std::string decoded_dump;
 	std::string cfg_dump;
-	bool        skip_dispatch = false;
 };
 
 struct CompileResult {

src/graphics/shader/recompiler/Tessellation.cpp

@@ -217,7 +217,7 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 			    2, ActiveAddress(*block, it, it->Arg(2), it->Arg(it->NumArgs() - 1u), resolved));
 		}
 	}
-	ConstantPropagationPass(program.blocks);
+	ConstantPropagationPass(program.blocks, program.wave_size);
 	uint32_t reads = 0, writes = 0, factors = 0;
 	for (auto* block: program.blocks) {
 		for (auto it = block->begin(); it != block->end(); ++it) {
@@ -317,7 +317,7 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 			inst.Invalidate();
 		}
 	}
-	ConstantPropagationPass(program.blocks);
+	ConstantPropagationPass(program.blocks, program.wave_size);
 	RemoveIdentities(program.blocks);
 	EliminateDeadCode(program.blocks);
 	LOGF("%s tessellation lowering: reads=%u writes=%u factors=%u\n",

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -122,6 +122,16 @@ void ValidateNativeProgram(const IR::Program& program) {
 			              program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer;
 		       });
 	};
+	const auto local_flat_handle = [&](const IR::Inst& handle) {
+		return !handle.Uses().empty() &&
+		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
+			       if (IR::AddressOpcodeInfoOf(use.user->GetOpcode()).access == IR::AddressAccess::None)
+				       return false;
+			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].kind == IR::ResourceKind::FlatLocal;
+		       });
+	};
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
 			const auto dense = inst.Flags<uint32_t>();
@@ -138,7 +148,7 @@ void ValidateNativeProgram(const IR::Program& program) {
 					if (planning_only_handle(inst)) {
 						break;
 					}
-					if (inst.NumArgs() != 2 || !program.info.uses_dma) {
+					if (inst.NumArgs() != 2 || (!program.info.uses_dma && !local_flat_handle(inst))) {
 						Fail(program, "typed address handle has invalid DMA metadata");
 					}
 					break;
@@ -188,11 +198,16 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (memory_index >= program.memory_info.size()) {
 					Fail(program, "address operation has invalid memory metadata");
 				}
-				if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+				const auto kind = program.memory_info[memory_index].kind;
+				if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
 					if (program.scratch_dwords == 0) {
 						Fail(program, "scratch operation has no per-thread storage");
 					}
 					requirements.function_scratch = true;
+					if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute &&
+					    program.stage != ShaderType::Mesh) {
+						requirements.function_lds = true;
+					}
 				} else if (address_access == IR::AddressAccess::Write) {
 					Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
 				}
@@ -229,7 +244,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
 					Fail(program, "shared operation has invalid resource kind");
 				}
-				if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicOr64) {
+				if (shared_access == IR::SharedAccess::Atomic &&
+				    IR::SharedComponentCount(inst.GetOpcode()) == 2u) {
 					if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) {
 						Fail(program, "64-bit shared atomics require compute LDS");
 					}
@@ -247,6 +263,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				}
 			}
 			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
 				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
 				case IR::ValueOpcode::ReadFirstLane:

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -444,6 +444,18 @@ uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
 	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
 }
 
+uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThanEqual);
+}
+
+uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThanEqual);
+}
+
+uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
+	return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThanEqual);
+}
+
 uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
 	return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
 }

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -0,0 +1,348 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+using Vec3 = std::array<uint32_t, 3>;
+
+uint32_t Extract(EmitterState& s, uint32_t type, uint32_t vector, uint32_t index) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeExtract, type, result, vector, index);
+	return result;
+}
+
+uint32_t Vector(EmitterState& s, uint32_t type, std::span<const uint32_t> components) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, type, result, components);
+	return result;
+}
+
+uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
+	const auto offset = Binary(s, spv::OpShiftLeftLogical, TypeU32(s), word, ConstantU32(s, 2));
+	const auto address = Binary(s, spv::OpIAdd, TypeScalarU64(s), block,
+	                            Unary(s, spv::OpUConvert, TypeScalarU64(s), offset));
+	const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpLoad, TypeU32(s), result, pointer, spv::MemoryAccessAlignedMask, 4u);
+	return result;
+}
+
+uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
+                  Vec3 origin, Vec3 direction) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
+		const auto value = Binary(s, code, f, x, y);
+		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
+		return value;
+	};
+	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
+	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
+	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
+	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
+	const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); };
+	// The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0).
+	const std::array vertices {
+		EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, kind, ConstantU32(s, 2)),
+		Select(s, u, is_kind(0), ConstantU32(s, 1),
+		       Select(s, u, is_kind(3), ConstantU32(s, 4), ConstantU32(s, 3))),
+		Select(s, u, is_kind(2), ConstantU32(s, 4),
+		       Select(s, u, is_kind(3), ConstantU32(s, 0), ConstantU32(s, 2))) };
+	std::array<Vec3, 3> positions;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto first = Binary(s, spv::OpIMul, u, vertices[i], ConstantU32(s, 3));
+		for (uint32_t axis = 0; axis < 3; ++axis)
+			positions[i][axis] = Unary(s, spv::OpBitcast, f,
+			    NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis))));
+	}
+	const auto flag = NodeWord(s, block, ConstantU32(s, 15));
+	const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1]));
+	const auto z_largest = cmp(spv::OpFOrdLessThan,
+	    Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2]));
+	const auto rotate = [&](Vec3 value) {
+		Vec3 result;
+		for (uint32_t i = 0; i < 3; ++i)
+			result[i] = Select(s, f, z_largest, value[i],
+			    Select(s, f, y_largest, value[(i + 2) % 3], value[(i + 1) % 3]));
+		return result;
+	};
+	origin = rotate(origin);
+	direction = rotate(direction);
+	std::array<Vec3, 3> projected;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto position = rotate(positions[i]);
+		Vec3 relative;
+		for (uint32_t axis = 0; axis < 3; ++axis)
+			relative[axis] = op(spv::OpFSub, position[axis], origin[axis]);
+		projected[i] = {op(spv::OpFSub, op(spv::OpFMul, relative[0], direction[2]),
+		                              op(spv::OpFMul, direction[0], relative[2])),
+		                op(spv::OpFSub, op(spv::OpFMul, relative[1], direction[2]),
+		                              op(spv::OpFMul, direction[1], relative[2])), relative[2]};
+	}
+	Vec3 edge, weighted;
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto& first = projected[(i + 1) % 3];
+		const auto& second = projected[(i + 2) % 3];
+		edge[i] = op(spv::OpFSub, op(spv::OpFMul, second[0], first[1]),
+		                        op(spv::OpFMul, second[1], first[0]));
+		weighted[i] = op(spv::OpFMul, edge[i], direction[2]);
+	}
+	auto numerator = op(spv::OpFAdd, op(spv::OpFAdd,
+	    op(spv::OpFMul, edge[0], projected[0][2]), op(spv::OpFMul, edge[1], projected[1][2])),
+	    op(spv::OpFMul, edge[2], projected[2][2]));
+	auto denominator = op(spv::OpFAdd, op(spv::OpFAdd, weighted[0], weighted[1]), weighted[2]);
+	uint32_t negative = ConstantBool(s, false), positive = negative;
+	for (auto value: edge) {
+		negative = either(negative, cmp(spv::OpFOrdLessThan, value, zero));
+		positive = either(positive, cmp(spv::OpFOrdGreaterThan, value, zero));
+	}
+	const auto winding = cmp(spv::OpFOrdGreaterThan, denominator, zero);
+	auto missed = either(both(negative, positive), cmp(spv::OpFOrdEqual, denominator, zero));
+	missed = either(missed, Unary(s, spv::OpIsNan, b, numerator));
+	missed = either(missed, cmp(spv::OpFOrdLessThan,
+	    Select(s, f, winding, numerator, Unary(s, spv::OpFNegate, f, numerator)), zero));
+	for (uint32_t i = 0; i < 3; ++i) {
+		const auto first = projected[(i + 1) % 3][1], second = projected[(i + 2) % 3][1];
+		const auto first_zero = cmp(spv::OpFOrdEqual, first, zero);
+		const auto horizontal = both(first_zero, cmp(spv::OpFOrdEqual, second, zero));
+		const auto below = either(cmp(spv::OpFOrdLessThan, first, zero),
+		    both(first_zero, cmp(spv::OpFOrdGreaterThan, second, zero)));
+		const auto right = Binary(s, spv::OpLogicalNotEqual, b, below, winding);
+		const auto excluded = Select(s, b, horizontal,
+		    cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right);
+		missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded));
+	}
+	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
+	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
+	const auto coordinate = [&](uint32_t shift) {
+		const auto index = Binary(s, spv::OpBitwiseAnd, u,
+		    Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3));
+		return Unary(s, spv::OpBitcast, u, Select(s, f,
+		    cmp(spv::OpIEqual, index, ConstantU32(s, 1)), weighted[1],
+		    Select(s, f, cmp(spv::OpIEqual, index, ConstantU32(s, 2)), weighted[2], weighted[0])));
+	};
+	const std::array result {
+		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)),
+		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)),
+		Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)),
+		Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) };
+	return Vector(s, TypeU32Vector(s, 4), result);
+}
+
+std::array<uint32_t, 6> Box(EmitterState& s, uint32_t first, uint32_t second,
+                           uint32_t half, uint32_t child) {
+	const auto f = TypeF32(s);
+	const auto half_label = s.builder.AllocateId(), full_label = s.builder.AllocateId();
+	const auto merge = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+	s.builder.AddFunction(spv::OpBranchConditional, half, half_label, full_label);
+	EmitLabel(s, half_label);
+	std::array<uint32_t, 6> packed, full, result;
+	for (uint32_t pair = 0; pair < 3; ++pair) {
+		const auto word = NodeWord(s, first, ConstantU32(s, 4 + child * 3 + pair));
+		const auto unpacked = EmitGlsl<GLSLstd450UnpackHalf2x16, IR::Type::F32x2>(s, word);
+		packed[pair * 2] = Extract(s, f, unpacked, 0);
+		packed[pair * 2 + 1] = Extract(s, f, unpacked, 1);
+	}
+	s.builder.AddFunction(spv::OpBranch, merge);
+	EmitLabel(s, full_label);
+	for (uint32_t component = 0; component < 6; ++component) {
+		const auto word = 4 + child * 6 + component;
+		full[component] = Unary(s, spv::OpBitcast, f,
+		    NodeWord(s, word < 16 ? first : second, ConstantU32(s, word % 16)));
+	}
+	s.builder.AddFunction(spv::OpBranch, merge);
+	EmitLabel(s, merge);
+	for (uint32_t component = 0; component < 6; ++component) {
+		result[component] = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpPhi, f, result[component], packed[component], half_label,
+		                      full[component], full_label);
+	}
+	return result;
+}
+
+uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
+               uint32_t grow, uint32_t sort, uint32_t extent, const Vec3& origin, const Vec3& inverse) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	std::array<uint32_t, 4> depths, indices;
+	Vec3 forward;
+	for (uint32_t axis = 0; axis < 3; ++axis)
+		forward[axis] = Binary(s, spv::OpFOrdGreaterThanEqual, b, inverse[axis], zero);
+	for (uint32_t child = 0; child < 4; ++child) {
+		const auto box = Box(s, first, second, half, child);
+		Vec3 near, far;
+		for (uint32_t axis = 0; axis < 3; ++axis) {
+			const auto lo = Binary(s, spv::OpFMul, f,
+			    Binary(s, spv::OpFSub, f, box[axis], origin[axis]), inverse[axis]);
+			const auto hi = Binary(s, spv::OpFMul, f,
+			    Binary(s, spv::OpFSub, f, box[axis + 3], origin[axis]), inverse[axis]);
+			near[axis] = Select(s, f, forward[axis], lo, hi);
+			far[axis] = Select(s, f, forward[axis], hi, lo);
+		}
+		const auto entry = EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s,
+		    EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s, near[0], near[1]), near[2]);
+		const auto exit = EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s,
+		    EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s, far[0], far[1]), far[2]);
+		// Valid hits have nonnegative exit distance. Saturate ULP growth at +infinity.
+		const auto bits = Unary(s, spv::OpBitcast, u, Select(s, f,
+		    Binary(s, spv::OpFOrdEqual, b, exit, zero), zero, exit));
+		const auto grown_bits = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+		    Binary(s, spv::OpIAdd, u, bits, grow), ConstantU32(s, 0x7f800000));
+		const auto grown = Unary(s, spv::OpBitcast, f, grown_bits);
+		auto hit = Binary(s, spv::OpLogicalAnd, b,
+		    Binary(s, spv::OpFOrdGreaterThanEqual, b, exit, zero),
+		    Binary(s, spv::OpFOrdLessThan, b, entry, extent));
+		hit = Binary(s, spv::OpLogicalAnd, b, hit,
+		    Binary(s, spv::OpFOrdLessThanEqual, b, entry, grown));
+		depths[child] = Unary(s, spv::OpBitcast, u, Select(s, f, hit,
+		    Select(s, f, Binary(s, spv::OpFOrdGreaterThan, b, entry, zero), entry, zero),
+		    ConstantF32(s, 0x7f800000)));
+		indices[child] = Select(s, u, hit, NodeWord(s, first, ConstantU32(s, child)), ConstantU32(s, ~0u));
+	}
+	// A select network keeps the original child order when sorting is disabled.
+	constexpr std::array<std::pair<uint32_t, uint32_t>, 5> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2}}};
+	for (size_t step = 0; step < network.size(); ++step) {
+		const auto [a, c] = network[step];
+		const auto swap = Binary(s, spv::OpLogicalAnd, b, sort,
+		    Binary(s, spv::OpUGreaterThan, b, depths[a], depths[c]));
+		const auto depth = depths[a], index = indices[a];
+		const auto later = [&](uint32_t position) {
+			return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) {
+				return pair.first == position || pair.second == position;
+			});
+		};
+		if (later(a)) depths[a] = Select(s, u, swap, depths[c], depth);
+		if (later(c)) depths[c] = Select(s, u, swap, depth, depths[c]);
+		indices[a] = Select(s, u, swap, indices[c], index);
+		indices[c] = Select(s, u, swap, index, indices[c]);
+	}
+	return Vector(s, TypeU32Vector(s, 4), indices);
+}
+
+} // namespace
+
+void DefineBvhIntersect(EmitterState& s) {
+	if (!s.requirements.bvh) return;
+	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
+	const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
+	s.bvh_intersect_function = s.builder.AllocateId();
+	s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
+	s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
+	                      spv::FunctionControlMaskNone, signature);
+	std::array<uint32_t, 6> args;
+	const std::array types {vec4, u, f, vec3, vec3, vec3};
+	for (uint32_t i = 0; i < args.size(); ++i) {
+		args[i] = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
+	}
+	EmitLabel(s, s.builder.AllocateId());
+	const auto descriptor = args[0], node = args[1], extent = args[2];
+	Vec3 origin, direction, inverse;
+	for (uint32_t i = 0; i < 3; ++i) {
+		origin[i] = Extract(s, f, args[3], i);
+		direction[i] = Extract(s, f, args[4], i);
+		inverse[i] = Extract(s, f, args[5], i);
+	}
+	std::array<uint32_t, 4> words;
+	for (uint32_t i = 0; i < 4; ++i) words[i] = Extract(s, u, descriptor, i);
+	const auto and_bits = [&](uint32_t value, uint32_t mask) {
+		return Binary(s, spv::OpBitwiseAnd, u, value, ConstantU32(s, mask));
+	};
+	const auto shr = [&](uint32_t value, uint32_t shift) {
+		return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
+	};
+	const auto kind = and_bits(node, 7);
+	const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
+	const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
+	const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
+	const auto bary = Binary(s, spv::OpINotEqual, b, and_bits(words[3], 0x01000000), ConstantU32(s, 0));
+	const auto sort = Binary(s, spv::OpINotEqual, b, and_bits(words[1], 0x80000000), ConstantU32(s, 0));
+	const auto grow = and_bits(shr(words[1], 23), 0xff);
+	const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
+	    DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
+	const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
+	const auto last_index = Binary(s, spv::OpIAdd, wide, index,
+	    Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
+	const auto address = Binary(s, spv::OpIAdd, wide, base,
+	    Binary(s, spv::OpShiftLeftLogical, wide, index, ConstantDeviceAddress(s, 6)));
+	const auto last_address = Binary(s, spv::OpIAdd, wide, base,
+	    Binary(s, spv::OpShiftLeftLogical, wide, last_index, ConstantDeviceAddress(s, 6)));
+	auto valid = Binary(s, spv::OpIEqual, b, shr(words[3], 28), ConstantU32(s, 8));
+	const auto require = [&](uint32_t condition) {
+		valid = Binary(s, spv::OpLogicalAnd, b, valid, condition);
+	};
+	require(Binary(s, spv::OpULessThanEqual, b, kind, ConstantU32(s, 5)));
+	require(Binary(s, spv::OpULessThanEqual, b, last_index, DeviceAddressFromWords(s, words[2], and_bits(words[3], 0x3ff))));
+	require(Binary(s, spv::OpULessThan, b, last_address, ConstantDeviceAddress(s, uint64_t{1} << 48)));
+	require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, extent)));
+	for (uint32_t i = 0; i < 3; ++i) {
+		for (const auto value: {origin[i], direction[i], inverse[i]})
+			require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsNan, b, value)));
+		require(Unary(s, spv::OpLogicalNot, b, Unary(s, spv::OpIsInf, b, origin[i])));
+	}
+	const std::array triangle_invalid {ConstantU32(s, 0x7f800000), ConstantU32(s, 0x3f800000),
+	                                  ConstantU32(s, ~0u), ConstantU32(s, 0)};
+	const std::array box_invalid {ConstantU32(s, ~0u), ConstantU32(s, ~0u),
+	                             ConstantU32(s, ~0u), ConstantU32(s, ~0u)};
+	std::array<uint32_t, 4> invalid_words;
+	for (uint32_t i = 0; i < 4; ++i)
+		invalid_words[i] = Select(s, u, triangle, triangle_invalid[i], box_invalid[i]);
+	const auto invalid = Vector(s, vec4, invalid_words);
+	const auto result = EmitValueOrDefaultIfCondition(s, valid, vec4, invalid, [&] {
+		const auto first = GetBdaPointer(s, address);
+		const auto second = EmitValueOrDefaultIfCondition(s, full, wide, ConstantDeviceAddress(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, wide, address, ConstantDeviceAddress(s, 64)));
+		});
+		const auto present = Binary(s, spv::OpLogicalAnd, b,
+		    Binary(s, spv::OpINotEqual, b, first, ConstantDeviceAddress(s, 0)),
+		    Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, full),
+		        Binary(s, spv::OpINotEqual, b, second, ConstantDeviceAddress(s, 0))));
+		return EmitValueOrDefaultIfCondition(s, present, vec4, invalid, [&] {
+			const auto tri_label = s.builder.AllocateId(), box_label = s.builder.AllocateId();
+			const auto merge = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+			s.builder.AddFunction(spv::OpBranchConditional, triangle, tri_label, box_label);
+			EmitLabel(s, tri_label);
+			const auto tri_value = Triangle(s, first, kind, bary, origin, direction);
+			const auto tri_exit = s.current_label;
+			s.builder.AddFunction(spv::OpBranch, merge);
+			EmitLabel(s, box_label);
+			const auto box_value = Boxes(s, first, second, half, grow, sort, extent, origin, inverse);
+			const auto box_exit = s.current_label;
+			s.builder.AddFunction(spv::OpBranch, merge);
+			EmitLabel(s, merge);
+			const auto value = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpPhi, vec4, value, tri_value, tri_exit, box_value, box_exit);
+			return value;
+		});
+	});
+	s.builder.AddFunction(spv::OpReturnValue, result);
+	s.builder.AddFunction(spv::OpFunctionEnd);
+}
+
+uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto* ray = ctx.ImageAddress(inst.Arg(1));
+	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
+		ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
+	const auto vector = [&](uint32_t first) {
+		Vec3 values;
+		for (uint32_t i = 0; i < 3; ++i)
+			values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
+		return Vector(s, TypeF32Vector(s, 3), values);
+	};
+	const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
+	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
+	const auto origin = vector(2), direction = vector(5), inverse = vector(8);
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
+	    ConstantU32CompositeZero(s, 4), [&] {
+		const auto result = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpFunctionCall, TypeU32Vector(s, 4), result,
+		    s.bvh_intersect_function, descriptor, node, extent, origin, direction, inverse);
+		return result;
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -539,19 +539,6 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 	return PackImageTexel(ctx, mem, texel);
 }
 
-spv::Op ImageAtomicOpcode(IR::ValueOpcode opcode) {
-	switch (opcode) {
-		case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
-		case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
-		case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
-		case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
-		case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
-		case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
-		case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
-		default: return spv::OpNop;
-	}
-}
-
 } // namespace
 
 void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -876,10 +863,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
 		return;
 	}
-	const auto atomic_opcode = ImageAtomicOpcode(op);
 	if (image_info.access == IR::ImageAccess::Atomic) {
 		const auto dimension = image.dimension;
-		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
+		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
 			           const auto pointer      = state.builder.AllocateId();
 			           const auto pointer_type = state.builder.Type(
 			               spv::OpTypePointer, spv::StorageClassImage, TypeU32(state));
@@ -896,12 +882,8 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 					                                   op == IR::ValueOpcode::ImageAtomicFMax32);
 				                               });
 			           }
-			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer,
-			                                     ConstantU32(state, spv::ScopeDevice),
-			                                     ConstantU32(state, spv::MemorySemanticsMaskNone),
-			                                     ctx.Arg(inst, 2));
-			           EmitDeviceAtomicMemoryBarrier(state);
+			           const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
+			           EmitAtomicMemoryBarrier(state, IR::ResourceKind::Image);
 			           return old;
 		           }));
 		return;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -111,6 +111,9 @@ uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitSLessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitULessThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
+uint32_t EmitUGreaterThanEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
 EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
 EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
@@ -133,6 +136,9 @@ uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
 }
 inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
+EMIT_NATIVE(FPOrdEqual64, OpFOrdEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdLessThanEqual64, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
+EMIT_NATIVE(FPOrdGreaterThanEqual64, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
 inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
 inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
 inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
@@ -149,6 +155,7 @@ inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
 EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
+EMIT_NATIVE(FPAdd64, OpFAdd, F64, uint32_t, uint32_t)
 EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t)
 inline constexpr auto EmitFPFma64 =
     EmitGlsl<GLSLstd450Fma, IR::Type::F64, uint32_t, uint32_t, uint32_t>;
@@ -222,7 +229,7 @@ void                  EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst
 void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
-void                  EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -258,6 +265,7 @@ inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicAnd64     = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicOr32      = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicOr64      = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicXor32     = EmitAtomic32;
@@ -279,6 +287,7 @@ inline constexpr auto EmitSharedAtomicFMin32    = EmitSharedFloatAtomic;
 inline constexpr auto EmitSharedAtomicFMax32    = EmitSharedFloatAtomic;
 inline constexpr auto EmitSharedAtomicSwap32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicIAdd32    = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicIAdd64    = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicISub32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicInc32     = EmitSharedIncDec;
 inline constexpr auto EmitSharedAtomicDec32     = EmitSharedIncDec;
@@ -288,20 +297,25 @@ inline constexpr auto EmitSharedAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicAnd32     = EmitAtomic32;
 inline constexpr auto EmitSharedAtomicOr32      = EmitAtomic32;
+inline constexpr auto EmitSharedAtomicOr64      = EmitSharedAtomic64;
 inline constexpr auto EmitSharedAtomicXor32     = EmitAtomic32;
 inline constexpr auto EmitDataAppend            = EmitAppendConsume;
 inline constexpr auto EmitDataConsume           = EmitAppendConsume;
 uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
 inline constexpr auto EmitImageQueryDimensions = EmitImage;
 inline constexpr auto EmitImageQueryLod        = EmitImage;
 inline constexpr auto EmitImageRead            = EmitImage;
 inline constexpr auto EmitImageWrite           = EmitImage;
 inline constexpr auto EmitImageSampleRaw       = EmitImage;
 inline constexpr auto EmitImageGatherRaw       = EmitImage;
+inline constexpr auto EmitImageAtomicCompareSwap32 = EmitImage;
 inline constexpr auto EmitImageAtomicSwap32    = EmitImage;
 inline constexpr auto EmitImageAtomicIAdd32    = EmitImage;
+inline constexpr auto EmitImageAtomicSMin32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMin32    = EmitImage;
+inline constexpr auto EmitImageAtomicSMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicUMax32    = EmitImage;
 inline constexpr auto EmitImageAtomicAnd32     = EmitImage;
 inline constexpr auto EmitImageAtomicOr32      = EmitImage;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -58,6 +58,7 @@ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
 const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 
 struct SpirvRequirements {
+	bool bvh                          = false;
 	bool subgroup_ballot              = false;
 	bool subgroup_shuffle             = false;
 	bool subgroup_local_invocation_id = false;
@@ -83,6 +84,20 @@ struct EmitterState {
 	Builder                                          builder;
 	const IR::Program&                               program;
 	ShaderStageInputInfo                             input_info;
+	uint32_t                                        void_type = 0;
+	uint32_t                                        bool_type = 0;
+	uint32_t                                        u32_type = 0;
+	uint32_t                                        native_u64_type = 0;
+	uint32_t                                        i32_type = 0;
+	uint32_t                                        f32_type = 0;
+	uint32_t                                        f64_type = 0;
+	uint32_t                                        u32_pair_type = 0;
+	uint32_t                                        i32_pair_type = 0;
+	uint32_t                                        function_type = 0;
+	std::array<uint32_t, 3>                          bool_vector_types {};
+	std::array<uint32_t, 3>                          u32_vector_types {};
+	std::array<uint32_t, 3>                          i32_vector_types {};
+	std::array<uint32_t, 3>                          f32_vector_types {};
 	std::array<uint32_t, 6>                          tess_variables {};
 	uint32_t                                         tess_inner_variable = 0;
 	uint32_t                                         tess_patch_base     = 0;
@@ -96,6 +111,7 @@ struct EmitterState {
 	uint32_t                                         bda_pagetable_variable  = 0;
 	uint32_t                                         fault_buffer_variable   = 0;
 	uint32_t                                         bda_pointer_function    = 0;
+	uint32_t                                         bvh_intersect_function  = 0;
 	uint32_t                                         gds_variable            = 0;
 	uint32_t                                         gds_length              = 0;
 	uint32_t                                         push_constant_variable  = 0;
@@ -420,7 +436,12 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits);
 uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
                                   uint32_t component, uint32_t raw);
 
-void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
+spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode);
+
+uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
+                             uint32_t scope);
+
+void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind);
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
                                     bool max_value);
@@ -487,6 +508,10 @@ uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 void EmitProgram(EmitterState& state);
 
 void DefineGetBdaPointer(EmitterState& state);
+void DefineBvhIntersect(EmitterState& state);
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
+uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value);
+uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high);
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
@@ -528,14 +553,7 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
 
 template <typename Fn>
 uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
-	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
-	const auto memory = [&] {
-		switch (kind) {
-			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
-			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
-			default: return spv::MemorySemanticsUniformMemoryMask;
-		}
-	}();
+	const auto scope     = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 	const auto preheader = state.builder.AllocateId();
 	const auto header    = state.builder.AllocateId();
 	const auto cont      = state.builder.AllocateId();
@@ -564,8 +582,7 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
 	EmitLabel(state, cont);
 	state.builder.AddFunction(spv::OpBranch, header);
 	EmitLabel(state, merge);
-	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
-	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
+	EmitAtomicMemoryBarrier(state, kind);
 	return observed;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -1,6 +1,7 @@
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
 
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <algorithm>
 
@@ -105,31 +106,23 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad
 	return result;
 }
 
-uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
-                            uint32_t high) {
-	auto& state     = ctx.state;
-	auto  immediate = static_cast<int32_t>(mem.offset);
+uint32_t AddAddressOffset(EmitterState& state, const IR::MemoryInfo& mem, uint32_t low,
+                          uint32_t& high) {
+	const auto immediate = static_cast<int32_t>(mem.offset);
+	if (immediate == 0) return low;
 	const auto immediate_low  = ConstantU32(state, static_cast<uint32_t>(immediate));
 	const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
-	low                      = AddU64Low(state, low, high, immediate_low, immediate_high, high);
+	return AddU64Low(state, low, high, immediate_low, immediate_high, high);
+}
+
+uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
+                            uint32_t high) {
+	auto& state = ctx.state;
+	low = AddAddressOffset(state, mem, low, high);
 	const auto valid = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
 	return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
 }
 
-uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
-	return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
-	                              static_cast<uint32_t>(value),
-	                              static_cast<uint32_t>(value >> 32u));
-}
-
-uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
-	const auto low64  = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
-	const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
-	                           Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
-	                           ConstantDeviceAddress(state, 32));
-	return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
-}
-
 uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
 	auto& state = ctx.state;
 	auto  low   = ctx.Arg(inst, 1);
@@ -186,17 +179,9 @@ void RecordBdaFault(EmitterState& state, uint32_t page) {
 	                          Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit));
 }
 
-uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
-	auto&      state  = ctx.state;
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
-	                          state.bda_pointer_function, address);
-	return result;
-}
-
 uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
 	auto&      state   = ctx.state;
-	const auto bda     = GetBdaPointer(ctx, address);
+	const auto bda     = GetBdaPointer(state, address);
 	const auto present =
 	    Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
 	return EmitValueOrZeroIfCondition(state, present, [&]() {
@@ -511,6 +496,50 @@ void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resour
 	                              resource.memory_access);
 }
 
+template <typename Fn>
+void ForEachLocalFlatAccess(ValueEmitContext& ctx, const IR::Inst& inst, Fn&& emit) {
+	auto& state = ctx.state;
+	auto mem = ctx.Memory(inst);
+	auto high = ctx.Arg(inst, 2);
+	const auto low = AddAddressOffset(state, mem, ctx.Arg(inst, 1), high);
+	const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), low, ConstantU32(state, 2));
+	const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), ConstantU32(state, 0),
+	    Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, 3)));
+	for (const auto kind: {IR::ResourceKind::Scratch, IR::ResourceKind::Lds}) {
+		mem.kind = kind;
+		const auto resource = PrepareMemoryResourceAccess(state, mem);
+		const auto aperture = kind == IR::ResourceKind::Scratch ? Decoder::PrivateApertureHigh
+		                                                       : Decoder::SharedApertureHigh;
+		const auto selected = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, aperture));
+		const auto valid = AndCondition(state, aligned,
+		    AndCondition(state, selected, EmitMemoryElementInBounds(state, resource, index)));
+		emit(resource, index, valid);
+	}
+}
+
+uint32_t LoadLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		uint32_t result = 0;
+		ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
+			const auto value = EmitValueOrZeroIfCondition(ctx.state, valid, [&]() {
+				return LoadWordInBounds(ctx, resource, index);
+			});
+			result = result == 0 ? value : Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), result, value);
+		});
+		return result;
+	});
+}
+
+void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
+	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		ForEachLocalFlatAccess(ctx, inst, [&](const auto& resource, uint32_t index, uint32_t valid) {
+			EmitIfCondition(ctx.state, valid, [&]() {
+				StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, inst.NumArgs() - 2));
+			});
+		});
+	});
+}
+
 void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                        const MemoryResourceAccess& resource, uint32_t data) {
 	const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
@@ -526,55 +555,6 @@ void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem)
 	});
 }
 
-spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
-	switch (opcode) {
-		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
-		case IR::ValueOpcode::BufferAtomicSwap32:
-		case IR::ValueOpcode::BufferAtomicSwap64:
-		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
-		case IR::ValueOpcode::BufferAtomicIAdd32:
-		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
-		case IR::ValueOpcode::BufferAtomicISub32:
-		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
-		case IR::ValueOpcode::BufferAtomicSMin32:
-		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
-		case IR::ValueOpcode::BufferAtomicUMin32:
-		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
-		case IR::ValueOpcode::BufferAtomicSMax32:
-		case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
-		case IR::ValueOpcode::BufferAtomicUMax32:
-		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
-		case IR::ValueOpcode::BufferAtomicAnd32:
-		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
-		case IR::ValueOpcode::BufferAtomicOr32:
-		case IR::ValueOpcode::BufferAtomicOr64:
-		case IR::ValueOpcode::SharedAtomicOr64:
-		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
-		case IR::ValueOpcode::BufferAtomicXor32:
-		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
-		default: return spv::OpNop;
-	}
-}
-
-uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
-                             uint32_t scope) {
-	const auto old = ctx.state.builder.AllocateId();
-	if (inst.GetOpcode() == IR::ValueOpcode::BufferAtomicCmpSwap32) {
-		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
-		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
-		ctx.state.builder.AddFunction(
-		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
-		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
-		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
-	} else {
-		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
-		ctx.state.builder.AddFunction(SpirvAtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old,
-		                              pointer, ConstantU32(ctx.state, scope),
-		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
-	}
-	return old;
-}
-
 template <typename Fn>
 uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
                           const IR::MemoryInfo& mem, Fn&& operation) {
@@ -961,6 +941,27 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 
 } // namespace
 
+uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
+	return state.builder.Constant(spv::OpConstant, TypeScalarU64(state),
+	                              static_cast<uint32_t>(value),
+	                              static_cast<uint32_t>(value >> 32u));
+}
+
+uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto low64  = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
+	const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
+	                           Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
+	                           ConstantDeviceAddress(state, 32));
+	return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
+}
+
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), result,
+	                          state.bda_pointer_function, address);
+	return result;
+}
+
 void DefineGetBdaPointer(EmitterState& state) {
 	if (!state.program.info.uses_dma) {
 		return;
@@ -1024,14 +1025,7 @@ uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto scope =
 		    mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
 		const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
-		if (mem.kind == IR::ResourceKind::Lds) {
-			const auto semantics =
-			    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
-			ctx.state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
-			                              ConstantU32(ctx.state, semantics));
-		} else {
-			EmitDeviceAtomicMemoryBarrier(ctx.state);
-		}
+		EmitAtomicMemoryBarrier(ctx.state, mem.kind);
 		return old;
 	});
 }
@@ -1059,13 +1053,13 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 			                                            TypeStorageBufferU64ElementPointer(state)),
 			            ConstantU32(state, spv::ScopeDevice),
 			            ConstantU32(state, spv::MemorySemanticsMaskNone), value);
-			        EmitDeviceAtomicMemoryBarrier(state);
+			        EmitAtomicMemoryBarrier(state, mem.kind);
 			        return Unary(state, spv::OpBitcast, TypeU64(state), old);
 		        });
 	    });
 }
 
-void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst) {
+void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	const auto& mem = ctx.Memory(inst);
 	EnsureLdsStorage(state);
@@ -1243,7 +1237,9 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto shared_components = IR::SharedComponentCount(op);
 	const auto address_info      = IR::AddressOpcodeInfoOf(op);
 	uint32_t   value;
-	if (buffer_components > 1u)
+	if (mem.kind == IR::ResourceKind::FlatLocal)
+		value = LoadLocalFlat(ctx, inst);
+	else if (buffer_components > 1u)
 		value = LoadWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		value = LoadWideShared(ctx, inst, shared_components);
@@ -1270,7 +1266,9 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto  buffer_components = IR::BufferComponentCount(op);
 	const auto  shared_components = IR::SharedComponentCount(op);
 	const auto  type              = inst.Arg(inst.NumArgs() - 2).GetType();
-	if (buffer_components > 1u)
+	if (mem.kind == IR::ResourceKind::FlatLocal)
+		StoreLocalFlat(ctx, inst);
+	else if (buffer_components > 1u)
 		StoreWideBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		StoreWideShared(ctx, inst, shared_components);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -306,11 +306,79 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 	}
 }
 
-void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
-	const auto semantics =
-	    spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask;
-	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice),
-	                          ConstantU32(state, semantics));
+spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
+	switch (opcode) {
+		case IR::ValueOpcode::ImageAtomicCompareSwap32:
+		case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
+		case IR::ValueOpcode::ImageAtomicSwap32:
+		case IR::ValueOpcode::BufferAtomicSwap32:
+		case IR::ValueOpcode::BufferAtomicSwap64:
+		case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
+		case IR::ValueOpcode::ImageAtomicIAdd32:
+		case IR::ValueOpcode::BufferAtomicIAdd32:
+		case IR::ValueOpcode::SharedAtomicIAdd64:
+		case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
+		case IR::ValueOpcode::BufferAtomicISub32:
+		case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
+		case IR::ValueOpcode::ImageAtomicSMin32:
+		case IR::ValueOpcode::BufferAtomicSMin32:
+		case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
+		case IR::ValueOpcode::ImageAtomicUMin32:
+		case IR::ValueOpcode::BufferAtomicUMin32:
+		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicSMax32:
+		case IR::ValueOpcode::BufferAtomicSMax32:
+		case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
+		case IR::ValueOpcode::ImageAtomicUMax32:
+		case IR::ValueOpcode::BufferAtomicUMax32:
+		case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32:
+		case IR::ValueOpcode::BufferAtomicAnd32:
+		case IR::ValueOpcode::BufferAtomicAnd64:
+		case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32:
+		case IR::ValueOpcode::BufferAtomicOr32:
+		case IR::ValueOpcode::BufferAtomicOr64:
+		case IR::ValueOpcode::SharedAtomicOr64:
+		case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32:
+		case IR::ValueOpcode::BufferAtomicXor32:
+		case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
+		default: return spv::OpNop;
+	}
+}
+
+uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t pointer,
+                             uint32_t scope) {
+	const auto opcode = SpirvAtomicOpcode(inst.GetOpcode());
+	const auto old    = ctx.state.builder.AllocateId();
+	if (opcode == spv::OpAtomicCompareExchange) {
+		const auto desired    = ctx.Arg(inst, inst.NumArgs() - 3);
+		const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
+		ctx.state.builder.AddFunction(
+		    spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
+		    ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
+		    ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator);
+	} else {
+		const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
+		ctx.state.builder.AddFunction(opcode, TypeU32(ctx.state), old, pointer,
+		                              ConstantU32(ctx.state, scope),
+		                              ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value);
+	}
+	return old;
+}
+
+void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
+	const auto scope  = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
+	const auto memory = [&] {
+		switch (kind) {
+			case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
+			case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
+			default: return spv::MemorySemanticsUniformMemoryMask;
+		}
+	}();
+	state.builder.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope),
+	                          ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
 }
 
 uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -6,19 +6,32 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t TypeVoid(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeVoid);
+	if (state.void_type == 0) {
+		state.void_type = state.builder.Type(spv::OpTypeVoid);
+	}
+	return state.void_type;
 }
 
 uint32_t TypeBool(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeBool);
+	if (state.bool_type == 0) {
+		state.bool_type = state.builder.Type(spv::OpTypeBool);
+	}
+	return state.bool_type;
 }
 
 uint32_t TypeBoolVector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeBool(state), components);
+	auto& type = state.bool_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeBool(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeU32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 32, 0);
+	if (state.u32_type == 0) {
+		state.u32_type = state.builder.Type(spv::OpTypeInt, 32, 0);
+	}
+	return state.u32_type;
 }
 
 uint32_t TypeU64(EmitterState& state) {
@@ -26,33 +39,55 @@ uint32_t TypeU64(EmitterState& state) {
 }
 
 uint32_t TypeScalarU64(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 64, 0);
+	if (state.native_u64_type == 0) {
+		state.native_u64_type = state.builder.Type(spv::OpTypeInt, 64, 0);
+	}
+	return state.native_u64_type;
 }
 
 uint32_t TypeU32Pair(EmitterState& state) {
-	const auto element = TypeU32(state);
-	return state.builder.Type(spv::OpTypeStruct, element, element);
+	if (state.u32_pair_type == 0) {
+		const auto element = TypeU32(state);
+		state.u32_pair_type = state.builder.Type(spv::OpTypeStruct, element, element);
+	}
+	return state.u32_pair_type;
 }
 
 uint32_t TypeI32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeInt, 32, 1);
+	if (state.i32_type == 0) {
+		state.i32_type = state.builder.Type(spv::OpTypeInt, 32, 1);
+	}
+	return state.i32_type;
 }
 
 uint32_t TypeI32Pair(EmitterState& state) {
-	const auto element = TypeI32(state);
-	return state.builder.Type(spv::OpTypeStruct, element, element);
+	if (state.i32_pair_type == 0) {
+		const auto element = TypeI32(state);
+		state.i32_pair_type = state.builder.Type(spv::OpTypeStruct, element, element);
+	}
+	return state.i32_pair_type;
 }
 
 uint32_t TypeF32(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFloat, 32);
+	if (state.f32_type == 0) {
+		state.f32_type = state.builder.Type(spv::OpTypeFloat, 32);
+	}
+	return state.f32_type;
 }
 
 uint32_t TypeF64(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFloat, 64);
+	if (state.f64_type == 0) {
+		state.f64_type = state.builder.Type(spv::OpTypeFloat, 64);
+	}
+	return state.f64_type;
 }
 
 uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
+	auto& type = state.u32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
@@ -61,11 +96,19 @@ uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
 }
 
 uint32_t TypeI32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeI32(state), components);
+	auto& type = state.i32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeI32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components) {
-	return state.builder.Type(spv::OpTypeVector, TypeF32(state), components);
+	auto& type = state.f32_vector_types[components - 2u];
+	if (type == 0) {
+		type = state.builder.Type(spv::OpTypeVector, TypeF32(state), components);
+	}
+	return type;
 }
 
 uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee) {
@@ -73,7 +116,10 @@ uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint3
 }
 
 uint32_t TypeFunction(EmitterState& state) {
-	return state.builder.Type(spv::OpTypeFunction, TypeVoid(state));
+	if (state.function_type == 0) {
+		state.function_type = state.builder.Type(spv::OpTypeFunction, TypeVoid(state));
+	}
+	return state.function_type;
 }
 
 uint32_t StorageRuntimeArrayType(EmitterState& state) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -243,6 +243,12 @@ void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::
 }
 
 void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
+	if (ctx.half != 0 && (inst.GetOpcode() == IR::ValueOpcode::Ballot ||
+	                     inst.GetOpcode() == IR::ValueOpcode::ReadFirstLane)) {
+		// Both operations already combine both emulated halves into one whole-wave result.
+		ctx.Define(inst, ctx.other_half->Result(inst));
+		return;
+	}
 	switch (inst.GetOpcode()) {
 #define VALUE_OPCODE(name, ...)                                                                    \
 	case IR::ValueOpcode::name: return Invoke(Emit##name, ctx, inst);
@@ -679,6 +685,7 @@ void EmitProgram(EmitterState& state) {
 		}
 	}
 	DefineGetBdaPointer(state);
+	DefineBvhIntersect(state);
 	for (const auto* block: program.blocks) {
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
 			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -192,8 +192,11 @@ constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
 
 constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
     {0x0fu, Opcode::IMAGE_ATOMIC_SWAP},
+    {0x10u, Opcode::IMAGE_ATOMIC_CMPSWAP},
     {0x11u, Opcode::IMAGE_ATOMIC_ADD},
+    {0x14u, Opcode::IMAGE_ATOMIC_SMIN},
     {0x15u, Opcode::IMAGE_ATOMIC_UMIN},
+    {0x16u, Opcode::IMAGE_ATOMIC_SMAX},
     {0x17u, Opcode::IMAGE_ATOMIC_UMAX},
     {0x18u, Opcode::IMAGE_ATOMIC_AND},
     {0x19u, Opcode::IMAGE_ATOMIC_OR},
@@ -225,6 +228,7 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
+		case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
@@ -253,6 +257,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
 	}
 
 	switch (opcode) {
+		case 0xe6u: return 11u;
 		case 0x0eu: return 1u;
 		case 0x01u:
 		case 0x09u: return ImageCoordComponents(dimension) + 1u;
@@ -355,11 +360,20 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		SetUnsupported(inst, Family::MIMG, opcode,
 		               "MIMG image gather requires exactly one dmask bit");
 	}
+	if (inst.opcode == Opcode::IMAGE_ATOMIC_CMPSWAP && inst.dmask != 0x3u) {
+		SetUnsupported(inst, Family::MIMG, opcode,
+		               "MIMG image compare-and-swap requires 32-bit DMASK 0x3");
+	}
 	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
 	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
+	if (opcode == 0xe6u &&
+	    (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
+		SetUnsupported(inst, Family::MIMG, opcode,
+		               "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
+	}
 
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -61,6 +61,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
     {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
     {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
+    {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
     {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
 };
 
@@ -106,7 +107,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
-    {0x4au, Opcode::DS_OR_B64, 2, 32},
+    {0x40u, Opcode::DS_ADD_U64, 2, 32},         {0x4au, Opcode::DS_OR_B64, 2, 32},
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
     {0x77u, Opcode::DS_READ2_B64, 4, 32},        {0x78u, Opcode::DS_READ2ST64_B64, 4, 32},
@@ -168,6 +169,7 @@ bool IsDsWriteOpcode(Opcode opcode) {
 bool IsDsAtomicOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_U64:
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -46,6 +46,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x09u, Opcode::S_WQM_B32},
     {0x0au, Opcode::S_WQM_B64},
     {0x0bu, Opcode::S_BREV_B32},
+    {0x0cu, Opcode::S_BREV_B64},
     {0x0fu, Opcode::S_BCNT1_I32_B32},
     {0x10u, Opcode::S_BCNT1_I32_B64},
     {0x13u, Opcode::S_FF1_I32_B32},
@@ -108,6 +109,7 @@ constexpr OpcodeMap SOPP_OPCODE_LIST[] = {
     {0x12u, Opcode::S_TRAP},
     {0x16u, Opcode::S_TTRACEDATA},
     {0x17u, Opcode::S_CBRANCH_CDBGSYS},
+    {0x19u, Opcode::S_CBRANCH_CDBGSYS_OR_USER},
     {0x20u, Opcode::S_INST_PREFETCH},
     {0x23u, Opcode::S_WAITCNT_DEPCTR},
 };

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -184,8 +184,9 @@ std::string FormatExp(const Instruction& inst) {
 
 bool IsConditionalBranch(Opcode opcode) {
 	switch (opcode) {
-		// DevKit NGG should be disabled.
-		case Opcode::S_CBRANCH_CDBGSYS: return false;
+		// Conditional shader debugging is disabled.
+		case Opcode::S_CBRANCH_CDBGSYS:
+		case Opcode::S_CBRANCH_CDBGSYS_OR_USER: return false;
 		case Opcode::S_CBRANCH_SCC0:
 		case Opcode::S_CBRANCH_SCC1:
 		case Opcode::S_CBRANCH_VCCZ:
@@ -253,6 +254,8 @@ void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) {
 		case 125u: operand.kind = OperandKind::Null; return;
 		case 126u: operand.kind = OperandKind::ExecLo; return;
 		case 127u: operand.kind = OperandKind::ExecHi; return;
+		case 235u: operand.kind = OperandKind::SharedBase; return;
+		case 237u: operand.kind = OperandKind::PrivateBase; return;
 		case 239u: operand.kind = OperandKind::PopsExitingWaveId; return;
 		case 248u:
 			operand.kind      = OperandKind::FloatInlineConstant;
@@ -403,7 +406,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 	program.instructions.clear();
 	program.instructions.reserve(code.size());
 	program.code = code;
-	program.has_bvh = false;
 
 	std::vector<bool> branch_targets;
 	for (uint32_t word_index = 0; word_index < code.size();) {
@@ -412,10 +414,6 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 
 		const auto& inst = program.instructions.back();
 		word_index += inst.word_count;
-		if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) {
-			program.has_bvh = true;
-			return;
-		}
 
 		if (IsDirectBranch(inst.opcode)) {
 			const auto target_index = inst.branch_target / sizeof(uint32_t);
@@ -454,6 +452,8 @@ std::string OperandToString(const Operand& operand) {
 		case OperandKind::Scc: text = "scc"; break;
 		case OperandKind::M0: text = "m0"; break;
 		case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break;
+		case OperandKind::SharedBase: text = "shared_base"; break;
+		case OperandKind::PrivateBase: text = "private_base"; break;
 		case OperandKind::Null: text = "null"; break;
 		default: text = "unknown"; break;
 	}
@@ -504,6 +504,7 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_ABS_I32:
 		case Opcode::S_BREV_B32:
+		case Opcode::S_BREV_B64:
 		case Opcode::S_BCNT1_I32_B32:
 		case Opcode::S_FLBIT_I32_B32:
 		case Opcode::S_FF1_I32_B32:
@@ -560,6 +561,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_CBRANCH_EXECZ:
 		case Opcode::S_CBRANCH_EXECNZ:
 		case Opcode::S_CBRANCH_CDBGSYS:
+		case Opcode::S_CBRANCH_CDBGSYS_OR_USER:
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc,
 			                                               magic_enum::enum_name(inst.opcode),
 			                                               inst.branch_target));
@@ -572,9 +574,12 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_STORE:
 		case Opcode::IMAGE_STORE_MIP:
+		case Opcode::IMAGE_ATOMIC_CMPSWAP:
 		case Opcode::IMAGE_ATOMIC_SWAP:
 		case Opcode::IMAGE_ATOMIC_ADD:
+		case Opcode::IMAGE_ATOMIC_SMIN:
 		case Opcode::IMAGE_ATOMIC_UMIN:
+		case Opcode::IMAGE_ATOMIC_SMAX:
 		case Opcode::IMAGE_ATOMIC_UMAX:
 		case Opcode::IMAGE_ATOMIC_AND:
 		case Opcode::IMAGE_ATOMIC_OR:
@@ -641,6 +646,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_SMAX:
 		case Opcode::BUFFER_ATOMIC_UMAX:
 		case Opcode::BUFFER_ATOMIC_AND:
+		case Opcode::BUFFER_ATOMIC_AND_X2:
 		case Opcode::BUFFER_ATOMIC_OR:
 		case Opcode::BUFFER_ATOMIC_OR_X2:
 		case Opcode::BUFFER_ATOMIC_XOR:
@@ -663,6 +669,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_STORE_DWORDX3:
 		case Opcode::FLAT_STORE_DWORDX4:
 		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_U64:
 		case Opcode::DS_ADD_RTN_U32:
 		case Opcode::DS_SUB_U32:
 		case Opcode::DS_SUB_RTN_U32:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -10,6 +10,10 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
+// PS5 flat stack addresses use these SH_MEM_BASES aperture tags in VA[63:32].
+constexpr uint32_t PrivateApertureHigh = 0x70000000u;
+constexpr uint32_t SharedApertureHigh = 0x80000000u;
+
 enum class Family {
 	Unknown,
 	SOP1,
@@ -43,6 +47,7 @@ enum class Opcode {
 	S_ABS_I32,
 	S_ABSDIFF_I32,
 	S_BREV_B32,
+	S_BREV_B64,
 	S_BCNT1_I32_B32,
 	S_BCNT1_I32_B64,
 	S_FF1_I32_B32,
@@ -240,6 +245,7 @@ enum class Opcode {
 	V_MAD_U32_U24,
 	V_MAD_U64_U32,
 	V_FMA_F64,
+	V_ADD_F64,
 	V_MUL_F64,
 	V_FMA_F32,
 	V_FMA_F16,
@@ -355,6 +361,10 @@ enum class Opcode {
 	V_CMP_NEQ_F32,
 	V_CMP_NLT_F32,
 	V_CMP_TRU_F32,
+	V_CMP_EQ_F64,
+	V_CMP_LE_F64,
+	V_CMPX_LE_F64,
+	V_CMPX_GE_F64,
 	V_CMPX_LT_F32,
 	V_CMPX_EQ_F32,
 	V_CMPX_LE_F32,
@@ -385,12 +395,18 @@ enum class Opcode {
 	V_CMP_NE_I16,
 	V_CMP_GE_I16,
 	V_CMPX_LT_I16,
+	V_CMPX_EQ_I16,
+	V_CMPX_LE_I16,
+	V_CMPX_GT_I16,
+	V_CMPX_NE_I16,
+	V_CMPX_GE_I16,
 	V_CMP_LT_F16,
 	V_CMP_EQ_F16,
 	V_CMP_LE_F16,
 	V_CMP_GT_F16,
 	V_CMP_LG_F16,
 	V_CMP_GE_F16,
+	V_CMP_NGE_F16,
 	V_CMP_NGT_F16,
 	V_CMP_NEQ_F16,
 	V_CMP_NLT_F16,
@@ -427,11 +443,16 @@ enum class Opcode {
 	V_CMP_GE_U32,
 	V_CMP_T_U32,
 	V_CMP_EQ_I64,
+	V_CMP_LT_I64,
+	V_CMP_LE_I64,
 	V_CMP_LT_U64,
 	V_CMP_EQ_U64,
+	V_CMP_LE_U64,
 	V_CMP_GT_U64,
 	V_CMP_NE_U64,
+	V_CMP_GE_U64,
 	V_CMPX_NE_I64,
+	V_CMPX_LE_U64,
 	V_CMPX_NE_U64,
 	V_CMPX_LT_U32,
 	V_CMPX_EQ_U32,
@@ -491,6 +512,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_SMAX,
 	BUFFER_ATOMIC_UMAX,
 	BUFFER_ATOMIC_AND,
+	BUFFER_ATOMIC_AND_X2,
 	BUFFER_ATOMIC_OR,
 	BUFFER_ATOMIC_OR_X2,
 	BUFFER_ATOMIC_XOR,
@@ -511,6 +533,7 @@ enum class Opcode {
 	FLAT_STORE_DWORDX3,
 	FLAT_STORE_DWORDX4,
 	DS_ADD_U32,
+	DS_ADD_U64,
 	DS_ADD_RTN_U32,
 	DS_SUB_U32,
 	DS_SUB_RTN_U32,
@@ -568,6 +591,7 @@ enum class Opcode {
 	DS_WRITE_B128,
 	DS_WRITE_ADDTID_B32,
 	DS_READ_ADDTID_B32,
+	IMAGE_BVH_INTERSECT_RAY,
 	IMAGE_GET_RESINFO,
 	IMAGE_GET_LOD,
 	IMAGE_LOAD,
@@ -575,8 +599,11 @@ enum class Opcode {
 	IMAGE_STORE,
 	IMAGE_STORE_MIP,
 	IMAGE_ATOMIC_SWAP,
+	IMAGE_ATOMIC_CMPSWAP,
 	IMAGE_ATOMIC_ADD,
+	IMAGE_ATOMIC_SMIN,
 	IMAGE_ATOMIC_UMIN,
+	IMAGE_ATOMIC_SMAX,
 	IMAGE_ATOMIC_UMAX,
 	IMAGE_ATOMIC_AND,
 	IMAGE_ATOMIC_OR,
@@ -608,6 +635,7 @@ enum class Opcode {
 	S_CBRANCH_EXECZ,
 	S_CBRANCH_EXECNZ,
 	S_CBRANCH_CDBGSYS,
+	S_CBRANCH_CDBGSYS_OR_USER,
 	S_SENDMSG,
 	S_SETREG_B32,
 	S_SLEEP,
@@ -634,6 +662,8 @@ enum class OperandKind {
 	Scc,
 	M0,
 	PopsExitingWaveId,
+	SharedBase,
+	PrivateBase,
 	Null,
 	Vgpr,
 };
@@ -745,7 +775,6 @@ struct Instruction {
 struct Program {
 	std::span<const uint32_t> code;
 	std::vector<Instruction>  instructions;
-	bool                     has_bvh = false;
 };
 
 // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -228,6 +228,8 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x19u, Opcode::V_CMPX_NGE_F32},       {0x1au, Opcode::V_CMPX_NLG_F32},
     {0x1bu, Opcode::V_CMPX_NGT_F32},       {0x1cu, Opcode::V_CMPX_NLE_F32},
     {0x1du, Opcode::V_CMPX_NEQ_F32},       {0x1eu, Opcode::V_CMPX_NLT_F32},
+    {0x22u, Opcode::V_CMP_EQ_F64, false},  {0x23u, Opcode::V_CMP_LE_F64, false},
+    {0x33u, Opcode::V_CMPX_LE_F64, false}, {0x36u, Opcode::V_CMPX_GE_F64, false},
     {0x80u, Opcode::V_CMP_F_I32},          {0x81u, Opcode::V_CMP_LT_I32},
     {0x82u, Opcode::V_CMP_EQ_I32},         {0x83u, Opcode::V_CMP_LE_I32},
     {0x84u, Opcode::V_CMP_GT_I32},         {0x85u, Opcode::V_CMP_NE_I32},
@@ -239,7 +241,9 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x92u, Opcode::V_CMPX_EQ_I32},        {0x93u, Opcode::V_CMPX_LE_I32},
     {0x94u, Opcode::V_CMPX_GT_I32},        {0x95u, Opcode::V_CMPX_NE_I32},
     {0x96u, Opcode::V_CMPX_GE_I32},        {0x98u, Opcode::V_CMPX_CLASS_F32},
-    {0x99u, Opcode::V_CMPX_LT_I16, false},
+    {0x99u, Opcode::V_CMPX_LT_I16, false}, {0x9au, Opcode::V_CMPX_EQ_I16, false},
+    {0x9bu, Opcode::V_CMPX_LE_I16, false}, {0x9cu, Opcode::V_CMPX_GT_I16, false},
+    {0x9du, Opcode::V_CMPX_NE_I16, false}, {0x9eu, Opcode::V_CMPX_GE_I16, false},
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
@@ -249,16 +253,19 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
     {0xc5u, Opcode::V_CMP_NE_U32},         {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},          {0xa2u, Opcode::V_CMP_EQ_I64, false},
+    {0xa1u, Opcode::V_CMP_LT_I64, false},  {0xa3u, Opcode::V_CMP_LE_I64, false},
     {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
     {0xd2u, Opcode::V_CMPX_EQ_U32},        {0xd3u, Opcode::V_CMPX_LE_U32},
     {0xd4u, Opcode::V_CMPX_GT_U32},        {0xd5u, Opcode::V_CMPX_NE_U32},
     {0xd6u, Opcode::V_CMPX_GE_U32},        {0xe1u, Opcode::V_CMP_LT_U64, false},
-    {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe4u, Opcode::V_CMP_GT_U64, false},
-    {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xe2u, Opcode::V_CMP_EQ_U64, false},  {0xe3u, Opcode::V_CMP_LE_U64, false},
+    {0xe4u, Opcode::V_CMP_GT_U64, false},  {0xe5u, Opcode::V_CMP_NE_U64, false},
+    {0xe6u, Opcode::V_CMP_GE_U64, false},  {0xf3u, Opcode::V_CMPX_LE_U64, false},
     {0xf5u, Opcode::V_CMPX_NE_U64, false}, {0xc9u, Opcode::V_CMP_LT_F16},
     {0xcau, Opcode::V_CMP_EQ_F16},         {0xcbu, Opcode::V_CMP_LE_F16},
     {0xccu, Opcode::V_CMP_GT_F16},         {0xcdu, Opcode::V_CMP_LG_F16},
-    {0xceu, Opcode::V_CMP_GE_F16},         {0xebu, Opcode::V_CMP_NGT_F16},
+    {0xceu, Opcode::V_CMP_GE_F16},         {0xe9u, Opcode::V_CMP_NGE_F16},
+    {0xebu, Opcode::V_CMP_NGT_F16},
     {0xedu, Opcode::V_CMP_NEQ_F16},        {0xeeu, Opcode::V_CMP_NLT_F16},
     {0xd9u, Opcode::V_CMPX_LT_F16},        {0xdau, Opcode::V_CMPX_EQ_F16},
     {0xdbu, Opcode::V_CMPX_LE_F16},        {0xdcu, Opcode::V_CMPX_GT_F16},
@@ -280,6 +287,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x147u, Opcode::V_CUBEMA_F32},
     {0x14bu, Opcode::V_FMA_F32},
     {0x14cu, Opcode::V_FMA_F64},
+    {0x164u, Opcode::V_ADD_F64},
     {0x165u, Opcode::V_MUL_F64},
     {0x148u, Opcode::V_BFE_U32},
     {0x149u, Opcode::V_BFE_I32},
@@ -829,6 +837,10 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMP_F_F32:
 		case Opcode::V_CMP_LT_F32:
 		case Opcode::V_CMP_EQ_F32:
+		case Opcode::V_CMP_EQ_F64:
+		case Opcode::V_CMP_LE_F64:
+		case Opcode::V_CMPX_LE_F64:
+		case Opcode::V_CMPX_GE_F64:
 		case Opcode::V_CMP_LE_F32:
 		case Opcode::V_CMP_GT_F32:
 		case Opcode::V_CMP_LG_F32:
@@ -861,6 +873,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMP_GT_F16:
 		case Opcode::V_CMP_LG_F16:
 		case Opcode::V_CMP_GE_F16:
+		case Opcode::V_CMP_NGE_F16:
 		case Opcode::V_CMP_NGT_F16:
 		case Opcode::V_CMP_NEQ_F16:
 		case Opcode::V_CMP_NLT_F16:
@@ -1139,7 +1152,9 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) {
 }
 
 bool SupportsVopcSdwa(Opcode opcode) {
-	return opcode != Opcode::UNSUPPORTED;
+	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 &&
+	       opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 &&
+	       opcode != Opcode::V_CMPX_GE_F64;
 }
 
 void DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
@@ -1204,6 +1219,7 @@ void DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
 
 uint32_t NativeVop3SourceCount(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
 		case Opcode::V_MUL_LO_U32:
 		case Opcode::V_MUL_HI_U32:
@@ -1355,6 +1371,7 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
 		case Opcode::V_MAX_F32:
 		case Opcode::V_MAC_F32:
 		case Opcode::V_MAD_F32:
+		case Opcode::V_ADD_F64:
 		case Opcode::V_MUL_F64:
 		case Opcode::V_FMA_F64:
 		case Opcode::V_FMA_F32:
@@ -1460,6 +1477,8 @@ void ApplyNativeVop3SourceModifiers(Instruction& inst, uint32_t abs, uint32_t ne
 
 bool IsVopcCompareExec(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::V_CMPX_LE_F64:
+		case Opcode::V_CMPX_GE_F64:
 		case Opcode::V_CMPX_LT_F32:
 		case Opcode::V_CMPX_EQ_F32:
 		case Opcode::V_CMPX_LE_F32:
@@ -1487,8 +1506,14 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_U32:
 		case Opcode::V_CMPX_GE_U32:
 		case Opcode::V_CMPX_NE_I64:
+		case Opcode::V_CMPX_LE_U64:
 		case Opcode::V_CMPX_NE_U64:
 		case Opcode::V_CMPX_LT_I16:
+		case Opcode::V_CMPX_EQ_I16:
+		case Opcode::V_CMPX_LE_I16:
+		case Opcode::V_CMPX_GT_I16:
+		case Opcode::V_CMPX_NE_I16:
+		case Opcode::V_CMPX_GE_I16:
 		case Opcode::V_CMPX_LT_U16:
 		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -1,5 +1,7 @@
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 
+#include <bit>
+
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
 void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar,
@@ -26,9 +28,20 @@ void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool valu
 
 void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                     IR::Type type, bool scalar, bool cmpx) {
-	const auto lhs = ReadOperand(inst.src0, type);
-	const auto rhs = ReadOperand(inst.src1, type);
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx);
+	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
+	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMPX_NE_I64;
+	const auto read = [&](const Decoder::Operand& operand) {
+		// RDNA2 expands signed 64-bit integer literals by sign extension.
+		if (signed_64 && operand.kind == Decoder::OperandKind::LiteralConstant) {
+			return IR::Value(static_cast<uint64_t>(
+			    static_cast<int64_t>(std::bit_cast<int32_t>(operand.value))));
+		}
+		return ReadOperand(operand, type);
+	};
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {read(inst.src0), read(inst.src1)})), scalar,
+	                  cmpx);
 }
 
 void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -40,11 +53,13 @@ void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::Valu
 
 void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                   bool half, bool cmpx) {
+	const auto type = IR::ArgTypeOf(opcode, 0);
 	const auto lhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
+	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, type);
 	const auto rhs =
-	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
-	const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
+	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, type);
+	const IR::FPCompareFlags flags{
+	    .flush_input_denorms = !half && type == IR::Type::F32 && flush_f32_inputs};
 	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
 }
 

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -558,9 +558,18 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
-	const auto result =
-	    ir.Emit(opcode, {resource, address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
-	            AddMemoryInfo(memory, inst.pc));
+	const auto data     = MemorySourceAt(inst, 0);
+	const auto flags    = AddMemoryInfo(memory, inst.pc);
+	IR::Value  result;
+	if (opcode == IR::ValueOpcode::ImageAtomicCompareSwap32) {
+		// VDATA supplies the replacement; VDATA+1 supplies the comparison.
+		result = ir.Emit(opcode,
+		                 {resource, address, ReadU32(data), ReadU32(OffsetOperand(data, 1u)),
+		                  ir.GetExec()},
+		                 flags);
+	} else {
+		result = ir.Emit(opcode, {resource, address, ReadU32(data), ir.GetExec()}, flags);
+	}
 	if (inst.glc) {
 		WriteOperand(inst.dst, result);
 	}
@@ -883,6 +892,15 @@ void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
 
 void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
+		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
+			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
+			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+			for (uint32_t component = 0; component < 4; ++component) {
+				WriteOperand(OffsetOperand(inst.dst, component),
+				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
+			}
+			return;
+		}
 		case Decoder::Opcode::S_LOAD_DWORD:
 		case Decoder::Opcode::S_LOAD_DWORDX2:
 		case Decoder::Opcode::S_LOAD_DWORDX4:
@@ -957,6 +975,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMax32);
 		case Decoder::Opcode::BUFFER_ATOMIC_AND:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd32);
+		case Decoder::Opcode::BUFFER_ATOMIC_AND_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicAnd64);
 		case Decoder::Opcode::BUFFER_ATOMIC_OR:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicOr32);
 		case Decoder::Opcode::BUFFER_ATOMIC_OR_X2:
@@ -970,6 +990,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 
 		case Decoder::Opcode::DS_ADD_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, false);
+		case Decoder::Opcode::DS_ADD_U64:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd64, false);
 		case Decoder::Opcode::DS_ADD_RTN_U32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicIAdd32, true);
 		case Decoder::Opcode::DS_SUB_U32:
@@ -1017,12 +1039,18 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
 
+		case Decoder::Opcode::IMAGE_ATOMIC_CMPSWAP:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicCompareSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_SWAP:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSwap32);
 		case Decoder::Opcode::IMAGE_ATOMIC_ADD:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicIAdd32);
+		case Decoder::Opcode::IMAGE_ATOMIC_SMIN:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMin32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMIN:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMin32);
+		case Decoder::Opcode::IMAGE_ATOMIC_SMAX:
+			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_UMAX:
 			return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32);
 		case Decoder::Opcode::IMAGE_ATOMIC_AND:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -155,6 +155,12 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BREV_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitReverse32, IR::Type::U32, false, false,
 			                     false);
+		case O::S_BREV_B64: {
+			const auto value = ReadU32Pair(inst.src0);
+			WriteU32Pair(inst.dst, {IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[1]})),
+			                        IR::U32(ir.Emit(IR::ValueOpcode::BitReverse32, {value[0]}))});
+			return;
+		}
 		case O::S_BCNT1_I32_B32:
 			return SimpleInteger(inst, IR::ValueOpcode::BitCount32, IR::Type::U32, false, false,
 			                     true);
@@ -231,6 +237,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_CBRANCH_EXECZ:
 		case O::S_CBRANCH_EXECNZ:
 		case O::S_CBRANCH_CDBGSYS:
+		case O::S_CBRANCH_CDBGSYS_OR_USER:
 		case O::S_ENDPGM: return;
 		default: return FailMissingTranslation(inst);
 	}

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -135,6 +135,8 @@ IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
 		case Decoder::OperandKind::IntegerInlineConstant:
 		case Decoder::OperandKind::FloatInlineConstant: return IR::U32(IR::Value(operand.value));
 		case Decoder::OperandKind::Null:
+		case Decoder::OperandKind::SharedBase:
+		case Decoder::OperandKind::PrivateBase:
 		case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u));
 		case Decoder::OperandKind::Sgpr:
 			return ir.GetScalarReg(static_cast<IR::ScalarReg>(operand.reg));
@@ -474,6 +476,12 @@ IR::U32 Translator::ReadU32(const Decoder::Operand& operand) {
 }
 
 std::array<IR::U32, 2> Translator::ReadU32Pair(const Decoder::Operand& operand) {
+	if (operand.kind == Decoder::OperandKind::PrivateBase ||
+	    operand.kind == Decoder::OperandKind::SharedBase) {
+		return {IR::U32(IR::Value(0u)), IR::U32(IR::Value(
+		    operand.kind == Decoder::OperandKind::PrivateBase ? Decoder::PrivateApertureHigh
+		                                                      : Decoder::SharedApertureHigh))};
+	}
 	if (operand.kind == Decoder::OperandKind::ExecLo) {
 		return {ir.GetExecLo(), ir.GetExecHi()};
 	}
@@ -886,13 +894,35 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CVT_F64_F32:
 		case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
+		case Decoder::Opcode::V_ADD_F64:
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
+		case Decoder::Opcode::V_CMP_EQ_F64:
+		case Decoder::Opcode::V_CMP_LE_F64:
+		case Decoder::Opcode::V_CMPX_LE_F64:
+		case Decoder::Opcode::V_CMPX_GE_F64:
+		case Decoder::Opcode::V_CMP_EQ_I64:
+		case Decoder::Opcode::V_CMP_LT_I64:
+		case Decoder::Opcode::V_CMP_LE_I64:
+		case Decoder::Opcode::V_CMP_LT_U64:
+		case Decoder::Opcode::V_CMP_EQ_U64:
+		case Decoder::Opcode::V_CMP_LE_U64:
+		case Decoder::Opcode::V_CMP_GT_U64:
+		case Decoder::Opcode::V_CMP_NE_U64:
+		case Decoder::Opcode::V_CMP_GE_U64:
+		case Decoder::Opcode::V_CMPX_NE_I64:
+		case Decoder::Opcode::V_CMPX_LE_U64:
+		case Decoder::Opcode::V_CMPX_NE_U64:
+			include_vector(inst.src0, 2u);
+			include_vector(inst.src1, 2u);
+			break;
 		default: break;
 	}
 	if (inst.family == Decoder::Family::DS) {
 		switch (inst.opcode) {
+			case Decoder::Opcode::DS_ADD_U64:
+			case Decoder::Opcode::DS_OR_B64:
 			case Decoder::Opcode::DS_WRITE_B64:
 			case Decoder::Opcode::DS_WRITE_B96:
 			case Decoder::Opcode::DS_WRITE_B128: include_vector(inst.src1, inst.data_dwords); break;
@@ -1126,12 +1156,13 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				return entry_ir.ISub(lhs, minimum(lhs, rhs));
 			};
 			const auto local = builtin(IR::StageInputKind::LocalInvocationIndex);
-			const auto primitive_chunk = entry_ir.IMul(builtin(IR::StageInputKind::WorkgroupId, 0),
-			                                           u32(mesh.primitives_per_group));
+			const auto group = builtin(IR::StageInputKind::WorkgroupId, 0);
+			const auto primitive_chunk = mesh.fast_launch ? group :
+			    entry_ir.IMul(group, u32(mesh.primitives_per_group));
 			const auto step  = u32(mesh.InputPrimitiveStep());
 			const auto size  = u32(mesh.InputPrimitiveSize());
-			const auto chunk = entry_ir.IMul(primitive_chunk, step);
-			const auto vertices =
+			const auto chunk = mesh.fast_launch ? group : entry_ir.IMul(primitive_chunk, step);
+			const auto vertices = mesh.fast_launch ? u32(mesh.vertices_per_group) :
 			    minimum(subtract_saturate(draw(0), chunk), u32(mesh.vertices_per_group));
 			const auto primitives = entry_ir.Select(
 			    entry_ir.ULessThan(vertices, size), u32(0),
@@ -1152,56 +1183,64 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			    entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(
 			                                                         primitive_count, u32(8)),
 			                                                     vertex_count)));
-			// GS adjacency addresses local ES records in LDS. Fans retain the draw's
-			// center in every subgroup; strip winding follows the global primitive.
-			const auto vertex = entry_ir.IMul(local, step);
-			auto       first  = vertex;
-			auto       second = u32(0);
-			auto       third  = u32(0);
-			if (mesh.InputPrimitiveSize() >= 2u) {
-				second = entry_ir.IAdd(vertex, u32(1));
+			if (mesh.fast_launch) {
+				// Fast launch broadcasts the subgroup's base vertex and instance to every lane.
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5), entry_ir.IAdd(draw(1), group));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(6),
+				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
+			} else {
+				// GS adjacency addresses local ES records in LDS. Fans retain the draw's
+				// center in every subgroup; strip winding follows the global primitive.
+				const auto vertex = entry_ir.IMul(local, step);
+				auto       first  = vertex;
+				auto       second = u32(0);
+				auto       third  = u32(0);
+				if (mesh.InputPrimitiveSize() >= 2u) {
+					second = entry_ir.IAdd(vertex, u32(1));
+				}
+				if (mesh.InputPrimitiveSize() == 3u) {
+					third = entry_ir.IAdd(vertex, u32(2));
+				}
+				auto input_vertex = entry_ir.IAdd(chunk, local);
+				if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriFan)) {
+					first = u32(0);
+					input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex);
+				} else if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriStrip)) {
+					const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1));
+					first = entry_ir.IAdd(first, parity);
+					second = entry_ir.ISub(second, parity);
+				}
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
+				                      entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
+				                                         entry_ir.ShiftLeftLogical(second, u32(18))));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
+				                      entry_ir.ShiftLeftLogical(third, u32(2)));
+				const auto index_bytes  = draw(3);
+				const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
+				const auto index_low    = draw(4);
+				const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
+				                                           entry_ir.IMul(input_vertex, index_bytes));
+				const auto index_resource = entry_ir.Emit(
+				    IR::ValueOpcode::GetAddressResource,
+				    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
+				const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
+				result.memory_info.push_back({.kind = IR::ResourceKind::Global});
+				const auto packed_index = entry_ir.Emit(
+				    IR::ValueOpcode::LoadAddressU32,
+				    {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0),
+				     entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))},
+				    IR::MemoryFlags {.index = memory_index});
+				const auto index = IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::BitFieldUExtract,
+				    {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)),
+				     entry_ir.IMul(index_bytes, u32(8))}));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
+				                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(8),
+				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
 			}
-			if (mesh.InputPrimitiveSize() == 3u) {
-				third = entry_ir.IAdd(vertex, u32(2));
-			}
-			auto input_vertex = entry_ir.IAdd(chunk, local);
-			if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriFan)) {
-				first = u32(0);
-				input_vertex = entry_ir.Select(entry_ir.IEqual(local, u32(0)), u32(0), input_vertex);
-			} else if (mesh.input_primitive == static_cast<uint32_t>(Prospero::PrimitiveType::kTriStrip)) {
-				const auto parity = entry_ir.BitwiseAnd(entry_ir.IAdd(primitive_chunk, local), u32(1));
-				first = entry_ir.IAdd(first, parity);
-				second = entry_ir.ISub(second, parity);
-			}
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(0),
-			                      entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(first, u32(2)),
-			                                         entry_ir.ShiftLeftLogical(second, u32(18))));
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
-			                      entry_ir.ShiftLeftLogical(third, u32(2)));
-			const auto index_bytes  = draw(3);
-			const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
-			const auto index_low    = draw(4);
-			const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
-			                                           entry_ir.IMul(input_vertex, index_bytes));
-			const auto index_resource = entry_ir.Emit(
-			    IR::ValueOpcode::GetAddressResource,
-			    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
-			const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
-			result.memory_info.push_back({.kind = IR::ResourceKind::Global});
-			const auto packed_index = entry_ir.Emit(
-			    IR::ValueOpcode::LoadAddressU32,
-			    {index_resource, entry_ir.BitwiseAnd(byte_offset, u32(~3u)), u32(0),
-			     entry_ir.LogicalAnd(indexed, entry_ir.ULessThan(local, vertices))},
-			    IR::MemoryFlags {.index = memory_index});
-			const auto index = IR::U32(entry_ir.Emit(
-			    IR::ValueOpcode::BitFieldUExtract,
-			    {packed_index, entry_ir.IMul(entry_ir.BitwiseAnd(byte_offset, u32(3)), u32(8)),
-			     entry_ir.IMul(index_bytes, u32(8))}));
-			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
-			                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
-			entry_ir.SetVectorReg(
-			    static_cast<IR::VectorReg>(8),
-			    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
 		} else if (options.stage == ShaderType::Local) {
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(2),

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -114,18 +114,40 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
 			return;
 
+		case O::V_CMP_LT_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMP_LE_I64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMP_LE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false);
+			return;
+		case O::V_CMPX_LE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, true);
+			return;
+		case O::V_CMP_GE_U64:
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false);
+			return;
+
 		case O::V_CMP_EQ_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
 			return;
 		case O::V_CMP_EQ_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
 			return;
+		case O::V_CMPX_EQ_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, true);
+			return;
 		case O::V_CMP_NE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
 			return;
 		case O::V_CMP_NE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
 			return;
+		case O::V_CMPX_NE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, true);
+			return;
 		case O::V_CMP_GT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
 			return;
@@ -150,9 +172,15 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_GT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
 			return;
+		case O::V_CMPX_GT_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, true);
+			return;
 		case O::V_CMP_GE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
 			return;
+		case O::V_CMPX_GE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, true);
+			return;
 		case O::V_CMP_LT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
 			return;
@@ -162,12 +190,27 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_LE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
 			return;
+		case O::V_CMPX_LE_I16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, true);
+			return;
 
 		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return;
 		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return;
 		case O::V_CMP_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
 			return;
+		case O::V_CMP_EQ_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false);
+			return;
+		case O::V_CMP_LE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, false);
+			return;
+		case O::V_CMPX_LE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, true);
+			return;
+		case O::V_CMPX_GE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, true);
+			return;
 		case O::V_CMPX_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
 			return;
@@ -270,6 +313,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMPX_LE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
 			return;
+		case O::V_CMP_NGE_F16:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true, false);
+			return;
 		case O::V_CMP_NGT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false);
 			return;
@@ -421,6 +467,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false);
 		case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true);
 		case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
+		case O::V_ADD_F64: return FloatBinary(inst, IR::ValueOpcode::FPAdd64, false);
 		case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false);
 		case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
 		case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);

src/graphics/shader/recompiler/ir/Program.cpp

@@ -142,6 +142,7 @@ CompiledShaderInfo Program::TakeCompiledInfo() && {
 	    .user_data_base  = user_data_base,
 	    .user_data_count = user_data_count,
 	    .scratch_dwords  = scratch_dwords,
+	    .has_address_writes = has_address_writes,
 	    .info            = std::move(info),
 	    .bindings        = std::move(bindings),
 	};

src/graphics/shader/recompiler/ir/ResourceSnapshot.h

@@ -3,6 +3,7 @@
 
 #include <array>
 #include <cstdint>
+#include <utility>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -34,6 +35,7 @@ struct ResourceSnapshot {
 	std::vector<DescriptorValue> samplers;
 	std::vector<uint32_t>        flattened_srt;
 	std::vector<uint32_t>        user_data;
+	std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 	UniformFill                 uniform_fill;
 };
 

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -30,6 +30,7 @@ enum class ResourceKind {
 	Buffer,
 	IndirectBuffer,
 	Flat,
+	FlatLocal,
 	Global,
 	Scratch,
 	Lds,
@@ -40,7 +41,8 @@ enum class ResourceKind {
 
 [[nodiscard]] constexpr bool IsAddressResourceKind(ResourceKind kind) {
 	return kind == ResourceKind::ScalarAddress || kind == ResourceKind::Flat ||
-	       kind == ResourceKind::Global || kind == ResourceKind::Scratch;
+	       kind == ResourceKind::FlatLocal || kind == ResourceKind::Global ||
+	       kind == ResourceKind::Scratch;
 }
 
 struct MemoryInfo {
@@ -497,6 +499,7 @@ struct ResourceBlock {
 	Value                 condition;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> sources;
+	std::vector<uint32_t> srt_reads;
 };
 
 // Stable shader metadata consumed by the renderer after native IR has been discarded.
@@ -508,6 +511,7 @@ struct CompiledShaderInfo {
 	uint32_t                      user_data_count     = 64;
 	uint32_t                      scratch_dwords      = 0;
 	uint32_t                      param_export_mask   = 0;
+	bool                          has_address_writes  = false;
 	ShaderInfo                    info;
 	BindingLayout                 bindings;
 };
@@ -562,7 +566,6 @@ struct ResourcePlan {
 	mutable std::vector<uint8_t>            visited_blocks;
 	mutable std::vector<uint32_t>           pending_blocks;
 	mutable std::vector<uint32_t>           material_keys;
-	mutable std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 };
 
 struct Program: ResourcePlan {

src/graphics/shader/recompiler/ir/Value.cpp

@@ -2,7 +2,7 @@
 
 #include <algorithm>
 #include <cstring>
-#include <limits>
+#include <memory>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
@@ -135,10 +135,16 @@ bool Value::operator==(const Value& other) const {
 }
 
 Inst::Inst(ValueOpcode value_opcode, uint64_t value_flags)
-    : opcode(value_opcode), flags(value_flags) {
-	const auto count = NumArgsOf(opcode);
-	if (count != std::numeric_limits<size_t>::max()) {
-		args.resize(count);
+    : opcode(value_opcode),
+      num_args(value_opcode == ValueOpcode::Phi ? PhiArity
+                                               : static_cast<uint8_t>(NumArgsOf(value_opcode))),
+      flags(value_flags) {
+	if (num_args == PhiArity) {
+		std::destroy_at(&fixed_args);
+		std::construct_at(&phi_args);
+	} else if (num_args > InlineArity) {
+		std::destroy_at(&fixed_args);
+		std::construct_at(&large_args, num_args);
 	}
 }
 
@@ -154,8 +160,8 @@ Type Inst::GetType() const {
 	if (opcode == ValueOpcode::Phi) {
 		return static_cast<Type>(flags);
 	}
-	if (opcode == ValueOpcode::Identity && !args.empty()) {
-		return args.front().GetType();
+	if (opcode == ValueOpcode::Identity && num_args != 0) {
+		return Arg(0).GetType();
 	}
 	return TypeOf(opcode);
 }
@@ -173,21 +179,24 @@ size_t Inst::UseCount() const {
 }
 
 size_t Inst::NumArgs() const {
-	return args.size();
+	return num_args == PhiArity ? phi_args.size() : num_args;
 }
 
 size_t Inst::NumPhiBlocks() const {
-	return phi_blocks.size();
+	return num_args == PhiArity ? phi_args.size() : 0;
 }
 
 Value Inst::Arg(size_t index) const {
-	EXIT_IF(index >= args.size());
-	return args[index];
+	EXIT_IF(index >= NumArgs());
+	if (num_args <= InlineArity) {
+		return fixed_args[index];
+	}
+	return num_args == PhiArity ? phi_args[index].second : large_args[index];
 }
 
 Block* Inst::PhiBlock(size_t index) const {
-	EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_blocks.size());
-	return phi_blocks[index];
+	EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_args.size());
+	return phi_args[index].first;
 }
 
 Block* Inst::Parent() const {
@@ -203,15 +212,17 @@ void Inst::SetParent(Block* block) {
 }
 
 void Inst::SetArg(size_t index, Value value) {
-	if (index >= args.size()) {
-		EXIT_IF(NumArgsOf(opcode) != std::numeric_limits<size_t>::max());
-		args.resize(index + 1);
-	}
-	const auto old = args[index];
+	const auto old = Arg(index);
 	if (auto* old_inst = old.TryInstruction(); old_inst != nullptr) {
 		RemoveUse(old_inst, index);
 	}
-	args[index] = value;
+	if (num_args <= InlineArity) {
+		fixed_args[index] = value;
+	} else if (num_args == PhiArity) {
+		phi_args[index].second = value;
+	} else {
+		large_args[index] = value;
+	}
 	if (auto* new_inst = value.TryInstruction(); new_inst != nullptr) {
 		AddUse(new_inst, index);
 	}
@@ -219,9 +230,8 @@ void Inst::SetArg(size_t index, Value value) {
 
 void Inst::AddPhiOperand(Block* predecessor, Value value) {
 	EXIT_IF(opcode != ValueOpcode::Phi);
-	const auto index = args.size();
-	args.push_back(value);
-	phi_blocks.push_back(predecessor);
+	const auto index = phi_args.size();
+	phi_args.emplace_back(predecessor, value);
 	if (auto* value_inst = value.TryInstruction(); value_inst != nullptr) {
 		AddUse(value_inst, index);
 	}
@@ -234,21 +244,12 @@ void Inst::ReplaceUsesWith(Value replacement, bool preserve) {
 	}
 	Invalidate();
 	if (preserve) {
-		ReplaceOpcode(ValueOpcode::Identity);
-		args.resize(1);
+		opcode = ValueOpcode::Identity;
+		num_args = 1;
 		SetArg(0, replacement);
 	}
 }
 
-void Inst::ReplaceOpcode(ValueOpcode value_opcode) {
-	opcode           = value_opcode;
-	const auto count = NumArgsOf(opcode);
-	if (count != std::numeric_limits<size_t>::max()) {
-		EXIT_IF(!args.empty() && args.size() != count);
-		args.resize(count);
-	}
-}
-
 void Inst::Invalidate() {
 	ClearArgs();
 	opcode = ValueOpcode::Void;
@@ -269,13 +270,21 @@ void Inst::RemoveUse(Inst* used, size_t operand) {
 }
 
 void Inst::ClearArgs() {
-	for (size_t index = 0; index < args.size(); index++) {
-		if (auto* value_inst = args[index].TryInstruction(); value_inst != nullptr) {
+	for (size_t index = 0; index < NumArgs(); index++) {
+		if (auto* value_inst = Arg(index).TryInstruction(); value_inst != nullptr) {
 			RemoveUse(value_inst, index);
 		}
 	}
-	args.clear();
-	phi_blocks.clear();
+	if (num_args == PhiArity) {
+		std::destroy_at(&phi_args);
+		std::construct_at(&fixed_args);
+	} else if (num_args > InlineArity) {
+		std::destroy_at(&large_args);
+		std::construct_at(&fixed_args);
+	} else {
+		fixed_args.fill(Value {});
+	}
+	num_args = 0;
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/Value.h

@@ -4,6 +4,7 @@
 #include "graphics/shader/recompiler/ir/Reg.h"
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h"
 
+#include <array>
 #include <bit>
 #include <cstdint>
 #include <cstring>
@@ -133,7 +134,6 @@ public:
 	void SetArg(size_t index, Value value);
 	void AddPhiOperand(Block* predecessor, Value value);
 	void ReplaceUsesWith(Value replacement, bool preserve = true);
-	void ReplaceOpcode(ValueOpcode opcode);
 	void Invalidate();
 
 	template <typename T>
@@ -156,13 +156,22 @@ private:
 	void RemoveUse(Inst* used, size_t operand);
 	void ClearArgs();
 
+	static constexpr uint8_t InlineArity = 4;
+	static constexpr uint8_t PhiArity = UINT8_MAX;
+
 	ValueOpcode         opcode;
+	uint8_t             num_args;
+	mutable uint32_t    evaluation_index = UINT32_MAX;
 	uint64_t            flags;
 	Block*              parent = nullptr;
-	std::vector<Value>  args;
-	std::vector<Block*> phi_blocks;
+	union {
+		std::array<Value, InlineArity> fixed_args {};
+		std::vector<Value> large_args;
+		std::vector<std::pair<Block*, Value>> phi_args;
+	};
 	std::vector<Use>    uses;
-	mutable uint32_t    evaluation_index = UINT32_MAX;
 };
 
+static_assert(sizeof(Inst) <= 112, "Inst operand storage unintentionally increased");
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -123,6 +123,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicSMax32:
 		case ValueOpcode::BufferAtomicUMax32:
 		case ValueOpcode::BufferAtomicAnd32:
+		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr32:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::BufferAtomicXor32:
@@ -135,6 +136,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
+		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
 		case ValueOpcode::StoreBufferU32x2: return 2u;
@@ -164,6 +166,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicFMax32:
 		case ValueOpcode::SharedAtomicSwap32:
 		case ValueOpcode::SharedAtomicIAdd32:
+		case ValueOpcode::SharedAtomicIAdd64:
 		case ValueOpcode::SharedAtomicISub32:
 		case ValueOpcode::SharedAtomicInc32:
 		case ValueOpcode::SharedAtomicDec32:
@@ -183,6 +186,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 
 uint32_t SharedComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
+		case ValueOpcode::SharedAtomicIAdd64:
 		case ValueOpcode::SharedAtomicOr64:
 		case ValueOpcode::LoadSharedU32x2:
 		case ValueOpcode::WriteSharedU32x2: return 2u;
@@ -216,9 +220,12 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
 			return {ImageAccess::Read, ImageResourceClass::Sampled, true};
 		case ValueOpcode::ImageWrite:
 			return {ImageAccess::Write, ImageResourceClass::Storage, false};
+		case ValueOpcode::ImageAtomicCompareSwap32:
 		case ValueOpcode::ImageAtomicSwap32:
 		case ValueOpcode::ImageAtomicIAdd32:
+		case ValueOpcode::ImageAtomicSMin32:
 		case ValueOpcode::ImageAtomicUMin32:
+		case ValueOpcode::ImageAtomicSMax32:
 		case ValueOpcode::ImageAtomicUMax32:
 		case ValueOpcode::ImageAtomicAnd32:
 		case ValueOpcode::ImageAtomicOr32:

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -9,7 +9,7 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-enum class ValueOpcode {
+enum class ValueOpcode : uint16_t {
 #define VALUE_OPCODE(name, ...) name,
 #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc"
 #undef VALUE_OPCODE

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -137,6 +137,9 @@ VALUE_OPCODE(SMedTri32, U32, U32, U32, U32)
 VALUE_OPCODE(UMedTri32, U32, U32, U32, U32)
 VALUE_OPCODE(SLessThan32, U1, U32, U32)
 VALUE_OPCODE(SLessThan64, U1, U64, U64)
+VALUE_OPCODE(SLessThanEqual64, U1, U64, U64)
+VALUE_OPCODE(ULessThanEqual64, U1, U64, U64)
+VALUE_OPCODE(UGreaterThanEqual64, U1, U64, U64)
 VALUE_OPCODE(ULessThan32, U1, U32, U32)
 VALUE_OPCODE(ULessThan64, U1, U64, U64)
 VALUE_OPCODE(IEqual32, U1, U32, U32)
@@ -155,6 +158,9 @@ VALUE_OPCODE(LogicalAnd, U1, U1, U1)
 VALUE_OPCODE(LogicalXor, U1, U1, U1)
 VALUE_OPCODE(LogicalNot, U1, U1)
 VALUE_OPCODE(FPOrdEqual32, U1, F32, F32)
+VALUE_OPCODE(FPOrdEqual64, U1, F64, F64)
+VALUE_OPCODE(FPOrdLessThanEqual64, U1, F64, F64)
+VALUE_OPCODE(FPOrdGreaterThanEqual64, U1, F64, F64)
 VALUE_OPCODE(FPUnordEqual32, U1, F32, F32)
 VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32)
 VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32)
@@ -173,6 +179,7 @@ VALUE_OPCODE(FPAdd32, F32, F32, F32)
 VALUE_OPCODE(FPSub32, F32, F32, F32)
 VALUE_OPCODE(FPFma32, F32, F32, F32, F32)
 VALUE_OPCODE(FPMul32, F32, F32, F32)
+VALUE_OPCODE(FPAdd64, F64, F64, F64)
 VALUE_OPCODE(FPMul64, F64, F64, F64)
 VALUE_OPCODE(FPFma64, F64, F64, F64, F64)
 VALUE_OPCODE(FPRecip64, F64, F64)
@@ -245,6 +252,7 @@ VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicAnd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicOr64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -267,6 +275,7 @@ VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1)
 VALUE_OPCODE(SharedAtomicISub32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicInc32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicDec32, U32, U32, U32, U1)
@@ -283,14 +292,18 @@ VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
+VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
 VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
 VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
 VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
+VALUE_OPCODE(ImageAtomicCompareSwap32, U32, ImageResource, ImageAddress, U32, U32, U1)
 VALUE_OPCODE(ImageAtomicSwap32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1)
+VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
 VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <bit>
 #include <cstdint>
+#include <unordered_map>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -575,6 +576,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 		case ValueOpcode::UGreaterThan64:
 			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; });
 			return;
+		case ValueOpcode::ULessThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; });
+			return;
+		case ValueOpcode::UGreaterThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; });
+			return;
+		case ValueOpcode::SLessThanEqual64:
+			FoldU64Compare(inst, [](uint64_t a, uint64_t b) {
+				return std::bit_cast<int64_t>(a) <= std::bit_cast<int64_t>(b);
+			});
+			return;
 		case ValueOpcode::SLessThan64:
 			FoldU64Compare(inst, [](uint64_t a, uint64_t b) {
 				return std::bit_cast<int64_t>(a) < std::bit_cast<int64_t>(b);
@@ -649,12 +661,110 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 	}
 }
 
+// In wave32, extracting this invocation's bit from a ballot recovers its predicate.
+// Keep that identity through scalar EXEC operations and their loop-carried mask Phis.
+class LaneMaskProjection {
+public:
+	explicit LaneMaskProjection(std::unordered_set<Inst*>& lowered_ancillary)
+	    : m_lowered_ancillary(lowered_ancillary) {}
+
+	void Fold(Inst& inst) {
+		if (inst.GetOpcode() != ValueOpcode::INotEqual32 || !Immediate(Arg(inst, 1), 0u)) return;
+		const auto* bit = Arg(inst, 0).TryInstruction();
+		if (bit == nullptr || bit->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
+		    !Immediate(Arg(*bit, 1), 1u)) return;
+		const auto* shift = Arg(*bit, 0).TryInstruction();
+		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftRightLogical32) return;
+		const auto* index = Arg(*shift, 1).TryInstruction();
+		if (index == nullptr || index->GetOpcode() != ValueOpcode::BitwiseAnd32 ||
+		    !Immediate(Arg(*index, 1), 31u)) return;
+		const auto* lane = Arg(*index, 0).TryInstruction();
+		if (lane == nullptr || lane->GetOpcode() != ValueOpcode::LaneId) return;
+		m_visited.clear();
+		m_grounded = false;
+		if (CanProject(Arg(*shift, 0)) && m_grounded) Replace(inst, Project(Arg(*shift, 0)));
+	}
+
+private:
+	static bool Immediate(Value value, uint32_t expected) {
+		return IsImmediate(value, Type::U32) && value.U32() == expected;
+	}
+
+	static Value BallotPredicate(const Inst& inst) {
+		if (inst.GetOpcode() != ValueOpcode::CompositeExtractU32x4 ||
+		    !Immediate(Arg(inst, 1), 0u)) return {};
+		const auto* source = Arg(inst, 0).TryInstruction();
+		return source != nullptr && source->GetOpcode() == ValueOpcode::Ballot
+		           ? Arg(*source, 0) : Value {};
+	}
+
+	bool CanProject(Value value) {
+		if (value.IsImmediate()) {
+			const bool supported = Immediate(value, 0u) || Immediate(value, UINT32_MAX);
+			m_grounded |= supported;
+			return supported;
+		}
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return false;
+		if (m_values.contains(inst) || !BallotPredicate(*inst).IsEmpty()) {
+			m_grounded = true;
+			return true;
+		}
+		if (!m_visited.insert(inst).second) return true;
+		const auto op = inst->GetOpcode();
+		if (op != ValueOpcode::BitwiseAnd32 && op != ValueOpcode::BitwiseOr32 &&
+		    op != ValueOpcode::BitwiseNot32 && op != ValueOpcode::SelectU32 && op != ValueOpcode::Phi)
+			return false;
+		for (size_t arg = op == ValueOpcode::SelectU32 ? 1u : 0u; arg < inst->NumArgs(); ++arg) {
+			if (!CanProject(Arg(*inst, arg))) return false;
+		}
+		return inst->NumArgs() != 0;
+	}
+
+	Value Project(Value value) {
+		if (value.IsImmediate()) return Value(value.U32() != 0u);
+		auto* source = value.TryInstruction();
+		if (const auto found = m_values.find(source); found != m_values.end()) return found->second.Resolve();
+		if (const auto predicate = BallotPredicate(*source); !predicate.IsEmpty()) return predicate;
+		auto* block = source->Parent();
+		const auto where = std::find_if(block->begin(), block->end(),
+		                               [source](const Inst& candidate) { return &candidate == source; });
+		const auto op = source->GetOpcode() == ValueOpcode::BitwiseAnd32 ? ValueOpcode::LogicalAnd
+		              : source->GetOpcode() == ValueOpcode::BitwiseOr32 ? ValueOpcode::LogicalOr
+		              : source->GetOpcode() == ValueOpcode::BitwiseNot32 ? ValueOpcode::LogicalNot
+		              : source->GetOpcode() == ValueOpcode::SelectU32 ? ValueOpcode::SelectU1
+		                                                               : ValueOpcode::Phi;
+		auto result = op == ValueOpcode::Phi ? block->PrependNewInst(where, op)
+		            : op == ValueOpcode::LogicalNot
+		                ? block->PrependNewInst(where, op, {Value(false)})
+		            : op == ValueOpcode::SelectU1
+		                ? block->PrependNewInst(where, op, {Arg(*source, 0), Value(false), Value(false)})
+		                : block->PrependNewInst(where, op, {Value(false), Value(false)});
+		m_values.emplace(source, Value(&*result));
+		if (op == ValueOpcode::Phi) result->SetFlags(Type::U1);
+		for (size_t arg = op == ValueOpcode::SelectU1 ? 1u : 0u; arg < source->NumArgs(); ++arg) {
+			const auto projected = Project(Arg(*source, arg));
+			if (op == ValueOpcode::Phi) result->AddPhiOperand(source->PhiBlock(arg), projected);
+			else result->SetArg(arg, projected);
+		}
+		FoldInstruction(*block, result, m_lowered_ancillary);
+		return Value(&*result).Resolve();
+	}
+
+	std::unordered_set<Inst*>& m_lowered_ancillary;
+	std::unordered_set<const Inst*> m_visited;
+	std::unordered_map<const Inst*, Value> m_values;
+	bool m_grounded = false;
+};
+
 } // namespace
 
-void ConstantPropagationPass(const BlockList& blocks) {
+void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size) {
 	std::unordered_set<Inst*> lowered_ancillary;
+	LaneMaskProjection mask_projection(lowered_ancillary);
 	for (auto* block: blocks) {
 		for (auto inst = block->begin(); inst != block->end(); ++inst) {
+			if (wave_size == 32u) mask_projection.Fold(*inst);
 			FoldInstruction(*block, inst, lowered_ancillary);
 		}
 	}

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h

@@ -4,6 +4,6 @@
 
 namespace Libs::Graphics::ShaderRecompiler::IR {
 
-void ConstantPropagationPass(const BlockList& blocks);
+void ConstantPropagationPass(const BlockList& blocks, uint32_t wave_size = 64);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -157,34 +157,15 @@ bool CaptureStrictRead(void* userdata, uint64_t address, std::span<uint32_t> val
 
 bool CaptureOrdinaryRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
 	auto& capture = *static_cast<ReadCapture*>(userdata);
-	if (!capture.source.read_memory(capture.source.userdata, address, values)) {
-		return false;
+	if (capture.source.read_memory != nullptr) {
+		if (!capture.source.read_memory(capture.source.userdata, address, values)) return false;
+	} else {
+		std::memcpy(values.data(), reinterpret_cast<const void*>(address), values.size_bytes());
 	}
 	capture.ranges.emplace_back(address, values.size_bytes());
 	return true;
 }
 
-bool WrittenBuffersDisjoint(const ResourcePlan& program, const ResourceSnapshot& snapshot,
-                            std::span<const std::pair<uint64_t, uint64_t>> reads) {
-	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!program.info.buffers[i].written) continue;
-		ShaderBufferResource buffer;
-		if (!DecodeBufferDescriptor(snapshot.buffers[i], buffer)) return false;
-		const auto base = buffer.Base48();
-		const auto size = buffer.GetSize();
-		if (size == 0u) continue;
-		if (size - 1u > AddressMask - base) return false;
-		for (const auto [address, bytes]: reads) {
-			if (bytes == 0u) continue;
-			if (address > AddressMask || bytes - 1u > AddressMask - address ||
-			    (address <= base ? base - address < bytes : address - base < size)) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
 const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
 	if (source >= program.descriptor_sources.size()) {
 		return nullptr;
@@ -638,9 +619,10 @@ bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan&
 	return true;
 }
 
-static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
-	// Any shader write may alias a scalar predicate read, including on a later loop visit.
-	if (program.blocks.size() != program.block_info.size() || HasShaderMemoryWrites(program)) {
+template <typename Predicate>
+static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, const Predicate& predicate) {
+	// The renderer checks captured scalar reads against final buffer and image write ranges.
+	if (program.blocks.size() != program.block_info.size() || program.has_address_writes) {
 		return {};
 	}
 	std::unordered_map<uint32_t, uint32_t> indices;
@@ -659,9 +641,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 			case CFG::TerminatorKind::Branch: successors.push_back(terminator.true_block); break;
 			case CFG::TerminatorKind::ConditionalBranch:
 				successors = {terminator.true_block, terminator.false_block};
-				if (ValidateRuntimeValue(program, info.condition, RuntimeValueType::Integer)) {
-					block.condition = info.condition;
-				}
+				block.condition = info.condition;
 				break;
 			case CFG::TerminatorKind::IndirectBranch:
 				successors = terminator.indirect_targets;
@@ -678,13 +658,17 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 		}
 		for (const auto& inst: *program.blocks[i]) {
 			const auto op     = inst.GetOpcode();
+			if (op == ValueOpcode::ReadConst) {
+				block.srt_reads.push_back(inst.Arg(1).Resolve().U32());
+				continue;
+			}
 			const auto buffer = BufferAccessOf(op);
 			const auto image  = ImageOpcodeInfoOf(op);
 			if (buffer == BufferAccess::None && image.access == ImageAccess::None) {
 				continue;
 			}
 			const auto& memory = program.memory_info.at(inst.Flags<MemoryFlags>().index);
-			if (memory.planning_only) {
+			if (memory.planning_only || memory.kind == ResourceKind::IndirectBuffer) {
 				continue;
 			}
 			if (buffer != BufferAccess::None) {
@@ -700,6 +684,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program) {
 		block.sources.erase(std::unique(block.sources.begin(), block.sources.end()),
 		                    block.sources.end());
 	}
+	for (auto& block: blocks) block.condition = predicate(block.condition);
 	if (std::ranges::none_of(
 	        blocks, [](const ResourceBlock& block) { return !block.condition.IsEmpty(); })) {
 		return {};
@@ -914,10 +899,38 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	for (const auto& read: program.srt_reads) {
 		plan.srt_reads.push_back({Clone(read.value), read.flat_offset});
 	}
-	plan.control_flow = ResourceControlFlow(program);
-	for (auto& block: plan.control_flow) {
-		block.condition = Clone(block.condition);
-	}
+	// A proven uniform factor can decide a branch even when its other lanes are unknown.
+	// Keep only that Boolean structure, never the varying shader dependency graph.
+	Value unknown;
+	std::function<Value(Value)> ClonePredicate = [&](Value value) -> Value {
+		value = value.Resolve();
+		if (value.IsEmpty()) return {};
+		if (ValidateRuntimeValue(program, value, RuntimeValueType::Integer)) return Clone(value);
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return {};
+		const auto op = inst->GetOpcode();
+		if (op == ValueOpcode::ConditionRef || op == ValueOpcode::LogicalNot) {
+			const auto operand = ClonePredicate(inst->Arg(0));
+			if (operand.IsEmpty() || op == ValueOpcode::ConditionRef) return operand;
+			auto& node = plan.value_storage.emplace_back(op);
+			node.SetArg(0, operand);
+			return Value(&node);
+		}
+		if (op != ValueOpcode::LogicalAnd && op != ValueOpcode::LogicalOr) return {};
+		auto left = ClonePredicate(inst->Arg(0));
+		auto right = ClonePredicate(inst->Arg(1));
+		if (left.IsEmpty() && right.IsEmpty()) return {};
+		if (left.IsEmpty() || right.IsEmpty()) {
+			if (unknown.IsEmpty()) unknown = Value(&plan.value_storage.emplace_back(ValueOpcode::UndefU1));
+			if (left.IsEmpty()) left = unknown;
+			if (right.IsEmpty()) right = unknown;
+		}
+		auto& node = plan.value_storage.emplace_back(op);
+		node.SetArg(0, left);
+		node.SetArg(1, right);
+		return Value(&node);
+	};
+	plan.control_flow = ResourceControlFlow(program, ClonePredicate);
 	plan.uniform_fill = AnalyzeUniformFill(program);
 	for (uint32_t i = 0; i < plan.uniform_fill.fill.words; ++i) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
@@ -936,12 +949,15 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
 		                   plan.clean_flat_slots);
 	}
-	if (masked_image) {
-		plan.resource_tracking_complete &= !program.has_address_writes &&
-		    !std::ranges::any_of(plan.info.images, &ImageResource::written);
-		plan.capture_specialization_reads =
-		    std::ranges::any_of(plan.info.buffers, &BufferResource::written);
+	plan.capture_specialization_reads = masked_image || !plan.control_flow.empty();
+	// Writable descriptor addresses must come from clean backing for the alias proof.
+	for (const auto& buffer: plan.info.buffers) {
+		if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
 	}
+	for (const auto& image: plan.info.images) {
+		if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
+	}
+	if (masked_image) plan.resource_tracking_complete &= !program.has_address_writes;
 	return plan;
 }
 
@@ -952,21 +968,22 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		return false;
 	}
 	const bool capture_reads = program.capture_specialization_reads;
-	auto& reads = program.specialization_reads;
+	auto& reads = snapshot.specialization_reads;
+	reads.clear();
 	ReadCapture capture {runtime, reads};
 	SrtRuntime observed = runtime;
 	if (capture_reads) {
-		reads.clear();
 		observed.userdata = &capture;
-		observed.read_specialization_memory = CaptureStrictRead;
-		if (observed.read_memory != nullptr) observed.read_memory = CaptureOrdinaryRead;
+		observed.read_specialization_memory = runtime.read_specialization_memory != nullptr
+		                                         ? CaptureStrictRead : nullptr;
+		observed.read_memory = CaptureOrdinaryRead;
 	}
 	SrtWalker clean(program, CleanRuntime(observed));
 	SrtWalker walker(program, observed, program.clean_flat_slots, &clean);
-	const auto active = clean.FindActiveSources();
 	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
 		return false;
 	}
+	const auto active = std::span<const uint8_t>(program.active_sources);
 	snapshot.uniform_fill = {};
 	const auto& fill = program.uniform_fill;
 	const auto words = fill.fill.words;
@@ -979,12 +996,12 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		snapshot.uniform_fill = fill.fill;
 		snapshot.uniform_fill.value = stored[0];
 	}
-	const auto evaluate = [&](uint32_t source, DescriptorValue& value) {
+	const auto evaluate = [&](uint32_t source, DescriptorValue& value, bool written = false) {
 		if (source >= program.descriptor_sources.size()) {
 			return false;
 		}
 		if (active.empty() || active[source]) {
-			return walker.EvaluateDescriptor(source, value);
+			return (written ? clean : walker).EvaluateDescriptor(source, value);
 		}
 		value = {};
 		value.dword_count = program.descriptor_sources[source].dword_count;
@@ -992,19 +1009,11 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	};
 	snapshot.buffers.resize(program.info.buffers.size());
 	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i])) {
+		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i],
+		              program.info.buffers[i].written)) {
 			return false;
 		}
 	}
-	if (capture_reads) {
-		for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-			const auto& buffer = program.info.buffers[i];
-			if (!buffer.written || (!active.empty() && !active[buffer.source])) continue;
-			DescriptorValue strict;
-			if (!clean.EvaluateDescriptor(buffer.source, strict) ||
-			    strict != snapshot.buffers[i]) return false;
-		}
-	}
 	snapshot.images.resize(program.info.images.size());
 	specialization.images.resize(program.info.images.size());
 	for (uint32_t i = 0; i < program.info.images.size(); ++i) {
@@ -1040,7 +1049,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 				return false;
 			}
 		} else {
-			if (!evaluate(image.source, snapshot.images[i])) {
+			if (!evaluate(image.source, snapshot.images[i], image.written)) {
 				return false;
 			}
 			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
@@ -1054,7 +1063,6 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			return false;
 		}
 	}
-	if (capture_reads && !WrittenBuffersDisjoint(program, snapshot, reads)) return false;
 	snapshot.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
 	return BuildResourceSpecialization(program, snapshot, specialization);
 }

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -44,6 +44,34 @@ uint32_t PossibleU32Bits(Value value) {
 	}
 }
 
+bool IsNoncanonicalFlatAddress(Value high, Value active) {
+	high = high.Resolve();
+	active = active.Resolve();
+	// A predicated VGPR write supplies its new value on exactly the lanes which
+	// execute this access. Do not discard a selection under a different mask.
+	while (const auto* select = high.TryInstruction()) {
+		if (select->GetOpcode() != ValueOpcode::SelectU32 || select->Arg(0).Resolve() != active) break;
+		high = select->Arg(1).Resolve();
+	}
+	uint32_t lower = 0, upper = UINT32_MAX;
+	if (high.IsImmediate() && high.GetType() == Type::U32) {
+		lower = upper = high.U32();
+	} else if (const auto* inst = high.TryInstruction();
+	           inst != nullptr && inst->GetOpcode() == ValueOpcode::UMedTri32) {
+		// The median lies between any two constant inputs, regardless of the third.
+		uint32_t count = 0;
+		for (size_t index = 0; index < inst->NumArgs(); ++index) {
+			const auto value = inst->Arg(index).Resolve();
+			if (!value.IsImmediate() || value.GetType() != Type::U32) continue;
+			if (count++ == 0) lower = upper = value.U32();
+			else { lower = std::min(lower, value.U32()); upper = std::max(upper, value.U32()); }
+		}
+		if (count < 2) return false;
+	}
+	// Leave one high DWORD on each side for the signed instruction offset/carry.
+	return lower > 0x00008000u && upper < 0xffff7fffu;
+}
+
 Value CanonicalizeSampleAdjustDword3(Value value) {
 	for (;;) {
 		value            = value.Resolve();
@@ -1649,6 +1677,10 @@ private:
 
 	void Collect(Inst& inst) {
 		const auto op           = inst.GetOpcode();
+		if (op == ValueOpcode::BvhIntersect) {
+			m_info.uses_dma = true;
+			return;
+		}
 		const auto buffer       = BufferAccessOf(op);
 		const auto address_info = AddressOpcodeInfoOf(op);
 		const auto image_info   = ImageOpcodeInfoOf(op);
@@ -1695,6 +1727,15 @@ private:
 			if (!IsAddressResourceKind(memory.kind)) {
 				Fail(flags.pc, "address operation has invalid resource kind");
 			}
+			if (memory.kind == ResourceKind::Flat && memory.address_is_full &&
+			    IsNoncanonicalFlatAddress(inst.Arg(2), inst.Arg(inst.NumArgs() - 1))) {
+				ValidateAddressHandle(inst.Arg(0), flags.pc);
+				if (memory.data_bits != 32u || m_program.scratch_dwords == 0) {
+					Fail(flags.pc, "local FLAT access requires DWORD data and per-thread scratch storage");
+				}
+				m_program.memory_info[flags.index].kind = ResourceKind::FlatLocal;
+				return;
+			}
 			if (memory.kind == ResourceKind::Scratch) {
 				handle = inst.Arg(0).Resolve().TryInstruction();
 				if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource ||

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -795,18 +795,26 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
-		case ValueOpcode::LogicalAnd:
-			if (binary()) {
-				result = (a != 0u) && (b != 0u);
+		case ValueOpcode::LogicalAnd: {
+			const bool left = Arg(inst, 0, a);
+			if (left && a == 0u) {
+				result = 0u;
 				return true;
 			}
-			return false;
-		case ValueOpcode::LogicalOr:
-			if (binary()) {
-				result = (a != 0u) || (b != 0u);
+			if (!Arg(inst, 1, b) || (b != 0u && !left)) return false;
+			result = b != 0u;
+			return true;
+		}
+		case ValueOpcode::LogicalOr: {
+			const bool left = Arg(inst, 0, a);
+			if (left && a != 0u) {
+				result = 1u;
 				return true;
 			}
-			return false;
+			if (!Arg(inst, 1, b) || (b == 0u && !left)) return false;
+			result = b != 0u;
+			return true;
+		}
 		case ValueOpcode::LogicalXor:
 			if (binary()) {
 				result = (a != 0u) != (b != 0u);
@@ -844,16 +852,31 @@ bool SrtWalker::EvaluateDescriptor(uint32_t source, DescriptorValue& result) {
 	return true;
 }
 
-std::span<const uint8_t> SrtWalker::FindActiveSources() {
-	if (m_program.control_flow.empty()) {
-		return {};
-	}
+bool SrtWalker::RefreshFlatBuffer(std::vector<uint32_t>& flat) {
+	if (!m_program.srt_plan_complete) return false;
+	const auto refresh = [&](uint32_t slot) {
+		if (slot >= m_program.srt_reads.size()) return false;
+		const auto& read = m_program.srt_reads[slot];
+		const bool clean = read.flat_offset < m_clean_flat_slots.size() &&
+		                   m_clean_flat_slots[read.flat_offset] != 0u;
+		if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr))
+			return false;
+		auto& evaluator = clean ? *m_clean_evaluator : *this;
+		return read.flat_offset < flat.size() && evaluator.Evaluate(read.value, flat[read.flat_offset]);
+	};
 	auto& active = m_program.active_sources;
+	if (m_program.control_flow.empty()) {
+		active.clear();
+		flat.resize(m_program.srt_reads.size());
+		for (uint32_t slot = 0; slot < m_program.srt_reads.size(); ++slot) {
+			if (!refresh(slot)) return false;
+		}
+		return true;
+	}
+	flat.assign(m_program.srt_reads.size(), 0u);
 	active.assign(m_program.descriptor_sources.size(), 1u);
 	for (const auto& block: m_program.control_flow) {
-		for (const auto source: block.sources) {
-			active.at(source) = 0u;
-		}
+		for (const auto source: block.sources) active.at(source) = 0u;
 	}
 	auto& visited = m_program.visited_blocks;
 	auto& pending = m_program.pending_blocks;
@@ -863,41 +886,22 @@ std::span<const uint8_t> SrtWalker::FindActiveSources() {
 	while (!pending.empty()) {
 		const auto index = pending.back();
 		pending.pop_back();
-		if (visited.at(index)) {
-			continue;
-		}
+		if (visited.at(index)) continue;
 		visited[index] = 1u;
 		const auto& block = m_program.control_flow[index];
-		for (const auto source: block.sources) {
-			active[source] = 1u;
+		for (const auto source: block.sources) active[source] = 1u;
+		for (const auto slot: block.srt_reads) {
+			if (!refresh(slot)) return false;
 		}
 		uint32_t condition = 0;
+		auto& predicate = m_clean_evaluator != nullptr ? *m_clean_evaluator : *this;
 		if (!block.condition.IsEmpty() && m_runtime.read_specialization_memory != nullptr &&
-		    Evaluate(block.condition, condition)) {
+		    predicate.Evaluate(block.condition, condition)) {
 			pending.push_back(block.successors[condition != 0u ? 0u : 1u]);
 		} else {
 			pending.insert(pending.end(), block.successors.begin(), block.successors.end());
 		}
 	}
-	return active;
-}
-
-bool SrtWalker::RefreshFlatBuffer(std::vector<uint32_t>& flat) {
-	if (!m_program.srt_plan_complete) {
-		return false;
-	}
-	flat.resize(m_program.srt_reads.size());
-	for (const auto& read: m_program.srt_reads) {
-		const bool clean = read.flat_offset < m_clean_flat_slots.size() &&
-		                   m_clean_flat_slots[read.flat_offset] != 0u;
-		if (clean && (m_clean_evaluator == nullptr || m_runtime.read_specialization_memory == nullptr)) {
-			return false;
-		}
-		auto& evaluator = clean ? *m_clean_evaluator : *this;
-		if (read.flat_offset >= flat.size() || !evaluator.Evaluate(read.value, flat[read.flat_offset])) {
-			return false;
-		}
-	}
 	return true;
 }
 

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -38,8 +38,7 @@ public:
 
 	bool Evaluate(Value value, uint32_t& result);
 	bool EvaluateDescriptor(uint32_t source, DescriptorValue& result);
-	// An empty span means that all sources are active.
-	std::span<const uint8_t> FindActiveSources();
+	// Refreshes reachable scalar reads and active descriptor sources in one walk.
 	bool RefreshFlatBuffer(std::vector<uint32_t>& flat);
 
 private:

src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp

@@ -165,7 +165,6 @@ Value InitialValue(M0Tag) {
 
 enum class ReadStep { Start, SetValue, PushPhiArgument };
 
-template <typename T>
 struct ReadState {
 	Block*   block = nullptr;
 	Value    result;
@@ -183,8 +182,8 @@ public:
 
 	template <typename T>
 	Value Read(T variable, Block* root) {
-		std::vector<ReadState<T>> stack {{}, {.block = root}};
-		const auto                prepare_phi = [&]() {
+		stack.assign({{}, {.block = root}});
+		const auto prepare_phi = [&]() {
 			auto&      state        = stack.back();
 			const auto predecessors = state.block->ImmPredecessors();
 			if (state.pred == predecessors.size()) {
@@ -290,6 +289,7 @@ private:
 
 	std::unordered_map<Block*, std::map<Variable, Inst*>> incomplete_phis;
 	DefTable                                              definitions;
+	std::vector<ReadState>                               stack;
 };
 
 void VisitInstruction(Pass& pass, Block* block, Inst& inst) {

src/graphics/shader/shader.cpp

@@ -41,44 +41,33 @@
 
 namespace Libs::Graphics {
 
-struct ShaderBinaryInfo {
-	uint8_t  signature[7];
-	uint8_t  version;
-	uint32_t pssl_or_cg  : 1;
-	uint32_t cached      : 1;
-	uint32_t type        : 4;
-	uint32_t source_type : 2;
-	uint32_t length      : 24;
-	uint8_t  chunk_usage_base_offset_dw;
-	uint8_t  num_input_usage_slots;
-	uint8_t  is_srt                 : 1;
-	uint8_t  is_srt_used_info_valid : 1;
-	uint8_t  is_extended_usage_info : 1;
-	uint8_t  reserved2              : 5;
-	uint8_t  reserved3;
-	uint32_t hash0;
-	uint32_t hash1;
-	uint32_t crc32;
+struct ShaderMapEntry {
+	ShaderMappedData data;
+	uint64_t hash;
 };
 
-static std::unique_ptr<std::unordered_map<uint64_t, ShaderMappedData>> g_shader_map;
+static std::unique_ptr<std::unordered_map<uint64_t, ShaderMapEntry>> g_shader_map;
 static std::mutex                                                      g_shader_map_mutex;
 
 void ShaderInit() {
 	EXIT_IF(g_shader_map != nullptr);
 
-	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMappedData>>();
+	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMapEntry>>();
 }
 
 void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 	EXIT_IF(g_shader_map == nullptr);
 
+	if (addr == 0 || data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
+		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n",
+		     addr, data.code_size_bytes);
+	}
+	const auto hash = XXH3_64bits(reinterpret_cast<const void*>(addr), data.code_size_bytes);
 	std::scoped_lock lock(g_shader_map_mutex);
-
-	(*g_shader_map)[addr] = data;
+	(*g_shader_map)[addr] = {data, hash};
 }
 
-static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
+static ShaderMapEntry ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT_IF(g_shader_map == nullptr);
 
 	std::scoped_lock lock(g_shader_map_mutex);
@@ -90,37 +79,15 @@ static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) {
 	EXIT("%s shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, addr);
 }
 
-static const ShaderBinaryInfo* GetBinaryInfo(const uint32_t* code) {
-	EXIT_IF(code == nullptr);
-
-	if (code[0] == 0xBEEB03FF) {
-		return reinterpret_cast<const ShaderBinaryInfo*>(code +
-		                                                 static_cast<size_t>(code[1] + 1) * 2);
-	}
-
-	return nullptr;
-}
-
-static uint64_t GetDeclaredShaderHash(uint64_t shader_addr) {
-	const auto* header = GetBinaryInfo(reinterpret_cast<const uint32_t*>(shader_addr));
-	return header != nullptr ? (static_cast<uint64_t>(header->hash1) << 32u) | header->hash0 : 0;
-}
-
-static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash,
+static ShaderParams GetShaderParams(uint64_t shader_addr, uint64_t hash,
 	                                std::span<const uint32_t> user_data,
 	                                const ShaderMappedData& data, uint32_t user_data_base = 0) {
-	if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
-		EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64
-		     " has invalid AGC shader_size=0x%08" PRIx32 "\n",
-		     label, declared_hash, shader_addr, data.code_size_bytes);
-	}
 	const auto code_words = data.code_size_bytes / sizeof(uint32_t);
 	const auto code = std::span {reinterpret_cast<const uint32_t*>(shader_addr), code_words};
 	ShaderParams params {
 	    .code            = code,
 	    .user_data_count = static_cast<uint32_t>(user_data.size()) + user_data_base,
-	    .hash            = declared_hash != 0 ? declared_hash
-	                                          : XXH3_64bits(code.data(), code.size_bytes()),
+	    .hash            = hash,
 	};
 	EXIT_IF(user_data.size() > HW::UserSgprInfo::SGPRS_MAX ||
 	        params.user_data_count > params.user_data.size());
@@ -414,6 +381,8 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 		r.fields[1]       = sharp[1];
 		r.fields[2]       = sharp[2];
 		r.fields[3]       = sharp[3];
+		EXIT_NOT_IMPLEMENTED(r.AddTid());
+		EXIT_NOT_IMPLEMENTED(r.SwizzleEnabled());
 		if (format != Prospero::VertexAttribFormat::kInvalid) {
 			const auto                   format_raw    = static_cast<uint32_t>(format);
 			const auto                   buffer_format = format_raw >> 2u;
@@ -650,7 +619,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 		key.insert(key.end(), {mesh.wave_size, mesh.host_subgroup_size, mesh.lds_size_dwords,
 		                       mesh.scratch_size_dwords, mesh.input_primitive,
 		                       mesh.primitives_per_group, mesh.vertices_per_group,
-		                       mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex});
+		                       mesh.max_vertices, mesh.max_primitives, mesh.provoking_vertex,
+		                       static_cast<uint32_t>(mesh.fast_launch)});
 	}
 	key.push_back(info.tess.input_control_points);
 	if (info.tess.input_control_points != 0) {
@@ -662,12 +632,9 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 	for (int i = 0; i < info.resources_num; i++) {
 		const auto& resource    = info.resources[i];
 		const auto& destination = info.resources_dst[i];
-		key.push_back(destination.register_start);
 		key.push_back(destination.registers_num);
-		key.push_back(destination.fetch_index);
 		key.push_back(static_cast<uint32_t>(destination.attr_id));
-		key.push_back(resource.fields[1] & 0xbfff0000u); // Stride and swizzle enable.
-		key.push_back(resource.fields[3] & 0x3087ffffu); // Channels, format, OOB, and add TID.
+		key.push_back(resource.fields[3] & 0x7ffffu); // Channels and format used by embedded fetch.
 	}
 }
 
@@ -726,11 +693,10 @@ void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector<uint32_
 ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             const HW::UserConfig& user_config, ShaderVertexInputInfo& info) {
 	const auto& sh     = context.GetShaderRegisters();
-	const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
 	const bool merged = (context.GetShaderStages() & 0x20u) != 0;
 	auto        params = GetShaderParams(
-	    regs.es_regs.data_addr, "ShaderRecompiler VS",
-	    GetDeclaredShaderHash(regs.es_regs.data_addr),
+	    regs.es_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data,
 	    merged ? 8u : 0u);
 	if (!merged) {
@@ -753,12 +719,14 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 	mesh.provoking_vertex    = context.GetModeControl().provoking_vtx_last ? 2u : 0u;
 	mesh.lds_size_dwords     = static_cast<uint32_t>(regs.gs_regs.rsrc2.lds_size) * 128u;
 	mesh.scratch_size_dwords = data.scratch_size_dwords;
+	const auto fast_launch = (context.GetShaderStages() >> 19u) & 3u;
+	EXIT_NOT_IMPLEMENTED(fast_launch > 1u);
+	mesh.fast_launch = fast_launch != 0;
 	if (data.type == Prospero::ShaderBinaryType::kGsFront) {
 		EXIT_IF(regs.gs_regs.data_addr == 0);
-		const auto back = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
+		const auto [back, back_hash] = ShaderGetMappedData(regs.gs_regs.data_addr, "ShaderGetInputInfoGS():");
 		const auto back_params =
-		    GetShaderParams(regs.gs_regs.data_addr, "ShaderRecompiler GS",
-		                    GetDeclaredShaderHash(regs.gs_regs.data_addr), {}, back);
+		    GetShaderParams(regs.gs_regs.data_addr, back_hash, {}, back);
 		params.back_code = back_params.code;
 		params.user_data[0] = static_cast<uint32_t>(regs.gs_regs.user_data_addr);
 		params.user_data[1] = static_cast<uint32_t>(regs.gs_regs.user_data_addr >> 32u);
@@ -766,9 +734,20 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 		params.hash = XXH3_64bits(hashes, sizeof(hashes));
 		mesh.scratch_size_dwords = std::max(mesh.scratch_size_dwords, back.scratch_size_dwords);
 	}
-	EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
-	                     regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
 	const auto& group = user_config.GetGeControl();
+	if (mesh.fast_launch) {
+		const auto& user_vgpr = user_config.GetGeUserVgprEn();
+		EXIT_NOT_IMPLEMENTED(data.type != Prospero::ShaderBinaryType::kGs ||
+		                     regs.gs_regs.rsrc1.gs_vgpr_component_count != 0u ||
+		                     regs.gs_regs.rsrc2.es_vgpr_component_count != 1u ||
+		                     user_config.GetPrimType() != Prospero::PrimitiveType::kPointList ||
+		                     group.primitive_group_size != 1u || group.vertex_group_size != 1u ||
+		                     user_vgpr.vgpr1 || user_vgpr.vgpr2 || user_vgpr.vgpr3 ||
+		                     mesh.max_vertices != sh.m_vgtGsMaxVertOut);
+	} else {
+		EXIT_NOT_IMPLEMENTED(regs.gs_regs.rsrc1.gs_vgpr_component_count != 3u ||
+		                     regs.gs_regs.rsrc2.es_vgpr_component_count != 3u);
+	}
 	if ((user_config.GetPrimType() != Prospero::PrimitiveType::kPointList &&
 	     user_config.GetPrimType() != Prospero::PrimitiveType::kLineList &&
 	     user_config.GetPrimType() != Prospero::PrimitiveType::kTriFan &&
@@ -781,7 +760,8 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
 		     mesh.input_primitive, sh.m_vgtGsOutPrimType, sh.m_vgtGsMaxVertOut,
 		     group.primitive_group_size, group.vertex_group_size, mesh.max_vertices);
 	}
-	mesh.max_primitives       = group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
+	mesh.max_primitives = mesh.fast_launch ? mesh.max_vertices :
+	                      group.primitive_group_size * (sh.m_vgtGsMaxVertOut - 2u);
 	mesh.primitives_per_group = std::min({static_cast<uint32_t>(group.primitive_group_size),
 	                                      mesh.InputPrimitiveCount(group.vertex_group_size),
 	                                      mesh.max_vertices / sh.m_vgtGsMaxVertOut});
@@ -797,9 +777,9 @@ std::array<ShaderParams, 3>
 PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& context,
                             std::array<ShaderVertexInputInfo, 3>& input_info) {
 	const auto& sh        = context.GetShaderRegisters();
-	const auto  local     = ShaderGetMappedData(regs.ls_regs.data_addr, "ShaderGetInputInfoLS():");
-	const auto  control   = ShaderGetMappedData(regs.hs_regs.data_addr, "ShaderGetInputInfoHS():");
-	const auto evaluation = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoTES():");
+	const auto [local, local_hash] = ShaderGetMappedData(regs.ls_regs.data_addr, "ShaderGetInputInfoLS():");
+	const auto [control, control_hash] = ShaderGetMappedData(regs.hs_regs.data_addr, "ShaderGetInputInfoHS():");
+	const auto [evaluation, evaluation_hash] = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoTES():");
 	EXIT_NOT_IMPLEMENTED(local.type != Prospero::ShaderBinaryType::kHsFront ||
 	                     control.type != Prospero::ShaderBinaryType::kHsBack ||
 	                     evaluation.type != Prospero::ShaderBinaryType::kGs);
@@ -810,12 +790,9 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
 	const auto local_users      = std::span(regs.hs_user_sgpr.value, regs.hs_regs.rsrc2.user_sgpr);
 	const auto evaluation_users = std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr);
 	std::array<ShaderParams, 3> params {
-	    GetShaderParams(regs.ls_regs.data_addr, "ShaderRecompiler LS",
-	                    GetDeclaredShaderHash(regs.ls_regs.data_addr), local_users, local),
-	    GetShaderParams(regs.hs_regs.data_addr, "ShaderRecompiler HS",
-	                    GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control, 8u),
-	    GetShaderParams(regs.es_regs.data_addr, "ShaderRecompiler TES",
-	                    GetDeclaredShaderHash(regs.es_regs.data_addr), evaluation_users,
+	    GetShaderParams(regs.ls_regs.data_addr, local_hash, local_users, local),
+	    GetShaderParams(regs.hs_regs.data_addr, control_hash, local_users, control, 8u),
+	    GetShaderParams(regs.es_regs.data_addr, evaluation_hash, evaluation_users,
 	                    evaluation),
 	};
 	// The fused HS back half receives its separate user-data address in s0:s1.
@@ -855,19 +832,19 @@ ShaderParams PrepareProgram(
     const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
     std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
     ShaderPixelInputInfo&                               ps_info) {
-	const auto data = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():");
 	ShaderGetStaticInputInfoPS(regs, sh, target_export_mapping, data, ps_info);
 	return GetShaderParams(
-	    regs.ps_regs.data_addr, "ShaderRecompiler PS", GetDeclaredShaderHash(regs.ps_regs.data_addr),
+	    regs.ps_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), data);
 }
 
 ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
                             ShaderComputeInputInfo& info) {
-	const auto data = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
+	const auto [data, hash] = ShaderGetMappedData(regs.cs_regs.data_addr, "ShaderGetInputInfoCS():");
 	ShaderGetStaticInputInfoCS(regs, sh, data, info);
 	return GetShaderParams(
-	    regs.cs_regs.data_addr, "ShaderRecompiler CS", GetDeclaredShaderHash(regs.cs_regs.data_addr),
+	    regs.cs_regs.data_addr, hash,
 	    std::span<const uint32_t>(regs.cs_user_sgpr.value, regs.cs_regs.user_sgpr), data);
 }
 

src/graphics/shader/shader.h

@@ -88,6 +88,7 @@ struct ShaderMeshInputInfo: ShaderWorkgroupInputInfo {
 	uint32_t max_vertices         = 0;
 	uint32_t max_primitives       = 0;
 	uint32_t provoking_vertex     = 0;
+	bool     fast_launch          = false;
 
 	[[nodiscard]] constexpr uint32_t InputPrimitiveSize() const {
 		switch (static_cast<Prospero::PrimitiveType>(input_primitive)) {

src/libs/audio.cpp

@@ -1,6 +1,5 @@
 #include "libs/audio.h"
 
-#include <SDL3/SDL.h>
 #include "common/assert.h"
 #include "common/emulatorConfig.h"
 #include "common/logging/log.h"
@@ -12,7 +11,9 @@
 #include "libs/errno.h"
 #include "libs/libs.h"
 
+#include <SDL3/SDL.h>
 #include <algorithm>
+#include <array>
 #include <cstring>
 #include <limits>
 #include <magic_enum.hpp>
@@ -37,6 +38,24 @@ constexpr uint64_t AUDIO_OUT_TARGET_LATENCY_US = 40000;
 constexpr int      AUDIO_IN_SILENT_STATE_DEVICE_NONE = 0x1;
 constexpr uint32_t AUDIO_IN_GRAIN_MAX_ASYNC          = 384;
 
+struct OutputClock {
+	uint64_t next_time = 0;
+	uint64_t remainder = 0;
+
+	uint64_t Advance(uint64_t now, uint32_t frames, uint32_t rate) {
+		// Preserve small scheduling errors; discard timing debt after a discontinuity.
+		if (next_time == 0 || (now > next_time && now - next_time > 100000)) {
+			next_time = now;
+			remainder = 0;
+		}
+		const auto deadline = next_time;
+		const auto duration = 1000000ULL * frames + remainder;
+		next_time += duration / rate;
+		remainder = duration % rate;
+		return deadline;
+	}
+};
+
 static bool audio_out_port_type_is_valid(int type) {
 	return (type >= AUDIO_OUT_PORT_TYPE_MAIN && type <= AUDIO_OUT_PORT_TYPE_PADSPK) ||
 	       type == AUDIO_OUT_PORT_TYPE_VIBRATION || type == AUDIO_OUT_PORT_TYPE_AUDIO3D ||
@@ -98,17 +117,17 @@ public:
 
 private:
 	struct PortOut {
-		bool     used             = false;
-		int      type             = 0;
-		uint32_t samples_num      = 0;
-		uint32_t freq             = 0;
-		Format   format           = Format::Unknown;
-		uint64_t last_output_time = 0;
-		bool     queue_primed     = false;
-		int      channels_num     = 0;
-		int      volume[12]       = {};
+		bool        used        = false;
+		int         type        = 0;
+		uint32_t    samples_num = 0;
+		uint32_t    freq        = 0;
+		Format      format      = Format::Unknown;
+		OutputClock clock;
+		bool        queue_primed = false;
+		int         channels_num = 0;
+		int         volume[12]   = {};
 
-		SDL_AudioStream*                     stream  = nullptr;
+		SDL_AudioStream*                      stream  = nullptr;
 		Controller::DualSenseHaptics::Stream* haptics = nullptr;
 	};
 
@@ -376,10 +395,7 @@ bool Audio::QueueSdlAudio(PortOut* port, const void* data, bool blocking) {
 		        : 2u;
 		min_queued_size           = prepared_size * std::clamp(buffers, 2u, 16u);
 		const auto wait_start      = LibKernel::KernelGetProcessTime();
-		auto queued                = SDL_GetAudioStreamQueued(port->stream);
-		if (queued < static_cast<int>(prepared_size)) {
-			port->queue_primed = false;
-		}
+		auto       queued          = SDL_GetAudioStreamQueued(port->stream);
 		while (queued > static_cast<int>(min_queued_size)) {
 			if (LibKernel::KernelGetProcessTime() - wait_start > 200000) {
 				SDL_ClearAudioStream(port->stream);
@@ -389,19 +405,20 @@ bool Audio::QueueSdlAudio(PortOut* port, const void* data, bool blocking) {
 			Common::Thread::SleepMicro(1000);
 			queued = SDL_GetAudioStreamQueued(port->stream);
 		}
-		if (port->queue_primed) {
-			const auto next_time = port->last_output_time + buffer_us;
-			const auto now       = LibKernel::KernelGetProcessTime();
-			if (next_time > now) {
-				Common::Thread::SleepMicro(next_time - now);
-			}
+		if (queued < static_cast<int>(prepared_size)) {
+			port->queue_primed = false;
 		}
 	}
 
 	if (!SDL_PutAudioStreamData(port->stream, prepared_data, static_cast<int>(prepared_size))) {
 		LOGF("AudioOut: SDL_PutAudioStreamData failed: %s\n", SDL_GetError());
+		port->queue_primed = false;
 		return false;
 	}
+	// Unpaced priming must not advance the playback deadline into the future.
+	if (!port->queue_primed) {
+		port->clock = {};
+	}
 	if (blocking && !port->queue_primed &&
 	    SDL_GetAudioStreamQueued(port->stream) >= static_cast<int>(min_queued_size)) {
 		port->queue_primed = true;
@@ -411,18 +428,21 @@ bool Audio::QueueSdlAudio(PortOut* port, const void* data, bool blocking) {
 }
 
 Audio::Id Audio::AudioOutOpen(int type, uint32_t samples_num, uint32_t freq, Format format) {
+	if (freq == 0) {
+		return Id::Invalid();
+	}
+
 	Common::LockGuard lock(m_mutex);
 
 	for (int id = 0; id < OUT_PORTS_MAX; id++) {
 		if (!m_out_ports[id].used) {
 			auto& port = m_out_ports[id];
 
-			port.used             = true;
-			port.type             = type;
-			port.samples_num      = samples_num;
-			port.freq             = freq;
-			port.format           = format;
-			port.last_output_time = 0;
+			port.used        = true;
+			port.type        = type;
+			port.samples_num = samples_num;
+			port.freq        = freq;
+			port.format      = format;
 
 			switch (format) {
 				case Format::Signed16bitMono:
@@ -524,61 +544,68 @@ bool Audio::AudioOutSetVolume(Id handle, uint32_t bitflag, const int* volume) {
 }
 
 uint32_t Audio::AudioOutOutputs(OutputParam* params, uint32_t num, bool blocking) {
-	EXIT_NOT_IMPLEMENTED(num == 0);
+	EXIT_NOT_IMPLEMENTED(num == 0 || num > OUT_PORTS_MAX);
 	EXIT_NOT_IMPLEMENTED(!AudioOutValid(params[0].handle));
 
-	const auto& first_port = m_out_ports[params[0].handle.GetId()];
-
-	uint64_t block_time   = (1000000 * first_port.samples_num) / first_port.freq;
-	uint64_t current_time = LibKernel::KernelGetProcessTime();
-
-	uint64_t max_wait_time = 0;
-
-	for (uint32_t i = 0; i < num; i++) {
-		uint64_t next_time = m_out_ports[params[i].handle.GetId()].last_output_time + block_time;
-		uint64_t wait_time = (next_time > current_time ? next_time - current_time : 0);
-		max_wait_time      = (wait_time > max_wait_time ? wait_time : max_wait_time);
-	}
-
-	bool any_port_has_device = false;
-	for (uint32_t i = 0; i < num; i++) {
-		if (m_out_ports[params[i].handle.GetId()].stream != nullptr) {
-			any_port_has_device = true;
-			break;
-		}
-	}
-
-	// One real output device is enough to pace the whole synchronized batch. Applying the fallback
-	// when a vibration port is present would rate-limit the device-backed ports to exactly 1x and
-	// prevent their SDL queues from building an underrun cushion.
-	if (blocking && max_wait_time != 0 && !any_port_has_device) {
-		Common::Thread::SleepMicro(max_wait_time);
-	}
+	const auto&                     first_port = m_out_ports[params[0].handle.GetId()];
+	std::array<bool, OUT_PORTS_MAX> paced {};
+	bool                            any_device = false;
+	uint64_t                        deadline   = 0;
 
 	for (uint32_t i = 0; i < num; i++) {
 		auto& port = m_out_ports[params[i].handle.GetId()];
 
 		uint64_t controller_queued_us = 0;
+		bool controller_uses_bluetooth = false;
 		if (port.haptics != nullptr) {
 			// Keep the stream alive against close and volume changes.
 			Common::LockGuard lock(m_mutex);
 			controller_queued_us = Controller::DualSenseHaptics::Queue(
 			    port.haptics, Controller::GetActiveControllerId(), params[i].data, port.samples_num,
 			    static_cast<uint32_t>(port.channels_num), FormatIsFloat(port.format), port.volume);
+			controller_uses_bluetooth =
+			    Controller::DualSenseHaptics::UsesBluetooth(port.haptics);
 		}
 		if (controller_queued_us == 0) {
 			// No DualSense took it (e.g. it was unplugged); a pad speaker port plays on the main
 			// output instead, and a vibration port has none.
-			QueueSdlAudio(&port, params[i].data, blocking);
-		} else if (blocking && port.type == AUDIO_OUT_PORT_TYPE_PADSPK &&
-		           controller_queued_us > AUDIO_OUT_TARGET_LATENCY_US) {
-			// Vibration never paces output; the speaker is audible, so pace it like other ports.
-			Common::Thread::SleepMicro(controller_queued_us - AUDIO_OUT_TARGET_LATENCY_US);
+			if (QueueSdlAudio(&port, params[i].data, blocking)) {
+				any_device = true;
+				paced[i]   = port.queue_primed;
+			}
+		} else {
+			port.queue_primed = false;
+			// Bluetooth HID has its own sender. Let the sample clock pace a batch
+			// without another audio device, instead of waiting on the HID queue.
+			if (port.type == AUDIO_OUT_PORT_TYPE_PADSPK && !controller_uses_bluetooth) {
+				any_device = true;
+				if (blocking && controller_queued_us > AUDIO_OUT_TARGET_LATENCY_US) {
+					// Limit a stalled USB endpoint to one audio block of waiting.
+					const uint64_t block_us = 1000000ULL * port.samples_num / port.freq;
+					deadline = std::max(deadline, LibKernel::KernelGetProcessTime() +
+					                                  std::min(controller_queued_us - AUDIO_OUT_TARGET_LATENCY_US,
+					                                           block_us));
+				}
+			}
 		}
 	}
 
+	// Submit the whole batch before pacing it. Audible devices own its cadence while
+	// priming; silent/vibration-only batches use the same sample clock as SDL output.
+	const auto now = LibKernel::KernelGetProcessTime();
 	for (uint32_t i = 0; i < num; i++) {
-		m_out_ports[params[i].handle.GetId()].last_output_time = LibKernel::KernelGetProcessTime();
+		auto& port = m_out_ports[params[i].handle.GetId()];
+		if (blocking && (paced[i] || !any_device)) {
+			deadline = std::max(deadline, port.clock.Advance(now, port.samples_num, port.freq));
+		} else {
+			port.clock = {};
+		}
+		if (!blocking) {
+			port.queue_primed = false;
+		}
+	}
+	if (deadline > now) {
+		Common::Thread::SleepMicro(deadline - now);
 	}
 
 	return first_port.samples_num;
@@ -826,6 +853,9 @@ int KYTY_SYSV_ABI AudioOutOpen(int user_id, int type, int index, uint32_t len, u
 	if (!audio_out_port_type_is_valid(type)) {
 		return AUDIO_OUT_ERROR_INVALID_PORT_TYPE;
 	}
+	if (freq == 0) {
+		return AUDIO_OUT_ERROR_INVALID_SAMPLE_FREQ;
+	}
 	EXIT_NOT_IMPLEMENTED(index != 0);
 
 	Audio::Format format       = Audio::Format::Unknown;
@@ -935,6 +965,9 @@ int KYTY_SYSV_ABI AudioOutOutputs(AudioOutOutputParam* param, uint32_t num) {
 	PRINT_NAME();
 
 	EXIT_NOT_IMPLEMENTED(param == nullptr);
+	if (num == 0 || num > Audio::OUT_PORTS_MAX) {
+		return AUDIO_OUT_ERROR_INVALID_SIZE;
+	}
 
 	Audio::OutputParam params[Audio::OUT_PORTS_MAX];
 

src/libs/dualSenseBluetooth.cpp

@@ -0,0 +1,467 @@
+#include "libs/dualSenseBluetooth.h"
+
+#include "common/logging/log.h"
+
+#include <SDL3/SDL.h>
+#include <SDL3/SDL_hidapi.h>
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <condition_variable>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <opus.h>
+#include <string>
+#include <vector>
+
+namespace Libs::Controller::DualSenseBluetooth {
+
+namespace {
+// Report 0x39 and its sub-packets are described by awalol/DS5Dongle (MIT):
+// https://github.com/awalol/DS5Dongle
+// Each report plays two 32-frame stereo haptic blocks at 3 kHz (21.333 ms).
+constexpr uint64_t PERIOD_NS   = 64000000000ULL / 3000;
+constexpr int      HAPTIC_RATE = 3000;
+constexpr int      HAPTIC_REPORT_FLOATS = 2 * 32 * 2;
+// Two 480-frame Opus packets occupy the same 21.333 ms slot. Resampling
+// 48 kHz input to 45 kHz before encoding at 48 kHz compensates for playback.
+constexpr int    SPEAKER_RATE = 45000;
+constexpr int    OPUS_FRAMES  = 480;
+constexpr int    OPUS_BYTES   = 200;
+constexpr size_t REPORT_BYTES = 547;
+constexpr int    MAX_QUEUE_MS = 80;
+constexpr uint64_t COALESCE_NS = 1000000;
+
+struct Session;
+std::mutex                              g_mutex;
+std::condition_variable                 g_wake;
+std::map<int, std::shared_ptr<Session>> g_sessions;
+SDL_Thread*                             g_thread   = nullptr;
+bool                                    g_stopping = false;
+void (*g_tick)()                                   = nullptr;
+
+template <typename Char>
+std::string Address(const Char* serial) {
+	std::string address;
+	if (serial == nullptr) {
+		return address;
+	}
+	for (; *serial != 0; ++serial) {
+		if (*serial >= '0' && *serial <= '9') {
+			address.push_back(static_cast<char>(*serial));
+		} else if (*serial >= 'a' && *serial <= 'f') {
+			address.push_back(static_cast<char>(*serial));
+		} else if (*serial >= 'A' && *serial <= 'F') {
+			address.push_back(static_cast<char>(*serial - 'A' + 'a'));
+		} else if (*serial != ':' && *serial != '-') {
+			return {};
+		}
+	}
+	return address.size() == 12 ? address : std::string {};
+}
+
+SDL_hid_device* OpenDevice(int controller) {
+	SDL_LockJoysticks();
+	auto*      pad      = SDL_GetGamepadFromID(static_cast<SDL_JoystickID>(controller));
+	const bool wireless = pad != nullptr && SDL_GetGamepadType(pad) == SDL_GAMEPAD_TYPE_PS5 &&
+	                      SDL_GetGamepadConnectionState(pad) == SDL_JOYSTICK_CONNECTION_WIRELESS;
+	const auto vendor  = wireless ? SDL_GetGamepadVendor(pad) : 0;
+	const auto product = wireless ? SDL_GetGamepadProduct(pad) : 0;
+	const auto address = wireless ? Address(SDL_GetGamepadSerial(pad)) : std::string {};
+	SDL_UnlockJoysticks();
+	if (vendor != 0x054c || (product != 0x0ce6 && product != 0x0df2) || address.empty()) {
+		return nullptr;
+	}
+	auto*       devices = SDL_hid_enumerate(vendor, product);
+	std::string path;
+	bool        ambiguous = false;
+	for (auto* device = devices; device != nullptr; device = device->next) {
+		if (device->bus_type == SDL_HID_API_BUS_BLUETOOTH && device->path != nullptr &&
+		    Address(device->serial_number) == address &&
+		    (device->usage_page == 0 || (device->usage_page == 1 && device->usage == 5))) {
+			if (!path.empty()) {
+				ambiguous = true;
+				break;
+			}
+			path = device->path;
+		}
+	}
+	SDL_hid_free_enumeration(devices);
+	return !ambiguous && !path.empty() ? SDL_hid_open_path(path.c_str()) : nullptr;
+}
+
+void FinishReport(std::array<uint8_t, REPORT_BYTES>& report) {
+	const uint8_t prefix = 0xa2;
+	auto          crc    = SDL_crc32(0, &prefix, 1);
+	crc                  = SDL_crc32(crc, report.data(), report.size() - 4);
+	for (size_t i = 0; i < 4; i++) {
+		report[report.size() - 4 + i] = static_cast<uint8_t>(crc >> (8 * i));
+	}
+}
+
+struct Session {
+	SDL_hid_device*       hid     = nullptr;
+	OpusEncoder*          encoder = nullptr;
+	std::vector<Stream*>  streams;
+	std::mutex            io_mutex;
+	std::atomic<bool>     active {true};
+	std::atomic<bool>     failed {false};
+	std::atomic<uint64_t> generation {0};
+	uint64_t              next_send        = 0;
+	uint64_t              pending_deadline = 0;
+	uint8_t               sequence  = 0;
+	uint8_t               counter   = 0;
+
+	~Session() {
+		if (encoder != nullptr) {
+			opus_encoder_destroy(encoder);
+		}
+		if (hid != nullptr) {
+			SDL_hid_close(hid);
+		}
+	}
+
+	bool CreateEncoder() {
+		if (encoder != nullptr) {
+			return true;
+		}
+		int error = OPUS_OK;
+		encoder   = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &error);
+		if (encoder == nullptr || error != OPUS_OK) {
+			return false;
+		}
+		if (opus_encoder_ctl(encoder, OPUS_SET_BITRATE(OPUS_BYTES * 8 * 100)) == OPUS_OK &&
+		    opus_encoder_ctl(encoder, OPUS_SET_VBR(0)) == OPUS_OK &&
+		    opus_encoder_ctl(encoder, OPUS_SET_COMPLEXITY(0)) == OPUS_OK) {
+			return true;
+		}
+		opus_encoder_destroy(encoder);
+		encoder = nullptr;
+		return false;
+	}
+};
+} // namespace
+
+struct Stream {
+	std::shared_ptr<Session> session;
+	SDL_AudioStream*         audio   = nullptr;
+	uint32_t                 freq    = 0;
+	bool                     speaker = false;
+};
+
+namespace {
+template <size_t N>
+void Mix(SDL_AudioStream* stream, std::array<float, N>& mixed) {
+	std::array<float, N> pcm {};
+	const int            bytes = SDL_GetAudioStreamData(stream, pcm.data(), sizeof(pcm));
+	for (int i = 0; i < std::max(0, bytes) / static_cast<int>(sizeof(float)); i++) {
+		mixed[i] += pcm[i];
+	}
+}
+
+struct PendingReport {
+	std::shared_ptr<Session>          session;
+	uint64_t                          generation;
+	std::array<uint8_t, REPORT_BYTES> data {};
+	std::array<float, 2 * OPUS_FRAMES * 2> audio {};
+	bool                              speaker = false;
+};
+
+bool PrepareReport(Session& session, PendingReport& pending, uint64_t now) {
+	bool has_data      = false;
+	bool ready         = false;
+	bool speaker_ready = false;
+	for (const auto* stream: session.streams) {
+		const bool available = SDL_GetAudioStreamAvailable(stream->audio) > 0;
+		// Both payloads cover 64 / 3000 seconds. Measure input duration so SDL's
+		// resampler lookahead does not make a complete block appear partial.
+		const auto frames =
+		    (static_cast<uint64_t>(stream->freq) * (HAPTIC_REPORT_FLOATS / 2) + HAPTIC_RATE - 1) /
+		    HAPTIC_RATE;
+		const bool full = available && SDL_GetAudioStreamQueued(stream->audio) >=
+		                                   static_cast<int>(frames * 2 * sizeof(float));
+		has_data |= available;
+		ready |= full;
+		speaker_ready |= stream->speaker && full;
+		pending.speaker |= stream->speaker;
+	}
+	if (!has_data) {
+		session.pending_deadline = 0;
+		return false;
+	}
+	if (!ready || (!speaker_ready && (pending.speaker || session.next_send == 0))) {
+		// Accumulate short blocks for at most one report period. A full stream
+		// must not wait on a partial peer; only briefly coalesce haptics with
+		// speaker audio arriving in the same emulated batch.
+		const auto deadline = now + (ready ? COALESCE_NS : PERIOD_NS);
+		session.pending_deadline =
+		    session.pending_deadline == 0 ? deadline : std::min(session.pending_deadline, deadline);
+		if (now < session.pending_deadline) {
+			return false;
+		}
+	}
+	session.pending_deadline = 0;
+	std::array<float, HAPTIC_REPORT_FLOATS> haptics {};
+	for (auto* stream: session.streams) {
+		if (stream->speaker) {
+			Mix(stream->audio, pending.audio);
+		} else {
+			Mix(stream->audio, haptics);
+		}
+	}
+	auto& report     = pending.data;
+	report[0]        = 0x39;
+	report[1]        = static_cast<uint8_t>(session.sequence << 4);
+	session.sequence = (session.sequence + 1) & 0x0f;
+	report[2]        = 0x91;
+	report[3]        = 6;
+	report[4]        = 0x7e;
+	std::fill_n(report.begin() + 5, 4, 48);
+	session.counter += 2;
+	report[9]  = session.counter;
+	report[10] = 0xd2;
+	report[11] = 64;
+	for (size_t i = 0; i < haptics.size(); i++) {
+		const auto value = std::isfinite(haptics[i]) ? haptics[i] : 0.0f;
+		report[12 + i]   = static_cast<uint8_t>(
+            static_cast<int>(std::lround(std::clamp(value, -1.0f, 1.0f) * 127)));
+	}
+	if (pending.speaker) {
+		report[140] = 0xd3;
+		report[141] = OPUS_BYTES;
+	}
+	return true;
+}
+
+bool EncodeReport(PendingReport& pending) {
+	if (!pending.session->active || pending.session->failed) {
+		return false;
+	}
+	if (pending.speaker) {
+		for (auto& sample: pending.audio) {
+			sample = std::isfinite(sample) ? std::clamp(sample, -1.0f, 1.0f) : 0.0f;
+		}
+		for (int packet = 0; packet < 2; packet++) {
+			if (opus_encode_float(pending.session->encoder,
+			                      pending.audio.data() + packet * OPUS_FRAMES * 2, OPUS_FRAMES,
+			                      pending.data.data() + 142 + packet * OPUS_BYTES,
+			                      OPUS_BYTES) != OPUS_BYTES) {
+				pending.session->failed = true;
+				return false;
+			}
+		}
+	}
+	FinishReport(pending.data);
+	return true;
+}
+
+std::vector<PendingReport> Prepare(uint64_t now) {
+	std::vector<PendingReport> pending;
+	{
+		std::lock_guard lock(g_mutex);
+		for (auto& [id, session]: g_sessions) {
+			if (session->failed || now < session->next_send) {
+				continue;
+			}
+			PendingReport report {session, session->generation.load()};
+			if (PrepareReport(*session, report, now)) {
+				pending.push_back(std::move(report));
+				// Regain timing after a late HID write without sending reports in a
+				// burst. Reset the cadence only on the first report.
+				session->next_send = session->next_send == 0
+				                         ? now + PERIOD_NS
+				                         : std::max(session->next_send + PERIOD_NS,
+				                                    now + PERIOD_NS / 2);
+			}
+		}
+	}
+	// Opus encoding can take milliseconds; the emulation audio thread must be able
+	// to queue its next block while this work runs.
+	pending.erase(std::remove_if(pending.begin(), pending.end(),
+	                             [](PendingReport& report) { return !EncodeReport(report); }),
+	              pending.end());
+	return pending;
+}
+
+void Send(const std::vector<PendingReport>& pending) {
+	for (const auto& report: pending) {
+		auto&           session = *report.session;
+		std::lock_guard io_lock(session.io_mutex);
+		if (!session.active || session.failed || report.generation != session.generation) {
+			continue;
+		}
+		if (SDL_hid_write(session.hid, report.data.data(), report.data.size()) !=
+		    static_cast<int>(report.data.size())) {
+			session.failed = true;
+			LOGF("DualSenseBluetooth: audio HID write failed: %s\n", SDL_GetError());
+		}
+	}
+}
+
+int SDLCALL Sender(void*) {
+	std::unique_lock lock(g_mutex);
+	while (!g_stopping) {
+		auto tick = g_tick;
+		lock.unlock();
+		auto pending = Prepare(SDL_GetTicksNS());
+		Send(pending);
+		if (tick != nullptr) {
+			tick();
+		}
+		lock.lock();
+		if (g_sessions.empty()) {
+			g_wake.wait(lock, [] { return g_stopping || !g_sessions.empty(); });
+		} else {
+			const auto now = SDL_GetTicksNS();
+			uint64_t   wait_ns = 10000000; // Check rumble expiry at most 10 ms late.
+			for (const auto& [id, session]: g_sessions) {
+				if (session->next_send > now) {
+					wait_ns = std::min(wait_ns, session->next_send - now);
+				}
+				if (session->pending_deadline > now) {
+					wait_ns = std::min(wait_ns, session->pending_deadline - now);
+				}
+			}
+			g_wake.wait_for(lock, std::chrono::nanoseconds(wait_ns));
+		}
+	}
+	return 0;
+}
+} // namespace
+
+Stream* Open(uint32_t freq, bool speaker, int controller, void (*tick)()) {
+	if (freq == 0 || freq > 192000) {
+		return nullptr;
+	}
+	std::lock_guard lock(g_mutex);
+	if (g_stopping) {
+		return nullptr;
+	}
+	auto                     it = g_sessions.find(controller);
+	std::shared_ptr<Session> session;
+	if (it != g_sessions.end() && !it->second->failed) {
+		session = it->second;
+	} else {
+		session      = std::make_shared<Session>();
+		session->hid = OpenDevice(controller);
+		if (session->hid == nullptr) {
+			return nullptr;
+		}
+	}
+	if (speaker && !session->CreateEncoder()) {
+		return nullptr;
+	}
+	const SDL_AudioSpec source {SDL_AUDIO_F32, 2, static_cast<int>(freq)};
+	const SDL_AudioSpec target {SDL_AUDIO_F32, 2, speaker ? SPEAKER_RATE : HAPTIC_RATE};
+	auto*               audio = SDL_CreateAudioStream(&source, &target);
+	if (audio == nullptr) {
+		return nullptr;
+	}
+	if (g_thread == nullptr) {
+		g_thread = SDL_CreateThread(Sender, "DualSense Bluetooth audio", nullptr);
+		if (g_thread == nullptr) {
+			SDL_DestroyAudioStream(audio);
+			return nullptr;
+		}
+	}
+	g_tick       = tick;
+	auto* stream = new Stream {session, audio, freq, speaker};
+	session->streams.push_back(stream);
+	session->pending_deadline = 0;
+	g_sessions[controller] = session;
+	g_wake.notify_one();
+	return stream;
+}
+
+uint64_t Queue(Stream* stream, const float* stereo, uint32_t frames) {
+	if (stream == nullptr || stereo == nullptr || frames == 0) {
+		return 0;
+	}
+	std::lock_guard lock(g_mutex);
+	if (!stream->session->active || stream->session->failed) {
+		return 0;
+	}
+	const uint32_t max_frames = stream->freq * MAX_QUEUE_MS / 1000;
+	if (frames > max_frames) {
+		stereo += static_cast<size_t>(frames - max_frames) * 2;
+		frames = max_frames;
+	}
+	int       queued = std::max(0, SDL_GetAudioStreamQueued(stream->audio));
+	const int bytes  = static_cast<int>(frames * 2 * sizeof(float));
+	if (queued + bytes > static_cast<int>(max_frames * 2 * sizeof(float))) {
+		// Discard complete old report slots, preserving the newest queued audio.
+		// Clearing the entire stream here creates an audible gap after a brief
+		// Windows Bluetooth write stall.
+		std::array<float, 2 * OPUS_FRAMES * 2> discarded {};
+		const int discard_bytes = stream->speaker
+		                              ? static_cast<int>(sizeof(discarded))
+		                              : static_cast<int>(HAPTIC_REPORT_FLOATS * sizeof(float));
+		const int target = static_cast<int>(max_frames * 2 * sizeof(float)) - bytes;
+		while (queued > target) {
+			if (SDL_GetAudioStreamData(stream->audio, discarded.data(), discard_bytes) <= 0) {
+				SDL_ClearAudioStream(stream->audio);
+				break;
+			}
+			queued = std::max(0, SDL_GetAudioStreamQueued(stream->audio));
+		}
+	}
+	queued = std::max(0, SDL_GetAudioStreamQueued(stream->audio));
+	if (!SDL_PutAudioStreamData(stream->audio, stereo, bytes)) {
+		return 0;
+	}
+	g_wake.notify_one();
+	return static_cast<uint64_t>(queued + bytes) * 1000000 / (stream->freq * 2 * sizeof(float));
+}
+
+void Close(Stream* stream) {
+	if (stream == nullptr) {
+		return;
+	}
+	auto session = stream->session;
+	{
+		std::lock_guard lock(g_mutex);
+		auto&           streams = session->streams;
+		streams.erase(std::remove(streams.begin(), streams.end(), stream), streams.end());
+		session->generation++;
+		session->pending_deadline = 0;
+		if (streams.empty()) {
+			session->active = false;
+			for (auto it = g_sessions.begin(); it != g_sessions.end(); ++it) {
+				if (it->second == session) {
+					g_sessions.erase(it);
+					break;
+				}
+			}
+		}
+		SDL_DestroyAudioStream(stream->audio);
+		delete stream;
+	}
+	// Drain a write already in flight before the caller changes speaker routing.
+	std::lock_guard io_lock(session->io_mutex);
+}
+
+void Shutdown() {
+	SDL_Thread* thread;
+	{
+		std::lock_guard lock(g_mutex);
+		g_stopping = true;
+		for (auto& [id, session]: g_sessions) {
+			session->active = false;
+		}
+		thread = g_thread;
+		g_wake.notify_one();
+	}
+	if (thread != nullptr) {
+		SDL_WaitThread(thread, nullptr);
+	}
+	std::lock_guard lock(g_mutex);
+	g_sessions.clear();
+	g_thread   = nullptr;
+	g_tick     = nullptr;
+	g_stopping = false;
+}
+
+} // namespace Libs::Controller::DualSenseBluetooth

src/libs/dualSenseBluetooth.h

@@ -0,0 +1,14 @@
+#ifndef KYTY_LIBS_DUALSENSE_BLUETOOTH_H_
+#define KYTY_LIBS_DUALSENSE_BLUETOOTH_H_
+
+#include <cstdint>
+
+namespace Libs::Controller::DualSenseBluetooth {
+struct Stream;
+Stream*  Open(uint32_t frequency, bool speaker, int controller, void (*tick)());
+uint64_t Queue(Stream* stream, const float* samples, uint32_t frames);
+void     Close(Stream* stream);
+void     Shutdown();
+} // namespace Libs::Controller::DualSenseBluetooth
+
+#endif

src/libs/dualSenseHaptics.cpp

@@ -2,6 +2,7 @@
 
 #include "common/logging/log.h"
 #include "common/threads.h"
+#include "libs/dualSenseBluetooth.h"
 
 #include <SDL3/SDL.h>
 #include <algorithm>
@@ -13,18 +14,20 @@
 namespace Libs::Controller::DualSenseHaptics {
 
 struct Stream {
-	uint32_t           freq       = 0;
-	bool               speaker    = false;
-	int                controller = -1;
-	SDL_AudioStream*   sdl        = nullptr;
-	SDL_AudioDeviceID  device     = 0;
-	uint64_t           next_check = 0;
-	std::vector<float> buffer;
+	uint32_t                    freq       = 0;
+	bool                        speaker    = false;
+	int                         controller = -1;
+	SDL_AudioStream*            sdl        = nullptr;
+	DualSenseBluetooth::Stream* bluetooth  = nullptr;
+	SDL_AudioDeviceID           device     = 0;
+	uint64_t                    next_check = 0;
+	std::vector<float>          buffer;
 };
 
 namespace {
 
-constexpr int      FRAME_BYTES = 4 * sizeof(float);
+constexpr int      FRAME_BYTES  = 4 * sizeof(float);
+constexpr uint32_t MAX_QUEUE_MS = 80;
 Common::Mutex      g_mutex;
 int                g_controller      = -1;
 int                g_haptics_streams = 0;
@@ -107,11 +110,22 @@ void SDLCALL UpdateRumble(void*, SDL_AudioStream*, int, int) {
 	}
 }
 
+void UpdateBluetoothRumble() {
+	UpdateRumble(nullptr, nullptr, 0, 0);
+}
+
 void CloseDevice(Stream* stream) {
+	const bool had_device = stream->sdl != nullptr || stream->bluetooth != nullptr;
+	if (stream->bluetooth != nullptr) {
+		DualSenseBluetooth::Close(stream->bluetooth);
+		stream->bluetooth = nullptr;
+	}
 	if (stream->sdl != nullptr) {
 		// Destroy outside g_mutex: SDL waits for any running get callback.
 		SDL_DestroyAudioStream(stream->sdl);
 		stream->sdl = nullptr;
+	}
+	if (had_device) {
 		Common::LockGuard lock(g_mutex);
 		if (!stream->speaker && --g_haptics_streams == 0 && g_haptics_until != 0) {
 			g_haptics_until = 0;
@@ -127,26 +141,34 @@ void CloseDevice(Stream* stream) {
 	}
 }
 
+bool IsWireless(int controller) {
+	SDL_LockJoysticks();
+	auto*      pad = SDL_GetGamepadFromID(static_cast<SDL_JoystickID>(controller));
+	const bool wireless =
+	    pad != nullptr && SDL_GetGamepadConnectionState(pad) == SDL_JOYSTICK_CONNECTION_WIRELESS;
+	SDL_UnlockJoysticks();
+	return wireless;
+}
+
 bool CanUseDevice(int controller) {
 	// SDL exposes no portable association between a gamepad and its USB audio endpoint.
 	// Require one wired DualSense; guessing with multiple pads could play on the wrong one.
 	SDL_LockJoysticks();
-	auto* pad = SDL_GetGamepadFromID(static_cast<SDL_JoystickID>(controller));
-	bool  usable =
-	    pad != nullptr && SDL_GetGamepadConnectionState(pad) == SDL_JOYSTICK_CONNECTION_WIRED;
-	int   count           = 0;
-	auto* pads            = SDL_GetGamepads(&count);
-	int   dualsense_count = 0;
-	bool  found           = false;
+	int   count                 = 0;
+	auto* pads                  = SDL_GetGamepads(&count);
+	int   wired_dualsense_count = 0;
+	bool  found                 = false;
 	for (int i = 0; i < count; i++) {
-		if (SDL_GetGamepadTypeForID(pads[i]) == SDL_GAMEPAD_TYPE_PS5) {
-			dualsense_count++;
+		auto* candidate = SDL_GetGamepadFromID(pads[i]);
+		if (candidate != nullptr && SDL_GetGamepadTypeForID(pads[i]) == SDL_GAMEPAD_TYPE_PS5 &&
+		    SDL_GetGamepadConnectionState(candidate) == SDL_JOYSTICK_CONNECTION_WIRED) {
+			wired_dualsense_count++;
 			found |= pads[i] == static_cast<SDL_JoystickID>(controller);
 		}
 	}
 	SDL_free(pads);
 	SDL_UnlockJoysticks();
-	return usable && found && dualsense_count == 1;
+	return found && wired_dualsense_count == 1;
 }
 
 SDL_AudioDeviceID FindDevice() {
@@ -169,32 +191,42 @@ SDL_AudioDeviceID FindDevice() {
 	return device;
 }
 
-void RefreshDevice(Stream* stream, int controller) {
-	if (stream->controller != controller) {
+void RefreshDevice(Stream* stream, int controller, bool wireless) {
+	if (stream->controller != controller || (wireless && stream->sdl != nullptr) ||
+	    (!wireless && stream->bluetooth != nullptr)) {
 		CloseDevice(stream);
 		stream->controller = controller;
 		stream->next_check = 0;
 	}
 	const auto now = SDL_GetTicks();
-	if (now < stream->next_check) {
+	if (stream->bluetooth != nullptr || now < stream->next_check) {
 		return;
 	}
 	stream->next_check = now + 2000;
-	const auto device  = FindDevice();
+	const auto device  = wireless ? 0 : FindDevice();
 	if (stream->device == device && stream->sdl != nullptr) {
 		return;
 	}
 	CloseDevice(stream);
 	stream->device = device;
-	if (device == 0) {
+	if (!wireless && device == 0) {
 		return;
 	}
-	const SDL_AudioSpec desired {SDL_AUDIO_F32, 4, static_cast<int>(stream->freq)};
-	auto*               sdl = SDL_OpenAudioDeviceStream(device, &desired, UpdateRumble, nullptr);
-	SDL_AudioSpec       actual {};
-	bool ready = sdl != nullptr &&
-	             SDL_GetAudioDeviceFormat(SDL_GetAudioStreamDevice(sdl), &actual, nullptr) &&
-	             actual.channels == 4 && SDL_ResumeAudioStreamDevice(sdl);
+	SDL_AudioStream*            sdl       = nullptr;
+	DualSenseBluetooth::Stream* bluetooth = nullptr;
+	bool                        ready;
+	if (wireless) {
+		bluetooth = DualSenseBluetooth::Open(stream->freq, stream->speaker, controller,
+		                                     UpdateBluetoothRumble);
+		ready     = bluetooth != nullptr;
+	} else {
+		const SDL_AudioSpec desired {SDL_AUDIO_F32, 4, static_cast<int>(stream->freq)};
+		SDL_AudioSpec       actual {};
+		sdl   = SDL_OpenAudioDeviceStream(device, &desired, UpdateRumble, nullptr);
+		ready = sdl != nullptr &&
+		        SDL_GetAudioDeviceFormat(SDL_GetAudioStreamDevice(sdl), &actual, nullptr) &&
+		        actual.channels == 4 && SDL_ResumeAudioStreamDevice(sdl);
+	}
 	if (ready) {
 		Common::LockGuard lock(g_mutex);
 		if (stream->speaker) {
@@ -207,14 +239,56 @@ void RefreshDevice(Stream* stream, int controller) {
 		}
 	}
 	if (ready) {
-		stream->sdl = sdl;
-		LOGF("DualSenseHaptics: playing on '%s'\n", SDL_GetAudioDeviceName(device));
+		stream->sdl       = sdl;
+		stream->bluetooth = bluetooth;
+		if (!wireless) {
+			LOGF("DualSenseHaptics: playing on '%s'\n", SDL_GetAudioDeviceName(device));
+		}
 	} else {
+		DualSenseBluetooth::Close(bluetooth);
 		SDL_DestroyAudioStream(sdl);
-		LOGF("DualSenseHaptics: cannot open quad output: %s\n", SDL_GetError());
+		if (!wireless) {
+			LOGF("DualSenseHaptics: cannot open quad output: %s\n", SDL_GetError());
+		}
 	}
 }
 
+uint64_t QueueUsbAudio(Stream* stream, uint32_t frames) {
+	auto      queued = std::max(0, SDL_GetAudioStreamQueued(stream->sdl));
+	const int bytes  = static_cast<int>(frames * FRAME_BYTES);
+	const int max_bytes =
+	    static_cast<int>(static_cast<uint64_t>(stream->freq) * FRAME_BYTES * MAX_QUEUE_MS / 1000);
+	if (queued + bytes > max_bytes) {
+		// Keep the newest sound after a brief device stall. Clearing the entire
+		// stream produces a gap that is much longer than one audio block.
+		SDL_AudioSpec              target {};
+		std::array<uint8_t, 16384> discarded {};
+		if (SDL_GetAudioStreamFormat(stream->sdl, nullptr, &target)) {
+			const int frame_bytes = SDL_AUDIO_FRAMESIZE(target);
+			if (frame_bytes > 0) {
+				const int chunk_frames =
+				    std::min<int>(static_cast<int>(discarded.size()) / frame_bytes,
+				                  static_cast<int>((static_cast<uint64_t>(frames) * target.freq +
+				                                    stream->freq - 1) /
+				                                   stream->freq));
+				while (queued + bytes > max_bytes && chunk_frames > 0 &&
+				       SDL_GetAudioStreamData(stream->sdl, discarded.data(),
+				                              chunk_frames * frame_bytes) > 0) {
+					queued = std::max(0, SDL_GetAudioStreamQueued(stream->sdl));
+				}
+			}
+		}
+		if (queued + bytes > max_bytes) {
+			SDL_ClearAudioStream(stream->sdl);
+			queued = 0;
+		}
+	}
+	if (!SDL_PutAudioStreamData(stream->sdl, stream->buffer.data(), bytes)) {
+		return 0;
+	}
+	return static_cast<uint64_t>(queued + bytes) * 1000000 / (stream->freq * FRAME_BYTES);
+}
+
 } // namespace
 
 Stream* Open(uint32_t freq, bool speaker) {
@@ -232,18 +306,30 @@ void Close(Stream* stream) {
 	}
 }
 
+bool UsesBluetooth(const Stream* stream) {
+	return stream != nullptr && stream->bluetooth != nullptr;
+}
+
 uint64_t Queue(Stream* stream, int controller, const void* data, uint32_t frames, uint32_t channels,
                bool is_float, const int* volume) {
 	if (stream == nullptr || data == nullptr || frames == 0 || channels == 0 || volume == nullptr) {
 		return 0;
 	}
-	if (!CanUseDevice(controller)) {
+	const auto max_frames =
+	    static_cast<uint32_t>(static_cast<uint64_t>(stream->freq) * MAX_QUEUE_MS / 1000);
+	const auto first_frame = frames > max_frames ? frames - max_frames : 0;
+	frames -= first_frame;
+	if (frames == 0) {
+		return 0;
+	}
+	const bool wireless = IsWireless(controller);
+	if (!wireless && !CanUseDevice(controller)) {
 		CloseDevice(stream);
 		stream->next_check = 0; // Look for the device as soon as a DualSense is active again.
 		return 0;
 	}
-	RefreshDevice(stream, controller);
-	if (stream->sdl == nullptr) {
+	RefreshDevice(stream, controller, wireless);
+	if (stream->sdl == nullptr && stream->bluetooth == nullptr) {
 		return 0;
 	}
 	if (stream->speaker) {
@@ -257,47 +343,42 @@ uint64_t Queue(Stream* stream, int controller, const void* data, uint32_t frames
 			return 0;
 		}
 	}
-	stream->buffer.assign(static_cast<size_t>(frames) * 4, 0.0f);
+	const uint32_t output_channels = wireless ? 2 : 4;
+	const uint32_t channel_offset  = wireless || stream->speaker ? 0 : 2;
+	stream->buffer.assign(static_cast<size_t>(frames) * output_channels, 0.0f);
 	bool audible = false;
 	for (uint32_t frame = 0; frame < frames; frame++) {
 		for (uint32_t ch = 0; ch < 2; ch++) {
 			const auto src_ch = channels == 1 ? 0 : ch;
-			const auto index  = static_cast<size_t>(frame) * channels + src_ch;
+			const auto index  = (static_cast<size_t>(first_frame) + frame) * channels + src_ch;
 			float      value  = is_float ? static_cast<const float*>(data)[index]
 			                             : static_cast<const int16_t*>(data)[index] / 32768.0f;
 			value *= volume[src_ch] / 32768.0f;
-			// The speaker plays the right front channel; the actuators play the back ones.
-			stream->buffer[static_cast<size_t>(frame) * 4 + ch + (stream->speaker ? 0 : 2)] = value;
-			audible |= std::fabs(value) > 1.0f / 1024;
+			// USB places the speaker in front and the actuators in back; Bluetooth takes stereo.
+			stream->buffer[static_cast<size_t>(frame) * output_channels + ch + channel_offset] =
+			    value;
+			audible |= std::isfinite(value) && std::fabs(value) > 1.0f / 1024;
 		}
 	}
-	auto queued = std::max(0, SDL_GetAudioStreamQueued(stream->sdl));
-	if (queued > static_cast<int>(stream->freq * FRAME_BYTES * 80 / 1000)) {
-		SDL_ClearAudioStream(stream->sdl);
-		queued = 0;
-	}
-	const int  bytes = static_cast<int>(frames * FRAME_BYTES);
 	const auto queued_us =
-	    static_cast<uint64_t>(queued + bytes) * 1000000 / (stream->freq * FRAME_BYTES);
+	    wireless ? DualSenseBluetooth::Queue(stream->bluetooth, stream->buffer.data(), frames)
+	             : QueueUsbAudio(stream, frames);
+	if (queued_us == 0) {
+		CloseDevice(stream);
+		stream->next_check = SDL_GetTicks() + 2000;
+		return 0;
+	}
 	if (audible && !stream->speaker) {
 		Common::LockGuard lock(g_mutex);
-		if (SDL_GetGamepadTypeForID(static_cast<SDL_JoystickID>(controller)) !=
-		    SDL_GAMEPAD_TYPE_PS5) {
-			return 0;
-		}
 		SelectController(controller);
-		const auto playback_now = SDL_GetTicks();
-		const bool starting     = g_haptics_until <= playback_now;
+		const auto now      = SDL_GetTicks();
+		const bool starting = g_haptics_until <= now;
 		g_haptics_until =
-		    std::max(g_haptics_until, playback_now + std::max<uint64_t>(queued_us / 1000, 250));
+		    std::max(g_haptics_until, now + std::max<uint64_t>(queued_us / 1000, 250));
 		if (starting) {
 			ApplyRumble();
 		}
 	}
-	if (!SDL_PutAudioStreamData(stream->sdl, stream->buffer.data(), bytes)) {
-		CloseDevice(stream);
-		return 0;
-	}
 	return queued_us;
 }
 
@@ -315,6 +396,7 @@ bool SetVibration(int controller, uint8_t large_motor, uint8_t small_motor) {
 }
 
 void Shutdown() {
+	DualSenseBluetooth::Shutdown();
 	Common::LockGuard lock(g_mutex);
 	SelectController(-1);
 	SelectSpeaker(-1);

src/libs/dualSenseHaptics.h

@@ -11,9 +11,11 @@ struct Stream;
 // ports on the actuators.
 Stream* Open(uint32_t freq, bool speaker);
 void    Close(Stream* stream);
+// True while this stream is routed through the asynchronous Bluetooth HID sender.
+bool UsesBluetooth(const Stream* stream);
 // Returns how long the queued audio still plays, in microseconds, or 0 when no DualSense took it
-// (the active pad has no usable USB audio device or its speaker cannot be routed).
-// Multiple DualSenses or matching audio endpoints are ambiguous and are not selected.
+// (the active pad has no usable controller audio device or its speaker cannot be routed).
+// Ambiguous USB or Bluetooth endpoints are not selected.
 uint64_t Queue(Stream* stream, int controller, const void* data, uint32_t frames, uint32_t channels,
                bool is_float, const int* volume);
 // Returns false for other gamepad types, which retain the normal rumble path.

src/libs/libKernel.cpp

@@ -1797,6 +1797,12 @@ int KYTY_SYSV_ABI KernelRtldThreadAtexitDecrement(uint64_t* /*c*/) {
 	return 0;
 }
 
+static uint64_t KYTY_SYSV_ABI KernelGetAvailableCpumask() {
+	PRINT_NAME();
+
+	return 0x1fff;
+}
+
 static KYTY_SYSV_ABI int KernelGetCurrentCpu() {
 	PRINT_NAME();
 
@@ -3340,6 +3346,7 @@ LIB_DEFINE(InitLibKernel_1) {
 	LIB_FUNC("vYU8P9Td2Zo", KernelAioInitializeImpl);
 	LIB_FUNC("WhCc1w3EhSI", LibKernel::KernelSetThreadAtexitReport);
 	LIB_FUNC("WkwEd3N7w0Y", LibKernel::KernelInstallExceptionHandler);
+	LIB_FUNC("La9uyZv4Kvw", LibKernel::KernelGetAvailableCpumask);
 	LIB_FUNC("g0VTBxfJyu0", LibKernel::KernelGetCurrentCpu);
 	LIB_FUNC("wzvqT4UqKX8", LibKernel::KernelLoadStartModule);
 	LIB_FUNC("Xjoosiw+XPI", LibKernel::KernelUuidCreate);

src/libs/network.cpp

@@ -997,14 +997,31 @@ static int ConvertHostSocketError(int error) {
 		case EAGAIN: posix_error = Posix::POSIX_EWOULDBLOCK; break;
 		case EBADF: posix_error = Posix::POSIX_EBADF; break;
 		case EFAULT: posix_error = Posix::POSIX_EFAULT; break;
+		case EINTR: posix_error = Posix::POSIX_EINTR; break;
 		case EINVAL: posix_error = Posix::POSIX_EINVAL; break;
+		case EISCONN: posix_error = Posix::POSIX_EISCONN; break;
 		case EMFILE: posix_error = Posix::POSIX_EMFILE; break;
+		case EMSGSIZE: posix_error = Posix::POSIX_EMSGSIZE; break;
 		case ENFILE: posix_error = Posix::POSIX_ENFILE; break;
 		case ENOBUFS: posix_error = Posix::POSIX_ENOBUFS; break;
 		case ENOMEM: posix_error = Posix::POSIX_ENOMEM; break;
+		case ENETDOWN: posix_error = Posix::POSIX_ENETDOWN; break;
+		case ENETRESET: posix_error = Posix::POSIX_ENETRESET; break;
+		case ENETUNREACH: posix_error = Posix::POSIX_ENETUNREACH; break;
+		case ENOTCONN: posix_error = Posix::POSIX_ENOTCONN; break;
 		case ENOTSOCK: posix_error = Posix::POSIX_ENOTSOCK; break;
+		case EOPNOTSUPP: posix_error = Posix::POSIX_EOPNOTSUPP; break;
 		case EPIPE: posix_error = Posix::POSIX_EPIPE; break;
 		case EPROTONOSUPPORT: posix_error = Posix::POSIX_EPROTONOSUPPORT; break;
+		case ESHUTDOWN: posix_error = Posix::POSIX_ESHUTDOWN; break;
+		case ETIMEDOUT: posix_error = Posix::POSIX_ETIMEDOUT; break;
+		case ECONNABORTED: posix_error = Posix::POSIX_ECONNABORTED; break;
+		case ECONNREFUSED: posix_error = Posix::POSIX_ECONNREFUSED; break;
+		case ECONNRESET: posix_error = Posix::POSIX_ECONNRESET; break;
+		case EDESTADDRREQ: posix_error = Posix::POSIX_EDESTADDRREQ; break;
+		case EHOSTUNREACH: posix_error = Posix::POSIX_EHOSTUNREACH; break;
+		case EINPROGRESS: posix_error = Posix::POSIX_EINPROGRESS; break;
+		case EALREADY: posix_error = Posix::POSIX_EALREADY; break;
 		default: break;
 	}
 #endif
@@ -2098,17 +2115,22 @@ int KYTY_SYSV_ABI Setsockopt(int s, int level, int optname, const void* optval,
 		return 0;
 	}
 
-#if defined(_WIN32)
 	constexpr int ORBIS_SO_NBIO = 0x1200;
 	if (ConvertSocketOptionLevel(level) == SOL_SOCKET && optname == ORBIS_SO_NBIO &&
 	    optlen >= sizeof(int)) {
-		u_long enabled = (*static_cast<const int*>(optval) != 0 ? 1 : 0);
-		if (ioctlsocket(socket, FIONBIO, &enabled) == SOCKET_ERROR) {
-			return SetHostSocketError();
-		}
-		return 0;
-	}
+		const bool enabled = *static_cast<const int*>(optval) != 0;
+#if defined(_WIN32)
+		u_long     mode   = enabled ? 1 : 0;
+		const bool failed = ioctlsocket(socket, FIONBIO, &mode) == SOCKET_ERROR;
 #else
+		const int  flags = ::fcntl(socket, F_GETFL, 0);
+		const bool failed =
+		    flags < 0 ||
+		    ::fcntl(socket, F_SETFL, enabled ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK)) != 0;
+#endif
+		return failed ? SetHostSocketError() : 0;
+	}
+#if !defined(_WIN32)
 	// Guest TCP options: IPPROTO_TCP=6, TCP_NODELAY=1.
 	if (level != 6 || optname != 1) {
 		return SetGuestSocketError(Posix::POSIX_ENOPROTOOPT);

tests/AudioOutTimingTests.cpp

@@ -0,0 +1,401 @@
+// Exercise production output scheduling with a deterministic clock and consuming SDL queues.
+#include "common/threads.h"
+#include "libs/dualSenseHaptics.h"
+
+#include <SDL3/SDL.h>
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+
+namespace {
+void Check(bool condition, const char* message) {
+	if (!condition) {
+		std::fprintf(stderr, "AudioOutTimingTests: %s\n", message);
+		std::abort();
+	}
+}
+
+struct Stream {
+	SDL_AudioSpec spec;
+	uint64_t      frames = 0; // Millionths of a frame retain exact simulated consumption.
+	uint64_t      updated;
+};
+
+uint64_t                             now       = 1000000;
+uint32_t                             oversleep = 0, processing = 0;
+std::vector<uint32_t>                sleeps;
+std::vector<std::unique_ptr<Stream>> streams;
+bool                                 fail_open = false, fail_put = false, stalled = false;
+bool                                 pad_connected = false, pad_bluetooth = false;
+uint64_t                             pad_queue_us = 0;
+int                                  clears = 0;
+
+Stream& GetStream(SDL_AudioStream* stream) {
+	return *reinterpret_cast<Stream*>(stream);
+}
+
+void Drain(Stream& stream) {
+	if (!stalled) {
+		const auto consumed = (now - stream.updated) * stream.spec.freq;
+		stream.frames -= std::min(stream.frames, consumed);
+	}
+	stream.updated = now;
+}
+} // namespace
+
+namespace Fake {
+bool InitSubSystem(SDL_InitFlags) {
+	return true;
+}
+void QuitSubSystem(SDL_InitFlags) {}
+bool ResumeAudioStreamDevice(SDL_AudioStream*) {
+	return true;
+}
+
+SDL_AudioStream* OpenAudioDeviceStream(SDL_AudioDeviceID, const SDL_AudioSpec* spec,
+                                       SDL_AudioStreamCallback, void*) {
+	if (fail_open) {
+		return nullptr;
+	}
+
+	streams.push_back(std::make_unique<Stream>(Stream {*spec, 0, now}));
+	return reinterpret_cast<SDL_AudioStream*>(streams.back().get());
+}
+
+void DestroyAudioStream(SDL_AudioStream* stream) {
+	const auto it = std::find_if(streams.begin(), streams.end(), [stream](const auto& value) {
+		return value.get() == &GetStream(stream);
+	});
+	Check(it != streams.end(), "unknown stream destroyed");
+	streams.erase(it);
+}
+
+int GetAudioStreamQueued(SDL_AudioStream* stream) {
+	auto& state = GetStream(stream);
+	Drain(state);
+	return static_cast<int>(state.frames / 1000000 * SDL_AUDIO_BYTESIZE(state.spec.format) *
+	                        state.spec.channels);
+}
+
+bool PutAudioStreamData(SDL_AudioStream* stream, const void*, int bytes) {
+	now += processing;
+	if (fail_put) {
+		return false;
+	}
+	auto& state = GetStream(stream);
+	Drain(state);
+	state.frames += static_cast<uint64_t>(bytes) * 1000000 /
+	                (SDL_AUDIO_BYTESIZE(state.spec.format) * state.spec.channels);
+	return true;
+}
+
+bool ClearAudioStream(SDL_AudioStream* stream) {
+	auto& state   = GetStream(stream);
+	state.frames  = 0;
+	state.updated = now;
+	clears++;
+	return true;
+}
+} // namespace Fake
+
+namespace Common {
+class TimingThread: public Thread {
+public:
+	static void SleepMicro(uint32_t micros) {
+		sleeps.push_back(micros);
+		now += micros + oversleep;
+	}
+};
+} // namespace Common
+
+#define Thread                      TimingThread
+#define SDL_InitSubSystem           Fake::InitSubSystem
+#define SDL_QuitSubSystem           Fake::QuitSubSystem
+#define SDL_OpenAudioDeviceStream   Fake::OpenAudioDeviceStream
+#define SDL_ResumeAudioStreamDevice Fake::ResumeAudioStreamDevice
+#define SDL_DestroyAudioStream      Fake::DestroyAudioStream
+#define SDL_GetAudioStreamQueued    Fake::GetAudioStreamQueued
+#define SDL_PutAudioStreamData      Fake::PutAudioStreamData
+#define SDL_ClearAudioStream        Fake::ClearAudioStream
+#include "libs/audio.cpp"
+#undef Thread
+#undef SDL_InitSubSystem
+#undef SDL_QuitSubSystem
+#undef SDL_OpenAudioDeviceStream
+#undef SDL_ResumeAudioStreamDevice
+#undef SDL_DestroyAudioStream
+#undef SDL_GetAudioStreamQueued
+#undef SDL_PutAudioStreamData
+#undef SDL_ClearAudioStream
+
+namespace Libs::Controller {
+int GetActiveControllerId() {
+	return 0;
+}
+
+namespace DualSenseHaptics {
+Stream* Open(uint32_t, bool) {
+	static int pad_stream;
+	return pad_connected ? reinterpret_cast<Stream*>(&pad_stream) : nullptr;
+}
+void     Close(Stream*) {}
+bool UsesBluetooth(const Stream* stream) {
+	return stream != nullptr && pad_bluetooth;
+}
+uint64_t Queue(Stream* stream, int, const void*, uint32_t, uint32_t, bool, const int*) {
+	return stream != nullptr ? pad_queue_us : 0;
+}
+} // namespace DualSenseHaptics
+} // namespace Libs::Controller
+
+namespace Libs::LibKernel {
+uint64_t KYTY_SYSV_ABI KernelGetProcessTime() {
+	return now;
+}
+} // namespace Libs::LibKernel
+
+namespace Loader::Timer {
+double GetTimeMs() {
+	return static_cast<double>(now) / 1000;
+}
+} // namespace Loader::Timer
+
+namespace {
+using Audio = Libs::Audio::Audio;
+
+struct Fixture {
+	Audio                   audio;
+	std::array<float, 2048> pcm {};
+
+	Fixture() {
+		Check(streams.empty(), "stream leaked between tests");
+		now        = 1000000;
+		processing = oversleep = 0;
+		clears                 = 0;
+		fail_open = fail_put = stalled = false;
+		pad_connected = pad_bluetooth = false;
+		pad_queue_us = 0;
+		sleeps.clear();
+	}
+
+	Audio::Id Open(uint32_t frames = 256, int type = 0) {
+		const auto id = audio.AudioOutOpen(type, frames, 48000, Audio::Format::FloatStereo);
+		Check(id.IsValid(), "port open failed");
+		return id;
+	}
+
+	void Output(Audio::Id port, bool blocking = true) {
+		Audio::OutputParam param {port, pcm.data()};
+		audio.AudioOutOutputs(&param, 1, blocking);
+	}
+
+	void Prime(Audio::Id port) {
+		const auto start = now;
+		for (int i = 0; i < 8; i++) {
+			Output(port);
+		}
+		Check(now == start, "priming was paced before the 40 ms cushion filled");
+		Output(port);
+		Check(now - start >= 5333 && now - start <= 5334,
+		      "priming accumulated a multi-block deadline");
+		Drain(*streams.front());
+		Check(streams.front()->frames >= uint64_t {7 * 256} * 1000000,
+		      "first paced output consumed the priming cushion");
+	}
+};
+
+void TestPrimingAndSteadyCadence() {
+	Fixture    f;
+	const auto port = f.Open();
+	f.Prime(port);
+	processing = 125;
+	oversleep  = 175;
+	for (int i = 0; i < 10; i++) {
+		f.Output(port);
+	}
+	const auto start = now;
+	for (int i = 0; i < 3000; i++) {
+		f.Output(port);
+	}
+	Check(now - start >= 15999999 && now - start <= 16000001,
+	      "processing, sleep overshoot, or fractional periods accumulated timing drift");
+}
+
+void TestUnderrunAndStalledQueueRecovery() {
+	Fixture    f;
+	const auto port = f.Open();
+	f.Prime(port);
+	now += 1000000;
+	f.Prime(port);
+
+	// A stopped device must clear its stale queue and restart priming, not catch up old deadlines.
+	stalled = true;
+	for (int i = 0; i < 3 && clears == 0; i++) {
+		f.Output(port);
+	}
+	Check(clears == 1, "stalled playback queue was not cleared");
+	stalled          = false;
+	const auto start = now;
+	for (int i = 0; i < 7; i++) {
+		f.Output(port);
+	}
+	Check(now == start, "queue clear did not restart unpaced priming");
+	f.Output(port);
+	Check(now - start >= 5333 && now - start <= 5334,
+	      "queue clear retained an old pacing deadline");
+}
+
+void TestFallbackClockAndLongGap() {
+	Fixture f;
+	fail_open       = true;
+	const auto port = f.Open();
+	f.Output(port);
+	const auto start = now;
+	oversleep        = 175;
+	for (int i = 0; i < 3000; i++) {
+		now += 125;
+		f.Output(port);
+	}
+	Check(now - start >= 16000174 && now - start <= 16000176,
+	      "device-free output accumulated timing drift");
+	now += 1000000;
+	f.Output(port);
+	const auto resumed = now;
+	f.Output(port);
+	Check(now - resumed >= 5333 && now - resumed <= 5509,
+	      "long interruption caused a burst of device-free catch-up submissions");
+}
+
+void TestAsyncDoesNotAccumulateDeadlines() {
+	Fixture    f;
+	const auto port  = f.Open();
+	const auto start = now;
+	for (int i = 0; i < 8; i++) {
+		f.Output(port, false);
+	}
+	Check(now == start, "asynchronous output slept");
+	f.Output(port);
+	Check(now - start <= 5334, "asynchronous submissions accumulated a blocking deadline");
+	for (int i = 0; i < 3; i++) {
+		f.Output(port);
+	}
+	now += 1000000;
+	f.Output(port, false);
+	const auto resumed = now;
+	for (int i = 0; i < 7; i++) {
+		f.Output(port);
+	}
+	Check(now == resumed, "async-to-blocking transition did not reset an emptied queue");
+}
+
+void TestSynchronizedBatchAndInactivePorts() {
+	Fixture            f;
+	const auto         vibration  = f.Open(256, 10);
+	const auto         main       = f.Open();
+	const auto         background = f.Open();
+	Audio::OutputParam batch[] {
+	    {vibration, f.pcm.data()}, {main, f.pcm.data()}, {background, f.pcm.data()}};
+	const auto start = now;
+	for (int i = 0; i < 8; i++) {
+		f.audio.AudioOutOutputs(batch, 3);
+	}
+	Check(now == start, "vibration fallback slowed a device-backed batch's priming");
+	for (int i = 0; i < 3; i++) {
+		sleeps.clear();
+		f.audio.AudioOutOutputs(batch, 3);
+		Check(sleeps.size() == 1 && sleeps.front() >= 5333 && sleeps.front() <= 5334,
+		      "synchronized streams accumulated separate pacing waits");
+	}
+
+	Audio::OutputParam inactive[] {{main, nullptr}, {vibration, f.pcm.data()}};
+	now += 1000000;
+	f.audio.AudioOutOutputs(inactive, 2);
+	const auto resumed = now;
+	f.audio.AudioOutOutputs(inactive, 2);
+	Check(now - resumed >= 5333 && now - resumed <= 5334,
+	      "a device without PCM disabled fallback pacing");
+}
+
+void TestFallbackUsesEachPortsPeriod() {
+	Fixture f;
+	fail_open                     = true;
+	const auto         short_port = f.Open(128);
+	const auto         long_port  = f.Open(256);
+	Audio::OutputParam batch[] {{short_port, f.pcm.data()}, {long_port, f.pcm.data()}};
+	f.audio.AudioOutOutputs(batch, 2);
+	const auto start = now;
+	f.audio.AudioOutOutputs(batch, 2);
+	Check(now - start >= 5333 && now - start <= 5334,
+	      "fallback batch used the first port's period for every port");
+}
+
+void TestFailedQueueUsesFallbackClock() {
+	Fixture    f;
+	const auto port = f.Open();
+	fail_put        = true;
+	f.Output(port);
+	const auto start = now;
+	for (int i = 0; i < 3; i++) {
+		f.Output(port);
+	}
+	Check(now - start == 16000, "repeated queue failures reset fallback pacing");
+}
+
+void TestControllerSpeakerPacing() {
+	Fixture f;
+	fail_open     = true; // No PC audio device paces the pad speaker.
+	pad_connected = true;
+	pad_queue_us  = 80000;
+	const auto speaker = f.Open(256, 4);
+
+	pad_bluetooth = true;
+	f.Output(speaker);
+	const auto bluetooth_start = now;
+	f.Output(speaker);
+	Check(now - bluetooth_start >= 5333 && now - bluetooth_start <= 5334,
+	      "Bluetooth speaker waited on its HID queue instead of the sample clock");
+
+	pad_bluetooth = false;
+	const auto usb_start = now;
+	f.Output(speaker);
+	Check(now - usb_start >= 5333 && now - usb_start <= 5334,
+	      "stalled USB speaker waited longer than one audio block");
+}
+
+void TestInvalidBatchSize() {
+	Libs::Audio::AudioOut::AudioOutOutputParam param {};
+	for (const auto count: {0u, 33u}) {
+		Check(Libs::Audio::AudioOut::AudioOutOutputs(&param, count) ==
+		          Libs::Audio::AUDIO_OUT_ERROR_INVALID_SIZE,
+		      "invalid batch size was not rejected before accessing ports");
+	}
+}
+
+void TestZeroOutputFrequency() {
+	Fixture f;
+	Check(Libs::Audio::AudioOut::AudioOutOpen(0, 0, 0, 256, 0, 4) ==
+	          Libs::Audio::AUDIO_OUT_ERROR_INVALID_SAMPLE_FREQ,
+	      "public output open accepted zero frequency");
+	Check(!f.audio.AudioOutOpen(10, 256, 0, Audio::Format::FloatStereo).IsValid(),
+	      "internal output open accepted zero frequency");
+	Check(f.Open().ToInt() == 1, "rejected output port occupied a handle");
+}
+} // namespace
+
+int main() {
+	TestPrimingAndSteadyCadence();
+	TestUnderrunAndStalledQueueRecovery();
+	TestFallbackClockAndLongGap();
+	TestAsyncDoesNotAccumulateDeadlines();
+	TestSynchronizedBatchAndInactivePorts();
+	TestFallbackUsesEachPortsPeriod();
+	TestFailedQueueUsesFallbackClock();
+	TestControllerSpeakerPacing();
+	TestInvalidBatchSize();
+	TestZeroOutputFrequency();
+	Check(streams.empty(), "output stream leaked");
+	std::puts("AudioOutTimingTests: all cases passed");
+}

tests/MemoryTrackerTests.cpp

@@ -238,14 +238,31 @@ struct TrackerHarness {
   MemoryTracker tracker;
 };
 
+uint8_t *AllocateFixedGuestRange(uint64_t size, uintptr_t offset) {
+  // Keep the mapping inside the tracker's guest address space and preserve the
+  // requested region alignment. A fixed address can be occupied by the host
+  // process (notably by the macOS runner's ASLR layout).
+  constexpr uintptr_t first_base = 0x0000000200000000ull;
+  constexpr uintptr_t stride = 0x0000000100000000ull;
+  for (uintptr_t attempt = 0; attempt < 256; attempt++) {
+    auto *wanted = reinterpret_cast<void *>(first_base + offset + attempt * stride);
+    auto *memory = static_cast<uint8_t *>(
+        VirtualAlloc(wanted, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+    if (memory == wanted) {
+      return memory;
+    }
+    if (memory != nullptr) {
+      Check(VirtualFree(memory, 0, MEM_RELEASE) != 0,
+            "release unexpected fixed allocation failed");
+    }
+  }
+  Check(false, "no free fixed guest address found");
+  return nullptr;
+}
+
 uint8_t *Allocate(PageManager &manager, uint64_t pages) {
-  constexpr uintptr_t base = 0x0000000200010000ull;
   const auto size = manager.GetPageSize() * pages;
-  auto *memory = static_cast<uint8_t *>(
-      VirtualAlloc(reinterpret_cast<void *>(base), size,
-                   MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
-  Check(memory == reinterpret_cast<void *>(base), "fixed VirtualAlloc failed");
-  return memory;
+  return AllocateFixedGuestRange(size, 0x10000);
 }
 
 void Release(uint8_t *memory) {
@@ -386,16 +403,10 @@ void TestCpuDirtyUpload() {
 }
 
 void TestRangeInvalidation() {
-  constexpr uintptr_t base = 0x0000000201000000ull;
   TrackerHarness harness;
   auto &tracker = harness.tracker;
-  auto &page_manager = harness.page_manager;
   constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2;
-  auto *memory = static_cast<uint8_t *>(
-      VirtualAlloc(reinterpret_cast<void *>(base), size,
-                   MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
-  Check(memory == reinterpret_cast<void *>(base),
-        "range invalidation allocation failed");
+  auto *memory = AllocateFixedGuestRange(size, 0x1000000);
   const auto address = reinterpret_cast<uint64_t>(memory);
 
   tracker.ForEachUploadRange(
@@ -614,16 +625,12 @@ void TestGpuDownloadProtectionMirrors() {
 }
 
 void TestCrossRegionUpload() {
-  constexpr uintptr_t base = 0x0000000200010000ull;
   constexpr uint64_t region_size = 4ull * 1024ull * 1024ull;
   TrackerHarness harness;
   auto &tracker = harness.tracker;
   auto &page_manager = harness.page_manager;
   const auto page_size = page_manager.GetPageSize();
-  auto *memory = static_cast<uint8_t *>(
-      VirtualAlloc(reinterpret_cast<void *>(base), region_size * 2,
-                   MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
-  Check(memory == reinterpret_cast<void *>(base), "fixed VirtualAlloc failed");
+  auto *memory = AllocateFixedGuestRange(region_size * 2, 0x10000);
   const auto address = reinterpret_cast<uint64_t>(memory);
   const auto boundary = (address + region_size - 1) & ~(region_size - 1);
   uint32_t ranges = 0;
@@ -871,6 +878,15 @@ void TestFullRegionGpuUnmarkBatching() {
         [&]() noexcept {
           (void)tracker.IsRegionCpuModified(address, page_size);
         });
+  } else if (std::strcmp(name, "recursive-tracking-lock") == 0) {
+    Libs::Graphics::TrackingSpinLock lock;
+    lock.lock();
+    lock.lock();
+  } else if (std::strcmp(name, "non-owner-tracking-unlock") == 0) {
+    Libs::Graphics::TrackingSpinLock lock;
+    lock.lock();
+    std::thread worker([&] { lock.unlock(); });
+    worker.join();
   }
   std::_Exit(0x7f);
 }
@@ -928,7 +944,8 @@ void CheckDeathCase(const char *name) {
 }
 
 void TestFatalPaths() {
-  for (const char *name : {"gpu-dirty-explicit-cpu", "reentrant-upload"}) {
+  for (const char *name : {"gpu-dirty-explicit-cpu", "reentrant-upload",
+                           "recursive-tracking-lock", "non-owner-tracking-unlock"}) {
     CheckDeathCase(name);
   }
 }

tests/PadHapticsTests.cpp

@@ -1,11 +1,17 @@
 // Exercise the production module with real SDL streams and fake USB endpoints.
 #include <SDL3/SDL.h>
+#include <opus.h>
 #include <algorithm>
 #include <array>
+#include <atomic>
+#include <chrono>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
 #include <memory>
+#include <numeric>
+#include <semaphore>
+#include <thread>
 #include <vector>
 
 namespace {
@@ -24,23 +30,32 @@ struct Rumble {
 	Uint16 large, small;
 	Uint32 duration;
 };
-std::vector<Device>                 devices;
-std::vector<SDL_AudioStream*>       streams;
-Rumble                              rumble {};
-Uint64                              now        = 1000;
-int                                 audio_refs = 0, opens = 0, rumble_calls = 0;
-bool                                fail_open = false, fail_resume = false;
-int                                 actual_channels = 4;
-SDL_AudioDeviceID                   opened_device   = 0;
-SDL_AudioSpec                       opened_spec {};
-std::array<Rumble, 4>               pad_rumble {};
-std::array<std::vector<uint8_t>, 4> effects;
-std::vector<SDL_JoystickID>         connected_pads {1};
-std::array<Rumble, 4>               cached_rumble {};
-bool                                fail_effect = false, wireless = false;
-SDL_AudioStream*                   default_stream = nullptr;
-bool                               fail_default_resume = false;
-int                                active_controller = 1;
+std::vector<Device>                   devices;
+std::vector<SDL_AudioStream*>         streams;
+Rumble                                rumble {};
+Uint64                                now        = 1000;
+int                                   audio_refs = 0, opens = 0, rumble_calls = 0;
+bool                                  fail_open = false, fail_resume = false;
+int                                   actual_channels = 4;
+SDL_AudioDeviceID                     opened_device   = 0;
+SDL_AudioSpec                         opened_spec {};
+std::array<Rumble, 4>                 pad_rumble {};
+std::array<std::vector<uint8_t>, 4>   effects;
+std::vector<SDL_JoystickID>           connected_pads {1};
+std::array<Rumble, 4>                 cached_rumble {};
+bool                                  fail_effect = false, wireless = false, wireless_pad3 = false;
+bool                                  hid_available = false, fail_hid_write = false;
+bool                                  duplicate_hid  = false;
+int                                   hid_opens      = 0;
+int                                   opened_hid_pad = 0;
+std::vector<std::array<uint8_t, 547>> hid_reports;
+std::atomic_bool                      block_opus_encode {false}, block_hid_write {false};
+std::atomic_int                       hid_closes {0};
+std::binary_semaphore                 encode_started {0}, allow_encode {0};
+std::binary_semaphore                 write_started {0}, allow_write {0};
+SDL_AudioStream*                      default_stream      = nullptr;
+bool                                  fail_default_resume = false;
+int                                   active_controller   = 1;
 } // namespace
 
 namespace Fake {
@@ -116,8 +131,9 @@ void DestroyAudioStream(SDL_AudioStream* stream) {
 		return;
 	}
 	const auto it = std::find(streams.begin(), streams.end(), stream);
-	Check(it != streams.end(), "stream destroyed twice");
-	streams.erase(it);
+	if (it != streams.end()) {
+		streams.erase(it);
+	}
 	if (stream == default_stream) {
 		default_stream = nullptr;
 	}
@@ -134,6 +150,73 @@ SDL_Gamepad* GetGamepadFromID(SDL_JoystickID id) {
 	return id == 1 || id == 3 ? reinterpret_cast<SDL_Gamepad*>(static_cast<uintptr_t>(id))
 	                          : nullptr;
 }
+SDL_GamepadType GetGamepadType(SDL_Gamepad* pad) {
+	return GetGamepadTypeForID(static_cast<SDL_JoystickID>(reinterpret_cast<uintptr_t>(pad)));
+}
+Uint16 GetGamepadVendor(SDL_Gamepad*) {
+	return 0x054c;
+}
+Uint16 GetGamepadProduct(SDL_Gamepad*) {
+	return 0x0ce6;
+}
+const char* GetGamepadSerial(SDL_Gamepad* pad) {
+	return reinterpret_cast<uintptr_t>(pad) == 3 ? "aa-bb-cc-dd-ee-03" : "aa-bb-cc-dd-ee-01";
+}
+Uint64 GetTicksNS() {
+	return now * 1000000;
+}
+SDL_Thread* CreateThread(SDL_ThreadFunction, const char*, void*) {
+	return reinterpret_cast<SDL_Thread*>(1);
+}
+void                 WaitThread(SDL_Thread*, int*) {}
+SDL_hid_device_info* HidEnumerate(unsigned short, unsigned short) {
+	if (!hid_available) {
+		return nullptr;
+	}
+	static wchar_t             serial1[] = L"AA:BB:CC:DD:EE:01";
+	static wchar_t             serial3[] = L"AA:BB:CC:DD:EE:03";
+	static char                path1[]   = "pad-1";
+	static char                path3[]   = "pad-3";
+	static SDL_hid_device_info pad3 {}, pad1 {};
+	pad3.path          = path3;
+	pad3.serial_number = duplicate_hid ? serial1 : serial3;
+	pad3.bus_type      = SDL_HID_API_BUS_BLUETOOTH;
+	pad1.path          = path1;
+	pad1.serial_number = serial1;
+	pad1.bus_type      = SDL_HID_API_BUS_BLUETOOTH;
+	pad1.next          = &pad3;
+	return &pad1;
+}
+void            HidFreeEnumeration(SDL_hid_device_info*) {}
+SDL_hid_device* HidOpenPath(const char* path) {
+	hid_opens++;
+	opened_hid_pad = path[4] == '1' ? 1 : 3;
+	return reinterpret_cast<SDL_hid_device*>(static_cast<uintptr_t>(opened_hid_pad));
+}
+void HidClose(SDL_hid_device*) {
+	hid_closes++;
+}
+int HidWrite(SDL_hid_device*, const unsigned char* data, size_t size) {
+	if (block_hid_write.exchange(false)) {
+		write_started.release();
+		allow_write.acquire();
+	}
+	if (fail_hid_write || size != 547) {
+		return -1;
+	}
+	std::array<uint8_t, 547> report {};
+	std::copy_n(data, report.size(), report.begin());
+	hid_reports.push_back(report);
+	return static_cast<int>(size);
+}
+opus_int32 OpusEncodeFloat(OpusEncoder* encoder, const float* pcm, int frames,
+                           unsigned char* packet, opus_int32 bytes) {
+	if (block_opus_encode.exchange(false)) {
+		encode_started.release();
+		allow_encode.acquire();
+	}
+	return ::opus_encode_float(encoder, pcm, frames, packet, bytes);
+}
 SDL_JoystickID* GetGamepads(int* count) {
 	*count    = static_cast<int>(connected_pads.size());
 	auto* ids = static_cast<SDL_JoystickID*>(
@@ -142,8 +225,10 @@ SDL_JoystickID* GetGamepads(int* count) {
 	ids[connected_pads.size()] = 0;
 	return ids;
 }
-SDL_JoystickConnectionState GetGamepadConnectionState(SDL_Gamepad*) {
-	return wireless ? SDL_JOYSTICK_CONNECTION_WIRELESS : SDL_JOYSTICK_CONNECTION_WIRED;
+SDL_JoystickConnectionState GetGamepadConnectionState(SDL_Gamepad* pad) {
+	return wireless || (wireless_pad3 && reinterpret_cast<uintptr_t>(pad) == 3)
+	           ? SDL_JOYSTICK_CONNECTION_WIRELESS
+	           : SDL_JOYSTICK_CONNECTION_WIRED;
 }
 bool RumbleGamepad(SDL_Gamepad* pad, Uint16 large, Uint16 small, Uint32 duration) {
 	const auto id = reinterpret_cast<uintptr_t>(pad);
@@ -173,22 +258,37 @@ bool SendGamepadEffect(SDL_Gamepad* pad, const void* data, int size) {
 }
 } // namespace Fake
 
-#define SDL_GetTicks                  Fake::GetTicks
-#define SDL_InitSubSystem             Fake::InitSubSystem
-#define SDL_QuitSubSystem             Fake::QuitSubSystem
-#define SDL_GetAudioPlaybackDevices   Fake::GetAudioPlaybackDevices
-#define SDL_GetAudioDeviceName        Fake::GetAudioDeviceName
-#define SDL_GetAudioDeviceFormat      Fake::GetAudioDeviceFormat
-#define SDL_OpenAudioDeviceStream     Fake::OpenAudioDeviceStream
-#define SDL_ResumeAudioStreamDevice   Fake::ResumeAudioStreamDevice
-#define SDL_DestroyAudioStream        Fake::DestroyAudioStream
-#define SDL_GetAudioStreamDevice      Fake::GetAudioStreamDevice
-#define SDL_GetGamepadTypeForID       Fake::GetGamepadTypeForID
+#define SDL_GetTicks                Fake::GetTicks
+#define SDL_InitSubSystem           Fake::InitSubSystem
+#define SDL_QuitSubSystem           Fake::QuitSubSystem
+#define SDL_GetAudioPlaybackDevices Fake::GetAudioPlaybackDevices
+#define SDL_GetAudioDeviceName      Fake::GetAudioDeviceName
+#define SDL_GetAudioDeviceFormat    Fake::GetAudioDeviceFormat
+#define SDL_OpenAudioDeviceStream   Fake::OpenAudioDeviceStream
+#define SDL_ResumeAudioStreamDevice Fake::ResumeAudioStreamDevice
+#define SDL_DestroyAudioStream      Fake::DestroyAudioStream
+#define SDL_GetAudioStreamDevice    Fake::GetAudioStreamDevice
+#define SDL_GetGamepadTypeForID     Fake::GetGamepadTypeForID
+#define SDL_GetGamepadType          Fake::GetGamepadType
+#define SDL_GetGamepadVendor        Fake::GetGamepadVendor
+#define SDL_GetGamepadProduct       Fake::GetGamepadProduct
+#define SDL_GetGamepadSerial        Fake::GetGamepadSerial
+#define SDL_GetTicksNS              Fake::GetTicksNS
+#undef SDL_CreateThread
+#define SDL_CreateThread              Fake::CreateThread
+#define SDL_WaitThread                Fake::WaitThread
+#define SDL_hid_enumerate             Fake::HidEnumerate
+#define SDL_hid_free_enumeration      Fake::HidFreeEnumeration
+#define SDL_hid_open_path             Fake::HidOpenPath
+#define SDL_hid_close                 Fake::HidClose
+#define SDL_hid_write                 Fake::HidWrite
+#define opus_encode_float             Fake::OpusEncodeFloat
 #define SDL_GetGamepadFromID          Fake::GetGamepadFromID
 #define SDL_RumbleGamepad             Fake::RumbleGamepad
 #define SDL_SendGamepadEffect         Fake::SendGamepadEffect
 #define SDL_GetGamepads               Fake::GetGamepads
 #define SDL_GetGamepadConnectionState Fake::GetGamepadConnectionState
+#include "libs/dualSenseBluetooth.cpp"
 #include "libs/dualSenseHaptics.cpp"
 #include "libs/audio.cpp"
 #undef SDL_GetTicks
@@ -202,6 +302,19 @@ bool SendGamepadEffect(SDL_Gamepad* pad, const void* data, int size) {
 #undef SDL_DestroyAudioStream
 #undef SDL_GetAudioStreamDevice
 #undef SDL_GetGamepadTypeForID
+#undef SDL_GetGamepadType
+#undef SDL_GetGamepadVendor
+#undef SDL_GetGamepadProduct
+#undef SDL_GetGamepadSerial
+#undef SDL_GetTicksNS
+#undef SDL_CreateThread
+#undef SDL_WaitThread
+#undef SDL_hid_enumerate
+#undef SDL_hid_free_enumeration
+#undef SDL_hid_open_path
+#undef SDL_hid_close
+#undef SDL_hid_write
+#undef opus_encode_float
 #undef SDL_GetGamepadFromID
 #undef SDL_RumbleGamepad
 #undef SDL_SendGamepadEffect
@@ -228,6 +341,7 @@ double GetTimeMs() {
 
 namespace {
 namespace Haptics = Libs::Controller::DualSenseHaptics;
+namespace Bluetooth = Libs::Controller::DualSenseBluetooth;
 using Port        = std::unique_ptr<Haptics::Stream, decltype(&Haptics::Close)>;
 constexpr std::array<int, 2>   unity {32768, 32768};
 constexpr std::array<float, 4> pcm {0.5f, 0.5f, 0.5f, 0.5f};
@@ -240,11 +354,17 @@ struct Fixture {
 		cached_rumble  = {};
 		effects        = {};
 		connected_pads = {1};
-		fail_effect = wireless = false;
-		default_stream = nullptr;
+		fail_effect = wireless = wireless_pad3 = false;
+		hid_available = fail_hid_write = false;
+		duplicate_hid                  = false;
+		hid_opens                      = 0;
+		hid_closes                     = 0;
+		opened_hid_pad                 = 0;
+		hid_reports.clear();
+		default_stream      = nullptr;
 		fail_default_resume = false;
-		active_controller = 1;
-		now                    = 1000;
+		active_controller   = 1;
+		now                 = 1000;
 		opens = rumble_calls = 0;
 		fail_open = fail_resume = false;
 		actual_channels         = 4;
@@ -255,8 +375,8 @@ struct Fixture {
 	}
 };
 
-Port Open(bool speaker = false) {
-	auto* stream = Haptics::Open(48000, speaker);
+Port Open(bool speaker = false, uint32_t freq = 48000) {
+	auto* stream = Haptics::Open(freq, speaker);
 	Check(stream != nullptr, "port open failed");
 	return Port(stream, Haptics::Close);
 }
@@ -516,6 +636,446 @@ void TestSpeakerRequiresUnambiguousUsbDevice() {
 	      "USB to Bluetooth change retained the USB speaker");
 }
 
+void TestBluetoothAudioAndIdentity() {
+	Fixture f;
+	wireless = hid_available = true;
+	connected_pads           = {1, 3};
+	devices.clear(); // Bluetooth does not enumerate a quad USB audio endpoint.
+	auto               vibration = Open();
+	auto               speaker   = Open(true);
+	std::vector<float> haptic(2048, 0.5f);
+	std::vector<float> sound(2048, 0.25f);
+	Check(Haptics::SetVibration(1, 100, 80) && pad_rumble[1].large == 100 * 0x101,
+	      "Bluetooth controller did not receive normal rumble");
+	Check(Haptics::Queue(vibration.get(), 1, haptic.data(), 1024, 2, true, unity.data()) != 0,
+	      "Bluetooth haptics did not open");
+	Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true, unity.data()) != 0,
+	      "Bluetooth speaker did not open");
+	Check(hid_opens == 1 && opened_hid_pad == 1 && effects[1][7] == 0x30,
+	      "Bluetooth audio was not matched to the active controller");
+	Check(pad_rumble[1].large == 0, "Bluetooth haptics did not suppress motor rumble");
+	auto pending = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
+	Check(pending.size() == 1, "Bluetooth streams were not combined in one report");
+	Libs::Controller::DualSenseBluetooth::Send(pending);
+	Check(hid_reports.size() == 1, "Bluetooth audio report was not sent");
+	const auto& report = hid_reports.back();
+	Check(report[0] == 0x39 && report[2] == 0x91 && report[3] == 6 && report[4] == 0x7e &&
+	          report[10] == 0xd2 && report[11] == 64 && report[140] == 0xd3 && report[141] == 200,
+	      "Bluetooth audio sub-packets are malformed");
+	Check(report[12] > 0 && report[13] > 0,
+	      "haptic waveform was not downsampled into the actuator channels");
+	uint8_t prefix = 0xa2;
+	auto    crc    = SDL_crc32(0, &prefix, 1);
+	crc            = SDL_crc32(crc, report.data(), report.size() - 4);
+	for (size_t i = 0; i < 4; i++) {
+		Check(report[report.size() - 4 + i] == static_cast<uint8_t>(crc >> (8 * i)),
+		      "Bluetooth audio CRC is invalid");
+	}
+	int   opus_error = OPUS_OK;
+	auto* decoder    = opus_decoder_create(48000, 2, &opus_error);
+	Check(decoder != nullptr && opus_error == OPUS_OK, "Opus decoder unavailable in test");
+	std::array<float, 960> decoded {};
+	Check(opus_decode_float(decoder, report.data() + 142, 200, decoded.data(), 480, 0) == 480,
+	      "Bluetooth speaker packet could not be decoded");
+	opus_decoder_destroy(decoder);
+	const auto mean = std::accumulate(decoded.begin(), decoded.end(), 0.0f) / decoded.size();
+	Check(mean > 0.05f && mean < 0.35f, "Bluetooth speaker audio was not preserved");
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000).empty(),
+	      "Bluetooth sender ignored report pacing");
+	now += 300;
+	Haptics::UpdateBluetoothRumble();
+	Check(pad_rumble[1].large == 100 * 0x101,
+	      "Bluetooth haptics did not restore normal rumble afterward");
+
+	// A different active DualSense has a different HID address, even if both are
+	// paired.
+	Check(Haptics::Queue(vibration.get(), 3, haptic.data(), 1024, 2, true, unity.data()) != 0 &&
+	          opened_hid_pad == 3,
+	      "Bluetooth stream followed the wrong controller");
+	Check(Haptics::Queue(speaker.get(), 3, sound.data(), 1024, 2, true, unity.data()) != 0,
+	      "Bluetooth speaker did not follow the selected controller");
+	Check(effects[1][7] == 0, "old Bluetooth controller remained speaker-routed");
+	wireless       = false;
+	connected_pads = {3};
+	devices        = {{20, "Speakers (DualSense Wireless Controller)", 4}};
+	Check(Haptics::Queue(speaker.get(), 3, sound.data(), 1024, 2, true, unity.data()) != 0 &&
+	          opened_device == 20 && effects[3][7] == 0x30,
+	      "Bluetooth to USB change did not reopen the USB speaker immediately");
+	speaker.reset();
+	vibration.reset();
+}
+
+void TestBluetoothFailureFallsBack() {
+	Fixture f;
+	wireless = hid_available   = true;
+	auto               speaker = Open(true);
+	std::vector<float> sound(2048, 0.25f);
+	Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true, unity.data()) != 0,
+	      "Bluetooth port did not initially open");
+	fail_hid_write = true;
+	auto pending   = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
+	Libs::Controller::DualSenseBluetooth::Send(pending);
+	Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true, unity.data()) == 0 &&
+	          effects[1][7] == 0,
+	      "failed HID write did not restore the normal audio fallback");
+	speaker.reset();
+}
+
+void TestBluetoothFormatsAndRoutingFailure() {
+	Fixture f;
+	wireless = hid_available = fail_effect = true;
+	auto speaker                           = Open(true);
+	Check(Haptics::Queue(speaker.get(), 1, pcm.data(), 2, 2, true, unity.data()) == 0 &&
+	          !Haptics::UsesBluetooth(speaker.get()) && hid_closes == 1 && effects[1].empty(),
+	      "failed Bluetooth speaker routing leaked a session or suppressed fallback");
+	fail_effect = false;
+	now += 2001;
+	std::vector<int16_t> mono(1024, 16384);
+	const int            volume = 16384;
+	Queue(speaker, mono.data(), 1024, 1, false, &volume);
+	auto pending = Bluetooth::Prepare(now * 1000000);
+	Check(pending.size() == 1 && hid_opens == 2 && effects[1][7] == 0x30 &&
+	          std::abs(pending[0].audio[200] - 0.25f) < 1e-4f &&
+	          std::abs(pending[0].audio[201] - 0.25f) < 1e-4f,
+	      "Bluetooth mono int16 volume mapping or speaker routing recovery failed");
+}
+
+void TestBluetoothEncodingDoesNotBlockAudioQueue() {
+	Fixture f;
+	wireless = hid_available = true;
+	auto speaker = Open(true);
+	std::vector<float> sound(2048, 0.25f);
+	Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true, unity.data()) != 0,
+	      "Bluetooth speaker did not open for concurrent encoding test");
+	block_opus_encode = true;
+	std::thread encoder([&] {
+		auto pending = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
+		Libs::Controller::DualSenseBluetooth::Send(pending);
+	});
+	encode_started.acquire();
+	std::binary_semaphore queue_finished {0};
+	std::atomic_bool      queue_ok {false};
+	std::thread audio([&] {
+		queue_ok = Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true,
+		                          unity.data()) != 0;
+		queue_finished.release();
+	});
+	const bool queued_without_encoder =
+	    queue_finished.try_acquire_for(std::chrono::seconds(2));
+	allow_encode.release();
+	audio.join();
+	encoder.join();
+	Check(queued_without_encoder && queue_ok,
+	      "Bluetooth audio queue waited for Opus encoding");
+	speaker.reset();
+}
+
+void TestBluetoothCloseCancelsPreparedReport() {
+	Fixture f;
+	wireless = hid_available     = true;
+	auto               speaker   = Open(true);
+	auto               vibration = Open();
+	std::vector<float> sound(2048, 0.25f);
+	Queue(speaker, sound.data(), 1024);
+	Queue(vibration, sound.data(), 1024);
+	const auto base    = now * 1000000;
+	auto       pending = Bluetooth::Prepare(base);
+	Check(pending.size() == 1, "Bluetooth report was not prepared before stream close");
+	speaker.reset();
+	Bluetooth::Send(pending);
+	Check(hid_reports.empty(), "closed speaker's prepared report was sent");
+	Queue(vibration, sound.data(), 1024);
+	Bluetooth::Send(Bluetooth::Prepare(base + Bluetooth::PERIOD_NS));
+	Check(hid_reports.size() == 1 && hid_reports.back()[140] == 0,
+	      "closing the speaker prevented the remaining haptic stream from sending");
+}
+
+void TestBluetoothCloseDuringEncoding() {
+	Fixture f;
+	wireless = hid_available   = true;
+	auto               speaker = Open(true);
+	std::vector<float> sound(2048, 0.25f);
+	Queue(speaker, sound.data(), 1024);
+	block_opus_encode = true;
+	std::thread encoder([&] { Bluetooth::Send(Bluetooth::Prepare(now * 1000000)); });
+	encode_started.acquire();
+	std::binary_semaphore close_finished {0};
+	std::thread           closer([&] {
+		speaker.reset();
+		close_finished.release();
+	});
+	const bool closed_without_encoder  = close_finished.try_acquire_for(std::chrono::seconds(2));
+	const bool hid_alive_during_encode = hid_closes == 0;
+	allow_encode.release();
+	closer.join();
+	encoder.join();
+	Check(closed_without_encoder && hid_alive_during_encode && hid_closes == 1 &&
+	          hid_reports.empty(),
+	      "closing during encoding blocked, destroyed a live session, or sent stale audio");
+}
+
+void TestBluetoothCloseWaitsForWriteWithoutBlockingQueue() {
+	Fixture f;
+	wireless = hid_available     = true;
+	auto               speaker   = Open(true);
+	auto               vibration = Open();
+	std::vector<float> sound(2048, 0.25f);
+	Queue(speaker, sound.data(), 1024);
+	Queue(vibration, sound.data(), 1024);
+	auto pending = Bluetooth::Prepare(now * 1000000);
+	Check(pending.size() == 1, "Bluetooth report was not prepared before blocked write");
+	block_hid_write = true;
+	std::thread sender([&] { Bluetooth::Send(pending); });
+	write_started.acquire();
+	std::binary_semaphore close_started {0}, close_finished {0}, queue_finished {0};
+	std::thread           closer([&] {
+		close_started.release();
+		speaker.reset();
+		close_finished.release();
+	});
+	close_started.acquire();
+	const bool closed_before_write = close_finished.try_acquire_for(std::chrono::milliseconds(50));
+	std::atomic_bool queue_ok {false};
+	std::thread      audio([&] {
+		queue_ok =
+		    Haptics::Queue(vibration.get(), 1, sound.data(), 1024, 2, true, unity.data()) != 0;
+		queue_finished.release();
+	});
+	const bool       queued_without_write = queue_finished.try_acquire_for(std::chrono::seconds(2));
+	allow_write.release();
+	audio.join();
+	closer.join();
+	sender.join();
+	Check(!closed_before_write && queued_without_write && queue_ok && hid_reports.size() == 1 &&
+	          effects[1][7] == 0,
+	      "close failed to drain the HID write independently of the audio queue");
+}
+
+void TestBluetoothSpeakerWaitsForItsAudioBlock() {
+	Fixture f;
+	wireless = hid_available = true;
+	auto vibration = Open();
+	auto speaker = Open(true);
+	std::vector<float> haptic(2048, 0.5f);
+	std::vector<float> sound(2048, 0.25f);
+	Check(Haptics::Queue(vibration.get(), 1, haptic.data(), 1024, 2, true, unity.data()) != 0,
+	      "Bluetooth vibration stream did not open");
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000).empty(),
+	      "haptics sent a silent speaker packet before the audio batch arrived");
+	Check(Haptics::Queue(speaker.get(), 1, sound.data(), 256, 2, true, unity.data()) != 0,
+	      "Bluetooth speaker did not accept a short block");
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000).empty(),
+	      "short Bluetooth speaker block was padded with silence immediately");
+	Check(Haptics::Queue(speaker.get(), 1, sound.data(), 768, 2, true, unity.data()) != 0,
+	      "Bluetooth speaker did not accept the remainder of its block");
+	auto reports = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
+	Check(reports.size() == 1 && reports[0].audio[100] > 0.2f && reports[0].data[12] > 0,
+	      "Bluetooth speaker and haptics were not combined into a complete report");
+	speaker.reset();
+	vibration.reset();
+}
+
+void TestBluetoothShortHapticBlock() {
+	Fixture f;
+	wireless = hid_available     = true;
+	auto               vibration = Open();
+	std::vector<float> sound(2048, 0.25f);
+	Queue(vibration, sound.data(), 256);
+	const auto base = now * 1000000;
+	Check(Bluetooth::Prepare(base).empty() && Bluetooth::Prepare(base + 1000000).empty(),
+	      "short haptic block was padded with silence before its remainder arrived");
+	Queue(vibration, sound.data(), 768);
+	Check(Bluetooth::Prepare(base + 1000000).empty(),
+	      "completed haptic block skipped its speaker coalescing window");
+	auto pending = Bluetooth::Prepare(base + 2000000);
+	Check(pending.size() == 1 && pending[0].data[12 + 100] > 0,
+	      "complete haptic block did not play after its remainder arrived");
+}
+
+void TestBluetoothIncrementalBatchCoalesces() {
+	Fixture f;
+	wireless = hid_available = true;
+	auto                   vibration = Open();
+	auto speaker = Open(true);
+	std::array<float, 512> sound;
+	sound.fill(0.25f);
+	for (int block = 0; block < 3; ++block) {
+		Queue(vibration, sound.data(), 256);
+		Queue(speaker, sound.data(), 256);
+		Check(Bluetooth::Prepare(now * 1000000).empty(),
+		      "incremental Bluetooth batch was padded before it was complete");
+		now += 5;
+	}
+	Queue(vibration, sound.data(), 256);
+	Check(Bluetooth::Prepare(now * 1000000).empty(),
+	      "completed haptics sent before the same batch's final speaker block arrived");
+	Queue(speaker, sound.data(), 256);
+	auto pending = Bluetooth::Prepare(now * 1000000);
+	Check(pending.size() == 1 && pending[0].audio[1800] > 0.2f && pending[0].data[12 + 100] > 0,
+	      "completed incremental speaker and haptics batch did not play immediately");
+}
+
+void TestBluetoothLowRateCompleteBlock() {
+	Fixture f;
+	wireless = hid_available           = true;
+	auto                       speaker = Open(true, 8000);
+	std::array<float, 171 * 2> sound;
+	sound.fill(0.25f);
+	Queue(speaker, sound.data(), 171);
+	auto pending = Bluetooth::Prepare(now * 1000000);
+	Check(pending.size() == 1 && pending[0].audio[1000] > 0.2f,
+	      "resampler lookahead delayed a complete low-rate speaker block");
+}
+
+void TestBluetoothCompleteStreamDoesNotWaitForPartialPeer() {
+	Fixture f;
+	wireless = hid_available    = true;
+	auto               complete = Open(true);
+	auto               partial  = Open(true);
+	std::vector<float> sound(2048, 0.25f);
+	Queue(complete, sound.data(), 1024);
+	Queue(partial, sound.data(), 256);
+	auto pending = Bluetooth::Prepare(now * 1000000);
+	Check(pending.size() == 1 && pending[0].audio[100] > 0.4f && pending[0].audio[1000] > 0.2f,
+	      "complete speaker stream waited for a partial peer instead of mixing available audio");
+}
+
+void TestBluetoothShortFinalBlock() {
+	for (bool speaker: {false, true}) {
+		Fixture f;
+		wireless = hid_available = true;
+		auto               port  = Open(speaker);
+		std::vector<float> sound(2048, 0.25f);
+		Queue(port, sound.data(), 256);
+		const auto base = now * 1000000;
+		Check(Bluetooth::Prepare(base).empty(),
+		      "short final Bluetooth block sent before more audio could arrive");
+		Check(Bluetooth::Prepare(base + Bluetooth::PERIOD_NS).size() == 1,
+		      "short final Bluetooth block was never played");
+	}
+}
+
+void TestBluetoothOverflowKeepsRecentAudio() {
+	Fixture f;
+	wireless = hid_available = true;
+	auto speaker = Open(true);
+	std::vector<float> sound(2048);
+	for (int block = 1; block <= 4; ++block) {
+		std::fill(sound.begin(), sound.end(), block * 0.1f);
+		Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true,
+		                     unity.data()) != 0,
+		      "Bluetooth speaker overflow queue failed");
+	}
+	auto first = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
+	Check(first.size() == 1 && first[0].audio[200] > 0.15f && first[0].audio[200] < 0.25f,
+	      "Bluetooth queue overflow lost all buffered audio instead of the oldest block");
+	auto second = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000 +
+	                                                        Libs::Controller::DualSenseBluetooth::PERIOD_NS);
+	Check(second.size() == 1 && second[0].audio[200] > 0.25f && second[0].audio[200] < 0.35f,
+	      "Bluetooth queue overflow did not preserve the next audio block");
+	speaker.reset();
+}
+
+void TestBluetoothHapticOverflowKeepsRecentAudio() {
+	Fixture f;
+	wireless = hid_available = true;
+	auto vibration = Open();
+	std::vector<float> haptic(2048);
+	for (int block = 1; block <= 4; ++block) {
+		std::fill(haptic.begin(), haptic.end(), block * 0.1f);
+		Check(Haptics::Queue(vibration.get(), 1, haptic.data(), 1024, 2, true,
+		                     unity.data()) != 0,
+		      "Bluetooth haptic overflow queue failed");
+	}
+	const auto base = now * 1000000;
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(base).empty(),
+	      "haptic-only report skipped its coalescing window");
+	auto first = Libs::Controller::DualSenseBluetooth::Prepare(base + 1000000);
+	Check(first.size() == 1 && first[0].data[12 + 32] > 18 && first[0].data[12 + 32] < 32,
+	      "Bluetooth haptic overflow discarded buffered waveform instead of one report");
+	vibration.reset();
+}
+
+void TestBluetoothRecoversAfterLateWrite() {
+	Fixture f;
+	wireless = hid_available = true;
+	auto speaker = Open(true);
+	std::vector<float> sound(2048, 0.25f);
+	for (int i = 0; i < 3; ++i) {
+		Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true,
+		                     unity.data()) != 0,
+		      "Bluetooth speaker could not buffer three reports");
+	}
+	const auto base = now * 1000000;
+	const auto period = Libs::Controller::DualSenseBluetooth::PERIOD_NS;
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(base).size() == 1,
+	      "first Bluetooth report was not prepared");
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(base + period + 10000000).size() == 1,
+	      "late Bluetooth report was not prepared");
+	Check(Libs::Controller::DualSenseBluetooth::Prepare(base + 2 * period).size() == 1,
+	      "Bluetooth sender did not recover its schedule after a late write");
+	speaker.reset();
+}
+
+void TestUsbOverflowKeepsRecentAudio() {
+	Fixture f;
+	auto speaker = Open(true);
+	std::vector<float> sound(2048);
+	for (int block = 1; block <= 4; ++block) {
+		std::fill(sound.begin(), sound.end(), block * 0.1f);
+		Check(Haptics::Queue(speaker.get(), 1, sound.data(), 1024, 2, true,
+		                     unity.data()) != 0,
+		      "USB speaker overflow queue failed");
+	}
+	std::array<float, 4> first {};
+	Check(!streams.empty() &&
+	          SDL_GetAudioStreamData(streams.back(), first.data(), sizeof(first)) ==
+	              sizeof(first) &&
+	          first[0] > 0.15f && first[0] < 0.25f,
+	      "USB queue overflow did not preserve the newest buffered audio");
+	// A single oversized input must also preserve only its newest 80 ms.
+	sound.assign(4096 * 2, 0.1f);
+	std::fill(sound.begin() + 256 * 2, sound.end(), 0.5f);
+	const auto queued = Haptics::Queue(speaker.get(), 1, sound.data(), 4096, 2, true, unity.data());
+	Check(queued == 80000 && SDL_GetAudioStreamQueued(streams.back()) == 3840 * 4 * sizeof(float) &&
+	          SDL_GetAudioStreamData(streams.back(), first.data(), sizeof(first)) ==
+	              sizeof(first) &&
+	          first[0] == 0.5f && first[1] == 0.5f,
+	      "oversized USB block did not retain only the newest 80 ms");
+	speaker.reset();
+}
+
+void TestBluetoothAmbiguousDevice() {
+	Fixture f;
+	wireless = hid_available = duplicate_hid = true;
+	auto speaker                             = Open(true);
+	Check(Haptics::Queue(speaker.get(), 1, pcm.data(), 2, 2, true, unity.data()) == 0 &&
+	          hid_opens == 0 && effects[1].empty(),
+	      "ambiguous Bluetooth devices were routed arbitrarily");
+	speaker.reset();
+}
+
+void TestUsbAndBluetoothFeatureParity() {
+	Fixture f;
+	connected_pads = {1, 3};
+	wireless_pad3 = hid_available       = true;
+	auto               wired_speaker    = Open(true);
+	auto               wireless_speaker = Open(true);
+	std::vector<float> sound(2048, 0.25f);
+	Check(Haptics::Queue(wired_speaker.get(), 1, sound.data(), 1024, 2, true, unity.data()) != 0 &&
+	          opened_device == 10,
+	      "Bluetooth pad disabled the wired pad speaker");
+	Check(Haptics::Queue(wireless_speaker.get(), 3, sound.data(), 1024, 2, true, unity.data()) !=
+	              0 &&
+	          opened_hid_pad == 3,
+	      "wired pad disabled the Bluetooth speaker");
+	Check(Haptics::Queue(wired_speaker.get(), 1, sound.data(), 1024, 2, true, unity.data()) != 0 &&
+	          effects[1][7] == 0x30 && effects[3][7] == 0,
+	      "returning to the wired speaker left Bluetooth routing active");
+	wireless_speaker.reset();
+	wired_speaker.reset();
+}
+
 void TestAudioSpeakerFallback() {
 	Fixture f;
 	Libs::Audio::Audio audio;
@@ -545,6 +1105,26 @@ void TestAudioSpeakerRouting() {
 	ExpectPcm({0.5f, 0.5f, 0, 0, 0.5f, 0.5f, 0, 0});
 }
 
+void TestBluetoothAudioSpeakerRouting() {
+	Fixture f;
+	wireless = hid_available = true;
+	devices.clear();
+	Libs::Audio::Audio audio;
+	const auto port = audio.AudioOutOpen(4, 1024, 48000, Libs::Audio::Audio::Format::FloatStereo);
+	Check(port.IsValid() && default_stream != nullptr, "Bluetooth pad speaker port did not open");
+	std::vector<float> sound(2048, 0.25f);
+	Libs::Audio::Audio::OutputParam output {port, sound.data()};
+	Check(audio.AudioOutOutputs(&output, 1, false) == 1024 && hid_opens == 1 &&
+	          SDL_GetAudioStreamQueued(default_stream) == 0,
+	      "Bluetooth pad speaker also played on the host output");
+	fail_hid_write = true;
+	auto pending = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
+	Libs::Controller::DualSenseBluetooth::Send(pending);
+	Check(audio.AudioOutOutputs(&output, 1, false) == 1024 &&
+	          SDL_GetAudioStreamQueued(default_stream) > 0,
+	      "Bluetooth write failure did not use the normal speaker output");
+}
+
 void TestAudioDefaultResumeFailure() {
 	Fixture f;
 	Libs::Audio::Audio audio;
@@ -564,6 +1144,7 @@ int main() {
 	SDL_SetHint(SDL_HINT_AUDIO_DRIVER, "dummy");
 	TestAudioSpeakerFallback();
 	TestAudioSpeakerRouting();
+	TestBluetoothAudioSpeakerRouting();
 	TestAudioDefaultResumeFailure();
 	TestFormatsAndVolume();
 	TestDiscoveryAndHotplug();
@@ -571,6 +1152,25 @@ int main() {
 	TestSpeakerUnplug();
 	TestSpeakerSwitchAndFailures();
 	TestSpeakerRequiresUnambiguousUsbDevice();
+	TestBluetoothAudioAndIdentity();
+	TestBluetoothFailureFallsBack();
+	TestBluetoothFormatsAndRoutingFailure();
+	TestBluetoothEncodingDoesNotBlockAudioQueue();
+	TestBluetoothCloseCancelsPreparedReport();
+	TestBluetoothCloseDuringEncoding();
+	TestBluetoothCloseWaitsForWriteWithoutBlockingQueue();
+	TestBluetoothSpeakerWaitsForItsAudioBlock();
+	TestBluetoothShortHapticBlock();
+	TestBluetoothIncrementalBatchCoalesces();
+	TestBluetoothLowRateCompleteBlock();
+	TestBluetoothCompleteStreamDoesNotWaitForPartialPeer();
+	TestBluetoothShortFinalBlock();
+	TestBluetoothOverflowKeepsRecentAudio();
+	TestBluetoothHapticOverflowKeepsRecentAudio();
+	TestBluetoothRecoversAfterLateWrite();
+	TestUsbOverflowKeepsRecentAudio();
+	TestBluetoothAmbiguousDevice();
+	TestUsbAndBluetoothFeatureParity();
 	TestFailuresAndBoundedQueue();
 	TestRumbleLeaseAndDuration();
 	TestSwitchStopsOldRumble();

tests/PageManagerTests.cpp

@@ -192,13 +192,25 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE,
   Check(memory == reinterpret_cast<void *>(test_address),
         "fixed low VirtualAlloc failed");
 #elif defined(__APPLE__)
-  mach_vm_address_t raw = test_address;
-  Check(mach_vm_allocate(mach_task_self(), &raw, size, VM_FLAGS_FIXED) ==
-            KERN_SUCCESS &&
-            mach_vm_protect(mach_task_self(), raw, size, false,
-                            static_cast<vm_prot_t>(ToHostProt(protection))) ==
-                KERN_SUCCESS,
-        "fixed low mach_vm_allocate failed");
+  mach_vm_address_t raw = 0;
+  bool allocated = false;
+  constexpr uintptr_t stride = 0x0000000100000000ull;
+  for (uintptr_t candidate = test_address;
+       Libs::Graphics::GuestRange{candidate, size}.Valid(); candidate += stride) {
+    raw = candidate;
+    if (mach_vm_allocate(mach_task_self(), &raw, size, VM_FLAGS_FIXED) !=
+        KERN_SUCCESS) {
+      continue;
+    }
+    if (mach_vm_protect(mach_task_self(), raw, size, false,
+                        static_cast<vm_prot_t>(ToHostProt(protection))) ==
+        KERN_SUCCESS) {
+      allocated = true;
+      break;
+    }
+    mach_vm_deallocate(mach_task_self(), raw, size);
+  }
+  Check(allocated, "no free fixed guest address found");
   auto *memory = reinterpret_cast<uint8_t *>(raw);
   AllocationSizes()[memory] = static_cast<size_t>(size);
 #else

tests/ResourceMaterializationTests.cpp

@@ -219,6 +219,56 @@ void TestUnbasedFlatCacheHitMaterializes() {
         "unbased FLAT plan produced unexpected descriptors");
 }
 
+void TestWrittenDescriptorUsesStrictReaderOnce() {
+  using namespace Libs::Graphics::ShaderRecompiler::IR;
+  Program program;
+  program.stage = Libs::Graphics::ShaderType::Compute;
+  program.srt_plan_complete = true;
+  program.resource_tracking_complete = true;
+  auto &block = AddValueBlock(program);
+  program.memory_info.push_back({.kind = ResourceKind::ScalarAddress});
+  auto &handle = block.AppendNewInst(ValueOpcode::GetAddressResource,
+                                     {Value(0x1000u), Value(0u)});
+  auto &offset = block.AppendNewInst(ValueOpcode::GetUserData,
+                                     {Value(static_cast<ScalarReg>(0))});
+  auto &read = block.AppendNewInst(ValueOpcode::LoadAddressU32,
+      {Value(&handle), Value(&offset), Value(0u), Value(false)});
+  read.SetFlags(MemoryFlags{.index = 0});
+  DescriptorSource source;
+  source.dwords = {Value(&read), Value(0u), Value(4u), Value(0u)};
+  source.dword_count = 4;
+  program.descriptor_sources.push_back(source);
+  program.info.buffers.push_back({.source = 0, .written = true});
+  auto plan = ExtractResourcePlan(program);
+  struct Reads { uint32_t ordinary = 0; uint32_t strict = 0; bool clean = false; } reads;
+  const std::array<uint32_t, 1> user_data{4u};
+  const SrtRuntime runtime{
+      .user_data = user_data,
+      .read_memory = +[](void *data, uint64_t, std::span<uint32_t> words) {
+        ++static_cast<Reads *>(data)->ordinary;
+        words[0] = 0x8000u;
+        return true;
+      },
+      .userdata = &reads,
+      .read_specialization_memory = +[](void *data, uint64_t address, std::span<uint32_t> words) {
+        auto &reads = *static_cast<Reads *>(data);
+        ++reads.strict;
+        if (!reads.clean || address != 0x1004u) return false;
+        words[0] = 0x8000u;
+        return true;
+      }};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
+            reads.ordinary == 0 && reads.strict == 1,
+        "GPU-dirty dynamic writable descriptor bypassed strict provenance");
+  reads.clean = true;
+  reads.strict = 0;
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            reads.ordinary == 0 && reads.strict == 1 && snapshot.buffers[0].dwords[0] == 0x8000u,
+        "writable descriptor was evaluated twice or scalar EXEC suppressed its read");
+}
+
 void TestFailedMaterializationRejectsStage() {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   auto plan = UserDataBufferPlan();
@@ -262,6 +312,7 @@ int main() {
   TestMappedSrtUsesDirectReaderByDefault();
   TestIntegerRuntimeValueFollowsSrtReads();
   TestUnbasedFlatCacheHitMaterializes();
+  TestWrittenDescriptorUsesStrictReaderOnce();
   TestFailedMaterializationRejectsStage();
   TestMixedSamplerDuplicatesTheCorrectSnapshot();
   std::puts("ResourceMaterializationTests: all cases passed");

tests/ResourceTrackingTests.cpp

@@ -1386,11 +1386,17 @@ void TestUniformizedMaterialImageKeys() {
                 specialization.images[0].indirect_mapping_offset + 3u] == 4096u,
         "material mask did not limit sparse descriptor reads");
   user_data[8] = first_material;
-  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
-        "written buffer alias with a material key was accepted");
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            std::ranges::find(snapshot.specialization_reads,
+                std::pair<uint64_t, uint64_t>{first_material, 4}) !=
+                snapshot.specialization_reads.end(),
+        "material key read was omitted from the renderer write-overlap check");
   user_data[8] = first_table;
-  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
-        "written buffer alias with an image record was accepted");
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            std::ranges::find(snapshot.specialization_reads,
+                std::pair<uint64_t, uint64_t>{first_table, 32}) !=
+                snapshot.specialization_reads.end(),
+        "image descriptor read was omitted from the renderer write-overlap check");
   CheckFatal([&] { make_plan(Variant::WrongUpdate); }, "not a valid runtime value",
              "non-clearing material mask was accepted");
   CheckFatal([&] { make_plan(Variant::WrongEquality); }, "not a valid runtime value",
@@ -2785,6 +2791,108 @@ void TestConditionalBufferMaterialization() {
   CheckActive();
 }
 
+void TestGuardedScalarDescriptorReads() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  for (const bool shared : {false, true}) {
+    for (const bool exec : {false, true}) {
+      Fixture fixture;
+      auto *entry = fixture.block;
+      auto *optional = fixture.AddBlock();
+      auto *done = fixture.AddBlock();
+      entry->AddBranch(optional);
+      entry->AddBranch(done);
+      optional->AddBranch(done);
+      fixture.program.block_info[0].terminator = {
+          .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 1, .false_block = 2};
+      fixture.program.block_info[1].terminator = {
+          .kind = CFG::TerminatorKind::Branch, .true_block = 2};
+      fixture.program.block_info[2].terminator.kind = CFG::TerminatorKind::Return;
+      const auto control = fixture.Buffer({fixture.UserData(0), fixture.UserData(1),
+                                          fixture.UserData(2), fixture.UserData(3)});
+      MemoryInfo scalar;
+      scalar.kind = ResourceKind::ScalarBuffer;
+      scalar.offset = 28;
+      const auto flag = fixture.Emit(ValueOpcode::ReadConstBuffer, {control, Value(0u)},
+                                     fixture.AddMemory(scalar, 4));
+      const auto lane = fixture.Emit(ValueOpcode::GetBuiltin,
+          {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
+      const auto varying = fixture.Emit(ValueOpcode::INotEqual32, {lane, Value(0u)});
+      const auto enabled = fixture.Emit(ValueOpcode::INotEqual32, {flag, Value(0u)});
+      fixture.program.block_info[0].condition =
+          fixture.Emit(ValueOpcode::LogicalAnd, {varying, enabled});
+      const auto root = fixture.Address(fixture.UserData(4), Value(0u));
+      const auto Load = [&](Block *block) {
+        fixture.block = block;
+        MemoryInfo address;
+        address.kind = ResourceKind::ScalarAddress;
+        const auto pointer = fixture.Emit(ValueOpcode::LoadAddressU32,
+            {root, Value(0u), Value(0u), Value(exec)}, fixture.AddMemory(address, 8));
+        const auto table = fixture.Address(pointer, Value(0u));
+        std::array<Value, 4> words;
+        for (uint32_t word = 0; word < words.size(); ++word) {
+          address.offset = 40 + word * 4;
+          words[word] = fixture.Emit(ValueOpcode::LoadAddressU32,
+              {table, Value(0u), Value(0u), Value(exec)}, fixture.AddMemory(address, 12));
+        }
+        scalar.offset = 0;
+        fixture.Emit(ValueOpcode::ReadConstBuffer, {fixture.Buffer(words), Value(0u)},
+                     fixture.AddMemory(scalar, 16));
+      };
+      Load(optional);
+      if (shared) Load(done);
+      fixture.block = done;
+      MemoryInfo store;
+      store.kind = ResourceKind::Buffer;
+      fixture.Emit(ValueOpcode::StoreBufferU32,
+          {fixture.Buffer({Value(0x9000u), Value(0u), Value(4u), Value(0u)}),
+           Value(0u), Value(0u), Value(0u), Value(1u), Value(true)}, fixture.AddMemory(store, 20));
+      fixture.PlanAndTrack();
+      auto plan = ExtractResourcePlan(fixture.program);
+      Check(!exec || (!plan.srt_reads.empty() && !plan.control_flow[1].srt_reads.empty() &&
+                      (!shared || !plan.control_flow[2].srt_reads.empty())),
+            "guarded shared-read fixture lost its flattened execution sites");
+      struct Memory { LinearTestMemory data; uint32_t null_reads = 0; } memory;
+      const auto Read = +[](void *data, uint64_t address, std::span<uint32_t> words) {
+        auto &memory = *static_cast<Memory *>(data);
+        if (address == 40) ++memory.null_reads;
+        return ReadLinearTestMemory(&memory.data, address, words);
+      };
+      const std::array<uint32_t, 5> user_data{0x1000u, 0u, 64u, 0u, 0x1040u};
+      const SrtRuntime runtime{.user_data = user_data, .read_memory = Read,
+                               .userdata = &memory, .read_specialization_memory = Read};
+      memory.data.words[0xa8 / 4] = 0x2000u;
+      memory.data.words[0xb0 / 4] = 4u;
+      ResourceSnapshot snapshot;
+      ResourceSpecialization specialization;
+      if (!shared) {
+        Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 0 &&
+                  std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }),
+              "disabled feature speculatively dereferenced its null BVH table");
+        memory.data.words[7] = 1;
+        Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 1,
+              "active feature accepted an unreadable BVH table");
+      }
+      memory.data.words[0x40 / 4] = 0x1080u;
+      memory.data.watched_address = 0x10a8u;
+      memory.data.watched_reads = 0;
+      Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+                std::ranges::any_of(snapshot.buffers, [](const auto& descriptor) {
+                  return descriptor.dwords[0] == 0x2000u;
+                }) &&
+                (!exec || (memory.data.watched_reads == 1 &&
+                           std::ranges::find(snapshot.flattened_srt, 0x2000u) != snapshot.flattened_srt.end())),
+            "active or shared descriptor read was omitted after refreshing the cached plan");
+      if (!shared) {
+        memory.data.words[7] = 0;
+        memory.data.words[0x40 / 4] = 0;
+        Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 1 &&
+                  std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }),
+              "cached plan retained reads after the feature was disabled");
+      }
+    }
+  }
+}
+
 void TestConservativeBufferReachability() {
   std::array<uint32_t, 8> user_data{
       0x1000, 16u << 16u, 1, 0x4dfac,
@@ -2797,11 +2905,14 @@ void TestConservativeBufferReachability() {
     auto plan = ConditionalBufferPlan(use);
     ResourceSnapshot snapshot;
     ResourceSpecialization specialization;
+    std::array<uint32_t, 4> expected{};
+    if (use != ConditionalBufferUse::Writable)
+      std::copy_n(user_data.begin() + 4, expected.size(), expected.begin());
     Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
               snapshot.buffers.size() == 2 &&
-              std::equal(user_data.begin() + 4, user_data.end(),
+              std::equal(expected.begin(), expected.end(),
                          snapshot.buffers[1].dwords.begin()),
-          "shared, loop-dependent, or writable-alias resource was pruned");
+          "shared or loop-dependent resource was pruned, or inactive scalar branch retained");
   }
 }
 
@@ -3187,6 +3298,7 @@ int main() {
     Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma);
     Run("buffer swizzle specialization", TestBufferSwizzleSpecialization);
     Run("conditional buffer materialization", TestConditionalBufferMaterialization);
+    Run("guarded scalar descriptor reads", TestGuardedScalarDescriptorReads);
     Run("conservative buffer reachability", TestConservativeBufferReachability);
     Run("conditional indirect image", TestConditionalIndirectImageMaterialization);
     Run("shader info and bindings", TestShaderInfoAndBindingLayout);

tests/ScalarProvenanceTests.cpp

@@ -281,6 +281,61 @@ void TestInvariantAndDivergentPhi() {
         "divergent phi was accepted");
 }
 
+void TestOperandMutationLifecycle() {
+  Fixture fixture(6);
+  const auto definition = fixture.Emit(ValueOpcode::IAdd32, {Value(0x55u), Value(0u)});
+  auto *source = definition.TryInstruction();
+  const auto inserted = fixture.Emit(ValueOpcode::BitFieldInsert,
+      {definition, definition, Value(4u), Value(4u)});
+  Check(source->UseCount() == 2, "inline operands lost distinct reverse-use indices");
+  inserted.TryInstruction()->SetArg(1, Value(3u));
+  Check(source->UseCount() == 1, "inline operand mutation did not detach its old use");
+  source->ReplaceUsesWith(Value(0xaau));
+  uint32_t result = 0;
+  Check(source->UseCount() == 0 &&
+            SrtWalker(fixture.program, {}).Evaluate(inserted, result) && result == 0x3au,
+        "inline operand replacement changed the evaluated value");
+
+  for (const auto opcode : {ValueOpcode::GetImageResource, ValueOpcode::MakeImageAddress}) {
+    auto &large = fixture.program.value_storage.emplace_back(opcode);
+    auto &replacement = fixture.program.value_storage.emplace_back(opcode);
+    const auto count = large.NumArgs();
+    for (size_t index = 0; index < count; ++index) {
+      large.SetArg(index, definition);
+      replacement.SetArg(index, Value(static_cast<uint32_t>(index)));
+    }
+    const auto user = fixture.Emit(ValueOpcode::Identity, {Value(&large)}, 0, 1);
+    Check(source->UseCount() == count, "large operands lost reverse-use indices");
+    large.ReplaceUsesWith(Value(&replacement));
+    Check(source->UseCount() == 0 && large.GetOpcode() == ValueOpcode::Identity &&
+              large.NumArgs() == 1 && large.Arg(0) == Value(&replacement) &&
+              user.Resolve() == Value(&replacement) && replacement.UseCount() == 2,
+          "large operand replacement did not preserve users or detach old operands");
+    large.Invalidate();
+    large.Invalidate();
+    Check(large.NumArgs() == 0 && replacement.UseCount() == 1,
+          "repeated invalidation detached a surviving large-value user");
+  }
+
+  auto &phi = fixture.BlockAt(5).AppendNewInst(ValueOpcode::Phi);
+  phi.SetFlags(Type::U32);
+  for (size_t index = 0; index < 5; ++index) {
+    phi.AddPhiOperand(&fixture.BlockAt(index), definition);
+  }
+  phi.SetArg(3, Value(7u));
+  Check(phi.NumArgs() == 5 && phi.NumPhiBlocks() == 5 && source->UseCount() == 4,
+        "Phi mutation lost its incoming values or reverse uses");
+  for (size_t index = 0; index < 5; ++index) {
+    Check(phi.PhiBlock(index) == &fixture.BlockAt(index),
+          "Phi mutation changed an incoming predecessor");
+  }
+  const auto sum = fixture.Emit(ValueOpcode::IAdd32, {Value(&phi), Value(5u)}, 0, 5);
+  phi.ReplaceUsesWith(Value(6u));
+  Check(source->UseCount() == 0 && phi.NumPhiBlocks() == 0 && phi.NumArgs() == 1 &&
+            SrtWalker(fixture.program, {}).Evaluate(sum, result) && result == 11,
+        "Phi replacement retained incoming uses or changed the consumer value");
+}
+
 void TestControlDependentStandaloneLoadStaysTyped() {
   Fixture fixture(3);
   const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress);
@@ -634,6 +689,7 @@ int main() {
     TestShaderBaseAndUserData();
     TestCarryAndBitFields();
     TestInvariantAndDivergentPhi();
+    TestOperandMutationLifecycle();
     TestControlDependentStandaloneLoadStaysTyped();
     TestRuntime64BitDescriptorOps();
     TestUniformFirstLaneSamplerLod();

tests/ShaderRayTracingGpuTests.inc

@@ -0,0 +1,229 @@
+TestCase FlatStackApertures(u32 wave_size) {
+  TestCase test;
+  test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64";
+  auto &code = test.code;
+  code = {EncodeSop1(0x04, 8, 237), // private base
+          EncodeSop1(0x04, 10, 235), // shared base
+          EncodeVop2(0x1a, 20, InlineU32(2), 0),
+          EncodeVop2(0x1b, 22, InlineU32(1), 0),
+          EncodeVop2(0x1a, 22, InlineU32(31), 22)};
+  AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22)); // Clamp the stack address to the local apertures.
+  AppendVMovU32(&code, 30, 100);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeFlat0(0x1c, 1), EncodeFlat1(0, 0x7d, 30, 20)});
+  AppendVMovU32(&code, 30, 200);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeDs0(0x0d), EncodeDs1(0, 30, 20)});
+  AppendVMovU32(&code, 30, 300);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)});
+  // FLAT writes must alias both explicit memory instructions, per invocation.
+  code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20),
+                           EncodeDs0(0x36), EncodeDs1(25, 0, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 0);
+  AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, wave_size);
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 2 * wave_size);
+  // Inactive lanes must preserve both stores and destination VGPRs.
+  AppendSMovLiteral(&code, 126, 0x55555555);
+  if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x55555555);
+  code.push_back(EncodeVop2(0x25, 22, InlineU32(1), 22));
+  AppendVop3(&code, 0x159, 21, 9, 11, Vgpr(22));
+  AppendVMovU32(&code, 30, 400);
+  code.push_back(EncodeVop2(0x25, 30, Vgpr(0), 30));
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20)});
+  code.insert(code.end(), {EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  code.push_back(EncodeSop1(0x04, 126, 193)); // -1 sign-extends across EXEC
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 3 * wave_size);
+  // Noncanonical addresses outside either aperture, aperture overflow, and
+  // offsets outside allocated local storage must not wrap into valid storage.
+  AppendVMovLiteral(&code, 21, 0x78000000);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 4 * wave_size);
+  code.push_back(EncodeVop1(0x01, 21, 11));
+  AppendVMovLiteral(&code, 20, 0xfffffffc);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0, 4), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0, 4), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 5 * wave_size);
+  AppendVMovU32(&code, 20, wave_size * 4);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 6 * wave_size);
+  AppendVMovU32(&code, 20, 1);
+  code.insert(code.end(), {EncodeFlat0(0x1c, 0), EncodeFlat1(0, 0x7d, 30, 20),
+                           EncodeFlat0(0x0c, 0), EncodeFlat1(26, 0x7d, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 26, 0, 9 * wave_size);
+  code.push_back(EncodeVop2(0x1a, 20, InlineU32(2), 0));
+  code.insert(code.end(), {EncodeFlat0(0x0c, 1), EncodeFlat1(24, 0x7d, 0, 20),
+                           EncodeDs0(0x36), EncodeDs1(25, 0, 20)});
+  AppendStoreVgprAtLaneDwordOffset(&code, 24, 0, 7 * wave_size);
+  AppendStoreVgprAtLaneDwordOffset(&code, 25, 0, 8 * wave_size);
+  AppendEnd(&code);
+  test.expected.resize(10 * wave_size);
+  for (u32 lane = 0; lane < wave_size; ++lane) {
+    const bool shared = lane & 1;
+    test.expected[lane] = (shared ? 100 : 300) + lane;
+    test.expected[wave_size + lane] = (shared ? 300 : 200) + lane;
+    test.expected[2 * wave_size + lane] = 300 + lane;
+    test.expected[3 * wave_size + lane] = (shared ? 300 : 400) + lane;
+    test.expected[7 * wave_size + lane] = (shared ? 100 : 400) + lane;
+    test.expected[8 * wave_size + lane] = (shared ? 300 : 200) + lane;
+  }
+  test.compute_info.wave_size = wave_size;
+  test.compute_info.threads_num[0] = wave_size;
+  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
+  test.compute_info.thread_ids_num = 1;
+  test.compute_info.scratch_size_dwords = wave_size;
+  test.compute_info.lds_size_dwords = wave_size;
+  test.has_compute_info = true;
+  test.forbidden_spirv = {"get_bda_pointer"};
+  return test;
+}
+
+TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) {
+  TestCase test;
+  test.name = barycentrics ? (sorted ? "BvhBarycentricSorted" : "BvhBarycentricUnsorted")
+                          : (sorted ? "BvhTriangleIdSorted" : "BvhTriangleIdUnsorted");
+  if (grow != 0) test.name = "BvhBoxExitGrowth";
+  constexpr u32 NodeStart = 1024, OutputStart = 512, LaneCount = 32;
+  constexpr uint64_t GuestBase = 0x10000;
+  const auto bits = [](float value) { return std::bit_cast<u32>(value); };
+  constexpr u32 Infinity = 0x7f800000, Invalid = UINT32_MAX, Flag = 0x04040404;
+  test.initial.resize(NodeStart + 258 * 16);
+  auto &data = test.initial;
+  // Pack five float3 vertices and the triangle flag into a 64-byte node.
+  const std::array<float, 15> vertices{-1, -1, 2, 1, -1, 2, 0, 1, 2, 2, 1, 2, 0, 3, 2};
+  for (u32 i = 0; i < vertices.size(); ++i) data[NodeStart + i] = bits(vertices[i]);
+  data[NodeStart + 15] = Flag;
+  constexpr std::array<u32, 4> children{0x88, 0x90, 0x98, 0xa0};
+  constexpr std::array<float, 4> z{5, 1, 3, 7};
+  constexpr std::array<u32, 4> half_z{0x4500, 0x3c00, 0x4200, 0x4700};
+  constexpr std::array<u32, 4> half_far{0x4600, 0x4000, 0x4400, 0x4800};
+  for (u32 child = 0; child < 4; ++child) {
+    data[NodeStart + 16 + child] = data[NodeStart + 32 + child] = children[child];
+    const auto fp16 = NodeStart + 20 + child * 3;
+    data[fp16] = 0xbc00bc00; // min.xy = -1,-1
+    data[fp16 + 1] = 0x3c000000 | half_z[child];
+    data[fp16 + 2] = (half_far[child] << 16) | 0x3c00;
+    const auto fp32 = NodeStart + 36 + child * 6;
+    const std::array<float, 6> bounds{-1, -1, z[child], 1, 1, z[child] + 1};
+    for (u32 i = 0; i < 6; ++i) data[fp32 + i] = bits(bounds[i]);
+  }
+  // A grazing ray enters one ULP after exiting on another axis.
+  for (u32 child = 0; child < 4; ++child) {
+    data[NodeStart + 64 + child] = children[child];
+    const std::array<float, 6> bounds = child == 0
+        ? std::array<float, 6>{0, 0, std::nextafter(1.0f, 2.0f), 1, 2, 2}
+        : child == 1 ? std::array<float, 6>{-1, 1, -1, -0.0f, 2, 2}
+                     : std::array<float, 6>{1, 1, 1, -1, -1, -1};
+    for (u32 i = 0; i < 6; ++i) data[NodeStart + 68 + child * 6 + i] = bits(bounds[i]);
+  }
+  // An FP32 node straddles two guest pages with a 64-byte gap in physical backing.
+  std::copy_n(data.begin() + NodeStart + 32, 16, data.begin() + NodeStart + 255 * 16);
+  std::copy_n(data.begin() + NodeStart + 48, 16, data.begin() + NodeStart + 257 * 16);
+  const std::array<std::array<float, 2>, 4> origins{{{0, 0}, {1, 0}, {.5f, 1.5f}, {-.25f, 1}}};
+  std::array<std::array<u32, 4>, LaneCount> expected{};
+  for (u32 lane = 0; lane < LaneCount; ++lane) {
+    const auto kind = lane % 8;
+    const auto input = lane * 16;
+    data[input] = kind < 4 ? kind : kind == 4 ? 12 : kind == 5 ? 21 : kind == 6 ? 6 : 0x1000;
+    data[input + 1] = bits(6);
+    data[input + 2] = bits(kind < 4 ? origins[kind][0] : 0);
+    data[input + 3] = bits(kind < 4 ? origins[kind][1] : 0);
+    data[input + 7] = bits(1);
+    data[input + 8] = data[input + 9] = Infinity;
+    data[input + 10] = bits(1);
+    if (kind < 4) {
+      const float denominator = kind == 3 ? -2 : -4;
+      const float i = kind == 0 || kind == 3 ? -1 : -2;
+      const float j = kind == 3 ? -.5f : -1;
+      expected[lane] = {bits(2 * denominator), bits(denominator),
+                        barycentrics ? bits(i) : Flag + kind,
+                        barycentrics ? bits(j) : 1};
+    } else if (kind < 6) {
+      expected[lane] = sorted ? std::array<u32, 4>{children[1], children[2], children[0], Invalid}
+                              : std::array<u32, 4>{children[0], children[1], children[2], Invalid};
+    } else {
+      expected[lane] = kind == 6 ? std::array<u32, 4>{Invalid, Invalid, Invalid, Invalid}
+                                 : std::array<u32, 4>{Infinity, bits(1), Invalid, 0};
+    }
+    if (lane >= 8 && lane < 12) {
+      data[input + 2] = 0x7fc00000; // invalid ray origin
+      expected[lane] = {Infinity, bits(1), Invalid, 0};
+    } else if (lane == 12 || lane == 13) {
+      data[input + 1] = bits(1); // nearest face equals the exclusive extent
+      expected[lane] = {Invalid, Invalid, Invalid, Invalid};
+    } else if (lane >= 16 && lane < 20) {
+      const u32 axis = lane & 1;
+      data[input + 2 + axis] = bits(origins[kind][axis] - 4);
+      data[input + 5 + axis] = bits(2);
+      data[input + 8 + axis] = bits(.5f);
+      // Dominant X/Y projection scales all intersection numerators and the
+      // denominator by four; the distance and barycentrics remain unchanged.
+      for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i)
+        expected[lane][i] = bits(std::bit_cast<float>(expected[lane][i]) * 4);
+    } else if (lane == 22) {
+      data[input] = 37; // node 4, FP32
+      data[input + 1] = bits(10);
+      for (u32 i = 5; i <= 10; ++i) data[input + i] = bits(1);
+      expected[lane] = {grow ? children[0] : Invalid, Invalid, Invalid, Invalid};
+    } else if (lane >= 24 && lane < 28) {
+      data[input + 4] = bits(4);
+      data[input + 7] = data[input + 10] = bits(-1);
+      for (u32 i = 0; i < (barycentrics ? 4u : 2u); ++i)
+        expected[lane][i] = bits(-std::bit_cast<float>(expected[lane][i]));
+    } else if (lane == 28) {
+      data[input] = 0;
+      data[input + 3] = bits(-1); // horizontal triangle edge: top-left rule rejects
+      expected[lane] = {Infinity, bits(1), barycentrics ? bits(-2) : Flag,
+                        barycentrics ? bits(-2) : 0};
+    } else if (lane == 29) {
+      data[input] = (255 << 3) | 5;
+    }
+  }
+  test.expected = test.initial;
+  for (u32 lane = 0; lane < LaneCount; ++lane) {
+    std::copy(expected[lane].begin(), expected[lane].end(),
+              test.expected.begin() + OutputStart + lane * 4);
+    auto bounded = expected[lane];
+    if (lane % 8 == 5) bounded = {Invalid, Invalid, Invalid, Invalid};
+    if (lane >= 20) bounded = {0x12345678, 0x12345679, 0x1234567a, 0x1234567b};
+    std::copy(bounded.begin(), bounded.end(),
+              test.expected.begin() + OutputStart + LaneCount * 4 + lane * 4);
+  }
+  auto &code = test.code;
+  // The resource is a GPU value, not a materialized host image descriptor.
+  AppendSMovLiteral(&code, 8, GuestBase >> 8);
+  AppendSMovLiteral(&code, 9, (sorted ? 0x80000000 : 0) | (grow << 23));
+  AppendSMovLiteral(&code, 10, 256);
+  AppendSMovLiteral(&code, 11, barycentrics ? 0x81000000 : 0x80000000);
+  code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0));
+  for (u32 offset : {0u, 4u, 8u}) {
+    code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4),
+                             EncodeMubuf1(4 + offset, 0, 1)});
+  }
+  code.insert(code.end(), {EncodeMimg0(0xe6, 15, 0, false, 0, true),
+                           EncodeMimg1(20, 4, 2)});
+  code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
+  for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 20 + i, 2, OutputStart + i);
+  // NSA addressing, the FP32 second-block bound, and inactive destination lanes.
+  AppendSMovLiteral(&code, 10, 2);
+  for (u32 i = 0; i < 4; ++i) AppendVMovLiteral(&code, 24 + i, 0x12345678 + i);
+  AppendSMovLiteral(&code, 126, 0x000fffff);
+  code.insert(code.end(), {EncodeMimg0(0xe6, 15, 3, false, 0, true),
+                           EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, 0x00000e0d});
+  code.push_back(EncodeSop1(0x04, 126, 193));
+  for (u32 i = 0; i < 4; ++i)
+    AppendStoreVgprAtLaneDwordOffset(&code, 24 + i, 2, OutputStart + LaneCount * 4 + i);
+  AppendEnd(&code);
+  test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)},
+                       {GuestBase + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}};
+  test.compute_info.wave_size = LaneCount;
+  test.compute_info.threads_num[0] = LaneCount;
+  test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
+  test.compute_info.thread_ids_num = 1;
+  test.has_compute_info = true;
+  test.required_spirv = {"get_bda_pointer"};
+  return test;
+}

tests/ShaderRayTracingTests.inc

@@ -1,87 +1,50 @@
-void TestRayTracingDispatchDetection() {
+void TestRayTracingInstructions() {
   namespace D = ShaderRecompiler::Decoder;
-  const auto check_decoded = [](std::span<const uint32_t> code, bool expected,
-                                const char *message) {
-    D::Program decoded;
-    D::DecodeProgram(code, decoded);
-    Check(decoded.has_bvh == expected, message);
-    if (expected) {
-      const auto &last = decoded.instructions.back();
-      Check(last.family == D::Family::MIMG &&
-                (last.opcode_id == 0xe6u || last.opcode_id == 0xe7u),
-            "decoder continued past the first BVH instruction");
-      const auto result = ShaderRecompiler::TranslateProgram(
-          code, MakeCompileOptions(ShaderType::Compute));
-      Check(result.skip_dispatch && result.program.blocks.empty() && result.cfg_dump.empty(),
-            "BVH compute shader reached IR translation");
-    }
-  };
-  const auto check = [&](std::initializer_list<uint32_t> code, bool expected,
-                         const char *message) {
-    check_decoded(std::span<const uint32_t>{code.begin(), code.size()}, expected, message);
-  };
-
-  check({0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
-        "captured contiguous BVH instruction did not skip dispatch");
-  check({0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
-         EncodeSopp(0x01)}, true,
-        "captured NSA BVH instruction did not skip dispatch");
-  check({0xf19c9f01u, 0x00081014u, EncodeSopp(0x01)}, true,
-        "RDNA2 BVH64 instruction did not skip dispatch");
-  check({0xbe8e210eu, 0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
-        "unsupported SWAPPC before BVH prevented dispatch detection");
-  check({0xf1989f01u, 0x00081014u, 0xc0000000u, EncodeSopp(0x01)}, true,
-        "unknown instruction after BVH prevented dispatch skipping");
-
-  for (uint32_t opcode : {0xe6u, 0xe7u}) {
-    for (uint32_t nsa = 0; nsa < 4; ++nsa) {
-      for (uint32_t slc = 0; slc < 2; ++slc) {
-        std::array<uint32_t, 6> code{
-            EncodeMimg0(opcode, 0xf, true) | (nsa << 1u) | (slc << 25u),
-            EncodeMimg1(0, 0, 0, 0)};
-        code[2 + nsa] = EncodeSopp(0x01);
-        check_decoded(std::span(code).first(3 + nsa), true,
-                      "BVH cache flags or NSA length prevented dispatch detection");
-      }
-    }
+  const std::array code{
+      0xf1989f01u, 0x00081014u,
+      EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(16, 0, 0),
+      0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
+      EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(16, 0, 0),
+      EncodeSopp(0x01)};
+  D::Program decoded;
+  D::DecodeProgram(code, decoded);
+  Check(decoded.instructions.size() == 5 &&
+            decoded.instructions.back().opcode == D::Opcode::S_ENDPGM,
+        "decoder stopped before instructions following BVH intersections");
+  for (const auto index : {0u, 2u}) {
+    const auto &inst = decoded.instructions[index];
+    Check(inst.opcode == D::Opcode::IMAGE_BVH_INTERSECT_RAY &&
+              inst.image_address_components == 11 && inst.data_dwords == 4 && inst.image_r128,
+          "BVH32 instruction lost its ray or descriptor width");
   }
-  for (uint32_t opcode : {0x66u, 0x67u, 0xe4u, 0xe5u, 0xe8u, 0xe9u}) {
-    check({EncodeMimg0(opcode, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
-          false, "non-BVH image opcode was incorrectly skipped");
+  Check(decoded.instructions[2].image_nsa_addr[9] == 58,
+        "BVH NSA ray components were not decoded");
+  const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute));
+  Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 2 &&
+            result.program.info.uses_dma && result.program.info.images.empty() &&
+            result.program.info.samplers.empty(),
+        "BVH intersections did not use GPU descriptors and raw guest memory");
+  CheckSpirvBinaryValidates(result.spirv);
+
+  const std::array literal{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
+  D::DecodeProgram(literal, decoded);
+  Check(decoded.instructions.size() == 2 && decoded.instructions[0].opcode == D::Opcode::S_MOV_B32,
+        "BVH-shaped scalar literal was decoded as an instruction");
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+  for (const auto words : {std::array{0xf19c9f01u, 0x00081014u}, // BVH64
+                           std::array{0xf1989f01u, 0x40081014u}, // A16
+                           std::array{0xf1981f01u, 0x00081014u}, // not R128
+                           std::array{0xf1989f01u, 0x80081014u}}) { // D16
+    const std::array unsupported{words[0], words[1], EncodeSopp(0x01)};
+    ExpectFatal([&]() { (void)RecompileForTest(unsupported, MakeCompileOptions(ShaderType::Compute)); },
+                "unsupported BVH encoding was silently accepted");
   }
-
-  check({EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}, false,
-        "scalar literal was mistaken for a BVH instruction");
-  check({EncodeMimg0(0x20, 0xf) | 6u, EncodeMimg1(0, 0, 0, 0),
-         0xf1989f01u, 0xf19c9f01u, 0u, EncodeSopp(0x01)}, false,
-        "NSA address words were mistaken for BVH instructions");
-  check({EncodeMimg0(0x20, 0xf), 0xf1989f01u, EncodeSopp(0x01)}, false,
-        "MIMG operand word was mistaken for a BVH instruction");
-  check({EncodeSopp(0x01), 0xf1989f01u, 0x00081014u}, false,
-        "data after the shader end was mistaken for a BVH instruction");
-  check({EncodeMimg0(0x20, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
-        false, "ordinary image shader was incorrectly skipped");
-
-  // An end instruction on one branch does not end another branch's code.
-  check({EncodeSopp(0x04, 1), EncodeSopp(0x01), 0xf1989f01u, 0x00081014u,
-         EncodeSopp(0x01)}, true,
-        "BVH at a branch target after an early end was not detected");
-
-  const std::array bvh_code{0xf1989f01u, 0x00081014u, 0xf19c9f01u, 0x00081014u,
-                            EncodeSopp(0x01)};
-  const std::array ordinary_code{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
-  D::Program reused;
-  D::DecodeProgram(bvh_code, reused);
-  Check(reused.has_bvh && reused.instructions.size() == 1,
-        "decoder did not stop at the first BVH instruction");
-  D::DecodeProgram(ordinary_code, reused);
-  Check(!reused.has_bvh, "decoder retained stale BVH metadata");
-  for (const auto stage : {ShaderType::Compute, ShaderType::Vertex, ShaderType::Pixel}) {
-    const auto options = MakeCompileOptions(stage);
-    Check(!ShaderRecompiler::TranslateProgram(ordinary_code, options).skip_dispatch,
-          "BVH-shaped literal marked an ordinary shader for skipping");
-    const auto compiled = RecompileForTest(ordinary_code, options);
-    Check(!compiled.spirv.empty(), "ordinary shader did not compile");
-    CheckSpirvBinaryValidates(compiled.spirv);
+  for (const auto nsa : {1u, 2u}) {
+    std::array<uint32_t, 5> truncated{0xf1989f01u | (nsa << 1), 0x00081014u};
+    truncated[2 + nsa] = EncodeSopp(0x01);
+    ExpectFatal([&]() {
+      (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute));
+    }, "BVH NSA instruction accepted fewer than eleven ray addresses");
   }
+#endif
 }

tests/shaderCfgTests.cpp

@@ -1884,37 +1884,41 @@ void TestSopkCompareImmediateExtension() {
   }
 }
 
-void TestDisabledSystemDebugBranch() {
+void TestDisabledDebugBranches() {
   using namespace ShaderRecompiler;
-  // Relocate PPSA08709 MS pc 0x530 to zero, retaining its displacement to the
-  // system validation helper immediately after the main shader's S_ENDPGM.
-  std::array<uint32_t, 274> shader;
-  shader.fill(EncodeSopp(0x00));
-  shader[0] = 0xbf97010fu;
-  shader[1] = EncodeVop1(0x01, 1, 129);
-  shader[2] = EncodeMubuf0(0x1c, 0, false);
-  shader[3] = EncodeMubuf1(1, 0, 0);
-  shader[271] = EncodeSopp(0x01);
-  shader[272] = 0xffffffffu; // System-only helper must not be decoded.
-  shader[273] = 0xffffffffu;
-  Decoder::Program decoded;
-  Decoder::DecodeProgram(shader, decoded);
-  Check(decoded.instructions.front().opcode == Decoder::Opcode::S_CBRANCH_CDBGSYS &&
-            decoded.instructions.front().branch_target == 0x440u &&
-            decoded.instructions.back().pc == 0x43cu &&
-            decoded.instructions.back().opcode == Decoder::Opcode::S_ENDPGM,
-        "disabled system debug branch reached its post-ENDPGM validation helper");
-  auto graph = CFG::BuildGraph(decoded);
-  Check(graph.blocks.size() == 1u && graph.FindBlockByPc(0x440u) == nullptr &&
-            graph.blocks.front().terminator.kind == CFG::TerminatorKind::Return,
-        "disabled system debug branch changed normal shader control flow");
-  auto options = MakeCompileOptions(ShaderType::Compute);
-  options.dump_ir = true;
-  auto result = RecompileForTest(shader, options);
-  Check((result.decoded_dump.find("S_CBRANCH_CDBGSYS 0x00000440") != std::string::npos) &&
-            (result.ir_dump.find("StoreBufferU32") != std::string::npos),
-        "disabled system debug branch lost its identity or fallthrough write");
-  CheckSpirvBinaryValidates(result.spirv);
+  // Both debug branches fall through when conditional shader debugging is disabled.
+  // Retain the post-ENDPGM helper displacement from PPSA08709 and exercise the
+  // OR_USER variant encountered in PPSA03671 with the same body.
+  for (const auto opcode : {Decoder::Opcode::S_CBRANCH_CDBGSYS,
+                            Decoder::Opcode::S_CBRANCH_CDBGSYS_OR_USER}) {
+    std::array<uint32_t, 274> shader;
+    shader.fill(EncodeSopp(0x00));
+    shader[0] = EncodeSopp(opcode == Decoder::Opcode::S_CBRANCH_CDBGSYS ? 0x17 : 0x19, 0x10f);
+    shader[1] = EncodeVop1(0x01, 1, 129);
+    shader[2] = EncodeMubuf0(0x1c, 0, false);
+    shader[3] = EncodeMubuf1(1, 0, 0);
+    shader[271] = EncodeSopp(0x01);
+    shader[272] = 0xffffffffu; // Debug-only helper must not be decoded.
+    shader[273] = 0xffffffffu;
+    Decoder::Program decoded;
+    Decoder::DecodeProgram(shader, decoded);
+    Check(decoded.instructions.front().opcode == opcode &&
+              decoded.instructions.front().branch_target == 0x440u &&
+              decoded.instructions.back().pc == 0x43cu &&
+              decoded.instructions.back().opcode == Decoder::Opcode::S_ENDPGM,
+          "disabled debug branch reached its post-ENDPGM validation helper");
+    auto graph = CFG::BuildGraph(decoded);
+    Check(graph.blocks.size() == 1u && graph.FindBlockByPc(0x440u) == nullptr &&
+              graph.blocks.front().terminator.kind == CFG::TerminatorKind::Return,
+          "disabled debug branch changed normal shader control flow");
+    auto options = MakeCompileOptions(ShaderType::Compute);
+    options.dump_ir = true;
+    auto result = RecompileForTest(shader, options);
+    Check((result.decoded_dump.find(" 0x00000440") != std::string::npos) &&
+              (result.ir_dump.find("StoreBufferU32") != std::string::npos),
+          "disabled debug branch lost its identity or fallthrough write");
+    CheckSpirvBinaryValidates(result.spirv);
+  }
 }
 
 void TestNewShaderRecompilerRdna2ScalarOpcodes() {
@@ -8580,11 +8584,17 @@ void TestNestedSelectionPreservesDescriptorSources() {
           "nested descriptor selection lost its structured resource bindings");
     for (size_t i = 0; i < result.program.info.buffers.size(); ++i) {
       const auto pc = result.program.info.buffers[i].first_use_pc;
-      const uint32_t expected = pc == 8u * 4u ? table[0]
-                                : pc == 14u * 4u ? table[4]
-                                : pc == direct_arm * 4u ? user_data[8]
-                                : pc == join * 4u ? user_data[4] : 0u;
-      Check(expected != 0u && result.resources.buffers[i].dwords[0] == expected,
+      Check(pc == 8u * 4u || pc == 14u * 4u || pc == direct_arm * 4u || pc == join * 4u,
+            "nested descriptor selection lost a native source location");
+      const auto selected_pc = (mode == 0u ? 8u : mode == 1u ? 14u : direct_arm) * 4u;
+      const auto *expected = pc == 8u * 4u ? table.data()
+                           : pc == 14u * 4u ? table.data() + 4
+                           : pc == direct_arm * 4u ? user_data.data() + 8 : user_data.data() + 4;
+      const auto &descriptor = result.resources.buffers[i];
+      const bool unused = pc != selected_pc && pc != join * 4u;
+      Check(descriptor.dword_count == 4 &&
+                (std::equal(expected, expected + 4, descriptor.dwords.begin()) ||
+                 (unused && std::ranges::all_of(descriptor.dwords, [](auto word) { return word == 0; }))),
             "nested descriptor selection chose another arm's descriptor");
     }
     CheckSpirvBinaryValidates(result.spirv);
@@ -8692,10 +8702,10 @@ void TestNativeScalarReadDescriptorPlanning() {
     auto options = MakeCompileOptions(ShaderType::Pixel);
     options.user_data = user_data;
     const auto result = RecompileForTest(shader, options, ReadHostTestMemory);
-    Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u &&
-              std::ranges::equal(result.resources.flattened_srt, table),
-          "scalar descriptor planning retained an unrelated native execution mask");
     const auto expected = mode == 2u ? table : std::array<uint32_t, 4>{};
+    Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u &&
+              std::ranges::equal(result.resources.flattened_srt, expected),
+          "scalar descriptor planning retained an unrelated native execution mask");
     Check(std::equal(expected.begin(), expected.end(), result.resources.buffers[0].dwords.begin()),
           "scalar descriptor planning selected the wrong native resource");
     CheckSpirvBinaryValidates(result.spirv);
@@ -10104,6 +10114,44 @@ void TestMergedShaderUserDataSnapshot() {
   Check(fan_input.mesh.primitives_per_group == 30 && fan_input.mesh.vertices_per_group == 32 &&
             fan_input.mesh.max_vertices == 256 && fan_input.mesh.max_primitives == 192,
         "captured triangle-fan GS configuration lost its subgroup assembly limits");
+  context.SetShaderStages(0x00482030u);
+  context.SetMaxOutputPerSubgroup(32);
+  context.SetGsMaxVertOut(32);
+  user_config.SetPrimitiveType(Prospero::PrimitiveType::kPointList);
+  user_config.SetGeControl({1, 1});
+  regs.gs_regs.rsrc1.gs_vgpr_component_count = 0;
+  regs.gs_regs.rsrc2.es_vgpr_component_count = 1;
+  ShaderVertexInputInfo fast_input{};
+  PrepareProgram(regs, context, user_config, fast_input);
+  Check(fast_input.mesh.fast_launch && fast_input.mesh.threads_num[0] == 32 &&
+            fast_input.mesh.primitives_per_group == 1 && fast_input.mesh.vertices_per_group == 1 &&
+            fast_input.mesh.max_vertices == 32 && fast_input.mesh.max_primitives == 32,
+        "captured fast-launch shader lost its per-group launch and output limits");
+  auto ordinary_input = fast_input;
+  ordinary_input.mesh.fast_launch = false;
+  Check(MakeStageStaticKey(fast_input) != MakeStageStaticKey(ordinary_input),
+        "fast launch shared the ordinary input-assembly shader key");
+
+  const uint32_t fast_code[] = {
+      EncodeSMovB32(124, 255), 0x20020u, // allocate the full 32 vertices and primitives
+      EncodeSopp(0x10, 9),
+      EncodeVop1(0x01, 0, 256 + 5),
+      EncodeVop1(0x01, 1, 256 + 6),
+      EncodeExp0(0x0c, 0xf, false), EncodeExp1(0, 1, 0, 1),
+      EncodeExp0(0x14, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01),
+  };
+  options.input_info.vertex = &fast_input;
+  options.wave_size = fast_input.mesh.wave_size;
+  options.back_code = {};
+  const auto fast_result = RecompileForTest(fast_code, options);
+  CheckSpirvBinaryValidates(fast_result.spirv);
+  Check(fast_result.program.memory_info.empty(),
+        "compiled fast-launch entry retained an unused index-buffer resource");
+  const auto fast_source = DisassembleSpirvBinary(fast_result.spirv);
+  Check(SpirvSourceHasInstructionUsing(fast_source, "OpExecutionMode", "OutputVertices 32") &&
+            SpirvSourceHasInstructionUsing(fast_source, "OpExecutionMode", "OutputPrimitivesEXT 32"),
+        "fast-launch SPIR-V truncated the mesh output capacity");
+
 }
 
 void TestEmbeddedFetchPreservesSharedScalarLoad() {
@@ -10152,6 +10200,39 @@ void TestEmbeddedFetchPreservesSharedScalarLoad() {
                        result.resources.buffers[0].dwords.begin()),
         "embedded fetch discarded a scalar-load component used by another buffer read");
   CheckSpirvBinaryValidates(result.spirv);
+
+  const auto key = MakeStageStaticKey(input);
+  const auto compile = [&](const ShaderVertexInputInfo &vertex) {
+    auto variant_options = options;
+    variant_options.input_info.vertex = &vertex;
+    return RecompileForTest(code, variant_options).spirv;
+  };
+  auto layout = input;
+  layout.resources_dst[0].register_start = 32;
+  layout.resources_dst[0].fetch_index = 1;
+  layout.resources[0].UpdateAddress48(0x56780000u);
+  layout.resources[0].fields[1] = 52u << 16u;
+  layout.resources[0].fields[2] = 128;
+  layout.resources[0].fields[3] |= 3u << 28u;
+  layout.buffers_num = 1;
+  layout.buffers[0] = {.addr = 0x56780000u, .stride = 52, .num_records = 128,
+                       .fetch_index = 1};
+  Check(compile(layout) == result.spirv,
+        "pipeline-only vertex layout changed the embedded-fetch shader");
+  Check(MakeStageStaticKey(layout) == key,
+        "pipeline-only vertex layout fragmented the shader module cache key");
+
+  auto formatted = input;
+  formatted.resources[0].fields[3] =
+      (static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32UInt) << 12u) |
+      DstSel(4, 5, 6, 7);
+  Check(compile(formatted) != result.spirv && MakeStageStaticKey(formatted) != key,
+        "vertex input numeric type lost its shader module specialization");
+  auto swizzled = input;
+  swizzled.resources[0].fields[3] =
+      (swizzled.resources[0].fields[3] & ~0xfffu) | DstSel(5, 4, 6, 7);
+  Check(compile(swizzled) != result.spirv && MakeStageStaticKey(swizzled) != key,
+        "vertex input destination selectors lost their shader module specialization");
 }
 
 void TestEmbeddedVertexFormatSwizzle() {
@@ -10203,7 +10284,7 @@ void TestEmbeddedVertexFormatSwizzle() {
     Decoder::Program decoded;
     Decoder::DecodeProgram(code, decoded);
     const uint32_t attribute = static_cast<uint32_t>(test.attribute_format) << 5u;
-    const uint32_t buffer[] = {0x10000000u, 16u << 16u, 1u,
+    uint32_t buffer[] = {0x10000000u, 16u << 16u, 1u,
                                (static_cast<uint32_t>(test.format) << 12u) | test.swizzle};
     HW::VertexShaderInfo regs{};
     regs.es_regs.data_addr = reinterpret_cast<uint64_t>(code);
@@ -10222,6 +10303,16 @@ void TestEmbeddedVertexFormatSwizzle() {
     ShaderMapUserData(regs.es_regs.data_addr, mapped);
     ShaderVertexInputInfo input{};
     PrepareProgram(regs, HW::Context{}, HW::UserConfig{}, input);
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+    if (&test == cases) {
+      for (const auto [word, flag] : {std::pair{1u, 1u << 31u}, std::pair{3u, 1u << 23u}}) {
+        buffer[word] |= flag;
+        ExpectFatal([&] { PrepareProgram(regs, HW::Context{}, HW::UserConfig{}, input); },
+                    "unsupported vertex fetch flags bypassed acquisition validation");
+        buffer[word] &= ~flag;
+      }
+    }
+#endif
     Frontend::EmbeddedFetchPlan fetch;
     fetch.loads.push_back({.pc = 0, .attrib_id = 0, .components = test.components});
     Frontend::TranslateOptions options{};
@@ -10274,8 +10365,10 @@ void TestMeshInputAssembly() {
     uint32_t wave_info, first, second, third, byte_offset, vertex_id;
     bool fetch;
     uint32_t wave_size = 64;
+    bool fast_launch = false;
+    uint32_t threads = 256;
   };
-  const Case cases[] = {
+  std::vector<Case> cases = {
       {Prospero::PrimitiveType::kTriList, 14, 177, 14, 2, 2, 0x1002, UINT32_MAX,
        0x40000309, 6, 7, 8, 340, 0xabcc, true},
       {Prospero::PrimitiveType::kTriList, 14, 177, 14, 9, 2, 0x1002, 0,
@@ -10323,14 +10416,30 @@ void TestMeshInputAssembly() {
       {Prospero::PrimitiveType::kTriStrip, 40, 40, 0, 32, 0, 0, 11,
        0x81000608, 32, 33, 34, 0, 43, false, 32},
   };
+  for (const uint32_t wave_size : {32u, 64u}) {
+    for (const uint32_t threads : {wave_size, 2u * wave_size}) {
+      for (const uint32_t group : {0u, 1u, 11u}) {
+        for (const uint32_t lane : {0u, wave_size - 1u, threads - 1u}) {
+          const uint32_t wave_info = ((threads / wave_size) << 28u) |
+                                     ((lane / wave_size) << 24u) |
+                                     (lane < wave_size ? 0x101u : 0u);
+          const uint32_t base_vertex = group == 11u ? UINT32_MAX : 11u;
+          cases.push_back({Prospero::PrimitiveType::kPointList, 1, 12, group, lane, 0, 0,
+                           base_vertex, wave_info, 0, 0, 0, 0, group + base_vertex, false,
+                           wave_size, true, threads});
+        }
+      }
+    }
+  }
   for (const auto &test : cases) {
     ShaderVertexInputInfo input{};
     auto &mesh = input.mesh;
     mesh.input_primitive = static_cast<uint32_t>(test.topology);
     mesh.wave_size = test.wave_size;
+    mesh.fast_launch = test.fast_launch;
     mesh.primitives_per_group = mesh.InputPrimitiveCount(test.capacity);
     mesh.vertices_per_group = mesh.InputVertexCount(mesh.primitives_per_group);
-    mesh.threads_num[0] = 256;
+    mesh.threads_num[0] = test.threads;
     mesh.threads_num[1] = mesh.threads_num[2] = 1;
     Decoder::Program decoded;
     CFG::Graph graph;
@@ -10363,16 +10472,21 @@ void TestMeshInputAssembly() {
       }
     }
     ConstantPropagationPass(program.blocks);
-    Check(load != nullptr && load->Arg(1).Resolve().U32() == test.byte_offset &&
-              load->Arg(3).Resolve().U1() == test.fetch,
-          "mesh index fetch address or active-lane predicate is wrong");
-    const auto *resource = load->Arg(0).ResolveInstruction();
-    Check(resource != nullptr && resource->Arg(0).Resolve().U32() == (test.address_low & ~3u) &&
-              resource->Arg(1).Resolve().U32() == 0x12 &&
-              program.memory_info[load->Flags<MemoryFlags>().index].kind == ResourceKind::Global,
-          "mesh index fetch lost its aligned guest address resource");
-    load->ReplaceUsesWith(Value(test.fetch ? 0xabcd0123u : 0u));
-    ConstantPropagationPass(program.blocks);
+    if (test.fast_launch) {
+      Check(load == nullptr && program.memory_info.empty(),
+            "fast launch emitted an ordinary input-assembly index resource");
+    } else {
+      Check(load != nullptr && load->Arg(1).Resolve().U32() == test.byte_offset &&
+                load->Arg(3).Resolve().U1() == test.fetch,
+            "mesh index fetch address or active-lane predicate is wrong");
+      const auto *resource = load->Arg(0).ResolveInstruction();
+      Check(resource != nullptr && resource->Arg(0).Resolve().U32() == (test.address_low & ~3u) &&
+                resource->Arg(1).Resolve().U32() == 0x12 &&
+                program.memory_info[load->Flags<MemoryFlags>().index].kind == ResourceKind::Global,
+            "mesh index fetch lost its aligned guest address resource");
+      load->ReplaceUsesWith(Value(test.fetch ? 0xabcd0123u : 0u));
+      ConstantPropagationPass(program.blocks);
+    }
     std::array<uint32_t, 9> vgprs{};
     uint32_t sgpr3 = 0;
     for (const auto &inst : *program.blocks.front()) {
@@ -10383,7 +10497,8 @@ void TestMeshInputAssembly() {
       }
     }
     Check(sgpr3 == test.wave_info && vgprs[0] == ((test.first << 2) | (test.second << 18)) &&
-              vgprs[1] == test.third * 4 && vgprs[5] == test.vertex_id && vgprs[8] == 9,
+              vgprs[1] == test.third * 4 && vgprs[5] == test.vertex_id &&
+              vgprs[test.fast_launch ? 6 : 8] == 9u,
           "mesh prolog changed input assembly, wave counts, vertex ID, or instance ID");
   }
 }
@@ -10623,35 +10738,6 @@ void TestNewShaderRecompilerClipDisabledPosition() {
   shifted.clip_space.offset[0] += 1.0f;
   Check(MakeStageStaticKey(clipped) != MakeStageStaticKey(shifted),
         "clip-disabled viewport transform is absent from the shader cache key");
-
-  ShaderVertexInputInfo layout_a{};
-  layout_a.resources_num = 1;
-  layout_a.buffers_num = 1;
-  layout_a.buffers[0].stride = 16;
-  auto layout_b = layout_a;
-  layout_b.buffers[0].stride = 32;
-  layout_b.buffers[0].fetch_index = 1;
-  layout_b.resources[0].UpdateAddress48(4);
-  Check(MakeStageStaticKey(layout_a) == MakeStageStaticKey(layout_b),
-        "pipeline-only vertex layout fragmented the shader module cache key");
-
-  const auto descriptor_state = [](const ShaderBufferResource &resource) {
-    return std::array<uint32_t, 9>{
-        resource.Stride(), resource.SwizzleEnabled(), resource.DstSelX(),
-        resource.DstSelY(), resource.DstSelZ(), resource.DstSelW(),
-        resource.RawFormat(), resource.OutOfBounds(), resource.AddTid()};
-  };
-  const auto key = MakeStageStaticKey(layout_a);
-  const auto state = descriptor_state(layout_a.resources[0]);
-  for (uint32_t word = 0; word < 4; word++) {
-    for (uint32_t bit = 0; bit < 32; bit++) {
-      auto changed = layout_a;
-      changed.resources[0].fields[word] ^= 1u << bit;
-      Check((MakeStageStaticKey(changed) == key) ==
-                (descriptor_state(changed.resources[0]) == state),
-            "vertex key lost a descriptor field or included runtime-only bits");
-    }
-  }
 }
 
 void TestNewShaderRecompilerAuxPositionExports() {
@@ -11479,6 +11565,124 @@ void TestValuePhiValidation() {
 #endif
 }
 
+void TestWave32MaskProjection() {
+  using namespace ShaderRecompiler::IR;
+  for (const uint32_t wave_size : {32u, 64u}) {
+    Program program;
+    program.stage = ShaderType::Compute;
+    program.wave_size = wave_size;
+    for (uint32_t id = 0; id < 3; ++id) {
+      program.block_storage.push_back(std::make_unique<Block>());
+      program.blocks.push_back(program.block_storage.back().get());
+      program.block_info.push_back({.id = id});
+    }
+    auto *entry = program.blocks[0];
+    auto *loop = program.blocks[1];
+    entry->AddBranch(loop);
+    loop->AddBranch(loop);
+    loop->AddBranch(program.blocks[2]);
+    program.block_info[0].terminator.kind = ShaderRecompiler::CFG::TerminatorKind::Branch;
+    program.block_info[0].terminator.true_block = 1;
+    program.block_info[1].terminator.kind = ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch;
+    program.block_info[1].terminator.true_block = 1;
+    program.block_info[1].terminator.false_block = 2;
+    program.block_info[2].terminator.kind = ShaderRecompiler::CFG::TerminatorKind::Return;
+    IREmitter e(entry);
+    const auto zero = U32(Value(0u));
+    const auto raw = e.GetUserData(static_cast<ScalarReg>(0));
+    const auto active = e.INotEqual(raw, zero);
+    const auto selected = e.IEqual(raw, U32(Value(7u)));
+    const auto ballot = [&](IREmitter &ir, U1 predicate, uint32_t component = 0u) {
+      return ir.CompositeExtract(ir.Emit(ValueOpcode::Ballot, {predicate}), component);
+    };
+    const auto thread_bit = [&](IREmitter &ir, U32 word, bool own_lane = true) {
+      const auto bit = own_lane
+          ? ir.BitwiseAnd(U32(ir.Emit(ValueOpcode::LaneId)), U32(Value(31u)))
+          : U32(Value(3u));
+      return ir.INotEqual(ir.BitwiseAnd(ir.ShiftRightLogical(word, bit), U32(Value(1u))), zero);
+    };
+    const auto active_mask = ballot(e, active);
+    const auto direct = thread_bit(e, active_mask);
+    const auto arbitrary = thread_bit(e, raw);
+    const auto mixed = thread_bit(e, e.BitwiseAnd(active_mask, raw));
+    const auto mixed_or = thread_bit(e, e.BitwiseOr(active_mask, raw));
+    const auto mixed_select = thread_bit(e, e.Select(selected, active_mask, raw));
+    const auto different_lane = thread_bit(e, active_mask, false);
+    const auto high_half = thread_bit(e, ballot(e, active, 1u));
+    const auto initial_mask = e.BitwiseAnd(active_mask, ballot(e, selected));
+    const auto raw_consumer = e.Emit(ValueOpcode::IAdd32, {initial_mask, Value(3u)});
+    const auto initial_predicate = e.LogicalAnd(active, selected);
+    const auto entry_size = entry->Instructions().size();
+    ConstantPropagationPass({entry}, wave_size);
+    Check(entry->Instructions().size() == entry_size,
+          "rejected mixed mask graph left unused projected instructions");
+    const auto combined_mask = e.BitwiseOr(active_mask, ballot(e, selected));
+    const auto combined = thread_bit(e, combined_mask);
+    const auto gated_mask = e.Select(selected, combined_mask, zero);
+    const auto gated = thread_bit(e, gated_mask);
+    const auto rejected = thread_bit(e, e.Select(selected, zero, combined_mask));
+    const auto combined_predicate = e.LogicalOr(active, selected);
+    const auto gated_predicate = e.LogicalAnd(selected, combined_predicate);
+    const auto rejected_predicate = e.Emit(ValueOpcode::SelectU1,
+                                          {selected, Value(false), combined_predicate});
+    const auto gated_raw_consumer = e.Emit(ValueOpcode::IAdd32, {gated_mask, Value(3u)});
+    auto &mask = loop->AppendNewInst(ValueOpcode::Phi);
+    mask.SetFlags(Type::U32);
+    auto &predicate = loop->AppendNewInst(ValueOpcode::Phi);
+    predicate.SetFlags(Type::U1);
+    IREmitter body(loop);
+    const auto matches = body.LogicalAnd(U1(Value(&predicate)), selected);
+    const auto remaining = body.BitwiseAnd(U32(Value(&mask)), body.BitwiseNot(ballot(body, matches)));
+    const auto remaining_predicate = body.LogicalAnd(U1(Value(&predicate)), body.LogicalNot(matches));
+    const auto remaining_live = thread_bit(body, remaining);
+    const auto live = thread_bit(body, U32(Value(&mask)));
+    mask.AddPhiOperand(entry, initial_mask);
+    mask.AddPhiOperand(loop, remaining);
+    predicate.AddPhiOperand(entry, initial_predicate);
+    predicate.AddPhiOperand(loop, remaining_predicate);
+    program.block_info[1].condition = remaining_predicate;
+
+    ConstantPropagationPass(program.blocks, wave_size);
+    if (wave_size == 32u) {
+      Check(direct.Resolve() == active.Resolve(), "wave32 ballot round trip lost its predicate");
+      Check(EquivalentValue(program, combined, combined_predicate),
+            "wave32 OR mask projection changed a lane predicate");
+      Check(EquivalentValue(program, gated, gated_predicate) &&
+                EquivalentValue(program, rejected, rejected_predicate),
+            "wave32 select mask projection changed a scalar gate or selected arm");
+      Check(EquivalentValue(program, live.Resolve(), Value(&predicate)),
+            "projected mask Phi differs from the existing per-lane EXEC Phi");
+      Check(EquivalentValue(program, remaining_live.Resolve(), remaining_predicate) &&
+                std::count_if(loop->begin(), loop->end(), [](const Inst &inst) {
+                  return inst.GetOpcode() == ValueOpcode::LogicalAnd;
+                }) == 3,
+            "backedge-first projection duplicated the loop's logical mask operation");
+      const auto *projected = live.Resolve().TryInstruction();
+      Check(projected != nullptr && projected->GetOpcode() == ValueOpcode::Phi &&
+                projected->GetType() == Type::U1 && projected->PhiBlock(0) == entry &&
+                projected->PhiBlock(1) == loop,
+            "mask projection lost loop Phi type or predecessor order");
+    } else {
+      Check(direct.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                live.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                combined.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                gated.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32 &&
+                rejected.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32,
+            "wave64 low-half mask was projected onto the wrong invocation");
+    }
+    for (const auto value : {arbitrary, mixed, mixed_or, mixed_select, different_lane, high_half}) {
+      Check(value.Resolve().TryInstruction()->GetOpcode() == ValueOpcode::INotEqual32,
+            "mask projection accepted an arbitrary mask, lane or ballot half");
+    }
+    Check(raw_consumer.Resolve().TryInstruction()->Arg(0).Resolve() == initial_mask.Resolve() &&
+              gated_raw_consumer.Resolve().TryInstruction()->Arg(0).Resolve() == gated_mask.Resolve(),
+          "mask projection changed a raw scalar mask consumer");
+    ResolveControlFlowIdentities(program);
+    RemoveIdentities(program.blocks);
+    ValidateProgram(program, true);
+  }
+}
+
 void TestU64ShiftConstantPropagation() {
   using namespace ShaderRecompiler::IR;
 
@@ -13503,11 +13707,11 @@ void TestPixelProgramCacheBindingIdentity() {
   Check(MakeStageStaticKey(no_depth_export) != MakeStageStaticKey(depth_export),
         "pixel shader identity omitted depth-export semantics");
 
-  auto check_program_identity = [&](const uint32_t *shader) {
+  auto check_program_identity = [&](std::span<const uint32_t> shader) {
     HW::PixelShaderInfo regs{};
-    regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(shader);
+    regs.ps_regs.data_addr = reinterpret_cast<uint64_t>(shader.data());
     ShaderMappedData mapped{};
-    mapped.code_size_bytes = sizeof(uint32_t);
+    mapped.code_size_bytes = static_cast<uint32_t>(shader.size_bytes());
     ShaderMapUserData(regs.ps_regs.data_addr, mapped);
 
     const std::array<Prospero::ColorComponentMapping, 8> identity_mappings{};
@@ -13522,7 +13726,8 @@ void TestPixelProgramCacheBindingIdentity() {
                                               second_info);
     CompilePixelRuntime(second_params, second_info);
     Check(
-        first_params.hash == second_params.hash &&
+        first_params.hash == XXH3_64bits(shader.data(), shader.size_bytes()) &&
+            first_params.hash == second_params.hash &&
             first_key == MakeStageStaticKey(second_info) &&
             first_info.stage.program != nullptr &&
             second_info.stage.program != nullptr,
@@ -13535,6 +13740,22 @@ void TestPixelProgramCacheBindingIdentity() {
   Check(request_01 == request_10,
         "relocated identical pixel programs did not share their source identity");
 
+  // Identical metadata after the scalar-literal prefix must not mask changes
+  // to the executable code when computing program identity.
+  std::array<uint32_t, 200> shader{};
+  shader[0] = 0xbeeb03ffu;
+  shader[1] = 0x5fu;
+  shader[2] = EncodeSMovB32(0, 129);
+  shader[3] = EncodeSopp(0x01);
+  shader[192] = 0x65726162u;
+  shader[193] = 0x746f6f66u;
+  shader[196] = 0x12345678u;
+  shader[197] = 0x9abcdef0u;
+  const auto first = check_program_identity(shader);
+  shader[2] = EncodeSMovB32(0, 130);
+  Check(check_program_identity(shader).first != first.first,
+        "re-registering changed code at the same address reused its old identity");
+
 }
 
 void TestGraphicsPushConstantPlacement() {
@@ -13637,6 +13858,30 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() {
   Check((result.ir_dump.find("GetScratchResource") != std::string::npos) &&
             result.program.scratch_dwords == 1,
         "SCRATCH incorrectly entered guest address tracking");
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+  // A local aperture proof must not admit a median which can still produce a
+  // canonical global address. Such stores need the existing ownership proof.
+  const uint32_t mixed_domain_store[] = {
+      EncodeSop1(0x04, 8, 237), // s_mov_b64 s[8:9], private_base
+      EncodeVop3Word0(0x159, 1),
+      EncodeVop3Word1(128, 9, 256 + 2), // median(0, private high, dynamic v2)
+      EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0),
+      EncodeSopp(0x01),
+  };
+  ExpectFatal([&]() { (void)RecompileForTest(mixed_domain_store, options); },
+              "mixed local/global FLAT store bypassed GPU ownership tracking");
+  const uint32_t changed_mask_store[] = {
+      EncodeSop1(0x04, 8, 237), EncodeSop1(0x04, 10, 235),
+      EncodeVop1(0x01, 1, 128), // canonical old address high
+      EncodeSop1(0x03, 126, 255), 0x55555555u,
+      EncodeVop3Word0(0x159, 1), EncodeVop3Word1(9, 11, 256 + 2),
+      EncodeSop1(0x04, 126, 193), // restore all lanes after the median
+      EncodeFlat0(0x1c, 0, 0), EncodeFlat1(0, 0x7d, 3, 0),
+      EncodeSopp(0x01),
+  };
+  ExpectFatal([&]() { (void)RecompileForTest(changed_mask_store, options); },
+              "FLAT proof discarded a differently predicated address value");
+#endif
 }
 
 void TestCompilerStageInputOwnership() {
@@ -14022,7 +14267,7 @@ int main() {
   using namespace Libs::Graphics;
 
   EnsureConfigInitialized();
-  TestRayTracingDispatchDetection();
+  TestRayTracingInstructions();
   TestResourceDescriptorClassification();
   TestShaderBufferResourceSize();
   TestNativeShaderResourceDependencies();
@@ -14125,7 +14370,7 @@ int main() {
   TestNewShaderRecompilerBufferLoadsGuardedByExec();
   TestNewShaderRecompilerBufferAtomicsGuardedByBounds();
   TestCapturedBufferAtomicsX2();
-  TestDisabledSystemDebugBranch();
+  TestDisabledDebugBranches();
   TestNewShaderRecompilerPixelImageSampleLodSelection();
   TestNewShaderRecompilerBranchConditionForms();
   TestNewShaderRecompilerSetpcBranch();
@@ -14143,6 +14388,7 @@ int main() {
   TestFinalSsaRejectsRegisterStatePseudos();
 #endif
   TestValuePhiValidation();
+  TestWave32MaskProjection();
   TestU64ShiftConstantPropagation();
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
   TestNativeWideValueValidation();