launcher: Raise DualSense speaker volume range for SpeakerFX (#1043)

controller: restore speaker midpoint and raise volume ceiling

3rdparty/cpuinfo/src/arm/mach/init.c

@@ -14,7 +14,7 @@
 #include <cpuinfo/internal-api.h>
 #include <cpuinfo/log.h>
 
-/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */
+/* Polyfill recent CPUFAMILY_ARM_* values for older platform headers */
 #ifndef CPUFAMILY_ARM_MONSOON_MISTRAL
 	#define CPUFAMILY_ARM_MONSOON_MISTRAL   0xE81E7EF6
 #endif

README.md

@@ -1,8 +1,8 @@
 # KytyPS5
 
-[![Build KytyPS5 (Windows)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (Linux)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
-[![Build KytyPS5 (macOS)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (Windows)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28Windows%29&label=Build%20KytyPS5%20%28Windows%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (Linux)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28Linux%29&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
+[![Build KytyPS5 (macOS)](https://img.shields.io/github/check-runs/KytyPS5/KytyPS5/main?nameFilter=Build%20KytyPS5%20%28macOS%29&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
 [![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20Linux%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
 [![Status](https://img.shields.io/badge/status-active%20development-orange.svg)](#current-status)
 [![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
@@ -72,8 +72,8 @@ or graphical glitches, so please include the version you tested when reporting a
       <img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
     </td>
     <td align="center">
-      <strong>UFC 5</strong><br>
-      <img src="docs/screenshots/ps5-06.png" width="300" alt="UFC 5 running in KytyPS5">
+      <strong>UFC 6</strong><br>
+      <img src="docs/screenshots/ps5-06.jpg" width="300" alt="UFC 6 running in KytyPS5">
     </td>
   </tr>
 </table>

src/common/emulatorConfig.h

@@ -48,7 +48,7 @@ struct ConfigOptions {
 	int32_t                user_id                     = DEFAULT_USER_ID;
 	std::string            audio_input_device;
 	std::optional<ControllerColor> controller_color;
-	uint32_t               controller_speaker_volume      = 100;
+	uint32_t               controller_speaker_volume      = 50;
 	uint32_t               controller_vibration_intensity = 100;
 	PresentMode            present_mode                = PresentMode::Mailbox;
 	int32_t                gpu_index                   = -1;

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -258,15 +258,8 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
 		}
 		auto& buffer = m_slot_buffers[FindBuffer(vaddr, size)];
 
-		// Widen nearby CPU reads so they share one GPU drain.
-		constexpr uint64_t WindowSize   = 512 * 1024;
-		const auto         buffer_begin = buffer.CpuAddress();
-		const auto         buffer_end   = buffer_begin + buffer.Size();
-		const auto window_begin = std::max(Common::AlignDown(vaddr, WindowSize), buffer_begin);
-		const auto window_end = std::min(std::max(window_begin + WindowSize, vaddr + size), buffer_end);
-
-		if (DownloadBufferMemory<false>(buffer, window_begin, window_end - window_begin)) {
-			m_memory_tracker.UnmarkRegionAsGpuModified(window_begin, window_end - window_begin);
+		if (DownloadBufferMemory<false>(buffer, vaddr, size)) {
+			m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
 		}
 		if (is_write) {
 			m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1037,6 +1037,8 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	auto info = destination.info;
 	if (image.depth_id) {
 		info.data            = image.info.data;
+		info.resources       = {image.stencil_subresources.level_count,
+		                        image.stencil_subresources.layer_count};
 		info.guest_format    = Prospero::BufferFormat::k8UInt;
 		info.bytes_per_block = 1;
 		if (info.IsTiled()) info.pitch = TileGetDepthPitch(info.extent.width, 1, 0);
@@ -1051,6 +1053,11 @@ void TextureCache::UploadImage(Image& image, Buffer& source, uint64_t source_off
 	auto copies = BuildDepthCopies(info, full_slice_size, image.depth_id
 	                                                        ? vk::ImageAspectFlagBits::eStencil
 	                                                        : vk::ImageAspectFlagBits::eDepth);
+	if (image.depth_id) {
+		for (auto& copy: copies) {
+			copy.imageSubresource.baseArrayLayer += image.stencil_subresources.base_layer;
+		}
+	}
 	TileManager::Result linear {source.Handle(), source_offset, source.Size() - source_offset};
 	if (info.IsTiled()) {
 		const auto tiles = BuildDepthTiles(info);
@@ -1239,7 +1246,8 @@ ImageId TextureCache::AssociateStencil(ImageId depth_id, GuestRange stencil) {
 	}
 	auto& record = m_slot_images[association];
 	TouchImage(record);
-	record.depth_id = depth_id;
+	record.depth_id             = depth_id;
+	record.stencil_subresources = depth.stencil_subresources;
 	return association;
 }
 
@@ -1445,6 +1453,18 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	TouchImage(image);
 	image.MarkGpuModified();
 	image.usage.depth_target = true;
+	if (desc.info.HasStencil()) {
+		const auto layers = image.info.resources.layers;
+		EXIT_IF(layers == 0 || image.info.data.size % layers != 0 ||
+		        desc.info.data.address < image.info.data.address);
+		const auto slice_size = image.info.data.size / layers;
+		const auto offset     = desc.info.data.address - image.info.data.address;
+		EXIT_IF(slice_size == 0 || offset % slice_size != 0 || offset / slice_size >= layers ||
+		        desc.info.resources.layers > layers - offset / slice_size);
+		image.stencil_subresources = {0, desc.info.resources.levels,
+		                             static_cast<uint32_t>(offset / slice_size),
+		                             desc.info.resources.layers};
+	}
 	image.info.stencil = desc.info.stencil;
 	image.info.metadata = desc.info.metadata;
 	if (desc.info.HasMetadata()) {
@@ -1491,9 +1511,11 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid image clear\n");
 	}
-	std::scoped_lock     lock {m_lock};
-	ImageId              selected {};
-	vk::ImageAspectFlags aspect {};
+	std::scoped_lock      lock {m_lock};
+	ImageId               selected {};
+	ImageId               stencil_id {};
+	vk::ImageAspectFlags  aspect {};
+	ImageSubresourceRange subresources;
 	for (const auto id: FindImagesInRegion(address, size, false)) {
 		auto owner = m_slot_images.try_get(id);
 		if (owner == nullptr) {
@@ -1501,10 +1523,13 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		}
 		vk::ImageAspectFlags candidate {};
 		ImageId              candidate_id = id;
+		ImageSubresourceRange candidate_subresources {
+		    0, owner->info.resources.levels, 0, owner->info.TransferLayers()};
 		if (owner->depth_id && owner->info.data.address == address &&
 		    owner->info.data.size == size) {
-			candidate    = vk::ImageAspectFlagBits::eStencil;
-			candidate_id = owner->depth_id;
+			candidate              = vk::ImageAspectFlagBits::eStencil;
+			candidate_id           = owner->depth_id;
+			candidate_subresources = owner->stencil_subresources;
 			owner        = m_slot_images.try_get(candidate_id);
 			if (owner == nullptr || owner->backing.image == nullptr || !owner->info.HasStencil()) {
 				continue;
@@ -1517,11 +1542,13 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		if (!candidate) {
 			continue;
 		}
-		if (selected && selected != candidate_id) {
+		if (selected && (selected != candidate_id || subresources != candidate_subresources)) {
 			return false;
 		}
-		selected = candidate_id;
-		aspect   = candidate;
+		selected     = candidate_id;
+		stencil_id   = candidate == vk::ImageAspectFlagBits::eStencil ? id : ImageId {};
+		aspect       = candidate;
+		subresources = candidate_subresources;
 	}
 	if (!selected) {
 		return false;
@@ -1543,7 +1570,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 		clear.depthStencil.stencil = stencil_clear;
 	}
 	ClearImage(command, selected, image.backing.format,
-	           {aspect, 0, image.info.resources.levels, 0, image.info.TransferLayers()}, clear);
+	           {aspect, subresources.base_level, subresources.level_count,
+	            subresources.base_layer, subresources.layer_count}, clear);
+	if (stencil_id) {
+		TrackImage(stencil_id);
+		CommitGpuWrite(m_slot_images[stencil_id]);
+	}
 	return true;
 }
 
@@ -1856,23 +1888,6 @@ void TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size) {
 	}
 }
 
-bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
-	if (!GuestRange {address, size}.Valid()) {
-		return false;
-	}
-	std::scoped_lock lock {m_lock};
-	for (const auto id: FindImagesInRegion(address, size, false)) {
-		const auto& image = m_slot_images[id];
-		// PPSA17168: S_LOAD_DWORD reads shader data at an address overlapping an old
-		// render target whose memory the CPU has reused. The cached image still retains
-		// its earlier GPU-modified flag.
-		if (!image.depth_id && image.IsGpuModified() && !image.IsDefinitelyCpuDirty()) {
-			return true;
-		}
-	}
-	return false;
-}
-
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 	const auto page_begin = Common::AlignDown(address, TRACKER_PAGE_SIZE);
 	const auto page_end   = Common::AlignUp(address + size, TRACKER_PAGE_SIZE);

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -61,7 +61,6 @@ public:
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
 	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
-	[[nodiscard]] bool IsRegionGpuModified(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
 	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);

src/graphics/host_gpu/renderer/image/image.cpp

@@ -673,7 +673,9 @@ Prospero::BufferFormat RenderTargetTransferFormat(uint32_t bytes_per_element) {
 } // namespace ImageOps
 
 Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
-    : info(image_info), m_graphics(graphics), m_scheduler(scheduler) {
+    : info(image_info),
+      stencil_subresources {0, image_info.resources.levels, 0, image_info.resources.layers},
+      m_graphics(graphics), m_scheduler(scheduler) {
 	KYTY_PROFILER_FUNCTION();
 	ImageOps::Validate(info);
 	m_cpu_dirty =

src/graphics/host_gpu/renderer/image/image.h

@@ -148,6 +148,8 @@ public:
 	uint64_t         track_addr     = 0;
 	uint64_t         track_addr_end = 0;
 	ImageId          depth_id {};
+	// The current stencil plane's mapping into the depth image, also retained by its association.
+	ImageSubresourceRange stencil_subresources;
 	uint64_t         tick_accessed_last = 0;
 	size_t           lru_id             = 0;
 

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -130,7 +130,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
 	const auto aligned_offset = Common::AlignDown(offset, alignment);
 	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
-	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
+	if (adjustment >= 256 || size > max_range - adjustment) {
 		EXIT("storage buffer offset adjustment is unsupported\n");
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -94,8 +94,10 @@ void PipelineCacheLog(fmt::format_string<Args...> format, Args&&... args) {
 }
 
 bool ReadShaderGuestMemory(void*, uint64_t address, std::span<uint32_t> values) {
+	// Scalar and unformatted buffer dependencies use the same backing as native raw loads.
+	// Image synchronization belongs to formatted buffer bindings, not these reads.
 	return !values.empty() &&
-	       Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, values.data(), values.size_bytes());
+	       Libs::LibKernel::Memory::TryReadBufferBacking(address, values.data(), values.size_bytes());
 }
 
 void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -272,22 +272,79 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		return;
 	}
 
-	const bool large_workgroup =
-	    (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512);
-	const bool                   has_sampler = !program.info.samplers.empty();
-	static std::atomic<uint32_t> dispatch_log_count {0};
-	if ((large_workgroup || has_sampler) &&
-	    dispatch_log_count.fetch_add(1, std::memory_order_relaxed) < 512) {
+	if (use_thread_dimensions) {
+		auto groups_from_threads = [](uint32_t threads, uint32_t group_size) {
+			return (threads == 0
+			            ? 0u
+			            : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u));
+		};
+
+		const uint32_t old_x = thread_group_x;
+		const uint32_t old_y = thread_group_y;
+		const uint32_t old_z = thread_group_z;
+		thread_group_x       = groups_from_threads(thread_group_x, cs_regs.cs_regs.num_thread_x);
+		thread_group_y       = groups_from_threads(thread_group_y, cs_regs.cs_regs.num_thread_y);
+		thread_group_z       = groups_from_threads(thread_group_z, cs_regs.cs_regs.num_thread_z);
+
+		static std::atomic<uint32_t> log_count {0};
+		if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
+			LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> "
+			     "groups %ux%ux%u\n",
+			     old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u),
+			     std::max(cs_regs.cs_regs.num_thread_y, 1u),
+			     std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y,
+			     thread_group_z);
+		}
+	}
+
+	buffer.EndRendering();
+	auto& pipeline =
+	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
+	auto& bindings = m_compute_bindings;
+	PrepareBindings(input_info.stage, bindings);
+	if (program.bindings.dispatch_thread_dword != ShaderRecompiler::IR::PushData::NoStart) {
+		std::copy(std::begin(input_info.dispatch_threads_num), std::end(input_info.dispatch_threads_num),
+		          bindings.shader_data.begin() + program.bindings.dispatch_thread_dword);
+	}
+	FindBuffers(bindings);
+	if (program.info.uses_dma) {
+		m_context.PrepareBda();
+	}
+	RebindImages(bindings);
+	RebindBuffers(bindings);
+
+	auto              vk_buffer        = buffer.Handle();
+	PreparedBindings* descriptor_stage = &bindings;
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
+	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
+	has_storage_writes =
+	    std::any_of(program.info.images.begin(), program.info.images.end(),
+	                [](const auto& image) {
+		                return image.written &&
+		                       image.resource_class ==
+		                           ShaderRecompiler::IR::ImageResourceClass::Storage;
+	                }) ||
+	    has_storage_writes;
+	if (has_storage_writes) {
+		// A host fence used to serialize every dispatch. Preserve its read-before-write ordering
+		// while allowing the queue to execute asynchronously.
+		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
+	}
+	if (Config::GraphicsDebugDumpEnabled()) {
 		const auto sampled_images = std::count_if(
 		    program.info.images.begin(), program.info.images.end(), [](const auto& image) {
 			    return image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled;
 		    });
 		const uint32_t frame_num = static_cast<uint32_t>(m_context.GetGpu().GetFrameNum());
 		LOGF("GraphicsRenderDispatchDirect: frame=%u shader=0x%016" PRIx64
+		     " hash=0x%016" PRIx64 " tick=%" PRIu64
 		     " groups=%ux%ux%u mode=0x%08" PRIx32 " local=%ux%ux%u "
 		     "buffers=%zu textures=%zu sampled=%zu storage=%zu samplers=%zu push=%u\n",
-		     frame_num, sh_ctx.GetCs().cs_regs.data_addr, thread_group_x, thread_group_y,
-		     thread_group_z, mode, input_info.threads_num[0], input_info.threads_num[1],
+		     frame_num, sh_ctx.GetCs().cs_regs.data_addr, program.shader_hash,
+		     m_context.GetCommandScheduler().CurrentTick(),
+		     thread_group_x, thread_group_y, thread_group_z, mode,
+		     input_info.threads_num[0], input_info.threads_num[1],
 		     input_info.threads_num[2], program.info.buffers.size(), program.info.images.size(),
 		     sampled_images, program.info.images.size() - sampled_images,
 		     program.info.samplers.size(),
@@ -334,61 +391,6 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 		}
 	}
 
-	if (use_thread_dimensions) {
-		auto groups_from_threads = [](uint32_t threads, uint32_t group_size) {
-			return (threads == 0
-			            ? 0u
-			            : (threads + std::max(group_size, 1u) - 1u) / std::max(group_size, 1u));
-		};
-
-		const uint32_t old_x = thread_group_x;
-		const uint32_t old_y = thread_group_y;
-		const uint32_t old_z = thread_group_z;
-		thread_group_x       = groups_from_threads(thread_group_x, cs_regs.cs_regs.num_thread_x);
-		thread_group_y       = groups_from_threads(thread_group_y, cs_regs.cs_regs.num_thread_y);
-		thread_group_z       = groups_from_threads(thread_group_z, cs_regs.cs_regs.num_thread_z);
-
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
-			LOGF("GraphicsRenderDispatchDirect: use-thread-dimensions %ux%ux%u / %ux%ux%u -> "
-			     "groups %ux%ux%u\n",
-			     old_x, old_y, old_z, std::max(cs_regs.cs_regs.num_thread_x, 1u),
-			     std::max(cs_regs.cs_regs.num_thread_y, 1u),
-			     std::max(cs_regs.cs_regs.num_thread_z, 1u), thread_group_x, thread_group_y,
-			     thread_group_z);
-		}
-	}
-
-	buffer.EndRendering();
-	auto& pipeline =
-	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
-	auto& bindings = m_compute_bindings;
-	PrepareBindings(input_info.stage, bindings);
-	FindBuffers(bindings);
-	if (program.info.uses_dma) {
-		m_context.PrepareBda();
-	}
-	RebindImages(bindings);
-	RebindBuffers(bindings);
-
-	auto              vk_buffer        = buffer.Handle();
-	PreparedBindings* descriptor_stage = &bindings;
-	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
-	               std::span {&descriptor_stage, 1u});
-	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
-	has_storage_writes =
-	    std::any_of(program.info.images.begin(), program.info.images.end(),
-	                [](const auto& image) {
-		                return image.written &&
-		                       image.resource_class ==
-		                           ShaderRecompiler::IR::ImageResourceClass::Storage;
-	                }) ||
-	    has_storage_writes;
-	if (has_storage_writes) {
-		// A host fence used to serialize every dispatch. Preserve its read-before-write ordering
-		// while allowing the queue to execute asynchronously.
-		ShaderWriteHazardBarrier(vk_buffer, vk::PipelineStageFlagBits::eComputeShader);
-	}
 	vk_buffer.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline.pipeline);
 	vk_buffer.dispatch(thread_group_x, thread_group_y, thread_group_z);
 

src/graphics/presentation/window/vulkanWindow.cpp

@@ -47,6 +47,12 @@ struct VulkanExtensions {
 	std::vector<vk::LayerProperties>     available_layers;
 };
 
+vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
+	vk::PhysicalDeviceVulkan11Features features {};
+	features.storageBuffer16BitAccess = VK_TRUE;
+	return features;
+}
+
 vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
 	vk::PhysicalDeviceVulkan12Features features {};
 	features.samplerMirrorClampToEdge  = VK_TRUE;
@@ -55,6 +61,8 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
 	features.shaderOutputViewportIndex = VK_TRUE;
 	features.bufferDeviceAddress       = VK_TRUE;
 	features.shaderBufferInt64Atomics  = VK_TRUE;
+	features.storageBuffer8BitAccess   = VK_TRUE;
+	features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
 	return features;
 }
 
@@ -206,6 +214,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		depth_clip_control.pNext = &depth_clip_enable;
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
+		vk::PhysicalDeviceVulkan11Features features11 {};
 #if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
 #else
@@ -213,6 +222,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		fragment_barycentric.pNext = &depth_clip_control;
 		features12.pNext           = &fragment_barycentric;
 #endif
+		features11.pNext       = features12.pNext;
+		features12.pNext       = &features11;
 		features13.pNext       = &features12;
 		device_features2.pNext = &features13;
 
@@ -256,6 +267,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge",
 		              required_features12.samplerMirrorClampToEdge);
+		check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
+		              WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
+		check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
+		              required_features12.storageBuffer8BitAccess);
 		check_feature(features12.timelineSemaphore, "timelineSemaphore",
 		              required_features12.timelineSemaphore);
 		check_feature(features12.shaderOutputLayer, "shaderOutputLayer",
@@ -266,6 +281,9 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		              required_features12.bufferDeviceAddress);
 		check_feature(features12.shaderBufferInt64Atomics, "shaderBufferInt64Atomics",
 		              required_features12.shaderBufferInt64Atomics);
+		check_feature(features12.shaderSampledImageArrayNonUniformIndexing,
+		              "shaderSampledImageArrayNonUniformIndexing",
+		              required_features12.shaderSampledImageArrayNonUniformIndexing);
 		check_feature(features13.robustImageAccess, "robustImageAccess");
 		check_feature(features13.dynamicRendering, "dynamicRendering",
 		              required_features13.dynamicRendering);
@@ -478,6 +496,9 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	workgroup_layout.pNext = &depth_clip_control;
 	features12.pNext = workgroup_layout_extension ? static_cast<void*>(&workgroup_layout)
 	                                             : static_cast<void*>(&depth_clip_control);
+	auto features11 = WindowContext::RequiredVulkan11Features();
+	features11.pNext = features12.pNext;
+	features12.pNext = &features11;
 	if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) {
 		Log::WriteToConsoleAndLog(fmt::format(
 		    "WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, "
@@ -560,8 +581,7 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.shaderInt64 = VK_TRUE;
 	device_features.shaderFloat64 =
 	    supported_features2.features.shaderFloat64 &&
-	    float_controls.shaderSignedZeroInfNanPreserveFloat64 &&
-	    float_controls.shaderRoundingModeRTEFloat32;
+	    float_controls.shaderSignedZeroInfNanPreserveFloat64;
 	// if (device_features.shaderFloat64 && !float_controls.shaderDenormPreserveFloat64) {
 	// 	Log::WriteToConsoleAndLog(
 	// 	    "WARNING: Vulkan device does not guarantee FP64 denormal preservation; "

src/graphics/presentation/window/window.cpp

@@ -362,11 +362,13 @@ static void GameEventDidEnterForeground(WindowLoopState& game) {
 	SetPause(game, false);
 }
 
-void WindowContext::Resize(uint32_t new_width, uint32_t new_height) {
-	EXIT_IF(new_width == 0 || new_height == 0);
+void WindowContext::Resize(int new_width, int new_height) {
+	if (new_width <= 0 || new_height <= 0) {
+		return;
+	}
 	Common::LockGuard lock(mutex);
-	graphic_ctx.screen_width  = new_width;
-	graphic_ctx.screen_height = new_height;
+	graphic_ctx.screen_width  = static_cast<uint32_t>(new_width);
+	graphic_ctx.screen_height = static_cast<uint32_t>(new_height);
 }
 
 void WindowContext::ProcessWindowEvent(const SDL_WindowEvent& event) {

src/graphics/presentation/window/windowInternal.h

@@ -34,6 +34,7 @@ struct WindowContext {
 	~WindowContext();
 	KYTY_CLASS_NO_COPY(WindowContext);
 
+	[[nodiscard]] static vk::PhysicalDeviceVulkan11Features RequiredVulkan11Features() noexcept;
 	[[nodiscard]] static vk::PhysicalDeviceVulkan12Features RequiredVulkan12Features() noexcept;
 	[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
 	[[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept;
@@ -42,7 +43,7 @@ struct WindowContext {
 	void                                                    RefreshSurfaceCapabilities();
 	void                                                    UpdateIcon();
 	void                                                    UpdateTitle();
-	void                                                    Resize(uint32_t width, uint32_t height);
+	void                                                    Resize(int width, int height);
 	void ProcessWindowEvent(const SDL_WindowEvent& event);
 	void ProcessDisplayEvent(const SDL_DisplayEvent& event);
 	void ProcessEvent(double time_seconds);

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -648,6 +648,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
 				})) {
 					continue;
 				}
+				const auto resource = inst.Flags<uint32_t>();
+				first               = resource < ir.info.buffers.size() &&
+				                              ir.info.buffers[resource].indirect_root == resource
+				                          ? 1u
+				                          : 0u;
 			} else if (op == IR::ValueOpcode::GetImageResource) {
 				const auto resource = inst.Flags<uint32_t>();
 				first = resource < ir.info.images.size() &&

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -85,9 +85,13 @@ void ValidateNativeProgram(const IR::Program& program) {
 	}
 	const auto has_shader_data_storage = present[static_cast<size_t>(Kind::ShaderData)];
 	const auto shader_data_dwords = program.bindings.ShaderDataDwords();
+	const auto user_data_dwords = program.bindings.user_data_registers.size();
+	const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart;
 	if ((program.bindings.UsesPushData() &&
 	     !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
-	    program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
+	    (has_dispatch_threads && (program.stage != ShaderType::Compute ||
+	                              program.bindings.dispatch_thread_dword != user_data_dwords)) ||
+	    program.bindings.memory_offset_dword != user_data_dwords + (has_dispatch_threads ? 3u : 0u) ||
 	    program.bindings.memory_offset_count != buffers.size() ||
 	    has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) ||
 	    !std::is_sorted(program.bindings.user_data_registers.begin(),
@@ -218,7 +222,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					Fail(program, "buffer operation has invalid memory metadata");
 				}
 				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind == IR::ResourceKind::IndirectBuffer) {
+				if (memory.kind == IR::ResourceKind::IndirectBuffer &&
+				    inst.GetOpcode() != IR::ValueOpcode::ReadConstBuffer) {
 					requirements.subgroup_local_invocation_id = true;
 				}
 				if (memory.kind == IR::ResourceKind::Buffer) {
@@ -226,6 +231,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					if (memory.resource >= program.info.buffers.size()) {
 						Fail(program, "buffer operation has invalid resource metadata");
 					}
+					const auto bits = StorageBufferElementBits(program, memory);
+					requirements.buffer_u8 |= bits == 8u;
+					requirements.buffer_u16 |= bits == 16u;
 					if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
 						if (program.stage != ShaderType::Compute) {
 							Fail(program, "buffer ADD_TID is only valid for compute shaders");
@@ -263,6 +271,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 				}
 			}
 			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break;
 				case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
 				case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
 				case IR::ValueOpcode::DppMoveU32:
@@ -288,6 +297,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					break;
 				}
 				case IR::ValueOpcode::SwizzleU32:
+				case IR::ValueOpcode::PermuteU32:
 				case IR::ValueOpcode::BpermuteU32: {
 					requirements.subgroup_ballot              = true;
 					requirements.subgroup_shuffle             = true;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -83,7 +83,7 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
 	}
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpAccessChain, result_ptr_type, pointer, variable_id,
-	                          ConstantU32(state, array_index));
+	                          array_index);
 	return pointer;
 }
 
@@ -152,8 +152,9 @@ uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
 	}
 }
 
-uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip) {
-	const auto pointer = ImageDescriptorPointer(state, resource, mip);
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip,
+                             uint32_t array_index) {
+	const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index);
 	const auto image = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpLoad, ImageType(state, state.program.info.images.at(resource)),
 	                          image, pointer);
@@ -167,30 +168,40 @@ uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
 	const auto pointer_type =
 	    state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type);
 	const auto pointer = DescriptorElementPointer(
-	    state, pointer_type, state.sampler_variable, array_index,
+	    state, pointer_type, state.sampler_variable, ConstantU32(state, array_index),
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer);
 	return sampler_id;
 }
 
-uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip) {
+uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip,
+                          uint32_t array_index) {
 	const auto& image_resource = state.program.info.images.at(resource);
 	EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
-	const auto  image          = LoadImageDescriptor(state, resource, mip);
+	const auto  image          = LoadImageDescriptor(state, resource, mip, array_index);
 	const auto  sampled_image = state.builder.AllocateId();
 	const auto  sampled_type =
 	    state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource));
 	state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
+	if (array_index != 0u) {
+		state.builder.RequireExtension("SPV_EXT_descriptor_indexing");
+		state.builder.RequireCapability(spv::CapabilityShaderNonUniform);
+		state.builder.RequireCapability(spv::CapabilitySampledImageArrayNonUniformIndexing);
+		state.builder.AddAnnotation(spv::OpDecorate, sampled_image, spv::DecorationNonUniform);
+	}
 	return sampled_image;
 }
 
-uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip) {
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip,
+                                uint32_t array_index) {
 	const auto& image = state.program.info.images.at(resource);
 	EXIT_IF(mip >= image.mip_count);
 	const auto kind = IR::DescriptorBindingForImage(image);
 	EXIT_IF(!kind.has_value());
-	const auto array_index  = ResourceForDescriptor(state, *kind, resource) + mip;
+	if (array_index == 0u) {
+		array_index = ConstantU32(state, ResourceForDescriptor(state, *kind, resource) + mip);
+	}
 	const auto pointer_type = state.builder.Type(
 	    spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
 	const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
@@ -213,7 +224,7 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
 		EmitWrite(0);
 		return;
 	}
-	EmitImageMipSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
+	EmitIndexSwitch(state, mip_lod, image.mip_count, 0, EmitWrite);
 }
 
 spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -226,13 +226,13 @@ void DefineBvhIntersect(EmitterState& s) {
 	if (!s.requirements.bvh) return;
 	const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s);
 	const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
-	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3);
 	s.bvh_intersect_function = s.builder.AllocateId();
 	s.builder.AddName(s.bvh_intersect_function, "bvh_intersect");
 	s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function,
 	                      spv::FunctionControlMaskNone, signature);
 	std::array<uint32_t, 6> args;
-	const std::array types {vec4, u, f, vec3, vec3, vec3};
+	const std::array types {vec4, wide, f, vec3, vec3, vec3};
 	for (uint32_t i = 0; i < args.size(); ++i) {
 		args[i] = s.builder.AllocateId();
 		s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]);
@@ -253,7 +253,7 @@ void DefineBvhIntersect(EmitterState& s) {
 	const auto shr = [&](uint32_t value, uint32_t shift) {
 		return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift));
 	};
-	const auto kind = and_bits(node, 7);
+	const auto kind = and_bits(Unary(s, spv::OpUConvert, u, node), 7);
 	const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4));
 	const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5));
 	const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4));
@@ -262,7 +262,7 @@ void DefineBvhIntersect(EmitterState& s) {
 	const auto grow = and_bits(shr(words[1], 23), 0xff);
 	const auto base = Binary(s, spv::OpShiftLeftLogical, wide,
 	    DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8));
-	const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3));
+	const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantDeviceAddress(s, 3));
 	const auto last_index = Binary(s, spv::OpIAdd, wide, index,
 	    Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0)));
 	const auto address = Binary(s, spv::OpIAdd, wide, base,
@@ -325,17 +325,23 @@ void DefineBvhIntersect(EmitterState& s) {
 uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& s = ctx.state;
 	const auto* ray = ctx.ImageAddress(inst.Arg(1));
-	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0)
-		ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function");
+	const auto node_words = inst.Flags<uint32_t>();
+	if (ray == nullptr || (node_words != 1u && node_words != 2u) ||
+	    ray->NumArgs() < node_words + 10u || s.bvh_intersect_function == 0)
+		ctx.Fail(inst, "BVH intersection requires its node pointer, ray components and shared function");
 	const auto vector = [&](uint32_t first) {
 		Vec3 values;
 		for (uint32_t i = 0; i < 3; ++i)
 			values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i));
 		return Vector(s, TypeF32Vector(s, 3), values);
 	};
-	const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0);
-	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1));
-	const auto origin = vector(2), direction = vector(5), inverse = vector(8);
+	const auto descriptor = ctx.Arg(inst, 0);
+	const auto node = node_words == 2u
+	                      ? DeviceAddressFromWords(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1))
+	                      : Unary(s, spv::OpUConvert, TypeScalarU64(s), ctx.Arg(*ray, 0));
+	const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words));
+	const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u),
+	           inverse = vector(node_words + 7u);
 	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4),
 	    ConstantU32CompositeZero(s, 4), [&] {
 		const auto result = s.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -366,18 +366,23 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
 			state.builder.AddFunction(spv::OpStore, state.point_size_variable, f32);
 			continue;
 		}
-		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
+		auto StoreDistance = [&](IR::StageOutputKind kind, uint32_t variable, uint32_t index) {
 			if (index == UINT32_MAX) {
 				return;
 			}
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain,
-			                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
-			                          pointer, variable, ConstantU32(state, index));
+			uint32_t pointer;
+			if (state.program.stage == ShaderType::Mesh) {
+				pointer = MeshOutputPointer(state, kind, index);
+			} else {
+				pointer = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpAccessChain,
+				                          TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
+				                          pointer, variable, ConstantU32(state, index));
+			}
 			state.builder.AddFunction(spv::OpStore, pointer, f32);
 		};
-		StoreDistance(state.clip_distance_variable, output.clip_distance);
-		StoreDistance(state.cull_distance_variable, output.cull_distance);
+		StoreDistance(IR::StageOutputKind::ClipDistance, state.clip_distance_variable, output.clip_distance);
+		StoreDistance(IR::StageOutputKind::CullDistance, state.cull_distance_variable, output.cull_distance);
 	}
 }
 
@@ -574,6 +579,14 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
 
 void EmitVoid(ValueEmitContext&) {}
 
+void EmitStoreCompletion(EmitterState& state) {
+	state.builder.AddFunction(
+	    spv::OpControlBarrier, ConstantU32(state, spv::ScopeSubgroup),
+	    ConstantU32(state, spv::ScopeDevice),
+	    ConstantU32(state, spv::MemorySemanticsReleaseMask | spv::MemorySemanticsUniformMemoryMask |
+	                          spv::MemorySemanticsImageMemoryMask));
+}
+
 void EmitBarrier(EmitterState& state) {
 	const auto tessellation = state.program.stage == ShaderType::TessellationControl;
 	if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
@@ -623,6 +636,14 @@ uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index)
 	return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
 }
 
+uint32_t EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto start = ctx.state.program.bindings.dispatch_thread_dword;
+	if (start == IR::PushData::NoStart || !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() >= 3u) {
+		ctx.Fail(inst, "invalid dispatch thread extent");
+	}
+	return EmitShaderDataDwordLoad(ctx.state, start + inst.Arg(0).U32());
+}
+
 uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
 	const auto result = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpUndef, TypeId(state, inst.GetType()), result);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -728,7 +728,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				return sample;
 			};
 			const auto sample = image.mip_mode == IR::ImageMipMode::Dynamic && image.mip_count > 1u
-			                        ? EmitImageMipSwitch(
+			                        ? EmitIndexSwitch(
 			                              state, GatherMip(ctx, inst, mem, *address, layout,
 			                                               image.mip_count),
 			                              image.mip_count, result_type, EmitGather)
@@ -776,13 +776,13 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
 		}
 		const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler);
-		const auto EmitSample = [&](uint32_t resource) {
+		const auto EmitSample = [&](uint32_t resource, uint32_t array_index = 0u) {
 			const auto& candidate = state.program.info.images[resource];
 			const auto coord =
 			    CoordF32(ctx, mem, *address, layout.coord,
 			             ImageDimensionInfoFor(candidate.dimension).coordinate_components,
 			             candidate.cube);
-			const auto            sampled = MakeSampledImage(state, resource, sampler_id);
+			const auto sampled = MakeSampledImage(state, resource, sampler_id, 0u, array_index);
 			const auto            sample  = state.builder.AllocateId();
 			std::vector<uint32_t> sample_operands {result_type, sample, sampled, coord};
 			if (dref) {
@@ -808,7 +808,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto* source = image.source < state.program.descriptor_sources.size()
 		                         ? &state.program.descriptor_sources[image.source]
 		                         : nullptr;
-		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
+		if (handle == nullptr || source == nullptr || !source->indirect_descriptor.has_value() ||
 		    handle->NumArgs() == 0u) {
 			ctx.Fail(inst, "has invalid indirect image key provenance");
 			return;
@@ -819,55 +819,42 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Fail(inst, "has no indirect image runtime mapping");
 			return;
 		}
-		const auto LoadMapping = [&](uint32_t index) {
-			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
-			                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
-			                          index);
-			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
-			return value;
-		};
-		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
-		auto       low      = ConstantU32(state, 0u);
-		auto       high     = LoadMapping(mapping);
-		auto       selected = ConstantU32(state, 0u);
-		for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
-		     iteration++) {
-			const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
-			const auto mid    = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
-			                           Binary(state, spv::OpIAdd, TypeU32(state), low, high),
-			                           ConstantU32(state, 1u));
-			const auto probe  = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), probe, active, mid,
-			                          ConstantU32(state, 0u));
-			const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
-			                          Binary(state, spv::OpIAdd, TypeU32(state),
-			                                 Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
-			                                        probe, ConstantU32(state, 1u)),
-			                                 ConstantU32(state, 1u)));
-			const auto mapped_key = LoadMapping(entry);
-			const auto candidate  = LoadMapping(
-			    Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
-			const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
-			const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
-			const auto next_selected = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_selected, match,
-			                          candidate, selected);
-			selected              = next_selected;
-			const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
-			const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
-			const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
-			                               Unary(state, spv::OpLogicalNot, TypeBool(state), less));
-			const auto next_low   = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    spv::OpSelect, TypeU32(state), next_low, take_upper,
-			    Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
-			low                  = next_low;
-			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpSelect, TypeU32(state), next_high, take_lower, mid,
-			                          high);
-			high = next_high;
+		const auto selected = EmitIndirectResourceIndex(
+		    state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
+		const auto kind = *IR::DescriptorBindingForImage(image);
+		const auto& binding = *IR::FindBinding(state.program.bindings, kind);
+		uint32_t first_child = 0;
+		bool homogeneous = true;
+		for (uint32_t ordinal = 0; ordinal < image.indirect_resources.size(); ++ordinal) {
+			const auto resource = image.indirect_resources[ordinal];
+			const auto& candidate = state.program.info.images[resource];
+			if (candidate.dimension != image.dimension || candidate.cube != image.cube ||
+			    IR::DescriptorBindingForImage(candidate) != kind ||
+			    candidate.mip_count != 1u) {
+				homogeneous = false;
+				break;
+			}
+			if (ordinal == 1u) first_child = ResourceForDescriptor(state, kind, resource);
+			const auto child_slot = first_child + ordinal - 1u;
+			if (ordinal > 1u && (child_slot >= binding.resources.size() ||
+			                     binding.resources[child_slot] != resource)) {
+				homogeneous = false;
+				break;
+			}
+		}
+		if (homogeneous) {
+			const auto child_index = Binary(state, spv::OpIAdd, TypeU32(state), selected,
+			                                ConstantU32(state, first_child - 1u));
+			const auto is_root = Binary(state, spv::OpIEqual, TypeBool(state), selected,
+			                            ConstantU32(state, 0u));
+			const auto array_index = state.builder.AllocateId();
+			state.builder.AddFunction(spv::OpSelect, TypeU32(state), array_index, is_root,
+			                          ConstantU32(state, ResourceForDescriptor(state, kind, mem.resource)),
+			                          child_index);
+			const auto sample = EmitSample(mem.resource, array_index);
+			const auto result = dref ? sample : UnpackImageTexel(ctx, mem, sample);
+			ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem));
+			return;
 		}
 		const auto            default_label = state.builder.AllocateId();
 		const auto            merge_label   = state.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -179,6 +179,7 @@ inline constexpr auto EmitFPFloor32     = EmitGlsl<GLSLstd450Floor, IR::Type::F3
 inline constexpr auto EmitFPCeil32      = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
 inline constexpr auto EmitFPTrunc32     = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
 inline constexpr auto EmitFPFract32     = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
+inline constexpr auto EmitFPFract64     = EmitGlsl<GLSLstd450Fract, IR::Type::F64, uint32_t>;
 uint32_t              EmitFPSin(EmitterState& state, uint32_t arg0);
 uint32_t              EmitFPCos(EmitterState& state, uint32_t arg0);
 #undef EMIT_NATIVE
@@ -188,16 +189,17 @@ void                  EmitVoid(ValueEmitContext&);
 inline constexpr auto EmitReference    = EmitVoid;
 inline constexpr auto EmitReferenceU32 = EmitVoid;
 inline constexpr auto EmitControlNop   = EmitVoid;
-inline constexpr auto EmitWaitcnt      = EmitVoid;
 inline constexpr auto EmitSendmsg      = EmitVoid;
 inline constexpr auto EmitTtraceData   = EmitVoid;
 inline constexpr auto EmitInstPrefetch = EmitVoid;
 void                  EmitBarrier(EmitterState& state);
+void                  EmitStoreCompletion(EmitterState& state);
 void                  EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
+uint32_t              EmitGetDispatchThreadExtent(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
 uint32_t              EmitUndefU1(EmitterState& state, const IR::Inst& inst);
 inline constexpr auto EmitUndefU8  = EmitUndefU1;
@@ -230,10 +232,12 @@ void                  EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& ins
 uint32_t              EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
+void                  EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitPermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -61,6 +61,7 @@ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
 struct SpirvRequirements {
 	bool bvh                          = false;
 	bool subgroup_ballot              = false;
+	bool subgroup_barrier             = false;
 	bool subgroup_shuffle             = false;
 	bool subgroup_local_invocation_id = false;
 	bool compute_derivatives          = false;
@@ -69,6 +70,8 @@ struct SpirvRequirements {
 	bool function_scratch             = false;
 	bool pixel_valid_mask             = false;
 	bool buffer_int64_atomics         = false;
+	bool buffer_u8                    = false;
+	bool buffer_u16                   = false;
 	bool shared_int64_atomics         = false;
 	bool coherent_buffers             = false;
 	bool float64                      = false;
@@ -107,6 +110,8 @@ struct EmitterState {
 	uint32_t                                         lane_count              = 1;
 	uint32_t                                         lane_half               = 0;
 	uint32_t                                         storage_buffer_variable = 0;
+	uint32_t                                         storage_buffer_u8_variable = 0;
+	uint32_t                                         storage_buffer_u16_variable = 0;
 	uint32_t                                         storage_buffer_u64_variable = 0;
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
 	uint32_t                                         bda_pagetable_variable  = 0;
@@ -167,10 +172,9 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
 uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
 uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
 uint32_t TypeFunction(EmitterState& state);
-uint32_t TypeStorageBufferPointer(EmitterState& state);
-uint32_t TypeStorageBufferElementPointer(EmitterState& state);
-uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
-uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
+uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits);
+uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits = 32);
+uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32);
 uint32_t TypePhysicalU32Pointer(EmitterState& state);
 uint32_t TypePushConstantElementPointer(EmitterState& state);
 uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
@@ -315,14 +319,16 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image);
 
 uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension);
 
-uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t mip = 0,
+                                uint32_t array_index = 0);
 
-uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0);
+uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0,
+                             uint32_t array_index = 0);
 
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler);
 
 uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id,
-                          uint32_t mip = 0);
+                          uint32_t mip = 0, uint32_t array_index = 0);
 
 void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
                            uint32_t texel);
@@ -348,7 +354,8 @@ uint32_t DefineInterfaceVariable(EmitterState& state, uint32_t type, spv::Storag
 void     DefineModule(EmitterState& state);
 void     DefineTessellationInterfaces(EmitterState& state);
 void     DefineTessellationExecutionModes(EmitterState& state);
-void     DefineMeshOutputs(EmitterState& state);
+void     DefineMeshOutputs(EmitterState& state, uint32_t clip_distance_count,
+                           uint32_t cull_distance_count);
 void     EmitMeshEntryPoint(EmitterState& state);
 void     EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index = 0);
@@ -392,6 +399,7 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
 uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
 
 Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
+uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem);
 
 void EmitMemoryOffsets(EmitterState& state);
 
@@ -402,21 +410,19 @@ struct MemoryResourceAccess {
 	IR::ResourceKind      kind             = IR::ResourceKind::None;
 	uint32_t              object_pointer   = 0;
 	uint32_t              length           = 0;
-	uint32_t              index_offset     = 0;
 	uint32_t              byte_offset      = 0;
-	bool                  add_index_offset = false;
+	uint32_t              element_bits     = 32;
 	spv::MemoryAccessMask memory_access    = spv::MemoryAccessMaskNone;
 };
 
+uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
+                                   uint32_t search_iterations, uint32_t default_resource);
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
 MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState&         state,
                                                         const IR::MemoryInfo& mem,
                                                         uint32_t variable, uint32_t pointer_type);
 
-uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
-                                uint32_t raw_index);
-
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
                                    uint32_t index);
 
@@ -517,17 +523,17 @@ uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high
 
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
-auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_count,
+auto EmitIndexSwitch(EmitterState& state, uint32_t index, uint32_t count,
                         uint32_t result_type, Fn&& emit) {
 	constexpr bool has_result = !std::is_void_v<std::invoke_result_t<Fn, uint32_t>>;
-	EXIT_IF(mip_count == 0u);
+	EXIT_IF(count == 0u);
 	const auto merge_label = state.builder.AllocateId();
-	std::vector<uint32_t> labels(mip_count);
-	std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
-	for (uint32_t mip = 0; mip < mip_count; mip++) {
-		labels[mip] = state.builder.AllocateId();
-		words.push_back(mip);
-		words.push_back(labels[mip]);
+	std::vector<uint32_t> labels(count);
+	std::vector<uint32_t> words {spv::OpSwitch, index, merge_label};
+	for (uint32_t item = 0; item < count; item++) {
+		labels[item] = state.builder.AllocateId();
+		words.push_back(item);
+		words.push_back(labels[item]);
 	}
 	if constexpr (has_result) {
 		words[2] = labels.front();
@@ -538,13 +544,13 @@ auto EmitImageMipSwitch(EmitterState& state, uint32_t mip_lod, uint32_t mip_coun
 	if constexpr (has_result) {
 		phi = {spv::OpPhi, result_type, state.builder.AllocateId()};
 	}
-	for (uint32_t mip = 0; mip < mip_count; mip++) {
-		EmitLabel(state, labels[mip]);
+	for (uint32_t item = 0; item < count; item++) {
+		EmitLabel(state, labels[item]);
 		if constexpr (has_result) {
-			phi.push_back(emit(mip));
+			phi.push_back(emit(item));
 			phi.push_back(state.current_label);
 		} else {
-			emit(mip);
+			emit(item);
 		}
 		state.builder.AddFunction(spv::OpBranch, merge_label);
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -4,6 +4,8 @@
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <algorithm>
+#include <bit>
+#include <utility>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
@@ -233,7 +235,15 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
 
 uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
 	if (mem.kind == IR::ResourceKind::Buffer) {
-		return BufferByteAddress(ctx, inst, mem);
+		const auto resource = ResourceForDescriptor(ctx.state, IR::DescriptorBindingKind::Buffers,
+		                                            mem.resource);
+		const auto prefix = ctx.state.memory_byte_offsets[resource];
+		const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state),
+		                            BufferByteAddress(ctx, inst, mem), prefix);
+		// The host binding prefix must not wrap an out-of-range guest address into the buffer.
+		return Select(ctx.state, TypeU32(ctx.state),
+		              Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), address, prefix),
+		              address, ConstantU32(ctx.state, UINT32_MAX));
 	}
 	if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
 		if (mem.offset == 0u) {
@@ -249,20 +259,14 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
 }
 
 uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	auto address = ByteAddress(ctx, inst, mem);
+	if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
+		// RDNA2 DS region addresses retain bits [15:2] after adding the byte offset.
+		address = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
+		                 ConstantU32(ctx.state, 0xffffu));
+	}
 	return Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state),
-	              ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2));
-}
-
-struct PreparedMemoryElement {
-	MemoryResourceAccess resource;
-	uint32_t             index = 0;
-};
-
-PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
-                                           uint32_t raw_index) {
-	auto       resource = PrepareMemoryResourceAccess(ctx.state, mem);
-	const auto index    = EmitMemoryElementIndex(ctx.state, resource, raw_index);
-	return {.resource = resource, .index = index};
+	              address, ConstantU32(ctx.state, 2));
 }
 
 uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
@@ -273,7 +277,7 @@ uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess&
 
 uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                           const MemoryResourceAccess& resource) {
-	const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
+	const auto index = DwordIndex(ctx, inst, mem);
 	return EmitValueOrZeroIfCondition(
 	    ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index),
 	    [&]() { return LoadWordInBounds(ctx, resource, index); });
@@ -289,10 +293,10 @@ uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me
 uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                              const MemoryResourceAccess& resource, uint32_t bits,
                              bool sign_extend) {
-	const auto address   = ByteAddress(ctx, inst, mem);
-	const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
-	                              ConstantU32(ctx.state, 2));
-	const auto index     = EmitMemoryElementIndex(ctx.state, resource, raw_index);
+	const auto address = ByteAddress(ctx, inst, mem);
+	const auto index = Binary(
+	    ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
+	    ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
 	return EmitValueOrZeroIfCondition(
 	    ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
 		    return LoadSubwordInBounds(ctx, resource, address, index, bits, sign_extend);
@@ -310,15 +314,23 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
 
 uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
                              uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) {
-	const auto word = LoadWordInBounds(ctx, resource, index);
-	const auto byte  = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
-	                          ConstantU32(ctx.state, 3));
-	const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
-	                          ConstantU32(ctx.state, 3));
-	const auto value =
-	    Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
-	           Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
-	           ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
+	uint32_t value;
+	if (resource.element_bits < 32u) {
+		const auto loaded = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(spv::OpLoad, TypeStorageBufferElement(ctx.state, bits), loaded,
+		                              EmitMemoryElementPointer(ctx.state, resource, index),
+		                              resource.memory_access);
+		value = Unary(ctx.state, spv::OpUConvert, TypeU32(ctx.state), loaded);
+	} else {
+		const auto word = LoadWordInBounds(ctx, resource, index);
+		const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
+		                         ConstantU32(ctx.state, 3));
+		const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
+		                          ConstantU32(ctx.state, 3));
+		value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
+		               Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
+		               ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
+	}
 	if (!sign_extend) return value;
 	const auto left = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), value,
 	                         ConstantU32(ctx.state, 32u - bits));
@@ -339,14 +351,6 @@ Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Memor
 	                 : StorageBufferFormat(ctx.state, mem);
 }
 
-IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, const Format::BufferFormatInfo& info,
-                                        uint32_t component) {
-	mem.offset += Format::GetFormatComponentByteOffset(info, component);
-	mem.data_dwords     = 1u;
-	mem.component_index = component;
-	return mem;
-}
-
 IR::MemoryInfo RebaseRawComponent(IR::MemoryInfo mem, uint32_t component) {
 	mem.offset += component * 4u;
 	mem.data_dwords     = 1u;
@@ -381,19 +385,17 @@ uint32_t FormattedConstant(ValueEmitContext& ctx, const Format::BufferFormatInfo
 }
 
 template <typename LoadWordFn, typename LoadSubwordFn>
-uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
-                                const Format::BufferFormatInfo& info,
-                                uint32_t output_component, LoadWordFn&& load_word,
+uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const Format::BufferFormatInfo& info,
+                                FormattedSource source, LoadWordFn&& load_word,
                                 LoadSubwordFn&& load_subword) {
-	const auto source = ResolveFormattedSource(ctx, mem, info, output_component);
 	if (source.kind != FormattedSourceKind::Memory) {
 		return FormattedConstant(ctx, info, source.kind);
 	}
 	const auto component = source.component;
 	const auto bits      = info.component_bits[component];
-	uint32_t   raw       = 0;
-	if (info.packed_bitfield) {
-		raw = load_word(component);
+	auto raw = info.packed_bitfield || bits == 32u ? load_word(component)
+	                                                : load_subword(component, bits);
+	if (info.packed_bitfield || (bits < 32u && IsSignedFormatComponent(info.type))) {
 		const auto type =
 		    IsSignedFormatComponent(info.type) ? TypeI32(ctx.state) : TypeU32(ctx.state);
 		const auto source_value =
@@ -402,49 +404,28 @@ uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem
 		ctx.state.builder.AddFunction(IsSignedFormatComponent(info.type) ? spv::OpBitFieldSExtract
 		                                                                 : spv::OpBitFieldUExtract,
 		                              type, extracted, source_value,
-		                              ConstantU32(ctx.state, info.component_bit_offset[component]),
+		                              ConstantU32(ctx.state, info.packed_bitfield
+		                                                         ? info.component_bit_offset[component]
+		                                                         : 0u),
 		                              ConstantU32(ctx.state, bits));
 		raw = type == TypeI32(ctx.state)
 		          ? Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), extracted)
 		          : extracted;
-	} else if (bits == 32u) {
-		raw = load_word(component);
-	} else {
-		raw = load_subword(component, bits, IsSignedFormatComponent(info.type));
 	}
 	return NormalizeFormatComponent(ctx.state, info, component, raw);
 }
 
-uint32_t FormattedLoadPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
-                               const IR::MemoryInfo& mem, uint32_t output_component,
-                               const MemoryResourceAccess& resource) {
-	const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem));
-	if (info.type == Format::ComponentType::Unknown) {
-		return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, output_component), resource);
-	}
-	return LoadFormattedComponent(
-	    ctx, mem, info, output_component,
-	    [&](uint32_t component) {
-		    return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component),
-		                            resource);
-	    },
-	    [&](uint32_t component, uint32_t bits, bool sign_extend) {
-		    return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component),
-		                               resource, bits, sign_extend);
-	    });
-}
-
-uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
-	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
-		const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
-		return FormattedLoadPrepared(ctx, inst, mem, 0u, resource);
-	});
-}
-
 void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                           const MemoryResourceAccess& resource, uint32_t address, uint32_t index,
                           uint32_t bits, uint32_t data) {
 	const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
+	if (resource.element_bits < 32u) {
+		ctx.state.builder.AddFunction(
+		    spv::OpStore, pointer,
+		    Unary(ctx.state, spv::OpUConvert, TypeStorageBufferElement(ctx.state, bits), data),
+		    resource.memory_access);
+		return;
+	}
 	const auto shift   = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
 	                            Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
 	                                   ConstantU32(ctx.state, 3)),
@@ -472,10 +453,10 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
 
 void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                           const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) {
-	const auto address   = ByteAddress(ctx, inst, mem);
-	const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
-	                              ConstantU32(ctx.state, 2));
-	const auto index     = EmitMemoryElementIndex(ctx.state, resource, raw_index);
+	const auto address = ByteAddress(ctx, inst, mem);
+	const auto index = Binary(
+	    ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
+	    ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
 	EmitIfCondition(
 	    ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
 		    StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data);
@@ -542,16 +523,58 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
 
 void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                        const MemoryResourceAccess& resource, uint32_t data) {
-	const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
+	const auto index = DwordIndex(ctx, inst, mem);
 	EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
 		StoreWordInBounds(ctx, resource, index, data);
 	});
 }
 
 void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
-	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
-		const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
-		StoreWordPrepared(ctx, inst, mem, resource, ctx.Arg(inst, inst.NumArgs() - 2));
+	auto& state = ctx.state;
+	EmitIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto data = ctx.Arg(inst, inst.NumArgs() - 2);
+		const auto store = [&](uint32_t resource) {
+			if (mem.kind == IR::ResourceKind::Buffer) {
+				const auto& buffer = state.program.info.buffers[resource];
+				if (buffer.packed_stride == 0u &&
+				    buffer.source < state.program.descriptor_sources.size() &&
+				    state.program.descriptor_sources[buffer.source].indirect_descriptor) return;
+			}
+			mem.resource = resource;
+			const auto access = PrepareMemoryResourceAccess(state, mem);
+			StoreWordPrepared(ctx, inst, mem, access, data);
+		};
+		if (mem.kind != IR::ResourceKind::Buffer ||
+		    state.program.info.buffers[mem.resource].indirect_root != mem.resource) {
+			store(mem.resource);
+			return;
+		}
+		const auto& buffer = state.program.info.buffers[mem.resource];
+		const auto* handle = inst.Arg(0).ResolveInstruction();
+		if (handle == nullptr || buffer.indirect_resources.empty() ||
+		    state.flattened_srt_variable == 0 || buffer.indirect_search_iterations == 0u) {
+			ctx.Fail(inst, "has no indirect buffer runtime mapping");
+			return;
+		}
+		const auto selected = EmitIndirectResourceIndex(
+		    state, ctx.Arg(*handle, 0), buffer.indirect_mapping_offset,
+		    buffer.indirect_search_iterations, UINT32_MAX);
+		const auto merge = state.builder.AllocateId();
+		std::vector<uint32_t> labels(buffer.indirect_resources.size());
+		std::vector<uint32_t> branches {spv::OpSwitch, selected, merge};
+		for (uint32_t i = 0; i < labels.size(); ++i) {
+			labels[i] = state.builder.AllocateId();
+			branches.push_back(i);
+			branches.push_back(labels[i]);
+		}
+		state.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
+		state.builder.AddFunction(branches);
+		for (uint32_t i = 0; i < labels.size(); ++i) {
+			EmitLabel(state, labels[i]);
+			store(buffer.indirect_resources[i]);
+			state.builder.AddFunction(spv::OpBranch, merge);
+		}
+		EmitLabel(state, merge);
 	});
 }
 
@@ -559,10 +582,11 @@ template <typename Fn>
 uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
                           const IR::MemoryInfo& mem, Fn&& operation) {
 	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
-		const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
+		const auto index = DwordIndex(ctx, inst, mem);
+		const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
 		return EmitValueOrZeroIfCondition(
-		    ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
-			    return operation(EmitMemoryElementPointer(ctx.state, access.resource, access.index));
+		    ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
+			    return operation(EmitMemoryElementPointer(ctx.state, resource, index));
 		    });
 	});
 }
@@ -597,83 +621,59 @@ uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) {
 struct PreparedFormattedMemory {
 	Format::BufferFormatInfo info;
 	MemoryResourceAccess     resource;
-	std::array<uint32_t, 4>  addresses {};
-	std::array<uint32_t, 4>  indices {};
+	uint32_t                 base = 0;
 	uint32_t                 in_bounds = 0;
 };
 
-enum class FormattedAccess { Load, Store };
-
 PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::Inst& inst,
-                                               const IR::MemoryInfo&       mem,
+                                               const IR::MemoryInfo& mem,
                                                const MemoryResourceAccess& resource,
-                                               const Format::BufferFormatInfo& info,
-                                               uint32_t components, FormattedAccess access) {
+                                               const Format::BufferFormatInfo& info) {
 	PreparedFormattedMemory plan;
-	plan.info     = info;
-	plan.resource = resource;
-	std::array<bool, 4> required_components {};
-	if (access == FormattedAccess::Load) {
-		for (uint32_t output = 0; output < components; output++) {
-			const auto source = ResolveFormattedSource(ctx, mem, plan.info, output);
-			if (source.kind == FormattedSourceKind::Memory) {
-				required_components[source.component] = true;
-			}
-		}
-	} else {
-		for (uint32_t component = 0; component < std::min(components, plan.info.component_count);
-		     component++) {
-			required_components[component] = true;
-		}
-	}
-	bool first_bound = true;
-	for (uint32_t component = 0; component < plan.info.component_count; component++) {
-		if (!required_components[component]) continue;
-		const auto byte_offset = Format::GetFormatComponentByteOffset(plan.info, component);
-		bool       reused      = false;
-		for (uint32_t previous = 0; previous < component; previous++) {
-			if (required_components[previous] &&
-			    Format::GetFormatComponentByteOffset(plan.info, previous) == byte_offset) {
-				plan.addresses[component] = plan.addresses[previous];
-				plan.indices[component]   = plan.indices[previous];
-				reused                    = true;
-				break;
-			}
-		}
-		if (reused) continue;
-		const auto component_mem  = RebaseFormattedComponent(mem, plan.info, component);
-		plan.addresses[component] = ByteAddress(ctx, inst, component_mem);
-		const auto raw_index      = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state),
-		                                   plan.addresses[component], ConstantU32(ctx.state, 2));
-		plan.indices[component]   = EmitMemoryElementIndex(ctx.state, resource, raw_index);
-		const auto component_bound =
-		    EmitMemoryElementInBounds(ctx.state, resource, plan.indices[component]);
-		if (first_bound) {
-			plan.in_bounds = component_bound;
-			first_bound    = false;
-		} else {
-			plan.in_bounds = AndCondition(ctx.state, plan.in_bounds, component_bound);
-		}
-	}
-	if (first_bound) plan.in_bounds = ConstantBool(ctx.state, true);
+	plan.info      = info;
+	plan.resource  = resource;
+	const auto start = ByteAddress(ctx, inst, mem);
+	// Check the whole, unaligned format span before aligning the actual memory address.
+	const auto end = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start,
+	                        ConstantU32(ctx.state, info.byte_size - 1u));
+	const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end,
+	                               ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
+	plan.in_bounds = AndCondition(
+	    ctx.state, Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start),
+	    EmitMemoryElementInBounds(ctx.state, plan.resource, last_index));
+	plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start,
+	                   ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u)));
 	return plan;
 }
 
+std::pair<uint32_t, uint32_t> FormattedComponentAddress(EmitterState& state,
+                                                       const PreparedFormattedMemory& plan,
+                                                       uint32_t component) {
+	// Components advance sequentially from one address; do not reapply buffer swizzling.
+	const auto offset = Format::GetFormatComponentByteOffset(plan.info, component);
+	const auto address = offset == 0u ? plan.base : Binary(
+	    state, spv::OpIAdd, TypeU32(state), plan.base, ConstantU32(state, offset));
+	return {address, Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
+	                        ConstantU32(state, std::countr_zero(plan.resource.element_bits / 8u)))};
+}
+
 uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                                const PreparedFormattedMemory& plan, uint32_t output_component) {
 	return LoadFormattedComponent(
-	    ctx, mem, plan.info, output_component,
+	    ctx, plan.info, ResolveFormattedSource(ctx, mem, plan.info, output_component),
 	    [&](uint32_t component) {
-		    return LoadWordInBounds(ctx, plan.resource, plan.indices[component]);
+		    return LoadWordInBounds(ctx, plan.resource,
+		                            FormattedComponentAddress(ctx.state, plan, component).second);
 	    },
-	    [&](uint32_t component, uint32_t bits, bool sign_extend) {
-		    return LoadSubwordInBounds(ctx, plan.resource, plan.addresses[component],
-		                               plan.indices[component], bits, sign_extend);
+	    [&](uint32_t component, uint32_t bits) {
+		    const auto [address, index] = FormattedComponentAddress(ctx.state, plan, component);
+		    return LoadSubwordInBounds(ctx, plan.resource, address, index, bits, false);
 	    });
 }
 
 uint32_t ConstructU32Composite(EmitterState& state, uint32_t components,
                                const std::array<uint32_t, 4>& values) {
+	if (components == 1u) return values[0];
 	const auto            result = state.builder.AllocateId();
 	std::vector<uint32_t> words {spv::OpCompositeConstruct, TypeU32Composite(state, components),
 	                             result};
@@ -692,6 +692,21 @@ uint32_t FormattedOutOfBoundsValue(ValueEmitContext& ctx, const IR::MemoryInfo&
 	return ConstructU32Composite(ctx.state, components, values);
 }
 
+uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
+		const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem));
+		if (info.type == Format::ComponentType::Unknown) {
+			return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource);
+		}
+		const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info);
+		return EmitValueOrDefaultIfCondition(
+		    ctx.state, plan.in_bounds, TypeU32(ctx.state),
+		    FormattedOutOfBoundsValue(ctx, mem, plan, 1u),
+		    [&]() { return LoadFormattedInBounds(ctx, mem, plan, 0u); });
+	});
+}
+
 uint32_t EncodeFormattedStoreComponent(ValueEmitContext& ctx,
                                        const Format::BufferFormatInfo& info,
                                        uint32_t component, uint32_t data) {
@@ -722,8 +737,7 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
                              const Format::BufferFormatInfo& info, uint32_t data,
                              uint32_t components) {
 	// RDNA2 formatted stores transfer the entire format, zero-filling missing VGPRs.
-	const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, info.component_count,
-	                                         FormattedAccess::Store);
+	const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info);
 	EmitIfCondition(ctx.state, plan.in_bounds, [&]() {
 		auto packed = ConstantU32(ctx.state, 0);
 		for (uint32_t component = 0; component < info.component_count; component++) {
@@ -743,14 +757,19 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
 				    ctx.state, packed, value,
 				    ConstantU32(ctx.state, info.component_bit_offset[component]),
 				    ConstantU32(ctx.state, bits));
-			} else if (bits == 8u || bits == 16u) {
-				StoreSubwordInBounds(ctx, mem, resource, plan.addresses[component],
-				                     plan.indices[component], bits, value);
 			} else {
-				StoreWordInBounds(ctx, resource, plan.indices[component], value);
+				const auto [address, index] = FormattedComponentAddress(ctx.state, plan, component);
+				if (bits == 8u || bits == 16u) {
+					StoreSubwordInBounds(ctx, mem, plan.resource, address, index, bits, value);
+				} else {
+					StoreWordInBounds(ctx, plan.resource, index, value);
+				}
 			}
 		}
-		if (info.packed_bitfield) StoreWordInBounds(ctx, resource, plan.indices[0], packed);
+		if (info.packed_bitfield) {
+			StoreWordInBounds(ctx, plan.resource,
+			                  FormattedComponentAddress(ctx.state, plan, 0u).second, packed);
+		}
 	});
 }
 
@@ -767,78 +786,193 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
 	});
 }
 
-uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
-	auto&       state   = ctx.state;
-	const auto& handle  = *inst.Arg(0).ResolveInstruction();
-	const auto  word1   = ctx.Arg(handle, 1);
-	const auto  records = ctx.Arg(handle, 2);
-	const auto  word3   = ctx.Arg(handle, 3);
-	const auto  field   = [&](uint32_t word, uint32_t first, uint32_t count) {
-		return EmitBitFieldUExtract(state, word, ConstantU32(state, first),
-		                            ConstantU32(state, count));
+uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&       state  = ctx.state;
+	const auto& handle = *inst.Arg(0).ResolveInstruction();
+	const auto  word1  = ctx.Arg(handle, 1);
+	const auto  stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16),
+	                                         ConstantU32(state, 14));
+	const auto size = Binary(
+	    state, spv::OpIMul, TypeScalarU64(state),
+	    Unary(state, spv::OpUConvert, TypeScalarU64(state),
+	          EmitUMax32(state, stride, ConstantU32(state, 1))),
+	    Unary(state, spv::OpUConvert, TypeScalarU64(state), ctx.Arg(handle, 2)));
+	const auto offset = Binary(
+	    state, spv::OpIAdd, TypeScalarU64(state),
+	    Unary(state, spv::OpUConvert, TypeScalarU64(state),
+	          Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1),
+	                 ConstantU32(state, ~3u))),
+	    ConstantDeviceAddress(state, ctx.Memory(inst).offset & ~3u));
+	const auto end = Binary(state, spv::OpIAdd, TypeScalarU64(state), offset,
+	                        ConstantDeviceAddress(state, sizeof(uint32_t)));
+	return EmitValueOrZeroIfCondition(
+	    state, Binary(state, spv::OpULessThanEqual, TypeBool(state), end, size), [&]() {
+		    const auto base = DeviceAddressFromWords(
+		        state, Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(handle, 0),
+		                      ConstantU32(state, ~3u)),
+		        Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1, ConstantU32(state, 0xffffu)));
+		    return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), base, offset));
+	    });
+}
+
+struct IndirectBufferAccess {
+	uint32_t address;
+	uint32_t offset;
+	uint32_t stride;
+	uint32_t swizzle;
+	uint32_t format;
+	uint32_t selector;
+	uint32_t mode;
+	uint32_t records;
+	uint32_t index_in_bounds;
+	uint32_t scalar_in_bounds;
+	uint32_t raw_records;
+	uint32_t raw_index_in_bounds;
+};
+
+IndirectBufferAccess PrepareIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&       state  = ctx.state;
+	const auto& handle = *inst.Arg(0).ResolveInstruction();
+	const auto word1   = ctx.Arg(handle, 1);
+	const auto records = ctx.Arg(handle, 2);
+	const auto word3   = ctx.Arg(handle, 3);
+	const auto field = [&](uint32_t word, uint32_t first, uint32_t count) {
+		return EmitBitFieldUExtract(state, word, ConstantU32(state, first), ConstantU32(state, count));
 	};
 	const auto nonzero = [&](uint32_t value) {
 		return Binary(state, spv::OpINotEqual, TypeBool(state), value, ConstantU32(state, 0));
 	};
-	const auto has_dword = [&](uint32_t offset, uint32_t size) {
-		return AndCondition(
-		    state,
-		    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size, ConstantU32(state, 4)),
-		    Binary(state, spv::OpULessThanEqual, TypeBool(state), offset,
-		           Binary(state, spv::OpISub, TypeU32(state), size, ConstantU32(state, 4))));
-	};
-	const auto stride       = field(word1, 16, 14);
-	const auto swizzle      = AndCondition(state, nonzero(stride), nonzero(field(word1, 31, 1)));
-	const auto index_stride = field(word3, 21, 2);
-	const auto add_tid      = nonzero(field(word3, 23, 1));
-	const auto index =
-	    Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
-	           Select(state, TypeU32(state), add_tid, BufferLane(state), ConstantU32(state, 0)));
+	const auto stride  = field(word1, 16, 14);
+	const auto swizzle = AndCondition(state, nonzero(stride), nonzero(field(word1, 31, 1)));
+	const auto index = Binary(
+	    state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
+	    Select(state, TypeU32(state), nonzero(field(word3, 23, 1)), BufferLane(state),
+	           ConstantU32(state, 0)));
 	const auto soffset = ctx.Arg(inst, 3);
-	const auto base    = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16));
-	const auto valid_format    = nonzero(field(word3, 12, 7));
-	const auto mode            = field(word3, 28, 2);
-	const auto index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records);
-	const auto scalar_in_bounds =
-	    Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records);
+	const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset,
+	                                            ctx.Memory(inst).offset, stride, swizzle,
+	                                            field(word3, 21, 2));
+	const auto base = DeviceAddressFromWords(state, ctx.Arg(handle, 0), field(word1, 0, 16));
 	const auto raw_records = Binary(state, spv::OpISub, TypeU32(state), records, soffset);
-	const auto raw_index_in_bounds =
-	    Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records);
+	return {
+	    .address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
+	                      Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte)),
+	    .offset = address.offset,
+	    .stride = stride,
+	    .swizzle = swizzle,
+	    .format = field(word3, 12, 7),
+	    .selector = ctx.Memory(inst).formatted ? field(word3, 0, 3) : 0u,
+	    .mode = field(word3, 28, 2),
+	    .records = records,
+	    .index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, records),
+	    .scalar_in_bounds = Binary(state, spv::OpULessThanEqual, TypeBool(state), soffset, records),
+	    .raw_records = raw_records,
+	    .raw_index_in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, raw_records),
+	};
+}
+
+uint32_t IndirectBufferInBounds(EmitterState& state, const IndirectBufferAccess& buffer,
+                                uint32_t displacement, uint32_t bytes, bool formatted) {
+	const auto offset = displacement == 0u ? buffer.offset : Binary(
+	    state, spv::OpIAdd, TypeU32(state), buffer.offset, ConstantU32(state, displacement));
+	const auto has_payload = [&](uint32_t size) {
+		return AndCondition(
+		    state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size,
+		                  ConstantU32(state, bytes)),
+		    Binary(state, spv::OpULessThanEqual, TypeBool(state), offset,
+		           Binary(state, spv::OpISub, TypeU32(state), size, ConstantU32(state, bytes))));
+	};
+	const auto structured = AndCondition(
+	    state, buffer.index_in_bounds,
+	    formatted ? has_payload(buffer.stride)
+	              : Binary(state, spv::OpULessThan, TypeBool(state), offset, buffer.stride));
+	// OOB_SELECT=3 reduces NUM_RECORDS by SOFFSET before its offset/index checks.
+	const auto raw = AndCondition(
+	    state, buffer.scalar_in_bounds,
+	    Select(state, TypeBool(state), buffer.swizzle,
+	           AndCondition(state, buffer.raw_index_in_bounds, has_payload(buffer.stride)),
+	           has_payload(buffer.raw_records)));
+	auto in_bounds = Select(
+	    state, TypeBool(state),
+	    Binary(state, spv::OpIEqual, TypeBool(state), buffer.mode, ConstantU32(state, 0)),
+	    structured, buffer.index_in_bounds);
+	in_bounds = Select(
+	    state, TypeBool(state),
+	    Binary(state, spv::OpULessThan, TypeBool(state), buffer.mode, ConstantU32(state, 2)),
+	    in_bounds,
+	    Select(state, TypeBool(state),
+	           Binary(state, spv::OpIEqual, TypeBool(state), buffer.mode, ConstantU32(state, 2)),
+	           Binary(state, spv::OpINotEqual, TypeBool(state), buffer.records, ConstantU32(state, 0)),
+	           raw));
+	return in_bounds;
+}
+
+uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
+	auto& state = ctx.state;
+	const auto buffer = PrepareIndirectBuffer(ctx, inst);
+	const auto valid_format = Binary(state, spv::OpINotEqual, TypeBool(state), buffer.format,
+	                                  ConstantU32(state, 0));
+	const auto base = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address,
+	                         ConstantDeviceAddress(state, ~uint64_t {3}));
 	std::array<uint32_t, 4> values {};
-	for (uint32_t component = 0; component < components; component++) {
-		const auto address = CalculateBufferAddress(state, index, ctx.Arg(inst, 2), soffset,
-		                                            ctx.Memory(inst).offset + component * 4u,
-		                                            stride, swizzle, index_stride);
-		// RDNA2 raw DWORD vectors check each component, unlike formatted accesses.
-		const auto structured_bounds =
-		    AndCondition(state, index_in_bounds,
-		                 Binary(state, spv::OpULessThan, TypeBool(state), address.offset, stride));
-		// OOB_SELECT=3 reduces NUM_RECORDS by SOFFSET before the offset/index check.
-		const auto raw_bounds = AndCondition(
-		    state, scalar_in_bounds,
-		    Select(state, TypeBool(state), swizzle,
-		           AndCondition(state, raw_index_in_bounds, has_dword(address.offset, stride)),
-		           has_dword(address.offset, raw_records)));
-		auto in_bounds =
-		    Select(state, TypeBool(state),
-		           Binary(state, spv::OpIEqual, TypeBool(state), mode, ConstantU32(state, 0)),
-		           structured_bounds, index_in_bounds);
-		in_bounds = Select(
-		    state, TypeBool(state),
-		    Binary(state, spv::OpULessThan, TypeBool(state), mode, ConstantU32(state, 2)),
-		    in_bounds,
-		    Select(state, TypeBool(state),
-		           Binary(state, spv::OpIEqual, TypeBool(state), mode, ConstantU32(state, 2)),
-		           nonzero(records), raw_bounds));
-		const auto guest =
-		    Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
-		           Unary(state, spv::OpUConvert, TypeScalarU64(state), address.byte));
-		values[component] = LoadBda(ctx, guest, AndCondition(state, valid_format, in_bounds), 32u);
+	for (uint32_t component = 0; component < components; ++component) {
+		const auto address = component == 0u ? base : Binary(
+		    state, spv::OpIAdd, TypeScalarU64(state), base,
+		    ConstantDeviceAddress(state, component * 4u));
+		values[component] = EmitValueOrZeroIfCondition(
+		    state, AndCondition(state, valid_format,
+		                        IndirectBufferInBounds(state, buffer, component * 4u, 4u, false)),
+		    [&]() { return LoadBdaDword(ctx, address); });
 	}
 	return ConstructU32Composite(state, components, values);
 }
 
-uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
+uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& state = ctx.state;
+	return EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto buffer = PrepareIndirectBuffer(ctx, inst);
+		return EmitIndexSwitch(
+		    state, buffer.format, static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32Float) + 1u,
+		    TypeU32(state), [&](uint32_t format) {
+			    const auto info = Format::GetFormatInfo(static_cast<Prospero::BufferFormat>(format));
+			    if (info.type == Format::ComponentType::Unknown) return ConstantU32(state, 0);
+			    const auto constant = Select(
+			        state, TypeU32(state),
+			        Binary(state, spv::OpIEqual, TypeBool(state), buffer.selector, ConstantU32(state, 1)),
+			        FormattedConstant(ctx, info, FormattedSourceKind::One), ConstantU32(state, 0));
+			    return EmitValueOrDefaultIfCondition(
+			        state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector,
+			                      ConstantU32(state, 4)),
+			        TypeU32(state), constant, [&]() {
+				        const auto in_bounds = IndirectBufferInBounds(state, buffer, 0u, info.byte_size, true);
+				        const auto base = Binary(
+				            state, spv::OpBitwiseAnd, TypeScalarU64(state), buffer.address,
+				            ConstantDeviceAddress(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u)));
+				        const auto load = [&](uint32_t component, uint32_t bits) {
+					        const auto offset = Format::GetFormatComponentByteOffset(info, component);
+					        const auto address = offset == 0u ? base : Binary(
+					            state, spv::OpIAdd, TypeScalarU64(state), base,
+					            ConstantDeviceAddress(state, offset));
+					        return LoadBda(ctx, address, in_bounds, bits);
+				        };
+				        const auto emit_component = [&](uint32_t component) {
+					        return LoadFormattedComponent(
+					            ctx, info, {FormattedSourceKind::Memory, component},
+					            [&](uint32_t source) { return load(source, 32u); }, load);
+				        };
+				        if (info.component_count == 1u) return emit_component(0u);
+				        const auto component = Binary(
+				            state, spv::OpUMod, TypeU32(state),
+				            Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)),
+				            ConstantU32(state, info.component_count));
+				        return EmitIndexSwitch(state, component, info.component_count, TypeU32(state),
+				                               emit_component);
+			        });
+		    });
+	});
+}
+
+uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
 	auto& state = ctx.state;
 	return EmitValueOrDefaultIfCondition(
 	    state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
@@ -851,8 +985,7 @@ uint32_t LoadWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co
 		    const auto info = Format::GetFormatInfo(
 		        mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid);
 		    if (info.type != Format::ComponentType::Unknown) {
-			    const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components,
-			                                             FormattedAccess::Load);
+			    const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info);
 			    return EmitValueOrDefaultIfCondition(
 			        state, plan.in_bounds, TypeU32Composite(state, components),
 			        FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() {
@@ -907,9 +1040,8 @@ uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co
 			                               ? base
 			                               : Binary(state, spv::OpIAdd, TypeU32(state), base,
 			                                        ConstantU32(state, component * 4u));
-			    const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+			    const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
 			                                  address, ConstantU32(state, 2));
-			    const auto index     = EmitMemoryElementIndex(state, resource, raw_index);
 			    values[component]    = EmitValueOrZeroIfCondition(
 			        state, EmitMemoryElementInBounds(state, resource, index),
 			        [&]() { return LoadWordInBounds(ctx, resource, index); });
@@ -928,9 +1060,8 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 			const auto address = component == 0u ? base
 			                                     : Binary(state, spv::OpIAdd, TypeU32(state), base,
 			                                              ConstantU32(state, component * 4u));
-			const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
+			const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
 			                              ConstantU32(state, 2));
-			const auto index = EmitMemoryElementIndex(state, resource, raw_index);
 			EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index),
 			                [&]() {
 				                StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, component + 1u));
@@ -988,7 +1119,7 @@ void DefineGetBdaPointer(EmitterState& state) {
 	                                  ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
 	const auto page          = Unary(state, spv::OpUConvert, TypeU32(state), page64);
 	const auto entry_pointer = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferU64ElementPointer(state),
+	state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64),
 	                          entry_pointer, state.bda_pagetable_variable, ConstantU32(state, 0),
 	                          page);
 	const auto base = state.builder.AllocateId();
@@ -1036,16 +1167,15 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitValueOrDefaultIfCondition(
 	    state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
 		    const auto resource = PrepareStorageBufferResourceAccess(
-		        state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
-		    const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
-		                                     ByteAddress(ctx, inst, mem), resource.byte_offset);
+		        state, mem, state.storage_buffer_u64_variable, TypeStorageBufferPointer(state, 64));
+		    const auto byte_address = ByteAddress(ctx, inst, mem);
 		    const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
 		                              ConstantU32(state, 3u));
 		    return EmitValueOrDefaultIfCondition(
 		        state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
 		        ConstantU64(state, 0), [&]() {
 			        const auto pointer = EmitStorageBufferElementPointer(
-			            state, resource, index, TypeStorageBufferU64ElementPointer(state));
+			            state, resource, index, TypeStorageBufferElementPointer(state, 64));
 			        const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
 			        EmitAtomicMemoryBarrier(state, mem.kind);
 			        return old;
@@ -1092,16 +1222,17 @@ void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto& mem       = ctx.Memory(inst);
 	const bool  max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
 	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
-		const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
+		const auto index = DwordIndex(ctx, inst, mem);
+		const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
 		EmitIfCondition(
-		    ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
+		    ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
 			    ctx.state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable,
 			                                  ctx.Arg(inst, 1));
 			    const auto data = ctx.state.builder.AllocateId();
 			    ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), data,
 			                                  ctx.scratch_u32_variable);
 			    AtomicUpdate(
-			        ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
+			        ctx.state, EmitMemoryElementPointer(ctx.state, resource, index),
 			        mem.kind, [&](uint32_t old) {
 				        const auto old_f =
 				            Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
@@ -1145,10 +1276,9 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 		           Binary(state, spv::OpIAdd, TypeU32(state), base, size),
 		           ConstantU32(state, 0xc000u)));
 	}
-	const auto raw_index =
+	const auto index =
 	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
 	const auto access = PrepareMemoryResourceAccess(state, mem);
-	const auto index  = EmitMemoryElementIndex(state, access, raw_index);
 	const auto exec   = ctx.Arg(inst, 1);
 	const auto ballot = ctx.Ballot(inst.Arg(1));
 	const auto low    = state.builder.AllocateId();
@@ -1207,6 +1337,10 @@ uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst) {
 void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto mem = ctx.Memory(inst);
 	if (mem.planning_only) return;
+	if (mem.kind == IR::ResourceKind::IndirectBuffer) {
+		ctx.Define(inst, LoadIndirectScalarBuffer(ctx, inst));
+		return;
+	}
 	auto& state        = ctx.state;
 	mem.kind           = IR::ResourceKind::ScalarBuffer;
 	auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
@@ -1215,7 +1349,10 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
 		index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
 	}
 	const auto access    = PrepareMemoryResourceAccess(state, mem);
-	const auto element   = EmitMemoryElementIndex(state, access, index);
+	// Scalar buffer addressing aligns the base and each offset independently.
+	const auto element = Binary(
+	    state, spv::OpIAdd, TypeU32(state), index,
+	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byte_offset, ConstantU32(state, 2)));
 	const auto condition = EmitMemoryElementInBounds(state, access, element);
 	ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
 		           return EmitNative<spv::OpLoad, IR::Type::U32>(
@@ -1233,8 +1370,9 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 	uint32_t   value;
 	if (mem.kind == IR::ResourceKind::FlatLocal)
 		value = LoadLocalFlat(ctx, inst);
-	else if (buffer_components > 1u)
-		value = LoadWideBuffer(ctx, inst, buffer_components);
+	else if (buffer_components > 1u ||
+	         (buffer_components == 1u && mem.kind == IR::ResourceKind::IndirectBuffer && !mem.formatted))
+		value = LoadBuffer(ctx, inst, buffer_components);
 	else if (shared_components > 1u)
 		value = LoadWideShared(ctx, inst, shared_components);
 	else if (mem.kind == IR::ResourceKind::ScalarAddress)
@@ -1244,7 +1382,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
 		value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),
 		                address_info.data_bits);
 	else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
-		value = FormattedLoad(ctx, inst, mem);
+		value = mem.kind == IR::ResourceKind::IndirectBuffer ? LoadIndirectFormattedX(ctx, inst)
+		                                                     : FormattedLoad(ctx, inst, mem);
 	else if (inst.GetType() == IR::Type::U8)
 		value = LoadSubword(ctx, inst, mem, 8, false);
 	else if (inst.GetType() == IR::Type::U16)
@@ -1282,6 +1421,15 @@ uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement);
 }
 
+void EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto keep = Unary(ctx.state, spv::OpNot, TypeU32(ctx.state), ctx.Arg(inst, 1));
+	EmitAtomicUpdate(ctx, inst, ctx.Memory(inst),
+	                 [keep](EmitterState& state, uint32_t old, uint32_t value) {
+		                 return Binary(state, spv::OpBitwiseOr, TypeU32(state),
+		                               Binary(state, spv::OpBitwiseAnd, TypeU32(state), old, keep), value);
+	                 });
+}
+
 uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0));
@@ -1291,6 +1439,52 @@ uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2));
 }
 
+uint32_t EmitPermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state   = ctx.state;
+	auto       lane    = EmitSubgroupLocalInvocationId(state);
+	if (state.lane_count == 2) {
+		lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane, ConstantU32(state, 31));
+	}
+	const auto address = ctx.Arg(inst, 1);
+	const auto word    = Binary(state, spv::OpShiftRightLogical, TypeU32(state), lane,
+	                            ConstantU32(state, 5));
+	const auto ballot_word = [&](uint32_t predicate) {
+		const auto ballot = state.builder.AllocateId();
+		const auto result = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
+		                          ConstantU32(state, spv::ScopeSubgroup), predicate);
+		state.builder.AddFunction(spv::OpVectorExtractDynamic, TypeU32(state), result, ballot, word);
+		return result;
+	};
+	// RDNA2 permutes independently within each 32-lane half. Intersect the source
+	// address bit ballots to find this destination's enabled writers without LDS.
+	auto writers = ballot_word(ctx.Arg(inst, 2));
+	for (uint32_t bit = 0; bit < 5; ++bit) {
+		const auto address_bit = Binary(state, spv::OpBitwiseAnd, TypeU32(state), address,
+		                                ConstantU32(state, 1u << (bit + 2)));
+		const auto mask = ballot_word(Binary(state, spv::OpINotEqual, TypeBool(state),
+		                                      address_bit, ConstantU32(state, 0)));
+		const auto lane_bit = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane,
+		                             ConstantU32(state, 1u << bit));
+		const auto selected = Select(
+		    state, TypeU32(state),
+		    Binary(state, spv::OpINotEqual, TypeBool(state), lane_bit, ConstantU32(state, 0)),
+		    mask, Unary(state, spv::OpNot, TypeU32(state), mask));
+		writers = Binary(state, spv::OpBitwiseAnd, TypeU32(state), writers, selected);
+	}
+	const auto active = Binary(state, spv::OpINotEqual, TypeBool(state), writers,
+	                           ConstantU32(state, 0));
+	const auto base = Binary(state, spv::OpBitwiseAnd, TypeU32(state), lane,
+	                         ConstantU32(state, ~31u));
+	const auto source = Select(state, TypeU32(state), active,
+	                           Binary(state, spv::OpBitwiseOr, TypeU32(state), base,
+	                                  EmitFindUMsb32(state, writers)), lane);
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), result,
+	                          ConstantU32(state, spv::ScopeSubgroup), ctx.Arg(inst, 0), source);
+	return Select(state, TypeU32(state), active, result, ConstantU32(state, 0));
+}
+
 uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&      state  = ctx.state;
 	const auto source = ctx.Arg(inst, 0);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -31,6 +31,51 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
 	return ret;
 }
 
+uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
+                                   uint32_t search_iterations, uint32_t default_resource) {
+	const auto LoadMapping = [&](uint32_t index) {
+		const auto pointer = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
+		                          pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+		                          index);
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
+		return value;
+	};
+	const auto mapping = ConstantU32(state, mapping_offset);
+	const auto count = LoadMapping(mapping);
+	const auto entry_at = [&](uint32_t index) {
+		return Binary(state, spv::OpIAdd, TypeU32(state), mapping,
+		              Binary(state, spv::OpIAdd, TypeU32(state),
+		                     Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
+		                            ConstantU32(state, 1u)), ConstantU32(state, 1u)));
+	};
+	auto low = ConstantU32(state, 0u);
+	auto high = count;
+	for (uint32_t iteration = 0; iteration < search_iterations; ++iteration) {
+		const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
+		const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
+		                        Binary(state, spv::OpIAdd, TypeU32(state), low, high),
+		                        ConstantU32(state, 1u));
+		const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
+		const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
+		                         LoadMapping(entry_at(probe)), key);
+		low = Select(state, TypeU32(state),
+		             Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
+		             Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
+		high = Select(state, TypeU32(state),
+		              Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
+		                     Unary(state, spv::OpLogicalNot, TypeBool(state), less)), mid, high);
+	}
+	const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
+	const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
+	const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
+	                          Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
+	const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
+	                                         ConstantU32(state, 1u)));
+	return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
+}
+
 uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
@@ -47,6 +92,15 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
 	return state.program.info.buffers[mem.resource].descriptor_format;
 }
 
+uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) {
+	if (!mem.formatted) return mem.data_bits;
+	const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
+	                              : program.info.buffers[mem.resource].descriptor_format;
+	const auto info = Format::GetFormatInfo(format);
+	return info.type == Format::ComponentType::Unknown || info.packed_bitfield
+	           ? 32u : info.component_bits[0];
+}
+
 void EmitMemoryOffsets(EmitterState& state) {
 	for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
 		const auto word =
@@ -149,24 +203,18 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
 			EXIT("physical address memory must use the BDA emitter\n");
 		case IR::ResourceKind::ScalarBuffer:
 		case IR::ResourceKind::Buffer: {
+			const auto bits = mem.kind == IR::ResourceKind::Buffer
+			                      ? StorageBufferElementBits(state.program, mem) : 32u;
+			const auto variable = bits == 8u ? state.storage_buffer_u8_variable
+			                      : bits == 16u ? state.storage_buffer_u16_variable
+			                                    : state.storage_buffer_variable;
 			access = PrepareStorageBufferResourceAccess(
-			    state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
-			access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
-			                                    ConstantU32(state, 2u));
-			access.add_index_offset = true;
+			    state, mem, variable, TypeStorageBufferPointer(state, bits));
+			access.element_bits = bits;
 			return access;
 		}
 		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
 	}
-	access.length = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
-	                          access.object_pointer, 0);
-	return access;
-}
-
-uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
-                                uint32_t raw_index) {
-	return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
 }
 
 uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
@@ -195,7 +243,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
 		return pointer;
 	}
 	return EmitStorageBufferElementPointer(state, access, index,
-	                                       TypeStorageBufferElementPointer(state));
+	                                       TypeStorageBufferElementPointer(state, access.element_bits));
 }
 
 uint32_t EmitStorageBufferElementPointer(EmitterState& state,

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMesh.cpp

@@ -25,43 +25,58 @@ uint32_t MeshLoad(EmitterState& state, uint32_t variable, spv::StorageClass stor
 }
 
 uint32_t MeshOutputType(EmitterState& state, IR::StageOutputKind kind) {
-	return kind == IR::StageOutputKind::Layer ? TypeU32(state) : TypeF32Vector(state, 4);
+	switch (kind) {
+		case IR::StageOutputKind::Layer: return TypeU32(state);
+		case IR::StageOutputKind::ClipDistance:
+		case IR::StageOutputKind::CullDistance: return TypeF32(state);
+		case IR::StageOutputKind::Position:
+		case IR::StageOutputKind::Parameter: return TypeF32Vector(state, 4);
+		default: EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(kind));
+	}
 }
 
 } // namespace
 
-void DefineMeshOutputs(EmitterState& state) {
+void DefineMeshOutputs(EmitterState& state, uint32_t clip_distance_count,
+                       uint32_t cull_distance_count) {
 	const auto& mesh = state.input_info.vertex->mesh;
 	for (auto& output: state.outputs) {
-		if (output.kind != IR::StageOutputKind::Position &&
-		    output.kind != IR::StageOutputKind::Parameter &&
-		    output.kind != IR::StageOutputKind::Layer) {
-			EXIT("unsupported mesh output kind=%u\n", static_cast<uint32_t>(output.kind));
+		const auto type = MeshOutputType(state, output.kind);
+		const bool clip = output.kind == IR::StageOutputKind::ClipDistance;
+		const bool cull = output.kind == IR::StageOutputKind::CullDistance;
+		auto& variable = clip ? state.clip_distance_variable
+		                 : cull ? state.cull_distance_variable : output.variable_id;
+		if (variable == 0) {
+			const auto element_type = clip || cull
+			                              ? state.builder.Type(spv::OpTypeArray, type,
+			                                                   ConstantU32(state, clip ? clip_distance_count
+			                                                                           : cull_distance_count))
+			                              : type;
+			variable = MeshArray(state, spv::StorageClassOutput, element_type,
+			                     output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives
+			                                                             : mesh.max_vertices);
+			state.interface_variables.push_back(variable);
+			state.builder.AddName(variable, output.debug_name.c_str());
+			if (output.kind == IR::StageOutputKind::Parameter) {
+				state.builder.AddAnnotation(spv::OpDecorate, variable,
+				                            spv::DecorationLocation, output.location);
+			} else {
+				const auto builtin = clip ? spv::BuiltInClipDistance
+				                     : cull ? spv::BuiltInCullDistance
+				                     : output.kind == IR::StageOutputKind::Layer ? spv::BuiltInLayer
+				                                                                 : spv::BuiltInPosition;
+				state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn, builtin);
+			}
+			if (output.kind == IR::StageOutputKind::Layer) {
+				state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationPerPrimitiveEXT);
+			}
 		}
-		const auto type    = MeshOutputType(state, output.kind);
-		output.variable_id = MeshArray(
-		    state, spv::StorageClassOutput, type,
-		    output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
+		output.variable_id = variable;
 		// Only Layer is read by another invocation, through the primitive's provoking vertex.
 		const bool shared = output.kind == IR::StageOutputKind::Layer;
 		output.mesh_data_variable =
 		    MeshArray(state, shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, type,
 		              shared ? mesh.max_vertices : state.lane_count);
-		state.interface_variables.push_back(output.variable_id);
-		state.builder.AddName(output.variable_id, output.debug_name.c_str());
-		if (output.kind == IR::StageOutputKind::Parameter) {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
-			                            spv::DecorationLocation, output.location);
-		} else {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id, spv::DecorationBuiltIn,
-			                            output.kind == IR::StageOutputKind::Layer
-			                                ? spv::BuiltInLayer
-			                                : spv::BuiltInPosition);
-		}
-		if (output.kind == IR::StageOutputKind::Layer) {
-			state.builder.AddAnnotation(spv::OpDecorate, output.variable_id,
-			                            spv::DecorationPerPrimitiveEXT); // PerPrimitiveEXT
-		}
 	}
 	state.mesh_allocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2);
 	state.mesh_primitive_data =
@@ -148,8 +163,16 @@ void EmitMeshEntryPoint(EmitterState& state) {
 				const auto value =
 				    MeshLoad(state, output.mesh_data_variable, spv::StorageClassPrivate, type,
 				             ConstantU32(state, half));
-				const auto pointer =
-				    MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				uint32_t pointer;
+				if (output.kind == IR::StageOutputKind::ClipDistance ||
+				    output.kind == IR::StageOutputKind::CullDistance) {
+					pointer = state.builder.AllocateId();
+					state.builder.AddFunction(
+					    spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, type), pointer,
+					    output.variable_id, index, ConstantU32(state, output.index));
+				} else {
+					pointer = MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
+				}
 				state.builder.AddFunction(spv::OpStore, pointer, value);
 			}
 		});

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -91,7 +91,8 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
 }
 
 uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
-	EXIT_IF(components < 2u || components > 4u);
+	EXIT_IF(components < 1u || components > 4u);
+	if (components == 1u) return TypeU32(state);
 	return components == 2u ? TypeU32Pair(state) : TypeU32Vector(state, components);
 }
 
@@ -122,46 +123,31 @@ uint32_t TypeFunction(EmitterState& state) {
 	return state.function_type;
 }
 
-uint32_t StorageRuntimeArrayType(EmitterState& state) {
-	return state.builder.DecoratedType(
-	    spv::OpTypeRuntimeArray,
-	    {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}}, TypeU32(state));
+uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) {
+	return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state)
+	                                                   : state.builder.Type(spv::OpTypeInt, bits, 0);
 }
 
-uint32_t StorageBufferType(EmitterState& state) {
+uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) {
+	return state.builder.DecoratedType(
+	    spv::OpTypeRuntimeArray,
+	    {{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}},
+	    TypeStorageBufferElement(state, bits));
+}
+
+uint32_t StorageBufferType(EmitterState& state, uint32_t bits = 32) {
 	return state.builder.DecoratedType(spv::OpTypeStruct,
 	                                   {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
 	                                    {spv::OpDecorate, {spv::DecorationBlock}}},
-	                                   StorageRuntimeArrayType(state));
+	                                   StorageRuntimeArrayType(state, bits));
 }
 
-uint32_t TypeStorageBufferPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state));
+uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits) {
+	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state, bits));
 }
 
-uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state));
-}
-
-uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
-	return state.builder.DecoratedType(
-	    spv::OpTypeRuntimeArray,
-	    {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}}, TypeScalarU64(state));
-}
-
-uint32_t StorageBufferU64Type(EmitterState& state) {
-	return state.builder.DecoratedType(spv::OpTypeStruct,
-	                                   {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
-	                                    {spv::OpDecorate, {spv::DecorationBlock}}},
-	                                   StorageU64RuntimeArrayType(state));
-}
-
-uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state));
-}
-
-uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
-	return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
+uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) {
+	return TypePointer(state, spv::StorageClassStorageBuffer, TypeStorageBufferElement(state, bits));
 }
 
 uint32_t TypePhysicalU32Pointer(EmitterState& state) {
@@ -243,27 +229,34 @@ void DefineDescriptors(EmitterState& state) {
 			case IR::DescriptorBindingKind::Buffers:
 				state.storage_buffer_variable =
 				    Define(ArrayType(StorageBufferType(state)), "buffers");
+				if (state.requirements.buffer_u8) {
+					state.storage_buffer_u8_variable =
+					    Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8");
+				}
+				if (state.requirements.buffer_u16) {
+					state.storage_buffer_u16_variable =
+					    Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16");
+				}
 				if (state.requirements.buffer_int64_atomics) {
 					state.storage_buffer_u64_variable =
-					    Define(ArrayType(StorageBufferU64Type(state)), "buffers_u64");
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
-					                            spv::DecorationAliased);
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
-					                            spv::DecorationAliased);
+					    Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64");
 				}
-				if (state.requirements.coherent_buffers) {
+				for (const auto variable: {state.storage_buffer_variable, state.storage_buffer_u8_variable,
+				                           state.storage_buffer_u16_variable, state.storage_buffer_u64_variable}) {
+					if (variable == 0) continue;
+					if (state.storage_buffer_u8_variable != 0 || state.storage_buffer_u16_variable != 0 ||
+					    state.storage_buffer_u64_variable != 0) {
+						state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
+					}
 					// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
 					// must participate in visibility for cache-bypassing polling loads.
-					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
-					                            spv::DecorationCoherent);
-					if (state.storage_buffer_u64_variable != 0) {
-						state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
-						                            spv::DecorationCoherent);
+					if (state.requirements.coherent_buffers) {
+						state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent);
 					}
 				}
 				break;
 			case IR::DescriptorBindingKind::BdaPagetable:
-				state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");
+				state.bda_pagetable_variable = Define(StorageBufferType(state, 64), "bda_pagetable");
 				break;
 			case IR::DescriptorBindingKind::FaultBuffer:
 				state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer");
@@ -333,8 +326,9 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value) {
 }
 
 uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
-	EXIT_IF(components < 2u || components > 4u);
+	EXIT_IF(components < 1u || components > 4u);
 	const auto            zero = ConstantU32(state, 0);
+	if (components == 1u) return zero;
 	std::vector<uint32_t> values(components, zero);
 	return state.builder.Constant(spv::OpConstantComposite, TypeU32Composite(state, components),
 	                              values);
@@ -541,7 +535,7 @@ void DefineOutputs(EmitterState& state) {
 		}
 	}
 	if (state.program.stage == ShaderType::Mesh) {
-		DefineMeshOutputs(state);
+		DefineMeshOutputs(state, clip_distance_count, cull_distance_count);
 		return;
 	}
 	if (state.program.stage == ShaderType::Vertex && clip_distance_count + cull_distance_count < 8u &&
@@ -667,6 +661,13 @@ void DefineModule(EmitterState& state) {
 
 	state.builder.RequireCapability(spv::CapabilityShader);
 	state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
+	if (state.requirements.buffer_u8) {
+		state.builder.RequireExtension("SPV_KHR_8bit_storage");
+		state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess);
+	}
+	if (state.requirements.buffer_u16) {
+		state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
+	}
 	if (state.program.info.uses_dma) {
 		state.builder.RequireCapability(spv::CapabilityInt64);
 		state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
@@ -701,8 +702,9 @@ void DefineModule(EmitterState& state) {
 	if (state.requirements.image_gather_extended) {
 		state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
 	}
-	if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
-	    state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
+	if (state.lane_count == 2 || state.requirements.subgroup_barrier ||
+	    state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
+	    state.requirements.subgroup_local_invocation_id) {
 		state.builder.RequireCapability(spv::CapabilityGroupNonUniform);
 	}
 	if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
@@ -737,15 +739,13 @@ void DefineModule(EmitterState& state) {
 	if (state.requirements.float64) {
 		EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
 		                     state.input_info.compute->float_mode != 0xc0);
-		// MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals.
+		// Use native rounding for MODE=0xc0, consistent with ordinary FP32 arithmetic.
 		state.builder.RequireCapability(spv::CapabilityFloat64);
-		state.builder.RequireCapability(spv::CapabilityRoundingModeRTE);
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
 		                               64u);
 		// FP64 denormal preservation is temporarily disabled.
 		// state.builder.RequireCapability(spv::CapabilityDenormPreserve);
 		// state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u);
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeRoundingModeRTE, 32u);
 	}
 	if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) {
 		uint32_t    local_x = state.requirements.compute_derivatives ? 2u : 1u;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -315,6 +315,7 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
 			auto& lane          = half == 0 ? ctx : *ctx.other_half;
 			ctx.state.lane_half = half;
 			if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier &&
+			                  inst.GetOpcode() != IR::ValueOpcode::StoreCompletion &&
 			                  inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) {
 				emit_instruction(lane, inst);
 			}

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -228,7 +228,8 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
-		case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
+		case 0xe6u:
+		case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
@@ -258,6 +259,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
 
 	switch (opcode) {
 		case 0xe6u: return 11u;
+		case 0xe7u: return 12u;
 		case 0x0eu: return 1u;
 		case 0x01u:
 		case 0x09u: return ImageCoordComponents(dimension) + 1u;
@@ -379,10 +381,10 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
-	if (opcode == 0xe6u &&
+	if (inst.opcode == Opcode::IMAGE_BVH_INTERSECT_RAY &&
 	    (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) {
 		SetUnsupported(inst, Family::MIMG, opcode,
-		               "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf");
+		               "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf");
 	}
 
 	DecodeVectorGpr(vdata, inst.dst);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -92,7 +92,8 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x05u, Opcode::DS_MIN_I32, 1, 32},          {0x06u, Opcode::DS_MAX_I32, 1, 32},
     {0x07u, Opcode::DS_MIN_U32, 1, 32},          {0x08u, Opcode::DS_MAX_U32, 1, 32},
     {0x09u, Opcode::DS_AND_B32, 1, 32},          {0x0au, Opcode::DS_OR_B32, 1, 32},
-    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0du, Opcode::DS_WRITE_B32, 1, 32},
+    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0cu, Opcode::DS_MSKOR_B32, 1, 32},
+    {0x0du, Opcode::DS_WRITE_B32, 1, 32},
     {0x0eu, Opcode::DS_WRITE2_B32, 2, 32},       {0x0fu, Opcode::DS_WRITE2ST64_B32, 2, 32},
     {0x12u, Opcode::DS_MIN_F32, 1, 32},          {0x13u, Opcode::DS_MAX_F32, 1, 32},
     {0x1eu, Opcode::DS_WRITE_B8, 1, 8},          {0x1fu, Opcode::DS_WRITE_B16, 1, 16},
@@ -116,7 +117,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0xa6u, Opcode::DS_READ_U16_D16, 1, 16},
     {0xa7u, Opcode::DS_READ_U16_D16_HI, 1, 16},
     {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
-    {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
+    {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32},     {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
     {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
     {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
 };
@@ -192,6 +193,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 		case Opcode::DS_OR_RTN_B32:
 		case Opcode::DS_XOR_B32:
 		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_WRXCHG_RTN_B32: return true;
 		default: return false;
 	}
@@ -203,8 +205,10 @@ uint32_t DsSourceCount(Opcode opcode) {
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
 		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_MIN_F32:
 		case Opcode::DS_MAX_F32: return 3u;
+		case Opcode::DS_PERMUTE_B32:
 		case Opcode::DS_BPERMUTE_B32: return 2u;
 		case Opcode::DS_READ_ADDTID_B32:
 		case Opcode::DS_CONSUME:
@@ -407,7 +411,8 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
 	if (inst.gds &&
-	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_BPERMUTE_B32 ||
+	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_PERMUTE_B32 ||
+	     inst.opcode == Opcode::DS_BPERMUTE_B32 ||
 	     inst.opcode == Opcode::DS_WRITE_ADDTID_B32 ||
 	     inst.opcode == Opcode::DS_READ_ADDTID_B32)) {
 		SetUnsupported(inst, Family::DS, opcode, "DS lane operation is available only for LDS");

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -86,7 +86,7 @@ constexpr OpcodeMap SOPK_OPCODE_LIST[] = {
     {0x08u, Opcode::S_CMP_LE_I32}, {0x09u, Opcode::S_CMP_EQ_U32}, {0x0au, Opcode::S_CMP_LG_U32},
     {0x0bu, Opcode::S_CMP_GT_U32}, {0x0cu, Opcode::S_CMP_GE_U32}, {0x0du, Opcode::S_CMP_LT_U32},
     {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32},    {0x10u, Opcode::S_MULK_I32},
-    {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT},    {0x18u, Opcode::S_WAITCNT},
+    {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT_VSCNT}, {0x18u, Opcode::S_WAITCNT},
     {0x19u, Opcode::S_WAITCNT},    {0x1au, Opcode::S_WAITCNT},
     {0x1bu, Opcode::S_SUBVECTOR_LOOP_BEGIN}, {0x1cu, Opcode::S_SUBVECTOR_LOOP_END},
 };
@@ -234,6 +234,14 @@ void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			inst.src_count         = 1;
 			return;
 		}
+		case Opcode::S_WAITCNT_VSCNT:
+			inst.dst.kind       = OperandKind::Null;
+			inst.src1           = inst.src0;
+			inst.src1.value     = word & 0xffffu;
+			inst.src1.signed_val = static_cast<int32_t>(inst.src1.value);
+			DecodeScalarSource(sdst, pc, inst.src0);
+			inst.src_count = 2;
+			return;
 		case Opcode::S_SETREG_B32:
 			inst.dst.kind        = OperandKind::Null;
 			inst.src1.kind       = OperandKind::LiteralConstant;

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -535,6 +535,11 @@ std::string InstructionToString(const Instruction& inst) {
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc,
 			                                               OperandToString(inst.src0).c_str(),
 			                                               OperandToString(inst.src1).c_str()));
+		case Opcode::S_WAITCNT_VSCNT:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               OperandToString(inst.src0),
+			                                               OperandToString(inst.src1)));
 		case Opcode::S_NOP:
 		case Opcode::S_WAITCNT:
 		case Opcode::S_WAITCNT_DEPCTR:
@@ -692,10 +697,12 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_OR_RTN_B32:
 		case Opcode::DS_XOR_B32:
 		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_MSKOR_B32:
 		case Opcode::DS_WRXCHG_RTN_B32:
 		case Opcode::DS_MIN_F32:
 		case Opcode::DS_MAX_F32:
 		case Opcode::DS_SWIZZLE_B32:
+		case Opcode::DS_PERMUTE_B32:
 		case Opcode::DS_BPERMUTE_B32:
 		case Opcode::DS_READ_I8:
 		case Opcode::DS_READ_U8:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -186,6 +186,7 @@ enum class Opcode {
 	V_RCP_F32,
 	V_RCP_IFLAG_F32,
 	V_FRACT_F32,
+	V_FRACT_F64,
 	V_TRUNC_F32,
 	V_CEIL_F32,
 	V_RNDNE_F32,
@@ -445,6 +446,7 @@ enum class Opcode {
 	V_CMP_EQ_I64,
 	V_CMP_LT_I64,
 	V_CMP_LE_I64,
+	V_CMP_NE_I64,
 	V_CMP_LT_U64,
 	V_CMP_EQ_U64,
 	V_CMP_LE_U64,
@@ -556,10 +558,12 @@ enum class Opcode {
 	DS_OR_RTN_B32,
 	DS_XOR_B32,
 	DS_XOR_RTN_B32,
+	DS_MSKOR_B32,
 	DS_WRXCHG_RTN_B32,
 	DS_MIN_F32,
 	DS_MAX_F32,
 	DS_SWIZZLE_B32,
+	DS_PERMUTE_B32,
 	DS_BPERMUTE_B32,
 	DS_CONSUME,
 	DS_APPEND,
@@ -625,6 +629,7 @@ enum class Opcode {
 
 	S_NOP,
 	S_WAITCNT,
+	S_WAITCNT_VSCNT,
 	S_WAITCNT_DEPCTR,
 	S_BARRIER,
 	S_BRANCH,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -123,6 +123,7 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
     {0x39u, Opcode::V_FFBH_U32},
     {0x3au, Opcode::V_FFBL_B32},
     {0x3bu, Opcode::V_FFBH_I32},
+    {0x3eu, Opcode::V_FRACT_F64},
     {0x3fu, Opcode::V_FREXP_EXP_I32_F32},
     {0x40u, Opcode::V_FREXP_MANT_F32},
     {0x42u, Opcode::V_MOVRELD_B32},
@@ -182,6 +183,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x39u, Opcode::V_FFBH_U32},
     {0x3au, Opcode::V_FFBL_B32},
     {0x3bu, Opcode::V_FFBH_I32},
+    {0x3eu, Opcode::V_FRACT_F64},
     {0x3fu, Opcode::V_FREXP_EXP_I32_F32},
     {0x40u, Opcode::V_FREXP_MANT_F32},
     {0x42u, Opcode::V_MOVRELD_B32},
@@ -254,6 +256,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0xc5u, Opcode::V_CMP_NE_U32},         {0xc6u, Opcode::V_CMP_GE_U32},
     {0xc7u, Opcode::V_CMP_T_U32},          {0xa2u, Opcode::V_CMP_EQ_I64, false},
     {0xa1u, Opcode::V_CMP_LT_I64, false},  {0xa3u, Opcode::V_CMP_LE_I64, false},
+    {0xa5u, Opcode::V_CMP_NE_I64, false},
     {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
     {0xd2u, Opcode::V_CMPX_EQ_U32},        {0xd3u, Opcode::V_CMPX_LE_U32},
     {0xd4u, Opcode::V_CMPX_GT_U32},        {0xd5u, Opcode::V_CMPX_NE_U32},
@@ -472,11 +475,18 @@ bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
 bool IsVop1FloatResultOpcode(Opcode opcode);
 bool IsVopcCompareExec(Opcode opcode);
 
+bool IsVop1Float64Opcode(Opcode opcode) {
+	return opcode == Opcode::V_CVT_F64_I32 || opcode == Opcode::V_CVT_F32_F64 ||
+	       opcode == Opcode::V_CVT_F64_F32 || opcode == Opcode::V_CVT_F64_U32 ||
+	       opcode == Opcode::V_RCP_F64 || opcode == Opcode::V_FRACT_F64;
+}
+
 bool IsVop1FloatSourceOpcode(Opcode opcode) {
 	switch (opcode) {
 		case Opcode::V_CVT_F32_F64:
 		case Opcode::V_CVT_F64_F32:
 		case Opcode::V_RCP_F64:
+		case Opcode::V_FRACT_F64:
 		case Opcode::V_MOV_B32:
 		case Opcode::V_CVT_F32_F16:
 		case Opcode::V_CVT_U32_F32:
@@ -551,6 +561,7 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
     // partial destination may combine with any source selector.
     {Opcode::V_NOT_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(),
      SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
+    {Opcode::V_BFREV_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_FFBL_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_FFBH_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
@@ -647,6 +658,10 @@ bool SupportsVop1Clamp(Opcode opcode) {
 }
 
 bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
+	if (IsVop1Float64Opcode(inst.opcode)) {
+		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support SDWA");
+		return false;
+	}
 	const auto dst_sel   = (modifier >> 8u) & 0x7u;
 	const auto dst_u     = (modifier >> 11u) & 0x3u;
 	const auto clamp     = (modifier >> 13u) & 0x1u;
@@ -735,9 +750,7 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
 
 void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                    uint32_t opcode, uint32_t vdst, Instruction& inst) {
-	if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 ||
-	    inst.opcode == Opcode::V_CVT_F64_F32 || inst.opcode == Opcode::V_CVT_F64_U32 ||
-	    inst.opcode == Opcode::V_RCP_F64) {
+	if (IsVop1Float64Opcode(inst.opcode)) {
 		SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP");
 		return;
 	}
@@ -1152,7 +1165,8 @@ VopcSdwaFields DecodeVopcSdwaFields(uint32_t modifier) {
 }
 
 bool SupportsVopcSdwa(Opcode opcode) {
-	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_EQ_F64 &&
+	return opcode != Opcode::UNSUPPORTED && opcode != Opcode::V_CMP_NE_I64 &&
+	       opcode != Opcode::V_CMP_EQ_F64 &&
 	       opcode != Opcode::V_CMP_LE_F64 && opcode != Opcode::V_CMPX_LE_F64 &&
 	       opcode != Opcode::V_CMPX_GE_F64;
 }

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -31,6 +31,7 @@ void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueO
 	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
 	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
 	                       inst.opcode == Decoder::Opcode::V_CMP_EQ_I64 ||
+	                       inst.opcode == Decoder::Opcode::V_CMP_NE_I64 ||
 	                       inst.opcode == Decoder::Opcode::V_CMPX_NE_I64;
 	const auto read = [&](const Decoder::Operand& operand) {
 		// RDNA2 expands signed 64-bit integer literals by sign extension.

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -181,8 +181,12 @@ void Translator::EmitControlNop() {
 	ir.Emit(IR::ValueOpcode::ControlNop);
 }
 
-void Translator::EmitWaitcnt() {
-	ir.Emit(IR::ValueOpcode::Waitcnt);
+void Translator::S_WAITCNT_VSCNT(const Decoder::Instruction& inst) {
+	const auto count = inst.src1.value & 63u;
+	if (inst.src0.kind != Decoder::OperandKind::Null || (count != 0u && count != 63u)) {
+		EXIT("unsupported partial or register-based S_WAITCNT_VSCNT at 0x%08x", inst.pc);
+	}
+	if (count == 0u) ir.Emit(IR::ValueOpcode::StoreCompletion);
 }
 
 void Translator::S_BARRIER() {
@@ -252,13 +256,14 @@ void Translator::ScalarSelect64(const Decoder::Instruction& inst,
 	const auto selected_mask_valid =
 	    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
 	                   {condition, ReadMaskValid(inst.src0), ReadMaskValid(false_source)}));
-	if (IsExecOrVcc(inst.dst)) {
-		WriteMask(inst.dst, selected_mask, true);
-		return;
-	}
 	WriteU32Pair(inst.dst,
 	             {ir.Select(condition, lhs[0], rhs[0]), ir.Select(condition, lhs[1], rhs[1])});
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+	// Scalar selects preserve the raw pair even when EXEC/VCC holds an address.
+	if (inst.dst.kind == Decoder::OperandKind::ExecLo) {
+		ir.SetExec(selected_mask);
+	} else if (inst.dst.kind == Decoder::OperandKind::VccLo) {
+		ir.SetVcc(selected_mask);
+	} else if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
 		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
 		ir.SetThreadBitScalarReg(dst, selected_mask);
 		ir.SetScalarMaskTag(dst, selected_mask_valid);

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -483,10 +483,11 @@ void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
 	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
 }
 
-void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
-	const auto hi      = ReadU32(inst.src0);
-	const auto lo      = ReadU32(inst.src1);
-	const auto shift   = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
+void Translator::V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset) {
+	const auto hi = ReadU32(inst.src0);
+	const auto lo = ReadU32(inst.src1);
+	auto shift    = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(byte_offset ? 3u : 31u)));
+	if (byte_offset) shift = ir.ShiftLeftLogical(shift, IR::U32(IR::Value(3u)));
 	const auto lo_part = ir.ShiftRightLogical(lo, shift);
 	const auto inverse =
 	    ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(32u)), shift), IR::U32(IR::Value(31u)));
@@ -496,21 +497,6 @@ void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(lo_part, hi_part));
 }
 
-void Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
-	const auto hi           = ReadU32(inst.src0);
-	const auto lo           = ReadU32(inst.src1);
-	const auto byte_offset  = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
-	const auto bit_offset   = ir.ShiftLeftLogical(byte_offset, IR::U32(IR::Value(3u)));
-	const auto concatenated = ir.ConstructU64(lo, hi);
-	const auto shifted =
-	    IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64,
-	                    {concatenated, ir.BitwiseAnd(bit_offset, IR::U32(IR::Value(63u)))}));
-	const auto in_range =
-	    IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, IR::Value(8u)}));
-	WriteOperand(DestinationOperand(inst),
-	             ir.Select(in_range, ExtractU64(shifted)[0], IR::U32(IR::Value(0u))));
-}
-
 void Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
 	const auto shift  = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -239,9 +239,6 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 			case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
 			case Decoder::Opcode::DS_WRITE_ADDTID_B32:
 				return index == 0u ? decoded.src1 : MakeM0Operand();
-			case Decoder::Opcode::DS_MIN_F32:
-			case Decoder::Opcode::DS_MAX_F32:
-				return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
 			case Decoder::Opcode::DS_WRITE_B8:
 			case Decoder::Opcode::DS_WRITE_B16:
 			case Decoder::Opcode::DS_WRITE_B8_D16_HI:
@@ -255,12 +252,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
 			case Decoder::Opcode::DS_WRITE2_B64:
 			case Decoder::Opcode::DS_WRITE2ST64_B64:
 				return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
-			default:
-				if (decoded.opcode >= Decoder::Opcode::DS_ADD_U32 &&
-				    decoded.opcode <= Decoder::Opcode::DS_WRXCHG_RTN_B32) {
-					return index == 0u ? decoded.src1 : decoded.src0;
-				}
-				return decoded.src0;
+			default: return decoded.src0;
 		}
 	}
 	return DecodedSourceAt(decoded, index);
@@ -580,12 +572,13 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                            bool returns_value) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
-	const auto address = ReadU32(MemorySourceAt(inst, 1));
-	const auto data_src = MemorySourceAt(inst, 0);
+	const auto address = ReadU32(inst.src0);
 	const IR::Value value =
-	    memory.data_dwords == 2u ? IR::Value(ReadU64(data_src)) : IR::Value(ReadU32(data_src));
-	const auto result  = ir.Emit(opcode, {address, value, ir.GetExec()},
-	                             AddMemoryInfo(memory, inst.pc));
+	    memory.data_dwords == 2u ? IR::Value(ReadU64(inst.src1)) : IR::Value(ReadU32(inst.src1));
+	const auto flags = AddMemoryInfo(memory, inst.pc);
+	const auto result = inst.src_count == 3u
+	                        ? ir.Emit(opcode, {address, value, ReadU32(inst.src2), ir.GetExec()}, flags)
+	                        : ir.Emit(opcode, {address, value, ir.GetExec()}, flags);
 	if (returns_value) {
 		WriteOperand(inst.dst, result);
 	}
@@ -857,14 +850,6 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
 	}
 }
 
-void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
-	const auto memory = MemoryInfoFromDecoded(inst);
-	ir.Emit(opcode,
-	        {ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)),
-	         ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()},
-	        AddMemoryInfo(memory, inst.pc));
-}
-
 void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	WriteOperand(inst.dst, ir.Emit(opcode,
@@ -894,9 +879,10 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
 	                                ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()}));
 }
 
-void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
+void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) {
 	const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset)));
-	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32,
+	WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32
+	                                       : IR::ValueOpcode::PermuteU32,
 	                               {ReadU32(inst.src1), address, ir.GetExec()}));
 }
 
@@ -904,7 +890,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
 		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
 			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
-			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()},
+			    inst.image_address_components - 10u);
 			for (uint32_t component = 0; component < 4; ++component) {
 				WriteOperand(OffsetOperand(inst.dst, component),
 				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)}));
@@ -1046,6 +1033,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, false);
 		case Decoder::Opcode::DS_XOR_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, true);
+		case Decoder::Opcode::DS_MSKOR_B32:
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicMaskedOr32, false);
 		case Decoder::Opcode::DS_WRXCHG_RTN_B32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
 
@@ -1108,11 +1097,12 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst);
 
 		case Decoder::Opcode::DS_MIN_F32:
-			return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMin32);
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false);
 		case Decoder::Opcode::DS_MAX_F32:
-			return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32);
+			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMax32, false);
 		case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst);
-		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_BPERMUTE_B32(inst);
+		case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false);
+		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true);
 		case Decoder::Opcode::DS_CONSUME:
 			return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume);
 		case Decoder::Opcode::DS_APPEND:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -18,7 +18,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return;
 		case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return;
 		case O::S_SETREG_B32: EmitControlNop(); return;
-		case O::S_WAITCNT: EmitWaitcnt(); return;
+		case O::S_WAITCNT_VSCNT: S_WAITCNT_VSCNT(inst); return;
+		case O::S_WAITCNT: return;
 
 		case O::S_AND_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
@@ -224,7 +225,7 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_SLEEP:
 		case O::S_SETPRIO:
 		case O::S_TRAP: EmitControlNop(); return;
-		case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return;
+		case O::S_WAITCNT_DEPCTR: return;
 		case O::S_BARRIER: S_BARRIER(); return;
 		case O::S_SENDMSG: S_SENDMSG(inst); return;
 		case O::S_TTRACEDATA: S_TTRACEDATA(); return;

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -896,7 +896,8 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_ADD_F64:
 		case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
-		case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
+		case Decoder::Opcode::V_RCP_F64:
+		case Decoder::Opcode::V_FRACT_F64: include_vector(inst.dst, 2u); [[fallthrough]];
 		case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
 		case Decoder::Opcode::V_CMP_EQ_F64:
 		case Decoder::Opcode::V_CMP_LE_F64:
@@ -905,6 +906,7 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
 		case Decoder::Opcode::V_CMP_EQ_I64:
 		case Decoder::Opcode::V_CMP_LT_I64:
 		case Decoder::Opcode::V_CMP_LE_I64:
+		case Decoder::Opcode::V_CMP_NE_I64:
 		case Decoder::Opcode::V_CMP_LT_U64:
 		case Decoder::Opcode::V_CMP_EQ_U64:
 		case Decoder::Opcode::V_CMP_LE_U64:
@@ -1104,6 +1106,16 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				                                  IR::U32(IR::Value(total_threads)));
 			}
 		}
+		if (options.stage == ShaderType::Compute &&
+		    options.input_info.compute->dispatch_thread_dimensions) {
+			for (uint32_t axis = 0; axis < 3u; axis++) {
+				const auto extent = IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::GetDispatchThreadExtent, {IR::Value(axis)}));
+				initial_exec = entry_ir.LogicalAnd(
+				    initial_exec,
+				    entry_ir.ULessThan(builtin(IR::StageInputKind::GlobalInvocationId, axis), extent));
+			}
+		}
 		entry_ir.SetExec(initial_exec);
 		const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec});
 		entry_ir.SetExecLo(entry_ir.CompositeExtract(initial_mask, 0));

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -105,11 +105,10 @@ private:
 	void DS_READ2(const Decoder::Instruction& inst);
 	void DS_WRITE(const Decoder::Instruction& inst);
 	void DS_WRITE2(const Decoder::Instruction& inst);
-	void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void DS_ADDTID(const Decoder::Instruction& inst, bool write);
 	void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
-	void DS_BPERMUTE_B32(const Decoder::Instruction& inst);
+	void DS_PERMUTE(const Decoder::Instruction& inst, bool backward);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -208,8 +207,7 @@ private:
 	void    V_BFI_B32(const Decoder::Instruction& inst);
 	void    S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
 	void    S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
-	void    V_ALIGNBIT_B32(const Decoder::Instruction& inst);
-	void    V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
+	void    V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset);
 	void    V_LSHL_ADD_U32(const Decoder::Instruction& inst);
 	void    V_ADD_LSHL_U32(const Decoder::Instruction& inst);
 	void    V_XAD_U32(const Decoder::Instruction& inst);
@@ -229,7 +227,7 @@ private:
 	void ScalarMinMax32(const Decoder::Instruction& inst, IR::ValueOpcode value_opcode,
 	                    IR::ValueOpcode compare_opcode);
 	void EmitControlNop();
-	void EmitWaitcnt();
+	void S_WAITCNT_VSCNT(const Decoder::Instruction& inst);
 	void S_BARRIER();
 	void S_SENDMSG(const Decoder::Instruction& inst);
 	void S_TTRACEDATA();

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -106,6 +106,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMP_GT_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
 			return;
+		case O::V_CMP_NE_I64:
 		case O::V_CMP_NE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
 			return;
@@ -453,6 +454,7 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_RCP_F64: return FloatUnary(inst, IR::ValueOpcode::FPRecip64);
 		case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32);
 		case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32);
+		case O::V_FRACT_F64: return FloatUnary(inst, IR::ValueOpcode::FPFract64);
 		case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32);
 		case O::V_CEIL_F32: return FloatUnary(inst, IR::ValueOpcode::FPCeil32);
 		case O::V_RNDNE_F32: return FloatUnary(inst, IR::ValueOpcode::FPRoundEven32);
@@ -591,8 +593,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_BFE_U32: return V_BFE_U32(inst, false);
 		case O::V_BFE_I32: return V_BFE_U32(inst, true);
 		case O::V_BFI_B32: return V_BFI_B32(inst);
-		case O::V_ALIGNBIT_B32: return V_ALIGNBIT_B32(inst);
-		case O::V_ALIGNBYTE_B32: return V_ALIGNBYTE_B32(inst);
+		case O::V_ALIGNBIT_B32: return V_ALIGN_B32(inst, false);
+		case O::V_ALIGNBYTE_B32: return V_ALIGN_B32(inst, true);
 		case O::V_LSHL_ADD_U32: return V_LSHL_ADD_U32(inst);
 		case O::V_ADD_LSHL_U32: return V_ADD_LSHL_U32(inst);
 		case O::V_XAD_U32: return V_XAD_U32(inst);

src/graphics/shader/recompiler/ir/Program.cpp

@@ -190,6 +190,18 @@ Value ResolveInvariantPhi(const ResourcePlan& program, Value value) {
 	return invariant;
 }
 
+Value ResolveActiveU32(Value value, Value active) {
+	for (uint32_t depth = 0; depth <= 32; ++depth) {
+		value = value.Resolve();
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 ||
+		    inst->Arg(0).Resolve() != active)
+			return value;
+		value = inst->Arg(1);
+	}
+	return {};
+}
+
 bool HasShaderMemoryWrites(const Program& program) {
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
@@ -399,7 +411,8 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
 				const auto& memory = program.memory_info[memory_index];
-				if (memory.kind != ResourceKind::ScalarBuffer) {
+				if (memory.kind != ResourceKind::ScalarBuffer &&
+				    memory.kind != ResourceKind::IndirectBuffer) {
 					return Fail(fmt::format("{} has an invalid scalar-memory resource kind",
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
@@ -457,7 +470,7 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 				}
 				if (memory.kind == ResourceKind::IndirectBuffer &&
 				    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) {
-					return Fail("indirect buffer requires a raw DWORD x2/x3/x4 load");
+					return Fail("indirect buffer requires a scalar, raw DWORD x1/x2/x3/x4, or formatted X load");
 				}
 				if (buffer_components > 1u &&
 				    (!vector_buffer || memory.data_bits != 32u ||

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -73,9 +73,13 @@ struct MemoryInfo {
 	bool                    planning_only                                         = false;
 
 	[[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const {
-		return !formatted && !typed && data_bits == 32u &&
-		       (opcode == ValueOpcode::LoadBufferU32x2 || opcode == ValueOpcode::LoadBufferU32x3 ||
-		        opcode == ValueOpcode::LoadBufferU32x4);
+		return !typed && data_bits == 32u &&
+		       (formatted ? opcode == ValueOpcode::LoadBufferU32
+		                  : opcode == ValueOpcode::ReadConstBuffer ||
+		                        opcode == ValueOpcode::LoadBufferU32 ||
+		                        opcode == ValueOpcode::LoadBufferU32x2 ||
+		                        opcode == ValueOpcode::LoadBufferU32x3 ||
+		                        opcode == ValueOpcode::LoadBufferU32x4);
 	}
 
 	bool operator==(const MemoryInfo& other) const = default;
@@ -97,6 +101,7 @@ struct ExportInfo {
 
 struct BufferResource {
 	static constexpr uint32_t NoImageAlias = UINT32_MAX;
+	static constexpr uint32_t NoIndirectBuffer = UINT32_MAX;
 
 	uint32_t               source             = 0;
 	uint32_t               first_use_pc       = 0;
@@ -110,6 +115,10 @@ struct BufferResource {
 	bool                   atomic             = false;
 	bool                   formatted          = false;
 	bool                   scalar             = false;
+	uint32_t               indirect_root              = NoIndirectBuffer;
+	uint32_t               indirect_mapping_offset    = 0;
+	uint32_t               indirect_search_iterations = 0;
+	std::vector<uint32_t>  indirect_resources;
 
 	bool operator==(const BufferResource& other) const = default;
 };
@@ -421,6 +430,7 @@ struct DescriptorBinding {
 
 struct BindingLayout {
 	uint32_t                       push_data_start_dword = PushData::NoStart;
+	uint32_t                       dispatch_thread_dword = PushData::NoStart;
 	uint32_t                       memory_offset_dword = 0;
 	uint32_t                       memory_offset_count = 0;
 	std::vector<uint32_t>          user_data_registers;
@@ -472,22 +482,24 @@ struct BlockInfo {
 };
 
 struct DescriptorSource {
-	struct IndirectImage {
+	struct IndirectDescriptor {
 		uint32_t material_source = UINT32_MAX;
 		uint32_t table_source    = 0;
 		uint32_t selector_stride = 0;
 		uint32_t selector_offset = 0;
 		uint32_t table_offset    = 0;
+		uint32_t selector_shift  = 0;
 		Value    key_count;
+		Value                 selector_first;
 		Value    selector_mask;
 		std::vector<uint32_t> sources;
 
-		bool operator==(const IndirectImage& other) const = default;
+		bool operator==(const IndirectDescriptor& other) const = default;
 	};
 
 	std::array<Value, 8>         dwords {};
 	uint32_t                     dword_count = 0;
-	std::optional<IndirectImage> indirect_image;
+	std::optional<IndirectDescriptor> indirect_descriptor;
 
 	bool operator==(const DescriptorSource& other) const = default;
 };
@@ -610,6 +622,7 @@ void  ValidateProgram(const Program& program, bool require_ssa);
 void  ResolveControlFlowIdentities(Program& program);
 bool  EquivalentValue(const ResourcePlan& program, Value left, Value right);
 Value ResolveInvariantPhi(const ResourcePlan& program, Value value);
+Value ResolveActiveU32(Value value, Value active);
 
 } // namespace Libs::Graphics::ShaderRecompiler::IR
 

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -93,6 +93,7 @@ bool HasSideEffects(ValueOpcode opcode) {
 		case ValueOpcode::SetAttribute:
 		case ValueOpcode::SetTessellationAttribute:
 		case ValueOpcode::MeshAllocate:
+		case ValueOpcode::StoreCompletion:
 		case ValueOpcode::Barrier: return true;
 		default: return false;
 	}
@@ -177,7 +178,8 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::SharedAtomicAnd32:
 		case ValueOpcode::SharedAtomicOr32:
 		case ValueOpcode::SharedAtomicOr64:
-		case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic;
+		case ValueOpcode::SharedAtomicXor32:
+		case ValueOpcode::SharedAtomicMaskedOr32: return SharedAccess::Atomic;
 		case ValueOpcode::DataAppend: return SharedAccess::Append;
 		case ValueOpcode::DataConsume: return SharedAccess::Consume;
 		default: return SharedAccess::None;

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -8,6 +8,7 @@ VALUE_OPCODE(ConditionRef, U1, U1)
 
 // Translation and register-state pseudo operations.
 VALUE_OPCODE(GetUserData, U32, ScalarReg)
+VALUE_OPCODE(GetDispatchThreadExtent, U32, U32)
 VALUE_OPCODE(GetShaderBase, U64)
 VALUE_OPCODE(MeshDrawParameter, U32, U32)
 VALUE_OPCODE(MeshAllocate, Void, U32)
@@ -202,6 +203,7 @@ VALUE_OPCODE(FPFloor32, F32, F32)
 VALUE_OPCODE(FPCeil32, F32, F32)
 VALUE_OPCODE(FPTrunc32, F32, F32)
 VALUE_OPCODE(FPFract32, F32, F32)
+VALUE_OPCODE(FPFract64, F64, F64)
 
 VALUE_OPCODE(LaneId, U32)
 VALUE_OPCODE(Ballot, U32x4, U1)
@@ -209,6 +211,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
 VALUE_OPCODE(ReadLane, U32, U32, U32)
 VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
 VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1)
+VALUE_OPCODE(PermuteU32, U32, U32, U32, U1)
 VALUE_OPCODE(BpermuteU32, U32, U32, U32, U1)
 
 // Typed shader resources and memory operations. Descriptor state is represented by SSA values;
@@ -287,6 +290,7 @@ VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1)
 VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
+VALUE_OPCODE(SharedAtomicMaskedOr32, Void, U32, U32, U32, U1)
 VALUE_OPCODE(DataAppend, U32, U32, U1)
 VALUE_OPCODE(DataConsume, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
@@ -316,7 +320,7 @@ VALUE_OPCODE(GetInterpolationParameter, U32, U32, U32, U32)
 VALUE_OPCODE(SetAttribute, Void, U32x4, U1)
 
 VALUE_OPCODE(ControlNop, Void)
-VALUE_OPCODE(Waitcnt, Void)
+VALUE_OPCODE(StoreCompletion, Void)
 VALUE_OPCODE(Barrier, Void)
 VALUE_OPCODE(Sendmsg, Void)
 VALUE_OPCODE(TtraceData, Void)

src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp

@@ -15,11 +15,16 @@ namespace {
 	std::abort();
 }
 
-std::vector<uint32_t> CollectUserData(const Program& program) {
+void CollectShaderData(const Program& program, BindingLayout& layout) {
 	std::array<bool, NumScalarRegs> registers {};
+	bool uses_dispatch_threads = false;
 	for (const auto* block: program.blocks) {
 		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) {
+			if (!inst.HasUses()) {
+				continue;
+			}
+			uses_dispatch_threads |= inst.GetOpcode() == ValueOpcode::GetDispatchThreadExtent;
+			if (inst.GetOpcode() != ValueOpcode::GetUserData) {
 				continue;
 			}
 			if (inst.Arg(0).GetType() != Type::ScalarReg) {
@@ -32,13 +37,16 @@ std::vector<uint32_t> CollectUserData(const Program& program) {
 			registers[index] = true;
 		}
 	}
-	std::vector<uint32_t> result;
 	for (uint32_t index = 0; index < registers.size(); index++) {
 		if (registers[index]) {
-			result.push_back(index);
+			layout.user_data_registers.push_back(index);
 		}
 	}
-	return result;
+	layout.memory_offset_dword = static_cast<uint32_t>(layout.user_data_registers.size());
+	if (uses_dispatch_threads) {
+		layout.dispatch_thread_dword = layout.memory_offset_dword;
+		layout.memory_offset_dword += 3u;
+	}
 }
 
 void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
@@ -74,6 +82,9 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
 			} else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) {
 				EXIT_IF(memory.resource >= program.info.buffers.size());
 				live_buffers.at(memory.resource) = true;
+				for (const auto child: program.info.buffers[memory.resource].indirect_resources) {
+					live_buffers.at(child) = true;
+				}
 			}
 		}
 	}
@@ -86,13 +97,15 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
 }
 
 bool UsesFlattenedSrt(const Program& program) {
+	const auto uses_mapping = [](const auto& resource) {
+		return resource.indirect_search_iterations != 0u;
+	};
 	return std::ranges::any_of(program.blocks, [](const Block* block) {
 		return std::ranges::any_of(*block, [](const Inst& inst) {
 			return inst.GetOpcode() == ValueOpcode::ReadConst;
 		});
-	}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) {
-		return image.indirect_search_iterations != 0u;
-	});
+	}) || std::ranges::any_of(program.info.buffers, uses_mapping) ||
+	       std::ranges::any_of(program.info.images, uses_mapping);
 }
 
 void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
@@ -104,8 +117,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
 	BindingLayout next;
 	std::vector<uint32_t> buffers;
 	const bool            uses_gds = CollectMemoryResources(program, buffers);
-	next.user_data_registers = CollectUserData(program);
-	next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
+	CollectShaderData(program, next);
 	next.memory_offset_count       = static_cast<uint32_t>(buffers.size());
 	next.push_data_start_dword =
 	    PushData::StartFor(push_data_start_dword, next.ShaderDataDwords());

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -199,6 +199,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
                       std::unordered_set<Inst*>& lowered_ancillary) {
 	auto& inst = *instruction;
 	switch (inst.GetOpcode()) {
+		case ValueOpcode::StoreBufferU32: {
+			const auto data = ResolveActiveU32(Arg(inst, 4), Arg(inst, 5));
+			if (!data.IsEmpty() && data != inst.Arg(4)) inst.SetArg(4, data);
+			return;
+		}
 		case ValueOpcode::Phi: FoldPhi(inst); return;
 		case ValueOpcode::SelectU1:
 			if (!FoldSelect(inst) && IsImmediate(Arg(inst, 2), Type::U1) &&
@@ -503,6 +508,17 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			}
 			return;
 		}
+		case ValueOpcode::BitReverse32: {
+			const auto value = Arg(inst, 0);
+			if (IsImmediate(value, Type::U32)) {
+				uint32_t source = value.U32(), reversed = 0;
+				for (uint32_t bit = 0; bit < 32u; ++bit, source >>= 1u) {
+					reversed = (reversed << 1u) | (source & 1u);
+				}
+				Replace(inst, Value(reversed));
+			}
+			return;
+		}
 		case ValueOpcode::BitCount32: {
 			const auto value = Arg(inst, 0);
 			if (IsImmediate(value, Type::U32)) {

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -20,7 +20,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
 namespace {
 
 constexpr uint64_t AddressMask            = 0x0000ffffffffffffull;
-constexpr uint64_t MaxIndirectImageProbes = 65536u;
+constexpr uint64_t MaxIndirectDescriptorProbes = 65536u;
 
 bool SpecializationFail(std::string_view message) {
 	std::fprintf(stderr, "shader resource specialization failed: %.*s\n",
@@ -224,14 +224,44 @@ bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
 	       runtime.read_specialization_memory(runtime.userdata, address, prefix);
 }
 
-bool MaterializeIndirectImage(const ResourcePlan& program,
-                              const DescriptorSource::IndirectImage& indirect,
-                              uint32_t image_index,
-                              const SrtRuntime& runtime, SrtWalker& clean,
-                              ResourceSnapshot& snapshot,
-                              ResourceSpecialization& specialization) {
+bool NormalizeIndirectStoreBuffer(DescriptorValue& value) {
+	ShaderBufferResource descriptor;
+	if (!DecodeBufferDescriptor(value, descriptor)) return false;
+	if (descriptor.Type() != 0u || descriptor.Format() == Prospero::BufferFormat::kInvalid ||
+	    descriptor.Base48() == 0u || descriptor.GetSize() == 0u) {
+		value.dwords.fill(0u);
+	} else if (descriptor.Stride() != 1u || descriptor.SwizzleEnabled() || descriptor.AddTid() ||
+	           descriptor.OutOfBounds() != 3u) {
+		return SpecializationFail("indirect buffer stores require linear byte bounds");
+	}
+	return true;
+}
+
+template <typename Specialization, typename Normalize>
+bool MaterializeIndirectDescriptor(const ResourcePlan&                         program,
+                                   const DescriptorSource::IndirectDescriptor& indirect,
+                                   uint32_t resource_index, uint32_t dword_count,
+                                   const SrtRuntime& runtime, SrtWalker& clean,
+                                   ResourceSnapshot&             snapshot,
+                                   std::vector<DescriptorValue>& descriptors,
+                                   std::vector<Specialization>&  specializations,
+                                   uint32_t maximum_resources, Normalize&& normalize) {
+	const auto  descriptor_bytes = dword_count * sizeof(uint32_t);
 	const auto& sources = indirect.sources;
 	auto& keys = program.material_keys;
+	const auto read_keys = [&](const ShaderBufferResource& material, uint64_t first,
+	                           uint64_t step, uint64_t count) {
+		keys.resize(count);
+		if (step == 4u) {
+			return ReadScalarTable(material.Base48(), material.GetSize(), first, runtime, keys);
+		}
+		for (auto& key: keys) {
+			if (!ReadScalarTable(material.Base48(), material.GetSize(), first, runtime, {&key, 1}))
+				return false;
+			first += step;
+		}
+		return true;
+	};
 	uint64_t table_base = 0;
 	uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
 	if (sources.empty()) {
@@ -257,12 +287,31 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 			const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
 			if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
 			if (table_value.dword_count != 2u || !evaluated ||
-			    key_count > MaxIndirectImageProbes ||
-			    uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
+			    key_count > MaxIndirectDescriptorProbes ||
+			    uint64_t {indirect.table_offset} + uint64_t {key_count} * descriptor_bytes >
+			        UINT32_MAX + 1ull) {
 				return false;
 			}
 			keys.resize(key_count);
 			std::iota(keys.begin(), keys.end(), 0u);
+		} else if (!indirect.selector_first.IsEmpty()) {
+			ShaderBufferResource material;
+			uint32_t             first = 0, count = 0;
+			if (!DecodeBufferDescriptor(material_value, material) || material.Type() != 0u ||
+			    (indirect.selector_shift != 0u && (material.Base48() & 3u) != 0u) ||
+			    material.SwizzleEnabled() ||
+			    material.AddTid() || material.OutOfBounds() != 0u ||
+			    uint64_t {indirect.selector_offset} + 4u > material.Stride() ||
+			    !clean.Evaluate(indirect.selector_first, first) ||
+			    !clean.Evaluate(indirect.key_count, count) || count > MaxIndirectDescriptorProbes ||
+			    uint64_t {first} + count > material.NumRecords())
+				return false;
+			if (count != 0u && (uint64_t {first} + count - 1u) * material.Stride() +
+			                           indirect.selector_offset + 4u >
+			                       uint64_t {UINT32_MAX} + 1u)
+				return false;
+			if (!read_keys(material, uint64_t {first} * material.Stride() + indirect.selector_offset,
+			               material.Stride(), count)) return false;
 		} else if (!indirect.selector_mask.IsEmpty()) {
 			uint32_t mask = 0;
 			uint32_t count = 0;
@@ -286,12 +335,9 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 				keys.push_back(key);
 				mask &= mask - 1u;
 			}
-			std::ranges::sort(keys);
-			keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 		} else {
 			ShaderBufferResource material;
-			if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
-			    material.Stride() != indirect.selector_stride) {
+			if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u) {
 				return false;
 			}
 			// The first aligned offset includes the immediate added after shader U32 arithmetic.
@@ -301,70 +347,68 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 			const auto size = material.GetSize();
 			const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
 			const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
-			if (probe_count > MaxIndirectImageProbes) {
+			if (probe_count > MaxIndirectDescriptorProbes) {
 				return false;
 			}
-			keys.reserve(static_cast<size_t>(probe_count) + 1u);
-			keys.push_back(0u);
-			for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
-				uint32_t key = 0;
-				if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
-					return false;
-				}
-				keys.push_back(key);
+			if (!read_keys(material, first, step, probe_count)) return false;
+			if (indirect.selector_shift == 0u) keys.push_back(0u);
+		}
+		if (indirect.material_source != UINT32_MAX) {
+			if (indirect.selector_shift != 0u) {
+				for (auto& key: keys) key >>= indirect.selector_shift;
+				keys.push_back(0u); // An out-of-range material load returns zero.
 			}
 			std::ranges::sort(keys);
 			keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 		}
 	}
 
-	const auto children_begin = snapshot.images.size();
+	const auto children_begin = descriptors.size();
 	const auto mapping_offset = snapshot.flattened_srt.size();
-	const auto root_image = specialization.images[image_index];
+	const auto root_resource  = specializations[resource_index];
 	const auto key_count = sources.empty() ? keys.size() : sources.size();
 	snapshot.flattened_srt.resize(mapping_offset + 1u + key_count * 2u);
 	snapshot.flattened_srt[mapping_offset] = static_cast<uint32_t>(key_count);
 	for (uint32_t entry = 0; entry < key_count; ++entry) {
 		const auto key = sources.empty() ? keys[entry] : entry;
 		DescriptorValue candidate;
-		candidate.dword_count = 8u;
+		candidate.dword_count = dword_count;
 		if (sources.empty()) {
-			const auto table_offset = (key << 5u) + indirect.table_offset;
-			if (!ReadScalarTable(table_base, table_size, table_offset, runtime, candidate.dwords)) {
+			const auto table_offset =
+			    static_cast<uint32_t>(key * descriptor_bytes) + indirect.table_offset;
+			if (!ReadScalarTable(table_base, table_size, table_offset, runtime,
+			                     std::span(candidate.dwords).first(dword_count))) {
 				return false;
 			}
 		} else if (!clean.EvaluateDescriptor(sources[entry], candidate)) {
 			return false;
 		}
-		if (NullImageDescriptor(candidate) ||
-		    !ValidImageDescriptor(candidate, program.info.images[image_index].r128)) {
-			candidate.dwords.fill(0);
-		}
+		if (!normalize(candidate)) return false;
 		uint32_t ordinal = 0;
 		if (entry == 0) {
-			snapshot.images[image_index] = candidate;
-		} else if (snapshot.images[image_index] != candidate) {
-			const auto found = std::find(snapshot.images.begin() + children_begin,
-			                             snapshot.images.end(), candidate);
-			ordinal = static_cast<uint32_t>(found - snapshot.images.begin() - children_begin + 1u);
-			if (found == snapshot.images.end()) {
-				if (snapshot.images.size() >= ShaderInfo::MaxImages) {
+			descriptors[resource_index] = candidate;
+		} else if (descriptors[resource_index] != candidate) {
+			const auto found =
+			    std::find(descriptors.begin() + children_begin, descriptors.end(), candidate);
+			ordinal = static_cast<uint32_t>(found - descriptors.begin() - children_begin + 1u);
+			if (found == descriptors.end()) {
+				if (descriptors.size() >= maximum_resources) {
 					return false;
 				}
-				snapshot.images.push_back(candidate);
-				auto child = root_image;
-				child.indirect_root = image_index;
-				specialization.images.push_back(child);
+				descriptors.push_back(candidate);
+				auto child          = root_resource;
+				child.indirect_root = resource_index;
+				specializations.push_back(child);
 			}
 		}
 		snapshot.flattened_srt[mapping_offset + 1u + entry * 2u] = key;
 		snapshot.flattened_srt[mapping_offset + 2u + entry * 2u] = ordinal;
 	}
-	if (snapshot.images.size() == children_begin) {
+	if (descriptors.size() == children_begin) {
 		snapshot.flattened_srt.resize(mapping_offset);
 	} else {
-		auto& root = specialization.images[image_index];
-		root.indirect_root = image_index;
+		auto& root                      = specializations[resource_index];
+		root.indirect_root              = resource_index;
 		root.indirect_mapping_offset = static_cast<uint32_t>(mapping_offset);
 		root.indirect_search_iterations = std::bit_width(key_count);
 	}
@@ -383,10 +427,21 @@ struct SamplerPlan {
 	uint32_t                                                  sampler_count = 0;
 };
 
+template <typename T, typename Keep>
+void CompactImages(std::vector<T>& images, Keep&& keep) {
+	size_t count = 0;
+	for (size_t index = 0; index < images.size(); ++index) {
+		if (!keep(index)) continue;
+		if (count != index) images[count] = std::move(images[index]);
+		++count;
+	}
+	images.resize(count);
+}
+
 struct ImageRemap {
 	explicit ImageRemap(const ResourceSpecialization& specialization)
-	    : source_count(static_cast<uint32_t>(specialization.images.size())) {
-		EXIT_IF(specialization.images.size() > indices.size());
+	    : indices(specialization.images.size()),
+	      source_count(static_cast<uint32_t>(specialization.images.size())) {
 		for (uint32_t index = 0; index < source_count; index++) {
 			indices[index] = specialization.images[index].fmask ? UINT32_MAX : count++;
 		}
@@ -403,16 +458,11 @@ struct ImageRemap {
 		if (count == source_count) {
 			return;
 		}
-		for (uint32_t index = 0; index < source_count; index++) {
-			if (indices[index] != UINT32_MAX && indices[index] != index) {
-				images[indices[index]] = std::move(images[index]);
-			}
-		}
-		images.resize(count);
+		CompactImages(images, [&](size_t index) { return indices[index] != UINT32_MAX; });
 	}
 
 private:
-	std::array<uint32_t, ShaderInfo::MaxImages> indices;
+	std::vector<uint32_t> indices;
 	uint32_t                                    source_count;
 	uint32_t                                    count = 0;
 };
@@ -559,7 +609,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			}
 		}
 	}
-	ImageRemap(specialization).Apply(snapshot.images);
+	CompactImages(snapshot.images, [&](size_t index) { return !specialization.images[index].fmask; });
 	return true;
 }
 
@@ -605,6 +655,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 		}
 	}
 	std::vector<ResourceBlock> blocks(program.blocks.size());
+	std::vector<std::vector<uint32_t>> predecessors(blocks.size());
 	for (uint32_t i = 0; i < blocks.size(); i++) {
 		auto&                 block      = blocks[i];
 		const auto&           info       = program.block_info[i];
@@ -628,6 +679,7 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 				return {};
 			}
 			block.successors.push_back(found->second);
+			predecessors[found->second].push_back(i);
 		}
 		for (const auto& inst: *program.blocks[i]) {
 			const auto op     = inst.GetOpcode();
@@ -657,7 +709,30 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 		block.sources.erase(std::unique(block.sources.begin(), block.sources.end()),
 		                    block.sources.end());
 	}
-	for (auto& block: blocks) block.condition = predicate(block.condition);
+	// Host branches only matter when they guard a later resource access. Leave
+	// data-only branches (for example a mutable flag checked before a trap) on the GPU.
+	std::vector<uint8_t> resource_reachable(blocks.size());
+	std::vector<uint32_t> pending;
+	for (uint32_t i = 0; i < blocks.size(); ++i) {
+		if (!blocks[i].sources.empty() || !blocks[i].srt_reads.empty()) {
+			resource_reachable[i] = 1;
+			pending.push_back(i);
+		}
+	}
+	while (!pending.empty()) {
+		const auto index = pending.back();
+		pending.pop_back();
+		for (const auto predecessor: predecessors[index]) {
+			if (resource_reachable[predecessor]) continue;
+			resource_reachable[predecessor] = 1;
+			pending.push_back(predecessor);
+		}
+	}
+	for (auto& block: blocks) {
+		block.condition = std::ranges::any_of(block.successors, [&](uint32_t successor) {
+			return resource_reachable[successor] != 0;
+		}) ? predicate(block.condition) : Value {};
+	}
 	if (std::ranges::none_of(
 	        blocks, [](const ResourceBlock& block) { return !block.condition.IsEmpty(); })) {
 		return {};
@@ -665,11 +740,33 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 	return blocks;
 }
 
+// ResolveComputeBufferFill proves complete workgroups and matching requested extents, so every
+// dispatch-bound predicate accepted here is true for every invocation covered by the fill.
+static bool IsFullDispatchPredicate(Value value, uint32_t depth = 0) {
+	value = value.Resolve();
+	if (value == Value(true)) return true;
+	const auto* inst = value.TryInstruction();
+	if (inst == nullptr || depth > 32) return false;
+	if (inst->GetOpcode() == ValueOpcode::LogicalAnd) {
+		return IsFullDispatchPredicate(inst->Arg(0), depth + 1) &&
+		       IsFullDispatchPredicate(inst->Arg(1), depth + 1);
+	}
+	if (inst->GetOpcode() != ValueOpcode::ULessThan32) return false;
+	const auto* global = inst->Arg(0).Resolve().TryInstruction();
+	const auto* extent = inst->Arg(1).Resolve().TryInstruction();
+	if (global == nullptr || global->GetOpcode() != ValueOpcode::GetBuiltin ||
+	    global->Arg(0).Resolve() != Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)) ||
+	    extent == nullptr || extent->GetOpcode() != ValueOpcode::GetDispatchThreadExtent)
+		return false;
+	const auto axis = global->Arg(1).Resolve();
+	return axis.IsImmediate() && axis.U32() < 3 && extent->Arg(0).Resolve() == axis;
+}
+
 // Nonnegative affine coefficients for constant, local and workgroup coordinates. Reject modular
 // arithmetic that could wrap; runtime coverage also bounds the largest invocation index.
-static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t axis,
+static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t axis, Value guard,
                                                       uint32_t depth = 0) {
-	value = value.Resolve();
+	value = ResolveActiveU32(value, guard);
 	if (depth > 32 || value.GetType() != Type::U32) {
 		return {};
 	}
@@ -693,8 +790,8 @@ static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t ax
 	    op != ValueOpcode::ShiftLeftLogical32) {
 		return {};
 	}
-	auto left  = FillIndex(inst->Arg(0), axis, depth + 1);
-	auto right = FillIndex(inst->Arg(1), axis, depth + 1);
+	auto left  = FillIndex(inst->Arg(0), axis, guard, depth + 1);
+	auto right = FillIndex(inst->Arg(1), axis, guard, depth + 1);
 	if (!left || !right) {
 		return {};
 	}
@@ -750,6 +847,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 	UniformFillPlan result;
 	result.fill.resource = memory.resource;
 	Value data;
+	Value guard(true);
 	if (store->GetOpcode() == ValueOpcode::ImageWrite) {
 		if (program.info.images.size() != 1 || memory.dmask != 1 || memory.data_bits != 32 ||
 		    memory.image_has_mip || memory.image_sample_flags != 0 || memory.image_r128 ||
@@ -760,7 +858,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 		const auto* address = store->Arg(1).ResolveInstruction();
 		if (address == nullptr || address->GetOpcode() != ValueOpcode::MakeImageAddress) return {};
 		for (uint32_t axis = 0; axis < 3; ++axis) {
-			const auto index = FillIndex(address->Arg(axis), axis);
+			const auto index = FillIndex(address->Arg(axis), axis, guard);
 			if (!index || (*index)[0] != 0) return {};
 			if (axis < 2) {
 				if ((*index)[1] != 1 || (*index)[2] == 0) return {};
@@ -782,20 +880,22 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 		                             ValueOpcode::StoreBufferU32x3, ValueOpcode::StoreBufferU32x4};
 		const auto           store_op = std::ranges::find(stores, op);
 		if (store_op == stores.end() || store->Arg(2).Resolve() != Value(0u) ||
-		    store->Arg(3).Resolve() != Value(0u) || store->Arg(5).Resolve() != Value(true))
+		    store->Arg(3).Resolve() != Value(0u))
 			return {};
+		guard = store->Arg(5).Resolve();
+		if (!IsFullDispatchPredicate(guard)) return {};
 		if (!memory.formatted || memory.typed || !memory.idxen || memory.offen || memory.offset != 0 ||
 		    memory.data_bits != 32 ||
 		    memory.data_dwords != static_cast<uint32_t>(store_op - stores.begin() + 1))
 			return {};
-		const auto address = FillIndex(store->Arg(1), 0);
+		const auto address = FillIndex(store->Arg(1), 0, guard);
 		if (!address || (*address)[0] != 0 || (*address)[1] != 1 || (*address)[2] == 0) return {};
 		result.fill.kind = UniformFillKind::Buffer;
 		result.fill.group_stride[0] = static_cast<uint32_t>((*address)[2]);
 		result.fill.words = memory.data_dwords;
 		data = store->Arg(4);
 	}
-	data = data.Resolve();
+	data = ResolveActiveU32(data, guard);
 	const auto*          vector = data.TryInstruction();
 	constexpr std::array composites {ValueOpcode::CompositeConstructU32x2,
 	                                 ValueOpcode::CompositeConstructU32x3,
@@ -804,7 +904,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
 	    (vector == nullptr || vector->GetOpcode() != composites[result.fill.words - 2]))
 		return {};
 	for (uint32_t i = 0; i < result.fill.words; ++i) {
-		const auto word = result.fill.words == 1 ? data : vector->Arg(i);
+		const auto word = ResolveActiveU32(result.fill.words == 1 ? data : vector->Arg(i), guard);
 		if (word.GetType() != Type::U32 ||
 		    !ValidateRuntimeValue(program, word, RuntimeValueType::Integer))
 			return {};
@@ -831,6 +931,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		if (source == nullptr) {
 			return value;
 		}
+		if (source->GetOpcode() == ValueOpcode::ReadFirstLane &&
+		    ValidateRuntimeValue(program, source->Arg(0))) {
+			// Uniform values need no new EXEC context; preserve their shared evaluation memo.
+			return Clone(source->Arg(0));
+		}
 		if (source->GetOpcode() == ValueOpcode::Phi) {
 			const auto invariant = ResolveInvariantPhi(program, value);
 			if (!invariant.IsEmpty() && invariant != value) {
@@ -840,6 +945,10 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		if (const auto found = cloned.find(source); found != cloned.end()) {
 			return Value(found->second);
 		}
+		if (source->GetOpcode() == ValueOpcode::LoadBufferU32) {
+			plan.requires_specialization_memory = true;
+			plan.capture_specialization_reads   = true;
+		}
 		auto& target =
 		    plan.value_storage.emplace_back(source->GetOpcode(), source->Flags<uint64_t>());
 		cloned.emplace(source, &target);
@@ -859,10 +968,13 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	for (const auto& source: program.descriptor_sources) {
 		auto& target          = plan.descriptor_sources.emplace_back();
 		target.dword_count    = source.dword_count;
-		target.indirect_image = source.indirect_image;
-		if (target.indirect_image.has_value()) {
-			target.indirect_image->key_count = Clone(target.indirect_image->key_count);
-			target.indirect_image->selector_mask = Clone(target.indirect_image->selector_mask);
+		target.indirect_descriptor = source.indirect_descriptor;
+		if (target.indirect_descriptor.has_value()) {
+			target.indirect_descriptor->key_count = Clone(target.indirect_descriptor->key_count);
+			target.indirect_descriptor->selector_first =
+			    Clone(target.indirect_descriptor->selector_first);
+			target.indirect_descriptor->selector_mask =
+			    Clone(target.indirect_descriptor->selector_mask);
 		}
 		for (uint32_t dword = 0; dword < source.dword_count; dword++) {
 			target.dwords[dword] = Clone(source.dwords[dword]);
@@ -909,35 +1021,35 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
 	}
 	plan.clean_flat_slots.resize(plan.srt_reads.size());
-	bool capture_image_reads = false;
-	for (const auto& image: plan.info.images) {
-		const auto* source = Source(plan, image.source);
-		if (source == nullptr || !source->indirect_image.has_value()) {
-			continue;
-		}
+	bool capture_indirect_reads = false;
+	for (const auto& source: plan.descriptor_sources) {
+		if (!source.indirect_descriptor.has_value()) continue;
+		const auto& indirect                = *source.indirect_descriptor;
 		plan.requires_specialization_memory = true;
-		capture_image_reads |= !source->indirect_image->selector_mask.IsEmpty() ||
-		                       !source->indirect_image->sources.empty();
-		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
-		                   plan.clean_flat_slots, source->indirect_image->selector_mask);
-		if (source->indirect_image->sources.empty()) {
-			MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
-			                   plan.clean_flat_slots);
+		capture_indirect_reads |= indirect.material_source != UINT32_MAX || !indirect.sources.empty();
+		MarkCleanFlatSlots(plan, Source(plan, indirect.material_source), plan.clean_flat_slots,
+		                   indirect.selector_mask);
+		MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.selector_first);
+		MarkCleanFlatSlots(plan, nullptr, plan.clean_flat_slots, indirect.key_count);
+		if (indirect.sources.empty()) {
+			MarkCleanFlatSlots(plan, Source(plan, indirect.table_source), plan.clean_flat_slots);
 		} else {
-			for (const auto candidate: source->indirect_image->sources) {
+			for (const auto candidate: indirect.sources) {
 				MarkCleanFlatSlots(plan, Source(plan, candidate), plan.clean_flat_slots);
 			}
 		}
 	}
-	plan.capture_specialization_reads = capture_image_reads || !plan.control_flow.empty();
-	// Writable descriptor addresses must come from clean backing for the alias proof.
-	for (const auto& buffer: plan.info.buffers) {
-		if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
+	plan.capture_specialization_reads |= capture_indirect_reads || !plan.control_flow.empty();
+	if (plan.capture_specialization_reads) {
+		// Clean writable addresses prove that shader writes cannot overlap captured reads.
+		for (const auto& buffer: plan.info.buffers) {
+			if (buffer.written) MarkCleanFlatSlots(plan, Source(plan, buffer.source), plan.clean_flat_slots);
+		}
+		for (const auto& image: plan.info.images) {
+			if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
+		}
 	}
-	for (const auto& image: plan.info.images) {
-		if (image.written) MarkCleanFlatSlots(plan, Source(plan, image.source), plan.clean_flat_slots);
-	}
-	if (capture_image_reads) plan.resource_tracking_complete &= !program.has_address_writes;
+	if (capture_indirect_reads) plan.resource_tracking_complete &= !program.has_address_writes;
 	return plan;
 }
 
@@ -959,7 +1071,8 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		observed.read_memory = CaptureOrdinaryRead;
 	}
 	SrtWalker clean(program, CleanRuntime(observed));
-	SrtWalker walker(program, observed, program.clean_flat_slots, &clean);
+	SrtWalker walker(program, observed, program.clean_flat_slots,
+	                 capture_reads || program.requires_specialization_memory ? &clean : nullptr);
 	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
 		return false;
 	}
@@ -981,19 +1094,34 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			return false;
 		}
 		if (active.empty() || active[source]) {
-			return (written ? clean : walker).EvaluateDescriptor(source, value);
+			return (capture_reads && written ? clean : walker).EvaluateDescriptor(source, value);
 		}
 		value = {};
 		value.dword_count = program.descriptor_sources[source].dword_count;
 		return true;
 	};
 	snapshot.buffers.resize(program.info.buffers.size());
-	specialization.buffers.clear();
-	specialization.buffers.reserve(program.info.buffers.size());
-	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
-		if (!evaluate(program.info.buffers[i].source, snapshot.buffers[i],
-		              program.info.buffers[i].written)) {
-			return false;
+	specialization.buffers.assign(program.info.buffers.size(), {});
+	for (uint32_t i = 0; i < snapshot.buffers.size(); ++i) {
+		const auto base_index =
+		    i < program.info.buffers.size() ? i : specialization.buffers[i].indirect_root;
+		if (base_index >= program.info.buffers.size()) return false;
+		const auto& base = program.info.buffers[base_index];
+		if (i < program.info.buffers.size()) {
+			const auto* source = Source(program, base.source);
+			if (source == nullptr) return false;
+			if (source->indirect_descriptor.has_value()) {
+				snapshot.buffers[i] = {.dword_count = 4u};
+				if (active.empty() || active[base.source]) {
+					if (!MaterializeIndirectDescriptor(
+					        program, *source->indirect_descriptor, i, 4u, observed, clean, snapshot,
+					        snapshot.buffers, specialization.buffers, ShaderInfo::MaxBuffers,
+					        NormalizeIndirectStoreBuffer))
+						return false;
+				}
+			} else if (!evaluate(base.source, snapshot.buffers[i], base.written)) {
+				return false;
+			}
 		}
 		auto&                descriptor_value = snapshot.buffers[i];
 		ShaderBufferResource descriptor;
@@ -1012,15 +1140,12 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		} else if (!swizzle) {
 			packed_stride &= ~(3u << 16u);
 		}
-		specialization.buffers.push_back({
-		    .packed_stride     = packed_stride,
-		    .descriptor_format = program.info.buffers[i].formatted
-		                             ? descriptor.Format()
-		                             : Prospero::BufferFormat::kInvalid,
-		    .descriptor_swizzle =
-		        program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
-		    .zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u,
-		});
+		auto& buffer         = specialization.buffers[i];
+		buffer.packed_stride = packed_stride;
+		buffer.descriptor_format =
+		    base.formatted ? descriptor.Format() : Prospero::BufferFormat::kInvalid;
+		buffer.descriptor_swizzle = base.formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7);
+		buffer.zero_stride_oob    = descriptor.OutOfBounds() == 0u && stride == 0u;
 	}
 	snapshot.images.resize(program.info.images.size());
 	specialization.images.resize(program.info.images.size());
@@ -1041,13 +1166,19 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		if (source == nullptr) {
 			return false;
 		}
-		if (source->indirect_image.has_value()) {
+		if (source->indirect_descriptor.has_value()) {
 			snapshot.images[i] = {.dword_count = 8u};
 			if (!active.empty() && !active[image.source]) {
 				continue;
 			}
-			const auto& indirect = *source->indirect_image;
-			if (!MaterializeIndirectImage(program, indirect, i, observed, clean, snapshot, specialization)) {
+			const auto& indirect = *source->indirect_descriptor;
+			if (!MaterializeIndirectDescriptor(
+			        program, indirect, i, 8u, observed, clean, snapshot, snapshot.images,
+			        specialization.images, UINT32_MAX, [&](DescriptorValue& value) {
+				        if (NullImageDescriptor(value) || !ValidImageDescriptor(value, image.r128))
+					        value.dwords.fill(0u);
+				        return true;
+			        })) {
 				return false;
 			}
 		} else {
@@ -1082,14 +1213,33 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
 	EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete ||
 	        program.binding_layout_complete);
-	EXIT_IF(program.info.buffers.size() != specialization.buffers.size() ||
+	EXIT_IF(program.info.buffers.size() > specialization.buffers.size() ||
 	        program.info.images.size() > specialization.images.size());
 
 	auto& buffers = program.info.buffers;
-	for (size_t index = 0; index < buffers.size(); index++) {
-		buffers[index].packed_stride      = specialization.buffers[index].packed_stride;
-		buffers[index].descriptor_format  = specialization.buffers[index].descriptor_format;
-		buffers[index].descriptor_swizzle = specialization.buffers[index].descriptor_swizzle;
+	const auto original_buffer_count = buffers.size();
+	buffers.reserve(specialization.buffers.size());
+	for (uint32_t index = 0; index < specialization.buffers.size(); ++index) {
+		const auto& source = specialization.buffers[index];
+		if (index >= buffers.size()) {
+			EXIT_IF(source.indirect_root >= original_buffer_count);
+			buffers.push_back(buffers[source.indirect_root]);
+		}
+		auto& buffer                      = buffers[index];
+		buffer.packed_stride              = source.packed_stride;
+		buffer.descriptor_format          = source.descriptor_format;
+		buffer.descriptor_swizzle         = source.descriptor_swizzle;
+		buffer.indirect_root              = source.indirect_root;
+		buffer.indirect_mapping_offset    = source.indirect_mapping_offset;
+		buffer.indirect_search_iterations = source.indirect_search_iterations;
+		buffer.indirect_resources.clear();
+	}
+	for (uint32_t index = 0; index < buffers.size(); ++index) {
+		const auto root = buffers[index].indirect_root;
+		if (root != BufferResource::NoIndirectBuffer) {
+			EXIT_IF(root >= buffers.size());
+			buffers[root].indirect_resources.push_back(index);
+		}
 	}
 	auto& images = program.info.images;
 	const auto original_image_count = images.size();

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h

@@ -13,6 +13,9 @@ struct ResourceSpecialization {
 		Prospero::BufferFormat descriptor_format               = Prospero::BufferFormat::kInvalid;
 		uint32_t               descriptor_swizzle              = DstSel(4, 5, 6, 7);
 		bool                   zero_stride_oob                 = false;
+		uint32_t               indirect_root                   = BufferResource::NoIndirectBuffer;
+		uint32_t               indirect_mapping_offset         = 0;
+		uint32_t               indirect_search_iterations      = 0;
 		bool                   operator==(const Buffer&) const = default;
 	};
 

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -130,6 +130,58 @@ public:
 		return true;
 	}
 
+	bool SelectsRange(const Inst& selected, Value active, const Inst& candidate,
+	                  const Inst& carried, const Inst& mask,
+	                  std::span<const std::pair<Value, bool>> exit_conditions) {
+		// Inactive lanes retain a previously selected index. Initially every lane is
+		// active; the bounded assignment establishes the invariant on each backedge.
+		m_exclude_bound = false;
+		std::vector<uint32_t> range_values;
+		const auto in_range = [&] {
+			const auto value = Unknown(Type::U32);
+			range_values.push_back(value);
+			return value;
+		};
+		const auto contains = [&](const auto& self, uint32_t node) -> uint32_t {
+			if (std::ranges::find(range_values, node) != range_values.end()) return 1u;
+			const auto entry = m_nodes[node];
+			if (entry.variable == UINT32_MAX) return 0u;
+			return Select(NodeFor(entry.variable, 0u, 1u), self(self, entry.high),
+			              self(self, entry.low));
+		};
+		const auto state = [&](bool in_body) {
+			m_values[1].clear();
+			const auto enabled = Unknown(Type::U1);
+			for (const auto& phi: *m_induction.Parent()) {
+				if (phi.GetOpcode() != ValueOpcode::Phi) continue;
+				m_values[1][&phi] = EquivalentValue(m_program, Value(const_cast<Inst*>(&phi)),
+				                                      Value(const_cast<Inst*>(&mask)))
+				                         ? enabled : Unknown(phi.GetType());
+			}
+			m_values[1][&carried] = Select(enabled, Unknown(Type::U32), in_range());
+			if (in_body) m_values[1][&m_bound] = 1u;
+			m_values[1][&candidate] = Select(Evaluate(Value(const_cast<Inst*>(&m_bound)), true),
+			                                  in_range(), Unknown(Type::U32));
+		};
+		for (uint32_t arm = 0; arm < carried.NumArgs(); ++arm) {
+			const auto initial = mask.Arg(arm).Resolve();
+			if (initial.IsImmediate() && initial.GetType() == Type::U1 && initial.U1()) continue;
+			state(true);
+			const auto retained = contains(contains, Evaluate(carried.Arg(arm), true));
+			if (Select(Evaluate(mask.Arg(arm), true), 1u, retained) != 1u) return false;
+		}
+		for (uint32_t arm = 0; arm < selected.NumArgs(); ++arm) {
+			state(false);
+			const auto value = Evaluate(selected.Arg(arm), true);
+			m_values[1][&selected] = value;
+			auto allowed = Evaluate(exit_conditions[arm].first, true);
+			if (!exit_conditions[arm].second) allowed = Select(allowed, 0u, 1u);
+			allowed = Select(allowed, Evaluate(active, true), 0u);
+			if (Select(allowed, contains(contains, value), 1u) != 1u) return false;
+		}
+		return true;
+	}
+
 private:
 	struct Node {
 		uint32_t variable = UINT32_MAX;
@@ -171,6 +223,10 @@ private:
 	}
 
 	uint32_t Compare(const Inst& inst, uint32_t left, uint32_t right) {
+		if (left == right) {
+			if (inst.GetOpcode() == ValueOpcode::IEqual32) return 1u;
+			if (inst.GetOpcode() == ValueOpcode::INotEqual32) return 0u;
+		}
 		const auto key = std::tuple {inst.GetOpcode(), inst.Flags<uint64_t>(), left, right};
 		if (const auto found = m_predicates.find(key); found != m_predicates.end()) return found->second;
 		const auto variable = std::min(m_nodes[left].variable, m_nodes[right].variable);
@@ -202,7 +258,7 @@ private:
 			return node;
 		}
 		const auto* inst = value.TryInstruction();
-		if (current && inst == &m_bound) return 0u;
+		if (current && m_exclude_bound && inst == &m_bound) return 0u;
 		auto& values = m_values[current];
 		if (const auto found = values.find(inst); found != values.end()) {
 			if (found->second == UINT32_MAX) found->second = Unknown(value.GetType());
@@ -268,6 +324,7 @@ private:
 	const Inst& m_bound;
 	const Block* m_incoming = nullptr;
 	uint32_t m_variables = 0;
+	bool m_exclude_bound = true;
 	std::vector<Node> m_nodes {{}, {}};
 	std::vector<std::pair<Value, uint32_t>> m_literals;
 	std::array<std::map<const Inst*, uint32_t>, 2> m_values;
@@ -294,9 +351,10 @@ public:
 		if (m_program.resource_tracking_complete) {
 			Fail(0, "resources already tracked");
 		}
+		FoldBoundedLoopSelectors();
 		PlanScalarReads();
 		EliminateDeadCode(m_program.blocks);
-		PlanIndirectImages();
+		PlanIndirectDescriptors();
 		for (auto* block: m_program.blocks) {
 			for (auto& inst: *block) {
 				Collect(inst);
@@ -313,17 +371,36 @@ public:
 				memory.sampler = patch.sampler;
 			}
 		}
-		for (const auto& plan: m_indirect_images) {
-			plan.handle->SetArg(0, plan.key);
-			for (uint32_t dword = 0; dword < 4u; dword++) {
-				plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]);
-			}
-			for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
-				plan.handle->SetArg(dword, plan.key);
+		for (const auto& plan: m_indirect_descriptors) {
+			if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) {
+				plan.handle->SetArg(0, plan.key);
+				for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u));
+			} else {
+				if (plan.reads[0] == nullptr) plan.handle->SetArg(0, plan.key);
+				for (uint32_t dword = 0; dword < 4u; dword++) {
+					plan.handle->SetArg(dword + 1u, plan.roots[dword + 4u]);
+				}
+				for (uint32_t dword = 5u; dword < plan.roots.size(); dword++) {
+					plan.handle->SetArg(dword, plan.key);
+				}
 			}
 			if (plan.reads[0] != nullptr) {
-				for (const auto index: plan.memory)
-					m_program.memory_info[index].planning_only = true;
+				for (uint32_t word = 0; word < plan.handle->NumArgs(); ++word)
+					m_program.memory_info[plan.memory[word]].planning_only = true;
+			}
+		}
+		// Project descriptor-only SSA onto its key. Existing Phis preserve dominance;
+		// native descriptor resolution excludes their stale incoming values at each use.
+		for (const auto& plan: m_indirect_descriptors) {
+			if (plan.handle->GetOpcode() != ValueOpcode::GetImageResource || plan.reads[0] == nullptr)
+				continue;
+			for (const auto* read: plan.reads) {
+				const auto first = std::ranges::find_if(m_indirect_descriptors, [&](const auto& candidate) {
+					return candidate.reads[0] == read;
+				});
+				const auto key = first != m_indirect_descriptors.end() ? first->key : Value(0u);
+				const auto uses = read->Uses();
+				for (const auto& use: uses) use.user->SetArg(use.operand, key);
 			}
 		}
 		m_program.descriptor_sources         = std::move(m_sources);
@@ -351,7 +428,7 @@ private:
 		Value planning_handle;
 	};
 
-	struct IndirectImagePlan {
+	struct IndirectDescriptorPlan {
 		Inst*                      handle = nullptr;
 		uint32_t                   source = 0;
 		Value                      key;
@@ -663,24 +740,17 @@ private:
 				}
 			}
 		}
-		for (auto* block: m_program.blocks) {
-			for (auto& inst: *block) {
-				uint32_t index = 0;
-				if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 &&
-				    ScalarReadMemory(inst, index) != nullptr && inst.Arg(1).Resolve().IsImmediate() &&
-				    ValidateRuntimeValue(m_program, Value(&inst)))
-					CollectScalarRead(Value(&inst), inst.Flags<MemoryFlags>().pc);
-			}
-		}
 		for (auto* read: m_scalar_reads) {
 			const auto flags = read->Flags<MemoryFlags>();
 			auto& memory = m_program.memory_info[flags.index];
-			memory.planning_only = true;
 			const auto* handle = read->Arg(0).Resolve().TryInstruction();
 			auto resolved = std::ranges::find_if(m_resolved_handles, [&](const auto& entry) {
 				return entry.handle == handle && entry.pc == flags.pc;
 			});
 			EXIT_IF(resolved == m_resolved_handles.end());
+			uint32_t bad_dword = 0;
+			if (!ValidateSource(resolved->source, bad_dword)) continue;
+			memory.planning_only = true;
 			uint32_t slot = 0;
 			for (; slot < m_program.srt_reads.size(); ++slot) {
 				const auto* other = m_program.srt_reads[slot].value.Resolve().TryInstruction();
@@ -755,16 +825,20 @@ private:
 		for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
 			const auto& current = m_sources[candidate];
 			if (current.dword_count != descriptor.dword_count ||
-			    current.indirect_image.has_value() != descriptor.indirect_image.has_value()) {
+			    current.indirect_descriptor.has_value() != descriptor.indirect_descriptor.has_value()) {
 				continue;
 			}
-			if (current.indirect_image.has_value()) {
-				const auto& a = *current.indirect_image;
-				const auto& b = *descriptor.indirect_image;
+			if (current.indirect_descriptor.has_value()) {
+				const auto& a = *current.indirect_descriptor;
+				const auto& b = *descriptor.indirect_descriptor;
 				if (a.material_source != b.material_source || a.table_source != b.table_source ||
 				    a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset ||
+				    a.selector_shift != b.selector_shift ||
 				    a.table_offset != b.table_offset || a.sources != b.sources ||
 				    !EquivalentValue(m_program, a.key_count, b.key_count) ||
+				    a.selector_first.IsEmpty() != b.selector_first.IsEmpty() ||
+				    (!a.selector_first.IsEmpty() &&
+				     !EquivalentValue(m_program, a.selector_first, b.selector_first)) ||
 				    a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() ||
 				    (!a.selector_mask.IsEmpty() &&
 				     !EquivalentValue(m_program, a.selector_mask, b.selector_mask))) continue;
@@ -790,10 +864,17 @@ private:
 		return true;
 	}
 
-	static bool UsesOnly(const Inst& value, std::span<const Inst* const> users) {
-		return !value.Uses().empty() && std::ranges::all_of(value.Uses(), [&](const Use& use) {
-			return std::ranges::find(users, use.user) != users.end();
-		});
+	static bool UsesOnlyImageDescriptors(const Inst& value) {
+		if (value.Uses().empty()) return false;
+		std::vector<const Inst*> values {&value};
+		for (size_t index = 0; index < values.size(); ++index) {
+			for (const auto& use: values[index]->Uses()) {
+				if (use.user->GetOpcode() == ValueOpcode::GetImageResource) continue;
+				if (use.user->GetOpcode() != ValueOpcode::Phi) return false;
+				if (std::ranges::find(values, use.user) == values.end()) values.push_back(use.user);
+			}
+		}
+		return true;
 	}
 
 	const MemoryInfo* ScalarReadMemory(const Inst& read, uint32_t& index) const {
@@ -856,38 +937,39 @@ private:
 		return ValidateSource(descriptor, bad_dword);
 	}
 
-	bool MatchMaterialOffset(Value value, Value& selector, uint32_t& stride,
-	                         uint32_t& offset) const {
-		value           = value.Resolve();
-		offset          = 0;
-		auto* candidate = value.TryInstruction();
-		if (candidate != nullptr && candidate->GetOpcode() == ValueOpcode::IAdd32 &&
-		    candidate->NumArgs() == 2u) {
-			uint32_t immediate = 0;
-			if (ImmediateU32(candidate->Arg(0), immediate)) {
-				value = candidate->Arg(1).Resolve();
-			} else if (ImmediateU32(candidate->Arg(1), immediate)) {
-				value = candidate->Arg(0).Resolve();
+	bool MatchMaterialOffset(Value value, uint32_t& stride, uint32_t& offset) const {
+		stride = 1u;
+		offset = 0u;
+		value = value.Resolve();
+		while (const auto* inst = value.TryInstruction()) {
+			if (inst->NumArgs() != 2u) break;
+			uint32_t constant = 0;
+			Value remaining;
+			if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
+			    ImmediateU32(inst->Arg(1), constant) && constant < 32u) {
+				stride *= 1u << constant;
+				remaining = inst->Arg(0);
+			} else if (inst->GetOpcode() == ValueOpcode::IAdd32 ||
+			           inst->GetOpcode() == ValueOpcode::IMul32) {
+				if (ImmediateU32(inst->Arg(0), constant)) remaining = inst->Arg(1);
+				else if (ImmediateU32(inst->Arg(1), constant)) remaining = inst->Arg(0);
+				else break;
+				if (inst->GetOpcode() == ValueOpcode::IAdd32) offset += stride * constant;
+				else stride *= constant;
 			} else {
-				return false;
+				break;
 			}
-			offset = immediate;
+			value = remaining.Resolve();
 		}
-		const auto* multiply = value.TryInstruction();
-		if (multiply == nullptr || multiply->GetOpcode() != ValueOpcode::IMul32 ||
-		    multiply->NumArgs() != 2u) {
-			return false;
-		}
-		if (ImmediateU32(multiply->Arg(0), stride)) {
-			selector = multiply->Arg(1).Resolve();
-		} else if (ImmediateU32(multiply->Arg(1), stride)) {
-			selector = multiply->Arg(0).Resolve();
-		} else {
-			return false;
-		}
-		const auto* selector_inst = selector.TryInstruction();
-		return stride != 0u && selector_inst != nullptr &&
-		       selector_inst->GetOpcode() == ValueOpcode::ReadFirstLane;
+		// Treat the remaining GPU-derived scalar as any U32 value.
+		return stride != 0u && value.GetType() == Type::U32 && !value.IsImmediate();
+	}
+
+	const Inst* MaterialKeyShift(Value key, uint32_t& amount) const {
+		const auto* shift = key.Resolve().TryInstruction();
+		return shift != nullptr && shift->GetOpcode() == ValueOpcode::ShiftRightLogical32 &&
+		               ImmediateU32(shift->Arg(1), amount) && amount != 0u && amount < 32u
+		           ? shift : nullptr;
 	}
 
 	enum class LaneQuantifier { Any, All };
@@ -928,7 +1010,7 @@ private:
 		return false;
 	}
 
-	bool MatchTableOffset(Value value, Value& key, uint32_t& offset) const {
+	bool MatchTableOffset(Value value, Value& key, uint32_t& offset, uint32_t shift = 5u) const {
 		offset = 0;
 		for (;;) {
 			const auto* inst = value.Resolve().TryInstruction();
@@ -937,7 +1019,7 @@ private:
 			}
 			uint32_t immediate;
 			if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
-			    ImmediateU32(inst->Arg(1), immediate) && immediate == 5u) {
+			    ImmediateU32(inst->Arg(1), immediate) && immediate == shift) {
 				key = inst->Arg(0).Resolve();
 				return key.GetType() == Type::U32;
 			}
@@ -996,19 +1078,35 @@ private:
 		return edge && edge->positive ? edge->condition : Value {};
 	}
 
+	bool GuardedOnEntry(const Block* block, const auto& stops, const auto& accepts) const {
+		std::vector<const Block*> pending {block};
+		for (size_t i = 0; i < pending.size(); ++i) {
+			const auto* current = pending[i];
+			if (stops(current) || current->ImmPredecessors().empty()) return false;
+			for (const auto* previous: current->ImmPredecessors()) {
+				const auto edge = ConditionalEdge(previous, current);
+				if (edge && accepts(*edge)) continue;
+				if (std::ranges::find(pending, previous) == pending.end())
+					pending.push_back(previous);
+			}
+		}
+		return true;
+	}
+
 	bool HasActiveLane(Value mask, const Block* use) const {
 		mask = mask.Resolve();
 		const auto incoming_is_nonempty = [&](const Block* from, const Block* to,
 		                                     Value incoming, const Block* header) {
-			for (size_t depth = 0; depth < m_program.blocks.size(); ++depth) {
-				const auto edge = ConditionalEdge(from, to);
-				if (edge && edge->positive && Implies(edge->condition, incoming)) return true;
-				if (from == header || from->ImmSuccessors().size() != 1u ||
-				    from->ImmPredecessors().size() != 1u) return false;
-				to = from;
-				from = from->ImmPredecessors()[0];
-			}
-			return false;
+			const auto accepts = [&](const EdgePredicate& edge) {
+				return edge.positive && Implies(edge.condition, incoming);
+			};
+			const auto edge = ConditionalEdge(from, to);
+			if (edge && accepts(*edge)) return true;
+			const auto* definition = incoming.Resolve().TryInstruction();
+			return GuardedOnEntry(from, [&](const Block* block) {
+				// A witness before this definition may belong to an earlier loop iteration.
+				return block == header || (definition != nullptr && block == definition->Parent());
+			}, accepts);
 		};
 		const auto* phi = mask.TryInstruction();
 		if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi &&
@@ -1241,9 +1339,7 @@ private:
 		return {};
 	}
 
-	bool MatchUniformizedMaterialKey(Value key, const Inst& image,
-	                                DescriptorSource::IndirectImage& indirect,
-	                                DescriptorSource& material_source) {
+	const Inst* UniformizedMaterialValue(Value key, const Inst& use, Value executing = {}) const {
 		Value guard;
 		Value local;
 		const auto* first = key.Resolve().TryInstruction();
@@ -1251,7 +1347,9 @@ private:
 		    first->NumArgs() == 2u) {
 			guard = first->Arg(1).Resolve();
 			// Empty EXEC selects lane zero, which may not have loaded a material key.
-			if (!HasActiveLane(guard, first->Parent())) return false;
+			// An executing store also witnesses nonempty EXEC at the lane read.
+			if (executing.IsEmpty() ? !HasActiveLane(guard, first->Parent())
+			                        : !Implies(executing, guard)) return nullptr;
 			local = first->Arg(0).Resolve();
 			const auto* phi = guard.TryInstruction();
 			if (phi != nullptr && phi->GetOpcode() == ValueOpcode::Phi &&
@@ -1265,13 +1363,21 @@ private:
 				}
 			}
 		} else {
-			guard = PositiveLaneWitness(image.Parent());
-			if (guard.IsEmpty()) return false;
+			guard = PositiveLaneWitness(use.Parent());
+			if (guard.IsEmpty()) return nullptr;
 			local = EqualLocalKey(guard, key);
 		}
 		const auto* selected = local.Resolve().TryInstruction();
 		if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 ||
-		    selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return false;
+		    selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return nullptr;
+		return selected;
+	}
+
+	bool MatchUniformizedMaterialKey(Value key, const Inst& image,
+	                                DescriptorSource::IndirectDescriptor& indirect,
+	                                DescriptorSource& material_source) {
+		const auto* selected = UniformizedMaterialValue(key, image);
+		if (selected == nullptr) return false;
 		const auto active = selected->Arg(0).Resolve();
 		const auto* read = selected->Arg(1).Resolve().TryInstruction();
 		if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadAddressU32 ||
@@ -1303,12 +1409,13 @@ private:
 		return true;
 	}
 
-	Value BoundedLoopCount(Value key, const Block* use) const {
+	const Inst* BoundedLoop(Value key, const Block* use, const auto& accepts_bound) const {
 		const auto* phi = key.Resolve().TryInstruction();
-		if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
-		    phi->GetType() != Type::U32 || phi->NumArgs() != 2u ||
+		if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
+		    phi->GetType() != Type::U32 || phi->NumArgs() != 2u || phi->NumPhiBlocks() != 2u ||
 		    m_program.blocks.size() != m_program.block_info.size()) return {};
 		const Block* increment_block = nullptr;
+		uint32_t initial_arm = 0;
 		for (uint32_t initial = 0; initial < 2u; ++initial) {
 			const auto zero = phi->Arg(initial).Resolve();
 			const auto* step = phi->Arg(initial ^ 1u).Resolve().TryInstruction();
@@ -1321,51 +1428,145 @@ private:
 			    (step->Arg(1).Resolve() == key &&
 			     ImmediateU32(step->Arg(0), increment) && increment == 1u)) {
 				increment_block = step->Parent();
+				initial_arm = initial;
 				break;
 			}
 		}
 		if (increment_block == nullptr) return {};
 
-		const auto guarded_on_entry = [&](const Block* block, const auto& accepts) {
-			std::vector<const Block*> pending {block};
-			for (size_t i = 0; i < pending.size(); ++i) {
-				const auto* current = pending[i];
-				if (current == phi->Parent() || current->ImmPredecessors().empty()) return false;
-				for (const auto* previous: current->ImmPredecessors()) {
-					const auto edge = ConditionalEdge(previous, current);
-					if (edge && accepts(*edge)) continue;
-					if (std::ranges::find(pending, previous) == pending.end())
-						pending.push_back(previous);
-				}
-			}
-			return true;
-		};
+		const auto stops = [&](const Block* block) { return block == phi->Parent(); };
 		for (const auto& use_of_key: phi->Uses()) {
 			const auto* compare = use_of_key.user;
-			if (compare->GetOpcode() != ValueOpcode::SLessThan32 || use_of_key.operand != 0u ||
-			    !ValidateRuntimeValue(m_program, compare->Arg(1))) continue;
-			if (!guarded_on_entry(use, [&](const EdgePredicate& edge) {
+			if ((compare->GetOpcode() != ValueOpcode::SLessThan32 &&
+			     compare->GetOpcode() != ValueOpcode::ULessThan32) || use_of_key.operand != 0u ||
+			    !accepts_bound(*compare, initial_arm)) continue;
+			if (!GuardedOnEntry(use != nullptr ? use : increment_block, stops, [&](const EdgePredicate& edge) {
 				return edge.positive && Implies(edge.condition, Value(use_of_key.user));
 			})) continue;
 			LoopBoundProof proof(m_program, *phi, *compare);
 			// The image bound and the increment guard are separate obligations: a
 			// skipped image alone does not prevent signed induction wraparound.
-			if (guarded_on_entry(increment_block, [&](const EdgePredicate& edge) {
+			if (GuardedOnEntry(increment_block, stops, [&](const EdgePredicate& edge) {
 				return proof.Excludes(edge.condition, edge.positive);
-			})) return compare->Arg(1);
+			})) return compare;
 		}
 		return {};
 	}
 
-	bool TryMakeIndirectImage(Inst& handle, IndirectImagePlan& plan) {
-		if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) {
-			return false;
+	const Inst* BoundedLoop(Value key, const Block* use) const {
+		return BoundedLoop(key, use, [&](const Inst& compare, uint32_t) {
+			return ValidateRuntimeValue(m_program, compare.Arg(1), RuntimeValueType::Integer);
+		});
+	}
+
+	void FoldBoundedLoopSelectors() {
+		for (auto* header: m_program.blocks) {
+			for (auto& induction: *header) {
+				const Inst* carried = nullptr;
+				uint32_t maximum = 0;
+				const auto* compare = BoundedLoop(Value(&induction), nullptr,
+				    [&](const Inst& test, uint32_t initial) {
+					if (test.GetOpcode() != ValueOpcode::ULessThan32) return false;
+					carried = test.Arg(1).Resolve().TryInstruction();
+					if (carried == nullptr || carried->GetOpcode() != ValueOpcode::Phi ||
+					    carried->GetType() != Type::U32 || carried->Parent() != header ||
+					    carried->NumArgs() != 2u || carried->NumPhiBlocks() != 2u ||
+					    carried->PhiBlock(0) != induction.PhiBlock(0) ||
+					    carried->PhiBlock(1) != induction.PhiBlock(1)) return false;
+					maximum = SelectorMaximum(carried->Arg(initial));
+					if (maximum == 0u || maximum == UINT32_MAX) return false;
+					const std::array assumptions {std::pair {static_cast<const Inst*>(&induction), maximum - 1u},
+					                             std::pair {carried, maximum}};
+					// Close the carried-bound invariant before using either assumption.
+					return SelectorMaximum(carried->Arg(initial ^ 1u), assumptions) <= maximum;
+				});
+				if (compare == nullptr) continue;
+				const std::array assumptions {std::pair {static_cast<const Inst*>(&induction), maximum - 1u},
+				                             std::pair {carried, maximum}};
+				// Native scalar branches advance the zero/+1 induction uniformly, so
+				// even an any-lane bound test limits it by the largest carried bound.
+				// Per-lane edges instead require the comparison on every executing lane.
+				for (auto* block: m_program.blocks) {
+					if (!GuardedOnEntry(block, [&](const Block* at) { return at == header; },
+					    [&](const EdgePredicate& edge) {
+						return edge.positive && Implies(edge.condition, Value(const_cast<Inst*>(compare)));
+					})) continue;
+					for (auto& inst: *block) {
+						if (inst.GetOpcode() == ValueOpcode::SelectU32 &&
+						    SelectorPredicateFalse(inst.Arg(0), assumptions))
+							inst.ReplaceUsesWith(inst.Arg(2));
+					}
+				}
+			}
 		}
+	}
+
+	bool BoundedSelectedIndex(Value index, Value active,
+	                          DescriptorSource::IndirectDescriptor& indirect) const {
+		const auto* selected = index.Resolve().TryInstruction();
+		if (selected == nullptr || selected->GetOpcode() != ValueOpcode::Phi ||
+		    selected->NumArgs() == 0u) return false;
+		std::vector<const Inst*> values {selected};
+		for (size_t i = 0; i < values.size(); ++i) {
+			const auto* value = values[i];
+			if (value->GetOpcode() != ValueOpcode::Phi &&
+			    value->GetOpcode() != ValueOpcode::SelectU32) continue;
+			for (size_t arm = value->GetOpcode() == ValueOpcode::SelectU32 ? 1u : 0u;
+			     arm < value->NumArgs(); ++arm) {
+				const auto* next = value->Arg(arm).Resolve().TryInstruction();
+				if (next != nullptr && std::ranges::find(values, next) == values.end())
+					values.push_back(next);
+			}
+		}
+		for (const auto* candidate: values) {
+			if (candidate->GetOpcode() != ValueOpcode::IAdd32) continue;
+			for (uint32_t arg = 0; arg < 2u; ++arg) {
+				const auto* bound = BoundedLoop(candidate->Arg(arg), candidate->Parent());
+				const auto first = candidate->Arg(arg ^ 1u);
+				if (bound == nullptr || !ValidateRuntimeValue(m_program, first, RuntimeValueType::Integer))
+					continue;
+				const auto* induction = candidate->Arg(arg).ResolveInstruction();
+				const Inst* carried = nullptr;
+				for (const auto* value: values) {
+					if (value->GetOpcode() == ValueOpcode::Phi && value->Parent() == induction->Parent()) {
+						if (carried != nullptr) { carried = nullptr; break; }
+						carried = value;
+					}
+				}
+				if (carried == nullptr || carried->NumArgs() != 2u ||
+				    carried->PhiBlock(0) != induction->PhiBlock(0) ||
+				    carried->PhiBlock(1) != induction->PhiBlock(1)) continue;
+				std::vector<std::pair<Value, bool>> exits;
+				for (size_t arm = 0; arm < selected->NumArgs(); ++arm) {
+					const auto edge = ConditionalEdge(selected->PhiBlock(arm), selected->Parent());
+					if (!edge || edge->lanes != LaneQuantifier::All) break;
+					exits.emplace_back(edge->condition, edge->positive);
+				}
+				if (exits.size() != selected->NumArgs()) continue;
+				for (const auto& mask: *induction->Parent()) {
+					if (mask.GetOpcode() != ValueOpcode::Phi || mask.GetType() != Type::U1 ||
+					    mask.NumArgs() != 2u || mask.PhiBlock(0) != carried->PhiBlock(0) ||
+					    mask.PhiBlock(1) != carried->PhiBlock(1)) continue;
+					const auto initial = induction->Arg(0).Resolve().IsImmediate() ? 0u : 1u;
+					const auto enabled = mask.Arg(initial).Resolve();
+					if (!enabled.IsImmediate() || enabled.GetType() != Type::U1 || !enabled.U1()) continue;
+					LoopBoundProof proof(m_program, *induction, *bound);
+					if (!proof.SelectsRange(*selected, active, *candidate, *carried, mask, exits)) continue;
+					indirect.selector_first = first;
+					indirect.key_count = bound->Arg(1);
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+	bool MatchDescriptorTable(Inst& handle, const DescriptorSource& descriptor,
+	                          IndirectDescriptorPlan& plan,
+	                          DescriptorSource& table_source, Value& key, uint32_t& table_offset) {
 		Inst* table_handle = nullptr;
-		Value key;
-		uint32_t table_offset = 0;
-		for (uint32_t dword = 0; dword < plan.reads.size(); ++dword) {
-			auto* read = handle.Arg(dword).Resolve().TryInstruction();
+		for (uint32_t dword = 0; dword < handle.NumArgs(); ++dword) {
+			auto* read = descriptor.dwords[dword].Resolve().TryInstruction();
 			if (read == nullptr) {
 				return false;
 			}
@@ -1385,7 +1586,7 @@ private:
 			    (memory->kind == ResourceKind::ScalarAddress && read->Parent() != handle.Parent()) ||
 			    (table_handle != nullptr &&
 			     !EquivalentValue(m_program, Value(table_handle), Value(current_handle))) ||
-			    !MatchTableOffset(read->Arg(1), current_key, offset) ||
+			    !MatchTableOffset(read->Arg(1), current_key, offset, handle.NumArgs() == 4u ? 4u : 5u) ||
 			    memory->offset > UINT32_MAX - offset) {
 				return false;
 			}
@@ -1398,20 +1599,64 @@ private:
 				return false;
 			}
 			table_handle = current_handle;
-			const std::array<const Inst*, 1> image_users {&handle};
-			if (!UsesOnly(*read, image_users)) {
+			if (handle.NumArgs() == 8u ? !UsesOnlyImageDescriptors(*read) :
+			    std::ranges::any_of(read->Uses(), [](const Use& use) {
+				return use.user->GetOpcode() != ValueOpcode::GetBufferResource ||
+				       std::ranges::any_of(use.user->Uses(), [](const Use& consumer) {
+					       return consumer.user->GetOpcode() != ValueOpcode::StoreBufferU32;
+				       });
+			    })) {
 				return false;
 			}
 			plan.memory[dword] = memory_index;
 			plan.reads[dword] = read;
 		}
 
-		DescriptorSource table_source;
 		if (!MakeRuntimeTableSource(*plan.reads[0], table_source)) {
 			return false;
 		}
+		return true;
+	}
+
+	bool MatchUniformizedBufferKey(Value key, const Inst& image,
+	                               DescriptorSource::IndirectDescriptor& indirect,
+	                               DescriptorSource& material_source) {
+		const auto* selected = UniformizedMaterialValue(key, image);
+		if (selected == nullptr) return false;
+		uint32_t amount = 0;
+		const auto* shift = MaterialKeyShift(selected->Arg(1), amount);
+		if (shift == nullptr) return false;
+		const auto* loaded = shift->Arg(0).Resolve().TryInstruction();
+		if (loaded == nullptr || loaded->GetOpcode() != ValueOpcode::SelectU32 ||
+		    !EquivalentValue(m_program, selected->Arg(0), loaded->Arg(0))) return false;
+		const auto* read = loaded->Arg(1).Resolve().TryInstruction();
+		if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32) return false;
+		const auto& memory = m_program.memory_info[read->Flags<MemoryFlags>().index];
+		uint32_t offset = 1u, scalar = 1u;
+		if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
+		    !memory.idxen || memory.offen ||
+		    !ImmediateU32(read->Arg(2), offset) || offset != 0u ||
+		    !ImmediateU32(read->Arg(3), scalar) || scalar != 0u ||
+		    !EquivalentValue(m_program, selected->Arg(0), read->Arg(4)) ||
+		    !MakeRuntimeTableSource(*read, material_source)) return false;
+		// GPU index arithmetic may wrap. Every record in the bounded V# is a candidate.
+		indirect.material_source = InternSource(material_source);
+		indirect.selector_first = Value(0u);
+		indirect.key_count = material_source.dwords[2];
+		indirect.selector_offset = memory.offset;
+		indirect.selector_shift = amount;
+		return true;
+	}
+
+	bool TryMakeIndirectImage(Inst& handle, const DescriptorSource& descriptor,
+	                          IndirectDescriptorPlan& plan) {
+		if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) return false;
+		DescriptorSource table_source;
+		Value key;
+		uint32_t table_offset = 0;
+		if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset)) return false;
 		DescriptorSource material_source;
-		DescriptorSource::IndirectImage indirect;
+		DescriptorSource::IndirectDescriptor indirect;
 		indirect.table_offset = table_offset;
 		if (table_source.dword_count == 2u) {
 			const auto* selector = key.Resolve().TryInstruction();
@@ -1420,8 +1665,9 @@ private:
 			    NonzeroOnEntry(selector->Arg(0), handle.Parent());
 			if (bitscan) {
 				indirect.key_count = Value(32u);
-			} else {
-				indirect.key_count = BoundedLoopCount(key, handle.Parent());
+			} else if (!m_shader_writes) {
+				if (const auto* bound = BoundedLoop(key, handle.Parent()))
+					indirect.key_count = bound->Arg(1);
 			}
 			if (indirect.key_count.IsEmpty() &&
 			    !MatchUniformizedMaterialKey(key, handle, indirect, material_source)) {
@@ -1429,8 +1675,10 @@ private:
 			}
 			if ((table_offset & 3u) != 0u ||
 			    (bitscan && table_offset > UINT32_MAX - (32u * 32u - 1u))) return false;
-		} else {
-			auto* material_read = key.Resolve().TryInstruction();
+		} else if (!MatchUniformizedBufferKey(key, handle, indirect, material_source)) {
+			const auto* key_shift = MaterialKeyShift(key, indirect.selector_shift);
+			auto* material_read =
+			    (key_shift != nullptr ? key_shift->Arg(0) : key).Resolve().TryInstruction();
 			uint32_t material_memory_index = 0;
 			const auto* memory = material_read != nullptr
 			                         ? ScalarReadMemory(*material_read, material_memory_index) : nullptr;
@@ -1438,19 +1686,13 @@ private:
 			    memory->offset > INT32_MAX || !MemoryIndexBelongsTo(material_memory_index, *material_read)) {
 				return false;
 			}
-			Value selector;
-			if (!MatchMaterialOffset(material_read->Arg(1), selector, indirect.selector_stride,
+			if (!MatchMaterialOffset(material_read->Arg(1), indirect.selector_stride,
 			                         indirect.selector_offset)) {
 				return false;
 			}
 			const auto step = std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u);
 			indirect.selector_offset =
 			    (static_cast<uint32_t>(indirect.selector_offset % step) & ~3u) + (memory->offset & ~3u);
-			const auto* shift = plan.reads[0]->Arg(1).Resolve().TryInstruction();
-			const std::array<const Inst*, 1> material_users {shift};
-			if (!UsesOnly(*material_read, material_users) || !UsesOnly(*shift, plan.reads)) {
-				return false;
-			}
 			const auto* material_handle = material_read->Arg(0).Resolve().TryInstruction();
 			if (material_handle == nullptr ||
 			    material_handle->GetOpcode() != ValueOpcode::GetBufferResource ||
@@ -1467,7 +1709,7 @@ private:
 		            image_source.dwords.begin());
 		std::copy_n(table_source.dwords.begin(), table_source.dword_count,
 		            image_source.dwords.begin() + 4u);
-		image_source.indirect_image = indirect;
+		image_source.indirect_descriptor = indirect;
 		plan.handle = &handle;
 		plan.source = InternSource(image_source);
 		plan.key = key;
@@ -1475,20 +1717,117 @@ private:
 		return true;
 	}
 
-	uint32_t SelectorMaximum(Value value) const {
+	bool TryMakeIndirectBuffer(Inst& handle, const DescriptorSource& descriptor,
+	                           IndirectDescriptorPlan& plan) {
+		if (handle.GetOpcode() != ValueOpcode::GetBufferResource || handle.NumArgs() != 4u ||
+		    handle.Uses().empty()) return false;
+		for (const auto& use: handle.Uses()) {
+			const auto& inst = *use.user;
+			if (inst.GetOpcode() != ValueOpcode::StoreBufferU32) return false;
+			const auto& memory = m_program.memory_info[inst.Flags<MemoryFlags>().index];
+			uint32_t scalar = 1u;
+			if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
+			    memory.idxen || !memory.offen || !ImmediateU32(inst.Arg(3), scalar) || scalar != 0u)
+				return false;
+		}
+		DescriptorSource table_source;
+		Value key;
+		uint32_t table_offset = 0;
+		if (!MatchDescriptorTable(handle, descriptor, plan, table_source, key, table_offset) ||
+		    table_source.dword_count != 4u || table_offset != 0u) return false;
+		const auto* lane_read = key.Resolve().TryInstruction();
+		if (lane_read == nullptr || lane_read->GetOpcode() != ValueOpcode::ReadFirstLane)
+			return false;
+		const Inst* selected = nullptr;
+		for (const auto& use: handle.Uses()) {
+			const auto* value = UniformizedMaterialValue(
+			    key, *use.user, use.user->Arg(use.user->NumArgs() - 1u));
+			if (value == nullptr || (selected != nullptr && selected != value)) return false;
+			selected = value;
+		}
+		const auto* read = selected->Arg(1).Resolve().TryInstruction();
+		uint32_t lane = 0;
+		if (read != nullptr && read->GetOpcode() == ValueOpcode::CompositeExtractU32x2) {
+			if (!ImmediateU32(read->Arg(1), lane) || lane >= 2u) return false;
+			read = read->Arg(0).Resolve().TryInstruction();
+			if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32x2) return false;
+		} else if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadBufferU32) {
+			return false;
+		}
+		const auto& memory = m_program.memory_info[read->Flags<MemoryFlags>().index];
+		uint32_t offset = 1u, scalar = 1u;
+		if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
+		    !memory.idxen || memory.offen || memory.offset > UINT32_MAX - lane * 4u ||
+		    !ImmediateU32(read->Arg(2), offset) || offset != 0u ||
+		    !ImmediateU32(read->Arg(3), scalar) || scalar != 0u ||
+		    !EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false;
+		DescriptorSource material_source;
+		DescriptorSource::IndirectDescriptor indirect;
+		if (!MakeRuntimeTableSource(*read, material_source) ||
+		    !BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false;
+		indirect.material_source = InternSource(material_source);
+		indirect.table_source = InternSource(table_source);
+		indirect.selector_offset = memory.offset + lane * 4u;
+		DescriptorSource source;
+		source.dword_count = 4u;
+		source.dwords.fill(Value(0u));
+		source.indirect_descriptor = indirect;
+		plan.handle = &handle;
+		plan.source = InternSource(source);
+		plan.key = key;
+		return true;
+	}
+
+	using SelectorBounds = std::span<const std::pair<const Inst*, uint32_t>>;
+
+	bool SelectorPredicateFalse(Value value, SelectorBounds bounds, uint32_t depth = 0) const {
+		if (depth > 32u) return false;
+		value = value.Resolve();
+		if (value.IsImmediate()) return value.GetType() == Type::U1 && !value.U1();
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return false;
+		if (inst->GetOpcode() == ValueOpcode::LogicalAnd)
+			return SelectorPredicateFalse(inst->Arg(0), bounds, depth + 1u) ||
+			       SelectorPredicateFalse(inst->Arg(1), bounds, depth + 1u);
+		if (inst->GetOpcode() == ValueOpcode::IEqual32) {
+			for (uint32_t arg = 0; arg < 2u; ++arg) {
+				uint32_t target;
+				if (ImmediateU32(inst->Arg(arg), target) &&
+				    SelectorMaximum(inst->Arg(arg ^ 1u), bounds, depth + 1u) < target) return true;
+			}
+		}
+		return false;
+	}
+
+	uint32_t SelectorMaximum(Value value, SelectorBounds bounds = {}, uint32_t depth = 0) const {
+		if (depth > 32u) return UINT32_MAX;
 		value = value.Resolve();
 		if (value.IsImmediate() && value.GetType() == Type::U32) return value.U32();
 		const auto* inst = value.TryInstruction();
 		if (inst == nullptr) return UINT32_MAX;
-		if (inst->GetOpcode() == ValueOpcode::FindILsb32)
-			return NonzeroOnEntry(inst->Arg(0), inst->Parent()) ? 31u : UINT32_MAX;
-		if (inst->GetOpcode() == ValueOpcode::UMin32)
-			return std::min(SelectorMaximum(inst->Arg(0)), SelectorMaximum(inst->Arg(1)));
-		uint32_t shift;
-		if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 &&
-		    ImmediateU32(inst->Arg(1), shift) && shift < 32u) {
-			const auto maximum = uint64_t {SelectorMaximum(inst->Arg(0))} << shift;
-			if (maximum <= UINT32_MAX) return static_cast<uint32_t>(maximum);
+		for (const auto& [phi, maximum]: bounds)
+			if (inst == phi) return maximum;
+		const auto maximum = [&](uint32_t arg) {
+			return SelectorMaximum(inst->Arg(arg), bounds, depth + 1u);
+		};
+		switch (inst->GetOpcode()) {
+			case ValueOpcode::FindILsb32:
+				return NonzeroOnEntry(inst->Arg(0), inst->Parent()) ? 31u : UINT32_MAX;
+			case ValueOpcode::UMin32:
+			case ValueOpcode::BitwiseAnd32: return std::min(maximum(0), maximum(1));
+			case ValueOpcode::SelectU32:
+				return SelectorPredicateFalse(inst->Arg(0), bounds, depth + 1u)
+				           ? maximum(2) : std::max(maximum(1), maximum(2));
+			case ValueOpcode::ShiftRightLogical32:
+			case ValueOpcode::ShiftLeftLogical32: {
+				uint32_t shift;
+				if (!ImmediateU32(inst->Arg(1), shift) || shift >= 32u) break;
+				if (inst->GetOpcode() == ValueOpcode::ShiftRightLogical32) return maximum(0) >> shift;
+				const auto result = uint64_t {maximum(0)} << shift;
+				if (result <= UINT32_MAX) return static_cast<uint32_t>(result);
+				break;
+			}
+			default: break;
 		}
 		return UINT32_MAX;
 	}
@@ -1515,7 +1854,7 @@ private:
 		return found;
 	}
 
-	bool TryMakeFiniteImage(Inst& handle, IndirectImagePlan& plan) {
+	bool TryMakeFiniteImage(Inst& handle, IndirectDescriptorPlan& plan) {
 		if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u)
 			return false;
 		if (std::ranges::any_of(handle.Uses(), [](const Use& use) {
@@ -1584,8 +1923,8 @@ private:
 		DescriptorSource image_source;
 		image_source.dword_count = 8u;
 		image_source.dwords.fill(Value(0u));
-		image_source.indirect_image.emplace(DescriptorSource::IndirectImage{});
-		auto& sources = image_source.indirect_image->sources;
+		image_source.indirect_descriptor.emplace(DescriptorSource::IndirectDescriptor{});
+		auto& sources = image_source.indirect_descriptor->sources;
 		for (auto& choice: choices) {
 			if (choice.branch != nullptr) continue;
 			const auto source = InternSource(choice.descriptor);
@@ -1621,37 +1960,50 @@ private:
 		return true;
 	}
 
-	const IndirectImagePlan* FindIndirectImage(const Inst& handle) const {
+	const IndirectDescriptorPlan* FindIndirectDescriptor(const Inst& handle) const {
 		const auto found =
-		    std::find_if(m_indirect_images.begin(), m_indirect_images.end(),
-		                 [&](const IndirectImagePlan& plan) {
+		    std::find_if(m_indirect_descriptors.begin(), m_indirect_descriptors.end(),
+		                 [&](const IndirectDescriptorPlan& plan) {
 			    return plan.handle == &handle;
 		    });
-		return found == m_indirect_images.end() ? nullptr : &*found;
+		return found == m_indirect_descriptors.end() ? nullptr : &*found;
 	}
 
 	bool IsIndirectPlanningMemory(uint32_t index) const {
-		return std::any_of(m_indirect_images.begin(), m_indirect_images.end(),
-		                   [&](const IndirectImagePlan& plan) {
+		return std::any_of(m_indirect_descriptors.begin(), m_indirect_descriptors.end(),
+		                   [&](const IndirectDescriptorPlan& plan) {
 			return plan.reads[0] != nullptr &&
-			       std::ranges::find(plan.memory, index) != plan.memory.end();
+			       std::find(plan.memory.begin(), plan.memory.begin() + plan.handle->NumArgs(), index) !=
+			           plan.memory.begin() + plan.handle->NumArgs();
 		});
 	}
 
-	void PlanIndirectImages() {
+	void PlanIndirectDescriptors() {
 		for (auto* block: m_program.blocks) {
 			for (auto& inst: *block) {
-				if (ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None ||
+				if ((ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None &&
+				     inst.GetOpcode() != ValueOpcode::StoreBufferU32) ||
 				    inst.NumArgs() == 0u) {
 					continue;
 				}
 				auto* handle = inst.Arg(0).Resolve().TryInstruction();
-				if (handle == nullptr || FindIndirectImage(*handle) != nullptr) {
+				if (handle == nullptr || FindIndirectDescriptor(*handle) != nullptr) {
 					continue;
 				}
-				IndirectImagePlan plan;
-				if (TryMakeIndirectImage(*handle, plan) || TryMakeFiniteImage(*handle, plan)) {
-					m_indirect_images.push_back(std::move(plan));
+				const auto flags = inst.Flags<MemoryFlags>();
+				const auto& memory = m_program.memory_info[flags.index];
+				DescriptorSource descriptor;
+				MakeSource(*handle, inst.GetOpcode() == ValueOpcode::StoreBufferU32 ? 4u : 8u,
+				           false, false, memory.resource * 4u, descriptor, flags.pc);
+				IndirectDescriptorPlan plan;
+				if (TryMakeIndirectImage(*handle, descriptor, plan) || TryMakeFiniteImage(*handle, plan) ||
+				    TryMakeIndirectBuffer(*handle, descriptor, plan)) {
+					for (const auto& previous: m_indirect_descriptors) {
+						if (plan.reads[0] != nullptr && previous.reads[0] == plan.reads[0] &&
+						    !EquivalentValue(m_program, previous.key, plan.key))
+							Fail(flags.pc, "shared descriptor read has incompatible keys");
+					}
+					m_indirect_descriptors.push_back(std::move(plan));
 				}
 			}
 		}
@@ -1858,12 +2210,18 @@ private:
 		uint32_t resource = 0;
 
 		if (buffer != BufferAccess::None) {
-			if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc,
+			handle = inst.Arg(0).Resolve().TryInstruction();
+			const auto* indirect = handle == nullptr ? nullptr : FindIndirectDescriptor(*handle);
+			if (indirect != nullptr) {
+				source = indirect->source;
+			} else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc,
 			               memory.resource * 4u, handle, source)) {
-				if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) {
+				if (memory.kind != (op == ValueOpcode::ReadConstBuffer ? ResourceKind::ScalarBuffer
+				                                                        : ResourceKind::Buffer) ||
+				    !memory.SupportsIndirectBufferLoad(op)) {
 					Fail(flags.pc,
 					     "buffer descriptor is not a valid runtime value; GPU-selected access "
-					     "requires a raw DWORD x2/x3/x4 load");
+					     "requires a scalar, raw DWORD x1/x2/x3/x4, or formatted X load");
 				}
 				m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer;
 				m_info.uses_dma                         = true;
@@ -1914,7 +2272,7 @@ private:
 			Fail(flags.pc, "image operation has invalid resource kind");
 		}
 		handle               = inst.Arg(0).Resolve().TryInstruction();
-		const auto* indirect = handle != nullptr ? FindIndirectImage(*handle) : nullptr;
+		const auto* indirect = handle != nullptr ? FindIndirectDescriptor(*handle) : nullptr;
 		if (indirect != nullptr) {
 			source = indirect->source;
 		} else {
@@ -1963,7 +2321,7 @@ private:
 			for (uint32_t image = 0; image < m_info.images.size(); image++) {
 				const auto* image_source = Source(m_info.images[image].source);
 				if (image_source == nullptr || image_source->dword_count != 8 ||
-				    image_source->indirect_image.has_value()) {
+				    image_source->indirect_descriptor.has_value()) {
 					continue;
 				}
 				bool alias = true;
@@ -1991,7 +2349,7 @@ private:
 	std::vector<DescriptorSource>              m_sources;
 	std::vector<HandlePatch>                   m_handle_patches;
 	std::vector<MemoryPatch>                   m_memory_patches;
-	std::vector<IndirectImagePlan>             m_indirect_images;
+	std::vector<IndirectDescriptorPlan>             m_indirect_descriptors;
 	std::vector<std::pair<const Inst*, Value>> m_descriptor_selections;
 	bool                                       m_shader_writes = false;
 };

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -23,9 +23,6 @@ namespace {
 constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
 
 bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
-	if (base > AddressMask) {
-		return false;
-	}
 	if (offset < 0) {
 		const auto magnitude = uint64_t {0} - static_cast<uint64_t>(offset);
 		if (magnitude > base) {
@@ -35,7 +32,7 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
 		return true;
 	}
 	const auto magnitude = static_cast<uint64_t>(offset);
-	if (magnitude > AddressMask - base) {
+	if (magnitude > UINT64_MAX - base) {
 		return false;
 	}
 	result = base + magnitude;
@@ -44,16 +41,29 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
 
 bool IsRawRead(const ResourcePlan& values, const Inst& inst) {
 	const auto op = inst.GetOpcode();
-	if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) {
+	if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer &&
+	    op != ValueOpcode::LoadBufferU32) {
 		return false;
 	}
 	const auto index = inst.Flags<MemoryFlags>().index;
 	if (index >= values.memory_info.size()) {
 		return false;
 	}
-	const auto kind = values.memory_info[index].kind;
-	return (op == ValueOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) ||
-	       (op == ValueOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer);
+	const auto& memory = values.memory_info[index];
+	if (op != ValueOpcode::LoadBufferU32) {
+		return (op == ValueOpcode::LoadAddressU32 && memory.kind == ResourceKind::ScalarAddress) ||
+		       (op == ValueOpcode::ReadConstBuffer && memory.kind == ResourceKind::ScalarBuffer);
+	}
+	if (inst.NumArgs() != 5u || memory.kind != ResourceKind::Buffer || memory.typed ||
+	    memory.formatted || memory.coherent || memory.data_bits != 32u ||
+	    memory.data_dwords != 1u || memory.idxen || memory.offen || memory.offset != 0u ||
+	    inst.Arg(4).GetType() != Type::U1) {
+		return false;
+	}
+	for (uint32_t arg = 1; arg <= 3; ++arg) {
+		if (inst.Arg(arg).Resolve() != Value(0u)) return false;
+	}
+	return true;
 }
 
 bool IsDescriptorHandle(ValueOpcode opcode) {
@@ -92,6 +102,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::ISub64:
 		case ValueOpcode::IMul32:
 		case ValueOpcode::IMul64:
+		case ValueOpcode::UMulHi:
 		case ValueOpcode::UMin32:
 		case ValueOpcode::ShiftLeftLogical32:
 		case ValueOpcode::ShiftLeftLogical64:
@@ -108,6 +119,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::SelectU32:
 		case ValueOpcode::SelectF32:
 		case ValueOpcode::ULessThan32:
+		case ValueOpcode::ULessThanEqual32:
 		case ValueOpcode::IEqual32:
 		case ValueOpcode::UGreaterThan32:
 		case ValueOpcode::SGreaterThanEqual32:
@@ -120,6 +132,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::FPOrdGreaterThanEqual32:
 		case ValueOpcode::FPIsNan32:
 		case ValueOpcode::FPMul32:
+		case ValueOpcode::FPRecipIFlag32:
 		case ValueOpcode::FPTrunc32: return true;
 		default: return false;
 	}
@@ -160,6 +173,7 @@ private:
 				default: return false;
 			}
 		}
+		if (require_uniform && !m_active_mask.IsEmpty() && value == m_active_mask) return true;
 		// Integer-only dependency checks do not depend on the active EXEC mask.
 		if (!require_uniform && m_validated_dependencies.contains(inst)) return true;
 		if (!m_visiting.insert(inst).second) {
@@ -249,7 +263,8 @@ private:
 			}
 			return finish(true);
 		}
-		if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
+		if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer ||
+		    op == ValueOpcode::LoadBufferU32) {
 			const auto  expected = op == ValueOpcode::LoadAddressU32
 			                           ? ValueOpcode::GetAddressResource
 			                           : ValueOpcode::GetBufferResource;
@@ -258,6 +273,11 @@ private:
 			    handle->GetOpcode() != expected) {
 				return finish(false);
 			}
+			if (op == ValueOpcode::LoadBufferU32) {
+				const auto guard = inst->Arg(4).Resolve();
+				return finish(Validate(inst->Arg(0)) &&
+				              ((!m_active_mask.IsEmpty() && guard == m_active_mask) || Validate(guard)));
+			}
 		} else if (op == ValueOpcode::CompositeExtractU64) {
 			const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
 			if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) {
@@ -344,6 +364,10 @@ bool SrtWalker::EvaluateWide(Value value, uint64_t& result) {
 			default: return false;
 		}
 	}
+	if (!m_active_mask.IsEmpty() && value == m_active_mask) {
+		result = 1u;
+		return true;
+	}
 	auto* inst = value.TryInstruction();
 	if (inst == nullptr) {
 		return false;
@@ -433,6 +457,16 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 		return false;
 	}
 	const auto& mem    = m_program.memory_info[flags.index];
+	const bool vector = inst.GetOpcode() == ValueOpcode::LoadBufferU32;
+	const auto guard = vector ? inst.Arg(4).Resolve() : Value {};
+	if (vector && (guard.IsImmediate() || guard != m_active_mask)) {
+		uint64_t enabled = 0;
+		if (!Arg(inst, 4, enabled)) return false;
+		if (enabled == 0u) {
+			result = 0u;
+			return true;
+		}
+	}
 	const auto* handle = inst.Arg(0).ResolveInstruction();
 	if (handle == nullptr) {
 		return false;
@@ -443,10 +477,10 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 	if (!Arg(*handle, 0, low) || !Arg(*handle, 1, high) || !Arg(inst, 1, offset)) {
 		return false;
 	}
-	const auto base      = ((high << 32u) | static_cast<uint32_t>(low)) & AddressMask;
+	const auto base      = (high << 32u) | static_cast<uint32_t>(low);
 	const auto immediate = static_cast<int64_t>(static_cast<int32_t>(mem.offset));
 	uint64_t   address   = 0;
-	if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
+	if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer || vector) {
 		uint64_t records = 0;
 		uint64_t word3   = 0;
 		if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) {
@@ -458,13 +492,22 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 		const auto byte_offset =
 		    (static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
 		const auto stride  = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
+		if (vector) {
+			// Only the uniform, unswizzled structured DWORD address is evaluated on the host.
+			if ((high & (1u << 31u)) != 0u || (word3 & ((1u << 23u) | 0xf0000000u)) != 0u)
+				return false;
+			if (stride == 0u || records == 0u || ((word3 >> 12u) & 0x7fu) == 0u) {
+				result = 0u;
+				return true;
+			}
+		}
 		const auto size = stride == 0u
 		                      ? static_cast<uint64_t>(static_cast<uint32_t>(records))
 		                      : static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
 		if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) {
 			return false;
 		}
-		address = (base & ~uint64_t {3}) + byte_offset;
+		address = (base & AddressMask & ~uint64_t {3}) + byte_offset;
 	} else {
 		const auto relative = (immediate & ~int64_t {3}) +
 		                      static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
@@ -473,11 +516,13 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
 		}
 	}
 	uint32_t word = 0;
-	if (m_runtime.read_memory != nullptr) {
-		if (!m_runtime.read_memory(m_runtime.userdata, address, {&word, 1})) {
+	const auto reader = vector ? m_runtime.read_specialization_memory : m_runtime.read_memory;
+	if (reader != nullptr) {
+		if (!reader(m_runtime.userdata, address, {&word, 1})) {
 			return false;
 		}
 	} else {
+		if (vector) return false;
 		std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
 	}
 	result = word;
@@ -505,6 +550,9 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 		case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
 		case ValueOpcode::Phi: return EvaluatePhi(inst, result);
 		case ValueOpcode::ReadFirstLane: {
+			if (!m_active_mask.IsEmpty() && inst.Arg(1).Resolve() == m_active_mask) {
+				return EvaluateWide(inst.Arg(0), result);
+			}
 			const auto clean_runtime = CleanRuntime(m_runtime);
 			SrtWalker  clean_active(m_program, clean_runtime, {}, nullptr, inst.Arg(1));
 			SrtWalker  active(m_program, m_runtime, m_clean_flat_slots, &clean_active,
@@ -537,7 +585,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 		}
 		case ValueOpcode::LoadAddressU32:
 		case ValueOpcode::ReadConstBuffer:
+		case ValueOpcode::LoadBufferU32:
 			if (IsRawRead(m_program, inst)) {
+				if (inst.GetOpcode() == ValueOpcode::LoadBufferU32 && m_clean_evaluator != nullptr &&
+				    m_clean_evaluator->m_active_mask == m_active_mask) {
+					return m_clean_evaluator->EvaluateWide(Value(const_cast<Inst*>(&inst)), result);
+				}
 				return EvaluateRawRead(inst, result);
 			}
 			break;
@@ -577,6 +630,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::UMulHi:
+			if (binary()) {
+				result = (static_cast<uint64_t>(static_cast<uint32_t>(a)) * static_cast<uint32_t>(b)) >> 32u;
+				return true;
+			}
+			return false;
 		case ValueOpcode::UMin32:
 			if (binary()) {
 				result = std::min(static_cast<uint32_t>(a), static_cast<uint32_t>(b));
@@ -612,6 +671,15 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::FPRecipIFlag32:
+			if (Arg(inst, 0, a)) {
+				const auto exponent = (a >> 23u) & 0xffu;
+				// Normal positive powers of two have exact normal reciprocals in every FP mode.
+				if ((a & 0x807fffffu) != 0u || exponent == 0u || exponent >= 254u) return false;
+				result = (254u - exponent) << 23u;
+				return true;
+			}
+			return false;
 		case ValueOpcode::FPIsNan32:
 			if (Arg(inst, 0, a)) {
 				result = std::isnan(Float32(a));
@@ -782,6 +850,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
 				return true;
 			}
 			return false;
+		case ValueOpcode::ULessThanEqual32:
+			if (binary()) {
+				result = static_cast<uint32_t>(a) <= static_cast<uint32_t>(b);
+				return true;
+			}
+			return false;
 		case ValueOpcode::UGreaterThan32:
 			if (binary()) {
 				result = static_cast<uint32_t>(a) > static_cast<uint32_t>(b);

src/kernel/memory.cpp

@@ -166,7 +166,8 @@ static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64
 }
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
-static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX;
+static uint32_t        g_test_backing_store_unmaps_before_failure = UINT32_MAX;
+static callback_func_t g_test_before_backing_map                   = nullptr;
 #endif
 
 #include "memoryAddressSpace.inc"
@@ -880,13 +881,15 @@ bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) {
 	       g_guest_address_space->TryReadBacking(vaddr, data, size);
 }
 
-bool TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size) {
+bool TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size) {
 	if (g_gpu_resources != nullptr && IsGpuAddressRange(vaddr, size)) {
-		if (!Graphics::GuestGpu::IsGpuThread() ||
-		    GetGpuResources().GetBufferCache().HasGpuDirtyBytes(vaddr, size) ||
-		    GetGpuResources().GetTextureCache().IsRegionGpuModified(vaddr, size)) {
+		if (!Graphics::GuestGpu::IsGpuThread()) {
 			return false;
 		}
+		auto& buffers = GetGpuResources().GetBufferCache();
+		if (buffers.HasGpuDirtyBytes(vaddr, size)) {
+			buffers.ReadMemory(vaddr, size);
+		}
 	}
 	return TryReadBacking(vaddr, data, size);
 }
@@ -3024,6 +3027,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 		return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode,
 		                                         &shared_failure);
 	};
+	std::vector<PhysicalMemory::AllocatedBlock> replaced_mappings;
+	auto restore_mappings = [&](size_t count, bool restore_views) {
+		for (size_t index = 0; index < count; ++index) {
+			const auto& old = replaced_mappings[index];
+			if (restore_views) {
+				EXIT_IF(!g_guest_address_space->MapBacking(old.map_vaddr, old.map_size,
+				                                           old.start_addr, old.mode));
+			}
+			EXIT_IF(!g_physical_memory->Map(old.map_vaddr, old.start_addr, old.map_size,
+			                               old.prot, old.mode, old.gpu_mode));
+			g_physical_memory->SetVirtualRangeName(old.map_vaddr, old.map_size, old.name);
+			g_physical_memory->SetVirtualRangeMemoryType(old.map_vaddr, old.map_size,
+			                                          old.memory_type);
+		}
+		MapGpuRange(in_addr, len);
+	};
 	if (fixed) {
 		if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1u)) != 0 ||
 		    (alignment != 0 && in_addr % alignment != 0)) {
@@ -3034,22 +3053,68 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 		}
 
 		std::vector<VirtualRanges::Range> reserved_ranges;
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 		if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) &&
 		    std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) {
-			    return range.type == VirtualRangeType::Reserved;
+			    return range.type == VirtualRangeType::Direct;
 		    })) {
+			uint64_t covered = 0;
+			{
+				Common::LockGuard lock(g_physical_memory->GetMutex());
+				for (const auto& mapping: g_physical_memory->GetMappings()) {
+					const auto start = std::max(in_addr, mapping.map_vaddr);
+					const auto end = std::min(in_addr + len, mapping.map_vaddr + mapping.map_size);
+					if (start >= end) {
+						continue;
+					}
+					auto old = mapping;
+					old.start_addr += start - old.map_vaddr;
+					old.map_vaddr = start;
+					old.size = old.map_size = end - start;
+					covered += old.map_size;
+					replaced_mappings.push_back(old);
+				}
+			}
+			if (covered != len) {
+				return KERNEL_ERROR_EBUSY;
+			}
 			UnmapGpuRange(in_addr, len);
-			reserved_target = true;
-		}
-		if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
-			reserved_target = true;
-		}
-		if (!reserved_target) {
-			return KERNEL_ERROR_ENOMEM;
-		}
-		if (map_shared_fixed(in_addr)) {
+			for (size_t index = 0; index < replaced_mappings.size(); ++index) {
+				const auto& old = replaced_mappings[index];
+				GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess;
+				if (!g_physical_memory->Unmap(old.map_vaddr, old.map_size, &old_gpu_mode)) {
+					restore_mappings(index, false);
+					return KERNEL_ERROR_EBUSY;
+				}
+			}
+			// Keep the old view accessible until mmap atomically replaces it. In particular,
+			// do not publish a temporary reserved/PROT_NONE range between two direct views.
+			if (!map_shared_fixed(in_addr)) {
+				restore_mappings(replaced_mappings.size(), true);
+				return KERNEL_ERROR_ENOMEM;
+			}
 			out_addr       = in_addr;
 			shared_backing = true;
+		} else
+#endif
+		{
+			if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) &&
+			    std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) {
+				    return range.type == VirtualRangeType::Reserved;
+			    })) {
+				UnmapGpuRange(in_addr, len);
+				reserved_target = true;
+			}
+			if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
+				reserved_target = true;
+			}
+			if (!reserved_target) {
+				return KERNEL_ERROR_ENOMEM;
+			}
+			if (map_shared_fixed(in_addr)) {
+				out_addr       = in_addr;
+				shared_backing = true;
+			}
 		}
 	} else {
 		constexpr size_t DEFAULT_ALIGNMENT = 0x4000;
@@ -3105,15 +3170,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 
 	if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) {
 		LOGF_COLOR(Log::Color::Red, "\t [Fail]\n");
-		EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		if (replaced_mappings.empty()) {
+			EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		} else {
+			restore_mappings(replaced_mappings.size(), true);
+		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
 	PhysicalMemory::AllocatedBlock mapped_block {};
 	g_physical_memory->Find(direct_memory_start, false, &mapped_block);
-	const bool published = reserved_target
+	const bool published = !replaced_mappings.empty() || reserved_target
 	                           ? g_virtual_ranges->ReplaceSpan(
-	                                 out_addr, len, VirtualRangeType::Reserved, direct_memory_start,
+	                                 out_addr, len, replaced_mappings.empty()
+	                                                    ? VirtualRangeType::Reserved
+	                                                    : VirtualRangeType::Direct,
+	                                 direct_memory_start,
 	                                 prot, mapped_block.memory_type, VirtualRangeType::Direct, "")
 	                           : g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot,
 	                                                   mapped_block.memory_type,
@@ -3121,7 +3193,11 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 	if (!published) {
 		GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess;
 		EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode));
-		EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		if (replaced_mappings.empty()) {
+			EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
+		} else {
+			restore_mappings(replaced_mappings.size(), true);
+		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
@@ -3504,6 +3580,10 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags,
 }
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+void TestBeforeNextBackingMap(callback_func_t callback) {
+	g_test_before_backing_map = callback;
+}
+
 void TestFailNextPhysicalMemoryUnmap() {
 	TestFailPhysicalMemoryUnmapAfter(0);
 }

src/kernel/memory.h

@@ -114,7 +114,7 @@ int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size,
 int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
 bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
 bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
-bool                   TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size);
+bool                   TryReadBufferBacking(uint64_t vaddr, void* data, uint64_t size);
 bool                   TryReadSparseBacking(uint64_t vaddr, void* data, uint64_t size);
 [[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size);
 void                   WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
@@ -191,6 +191,7 @@ bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory
 bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+void     TestBeforeNextBackingMap(callback_func_t callback);
 void     TestFailNextPhysicalMemoryUnmap();
 void     TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
 void     TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);

src/kernel/memoryAddressSpace.inc

@@ -231,9 +231,8 @@ public:
 		return true;
 	}
 
-	bool MapExistingPlaceholderFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
-	                                 VirtualMemory::Mode mode,
-	                                 FailureReason*      failure_reason = nullptr) {
+	bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
+	              VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) {
 		if (!IsAvailable() || !RangeOk(backing_offset, size)) {
 			SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable
 			                                          : FailureReason::PhysicalRangeOutOfBounds);
@@ -257,14 +256,13 @@ public:
 		SetFailure(failure_reason, FailureReason::None);
 		return true;
 #else
+		std::lock_guard lock(m_mutex);
 		void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason);
 		if (mapped == nullptr || reinterpret_cast<uint64_t>(mapped) != vaddr) {
 			return false;
 		}
-		{
-			std::lock_guard lock(m_mutex);
-			EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second);
-		}
+		RemoveMappingsUnlocked(vaddr, size);
+		EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second);
 		SetFailure(failure_reason, FailureReason::None);
 		return true;
 #endif
@@ -302,8 +300,7 @@ public:
 			bool part_placeholder_preserved = false;
 			if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) {
 				for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
-					EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
-					                                     it->mode));
+					EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
 				}
 				if (placeholder_preserved != nullptr) {
 					*placeholder_preserved = false;
@@ -320,8 +317,7 @@ public:
 		// views into one placeholder before publishing it there.
 		if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) {
 			for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
-				EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset,
-				                                     it->mode));
+				EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
 			}
 			return false;
 		}
@@ -363,12 +359,10 @@ public:
 			return false;
 		}
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		const uint64_t left_size  = vaddr - old.vaddr;
 		const uint64_t right_addr = vaddr + size;
-		const uint64_t old_end    = old.vaddr + old.size;
-		const uint64_t right_size = old_end - right_addr;
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		const uint64_t right_size = old.vaddr + old.size - right_addr;
 		const bool exact_unmap = (vaddr == old.vaddr && size == old.size);
 		if (exact_unmap) {
 			const bool unmapped =
@@ -393,14 +387,11 @@ public:
 				ok = SplitPlaceholder(vaddr, size) && ok;
 			}
 			if (left_size != 0) {
-				ok = MapExistingPlaceholderFixed(old.vaddr, left_size, old.backing_offset,
-				                                 old.mode) &&
-				     ok;
+				ok = MapFixed(old.vaddr, left_size, old.backing_offset, old.mode) && ok;
 			}
 			if (right_size != 0) {
-				ok = MapExistingPlaceholderFixed(right_addr, right_size,
-				                                 old.backing_offset + (right_addr - old.vaddr),
-				                                 old.mode) &&
+				ok = MapFixed(right_addr, right_size,
+				              old.backing_offset + (right_addr - old.vaddr), old.mode) &&
 				     ok;
 			}
 			if (ok) {
@@ -418,8 +409,7 @@ public:
 			}
 			EXIT_IF(VirtualFree(reinterpret_cast<void*>(old.vaddr), old.size,
 			                    MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) == 0);
-			EXIT_IF(
-			    !MapExistingPlaceholderFixed(old.vaddr, old.size, old.backing_offset, old.mode));
+			EXIT_IF(!MapFixed(old.vaddr, old.size, old.backing_offset, old.mode));
 			return false;
 		}
 		std::lock_guard lock(m_mutex);
@@ -433,16 +423,7 @@ public:
 		}
 
 		std::lock_guard lock(m_mutex);
-		m_maps.erase(old.vaddr);
-		if (left_size != 0) {
-			m_maps.emplace(old.vaddr,
-			               MapEntry {old.vaddr, left_size, old.backing_offset, old.mode});
-		}
-		if (right_size != 0) {
-			m_maps.emplace(right_addr,
-			               MapEntry {right_addr, right_size,
-			                         old.backing_offset + right_addr - old.vaddr, old.mode});
-		}
+		RemoveMappingsUnlocked(vaddr, size);
 
 		if (placeholder_preserved != nullptr) {
 			*placeholder_preserved = true;
@@ -516,6 +497,33 @@ private:
 		VirtualMemory::Mode mode           = VirtualMemory::Mode::NoAccess;
 	};
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	void RemoveMappingsUnlocked(uint64_t vaddr, uint64_t size) {
+		const auto end = vaddr + size;
+		auto       it = m_maps.upper_bound(vaddr);
+		if (it != m_maps.begin()) {
+			--it;
+		}
+		while (it != m_maps.end() && it->first < end) {
+			const auto old     = it->second;
+			const auto old_end = old.vaddr + old.size;
+			if (old_end <= vaddr) {
+				++it;
+				continue;
+			}
+			it = m_maps.erase(it);
+			if (old.vaddr < vaddr) {
+				m_maps.emplace(old.vaddr,
+				               MapEntry {old.vaddr, vaddr - old.vaddr, old.backing_offset, old.mode});
+			}
+			if (end < old_end) {
+				m_maps.emplace(end, MapEntry {end, old_end - end,
+				                             old.backing_offset + end - old.vaddr, old.mode});
+			}
+		}
+	}
+#endif
+
 	bool FindContainingUnlocked(uint64_t vaddr, uint64_t size, MapEntry* out) const {
 		EXIT_IF(out == nullptr);
 
@@ -621,6 +629,12 @@ private:
 
 	void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode,
 	              FailureReason* failure_reason) {
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+		if (const auto callback = g_test_before_backing_map) {
+			g_test_before_backing_map = nullptr;
+			callback(vaddr, size);
+		}
+#endif
 		void* ptr = mmap(reinterpret_cast<void*>(vaddr), size, GetProtect(mode),
 		                 MAP_SHARED | MAP_FIXED, m_fd, static_cast<off_t>(backing_offset));
 		if (ptr == MAP_FAILED) {
@@ -778,7 +792,12 @@ public:
 	                VirtualMemory::Mode               mode,
 	                GuestBackingStore::FailureReason* failure_reason = nullptr) {
 		std::lock_guard lock(m_mutex);
-		if (!ConsumeUnlocked(vaddr, size)) {
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+		const bool replacing = ContainsMappedUnlocked(vaddr, size, MappingType::Backing);
+#else
+		constexpr bool replacing = false;
+#endif
+		if (!replacing && !ConsumeUnlocked(vaddr, size)) {
 			if (failure_reason != nullptr) {
 				*failure_reason = GuestBackingStore::FailureReason::AddressRangeUnavailable;
 			}
@@ -787,12 +806,15 @@ public:
 		const auto host_mode = VirtualMemory::IsExecute(mode)
 		                           ? VirtualMemory::Mode::ExecuteReadWrite
 		                           : VirtualMemory::Mode::ReadWrite;
-		if (m_backing->MapExistingPlaceholderFixed(vaddr, size, backing_offset, host_mode,
-		                                           failure_reason)) {
-			EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing));
+		if (m_backing->MapFixed(vaddr, size, backing_offset, host_mode, failure_reason)) {
+			if (!replacing) {
+				EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing));
+			}
 			return true;
 		}
-		AddFreeUnlocked(vaddr, size);
+		if (!replacing) {
+			AddFreeUnlocked(vaddr, size);
+		}
 		return false;
 	}
 

src/launcher/CMakeLists.txt

@@ -21,6 +21,10 @@ add_executable(launcher WIN32
 
 
 target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version common)
+if(LINUX)
+	find_package(Qt6 COMPONENTS DBus REQUIRED)
+	target_link_libraries(launcher Qt6::DBus)
+endif()
 if(WIN32)
 target_link_libraries(launcher setupapi)
 endif()

src/launcher/forms/configuration_edit_dialog.ui

@@ -359,16 +359,16 @@
        <layout class="QHBoxLayout" name="controllerVolumeLayout">
         <item>
          <widget class="QSlider" name="slider_controller_volume">
-          <property name="toolTip"><string>Scale the DualSense controller speaker over USB and Bluetooth</string></property>
+          <property name="toolTip"><string>Scale the DualSense speaker over USB and Bluetooth. 50% is the previous default; 100% adds 6 dB of gain.</string></property>
           <property name="maximum"><number>100</number></property>
-          <property name="value"><number>100</number></property>
+          <property name="value"><number>50</number></property>
           <property name="orientation"><enum>Qt::Horizontal</enum></property>
          </widget>
         </item>
         <item>
          <widget class="QLabel" name="label_controller_volume_value">
           <property name="minimumSize"><size><width>40</width><height>0</height></size></property>
-          <property name="text"><string>100%</string></property>
+          <property name="text"><string>50%</string></property>
          </widget>
         </item>
        </layout>

src/launcher/include/configuration.h

@@ -48,7 +48,7 @@ QString EnumToText(T value) {
 
 struct ControllerSettings {
 	QString color;
-	int     speaker_volume      = 100;
+	int     speaker_volume      = 50;
 	int     vibration_intensity = 100;
 
 	void WriteSettings(QSettings* s) const {
@@ -60,13 +60,13 @@ struct ControllerSettings {
 	void ReadSettings(QSettings* s) {
 		const QColor saved_color(s->value("controller_color").toString());
 		color = saved_color.isValid() ? saved_color.name(QColor::HexRgb) : QString {};
-		const auto read_percent = [s](const char* key) {
+		const auto read_percent = [s](const char* key, int fallback) {
 			bool      ok    = false;
-			const int value = s->value(key, 100).toInt(&ok);
-			return ok ? qBound(0, value, 100) : 100;
+			const int value = s->value(key, fallback).toInt(&ok);
+			return ok ? qBound(0, value, 100) : fallback;
 		};
-		speaker_volume      = read_percent("controller_speaker_volume");
-		vibration_intensity = read_percent("controller_vibration_intensity");
+		speaker_volume      = read_percent("controller_speaker_volume", 50);
+		vibration_intensity = read_percent("controller_vibration_intensity", 100);
 	}
 };
 

src/launcher/src/configurationListWidget.cpp

@@ -36,6 +36,7 @@
 #include <QPainter>
 #include <QPalette>
 #include <QPointer>
+#include <QProcess>
 #include <QPushButton>
 #include <QRegularExpression>
 #include <QSet>
@@ -47,6 +48,12 @@
 #include <QUrl>
 #include <QtCore>
 
+#ifdef __linux__
+#include <QDBusConnection>
+#include <QDBusMessage>
+#include <QDBusPendingCallWatcher>
+#endif
+
 #include <memory>
 
 #include "ui_configuration_list_widget.h"
@@ -837,16 +844,52 @@ void ConfigurationListWidget::open_game_folder() {
 		return;
 	}
 
-	const auto base = item->GetInfo().basedir;
-	const QDir game_dir(GameContent::IsArchive(base) ? QFileInfo(base).absolutePath() : base);
+	const auto base       = item->GetInfo().basedir;
+	const bool is_archive = GameContent::IsArchive(base);
+	const QDir game_dir(is_archive ? QFileInfo(base).absolutePath() : base);
 	if (!game_dir.exists()) {
 		QMessageBox::warning(this, tr("Open game folder"), tr("Game folder does not exist."));
 		return;
 	}
 
-	if (!QDesktopServices::openUrl(QUrl::fromLocalFile(game_dir.absolutePath()))) {
-		QMessageBox::warning(this, tr("Open game folder"), tr("Could not open game folder."));
+	const auto open_directory = [this, game_dir] {
+		if (!QDesktopServices::openUrl(QUrl::fromLocalFile(game_dir.absolutePath()))) {
+			QMessageBox::warning(this, tr("Open game folder"), tr("Could not open game folder."));
+		}
+	};
+	if (is_archive) {
+		const auto path = QFileInfo(base).absoluteFilePath();
+#if defined(_WIN32)
+		QProcess explorer;
+		explorer.setProgram("explorer.exe");
+		explorer.setNativeArguments(
+		    QStringLiteral("/select,\"%1\"").arg(QDir::toNativeSeparators(path)));
+		if (explorer.startDetached()) {
+			return;
+		}
+#elif defined(__APPLE__)
+		if (QProcess::startDetached("/usr/bin/open", {"-R", path})) {
+			return;
+		}
+#elif defined(__linux__)
+		auto request = QDBusMessage::createMethodCall("org.freedesktop.FileManager1",
+		                                              "/org/freedesktop/FileManager1",
+		                                              "org.freedesktop.FileManager1", "ShowItems");
+		request << QStringList {QUrl::fromLocalFile(path).toString(QUrl::FullyEncoded)}
+		        << QString();
+		auto* watcher = new QDBusPendingCallWatcher(
+		    QDBusConnection::sessionBus().asyncCall(request, 5000), this);
+		connect(watcher, &QDBusPendingCallWatcher::finished, this,
+		        [open_directory](QDBusPendingCallWatcher* call) {
+			        call->deleteLater();
+			        if (call->isError()) {
+				        open_directory();
+			        }
+		        });
+		return;
+#endif
 	}
+	open_directory();
 }
 
 void ConfigurationListWidget::remove_save_data() {

src/libs/controller.cpp

@@ -224,7 +224,9 @@ void GameController::CycleSetting(Setting setting) {
 float GameController::GetSettingScale(Setting setting) const {
 	const auto step = m_setting_steps[static_cast<size_t>(setting)].load(std::memory_order_relaxed);
 	if (setting == Setting::SpeakerVolume) {
-		return SPEAKER_VOLUME[step] * (Config::GetControllerSpeakerVolume() / 100.0f);
+		// The old speaker level matches the PS5 slider's midpoint. Give the upper
+		// half of the global slider up to 6 dB of extra software gain.
+		return SPEAKER_VOLUME[step] * (Config::GetControllerSpeakerVolume() / 50.0f);
 	}
 	if (setting == Setting::VibrationIntensity) {
 		return INTENSITY[step] * (Config::GetControllerVibrationIntensity() / 100.0f);

src/libs/dualSenseHaptics.cpp

@@ -178,7 +178,11 @@ SDL_AudioDeviceID FindDevice() {
 	for (int i = 0; i < count; i++) {
 		const char*   name = SDL_GetAudioDeviceName(devices[i]);
 		SDL_AudioSpec spec {};
-		if (name != nullptr && SDL_strcasestr(name, "DualSense") != nullptr &&
+		const bool is_dualsense_name =
+		    name != nullptr &&
+		    (SDL_strcasestr(name, "DualSense") != nullptr ||
+		     SDL_strcasestr(name, "Wireless Controller") != nullptr);
+		if (is_dualsense_name &&
 		    SDL_GetAudioDeviceFormat(devices[i], &spec, nullptr) && spec.channels == 4) {
 			if (device != 0) {
 				device = 0; // Multiple matching endpoints cannot be associated reliably either.

src/loader/guestInstructionPatcher.cpp

@@ -590,7 +590,10 @@ DecodedFunction DecodeFunction(uintptr_t function_start, uintptr_t function_end,
 	return function;
 }
 
-RedZoneMask TranslateRedZoneMask(const RedZoneMask& mask, s64 stack_pointer_delta) {
+std::optional<RedZoneMask> TranslateRedZoneMask(const RedZoneMask& mask, s64 stack_pointer_delta) {
+	if (stack_pointer_delta == 0) {
+		return mask;
+	}
 	RedZoneMask translated;
 	for (size_t bit = 0; bit < GuestRedZoneSize; ++bit) {
 		if (!mask.test(bit)) {
@@ -598,9 +601,12 @@ RedZoneMask TranslateRedZoneMask(const RedZoneMask& mask, s64 stack_pointer_delt
 		}
 		const s64 after_offset  = static_cast<s64>(bit) - static_cast<s64>(GuestRedZoneSize);
 		const s64 before_offset = after_offset + stack_pointer_delta;
-		if (before_offset >= -static_cast<s64>(GuestRedZoneSize) && before_offset < 0) {
-			translated.set(static_cast<size_t>(before_offset + static_cast<s64>(GuestRedZoneSize)));
+		if (before_offset < -static_cast<s64>(GuestRedZoneSize) || before_offset >= 0) {
+			// A later inverse RSP adjustment can bring this byte back into the red zone.
+			// Losing its stack-slot identity makes the bounded analysis inconclusive.
+			return std::nullopt;
 		}
+		translated.set(static_cast<size_t>(before_offset + static_cast<s64>(GuestRedZoneSize)));
 	}
 	return translated;
 }
@@ -610,13 +616,16 @@ void AnalyzeRedZoneLiveness(DecodedFunction& function) {
 		return;
 	}
 
+	const auto protect_function = [&] {
+		for (auto& [_, decoded]: function.instructions) {
+			decoded.red_zone_live.set();
+		}
+	};
 	if (function.has_indirect_branch || function.requires_conservative_red_zone_tracking ||
 	    std::ranges::any_of(function.branch_targets, [&function](uintptr_t target) {
 		    return !function.instructions.contains(target);
 	    })) {
-		for (auto& [_, decoded]: function.instructions) {
-			decoded.red_zone_live.set();
-		}
+		protect_function();
 		return;
 	}
 
@@ -653,12 +662,14 @@ void AnalyzeRedZoneLiveness(DecodedFunction& function) {
 				add_successor(next_address);
 			}
 
-			const RedZoneMask translated_live_out =
-			    decoded.stack_pointer_delta.has_value()
-			        ? TranslateRedZoneMask(live_out, *decoded.stack_pointer_delta)
-			        : live_out;
+			const auto translated_live_out =
+			    TranslateRedZoneMask(live_out, decoded.stack_pointer_delta.value_or(0));
+			if (!translated_live_out) {
+				protect_function();
+				return;
+			}
 			const RedZoneMask new_live_in =
-			    decoded.red_zone_use | (translated_live_out & ~decoded.red_zone_def);
+			    decoded.red_zone_use | (*translated_live_out & ~decoded.red_zone_def);
 			if (new_live_in != live_in[reverse_index]) {
 				live_in[reverse_index] = new_live_in;
 				changed                = true;

src/main.cpp

@@ -54,7 +54,7 @@ static void PrintUsage() {
 	         Config::DEFAULT_USER_ID);
 	::printf("  --mic <name>                        Capture from this microphone; omit for silence.\n");
 	::printf("  --controller-color <#RRGGBB>        Override the controller lightbar color.\n");
-	::printf("  --controller-volume <0-100>         DualSense speaker volume. Default: 100.\n");
+	::printf("  --controller-volume <0-100>         DualSense speaker volume. Default: 50.\n");
 	::printf("  --controller-vibration <0-100>      DualSense vibration intensity. Default: 100.\n");
 	::printf(
 	    "  --present-mode <value>               Fifo, Mailbox, or Immediate. Default: Mailbox.\n");

tests/ControllerSettingsTests.cpp

@@ -112,7 +112,7 @@ struct Controller {
 		Initialize();
 		Connect(1);
 		Check(GetSettingScale(Setting::SpeakerVolume) ==
-		              Config::GetControllerSpeakerVolume() / 100.0f &&
+		              Config::GetControllerSpeakerVolume() / 50.0f &&
 		          GetSettingScale(Setting::VibrationIntensity) ==
 		              Config::GetControllerVibrationIntensity() / 100.0f &&
 		          GetSettingScale(Setting::TriggerEffectIntensity) == 1.0f,
@@ -165,7 +165,7 @@ void TestSettingCycles() {
 
 void TestGlobalControllerLevels() {
 	Config::ConfigOptions options;
-	options.controller_speaker_volume      = 50;
+	options.controller_speaker_volume      = 25;
 	options.controller_vibration_intensity = 25;
 	Config::Load(options);
 	{
@@ -190,6 +190,23 @@ void TestGlobalControllerLevels() {
 		Check(rumble.back().large == 17 * 257 && rumble.back().small == 8 * 257,
 		      "cycling vibration did not apply the combined intensity to cached rumble");
 	}
+	options.controller_speaker_volume = 50;
+	Config::Load(options);
+	{
+		Controller controller;
+		Check(GetSettingScale(Setting::SpeakerVolume) == 1.0f,
+		      "speaker midpoint did not preserve the previous default level");
+	}
+	options.controller_speaker_volume = 100;
+	Config::Load(options);
+	{
+		Controller controller;
+		Check(GetSettingScale(Setting::SpeakerVolume) == 2.0f,
+		      "speaker maximum did not add 6 dB of gain");
+		CycleSetting(Setting::SpeakerVolume);
+		Check(GetSettingScale(Setting::SpeakerVolume) == 0.0f,
+		      "speaker maximum prevented the hotkey from muting");
+	}
 	options.controller_speaker_volume      = 0;
 	options.controller_vibration_intensity = 0;
 	Config::Load(options);

tests/KernelFileSystemTests.cpp

@@ -675,7 +675,7 @@ void CheckAmprOrdering(Loader::SymbolDatabase &symbols, uint32_t file_id) {
             "initialize AMPR command buffer");
     }
   };
-  // SDK WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<.
+  // WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<.
   struct Comparison { uint8_t compare; uint64_t blocked, reference, released; };
   constexpr std::array comparisons {
       Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3,

tests/PadHapticsTests.cpp

@@ -438,7 +438,15 @@ void TestDiscoveryAndHotplug() {
 	devices[0].channels = 4;
 	now += 2001;
 	Queue(port, pcm.data());
-	Check(streams.size() == 1 && opened_device == 10, "late USB connection was not discovered");
+	devices.clear();
+	now += 2001;
+	Queue(port, pcm.data());
+	Check(streams.empty(), "disconnected endpoint retained its stream");
+	devices = {{10, "Wireless Controller", 4}};
+	now += 2001;
+	Queue(port, pcm.data());
+	Check(streams.size() == 1 && opened_device == 10,
+	      "Linux Wireless Controller endpoint was not discovered");
 	devices.clear();
 	now += 2001;
 	Queue(port, pcm.data());
@@ -1140,6 +1148,9 @@ void TestAudioSpeakerRouting() {
 	speaker_scale = 0.5f;
 	audio.AudioOutOutputs(&output, 1, false);
 	ExpectPcm({0.25f, 0.25f, 0, 0, 0.25f, 0.25f, 0, 0});
+	speaker_scale = 2.0f;
+	audio.AudioOutOutputs(&output, 1, false);
+	ExpectPcm({1.0f, 1.0f, 0, 0, 1.0f, 1.0f, 0, 0});
 	speaker_scale = 0.0f;
 	audio.AudioOutOutputs(&output, 1, false);
 	ExpectPcm({});
@@ -1157,7 +1168,7 @@ void TestAudioFallbackGain() {
 		Fixture            f;
 		Libs::Audio::Audio audio;
 		active_controller = 2;
-		speaker_scale     = 0.5f;
+		speaker_scale     = 2.0f;
 		const std::array<int16_t, 4> integer_pcm {16384, 16384, 16384, 16384};
 		const auto  format = is_float ? Libs::Audio::Audio::Format::FloatStereo
 		                              : Libs::Audio::Audio::Format::Signed16bitStereo;
@@ -1168,7 +1179,7 @@ void TestAudioFallbackGain() {
 			audio.AudioOutSetVolume(port, 3, volume.data());
 			Libs::Audio::Audio::OutputParam output {port, data};
 			audio.AudioOutOutputs(&output, 1, false);
-			const float          expected = type == 4 ? 0.125f : 0.25f;
+			const float          expected = type == 4 ? 0.5f : 0.25f;
 			std::array<float, 4> actual {};
 			if (is_float) {
 				Check(SDL_GetAudioStreamData(default_stream, actual.data(), sizeof(actual)) ==
@@ -1223,14 +1234,14 @@ void TestBluetoothAudioSpeakerRouting() {
 	const auto port = audio.AudioOutOpen(4, 1024, 48000, Libs::Audio::Audio::Format::FloatStereo);
 	Check(port.IsValid() && default_stream != nullptr, "Bluetooth pad speaker port did not open");
 	std::vector<float> sound(2048, 0.25f);
-	speaker_scale = 0.5f;
+	speaker_scale = 2.0f;
 	Libs::Audio::Audio::OutputParam output {port, sound.data()};
 	Check(audio.AudioOutOutputs(&output, 1, false) == 1024 && hid_opens == 1 &&
 	          SDL_GetAudioStreamQueued(default_stream) == 0,
 	      "Bluetooth pad speaker also played on the host output");
 	fail_hid_write = true;
 	auto pending = Libs::Controller::DualSenseBluetooth::Prepare(now * 1000000);
-	Check(pending.size() == 1 && std::abs(pending[0].audio[200] - 0.125f) < 1e-4f,
+	Check(pending.size() == 1 && std::abs(pending[0].audio[200] - 0.5f) < 1e-4f,
 	      "Bluetooth speaker did not apply its setting gain");
 	Libs::Controller::DualSenseBluetooth::Send(pending);
 	Check(audio.AudioOutOutputs(&output, 1, false) == 1024 &&

tests/ResourceMaterializationTests.cpp

@@ -1,8 +1,10 @@
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
+#include <bit>
 #include <cstdio>
 #include <cstdlib>
+#include <limits>
 #include <memory>
 
 namespace {
@@ -30,7 +32,8 @@ AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) {
   return *result;
 }
 
-Libs::Graphics::ShaderRecompiler::IR::ResourcePlan SrtPlan(uint64_t address) {
+Libs::Graphics::ShaderRecompiler::IR::ResourcePlan SrtPlan(uint64_t address,
+                                                       bool buffer = false) {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   Program program;
   program.stage = Libs::Graphics::ShaderType::Compute;
@@ -39,16 +42,22 @@ Libs::Graphics::ShaderRecompiler::IR::ResourcePlan SrtPlan(uint64_t address) {
   auto &value_block = AddValueBlock(program);
 
   MemoryInfo memory;
-  memory.kind = ResourceKind::ScalarAddress;
+  memory.kind = buffer ? ResourceKind::ScalarBuffer : ResourceKind::ScalarAddress;
   memory.planning_only = true;
   program.memory_info.push_back(memory);
   const auto low = Value(static_cast<uint32_t>(address));
   const auto high = Value(static_cast<uint32_t>(address >> 32u));
-  auto &handle =
-      value_block.AppendNewInst(ValueOpcode::GetAddressResource, {low, high});
-  auto &raw = value_block.AppendNewInst(
-      ValueOpcode::LoadAddressU32,
-      {Value(&handle), Value(0u), Value(0u), Value(true)});
+  auto &handle = buffer
+                     ? value_block.AppendNewInst(ValueOpcode::GetBufferResource,
+                                                {low, high, Value(4u), Value(0u)})
+                     : value_block.AppendNewInst(ValueOpcode::GetAddressResource,
+                                                {low, high});
+  auto &raw = buffer
+                  ? value_block.AppendNewInst(ValueOpcode::ReadConstBuffer,
+                                             {Value(&handle), Value(0u)})
+                  : value_block.AppendNewInst(
+                        ValueOpcode::LoadAddressU32,
+                        {Value(&handle), Value(0u), Value(0u), Value(true)});
   raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40});
   program.srt_reads.push_back({Value(&raw), 0});
 
@@ -163,6 +172,32 @@ void TestMappedSrtUsesDirectReaderByDefault() {
         "cache rematerialization did not use the direct reader by default");
 }
 
+void TestRawScalarAddressPreservesHighBits() {
+  using namespace Libs::Graphics::ShaderRecompiler::IR;
+  constexpr uint64_t address = 0xabcd000156780003ull;
+  for (const bool buffer : {false, true}) {
+    auto plan = SrtPlan(address, buffer);
+    uint64_t observed = 0;
+    const SrtRuntime runtime{
+        .read_memory = [](void *userdata, uint64_t requested,
+                          std::span<uint32_t> words) {
+          if (words.size() != 1) return false;
+          *static_cast<uint64_t *>(userdata) = requested;
+          words[0] = 0x12345678;
+          return true;
+        },
+        .userdata = &observed};
+    ResourceSnapshot snapshot;
+    ResourceSpecialization specialization;
+    Check(MaterializeResources(plan, runtime, snapshot, specialization),
+          "scalar address materialization failed");
+    Check(observed == (buffer ? 0x0000000156780000ull : address & ~uint64_t{3}),
+          "raw pointer high bits and buffer descriptor flags were conflated");
+    Check(snapshot.flattened_srt == std::vector<uint32_t>{0x12345678},
+          "scalar address read did not reach its flattened slot");
+  }
+}
+
 void TestIntegerRuntimeValueFollowsSrtReads() {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   auto plan = SrtPlan(0x10000);
@@ -212,6 +247,113 @@ void TestIntegerRuntimeValueFollowsSrtReads() {
         "cyclic SRT read-first-lane dependency was accepted");
 }
 
+void TestUniformVectorDescriptorRead() {
+  using namespace Libs::Graphics::ShaderRecompiler::IR;
+  Program program;
+  program.stage = Libs::Graphics::ShaderType::Compute;
+  program.srt_plan_complete = program.resource_tracking_complete = true;
+  auto &block = AddValueBlock(program);
+  auto &handle = block.AppendNewInst(ValueOpcode::GetBufferResource,
+      {Value(0x1000u), Value(4u << 16u), Value(1u), Value(0x16204u)});
+  program.memory_info.push_back({.kind = ResourceKind::Buffer});
+  auto &count = block.AppendNewInst(ValueOpcode::LoadBufferU32,
+      {Value(&handle), Value(0u), Value(0u), Value(0u), Value(true)});
+  count.SetFlags(MemoryFlags{.index = 0});
+  // The captured indirect kernel shares one read across sibling scalar lane reads,
+  // enclosed by a different EXEC mask. Its resource plan must share that read too.
+  auto &lane = block.AppendNewInst(ValueOpcode::GetBuiltin,
+      {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
+  auto &active = block.AppendNewInst(ValueOpcode::ULessThan32, {Value(&lane), Value(32u)});
+  auto &first = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(true)});
+  auto &next = block.AppendNewInst(ValueOpcode::IAdd32, {Value(&count), Value(1u)});
+  auto &second = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&next), Value(true)});
+  auto &sum = block.AppendNewInst(ValueOpcode::IAdd32, {Value(&first), Value(&second)});
+  auto &small = block.AppendNewInst(ValueOpcode::ULessThanEqual32, {Value(&count), Value(72u)});
+  auto &selected = block.AppendNewInst(ValueOpcode::SelectU32, {Value(&small), Value(&sum), Value(&count)});
+  auto &masked = block.AppendNewInst(ValueOpcode::SelectU32, {Value(&active), Value(&selected), Value(0u)});
+  auto &records = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&masked), Value(&active)});
+  DescriptorSource source;
+  source.dword_count = 4;
+  source.dwords = {Value(0x2000u), Value(4u << 16u), Value(&records), Value(0x16204u)};
+  program.descriptor_sources.push_back(source);
+  program.info.buffers.push_back({.source = 0, .written = true});
+  Check(ValidateRuntimeValue(program, Value(&count)), "uniform DWORD count was rejected");
+  struct Reads { uint32_t value = 72; uint32_t strict = 0; uint32_t ordinary = 0; bool clean = true; } reads;
+  const SrtRuntime runtime{
+      .read_memory = [](void *data, uint64_t, std::span<uint32_t> words) {
+        ++static_cast<Reads *>(data)->ordinary;
+        words[0] = 999;
+        return true;
+      },
+      .userdata = &reads,
+      .read_specialization_memory = [](void *data, uint64_t address, std::span<uint32_t> words) {
+        auto &reads = *static_cast<Reads *>(data);
+        ++reads.strict;
+        if (!reads.clean || address != 0x1000u || words.size() != 1) return false;
+        words[0] = reads.value;
+        return true;
+      }};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  for (const bool written : {false, true}) {
+    program.info.buffers[0].written = written;
+    auto plan = ExtractResourcePlan(program);
+    Check(plan.control_flow.empty() && plan.requires_specialization_memory &&
+              plan.capture_specialization_reads,
+          "vector descriptor read depended on incidental control-flow capture");
+    reads = {};
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.buffers[0].dwords[2] == 145 && reads.strict == 1 && reads.ordinary == 0 &&
+              snapshot.specialization_reads ==
+                  std::vector<std::pair<uint64_t, uint64_t>>{{0x1000u, 4u}},
+          "vector descriptor input was not read and captured exactly once");
+    reads.clean = false;
+    reads.strict = 0;
+    Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
+              reads.strict == 1 && reads.ordinary == 0,
+          "dirty vector descriptor input fell back to an ordinary memory read");
+  }
+  count.SetArg(4, Value(false));
+  auto &inactive = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(false)});
+  reads.strict = 0;
+  uint32_t result = 99;
+  Check(SrtWalker(program, runtime).Evaluate(Value(&inactive), result) &&
+            result == 0 && reads.strict == 0 && reads.ordinary == 0,
+        "literal false EXEC read vector memory");
+  count.SetArg(4, Value(true));
+  handle.SetArg(3, Value(0x204u));
+  Check(SrtWalker(program, runtime).Evaluate(Value(&count), result) &&
+            result == 0 && reads.strict == 0 && reads.ordinary == 0,
+        "invalid vector buffer format read memory");
+  handle.SetArg(3, Value(0x16204u));
+  count.SetArg(1, Value(&lane));
+  Check(!ValidateRuntimeValue(program, Value(&count)),
+        "varying vector address was treated as a uniform descriptor read");
+}
+
+void TestExactReciprocalDescriptorArithmetic() {
+  using namespace Libs::Graphics::ShaderRecompiler::IR;
+  Program program;
+  auto &block = AddValueBlock(program);
+  for (const float divisor : {64.f, 128.f, 256.f, 512.f,
+                              std::numeric_limits<float>::min(),
+                              std::bit_cast<float>(253u << 23u)}) {
+    auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)});
+    uint32_t result = 0;
+    Check(SrtWalker(program, {}).Evaluate(Value(&reciprocal), result) &&
+              result == std::bit_cast<uint32_t>(1.f / divisor),
+          "power-of-two reciprocal was not exact");
+  }
+  for (const float divisor : {0.f, 3.f, -64.f, std::numeric_limits<float>::infinity(),
+                              std::numeric_limits<float>::denorm_min(),
+                              std::bit_cast<float>(254u << 23u)}) {
+    auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)});
+    uint32_t result = 0;
+    Check(!SrtWalker(program, {}).Evaluate(Value(&reciprocal), result),
+          "unsupported reciprocal rounding or exceptional input was accepted");
+  }
+}
+
 void TestUnbasedFlatCacheHitMaterializes() {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   auto plan = UnbasedFlatPlan();
@@ -227,6 +369,7 @@ void TestWrittenDescriptorUsesStrictReaderOnce() {
   using namespace Libs::Graphics::ShaderRecompiler::IR;
   Program program;
   program.stage = Libs::Graphics::ShaderType::Compute;
+  program.user_data_count = 1;
   program.srt_plan_complete = true;
   program.resource_tracking_complete = true;
   auto &block = AddValueBlock(program);
@@ -243,7 +386,32 @@ void TestWrittenDescriptorUsesStrictReaderOnce() {
   source.dword_count = 4;
   program.descriptor_sources.push_back(source);
   program.info.buffers.push_back({.source = 0, .written = true});
+  // A host-evaluable branch captures resource reads and needs the writable
+  // descriptor's clean provenance for the renderer's disjointness proof.
+  auto &condition = block.AppendNewInst(ValueOpcode::IEqual32,
+                                        {Value(&offset), Value(4u)});
+  auto &store_block = AddValueBlock(program);
+  AddValueBlock(program);
+  program.block_info[0].condition = Value(&condition);
+  program.block_info[0].terminator.kind =
+      Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch;
+  program.block_info[0].terminator.true_block = 1;
+  program.block_info[0].terminator.false_block = 2;
+  program.block_info[1].id = 1;
+  program.block_info[1].terminator.kind =
+      Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return;
+  program.block_info[2].id = 2;
+  program.block_info[2].terminator.kind =
+      Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return;
+  program.memory_info.push_back({.kind = ResourceKind::Buffer, .resource = 0});
+  auto &output = store_block.AppendNewInst(ValueOpcode::GetBufferResource,
+      {source.dwords[0], source.dwords[1], source.dwords[2], source.dwords[3]});
+  store_block.AppendNewInst(ValueOpcode::StoreBufferU32,
+      {Value(&output), Value(0u), Value(0u), Value(0u), Value(1u), Value(true)})
+      .SetFlags(MemoryFlags{.index = 1});
   auto plan = ExtractResourcePlan(program);
+  Check(plan.capture_specialization_reads,
+        "conditional writable descriptor lost its alias proof");
   struct Reads { uint32_t ordinary = 0; uint32_t strict = 0; bool clean = false; } reads;
   const std::array<uint32_t, 1> user_data{4u};
   const SrtRuntime runtime{
@@ -269,7 +437,9 @@ void TestWrittenDescriptorUsesStrictReaderOnce() {
   reads.clean = true;
   reads.strict = 0;
   Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
-            reads.ordinary == 0 && reads.strict == 1 && snapshot.buffers[0].dwords[0] == 0x8000u,
+            reads.ordinary == 0 && reads.strict == 1 && snapshot.buffers[0].dwords[0] == 0x8000u &&
+            snapshot.specialization_reads ==
+                std::vector<std::pair<uint64_t, uint64_t>>{{0x1004u, 4u}},
         "writable descriptor was evaluated twice or scalar EXEC suppressed its read");
 }
 
@@ -313,7 +483,7 @@ void TestFiniteImageRefreshReusesScalarReads() {
   DescriptorSource root;
   root.dword_count = 8;
   root.dwords.fill(Value(0u));
-  root.indirect_image.emplace(DescriptorSource::IndirectImage{}).sources = {0, 1, 2, 1};
+  root.indirect_descriptor.emplace(DescriptorSource::IndirectDescriptor{}).sources = {0, 1, 2, 1};
   program.descriptor_sources.push_back(root);
   program.info.images.push_back({
       .source = 3,
@@ -426,7 +596,10 @@ void DbgExit(int) { std::abort(); }
 
 int main() {
   TestMappedSrtUsesDirectReaderByDefault();
+  TestRawScalarAddressPreservesHighBits();
   TestIntegerRuntimeValueFollowsSrtReads();
+  TestUniformVectorDescriptorRead();
+  TestExactReciprocalDescriptorArithmetic();
   TestUnbasedFlatCacheHitMaterializes();
   TestWrittenDescriptorUsesStrictReaderOnce();
   TestFailedMaterializationRejectsStage();

tests/ResourceTrackingTests.cpp

@@ -306,7 +306,7 @@ void TestInvariantIndirectImageMaterialization() {
   const auto source = fixture->program.info.images[0].source;
   Check(source < fixture->program.descriptor_sources.size() &&
             fixture->program.descriptor_sources[source]
-                .indirect_image.has_value(),
+                .indirect_descriptor.has_value(),
         "indirect image source was not retained for runtime proof");
   const auto image_handle =
       std::ranges::find_if(*fixture->block, [](const Inst &inst) {
@@ -668,7 +668,7 @@ void TestGuardedDirectImageTable() {
                  fixture.AddMemory(memory, 0x128));
     fixture.PlanAndTrack();
     const auto source = fixture.program.info.images[0].source;
-    const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
+    const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
     Check(indirect && indirect->material_source == UINT32_MAX &&
               indirect->selector_stride == 0u && indirect->table_offset == 344u &&
               indirect->key_count.Resolve().IsImmediate() &&
@@ -966,7 +966,7 @@ void TestBoundedComputeImageLoop() {
                  fixture.AddMemory(sample, 0x29c));
     fixture.PlanAndTrack();
     const auto source = fixture.program.info.images[0].source;
-    const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
+    const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
     Check(indirect && indirect->material_source == UINT32_MAX &&
               indirect->table_offset == 0x6b0u &&
               indirect->key_count.Resolve() == count.Resolve(),
@@ -1321,7 +1321,7 @@ void TestUniformizedMaterialImageKeys() {
                  fixture.AddMemory(output_memory, 0x1b00u), done);
     fixture.PlanAndTrack();
     const auto source = fixture.program.info.images[0].source;
-    const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
+    const auto &indirect = fixture.program.descriptor_sources[source].indirect_descriptor;
     Check(indirect && indirect->selector_stride == 0x90u &&
               indirect->selector_offset == 0xc00u &&
               indirect->table_offset == 0x20e0u &&
@@ -1340,7 +1340,7 @@ void TestUniformizedMaterialImageKeys() {
   plan = make_plan(Variant::FirstLaneAndNot);
   Check(plan.requires_specialization_memory &&
             plan.descriptor_sources[plan.info.images[0].source]
-                .indirect_image->selector_mask.Resolve().TryInstruction() != nullptr,
+                .indirect_descriptor->selector_mask.Resolve().TryInstruction() != nullptr,
         "uniformized image mask did not survive extraction");
   LinearTestMemory memory;
   memory.words.resize(0x23000u / 4u);
@@ -2352,7 +2352,7 @@ void TestDynamicSrtReadRemainsExplicit() {
         "unified memory-offset layout is inconsistent");
 }
 
-void TestPhiValidation() {
+void TestWritableDescriptorPhi() {
   Fixture fixture;
   auto *left = fixture.block;
   auto *right = fixture.AddBlock();
@@ -2370,16 +2370,16 @@ void TestPhiValidation() {
                                    MemoryFlags{0, 20}, merge);
   MemoryInfo memory;
   memory.kind = ResourceKind::Buffer;
-  fixture.Emit(ValueOpcode::LoadBufferU32,
-               {handle, Value(0u), Value(0u), Value(0u), Value(true)},
+  fixture.Emit(ValueOpcode::StoreBufferU32,
+               {handle, Value(0u), Value(0u), Value(0u), Value(1u), Value(true)},
                fixture.AddMemory(memory, 20), merge);
 
   CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value",
-             "control-dependent descriptor phi was accepted");
+             "control-dependent writable descriptor phi was accepted");
   Check(!fixture.program.resource_tracking_complete &&
             fixture.program.info.buffers.empty() &&
             fixture.program.descriptor_sources.empty(),
-        "control-dependent descriptor phi was not rejected transactionally");
+        "control-dependent writable descriptor phi was not rejected transactionally");
 }
 
 ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi,
@@ -2577,7 +2577,7 @@ void TestFiniteImagePhiCycle() {
                fixture.AddMemory(memory, 4));
   fixture.PlanAndTrack();
   const auto source = fixture.program.info.images[0].source;
-  const auto &finite = fixture.program.descriptor_sources[source].indirect_image;
+  const auto &finite = fixture.program.descriptor_sources[source].indirect_descriptor;
   Check(finite && finite->sources.size() == 2,
         "cyclic image selection did not retain two complete descriptor candidates");
   const auto *key = image.Instruction()->Arg(0).Instruction();
@@ -2659,7 +2659,7 @@ void TestFiniteImageBitScanSentinel() {
     }
     fixture.PlanAndTrack();
     const auto source = fixture.program.info.images[0].source;
-    const auto &finite = fixture.program.descriptor_sources[source].indirect_image;
+    const auto &finite = fixture.program.descriptor_sources[source].indirect_descriptor;
     Check(finite && finite->sources.size() == 1 &&
               image.Instruction()->Arg(0).Instruction()->NumPhiBlocks() == 2,
           "nonzero bit scan retained its impossible sentinel or removed a Phi edge");
@@ -2728,6 +2728,119 @@ void TestInvariantLoopPhi() {
         "loop-invariant descriptor phi was not evaluated through typed SSA");
 }
 
+void TestBufferStoreUsesItsOwnActiveValue() {
+  Fixture fixture;
+  const auto lane = fixture.Emit(ValueOpcode::GetBuiltin,
+      {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
+  const auto guard = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(16u)});
+  const auto other_guard = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(32u)});
+  const auto inner = fixture.Emit(ValueOpcode::SelectU32, {guard, Value(9u), lane});
+  const auto data = fixture.Emit(ValueOpcode::SelectU32, {guard, inner, lane});
+  const auto handle = fixture.Buffer({Value(0x1000u), Value(4u << 16u),
+                                       Value(64u), Value(0x16204u)});
+  MemoryInfo memory;
+  memory.kind = ResourceKind::Buffer;
+  const auto store = [&](Value active) {
+    return fixture.Emit(ValueOpcode::StoreBufferU32,
+        {handle, Value(0u), Value(0u), Value(0u), data, active},
+        fixture.AddMemory(memory, 0));
+  };
+  const auto matching = store(guard);
+  const auto different = store(other_guard);
+  ConstantPropagationPass(fixture.program.blocks);
+  Check(matching.Instruction()->Arg(4).Resolve() == Value(9u),
+        "buffer store retained data from its inactive lanes");
+  Check(different.Instruction()->Arg(4).Resolve() == data.Resolve() &&
+            data.Resolve().Instruction()->GetOpcode() == ValueOpcode::SelectU32,
+        "buffer store changed a shared value outside its own EXEC mask");
+}
+
+void TestBoundedRelativeRegisterWrites() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  enum class Variant { Bounded, BoundClobber, DescriptorClobber, EntryBypass };
+  for (const auto variant : {Variant::Bounded, Variant::BoundClobber,
+                             Variant::DescriptorClobber, Variant::EntryBypass}) {
+    Fixture fixture;
+    auto *entry = fixture.block;
+    auto *header = fixture.AddBlock();
+    auto *body = fixture.AddBlock();
+    auto *exit = fixture.AddBlock();
+    entry->AddBranch(header);
+    if (variant == Variant::EntryBypass) entry->AddBranch(body);
+    header->AddBranch(body);
+    header->AddBranch(exit);
+    body->AddBranch(header);
+    fixture.program.block_info[0].terminator = {
+        .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
+                                               : CFG::TerminatorKind::Branch,
+        .true_block = 1u, .false_block = 2u};
+    fixture.program.block_info[1].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 2u, .false_block = 3u};
+    fixture.program.block_info[2].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+    fixture.program.block_info[3].terminator.kind = CFG::TerminatorKind::Return;
+
+    const auto lane = fixture.Emit(ValueOpcode::GetBuiltin,
+        {Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
+    const auto enabled = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(32u)});
+    fixture.program.block_info[0].condition = enabled;
+    const auto chunk = fixture.Emit(ValueOpcode::SelectU32,
+                                    {enabled, Value(64u), Value(512u)});
+    const auto initial_bound = fixture.Emit(ValueOpcode::ShiftRightLogical32,
+                                           {chunk, Value(6u)});
+    const auto initial_records = fixture.UserData(2u);
+    const auto base = fixture.UserData(0u);
+    auto &counter = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+    auto &bound = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+    auto &records = header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+    const auto in_range = fixture.Emit(ValueOpcode::ULessThan32,
+                                       {Value(&counter), Value(&bound)}, 0, header);
+    fixture.program.block_info[1].condition = fixture.Emit(
+        ValueOpcode::ConditionRef, {in_range}, CFG::BranchCondition::VccNonZero, header);
+    const auto m0 = fixture.Emit(ValueOpcode::BitwiseAnd32,
+        {fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+                       {Value(&counter), Value(1u)}, 0, body), Value(255u)}, 0, body);
+    const auto relative_write = [&](Value old_value, uint32_t offset) {
+      const auto selected = fixture.Emit(ValueOpcode::IEqual32,
+                                           {m0, Value(offset)}, 0, body);
+      const auto active = fixture.Emit(ValueOpcode::LogicalAnd,
+                                        {enabled, selected}, 0, body);
+      return fixture.Emit(ValueOpcode::SelectU32, {active, lane, old_value}, 0, body);
+    };
+    bound.AddPhiOperand(entry, initial_bound);
+    bound.AddPhiOperand(body, relative_write(Value(&bound),
+        variant == Variant::BoundClobber ? 14u : 22u));
+    records.AddPhiOperand(entry, initial_records);
+    records.AddPhiOperand(body, relative_write(Value(&records),
+        variant == Variant::DescriptorClobber ? 14u : 21u));
+    counter.AddPhiOperand(entry, Value(0u));
+    counter.AddPhiOperand(body, fixture.Emit(ValueOpcode::IAdd32,
+                                             {Value(&counter), Value(1u)}, 0, body));
+    const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
+        {base, Value(4u << 16u), Value(&records), Value(0x16204u)},
+        MemoryFlags{0, 0x3e8}, exit);
+    MemoryInfo memory;
+    memory.kind = ResourceKind::Buffer;
+    fixture.Emit(ValueOpcode::StoreBufferU32,
+        {handle, Value(0u), Value(0u), Value(0u), Value(1u), Value(true)},
+        fixture.AddMemory(memory, 0x3e8), exit);
+    if (variant != Variant::Bounded) {
+      CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value",
+                 "relative register proof discarded a possible loop clobber");
+      continue;
+    }
+    fixture.PlanAndTrack();
+    const std::array<uint32_t, 3> user_data{0x1000u, 0u, 72u};
+    SrtRuntime runtime{.user_data = user_data};
+    DescriptorValue descriptor;
+    Check(SrtWalker(fixture.program, runtime).EvaluateDescriptor(
+              fixture.program.info.buffers[0].source, descriptor) &&
+              descriptor.dwords[2] == 72u,
+          "impossible relative writes left a false descriptor dependency");
+  }
+}
+
 void TestDmaAddressMaterialization() {
   Fixture fixture;
   const auto based =
@@ -2927,6 +3040,79 @@ ResourcePlan ConditionalBufferPlan(ConditionalBufferUse use) {
   return ExtractResourcePlan(fixture.program);
 }
 
+void TestNativeScalarAtomicDataDoesNotEnterResourcePlan() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  Fixture fixture;
+  auto *entry = fixture.block;
+  for (uint32_t i = 1; i < 5; ++i) fixture.AddBlock();
+  fixture.program.block_info[0].terminator = {
+      .kind = CFG::TerminatorKind::ConditionalBranch, .true_block = 1, .false_block = 4};
+  for (uint32_t i = 1; i < 4; ++i) {
+    fixture.program.block_info[i].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = i + 1};
+  }
+  fixture.program.block_info[4].terminator.kind = CFG::TerminatorKind::Return;
+
+  const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1));
+  MemoryInfo scalar;
+  scalar.kind = ResourceKind::ScalarAddress;
+  scalar.component_count = 2;
+  scalar.offset = 136;
+  const auto low = fixture.Emit(ValueOpcode::LoadAddressU32,
+      {table, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(scalar, 0x560));
+  scalar.offset = 140;
+  scalar.component_index = 1;
+  const auto high = fixture.Emit(ValueOpcode::LoadAddressU32,
+      {table, Value(0u), Value(0u), Value(true)}, fixture.AddMemory(scalar, 0x560));
+  const auto address = fixture.Address(low, high);
+  scalar.offset = 24;
+  scalar.component_count = 1;
+  scalar.component_index = 0;
+  const auto data_flags = fixture.AddMemory(scalar, 0x584);
+  const auto data = fixture.Emit(ValueOpcode::LoadAddressU32,
+      {address, Value(0u), Value(0u), Value(true)}, data_flags);
+  const auto descriptor = fixture.Buffer({low,
+      fixture.Emit(ValueOpcode::BitwiseOr32, {high, Value(256u << 16u)}),
+      Value(1u), Value(90628u)}, 0x58c);
+  MemoryInfo vector;
+  vector.kind = ResourceKind::Buffer;
+  vector.offset = 24;
+  vector.data_dwords = 2;
+  fixture.Emit(ValueOpcode::BufferAtomicOr64,
+      {descriptor, Value(0u), Value(0u), Value(0u), Value(uint64_t{1}), Value(true)},
+      fixture.AddMemory(vector, 0x58c));
+  const auto flag = fixture.Emit(ValueOpcode::BitwiseAnd32, {data, Value(16u)});
+  fixture.program.block_info[0].condition =
+      fixture.Emit(ValueOpcode::INotEqual32, {flag, Value(0u)});
+  fixture.PlanAndTrack();
+  Check(data.Instruction()->Parent() == entry &&
+            !fixture.program.memory_info[data_flags.index].planning_only &&
+            fixture.program.info.buffers.size() == 1 && fixture.program.info.buffers[0].atomic,
+        "mutable scalar data was replaced by a host snapshot before its native atomic write");
+  auto plan = ExtractResourcePlan(fixture.program);
+  Check(plan.srt_reads.size() == 2 && plan.control_flow.empty() &&
+            !plan.capture_specialization_reads,
+        "resource-free successor chain retained the shader's scalar data predicate");
+  struct Reads { uint32_t descriptors = 0; uint32_t data = 0; } reads;
+  const auto Read = +[](void *userdata, uint64_t address, std::span<uint32_t> words) {
+    auto &reads = *static_cast<Reads *>(userdata);
+    if (address == 0x2018u) { ++reads.data; return false; }
+    if (words.size() != 1 || (address != 0x1088u && address != 0x108cu)) return false;
+    ++reads.descriptors;
+    words[0] = address == 0x1088u ? 0x2000u : 0u;
+    return true;
+  };
+  const std::array<uint32_t, 2> user_data{0x1000u, 0u};
+  const SrtRuntime runtime{.user_data = user_data, .read_memory = Read, .userdata = &reads,
+                           .read_specialization_memory = Read};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            reads.descriptors == 2 && reads.data == 0 && snapshot.specialization_reads.empty() &&
+            snapshot.buffers[0].dwords[0] == 0x2000u,
+        "resource materialization read mutable scalar data or lost its descriptor pointer");
+}
+
 void TestConditionalBufferMaterialization() {
   auto plan = ConditionalBufferPlan(ConditionalBufferUse::Optional);
   // GTA III leaves packet words in s[12:15] when its scalar control word is zero.
@@ -3474,15 +3660,18 @@ int main() {
     Run("draw-uniform scalar image", TestUniformScalarBufferImage);
     Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization);
     Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
-    Run("phi validation", TestPhiValidation);
+    Run("writable descriptor phi", TestWritableDescriptorPhi);
     Run("conditional sampler phi", TestConditionalSamplerPhi);
     Run("finite image phi cycle", TestFiniteImagePhiCycle);
     Run("finite image bit scan sentinel", TestFiniteImageBitScanSentinel);
     Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
     Run("invariant loop phi", TestInvariantLoopPhi);
+    Run("buffer store active value", TestBufferStoreUsesItsOwnActiveValue);
+    Run("bounded relative register writes", TestBoundedRelativeRegisterWrites);
     Run("DMA address materialization", TestDmaAddressMaterialization);
     Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma);
     Run("buffer swizzle specialization", TestBufferSwizzleSpecialization);
+    Run("native scalar atomic data", TestNativeScalarAtomicDataDoesNotEnterResourcePlan);
     Run("conditional buffer materialization", TestConditionalBufferMaterialization);
     Run("guarded scalar descriptor reads", TestGuardedScalarDescriptorReads);
     Run("conservative buffer reachability", TestConservativeBufferReachability);

tests/ShaderRayTracingGpuTests.inc

@@ -3,7 +3,7 @@ TestCase FlatStackApertures(u32 wave_size) {
   test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64";
   auto &code = test.code;
   code = {EncodeSop1(0x04, 8, 237), // private base
-          EncodeSop1(0x04, 10, 235), // shared base
+          EncodeSop1(0x0b, 11, InlineU32(1)), // S_BREV_B32 shared aperture, as in GTA.
           EncodeVop2(0x1a, 20, InlineU32(2), 0),
           EncodeVop2(0x1b, 22, InlineU32(1), 0),
           EncodeVop2(0x1a, 22, InlineU32(31), 22)};
@@ -77,17 +77,21 @@ TestCase FlatStackApertures(u32 wave_size) {
   test.compute_info.scratch_size_dwords = wave_size;
   test.compute_info.lds_size_dwords = wave_size;
   test.has_compute_info = true;
+  test.decoded_counts = {{"S_BREV_B32", 1}};
+  test.ir_counts = {{" = BitReverse32 ", 0}};
   test.forbidden_spirv = {"get_bda_pointer"};
   return test;
 }
 
-TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) {
+TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0, bool wide = false) {
   TestCase test;
   test.name = barycentrics ? (sorted ? "BvhBarycentricSorted" : "BvhBarycentricUnsorted")
                           : (sorted ? "BvhTriangleIdSorted" : "BvhTriangleIdUnsorted");
   if (grow != 0) test.name = "BvhBoxExitGrowth";
+  if (wide) test.name = "Bvh64HighNodePointers";
   constexpr u32 NodeStart = 1024, OutputStart = 512, LaneCount = 32;
   constexpr uint64_t GuestBase = 0x10000;
+  const u32 node_base = wide ? 1u << 29 : 0;
   const auto bits = [](float value) { return std::bit_cast<u32>(value); };
   constexpr u32 Infinity = 0x7f800000, Invalid = UINT32_MAX, Flag = 0x04040404;
   test.initial.resize(NodeStart + 258 * 16);
@@ -181,6 +185,19 @@ TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) {
     } else if (lane == 29) {
       data[input] = (255 << 3) | 5;
     }
+    if (wide) {
+      // Keep the same small backing while placing nodes beyond the 32-bit pointer range.
+      std::copy_backward(data.begin() + input + 1, data.begin() + input + 11,
+                         data.begin() + input + 12);
+      data[input + 1] = 1;
+      if (lane == 14 || lane == 15) {
+        data[input] = lane == 14 ? 0 : 5;
+        // Neither an oversized index nor a shifted-address wrap may read node zero.
+        data[input + 1] = lane == 14 ? 0x00001000 : 0x20000000;
+        expected[lane] = lane == 14 ? std::array<u32, 4>{Infinity, bits(1), Invalid, 0}
+                                    : std::array<u32, 4>{Invalid, Invalid, Invalid, Invalid};
+      }
+    }
   }
   test.expected = test.initial;
   for (u32 lane = 0; lane < LaneCount; ++lane) {
@@ -196,29 +213,31 @@ TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) {
   // The resource is a GPU value, not a materialized host image descriptor.
   AppendSMovLiteral(&code, 8, GuestBase >> 8);
   AppendSMovLiteral(&code, 9, (sorted ? 0x80000000 : 0) | (grow << 23));
-  AppendSMovLiteral(&code, 10, 256);
+  AppendSMovLiteral(&code, 10, node_base + 256);
   AppendSMovLiteral(&code, 11, barycentrics ? 0x81000000 : 0x80000000);
   code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0));
   for (u32 offset : {0u, 4u, 8u}) {
-    code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4),
+    code.insert(code.end(), {EncodeMubuf0(offset == 8 && !wide ? 0x0f : 0x0e, offset * 4),
                              EncodeMubuf1(4 + offset, 0, 1)});
   }
-  code.insert(code.end(), {EncodeMimg0(0xe6, 15, 0, false, 0, true),
+  code.insert(code.end(), {EncodeMimg0(wide ? 0xe7 : 0xe6, 15, 0, false, 0, true),
                            EncodeMimg1(20, 4, 2)});
   code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0));
   for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 20 + i, 2, OutputStart + i);
   // NSA addressing, the FP32 second-block bound, and inactive destination lanes.
-  AppendSMovLiteral(&code, 10, 2);
+  AppendSMovLiteral(&code, 10, node_base + 2);
   for (u32 i = 0; i < 4; ++i) AppendVMovLiteral(&code, 24 + i, 0x12345678 + i);
   AppendSMovLiteral(&code, 126, 0x000fffff);
-  code.insert(code.end(), {EncodeMimg0(0xe6, 15, 3, false, 0, true),
-                           EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, 0x00000e0d});
+  code.insert(code.end(), {EncodeMimg0(wide ? 0xe7 : 0xe6, 15, 3, false, 0, true),
+                           EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09,
+                           wide ? 0x000f0e0du : 0x00000e0du});
   code.push_back(EncodeSop1(0x04, 126, 193));
   for (u32 i = 0; i < 4; ++i)
     AppendStoreVgprAtLaneDwordOffset(&code, 24 + i, 2, OutputStart + LaneCount * 4 + i);
   AppendEnd(&code);
-  test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)},
-                       {GuestBase + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}};
+  const auto nodes_address = GuestBase + (uint64_t{node_base} << 6);
+  test.bda_mappings = {{nodes_address, NodeStart * sizeof(u32)},
+                       {nodes_address + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}};
   test.compute_info.wave_size = LaneCount;
   test.compute_info.threads_num[0] = LaneCount;
   test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;

tests/ShaderRayTracingTests.inc

@@ -5,10 +5,15 @@ void TestRayTracingInstructions() {
       EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(16, 0, 0),
       0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
       EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(16, 0, 0),
+      0xf19c9f01u, 0x00081014u,
+      EncodeMubuf0(0x1e, 32, false), EncodeMubuf1(16, 0, 0),
+      // Captured 016c1ab44ee6a526: pointer v2:v7 and final inverse component v5.
+      0xf19c9f07u, 0x00090202u, 0x28263607u, 0x221f1e29u, 0x00050403u,
+      EncodeMubuf0(0x1e, 48, false), EncodeMubuf1(2, 0, 0),
       EncodeSopp(0x01)};
   D::Program decoded;
   D::DecodeProgram(code, decoded);
-  Check(decoded.instructions.size() == 5 &&
+  Check(decoded.instructions.size() == 9 &&
             decoded.instructions.back().opcode == D::Opcode::S_ENDPGM,
         "decoder stopped before instructions following BVH intersections");
   for (const auto index : {0u, 2u}) {
@@ -17,10 +22,20 @@ void TestRayTracingInstructions() {
               inst.image_address_components == 11 && inst.data_dwords == 4 && inst.image_r128,
           "BVH32 instruction lost its ray or descriptor width");
   }
+  for (const auto index : {4u, 6u}) {
+    const auto &inst = decoded.instructions[index];
+    Check(inst.opcode == D::Opcode::IMAGE_BVH_INTERSECT_RAY && inst.opcode_id == 0xe7 &&
+              inst.image_address_components == 12 && inst.data_dwords == 4 && inst.image_r128,
+          "BVH64 instruction lost its full node pointer or ray width");
+  }
+  Check(decoded.instructions[6].word_count == 5 &&
+            decoded.instructions[6].image_nsa_addr[0] == 7 &&
+            decoded.instructions[6].image_nsa_addr[10] == 5,
+        "captured BVH64 NSA pointer or final ray component was not decoded");
   Check(decoded.instructions[2].image_nsa_addr[9] == 58,
         "BVH NSA ray components were not decoded");
   const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute));
-  Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 2 &&
+  Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 4 &&
             result.program.info.uses_dma && result.program.info.images.empty() &&
             result.program.info.samplers.empty(),
         "BVH intersections did not use GPU descriptors and raw guest memory");
@@ -31,20 +46,22 @@ void TestRayTracingInstructions() {
   Check(decoded.instructions.size() == 2 && decoded.instructions[0].opcode == D::Opcode::S_MOV_B32,
         "BVH-shaped scalar literal was decoded as an instruction");
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-  for (const auto words : {std::array{0xf19c9f01u, 0x00081014u}, // BVH64
-                           std::array{0xf1989f01u, 0x40081014u}, // A16
+  for (const auto words : {std::array{0xf19c9f01u, 0x40081014u}, // BVH64 A16
+                           std::array{0xf1989f01u, 0x40081014u}, // BVH32 A16
                            std::array{0xf1981f01u, 0x00081014u}, // not R128
                            std::array{0xf1989f01u, 0x80081014u}}) { // D16
     const std::array unsupported{words[0], words[1], EncodeSopp(0x01)};
     ExpectFatal([&]() { (void)RecompileForTest(unsupported, MakeCompileOptions(ShaderType::Compute)); },
                 "unsupported BVH encoding was silently accepted");
   }
-  for (const auto nsa : {1u, 2u}) {
-    std::array<uint32_t, 5> truncated{0xf1989f01u | (nsa << 1), 0x00081014u};
-    truncated[2 + nsa] = EncodeSopp(0x01);
-    ExpectFatal([&]() {
-      (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute));
-    }, "BVH NSA instruction accepted fewer than eleven ray addresses");
+  for (const auto encoding : {0xf1989f01u, 0xf19c9f01u}) {
+    for (const auto nsa : {1u, 2u}) {
+      std::array<uint32_t, 5> truncated{encoding | (nsa << 1), 0x00081014u};
+      truncated[2 + nsa] = EncodeSopp(0x01);
+      ExpectFatal([&]() {
+        (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute));
+      }, "BVH NSA instruction accepted an incomplete ray address");
+    }
   }
 #endif
 }

tests/VirtualMemoryAllocationTests.cpp

@@ -207,30 +207,6 @@ void TestWindowsGuestRedZoneStaticPatcher() {
 	    Common::VirtualMemory::Mode::ExecuteReadWrite, "red_zone_patcher_test");
 	Check(test, mapping != 0, "failed to allocate patch test code");
 
-	std::vector<uint8_t> code;
-	const auto emit = [&code](std::initializer_list<uint8_t> bytes) {
-		code.insert(code.end(), bytes.begin(), bytes.end());
-	};
-	const auto emit64 = [&code](uint64_t value) {
-		const auto offset = code.size();
-		code.resize(offset + sizeof(value));
-		std::memcpy(code.data() + offset, &value, sizeof(value));
-	};
-	emit({0x48, 0xb8});
-	emit64(SENTINEL);                         // movabs rax, sentinel
-	emit({0x48, 0x89, 0x44, 0x24, 0xe8});     // mov [rsp-0x18], rax
-	emit({0x48, 0x8b, 0x07});                 // mov rax, [rdi] (faultable, 3 bytes)
-	emit({0x48, 0x8b, 0x44, 0x24, 0xe8});     // mov rax, [rsp-0x18]
-	emit({0x48, 0xb9});
-	emit64(SENTINEL);                         // movabs rcx, sentinel
-	emit({0x48, 0x39, 0xc8});                 // cmp rax, rcx
-	emit({0x0f, 0x94, 0xc0});                 // sete al
-	emit({0x0f, 0xb6, 0xc0, 0xc3});           // movzx eax, al; ret
-	Check(test, code.size() < CODE_SIZE, "generated patch test code is too large");
-	std::memcpy(reinterpret_cast<void*>(mapping), code.data(), code.size());
-	Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.size()),
-	      "failed to flush generated test code");
-
 	g_red_zone_fault_page = VirtualAlloc(nullptr, 0x1000, MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS);
 	Check(test, g_red_zone_fault_page != nullptr, "failed to allocate fault page");
 	auto* handler = AddVectoredExceptionHandler(1, RedZoneFaultHandler);
@@ -238,18 +214,63 @@ void TestWindowsGuestRedZoneStaticPatcher() {
 
 	using GuestFunction = uint64_t(KYTY_SYSV_ABI*)(const uint64_t*);
 	const auto function = reinterpret_cast<GuestFunction>(mapping);
-	const bool unpatched_was_corrupted = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 0;
-
-	DWORD old_protection = 0;
-	Check(test, VirtualProtect(g_red_zone_fault_page, 0x1000, PAGE_NOACCESS, &old_protection) != FALSE,
-	      "failed to reset fault page protection");
-	Loader::RegisterGuestInstructionPatchModule(
-	    reinterpret_cast<void*>(mapping), CODE_SIZE, reinterpret_cast<void*>(mapping + CODE_SIZE),
-	    TRAMPOLINE_SIZE);
 	const std::array<uintptr_t, 1> function_starts = {static_cast<uintptr_t>(mapping)};
-	const auto result = Loader::PatchGuestInstructions(
-	    mapping, code.size(), function_starts, true, false);
-	const bool patched_preserved = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 1;
+	const auto run_fault = [&] {
+		DWORD old_protection = 0;
+		Check(test, VirtualProtect(g_red_zone_fault_page, 0x1000, PAGE_NOACCESS, &old_protection) != FALSE,
+		      "failed to reset fault page protection");
+		return function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 1;
+	};
+	bool reproduced = true;
+	bool covered = true;
+	bool preserved = true;
+	// The partial excursions discard opposite halves of [rsp-24, rsp-16). Repair
+	// only the retained half before moving RSP, leaving the discarded half live
+	// across the fault. The zero excursion retains the original fixed-stack case.
+	for (const auto [excursion, live]: {std::pair {0, true}, {0, false}, {8, true}, {20, true},
+	                                    {128, true}, {256, true}, {-108, true}, {-128, true},
+	                                    {-256, true}}) {
+		Xbyak::CodeGenerator code(CODE_SIZE, reinterpret_cast<void*>(mapping));
+		code.mov(code.rax, SENTINEL);
+		code.mov(code.qword[code.rsp - 24], code.rax);
+		code.mov(code.rax, code.ptr[code.rdi]); // Faultable three-byte instruction.
+		if (!live) {
+			code.mov(code.rax, SENTINEL);
+			code.mov(code.qword[code.rsp - 24], code.rax);
+		} else if (excursion < 0) {
+			code.mov(code.dword[code.rsp - 20], static_cast<uint32_t>(SENTINEL >> 32));
+		} else if (excursion > 0) {
+			code.mov(code.dword[code.rsp - 24], static_cast<uint32_t>(SENTINEL));
+		}
+		if (excursion != 0) {
+			code.lea(code.rsp, code.ptr[code.rsp - excursion]);
+			code.lea(code.rsp, code.ptr[code.rsp + excursion]);
+		}
+		code.mov(code.rax, code.qword[code.rsp - 24]);
+		code.mov(code.rcx, SENTINEL);
+		code.cmp(code.rax, code.rcx);
+		code.sete(code.al);
+		code.movzx(code.eax, code.al);
+		code.ret();
+		Check(test, Xbyak::GetError() == 0, "failed to generate red-zone patch fixture");
+		Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()),
+		      "failed to flush generated test code");
+		reproduced &= run_fault() == !live;
+		Loader::RegisterGuestInstructionPatchModule(
+		    reinterpret_cast<void*>(mapping), CODE_SIZE, reinterpret_cast<void*>(mapping + CODE_SIZE),
+		    TRAMPOLINE_SIZE);
+		const auto result = Loader::PatchGuestInstructions(
+		    mapping, code.getSize(), function_starts, true, false);
+		const bool intact = run_fault();
+		const uint64_t expected_patches = live ? 1 : 0;
+		covered &= result.red_zone_function_count == 1 &&
+		           result.memory_instruction_count == expected_patches &&
+		           result.patched_memory_instruction_count == expected_patches &&
+		           result.unrelocatable_memory_instruction_count == 0;
+		preserved &= intact;
+		std::printf("[host]    red-zone excursion=%d live=%d patched=%llu preserved=%d\n", excursion, live,
+		            static_cast<unsigned long long>(result.patched_memory_instruction_count), intact);
+	}
 
 	Loader::UnregisterGuestInstructionPatchModule(reinterpret_cast<void*>(mapping));
 	RemoveVectoredExceptionHandler(handler);
@@ -257,12 +278,9 @@ void TestWindowsGuestRedZoneStaticPatcher() {
 	g_red_zone_fault_page = nullptr;
 	const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(mapping, CODE_SIZE + TRAMPOLINE_SIZE);
 
-	Check(test, unpatched_was_corrupted, "test harness did not reproduce red-zone corruption");
-	Check(test, result.red_zone_function_count == 1 && result.memory_instruction_count >= 1 &&
-	                result.patched_memory_instruction_count >= 1 &&
-	                result.unrelocatable_memory_instruction_count == 0,
-	      "static patcher did not cover the faultable instruction");
-	Check(test, patched_preserved, "patched fault still corrupted the guest red zone");
+	Check(test, reproduced, "test harness did not reproduce red-zone corruption");
+	Check(test, covered, "static patcher did not cover the faultable instruction");
+	Check(test, preserved, "patched fault still corrupted the guest red zone");
 	Check(test, freed, "failed to free patch test code");
 	std::printf("[host]    %-48s ok\n", test);
 }
@@ -1949,6 +1967,122 @@ void TestDirectMapUnmapReusesHostAddress() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+#if defined(__linux__)
+void TestFixedDirectReplacementPreservesAccess() {
+	using namespace Libs::LibKernel::Memory;
+	using Common::VirtualMemory::Mode;
+	const char* test = "FixedDirectReplacementPreservesAccess";
+	constexpr uint64_t page = SceKernelPageSize;
+	int64_t physical = 0;
+	CheckOk(test, KernelAllocateDirectMemory(0, KernelGetDirectMemorySize(), page * 4, page,
+	                                        SceKernelMtypeC, &physical),
+	        "KernelAllocateDirectMemory");
+	void* alias = nullptr;
+	void* address = nullptr;
+	CheckOk(test, KernelMapNamedDirectMemory(&alias, page * 4, SceKernelProtCpuRw, 0, physical,
+	                                        page, "replacement_alias"),
+	        "KernelMapNamedDirectMemory(alias)");
+	CheckOk(test, KernelMapNamedDirectMemory(&address, page * 3, SceKernelProtCpuRw, 0, physical,
+	                                        page, "replacement_original"),
+	        "KernelMapNamedDirectMemory(original)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	const auto alias_base = reinterpret_cast<uint64_t>(alias);
+	for (uint64_t i = 0; i < 4; ++i) {
+		*reinterpret_cast<uint64_t*>(alias_base + i * page) = (i + 1) * 0x11;
+	}
+	const auto replace = [&](uint64_t offset, uint64_t size, int protection, const char* name) {
+		void* middle = reinterpret_cast<void*>(base + page);
+		const int result = KernelMapNamedDirectMemory(&middle, size, protection, SceKernelMapFixed,
+		                                             physical + offset, page, name);
+		if (result == OK) {
+			Check(test, reinterpret_cast<uint64_t>(middle) == base + page,
+			      "fixed replacement moved the destination");
+		}
+		return result;
+	};
+	static unsigned observations = 0;
+	observations = 0;
+	const auto observe_old_mapping = [](uintptr_t start, size_t size) {
+		++observations;
+		uint64_t value = 0;
+		ssize_t bytes = -1;
+		std::thread reader([&] {
+			iovec local {&value, sizeof(value)};
+			iovec remote {reinterpret_cast<void*>(start), sizeof(value)};
+			bytes = process_vm_readv(getpid(), &local, 1, &remote, 1, 0);
+		});
+		reader.join();
+		Check("FixedDirectReplacementPreservesAccess",
+		      size == SceKernelPageSize && bytes == sizeof(value) && value == 0x22,
+		      "fixed replacement exposed an inaccessible or changed old mapping before mmap");
+	};
+	Check(test, ProtectGuestHostMemory(base, page, Mode::Read) &&
+	                ProtectGuestHostMemory(base + page * 2, page, Mode::NoAccess),
+	      "could not protect neighboring pages");
+	TestBeforeNextBackingMap(observe_old_mapping);
+	CheckOk(test, replace(page, page, SceKernelProtCpuRead, "replacement_same"),
+	        "KernelMapNamedDirectMemory(same backing)");
+	ExpectRange(test, Query(test, base + page), base + page, base + page * 2,
+	            SceKernelProtCpuRead, 0, 1, 0, 1, "replacement_same", physical + page);
+	TestBeforeNextBackingMap(observe_old_mapping);
+	CheckOk(test, replace(page * 3, page, SceKernelProtCpuRw, "replacement_changed"),
+	        "KernelMapNamedDirectMemory(different backing)");
+	Check(test, observations == 2, "pre-map observations did not execute");
+	Check(test, *reinterpret_cast<uint64_t*>(base + page) == 0x44,
+	      "replacement did not select the new physical offset");
+	*reinterpret_cast<uint64_t*>(base + page) = 0x45;
+	Check(test, *reinterpret_cast<uint64_t*>(alias_base + page * 3) == 0x45 &&
+	                *reinterpret_cast<uint64_t*>(alias_base + page) == 0x22,
+	      "replacement lost aliasing or changed the previous backing");
+	uint64_t value = 0;
+	iovec local {&value, sizeof(value)};
+	iovec left {reinterpret_cast<void*>(base), sizeof(value)};
+	iovec right {reinterpret_cast<void*>(base + page * 2), sizeof(value)};
+	Check(test, process_vm_readv(getpid(), &local, 1, &left, 1, 0) == sizeof(value) &&
+	                value == 0x11 && process_vm_writev(getpid(), &local, 1, &left, 1, 0) == -1 &&
+	                process_vm_readv(getpid(), &local, 1, &right, 1, 0) == -1,
+	      "middle replacement changed a neighboring page's bytes or host protection");
+	Check(test, ProtectGuestHostMemory(base, page * 3, Mode::ReadWrite),
+	      "could not restore neighboring permissions");
+	const auto check_preserved = [&] {
+		Check(test, *reinterpret_cast<uint64_t*>(base) == 0x11 &&
+		                *reinterpret_cast<uint64_t*>(base + page) == 0x45 &&
+		                *reinterpret_cast<uint64_t*>(base + page * 2) == 0x33,
+		      "failed replacement changed bytes");
+		for (uint64_t i = 0; i < 3; ++i) {
+			Check(test, !TestPlaceholderRangeIsFree(base + i * page, page),
+			      "failed replacement released part of the owned mapping");
+		}
+		ExpectRange(test, Query(test, base), base, base + page, SceKernelProtCpuRw,
+		            0, 1, 0, 1, "replacement_original", physical);
+		ExpectRange(test, Query(test, base + page), base + page, base + page * 2,
+		            SceKernelProtCpuRw, 0, 1, 0, 1, "replacement_changed", physical + page * 3);
+		ExpectRange(test, Query(test, base + page * 2), base + page * 2, base + page * 3,
+		            SceKernelProtCpuRw, 0, 1, 0, 1, "replacement_original", physical + page * 2);
+	};
+	TestFailPhysicalMemoryUnmapAfter(1);
+	CheckFailed(test, replace(page, page * 2, SceKernelProtCpuRead, "replacement_span"),
+	            "KernelMapNamedDirectMemory(second-view removal failure)");
+	check_preserved();
+	TestFailNextVirtualRangeReplacement();
+	CheckFailed(test, replace(page, page * 2, SceKernelProtCpuRead, "replacement_span"),
+	            "KernelMapNamedDirectMemory(publication failure)");
+	check_preserved();
+	CheckOk(test, replace(page, page * 2, SceKernelProtCpuRead, "replacement_span"),
+	        "KernelMapNamedDirectMemory(multiple backing views)");
+	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x11 &&
+	                *reinterpret_cast<uint64_t*>(base + page) == 0x22 &&
+	                *reinterpret_cast<uint64_t*>(base + page * 2) == 0x33,
+	      "multi-view replacement changed the prefix or chose incorrect backing offsets");
+	ExpectRange(test, Query(test, base + page), base + page, base + page * 3,
+	            SceKernelProtCpuRead, 0, 1, 0, 1, "replacement_span", physical + page);
+	CheckOk(test, KernelMunmap(base, page * 3), "KernelMunmap(replacement)");
+	CheckOk(test, KernelMunmap(alias_base, page * 4), "KernelMunmap(alias)");
+	CheckOk(test, KernelReleaseDirectMemory(physical, page * 4), "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+#endif
+
 void TestFixedReserveReplacesPartialDirectMapping() {
 	const char*        test         = "FixedReserveReplacesPartialDirectMapping";
 	constexpr uint64_t page_count   = 13;
@@ -4044,6 +4178,12 @@ void TestSmallFiberStacksAndMigration() {
 
 int main(int argc, char** argv) {
 	InitSubsystems();
+#if defined(__linux__)
+	if (argc == 2 && std::strcmp(argv[1], "--fixed-direct-replacement-only") == 0) {
+		RunTest(TestFixedDirectReplacementPreservesAccess);
+		return g_failed_tests == 0 ? 0 : 1;
+	}
+#endif
 #if defined(__x86_64__) || defined(_M_X64)
 	if (argc == 2 && std::strcmp(argv[1], "--fiber-only") == 0) {
 		RunTest(TestSmallFiberStacksAndMigration);
@@ -4121,6 +4261,9 @@ int main(int argc, char** argv) {
 	RunTest(TestHintlessDirectMapUsesCanonicalGuestBase);
 	RunTest(TestDirectMemoryContentPersistsAcrossRemap);
 	RunTest(TestDirectMapUnmapReusesHostAddress);
+#if defined(__linux__)
+	RunTest(TestFixedDirectReplacementPreservesAccess);
+#endif
 	RunTest(TestFixedReserveReplacesPartialDirectMapping);
 	RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);
 	RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);

tests/shaderCfgTests.cpp

@@ -1817,28 +1817,71 @@ void TestNggVertexEntryState() {
   }
 }
 
-void TestNewShaderRecompilerSopkWaitcntMarkers() {
-  const uint32_t shader[] = {
-      EncodeSopk(0x17, 125, 0xffff), // s_waitcnt_vscnt null, 0xffff
-      EncodeSopk(0x18, 125, 0),      // s_waitcnt_vmcnt null, 0
-      EncodeSopk(0x19, 125, 0),      // s_waitcnt_expcnt null, 0
-      EncodeSopk(0x1a, 125, 0),      // s_waitcnt_lgkmcnt null, 0
-      EncodeSopp(0x01, 0),           // s_endpgm
-  };
+void TestNativeStoreCompletion() {
+  using namespace ShaderRecompiler;
+  const std::array register_wait = {EncodeSopk(0x17, 4, static_cast<int16_t>(0x9234)), EncodeSopp(0x01)};
+  Decoder::Program decoded;
+  Decoder::DecodeProgram(register_wait, decoded);
+  const auto &wait = decoded.instructions.front();
+  Check(wait.opcode == Decoder::Opcode::S_WAITCNT_VSCNT &&
+            wait.src0.kind == Decoder::OperandKind::Sgpr && wait.src0.reg == 4 &&
+            wait.src1.value == 0x9234u,
+        "store completion lost its counter kind, source, or unsigned immediate");
 
   auto options = MakeCompileOptions(ShaderType::Compute);
   options.dump_ir = true;
+  ShaderComputeInputInfo compute{};
+  compute.threads_num[1] = compute.threads_num[2] = 1;
+  options.input_info.compute = &compute;
+  for (const uint32_t wave_size : {32u, 64u}) {
+    compute.threads_num[0] = compute.wave_size = options.wave_size = wave_size;
+    const std::array barrier_only = {0xbbfd0000u, EncodeSopp(0x01)};
+    const auto minimal = RecompileForTest(barrier_only, options);
+    CheckSpirvBinaryValidates(minimal.spirv);
+    Check(SpirvContainsCapability(minimal.spirv, 61),
+          "standalone store completion lacks GroupNonUniform capability");
 
-  auto result = RecompileForTest(shader, options);
-  Check((result.decoded_dump.find("s_waitcnt 0") != std::string::npos),
-        "new decoder did not decode SOPK waitcnt marker");
-  Check((result.decoded_dump.find("s_waitcnt 65535") != std::string::npos),
-        "SOPK waitcnt marker immediate was not kept unsigned");
-  Check((result.ir_dump.find("Waitcnt null, 0x00000000") != std::string::npos),
-        "SOPK waitcnt did not lower to an IR marker");
-  Check((result.ir_dump.find("Waitcnt null, 0x0000ffff") != std::string::npos),
-        "SOPK waitcnt marker immediate was not translated as 16-bit unsigned");
-  CheckSpirvBinaryValidates(result.spirv);
+    // Captured 5b10:0x2464 drains node stores before the next parent atomic.
+    const std::array publication = {
+        EncodeVop1(0x01, 1, 129),
+        EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(1, 0, 0),
+        0xbbfd0000u,
+        EncodeMubuf0(0x32, 4, false, true), EncodeMubuf1(1, 0, 0),
+        EncodeSopp(0x01),
+    };
+    const auto result = RecompileForTest(publication, options);
+    CheckSpirvBinaryValidates(result.spirv);
+    const auto store = result.ir_dump.find("StoreBufferU32");
+    const auto completion = result.ir_dump.find("StoreCompletion");
+    const auto atomic = result.ir_dump.find("BufferAtomicIAdd32");
+    Check(store < completion && completion < atomic && atomic != std::string::npos,
+          "store completion moved outside its native publication order");
+    const auto source = DisassembleSpirvBinary(result.spirv);
+    const auto barrier = source.find("OpControlBarrier %uint_3 %uint_1 %uint_2116");
+    Check(barrier != std::string::npos &&
+              source.find("OpStore") < barrier &&
+              barrier < source.find("OpAtomicIAdd") &&
+              source.find("OpControlBarrier", barrier + 1) == std::string::npos,
+          "store completion must synchronize the subgroup once after both wave halves");
+  }
+
+  const std::array no_store_wait = {
+      EncodeSopk(0x17, 125, 0xffff), // Effective VSCNT 63 waits for nothing.
+      EncodeSopk(0x18, 125, 0), EncodeSopk(0x19, 125, 0),
+      EncodeSopk(0x1a, 125, 0), EncodeSopp(0x01),
+  };
+  const auto no_wait = RecompileForTest(no_store_wait, options);
+  const auto source = DisassembleSpirvBinary(no_wait.spirv);
+  Check(source.find("OpControlBarrier") == std::string::npos &&
+            source.find("OpMemoryBarrier") == std::string::npos,
+        "other wait counters or VSCNT 63 became a store completion barrier");
+#ifndef _WIN32
+  for (const uint32_t word : {EncodeSopk(0x17, 125, 1), register_wait[0]}) {
+    const std::array unsupported = {word, EncodeSopp(0x01)};
+    ExpectFatal([&] { (void)RecompileForTest(unsupported, options); },
+                "unsupported partial or register-based store wait was dropped");
+  }
+#endif
 }
 
 void TestSopkCompareImmediateExtension() {
@@ -3536,6 +3579,48 @@ void TestNewShaderRecompilerCapturedVop1SdwaByteConvert() {
         "V_CVT_F16_U16 accepted unimplemented SDWA byte sign extension");
 }
 
+void TestNewShaderRecompilerCapturedFractF64() {
+  using namespace ShaderRecompiler::Decoder;
+  const uint32_t captured[] = {0x7e087d04u};
+  Instruction decoded;
+  DecodeInstruction(captured, 0, decoded);
+  Check(decoded.family == Family::VOP1 && decoded.opcode == Opcode::V_FRACT_F64 &&
+            decoded.opcode_id == 0x3e && decoded.word_count == 1 &&
+            decoded.dst.kind == OperandKind::Vgpr && decoded.dst.reg == 4 &&
+            decoded.src0.kind == OperandKind::Vgpr && decoded.src0.reg == 4,
+        "captured V_FRACT_F64 did not decode its in-place register pair");
+  for (const uint32_t opcode : {0x3eu, 0x2fu}) {
+    for (const uint32_t source : {249u, 250u}) {
+      const uint32_t invalid[] = {EncodeVop1(opcode, 4, source),
+          source == 249u ? 0x00060604u : EncodeVop1Dpp(4, 0xe4)};
+      DecodeInstruction(invalid, 0, decoded);
+      Check(decoded.opcode == Opcode::UNSUPPORTED,
+            "FP64 VOP1 accepted an unsupported SDWA or DPP encoding");
+    }
+  }
+}
+
+void TestNewShaderRecompilerVop1SdwaBfrev() {
+  using namespace ShaderRecompiler::Decoder;
+  const uint32_t captured[] = {0x7e0070f9u, 0x00040600u};
+  Instruction decoded;
+  DecodeInstruction(captured, 0, decoded);
+  Check(decoded.opcode == Opcode::V_BFREV_B32 && decoded.word_count == 2 &&
+            decoded.dst.kind == OperandKind::Vgpr && decoded.dst.reg == 0 &&
+            decoded.dst.sdwa_sel == 6 && decoded.src0.kind == OperandKind::Vgpr &&
+            decoded.src0.reg == 0 && decoded.src0.sdwa_sel == 4 &&
+            !decoded.src0.sdwa_sext,
+        "captured V_BFREV_B32 SDWA source metadata is incorrect");
+  for (const uint32_t modifier : {0x00070600u, 0x00140600u, 0x00240600u,
+                                 0x00042600u, 0x00044600u, 0x00040400u,
+                                 0x00041e00u}) {
+    const uint32_t invalid[] = {captured[0], modifier};
+    DecodeInstruction(invalid, 0, decoded);
+    Check(decoded.word_count == 2 && decoded.opcode == Opcode::UNSUPPORTED,
+          "V_BFREV_B32 SDWA accepted unsupported selectors or modifiers");
+  }
+}
+
 void TestNewShaderRecompilerVop1SdwaNotDestination() {
   auto options = MakeCompileOptions(ShaderType::Pixel);
 
@@ -4529,6 +4614,16 @@ void TestNewShaderDecoderArchitecture() {
   Check(ds.opcode == Opcode::DS_READ_B32 && ds.gds,
         "DS decoder lost the GFX10 opcode or GDS fields");
 
+  const uint32_t masked_or_code[] = {0xd8300144u, 0x000a0802u};
+  Instruction masked_or;
+  ShaderRecompiler::Decoder::DecodeInstruction(masked_or_code, 0u, masked_or);
+  Check(masked_or.opcode == Opcode::DS_MSKOR_B32 && !masked_or.gds &&
+            masked_or.word_count == 2u && masked_or.src_count == 3u &&
+            masked_or.offset == 0x144u && masked_or.src0.reg == 2u &&
+            masked_or.src1.reg == 8u && masked_or.src2.reg == 10u &&
+            masked_or.data_dwords == 1u && masked_or.data_bits == 32u,
+        "DS decoder misdecoded captured masked OR operands");
+
   for (bool decrement : {false, true}) {
     const uint32_t words[] = {decrement ? 0xd8920004u : 0xd88e0004u, 0x03000302u};
     Instruction atomic;
@@ -4628,8 +4723,8 @@ void TestNewShaderDecoderArchitecture() {
 
 void TestNewShaderRecompilerRejectsDppOn64BitCompares() {
   const uint32_t opcodes[] = {
-      0xa2u, 0xb5u, 0xe2u, 0xe4u, 0xe5u,
-      0xf5u}; // eq_i64, cmpx_ne_i64, eq_u64, gt_u64, ne_u64, cmpx_ne_u64
+      0xa2u, 0xa5u, 0xb5u, 0xe2u, 0xe4u, 0xe5u,
+      0xf5u}; // eq_i64, ne_i64, cmpx_ne_i64, eq_u64, gt_u64, ne_u64, cmpx_ne_u64
   for (const auto opcode : opcodes) {
     const uint32_t shader[] = {
         EncodeVopc(opcode, 250u, 0u), // DPP escape in SRC0
@@ -4647,6 +4742,13 @@ void TestNewShaderRecompilerRejectsDppOn64BitCompares() {
     Check((compare.unsupported_reason.find("VOPC DPP modifier is not supported for opcode") != std::string::npos),
           "64-bit VOPC DPP rejection reason was not explicit");
   }
+  const uint32_t sdwa[] = {EncodeVopc(0xa5u, 249u, 0u), 0x06060000u};
+  ShaderRecompiler::Decoder::Instruction compare;
+  ShaderRecompiler::Decoder::DecodeInstruction(sdwa, 0u, compare);
+  Check(compare.opcode == ShaderRecompiler::Decoder::Opcode::UNSUPPORTED &&
+            compare.word_count == 2u &&
+            compare.unsupported_reason.find("VOPC SDWA modifier is not supported") != std::string::npos,
+        "V_CMP_NE_I64 accepted an illegal SDWA encoding");
 }
 
 void TestNewShaderRecompilerCapturedVopcSdwaCmpxClass() {
@@ -7809,12 +7911,11 @@ void TestNewShaderRecompilerCfgLoopBreakContinue() {
   CheckSpirvBinaryValidates(result.spirv);
 }
 
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
 void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() {
   const uint32_t shader[] = {
       EncodeSMovB32(0, 128), // preheader: s0 = 0
       EncodeSmem0(0x08, 8, 4),
-      0u,                          // loop: s_buffer_load_dword s8, s[4:7]
+      0u,                          // loop: s_buffer_load_dword s8, s[8:11], s0
       EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1
       EncodeSopc(0x0a, 0, 130),    // s_cmp_lt_u32 s0, 2
       EncodeSopp(0x05, 0xfffbu),   // s_cbranch_scc1 loop
@@ -7824,14 +7925,25 @@ void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() {
       0xbf810000u,
   };
 
+  std::array<uint32_t, 64> user_data{};
+  for (const uint32_t base : {8u, 48u}) {
+    user_data[base] = base == 8u ? 0x1000u : 0x2000u;
+    user_data[base + 2] = 64u;
+    user_data[base + 3] = 0x30005204u;
+  }
   auto options = MakeCompileOptions(ShaderType::Compute);
-  options.dump_ir = true;
-  ExpectFatal([&] { (void)RecompileForTest(shader, options); },
-              "self-modifying scalar-buffer descriptor did not terminate "
-              "compilation");
+  options.user_data = user_data;
+  const auto result = RecompileForTest(shader, options);
+  Check(result.program.info.uses_dma && !result.program.dispatcher_fallback &&
+            SpirvContainsOpcode(result.spirv, 246),
+        "self-modifying scalar descriptor lost its structured GPU loop");
+  Check(result.program.info.buffers.size() == 1u && result.program.info.buffers[0].written &&
+            result.resources.flattened_srt.empty(),
+        "self-modifying scalar descriptor was flattened or bound on the host");
+  CheckSpirvBinaryValidates(result.spirv);
 }
 
-void TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher() {
+void TestNewShaderRecompilerCfgLoopHeaderBufferLoadStructured() {
   const uint32_t shader[] = {
       EncodeSMovB32(0, 128), // preheader: s0 = 0
       EncodeMubuf0(0x0c),
@@ -7844,13 +7956,15 @@ void TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher() {
       0xbf810000u,
   };
 
-  auto options = MakeCompileOptions(ShaderType::Compute);
-  options.dump_ir = true;
-  ExpectFatal([&] { (void)RecompileForTest(shader, options); },
-              "self-modifying vector-buffer descriptor did not terminate "
-              "compilation");
+  const auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute));
+  Check(result.program.info.uses_dma && !result.program.dispatcher_fallback &&
+            SpirvContainsOpcode(result.spirv, 246),
+        "loop-carried vector descriptor lost its structured GPU loop");
+  Check(result.program.info.buffers.size() == 1u && result.program.info.buffers[0].written &&
+            result.resources.flattened_srt.empty() && result.resources.specialization_reads.empty(),
+        "loop-carried vector descriptor was evaluated on the host");
+  CheckSpirvBinaryValidates(result.spirv);
 }
-#endif
 
 void TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured() {
   const uint32_t shader[] = {
@@ -8745,6 +8859,46 @@ void TestNativeScalarReadDescriptorPlanning() {
   }
 }
 
+void TestScalarSelectedVccBufferAddress() {
+  using namespace ShaderRecompiler::IR;
+  // 68ca7b6fd9c44cfa selects an ordinary 64-bit address through VCC before
+  // copying its words into a writable buffer descriptor.
+  const uint32_t shader[] = {
+      EncodeSopc(0x07, 4, 128), EncodeSop2(0x0b, 106, 0, 2),
+      EncodeVop1(0x01, 1, 106), EncodeVop1(0x02, 8, 257),
+      EncodeSMovB32(9, 107), EncodeSMovB32(10, 132),
+      EncodeSMovB32(11, 255), 3u << 28u,
+      EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(0, 2, 0),
+      EncodeSopp(0x01),
+  };
+  std::array<uint32_t, 5> userdata{0x12345670u, 0x12u, 0xb27c0000u, 0x30u, 0u};
+  for (const uint32_t wave_size : {32u, 64u}) {
+    auto options = MakeCompileOptions(ShaderType::Compute);
+    auto input = *options.input_info.compute;
+    input.threads_num[0] = input.wave_size = options.wave_size = wave_size;
+    options.input_info.compute = &input;
+    options.user_data = userdata;
+    auto translated = ShaderRecompiler::TranslateProgram(shader, options);
+    auto plan = ExtractResourcePlan(translated.program);
+    ResourceSnapshot snapshot;
+    ResourceSpecialization specialization;
+    for (const uint32_t condition : {0u, 1u}) {
+      userdata[4] = condition;
+      Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization),
+            "scalar-selected VCC address did not materialize");
+      const auto offset = condition ? 0u : 2u;
+      Check(snapshot.buffers.size() == 1 && snapshot.flattened_srt.empty() &&
+                snapshot.specialization_reads.empty() &&
+                snapshot.buffers[0].dwords[0] == userdata[offset] &&
+                snapshot.buffers[0].dwords[1] == userdata[offset + 1],
+            "VCC selection replaced raw address words with a lane ballot");
+    }
+    const auto compiled = ShaderRecompiler::CompileProgram(
+        std::move(translated), options, specialization);
+    CheckSpirvBinaryValidates(compiled.spirv);
+  }
+}
+
 void TestNativeGuardedSamplerSource() {
   constexpr uint32_t nested = 12, sample = 18, join = 20, end = 22;
   const uint32_t shader[] = {
@@ -8785,8 +8939,113 @@ void TestNativeGuardedSamplerSource() {
   }
 }
 
-void TestNativeDescriptorProvenanceRejectsGpuSelection() {
+void TestBoundedMaterialBufferStores(bool scalar_key) {
+  // Captured batch search and descriptor waterfall, with deformation arithmetic removed.
+  const uint32_t shader[] = {
+      0xbfa00003u, 0xd7460002u, 0x04010a04u, 0xf4080101u,
+      0xfa000080u, 0xbe8e0380u, 0xbe8f037eu, 0xbf8cc07fu,
+      0xf4240302u, 0xfa000014u, 0xf4080201u, 0xfa000020u,
+      0xbf880019u, 0x7e3602c1u, 0xbf8cc07fu, 0xbf0a0d0eu,
+      0xbf800000u, 0x8506807eu, 0xbeea0306u, 0xbefe0306u,
+      0xbf860011u, 0x816b0e0cu, 0x810e810eu, 0x936aff6bu,
+      0x00000050u, 0x7e36026bu, 0x816aa46au, 0xf4240104u,
+      0xd4000000u, 0xbf8cc07fu, 0xbeea0404u, 0x816a0405u,
+      0x7d8604f9u, 0x06868405u, 0x7d88046au, 0x876a6a04u,
+      0x8a7e6a06u, 0xbf89ffe6u, 0xbefe030fu, 0x7daa36c1u,
+      0xbf880013u, 0xbf8cc07fu, 0xe0342030u, 0x8002171bu,
+      0xf4080100u, 0xfa000020u, 0x7e020281u, 0x7ed40518u,
+      0xbeeb037eu, 0x7da4306au, 0x8f6a846au, 0x7e020280u,
+      0xf4280002u, 0xd4000000u, 0xbf8cc07fu, 0xe0701000u,
+      0x80003405u, 0xbefe036bu, 0x7daa0280u, 0xbf89fff3u,
+      0xbf810000u,
+  };
+  std::array<uint32_t, 8 * 20> material{};
+  std::array<uint32_t, 16 * 4> table{};
+  std::array<uint32_t, 8> batch{};
+  std::array<uint32_t, 12> table_root{};
+  std::array<uint32_t, 36> batch_root{};
+  const auto buffer = [](uint32_t* words, const void* data, uint32_t stride,
+                         uint32_t count, uint32_t mode = 0u) {
+    const auto address = reinterpret_cast<uint64_t>(data);
+    words[0] = static_cast<uint32_t>(address);
+    words[1] = static_cast<uint32_t>(address >> 32u) | (stride << 16u);
+    words[2] = count;
+    words[3] = 0x5004u | (mode << 28u);
+  };
+  batch[5] = 1u;
+  batch[6] = 2u;
+  material[20 + 9] = 32u;
+  material[20 + 13] = 3u;
+  material[40 + 9] = 32u;
+  material[40 + 10] = 32u;
+  material[40 + 13] = 5u;
+  buffer(table.data() + 12u, reinterpret_cast<void*>(0x100000u), 1u, 128u, 3u);
+  buffer(table.data() + 20u, reinterpret_cast<void*>(0x200000u), 1u, 128u, 3u);
+  buffer(table_root.data() + 8u, table.data(), 16u, 16u);
+  buffer(batch_root.data() + 8u, material.data(), 80u, 8u);
+  buffer(batch_root.data() + 32u, batch.data(), 16u, 2u);
+  const auto table_address = reinterpret_cast<uint64_t>(table_root.data());
+  const auto batch_address = reinterpret_cast<uint64_t>(batch_root.data());
+  const std::array<uint32_t, 4> user_data{
+      static_cast<uint32_t>(table_address), static_cast<uint32_t>(table_address >> 32u),
+      static_cast<uint32_t>(batch_address), static_cast<uint32_t>(batch_address >> 32u)};
+  auto options = MakeCompileOptions(ShaderType::Compute);
+  options.wave_size = 32u;
+  options.user_data = user_data;
+  auto code = std::to_array(shader);
+  if (scalar_key) {
+    code[0xa8u / 4u] = EncodeMubuf0(0x0cu, 52u);
+    code[0xacu / 4u] = EncodeMubuf1(24u, 2u, 27u);
+  }
+  auto translated = ShaderRecompiler::TranslateProgram(code, options);
+  auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
+  ShaderRecompiler::IR::ResourceSnapshot snapshot;
+  ShaderRecompiler::IR::ResourceSpecialization specialization;
+  const ShaderRecompiler::IR::SrtRuntime runtime{
+      .user_data = user_data, .read_memory = ReadHostTestMemory,
+      .read_specialization_memory = ReadHostTestMemory};
+  const auto materialize = [&] {
+    return ShaderRecompiler::IR::MaterializeResources(plan, runtime, snapshot, specialization);
+  };
+  Check(materialize(), "bounded material range failed to materialize");
+  const auto& buffers = specialization.buffers;
+  const auto root = std::ranges::find_if(buffers, [](const auto& resource) {
+    return resource.indirect_root != ShaderRecompiler::IR::BufferResource::NoIndirectBuffer;
+  });
+  Check(root != buffers.end() && snapshot.flattened_srt[root->indirect_mapping_offset] == 2u &&
+            buffers.size() == plan.info.buffers.size() + 1u,
+        "bounded material range did not select exactly its two writable buffers");
+  const auto root_index = static_cast<size_t>(root - buffers.begin());
+  const auto first_specialization = specialization;
+  table[12] += 0x1000u;
+  Check(materialize() && specialization == first_specialization,
+        "buffer address changes created a shader permutation");
+  batch[6] = 0u;
+  Check(materialize() && specialization.buffers[root_index].indirect_root ==
+            ShaderRecompiler::IR::BufferResource::NoIndirectBuffer &&
+            std::ranges::all_of(snapshot.buffers[root_index].dwords,
+                               [](uint32_t word) { return word == 0u; }) &&
+            specialization.buffers.size() == plan.info.buffers.size(),
+        "empty material batch retained writable candidates");
+  batch[6] = 2u;
+  batch[5] = 7u;
+  Check(!materialize(), "material batch exceeded the descriptor record count");
+  batch[5] = 1u;
+  batch_root[11] |= 1u << 30u;
+  Check(!materialize(), "material selection accepted a non-buffer descriptor type");
+  batch_root[11] &= ~(1u << 30u);
+  Check(materialize(), "restored material batch failed to materialize");
+  auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, specialization, 0u);
+  CheckSpirvBinaryValidates(result.spirv);
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
+  auto unbounded = code;
+  unbounded[0x3cu / 4u] = EncodeSopc(0x0bu, 14u, 13u); // s_cmp_le_u32 admits wraparound.
+  ExpectFatal([&] { ShaderRecompiler::TranslateProgram(unbounded, options); },
+              "material selection accepted an inclusive induction bound");
+#endif
+}
+
+void TestNativeDescriptorProvenanceKeepsGpuSelection() {
   const std::array<uint32_t, 13> selected{
       EncodeSmem0(0x02, 8, 6), 125u << 25u,
       EncodeVop1(0x01, 1, 128),
@@ -8812,7 +9071,15 @@ void TestNativeDescriptorProvenanceRejectsGpuSelection() {
       EncodeSopp(0x07, 0xfff6u),
       EncodeSopp(0x01),
   };
+  const std::array<uint32_t, 8> table{0x1000u, 0u, 4u, 3u << 28u,
+                                     0x2000u, 0u, 4u, 3u << 28u};
   std::array<uint32_t, 14> user_data{};
+  user_data[0] = 0x3000u;
+  user_data[2] = 64u;
+  user_data[3] = 0x30005204u;
+  const auto address = reinterpret_cast<uint64_t>(table.data());
+  user_data[12] = static_cast<uint32_t>(address);
+  user_data[13] = static_cast<uint32_t>(address >> 32u);
   ShaderComputeInputInfo input_info{};
   input_info.thread_ids_num = 1;
   input_info.threads_num[0] = 64;
@@ -8821,10 +9088,23 @@ void TestNativeDescriptorProvenanceRejectsGpuSelection() {
   options.input_info.compute = &input_info;
   options.user_data = user_data;
   for (const auto &shader : {selected, partial, loop}) {
-    ExpectFatal([&] { ShaderRecompiler::TranslateProgram(shader, options); },
-                "native descriptor provenance accepted a GPU-selected or loop-carried value");
+    const auto result = RecompileForTest(shader, options, ReadHostTestMemory);
+    Check(result.program.info.uses_dma && result.program.info.buffers.size() == 1u &&
+              result.program.info.buffers[0].written,
+          "GPU-selected scalar descriptor was materialized as a host binding");
+    uint32_t indirect_reads = 0;
+    for (const auto *block : result.program.blocks) {
+      for (const auto &inst : *block) {
+        if (inst.GetOpcode() != ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer) continue;
+        const auto &memory = result.program.memory_info[
+            inst.Flags<ShaderRecompiler::IR::MemoryFlags>().index];
+        indirect_reads += memory.kind == ShaderRecompiler::IR::ResourceKind::IndirectBuffer &&
+                          !memory.planning_only;
+      }
+    }
+    Check(indirect_reads == 1u, "GPU-selected scalar read was flattened or removed");
+    CheckSpirvBinaryValidates(result.spirv);
   }
-#endif
 }
 
 void TestCfgSiblingSharedExit() {
@@ -9924,13 +10204,15 @@ void TestMeshExportStorage() {
       EncodeExp0(0x21, 0xf, false), EncodeExp1(0, 0, 0, 0),
       EncodeExp0(0x22, 0xf, false), EncodeExp1(0, 0, 0, 0),
       EncodeExp0(0x0d, 0x4, false), EncodeExp1(0, 0, 0, 0), // Layer
+      EncodeExp0(0x0e, 0x9, false), EncodeExp1(0, 0, 0, 0), // Clip distances 0, 3
+      EncodeExp0(0x0f, 0x6, false), EncodeExp1(0, 0, 0, 0), // Cull distances 1, 2
       EncodeExp0(0x14, 0x1), EncodeExp1(0, 0, 0, 0), // primitive
       EncodeSopp(0x01),
   };
   std::vector<uint32_t> monolithic(std::begin(front), std::begin(front) + 2);
   monolithic.insert(monolithic.end(), std::begin(back), std::end(back));
   ShaderVertexInputInfo input{};
-  input.pa_cl_vs_out_cntl = (1u << 21u) | (1u << 18u);
+  input.pa_cl_vs_out_cntl = (1u << 21u) | (1u << 18u) | 0x00c0f00fu;
   auto &mesh = input.mesh;
   mesh.threads_num[0] = 192;
   mesh.threads_num[1] = mesh.threads_num[2] = 1;
@@ -9989,6 +10271,19 @@ void TestMeshExportStorage() {
                                               "vsharp %uint_0 %uint_4294967295"),
           "mesh draw prefix was lost or spilled shader data wrapped into push constants");
     const auto &binary = result.spirv;
+    for (const auto builtin : {3u, 4u}) {
+      auto expected = builtin == 3u ? std::vector<uint32_t>{0, 3}
+                                    : std::vector<uint32_t>{1, 2};
+      if (subgroup_size == 32) {
+        expected.push_back(expected[0]);
+        expected.push_back(expected[1]);
+        std::sort(expected.begin(), expected.end());
+      }
+      Check(SpirvDecorationValueCount(binary, 11u, builtin) == 1 &&
+                SpirvStoredBuiltInElements(binary, builtin) == expected &&
+                SpirvContainsCapability(binary, builtin == 3u ? 32u : 33u),
+            "mesh distances lost their shared BuiltIn array, plane index or logical lane");
+    }
     std::vector<uint32_t> sizes(binary[3]), constants(binary[3]);
     uint32_t shared_bytes = 0, private_bytes = 0;
     for (size_t i = 5; i < binary.size(); i += binary[i] >> 16u) {
@@ -10021,7 +10316,7 @@ void TestMeshExportStorage() {
     // when its four vertex exports and primitive exports are shared arrays.
     Check(shared_bytes == 3840u * 4u + 192u * 4u + 8u && shared_bytes <= 28672u,
           "mesh staging must retain guest LDS, shared Layer and allocation within the host budget");
-    Check(private_bytes == (4u * 16u + 4u) * (64u / subgroup_size),
+    Check(private_bytes == (4u * 16u + 4u + 4u * 4u) * (64u / subgroup_size),
           "mesh vertex and primitive exports lost their separate logical-lane storage");
   }
 }
@@ -13025,6 +13320,75 @@ void TestComputeImageFill() {
     Run(mutation);
 }
 
+void TestGpuProducedWritableDescriptor() {
+  using namespace ShaderRecompiler::IR;
+  // Reduced from 994af59d50bedca8: the GPU-built BVH header supplies a relative
+  // output address and record count. CMPX changes EXEC without clobbering VCC.
+  const uint32_t shader[] = {
+      0xf4041a81u, 0xfa000028u, // s_load_dwordx2 vcc, s[2:3], 0x28
+      0xf4000281u, 0xfa000074u, // s_load_dword s10, s[2:3], 0x74
+      0xbe8b03ffu, 0x00016204u, // s_mov_b32 s11, 0x16204
+      0xbf8cc07fu,             // s_waitcnt lgkmcnt(0)
+      0x8008026au,             // s_add_u32 s8, vcc_lo, s2
+      EncodeVopc(0xd5, 129, 0), // v_cmpx_ne_u32 1, v0
+      0x8209036bu,             // s_addc_u32 s9, vcc_hi, s3
+      0xbe891d97u,             // s_bitset1_b32 s9, 23 (stride = 128)
+      EncodeMubuf0(0x1c), EncodeMubuf1(0, 2, 0),
+      EncodeSopp(0x01),
+  };
+  const std::array<uint32_t, 4> user_data{0u, 0u, 0x00315000u, 3u};
+  auto options = MakeCompileOptions(ShaderType::Compute);
+  options.user_data = user_data;
+  auto translated = ShaderRecompiler::TranslateProgram(shader, options);
+  auto plan = ExtractResourcePlan(translated.program);
+  Check(plan.info.buffers.size() == 1 && plan.info.buffers[0].written &&
+            !plan.capture_specialization_reads && plan.srt_reads.size() == 3,
+        "native GPU-built writable descriptor acquired an unnecessary alias proof");
+  struct Reads {
+    std::array<uint32_t, 3> header{0x26c0u, 0u, 151u};
+    uint32_t ordinary = 0;
+    uint32_t strict = 0;
+  } reads;
+  const SrtRuntime runtime{
+      .user_data = user_data,
+      .read_memory = +[](void *data, uint64_t address, std::span<uint32_t> words) {
+        if (words.size() != 1) return false;
+        auto &reads = *static_cast<Reads *>(data);
+        const uint32_t index = address == 0x300315028ull ? 0u
+                             : address == 0x30031502cull ? 1u
+                             : address == 0x300315074ull ? 2u : 3u;
+        if (index == 3u) return false;
+        ++reads.ordinary;
+        words[0] = reads.header[index];
+        return true;
+      },
+      .userdata = &reads,
+      .read_specialization_memory = +[](void *data, uint64_t, std::span<uint32_t>) {
+        ++static_cast<Reads *>(data)->strict;
+        return false; // Current GPU bytes are available only through the ordinary reader.
+      }};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  const auto check = [&](uint32_t low, uint32_t high, uint32_t count, uint32_t reads_expected) {
+    Check(MaterializeResources(plan, runtime, snapshot, specialization),
+          "GPU-produced direct writable descriptor was rejected");
+    Check(snapshot.buffers.size() == 1 &&
+              snapshot.buffers[0].dwords[0] == low &&
+              snapshot.buffers[0].dwords[1] == high &&
+              snapshot.buffers[0].dwords[2] == count &&
+              snapshot.buffers[0].dwords[3] == 0x16204u &&
+              snapshot.specialization_reads.empty() && reads.strict == 0 &&
+              reads.ordinary == reads_expected,
+          "GPU-produced descriptor lost current scalar data or repeated its reads");
+  };
+  check(0x003176c0u, 0x00800003u, 151u, 3u);
+  const auto previous_specialization = specialization;
+  reads.header = {0xffd00000u, 0u, 302u}; // Exercise the low-address carry on refresh.
+  check(0x00015000u, 0x00800004u, 302u, 6u);
+  Check(specialization == previous_specialization,
+        "GPU-updated address or record count created a shader permutation");
+}
+
 void TestTypedDescriptorRealCarryAndScalarLoads() {
   const uint32_t carry_shader[] = {
       EncodeSop1(0x1f, 0, 0),      // s_getpc_b64 s[0:1]
@@ -14328,6 +14692,8 @@ int main() {
   TestNewShaderRecompilerDsReadWrite2Translation();
   TestNewShaderRecompilerDsWideAndAtomicTranslation();
   TestNewShaderRecompilerCapturedVop1SdwaByteConvert();
+  TestNewShaderRecompilerCapturedFractF64();
+  TestNewShaderRecompilerVop1SdwaBfrev();
   TestNewShaderRecompilerVop1SdwaNotDestination();
   TestNewShaderRecompilerScalarMemoryBindingDomains();
   // Opcode semantics and optimized SPIR-V are exercised by
@@ -14338,6 +14704,7 @@ int main() {
   TestNewShaderDecoderArchitecture();
   TestImageAddressOperands();
   TestSopkCompareImmediateExtension();
+  TestNativeStoreCompletion();
   TestNewShaderRecompilerCapturedVopcSdwaCmpxClass();
   TestNewShaderRecompilerCapturedVopcSdwaCmpxLtU16();
   TestNewShaderRecompilerIrLookupMissFailsExplicitly();
@@ -14355,10 +14722,8 @@ int main() {
   TestNewShaderRecompilerCfgTerminalExitMergePS();
   TestNewShaderRecompilerCfgPostEndTargetMergePS();
   TestNewShaderRecompilerCfgLoopBreakContinue();
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
   TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured();
-  TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher();
-#endif
+  TestNewShaderRecompilerCfgLoopHeaderBufferLoadStructured();
   TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured();
   TestNewShaderRecompilerCfgLoopHeaderDsReadStructured();
   TestNewShaderRecompilerCfgLoopHeaderDsRead2B64Structured();
@@ -14382,8 +14747,10 @@ int main() {
   TestSharedExitPreservesNativeDescriptorSources();
   TestNestedSelectionPreservesDescriptorSources();
   TestNativeScalarReadDescriptorPlanning();
+  TestScalarSelectedVccBufferAddress();
   TestNativeGuardedSamplerSource();
-  TestNativeDescriptorProvenanceRejectsGpuSelection();
+  for (const bool scalar_key : {false, true}) TestBoundedMaterialBufferStores(scalar_key);
+  TestNativeDescriptorProvenanceKeepsGpuSelection();
   TestCfgSiblingSharedExit();
   TestNewShaderRecompilerCfgNestedTailEarlyExit();
   TestNewShaderRecompilerCfgSharedReturnAfterNestedSelections();
@@ -14439,6 +14806,7 @@ int main() {
   TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled();
   TestTypedDescriptorRealWideMoveTranslation();
   TestComputeImageFill();
+  TestGpuProducedWritableDescriptor();
   TestTypedDescriptorRealCarryAndScalarLoads();
   TestSrtWalkerRealSmemTranslation();
   TestSrtWalkerVccBaseTranslation();