Capture PCM at set time when multiple AudioOut2 ports share a buffer (#826)

* capture PCM at set time when ports share a buffer

* Snapshot size limit and open device check

* Fall back to PCM pointer on stale snapshot

* audio: own AudioOut2 PCM at attribute set time

---------

Co-authored-by: nmzik <Nmzik@mail.ru>

CMakeLists.txt

@@ -465,6 +465,13 @@ add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
 target_link_libraries(audio_out2_port_tests common fmt::fmt)
 target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
 
+add_executable(ngs2_sampler_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/Ngs2SamplerTests.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
+target_link_libraries(ngs2_sampler_tests common fmt::fmt LibAtrac9)
+target_include_directories(ngs2_sampler_tests PRIVATE ${inc_headers})
+
 add_executable(save_data_memory_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/SaveDataMemoryTests.cpp"
 )
@@ -555,6 +562,7 @@ if(BUILD_TESTING)
 	add_test(NAME kernel_file_system COMMAND $<TARGET_FILE:kernel_file_system_tests>)
 	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
+	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
 	add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME shader_recompiler_alignbyte
@@ -620,6 +628,7 @@ if(BUILD_TESTING)
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
 		audio_out2_port_tests
+		ngs2_sampler_tests
 	)
 endif()
 
@@ -677,7 +686,10 @@ if(KYTY_CLANG_CL)
 	)
 	add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:kyty_emulator>/libwinpthread-1.dll)
 elseif(WIN32 OR LINUX)
-	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
+	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS}")
+	# LINK_FLAGS is a space-delimited string: a -Map path containing spaces would
+	# be split into bogus arguments, so pass it as a discrete link option instead.
+	target_link_options(kyty_emulator PRIVATE "-Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
 endif()
 
 if(LINUX)

README.md

@@ -4,13 +4,13 @@
 [![Build KytyPS5 (Linux)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28Linux%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
 [![Build KytyPS5 (macOS)](https://img.shields.io/github/actions/workflow/status/KytyPS5/KytyPS5/build.yml?branch=main&event=push&label=Build%20KytyPS5%20%28macOS%29)](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
 [![Platform](https://img.shields.io/badge/platform-Windows%20x64%20%7C%20Linux%20x64%20%7C%20macOS%20x86__64-0078D4.svg)](#system-requirements)
-[![Status](https://img.shields.io/badge/status-early%20development-orange.svg)](#current-status)
+[![Status](https://img.shields.io/badge/status-active%20development-orange.svg)](#current-status)
 [![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
 
 KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
 with experimental macOS support. It is based on a heavily modified version of
-[Kyty](https://github.com/InoriRus/Kyty). The project is in an early stage of development, so
-compatibility is limited and behavior may change significantly between builds.
+[Kyty](https://github.com/InoriRus/Kyty). The project is in active development, and behavior
+can change significantly between builds.
 
 > [!IMPORTANT]
 > KytyPS5 is not affiliated with Sony Interactive Entertainment or PlayStation. The project does
@@ -20,12 +20,12 @@ compatibility is limited and behavior may change significantly between builds.
 ## Current Status
 
 KytyPS5 can boot 2D games and a selection of 3D games, including titles built with Unreal Engine
-4/5, Unity, and custom engines. No external low-level emulation modules are currently required.
+4/5, Unity, and custom engines. External low-level emulation modules are neither required nor
+planned.
 
-Development is focused on compatibility and boot reliability.
+Development is currently focused on expanding game compatibility and improving boot reliability.
 
-Windows is the primary platform and receives the most testing. Linux builds and runs; see
-[Building on Linux](#building-on-linux).
+Windows and Linux are the primary platforms and receive the most testing.
 
 macOS support is experimental. The emulator is built for x86-64 and runs on Apple Silicon under
 Rosetta 2, with Vulkan provided by MoltenVK. A small number of titles have been verified in-game
@@ -36,16 +36,16 @@ Community game test results are available in the
 
 ## Bugs and Issues
 
-The project is in an early stage, so please be mindful when opening new issues. Expect crashes,
-graphical glitches, low compatibility, and poor performance.
+Compatibility, stability, and performance can vary between versions. You may encounter crashes
+or graphical glitches, so please include the version you tested when reporting an issue.
 
 ## Screenshots
 
 <table align="center">
   <tr>
     <td align="center">
-      <strong>Disgaea 6</strong><br>
-      <img src="docs/screenshots/ps5-01.png" width="300" alt="Disgaea 6 running in KytyPS5">
+      <strong>Astro Bot</strong><br>
+      <img src="docs/screenshots/ps5-01.png" width="300" alt="Astro Bot running in KytyPS5">
     </td>
     <td align="center">
       <strong>Dreaming Sarah</strong><br>
@@ -64,12 +64,12 @@ graphical glitches, low compatibility, and poor performance.
   </tr>
   <tr>
     <td align="center">
-      <strong>Hellboy</strong><br>
-      <img src="docs/screenshots/ps5-02.png" width="300" alt="Hellboy running in KytyPS5">
+      <strong>Demon's Souls</strong><br>
+      <img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
     </td>
     <td align="center">
-      <strong>Paleo Pines</strong><br>
-      <img src="docs/screenshots/ps5-06.png" width="300" alt="Paleo Pines running in KytyPS5">
+      <strong>Hellboy</strong><br>
+      <img src="docs/screenshots/ps5-06.png" width="300" alt="Hellboy running in KytyPS5">
     </td>
   </tr>
 </table>
@@ -79,7 +79,7 @@ graphical glitches, low compatibility, and poor performance.
 ## Contributing
 
 Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
-issues first, then use the **Game Emulation Bug Report** template and attach the complete log file.
+issues first, then use the **Game Emulation Status Report** template and attach the complete log file.
 
 Code contributions should be focused, build successfully on the platforms they touch, and include
 relevant tests where practical. Windows is the primary target, so a change that alters shared code
@@ -132,7 +132,7 @@ the Vulkan/SPIR-V validation rules.
 ### Build requirements (Windows)
 
 - Git
-- CMake 3.12 or newer
+- CMake 3.22.1 or newer
 - Ninja
 - Visual Studio 2022 or Build Tools 2022 with the **Desktop development with C++** workload and
   **C++ Clang tools for Windows** component
@@ -178,7 +178,7 @@ sudo apt-get install --no-install-recommends \
   libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
 ```
 
-Qt 6 (Concurrent, Network, Widgets) is also required — either the distribution packages
+Qt 6 (Concurrent, Network, Widgets) is required for the launcher — either the distribution packages
 (`qt6-base-dev`) or an official Qt installation.
 
 ```bash
@@ -197,6 +197,18 @@ The install step copies the Qt libraries and plugins next to the binaries, so
 from the pinned [KytyPS5 FFmpeg core](https://github.com/KytyPS5/ext-ffmpeg-core)
 release, including VP9 and WebM support. System FFmpeg packages are not required.
 
+To build `kyty_emulator` and the `kyty_tests` target without Qt, use a separate build directory:
+
+```bash
+git submodule update --init --recursive
+
+cmake -S . -B _Build/linux-no-qt -G Ninja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+  -DKYTY_BUILD_LAUNCHER=OFF
+
+cmake --build _Build/linux-no-qt --target kyty_emulator kyty_tests --parallel
+```
+
 As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
 time.
 
@@ -370,5 +382,6 @@ licenses included with those components.
 
 - [InoriRus/Kyty](https://github.com/InoriRus/Kyty) — KytyPS5 is based on a heavily modified version
   of the original Kyty project.
-- [shadps4-emu/shadPS4](https://github.com/shadps4-emu/shadPS4) — reference for memory-model
-  understanding and the AVPlayer implementation.
+- [shadps4-emu/shadPS4](https://github.com/shadps4-emu/shadPS4) — reference for understanding PS4
+  memory behavior, GPU resource aliasing and cache coherency,
+  and the AVPlayer implementation.

src/common/timer.cpp

@@ -38,16 +38,7 @@ bool Timer::IsPaused() const {
 
 // return time in milliseconds
 double Timer::GetTimeMs() const {
-	if (m_is_paused) {
-		return 1000.0 * (static_cast<double>(m_PauseTime - m_StartTime)) /
-		       static_cast<double>(m_Frequency);
-	}
-
-	uint64_t current_time = 0;
-	SysQueryPerformanceCounter(&current_time);
-
-	return 1000.0 * (static_cast<double>(current_time - m_StartTime)) /
-	       static_cast<double>(m_Frequency);
+	return GetTimeS() * 1000.0;
 }
 
 // return time in seconds

src/graphics/guest_gpu/graphicsRun.cpp

@@ -976,6 +976,16 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 	const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs);
 	EXIT_NOT_IMPLEMENTED(stride_in_bytes < args_size);
 
+	uint64_t index_size = 0;
+	if (indexed) {
+		switch (m_index_type_and_size) {
+			case 0: index_size = 2; break;
+			case 1: index_size = 4; break;
+			case 2: index_size = 1; break;
+			default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
+		}
+	}
+
 	for (uint32_t i = 0; i < draw_count; i++) {
 		const auto args_addr = m_draw_indirect_args_base_addr + data_offset +
 		                       static_cast<uint64_t>(i) * stride_in_bytes;
@@ -993,14 +1003,6 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 		auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr);
 
-		uint64_t index_size = 0;
-		switch (m_index_type_and_size) {
-			case 0: index_size = 2; break;
-			case 1: index_size = 4; break;
-			case 2: index_size = 1; break;
-			default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
-		}
-
 		auto* index_addr = reinterpret_cast<const void*>(
 		    m_index_base_addr + static_cast<uint64_t>(args->start_index_location) * index_size);
 
@@ -1203,8 +1205,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 			}
 			break;
 		case 0x02:
+		case 0x04:
 			if constexpr (sizeof(T) == sizeof(uint32_t)) {
-				if (eop_event_type == 0x2f && event_index == 0x06) {
+				if (event_write_source == 0x02 && eop_event_type == 0x2f && event_index == 0x06) {
 					switch (cache_action) {
 						case 0x00: write32(false); return;
 						case 0x38: write32(true); return;
@@ -1212,6 +1215,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					}
 				}
 			} else {
+				if (event_write_source == 0x04) {
+					value = Sync::ReadReferenceClock();
+				}
 				auto write64 = [&](bool with_writeback) {
 					auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
 					std::memcpy(dst, &value, sizeof(value));
@@ -1268,7 +1274,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 							case 0x14:
 							case 0x28:
 								if (((eop_event_type == 0x04 || eop_event_type == 0x28) &&
-								     event_index == 0x05 && !with_interrupt) ||
+								     event_index == 0x05) ||
 								    (event_index == 0x00)) {
 									write64(true);
 									return;
@@ -1300,45 +1306,6 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 				}
 			}
 			break;
-		case 0x04:
-			if constexpr (sizeof(T) == sizeof(uint64_t)) {
-				const auto clock = Sync::ReadReferenceClock();
-				auto*      dst   = static_cast<uint64_t*>(dst_gpu_addr);
-				std::memcpy(dst, &clock, sizeof(clock));
-				switch (cache_action) {
-					case 0x00:
-						if ((eop_event_type == 0x04 && event_index == 0x05) ||
-						    (eop_event_type == 0x28 && event_index == 0x00)) {
-							if (with_interrupt) {
-								Sync::WriteAtEndOfPipeWithInterrupt64(
-								    m_submit_id, command, dst, clock, m_interrupt_event_id,
-								    interrupt_context_id);
-							} else {
-								Sync::WriteAtEndOfPipeClockCounter(m_submit_id, command,
-								                                   dst, clock);
-							}
-							return;
-						}
-						break;
-					case 0x38:
-						if ((eop_event_type == 0x04 &&
-						     (event_index == 0x00 || event_index == 0x05)) ||
-						    (eop_event_type == 0x28 && event_index == 0x00)) {
-							if (with_interrupt) {
-								Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
-								    m_submit_id, command, dst, clock, m_interrupt_event_id,
-								    interrupt_context_id);
-							} else {
-								Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
-								    m_submit_id, command, dst, clock);
-							}
-							return;
-						}
-						break;
-					default: break;
-				}
-			}
-			break;
 		default: break;
 	}
 

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -107,36 +107,6 @@ constexpr uint64_t NumFramesBeforeRemoval = 32;
 	return true;
 }
 
-[[nodiscard]] const char* BindingTypeName(TextureCache::BindingType type) {
-	switch (type) {
-		case TextureCache::BindingType::Texture: return "Texture";
-		case TextureCache::BindingType::Storage: return "StorageTexture";
-		case TextureCache::BindingType::RenderTarget: return "ColorTarget";
-		case TextureCache::BindingType::DepthTarget: return "DepthTarget";
-		case TextureCache::BindingType::VideoOut: return "VideoOut";
-	}
-	return "Image";
-}
-
-void NameImageBinding(GraphicContext& graphics, Image& image, vk::ImageView view,
-                      TextureCache::BindingType type, const ImageViewInfo& view_info) {
-	const auto* role = BindingTypeName(type);
-	SetVulkanObjectNameF(
-	    graphics.device, image.backing.image,
-	    "Kyty.{}.Image[guest=0x{:016x} size=0x{:x} extent={}x{}x{} format={} mips={} layers={} "
-	    "samples={}]",
-	    role, image.info.data.address, image.info.data.size, image.info.extent.width,
-	    image.info.extent.height, image.info.extent.depth,
-	    static_cast<uint32_t>(image.info.pixel_format), image.info.resources.levels,
-	    image.info.resources.layers, image.info.samples);
-	SetVulkanObjectNameF(
-	    graphics.device, view,
-	    "Kyty.{}.View[guest=0x{:016x} format={} aspect=0x{:x} mip={}+{} layer={}+{}]", role,
-	    image.info.data.address, static_cast<uint32_t>(view_info.format),
-	    static_cast<vk::ImageAspectFlags::MaskType>(view_info.aspect), view_info.base_level,
-	    view_info.level_count, view_info.base_layer, view_info.layer_count);
-}
-
 [[nodiscard]] std::vector<vk::BufferImageCopy> BuildDepthCopies(const ImageInfo& info,
                                                               uint64_t slice_stride,
                                                               vk::ImageAspectFlags aspect) {
@@ -1443,9 +1413,7 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
 			break;
 		default: EXIT("TextureCache: invalid texture binding\n");
 	}
-	const auto view = image.FindView(desc.view_info);
-	NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
-	return view;
+	return image.FindView(desc.view_info);
 }
 
 vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc) {
@@ -1463,9 +1431,7 @@ vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc)
 	RefreshImage(id);
 	CommitGpuWrite(image);
 	TrackImageDownload(id, image);
-	const auto view = image.FindView(desc.view_info);
-	NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
-	return view;
+	return image.FindView(desc.view_info);
 }
 
 vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
@@ -1492,9 +1458,7 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
 	if (desc.info.HasStencil()) {
 		RefreshImage(AssociateStencil(id, desc.info.stencil));
 	}
-	const auto view = image.FindView(desc.view_info);
-	NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
-	return view;
+	return image.FindView(desc.view_info);
 }
 
 void TextureCache::MarkGpuWritten(ImageId id) {

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -41,13 +41,42 @@ struct RenderDepthInfo {
 	[[nodiscard]] vk::ImageAspectFlags AttachmentWriteAspects() const;
 };
 
+inline vk::ImageAspectFlags DepthFeedbackAspects(vk::ImageAspectFlags draw_writes,
+                                                 const ImageViewInfo& target,
+                                                 const ImageViewInfo& sampled) {
+	if (!ImageRangeOverlaps(target.base_level, target.level_count, sampled.base_level,
+	                        sampled.level_count) ||
+	    !ImageRangeOverlaps(target.base_layer, target.layer_count, sampled.base_layer,
+	                        sampled.layer_count)) {
+		return {};
+	}
+	return draw_writes & sampled.aspect;
+}
+
+inline vk::ImageAspectFlags DepthReadableAspects(vk::ImageLayout layout) {
+	switch (layout) {
+		case vk::ImageLayout::eDepthReadOnlyOptimal:
+		case vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal:
+			return vk::ImageAspectFlagBits::eDepth;
+		case vk::ImageLayout::eStencilReadOnlyOptimal:
+		case vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal:
+			return vk::ImageAspectFlagBits::eStencil;
+		case vk::ImageLayout::eDepthStencilReadOnlyOptimal:
+		case vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT:
+		case vk::ImageLayout::eGeneral:
+			return vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil;
+		default:
+			return {};
+	}
+}
+
 inline vk::ImageLayout depth_attachment_layout(const RenderDepthInfo& depth) {
 	const auto available     = ImageViewOps::DepthAspectMask(depth.desc.view_info.format);
 	const auto writes        = depth.AttachmentWriteAspects();
 	const bool has_depth     = static_cast<bool>(available & vk::ImageAspectFlagBits::eDepth);
 	const bool has_stencil   = static_cast<bool>(available & vk::ImageAspectFlagBits::eStencil);
-	// LoadOp clears are separate from guest depth writes.
-	const bool depth_write   = depth.depth_write_enable;
+	// The attachment layout must permit load clears as well as draw writes.
+	const bool depth_write   = static_cast<bool>(writes & vk::ImageAspectFlagBits::eDepth);
 	const bool stencil_write = static_cast<bool>(writes & vk::ImageAspectFlagBits::eStencil);
 	if (!has_stencil) {
 		return depth_write ? vk::ImageLayout::eDepthAttachmentOptimal

src/graphics/host_gpu/renderer/image/image.cpp

@@ -690,6 +690,12 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
 		     create.extent.width, create.extent.height, create.extent.depth,
 		     static_cast<int>(create.format), create.arrayLayers, create.mipLevels);
 	}
+	SetVulkanObjectNameF(
+	    graphics.device, backing.image,
+	    "Kyty.Image[guest=0x{:016x} size=0x{:x} extent={}x{}x{} format={} mips={} layers={} samples={}]",
+	    info.data.address, info.data.size, info.extent.width, info.extent.height, info.extent.depth,
+	    static_cast<uint32_t>(info.pixel_format), info.resources.levels, info.resources.layers,
+	    info.samples);
 }
 
 uint64_t Image::HashGuestEdges() const {

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -329,7 +329,9 @@ struct VideoOutFormatPolicy {
 	VideoOutPixelFormatInfo info;
 };
 
-inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {{
+inline constexpr uint64_t VIDEO_OUT_PIXEL_FORMAT_R10_G10_B10_A2_BT2100_PQ = 0x8100070422000000ull;
+
+inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {{
     {0x8000000022000000ull,
      {vk::Format::eR8G8B8A8Srgb, Prospero::BufferFormat::k8_8_8_8Srgb, 4, false}},
     {0x8000000000000000ull,
@@ -338,6 +340,8 @@ inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {
      {vk::Format::eA2B10G10R10UnormPack32, Prospero::BufferFormat::k10_10_10_2UNorm, 4, false}},
     {0x8100000000000000ull,
      {vk::Format::eA2R10G10B10UnormPack32, Prospero::BufferFormat::k10_10_10_2UNorm, 4, false}},
+    {VIDEO_OUT_PIXEL_FORMAT_R10_G10_B10_A2_BT2100_PQ,
+     {vk::Format::eA2B10G10R10UnormPack32, Prospero::BufferFormat::k10_10_10_2UNorm, 4, false}},
     {0xc001000622000000ull,
      {vk::Format::eR16G16B16A16Sfloat, Prospero::BufferFormat::k16_16_16_16Float, 8, false}},
     {0xc001000600000000ull,

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -391,6 +391,12 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     view_info.level_count, view_info.base_layer, view_info.layer_count,
 		     static_cast<vk::ImageUsageFlags::MaskType>(view_info.usage));
 	}
+	SetVulkanObjectNameF(
+	    m_graphics.device, view,
+	    "Kyty.ImageView[guest=0x{:016x} format={} aspect=0x{:x} mip={}+{} layer={}+{}]",
+	    info.data.address, static_cast<uint32_t>(normalized.format),
+	    static_cast<vk::ImageAspectFlags::MaskType>(normalized.aspect), normalized.base_level,
+	    normalized.level_count, normalized.base_layer, normalized.layer_count);
 	views.push_back({normalized, view});
 	return view;
 }

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -17,6 +17,7 @@
 #include "graphics/host_gpu/hostMemory.h"
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
+#include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
@@ -749,18 +750,27 @@ void RenderExecutor::ResetBindings() {
 	m_bound_images.clear();
 }
 
-PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime) {
+void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
+                                     PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(!runtime);
 	const auto& program  = *runtime.program;
 	const auto& snapshot = *runtime.resources;
-	PreparedBindings prepared;
 	prepared.runtime = &runtime;
-	prepared.images.reserve(program.info.images.size());
+	prepared.gds = {nullptr, 0, VK_WHOLE_SIZE};
+	prepared.flattened_srt = {};
+	prepared.shader_data_buffer = {};
+	prepared.buffer_sources.clear();
+	prepared.buffers.clear();
+	prepared.images.resize(program.info.images.size());
+	prepared.samplers.clear();
+	prepared.shader_data.clear();
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]);
 		BindImage(binding.image_id, binding.desc.type == TextureCache::BindingType::Storage);
-		prepared.images.push_back(std::move(binding));
+		binding.mip_views.swap(prepared.images[i].mip_views);
+		binding.mip_views.clear();
+		prepared.images[i] = std::move(binding);
 	}
 	prepared.samplers.reserve(program.info.samplers.size());
 	for (uint32_t i = 0; i < program.info.samplers.size(); i++) {
@@ -775,7 +785,6 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti
 	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) {
 		prepared.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle();
 	}
-	return prepared;
 }
 
 void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
@@ -989,19 +998,7 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 					    std::ranges::find(image.views, binding.image_view, &CachedImageView::view);
 					EXIT_IF(storage || host_view == image.views.end());
 					const auto aspect = host_view->info.aspect;
-					const bool depth_feedback =
-					    layout == vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT &&
-					    program.stage == ShaderType::Pixel;
-					const bool depth_read =
-					    depth_feedback || layout == vk::ImageLayout::eDepthReadOnlyOptimal ||
-					    layout == vk::ImageLayout::eDepthStencilReadOnlyOptimal ||
-					    layout == vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal;
-					const bool stencil_read =
-					    layout == vk::ImageLayout::eStencilReadOnlyOptimal ||
-					    layout == vk::ImageLayout::eDepthStencilReadOnlyOptimal ||
-					    layout == vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal;
-					if ((aspect & vk::ImageAspectFlagBits::eDepth && !depth_read) ||
-					    (aspect & vk::ImageAspectFlagBits::eStencil && !stencil_read)) {
+					if (aspect & ~DepthReadableAspects(layout)) {
 						EXIT("sampling a writable depth/stencil attachment aspect\n");
 					}
 				}

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -204,6 +204,7 @@ struct PipelineCache::ProgramCache {
 		ShaderRecompiler::IR::ResourceSnapshot       resources;
 		ShaderRecompiler::IR::ResourceSpecialization specialization;
 		std::vector<Permutation>                    permutations;
+		bool                                        skip_dispatch = false;
 	};
 
 	struct ProgramKeyHash {
@@ -283,6 +284,9 @@ struct PipelineCache::ProgramCache {
 		lookup_key.code_size       = static_cast<uint32_t>(params.code.size());
 		BuildStageStaticKey(input_info, lookup_key.static_state);
 		auto                                         entry = programs.find(lookup_key);
+		if (entry != programs.end() && entry->second.skip_dispatch) {
+			return {};
+		}
 		const ShaderRecompiler::IR::SrtRuntime       runtime {
 		    .user_data                  = user_data,
 		    .shader_base                = params.Base(),
@@ -348,6 +352,11 @@ struct PipelineCache::ProgramCache {
 			options.wave_size = input_info.wave_size;
 		}
 		auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
+		if (translated.skip_dispatch) {
+			entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
+			entry->second.skip_dispatch = true;
+			return {};
+		}
 		if (entry == programs.end()) {
 			entry = programs.try_emplace(lookup_key,
 			    ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -368,6 +368,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	// MoltenVK lacks VK_EXT_depth_clip_enable; omit the depth-clip struct on macOS and accept
 	// Vulkan's default depth clipping (enabled) instead of the PS5's clamp behavior.
 #if !defined(__APPLE__)
+	// The DB clamps depth to the viewport range after polygon offset is applied.
+	rasterizer.depthClampEnable = VK_TRUE;
 	rasterizer.pNext = &clip_ext;
 #endif
 	vk::PipelineRasterizationProvokingVertexStateCreateInfoEXT provoking_vertex {};

src/graphics/host_gpu/renderer/render.h

@@ -160,7 +160,7 @@ public:
 	void DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, uint64_t args_addr,
 	                      uint32_t mode);
 
-	[[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime);
+	void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared);
 	void                           FindBuffers(PreparedBindings& bindings);
 	void                           RebindBuffers(PreparedBindings& bindings);
 	void                           RebindImages(PreparedBindings& bindings);
@@ -196,7 +196,8 @@ private:
 	                         bool primitive_restart_enable);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth,
-	                                               const std::optional<PreparedBindings>& pixel = std::nullopt);
+	                                               vk::ImageAspectFlags& feedback_aspects,
+	                                               std::span<PreparedBindings* const> stages = {});
 	[[nodiscard]] bool        ResolveColorTargets(CommandBuffer& buffer,
 	                                              uint32_t render_target_slice_offset);
 	void                      BindImage(ImageId id, bool storage);
@@ -209,6 +210,8 @@ private:
 	                                              uint32_t group_y, uint32_t group_z, uint32_t mode);
 
 	RenderContext&                        m_context;
+	GraphicsBindings                     m_graphics_bindings;
+	PreparedBindings                     m_compute_bindings;
 	std::vector<ImageId>                  m_bound_images;
 	std::vector<vk::DescriptorBufferInfo> m_descriptor_buffers;
 	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -256,6 +256,10 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	input_info.dispatch_thread_dimensions = use_thread_dimensions;
 	const auto compute_program =
 	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
+	if (!compute_program) {
+		// Temporary until RT is implemented.
+		return;
+	}
 	if (use_thread_dimensions) {
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
@@ -362,7 +366,8 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
 	buffer.EndRendering();
 	auto& pipeline =
 	    m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
-	auto bindings = PrepareBindings(input_info.stage);
+	auto& bindings = m_compute_bindings;
+	PrepareBindings(input_info.stage, bindings);
 	FindBuffers(bindings);
 	if (program.info.uses_dma) {
 		m_context.PrepareBda();
@@ -412,9 +417,14 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
 	ShaderComputeInputInfo input_info {};
 	const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
 	    cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
+	if (!compute_program) {
+		// Temporary until RT is implemented.
+		return;
+	}
 	buffer.EndRendering();
 	auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
-	auto bindings = PrepareBindings(input_info.stage);
+	auto& bindings = m_compute_bindings;
+	PrepareBindings(input_info.stage, bindings);
 	FindBuffers(bindings);
 	const auto& program = *input_info.stage.program;
 	if (program.info.uses_dma) {

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -453,8 +453,10 @@ struct DrawCallInfo {
 
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
                                                  uint32_t color_count, RenderDepthInfo& depth,
-                                                 const std::optional<PreparedBindings>& pixel) {
+                                                 vk::ImageAspectFlags& feedback_aspects,
+                                                 std::span<PreparedBindings* const> stages) {
 	EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
+	feedback_aspects = {};
 	auto&       cache = m_context.GetTextureCache();
 	RenderState state {};
 	state.width                 = std::numeric_limits<uint32_t>::max();
@@ -471,15 +473,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		}
 		const auto image_view = cache.FindRenderTarget(target.image_id, target.desc);
 		auto&      image      = cache.GetImage(target.image_id);
-		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
-		                     "Kyty.MRT{}.Image[guest=0x{:016x} size=0x{:x} format={}]",
-		                     target.target_slot, image.info.data.address, image.info.data.size,
-		                     static_cast<uint32_t>(image.info.pixel_format));
-		SetVulkanObjectNameF(m_context.GetGraphics().device, image_view,
-		                     "Kyty.MRT{}.View[guest=0x{:016x} mip={} layer={}+{}]",
-		                     target.target_slot, image.info.data.address,
-		                     target.desc.view_info.base_level, target.desc.view_info.base_layer,
-		                     target.desc.view_info.layer_count);
 		EXIT_IF(image.backing.samples != target.desc.info.samples || image_view == nullptr);
 		const auto& view   = target.desc.view_info;
 		const auto  layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
@@ -520,37 +513,30 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			EXIT("failed to consume HTile clear state\n");
 		}
 		auto& image = cache.GetImage(depth.image_id);
-		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
-		                     "Kyty.DepthTarget.Image[guest=0x{:016x} size=0x{:x} format={}]",
-		                     image.info.data.address, image.info.data.size,
-		                     static_cast<uint32_t>(image.info.pixel_format));
-		SetVulkanObjectNameF(m_context.GetGraphics().device, image_view,
-		                     "Kyty.DepthTarget.View[guest=0x{:016x} layer={}+{}]",
-		                     image.info.data.address, depth.desc.view_info.base_layer,
-		                     depth.desc.view_info.layer_count);
 		EXIT_IF(image_view == nullptr || image.backing.samples != depth.desc.info.samples);
-		const bool feedback = depth.depth_write_enable && pixel &&
-		    std::ranges::any_of(pixel->images, [&](const TextureBinding& binding) {
-			    if (binding.image_id != depth.image_id ||
-			        binding.desc.type != TextureCache::BindingType::Texture) {
-				    return false;
-			    }
-			    const auto native =
-			        std::ranges::find(image.views, binding.image_view, &CachedImageView::view);
-			    EXIT_IF(native == image.views.end());
-			    const auto& sampled = native->info;
-			    const auto& target = depth.desc.view_info;
-			    return (sampled.aspect & vk::ImageAspectFlagBits::eDepth) &&
-			           ImageRangeOverlaps(sampled.base_level, sampled.level_count,
-			                              target.base_level, target.level_count) &&
-			           ImageRangeOverlaps(sampled.base_layer, sampled.layer_count,
-			                              target.base_layer, target.layer_count);
-		    });
-		if (feedback && !m_context.GetGraphics().attachment_feedback_loop_enabled) {
+		const auto draw_writes = depth.AttachmentWriteAspects();
+		vk::ImageAspectFlags sampled_aspects;
+		for (const auto* stage: stages) {
+			for (const auto& binding: stage->images) {
+				if (binding.image_id != depth.image_id ||
+				    binding.desc.type != TextureCache::BindingType::Texture) continue;
+				const auto native =
+				    std::ranges::find(image.views, binding.image_view, &CachedImageView::view);
+				EXIT_IF(native == image.views.end());
+				sampled_aspects |= native->info.aspect;
+				feedback_aspects |= DepthFeedbackAspects(draw_writes, depth.desc.view_info,
+				                                         native->info);
+			}
+		}
+		if (feedback_aspects && !m_context.GetGraphics().attachment_feedback_loop_enabled) {
 			EXIT("depth attachment feedback loop is not supported by the host\n");
 		}
-		const auto layout = feedback ? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
-		                             : depth_attachment_layout(depth);
+		auto layout = depth_attachment_layout(depth);
+		if (sampled_aspects & ~DepthReadableAspects(layout)) {
+			layout = m_context.GetGraphics().attachment_feedback_loop_enabled
+			             ? vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
+			             : vk::ImageLayout::eGeneral;
+		}
 		// The attachment store writes even when guest depth/stencil tests do not.
 		const auto access = vk::AccessFlagBits2::eDepthStencilAttachmentRead |
 		                    vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
@@ -683,11 +669,13 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer&               b
 	        vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX);
 
 	// Collect the non-empty guest vertex ranges.
+	std::array<uint64_t, ShaderVertexInputInfo::RES_MAX>          sizes {};
 	std::array<VertexBufferRange, ShaderVertexInputInfo::RES_MAX> ranges {};
 	uint32_t                                                      range_count = 0;
 	for (int i = 0; i < vs_input_info.buffers_num; i++) {
 		const auto& vertex = vs_input_info.buffers[i];
 		const auto  size   = VertexBufferDescriptorSize(vertex, vs_input_info);
+		sizes[i]           = size;
 		if (size == 0) {
 			continue;
 		}
@@ -736,7 +724,7 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer&               b
 	vk::Buffer null_buffer = nullptr;
 	for (int i = 0; i < vs_input_info.buffers_num; i++) {
 		const auto& vertex = vs_input_info.buffers[i];
-		const auto  size   = VertexBufferDescriptorSize(vertex, vs_input_info);
+		const auto  size   = sizes[i];
 		if (size == 0) {
 			if (null_buffer == nullptr) {
 				null_buffer = cache.GetBuffer(NULL_BUFFER_ID).Handle();
@@ -1073,15 +1061,16 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		                              index_source.guest_element_size);
 	}
 	LogDrawPhase(draw.Name(), "PrepareBindings");
-	GraphicsBindings                 bindings;
+	auto&                            bindings = m_graphics_bindings;
 	std::array<PreparedBindings*, 4> descriptor_stages {};
 	uint32_t                         stage_count = 0;
 	for (uint32_t i = 0; i < vertex_stages.size(); i++) {
-		bindings.vertex[i]               = PrepareBindings(state.vertex_info[i].stage);
+		PrepareBindings(state.vertex_info[i].stage, bindings.vertex[i]);
 		descriptor_stages[stage_count++] = &bindings.vertex[i];
 	}
 	if (state.ps_active) {
-		bindings.pixel.emplace(PrepareBindings(state.ps_input_info.stage));
+		if (!bindings.pixel) bindings.pixel.emplace();
+		PrepareBindings(state.ps_input_info.stage, *bindings.pixel);
 		descriptor_stages[stage_count++] = &*bindings.pixel;
 	}
 	const auto stages = std::span {descriptor_stages.data(), stage_count};
@@ -1100,9 +1089,10 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	    std::span {state.color_info, state.color_count}, state.depth_info, vertex_stages, buffer,
 	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
 	    state.programs);
+	vk::ImageAspectFlags feedback_aspects;
 	const auto rendering =
 	    AcquireRenderTargets(buffer, state.color_info, state.color_count, state.depth_info,
-	                         bindings.pixel);
+	                         feedback_aspects, stages);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest
@@ -1112,7 +1102,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	if (!mesh_active) {
 		CommitVertexBuffers(vk_buffer, vertex_bindings);
 	}
-	if (bindings.pixel && !draw.IsIndexed()) {
+	if (state.ps_active && !draw.IsIndexed()) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 	}
 	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline, stages);
@@ -1134,11 +1124,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 
 	SetGraphicsDynamicParams(buffer, vk_buffer, vertex_stages.back(), state.depth_info, rendering);
 	if (m_context.GetGraphics().attachment_feedback_loop_enabled) {
-		vk_buffer.setAttachmentFeedbackLoopEnableEXT(
-		    rendering.depth_stencil_attachment.image_layout ==
-		            vk::ImageLayout::eAttachmentFeedbackLoopOptimalEXT
-		        ? vk::ImageAspectFlags {vk::ImageAspectFlagBits::eDepth}
-		        : vk::ImageAspectFlags {});
+		vk_buffer.setAttachmentFeedbackLoopEnableEXT(feedback_aspects);
 	}
 
 	LogDrawPhase(draw.Name(), "BeginRendering");

src/graphics/host_gpu/renderer/sync.cpp

@@ -112,26 +112,6 @@ void WriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst
 	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Write);
 }
 
-void WriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst_gpu_addr,
-                                  uint64_t value) {
-	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), 0,
-	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Write);
-
-	LOGF_COLOR(Log::Color::BrightGreen,
-	           "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
-	           reinterpret_cast<uint64_t>(dst_gpu_addr), value);
-}
-
-void WriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, CommandBuffer& buffer,
-                                               uint64_t* dst_gpu_addr, uint64_t value) {
-	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), 0,
-	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::WriteBack);
-
-	LOGF_COLOR(Log::Color::BrightGreen,
-	           "EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
-	           reinterpret_cast<uint64_t>(dst_gpu_addr), value);
-}
-
 void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
                                      uint64_t* dst_gpu_addr, uint64_t value) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,

src/graphics/host_gpu/renderer/sync.h

@@ -25,10 +25,6 @@ void WriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst
                         uint64_t value);
 void WriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                            uint32_t dw_offset, uint32_t dw_num);
-void WriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst_gpu_addr,
-                                  uint64_t value);
-void WriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, CommandBuffer& buffer,
-                                               uint64_t* dst_gpu_addr, uint64_t value);
 void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
                                      uint32_t* dst_gpu_addr, uint32_t value);
 void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,

src/graphics/presentation/videoOut.cpp

@@ -24,6 +24,8 @@
 
 #include <algorithm>
 #include <array>
+#include <atomic>
+#include <fmt/format.h>
 #include <list>
 #include <thread>
 #include <vector>
@@ -541,6 +543,14 @@ Graphics::ImageInfo BufferAttributeGroup::ImageInfo(const VideoOutBuffer& buffer
 	if (!Graphics::DecodeVideoOutPixelFormat(attribute.pixel_format, pixel_format)) {
 		EXIT("unsupported video-out pixel format: 0x%016" PRIx64 "\n", attribute.pixel_format);
 	}
+	if (attribute.pixel_format == Graphics::VIDEO_OUT_PIXEL_FORMAT_R10_G10_B10_A2_BT2100_PQ) {
+		static std::atomic_flag warned = ATOMIC_FLAG_INIT;
+		if (!warned.test_and_set(std::memory_order_relaxed)) {
+			Log::WriteToConsoleAndLog(fmt::format(
+			    "Warning: HDR-to-SDR conversion is not implemented; displaying PQ video-out "
+			    "pixels unchanged (format=0x{:016x}).\n", attribute.pixel_format));
+		}
+	}
 	const auto tile_mode = Graphics::Prospero::TileMode::kRenderTarget;
 	const auto pitch =
 	    Graphics::TileGetTexturePitch(pixel_format.guest_format, attribute.width, tile_mode);
@@ -1546,7 +1556,9 @@ KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle) {
 	VideoOutFlipStatus status {};
 	DriverState().GetFlipQueue().GetFlipStatus(*ctx, status);
 
-	LOGF("\t flipPendingNum = %d\n", status.flipPendingNum);
+	if (Config::GraphicsDebugDumpEnabled()) {
+		LOGF("\t flipPendingNum = %d\n", status.flipPendingNum);
+	}
 
 	return status.flipPendingNum;
 }
@@ -1680,21 +1692,6 @@ KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* stat
 	return OK;
 }
 
-KYTY_SYSV_ABI int VideoOutGetVrrStatus(int handle, int32_t* status) {
-	PRINT_NAME();
-
-	if (status == nullptr) {
-		return VIDEO_OUT_ERROR_INVALID_ADDRESS;
-	}
-	if (DriverState().Get(handle) == nullptr) {
-		return VIDEO_OUT_ERROR_INVALID_HANDLE;
-	}
-
-	// Kyty currently presents at a fixed refresh rate and does not negotiate VRR.
-	*status = 0;
-	return OK;
-}
-
 static int ValidateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options,
                                 void* reserved_ptr, uint64_t reserved) {
 	if (!DriverState().IsOpened(handle)) {

src/graphics/presentation/videoOut.h

@@ -87,7 +87,6 @@ KYTY_SYSV_ABI int VideoOutGetEventData(const LibKernel::EventQueue::KernelEvent*
 KYTY_SYSV_ABI int VideoOutGetEventCount(const LibKernel::EventQueue::KernelEvent* ev);
 KYTY_SYSV_ABI int VideoOutWaitVblank(int handle);
 KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status);
-KYTY_SYSV_ABI int VideoOutGetVrrStatus(int handle, int32_t* status);
 KYTY_SYSV_ABI int VideoOutInitializeOutputOptions(VideoOutOutputOptions* options);
 KYTY_SYSV_ABI int VideoOutIsOutputSupported(int handle, uint64_t mode,
                                             const VideoOutOutputOptions* options,

src/graphics/presentation/window/vulkanWindow.cpp

@@ -238,6 +238,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 #endif
 		}
 #if !defined(__APPLE__)
+		if (device_features2.features.depthClamp != VK_TRUE) {
+			LOGF("depthClamp is not supported\n");
+			skip_device = true;
+		}
 		if (fragment_barycentric.fragmentShaderBarycentric != VK_TRUE) {
 			LOGF("fragmentShaderBarycentric is not supported\n");
 			skip_device = true;
@@ -607,6 +611,7 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.robustBufferAccess       = VK_TRUE;
 #if !defined(__APPLE__)
 	device_features.depthBounds = VK_TRUE; // unsupported by MoltenVK
+	device_features.depthClamp  = VK_TRUE;
 #endif
 	device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
 	device_features.shaderImageGatherExtended            = VK_TRUE;

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -20,6 +20,7 @@
 
 #include <algorithm>
 #include <array>
+#include <atomic>
 #include <chrono>
 #include <fmt/format.h>
 #include <map>
@@ -523,6 +524,20 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
 
+	// Temporary workaround for games that compile ray-tracing shaders before
+	// the player can select a mode without ray tracing.
+	if (options.stage == ShaderType::Compute && decoded.has_bvh) {
+		static std::atomic_flag warned = ATOMIC_FLAG_INIT;
+		if (!warned.test_and_set(std::memory_order_relaxed)) {
+			const auto& bvh = decoded.instructions.back();
+			Log::WriteToConsoleAndLog(fmt::format(
+			    "Warning: ray tracing is not implemented; skipping compute dispatches containing "
+			    "BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n",
+			    options.shader_hash, bvh.pc, bvh.opcode_id));
+		}
+		return {.skip_dispatch = true};
+	}
+
 	std::string decoded_dump;
 	if (options.dump_ir) {
 		decoded_dump = Decoder::ProgramToString(decoded);
@@ -615,6 +630,7 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
 CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options,
                              const IR::ResourceSpecialization& specialization,
                              uint32_t push_data_start_dword) {
+	EXIT_IF(translated.skip_dispatch);
 	const auto emit_begin = std::chrono::steady_clock::now();
 	auto& ir = translated.program;
 	IR::ApplyResourceSpecialization(ir, specialization);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -28,6 +28,7 @@ struct TranslateResult {
 	IR::Program program;
 	std::string decoded_dump;
 	std::string cfg_dump;
+	bool        skip_dispatch = false;
 };
 
 struct CompileResult {

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -227,6 +227,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
 					requirements.subgroup_local_invocation_id = true;
 				}
 				if (memory.kind == IR::ResourceKind::Buffer) {
+					requirements.coherent_buffers |= memory.coherent;
 					if (memory.resource >= program.info.buffers.size()) {
 						Fail(program, "buffer operation has invalid resource metadata");
 					}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -784,11 +784,11 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		                         ? &state.program.descriptor_sources[image.source]
 		                         : nullptr;
 		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
-		    source->indirect_image->key_arg >= handle->NumArgs()) {
+		    handle->NumArgs() == 0u) {
 			ctx.Fail(inst, "has invalid indirect image key provenance");
 			return;
 		}
-		const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg));
+		const auto key = ctx.Def(handle->Arg(0));
 		if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u ||
 		    image.indirect_resources.size() < 2u) {
 			ctx.Fail(inst, "has no indirect image runtime mapping");

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -67,6 +67,7 @@ struct SpirvRequirements {
 	bool function_scratch             = false;
 	bool pixel_valid_mask             = false;
 	bool buffer_int64_atomics         = false;
+	bool coherent_buffers             = false;
 };
 
 SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
@@ -371,12 +372,13 @@ void EmitMemoryOffsets(EmitterState& state);
 uint32_t LdsDwordCount(const EmitterState& state);
 
 struct MemoryResourceAccess {
-	IR::ResourceKind kind             = IR::ResourceKind::None;
-	uint32_t         object_pointer   = 0;
-	uint32_t         length           = 0;
-	uint32_t         index_offset     = 0;
-	uint32_t         byte_offset      = 0;
-	bool             add_index_offset = false;
+	IR::ResourceKind      kind             = IR::ResourceKind::None;
+	uint32_t              object_pointer   = 0;
+	uint32_t              length           = 0;
+	uint32_t              index_offset     = 0;
+	uint32_t              byte_offset      = 0;
+	bool                  add_index_offset = false;
+	spv::MemoryAccessMask memory_access    = spv::MemoryAccessMaskNone;
 };
 
 MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -313,7 +313,8 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
                           uint32_t index) {
 	const auto value   = ctx.state.builder.AllocateId();
 	const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
-	ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, pointer);
+	ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, pointer,
+	                              resource.memory_access);
 	return value;
 }
 
@@ -498,21 +499,21 @@ void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me
 	});
 }
 
+void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index,
+                       uint32_t data) {
+	ctx.state.builder.AddFunction(spv::OpStore,
+	                              EmitMemoryElementPointer(ctx.state, resource, index), data,
+	                              resource.memory_access);
+}
+
 void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
                        const MemoryResourceAccess& resource, uint32_t data) {
 	const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
 	EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
-		ctx.state.builder.AddFunction(spv::OpStore,
-		                              EmitMemoryElementPointer(ctx.state, resource, index), data);
+		StoreWordInBounds(ctx, resource, index, data);
 	});
 }
 
-void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index,
-                       uint32_t data) {
-	ctx.state.builder.AddFunction(spv::OpStore,
-	                              EmitMemoryElementPointer(ctx.state, resource, index), data);
-}
-
 void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
 	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
 		const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
@@ -710,10 +711,14 @@ void StoreFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
                             uint32_t data) {
 	if (component >= plan.info.component_count) return;
 	const auto bits = plan.info.component_bits[component];
-	if (plan.info.type == Format::ComponentType::Snorm && bits == 16u) {
+	if (bits == 16u && (plan.info.type == Format::ComponentType::Snorm ||
+	                    plan.info.type == Format::ComponentType::Float)) {
 		const auto value = EmitBitCastF32U32(ctx.state, data);
-		data = EmitPackSnorm2x16(
-		    ctx.state, EmitCompositeConstructF32x2(ctx.state, value, ConstantF32Value(ctx.state, 0.0f)));
+		const auto pair = EmitCompositeConstructF32x2(ctx.state, value,
+		                                               ConstantF32Value(ctx.state, 0.0f));
+		data = plan.info.type == Format::ComponentType::Float
+		           ? EmitPackHalf2x16(ctx.state, pair)
+		           : EmitPackSnorm2x16(ctx.state, pair);
 	}
 	if (bits == 8u || bits == 16u) {
 		StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses[component],

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -86,7 +86,9 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
 	}
 	const auto array_index =
 	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
-	MemoryResourceAccess access {.kind = mem.kind};
+	MemoryResourceAccess access {
+	    .kind = mem.kind,
+	    .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone};
 	access.object_pointer = state.builder.AllocateId();
 	state.builder.AddFunction(spv::OpAccessChain, pointer_type, access.object_pointer, variable,
 	                          ConstantU32(state, array_index));

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -201,6 +201,16 @@ void DefineDescriptors(EmitterState& state) {
 					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
 					                            spv::DecorationAliased);
 				}
+				if (state.requirements.coherent_buffers) {
+					// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
+					// must participate in visibility for cache-bypassing polling loads.
+					state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
+					                            spv::DecorationCoherent);
+					if (state.storage_buffer_u64_variable != 0) {
+						state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
+						                            spv::DecorationCoherent);
+					}
+				}
 				break;
 			case IR::DescriptorBindingKind::BdaPagetable:
 				state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -1389,6 +1389,15 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 	if (construct_predecessors.empty() || (!force_split && !has_external_predecessor)) {
 		return false;
 	}
+	if (construct_predecessors.size() == 1u) {
+		const auto* predecessor = graph.FindBlock(construct_predecessors.front());
+		if (predecessor != nullptr && predecessor->inst_begin == predecessor->inst_end &&
+		    predecessor->terminator.kind == TerminatorKind::Branch &&
+		    predecessor->terminator.true_block == merge &&
+		    predecessor->start_pc == merge_block->start_pc) {
+			return false;
+		}
+	}
 	const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge);
 	auto*      synthetic_block = graph.FindBlock(synthetic_merge);
 	if (synthetic_block != nullptr) {

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -21,6 +21,7 @@ constexpr MemoryOpcodeInfo SMEM_OPCODE_LIST[] = {
     {0x04u, Opcode::S_LOAD_DWORDX16, 16, 32},      {0x08u, Opcode::S_BUFFER_LOAD_DWORD, 1, 32},
     {0x09u, Opcode::S_BUFFER_LOAD_DWORDX2, 2, 32}, {0x0au, Opcode::S_BUFFER_LOAD_DWORDX4, 4, 32},
     {0x0bu, Opcode::S_BUFFER_LOAD_DWORDX8, 8, 32}, {0x0cu, Opcode::S_BUFFER_LOAD_DWORDX16, 16, 32},
+    {0x25u, Opcode::S_MEMREALTIME, 2, 32},
 };
 
 constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
@@ -245,6 +246,10 @@ void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 
 	DecodeScalarDestination(sdst, pc, inst.dst);
+	if (inst.opcode == Opcode::S_MEMREALTIME) {
+		inst.src_count = 0;
+		return;
+	}
 	// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
 	// pair index; their descriptor operand consumes four SGPRs from that base.
 	DecodeScalarSource(sbase * 2u, pc, inst.src0);
@@ -268,6 +273,7 @@ void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
 	inst.idxen       = ((word0 >> 13u) & 1u) != 0;
 	inst.offen       = ((word0 >> 12u) & 1u) != 0;
 	inst.glc         = ((word0 >> 14u) & 1u) != 0;
+	inst.dlc         = ((word0 >> 15u) & 1u) != 0;
 	inst.slc         = ((word1 >> 22u) & 1u) != 0;
 	inst.family      = Family::MUBUF;
 	inst.opcode_id   = opcode;
@@ -303,6 +309,7 @@ void DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
 	inst.idxen         = ((word0 >> 13u) & 1u) != 0;
 	inst.offen         = ((word0 >> 12u) & 1u) != 0;
 	inst.glc           = ((word0 >> 14u) & 1u) != 0;
+	inst.dlc           = ((word0 >> 15u) & 1u) != 0;
 	inst.slc           = ((word1 >> 22u) & 1u) != 0;
 	inst.family        = Family::MTBUF;
 	inst.opcode_id     = opcode;

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -73,11 +73,11 @@ std::string FormatMemory(const Instruction& inst) {
 		text += OperandToString(*sources[i]);
 	}
 	text += fmt::format(" ; offset={} offset2={} dwords={} bits={} dfmt={} nfmt={} signed={} "
-	                    "typed={} formatted={} segment={} glc={} slc={} idxen={} offen={}",
+	                    "typed={} formatted={} segment={} glc={} dlc={} slc={} idxen={} offen={}",
 	                    inst.offset, inst.secondary_offset, inst.data_dwords, inst.data_bits,
 	                    inst.data_format, inst.number_format, inst.data_signed ? 1u : 0u,
 	                    inst.typed ? 1u : 0u, inst.formatted ? 1u : 0u, inst.memory_segment,
-	                    inst.glc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u,
+	                    inst.glc ? 1u : 0u, inst.dlc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u,
 	                    inst.offen ? 1u : 0u);
 	return text;
 }
@@ -403,6 +403,7 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 	program.instructions.clear();
 	program.instructions.reserve(code.size());
 	program.code = code;
+	program.has_bvh = false;
 
 	std::vector<bool> branch_targets;
 	for (uint32_t word_index = 0; word_index < code.size();) {
@@ -411,6 +412,10 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
 
 		const auto& inst = program.instructions.back();
 		word_index += inst.word_count;
+		if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) {
+			program.has_bvh = true;
+			return;
+		}
 
 		if (IsDirectBranch(inst.opcode)) {
 			const auto target_index = inst.branch_target / sizeof(uint32_t);

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -354,6 +354,7 @@ enum class Opcode {
 	V_CMPX_GT_F32,
 	V_CMPX_LG_F32,
 	V_CMPX_GE_F32,
+	V_CMPX_O_F32,
 	V_CMPX_NGE_F32,
 	V_CMPX_NLG_F32,
 	V_CMPX_NGT_F32,
@@ -404,6 +405,7 @@ enum class Opcode {
 	V_CMP_LE_U16,
 	V_CMP_GT_U16,
 	V_CMPX_LT_U16,
+	V_CMPX_EQ_U16,
 	V_CMPX_GT_U16,
 	V_CMP_NE_U16,
 	V_CMP_GE_U16,
@@ -439,6 +441,7 @@ enum class Opcode {
 	S_BUFFER_LOAD_DWORDX4,
 	S_BUFFER_LOAD_DWORDX8,
 	S_BUFFER_LOAD_DWORDX16,
+	S_MEMREALTIME,
 	BUFFER_LOAD_FORMAT_X,
 	BUFFER_LOAD_FORMAT_XY,
 	BUFFER_LOAD_FORMAT_XYZ,
@@ -711,6 +714,7 @@ struct Instruction {
 	bool           formatted                                    = false;
 	bool           gds                                          = false;
 	bool           glc                                          = false;
+	bool           dlc                                          = false;
 	bool           slc                                          = false;
 	bool           idxen                                        = false;
 	bool           offen                                        = false;
@@ -729,6 +733,7 @@ struct Instruction {
 struct Program {
 	std::span<const uint32_t> code;
 	std::vector<Instruction>  instructions;
+	bool                     has_bvh = false;
 };
 
 // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -40,7 +40,7 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
     {0x0fu, Opcode::V_MIN_F32},
     {0x10u, Opcode::V_MAX_F32},
     {0x11u, Opcode::V_MIN_I32},
-    {0x12u, Opcode::V_MAX_I32},
+    {0x12u, Opcode::V_MAX_I32, Vop2SdwaProfile::IntegerFullDestination},
     {0x13u, Opcode::V_MIN_U32, Vop2SdwaProfile::IntegerPartialDestination},
     {0x14u, Opcode::V_MAX_U32, Vop2SdwaProfile::IntegerFullDestination},
     {0x15u, Opcode::V_LSHR_B32},
@@ -214,6 +214,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0x11u, Opcode::V_CMPX_LT_F32},        {0x12u, Opcode::V_CMPX_EQ_F32},
     {0x13u, Opcode::V_CMPX_LE_F32},        {0x14u, Opcode::V_CMPX_GT_F32},
     {0x15u, Opcode::V_CMPX_LG_F32},        {0x16u, Opcode::V_CMPX_GE_F32},
+    {0x17u, Opcode::V_CMPX_O_F32},
     {0x19u, Opcode::V_CMPX_NGE_F32},       {0x1au, Opcode::V_CMPX_NLG_F32},
     {0x1bu, Opcode::V_CMPX_NGT_F32},       {0x1cu, Opcode::V_CMPX_NLE_F32},
     {0x1du, Opcode::V_CMPX_NEQ_F32},       {0x1eu, Opcode::V_CMPX_NLT_F32},
@@ -231,7 +232,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
-    {0xb9u, Opcode::V_CMPX_LT_U16, false},
+    {0xb9u, Opcode::V_CMPX_LT_U16, false}, {0xbau, Opcode::V_CMPX_EQ_U16, false},
     {0xbcu, Opcode::V_CMPX_GT_U16},        {0xc0u, Opcode::V_CMP_F_U32},
     {0xc1u, Opcode::V_CMP_LT_U32},         {0xc2u, Opcode::V_CMP_EQ_U32},
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
@@ -521,8 +522,10 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
     {Opcode::V_CVT_F32_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_I32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_UBYTE0, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
+    // The source field extract and the destination insert are independent (Translate.cpp), so a
+    // partial destination may combine with any source selector.
     {Opcode::V_NOT_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(),
-     SdwaSelFull(), false},
+     SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
     {Opcode::V_FFBL_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
     {Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
     {Opcode::V_CVT_F16_F32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
@@ -820,6 +823,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
 		case Opcode::V_CMPX_GT_F32:
 		case Opcode::V_CMPX_LG_F32:
 		case Opcode::V_CMPX_GE_F32:
+		case Opcode::V_CMPX_O_F32:
 		case Opcode::V_CMPX_NGE_F32:
 		case Opcode::V_CMPX_NLG_F32:
 		case Opcode::V_CMPX_NGT_F32:
@@ -1432,6 +1436,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_GT_F32:
 		case Opcode::V_CMPX_LG_F32:
 		case Opcode::V_CMPX_GE_F32:
+		case Opcode::V_CMPX_O_F32:
 		case Opcode::V_CMPX_NGE_F32:
 		case Opcode::V_CMPX_NLG_F32:
 		case Opcode::V_CMPX_NGT_F32:
@@ -1454,6 +1459,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_I64:
 		case Opcode::V_CMPX_NE_U64:
 		case Opcode::V_CMPX_LT_U16:
+		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:
 		case Opcode::V_CMPX_LT_F16:
 		case Opcode::V_CMPX_EQ_F16:

src/graphics/shader/recompiler/frontend/translate/Attribute.cpp

@@ -106,12 +106,12 @@ void Translator::EXP(const Decoder::Instruction& inst) {
 	ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
 }
 
-bool Translator::EmitInterpolation(const Decoder::Instruction& inst) {
+void Translator::EmitInterpolation(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return true;
-		case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return true;
-		case Decoder::Opcode::V_INTERP_MOV_F32: V_INTERP_MOV_F32(inst); return true;
-		default: return false;
+		case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return;
+		case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return;
+		case Decoder::Opcode::V_INTERP_MOV_F32: V_INTERP_MOV_F32(inst); return;
+		default: return FailMissingTranslation(inst);
 	}
 }
 

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -47,12 +47,12 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc
 	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
 }
 
-void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered) {
+void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) {
 	const auto lhs       = IR::F32(ReadOperand(inst.src0, IR::Type::F32));
 	const auto rhs       = IR::F32(ReadOperand(inst.src1, IR::Type::F32));
 	const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})),
 	                                    IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs})));
-	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, false);
+	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, cmpx);
 }
 
 void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx) {

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -302,23 +302,95 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
 }
 
 void Translator::S_WQM(const Decoder::Instruction& inst, bool wide) {
-	const auto source = wide ? ReadU64(inst.src0)
-	                         : ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
+	const bool source_is_mask = inst.src0.kind == Decoder::OperandKind::Sgpr ||
+	                            inst.src0.kind == Decoder::OperandKind::ExecLo ||
+	                            inst.src0.kind == Decoder::OperandKind::VccLo ||
+	                            (!wide && IsExecOrVcc(inst.src0));
+	const auto source_valid = ReadMaskValid(inst.src0);
+	auto source_live = IR::U1(IR::Value(false));
+	if (source_is_mask) {
+		if (inst.src0.kind == Decoder::OperandKind::VccLo ||
+		    inst.src0.kind == Decoder::OperandKind::VccHi) {
+			source_live = ir.GetVcc();
+		} else {
+			source_live = ReadMask(inst.src0);
+		}
+	}
+	const auto retained_live = ir.LogicalAnd(source_valid, source_live);
+
+	IR::U64 source;
+	if (wide) {
+		source = ReadU64(inst.src0);
+	} else {
+		source = ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
+	}
 	const auto result = IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {source}));
 	if (!wide) {
 		const auto low = ir.CompositeExtract(result, 0);
+		const bool destination_is_mask = IsExecOrVcc(inst.dst);
+		const bool destination_is_exec = inst.dst.kind == Decoder::OperandKind::ExecLo ||
+		                                 inst.dst.kind == Decoder::OperandKind::ExecHi;
+		auto old_live = IR::U1(IR::Value(false));
+		if (destination_is_mask) {
+			if (destination_is_exec) {
+				old_live = ir.GetExec();
+			} else {
+				old_live = ir.GetVcc();
+			}
+		}
+
 		// A 32-bit write preserves the other EXEC/VCC half and invalidates any
 		// overlapping SGPR-pair mask provenance through the ordinary scalar path.
 		WriteRawU32(inst.dst, low);
+		if (destination_is_mask) {
+			const auto is_high_half = [](const Decoder::Operand& operand) {
+				return operand.kind == Decoder::OperandKind::ExecHi ||
+				       operand.kind == Decoder::OperandKind::VccHi;
+			};
+			const bool destination_is_high = is_high_half(inst.dst);
+			auto affected_lanes = IR::U1(IR::Value(!destination_is_high));
+			if (program.wave_size == 64u) {
+				const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
+				affected_lanes = ir.ULessThan(lane, IR::U32(IR::Value(32u)));
+				if (destination_is_high) {
+					affected_lanes = ir.LogicalNot(affected_lanes);
+				}
+			}
+
+			IR::U1 live;
+			if (destination_is_exec) {
+				live = ir.GetExec();
+			} else {
+				live = ir.GetVcc();
+			}
+			// A cross-half copy refers to different lanes, so only scalar bits apply.
+			if (is_high_half(inst.src0) == destination_is_high) {
+				live = ir.LogicalOr(retained_live, live);
+			}
+			live = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {affected_lanes, live, old_live}));
+			if (destination_is_exec) {
+				ir.SetExec(live);
+			} else {
+				ir.SetVcc(live);
+			}
+		}
 		ir.SetScc(ir.INotEqual(low, IR::U32(IR::Value(0u))));
 		return;
 	}
-	const auto mask_valid = ReadMaskValid(inst.src0);
+
+	// WQM adds live lanes but never removes a source lane. Keep that fact separate
+	// from the scalar mask: reconstructing an entry-true predicate from a ballot
+	// otherwise introduces a spurious dependency into every vector write.
+	const auto live = ir.LogicalOr(retained_live, ThreadBit(ExtractU64(result)));
 	WriteOperand(DestinationOperand(inst), result);
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+	if (inst.dst.kind == Decoder::OperandKind::ExecLo) {
+		ir.SetExec(live);
+	} else if (inst.dst.kind == Decoder::OperandKind::VccLo) {
+		ir.SetVcc(live);
+	} else if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
 		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
-		ir.SetThreadBitScalarReg(dst, ThreadBit(ExtractU64(result)));
-		ir.SetScalarMaskTag(dst, mask_valid);
+		ir.SetThreadBitScalarReg(dst, live);
+		ir.SetScalarMaskTag(dst, source_valid);
 	}
 	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
 }

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -3,6 +3,11 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
+void Translator::FailMissingTranslation(const Decoder::Instruction& inst) {
+	EXIT("opcode %s at pc 0x%08x has no IR translation",
+	     Decoder::InstructionToString(inst).c_str(), inst.pc);
+}
+
 void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
 	current_opcode = inst.opcode;
 	current_pc     = inst.pc;
@@ -16,35 +21,26 @@ void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
 		default: break;
 	}
 
-	bool translated = false;
 	switch (inst.family) {
 		case Decoder::Family::SOP1:
 		case Decoder::Family::SOP2:
 		case Decoder::Family::SOPK:
 		case Decoder::Family::SOPC:
-		case Decoder::Family::SOPP: translated = EmitScalar(inst); break;
+		case Decoder::Family::SOPP: return EmitScalar(inst);
 		case Decoder::Family::VOP1:
 		case Decoder::Family::VOP2:
 		case Decoder::Family::VOP3:
 		case Decoder::Family::VOP3P:
-		case Decoder::Family::VOPC: translated = EmitVector(inst); break;
+		case Decoder::Family::VOPC: return EmitVector(inst);
 		case Decoder::Family::SMEM:
 		case Decoder::Family::MUBUF:
 		case Decoder::Family::MTBUF:
 		case Decoder::Family::FLAT:
 		case Decoder::Family::DS:
-		case Decoder::Family::MIMG: translated = EmitMemory(inst); break;
-		case Decoder::Family::VINTRP: translated = EmitInterpolation(inst); break;
-		case Decoder::Family::EXP:
-			EXP(inst);
-			translated = true;
-			break;
-		default: break;
-	}
-
-	if (!translated) {
-		EXIT("opcode %s at pc 0x%08x has no IR translation",
-		     Decoder::InstructionToString(inst).c_str(), inst.pc);
+		case Decoder::Family::MIMG: return EmitMemory(inst);
+		case Decoder::Family::VINTRP: return EmitInterpolation(inst);
+		case Decoder::Family::EXP: return EXP(inst);
+		default: return FailMissingTranslation(inst);
 	}
 }
 

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -5,7 +5,7 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                bool accumulator, bool quiet_snan) {
 	const auto translate_lane = [&](bool high) {
 		const auto lhs = ReadF16LaneAsF32(inst.src0, high, true);
@@ -43,16 +43,15 @@ bool Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode
 		result = PackU16Lanes(override_lane(false), override_lane(true));
 	}
 	WriteOperand(raw, result);
-	return true;
 }
 
-bool Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                               bool invalid_negative) {
 	const auto argument = ReadF16AsF32(inst.src0);
 	auto       result   = IR::F32(ir.Emit(opcode, {argument}));
 	if (!invalid_negative) {
 		WriteF16(DestinationOperand(inst), result);
-		return true;
+		return;
 	}
 	const auto negative =
 	    IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {argument, IR::Value::F32(0.0f)}));
@@ -60,10 +59,9 @@ bool Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode
 	const auto bits    = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
 	const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
 	Write16Bits(DestinationOperand(inst), ir.Select(negative, invalid, bits));
-	return true;
 }
 
-bool Translator::Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto argument = ReadF16AsF32(inst.src0);
 	const auto magnitude =
 	    ir.BitwiseAnd(ir.BitCastU32(argument), IR::U32(IR::Value(0x7fffffffu)));
@@ -96,18 +94,16 @@ bool Translator::Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode o
 	const auto bits    = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
 	const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
 	Write16Bits(DestinationOperand(inst), ir.Select(infinite, invalid, bits));
-	return true;
 }
 
-bool Translator::Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                bool reverse) {
 	const auto lhs = ReadF16AsF32(reverse ? inst.src1 : inst.src0);
 	const auto rhs = ReadF16AsF32(reverse ? inst.src0 : inst.src1);
 	WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {lhs, rhs})));
-	return true;
 }
 
-bool Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                 bool accumulator, bool mix) {
 	std::array<IR::Value, 3> args;
 	for (uint32_t index = 0; index < args.size(); index++) {
@@ -115,16 +111,14 @@ bool Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcod
 		args[index] = mix ? IR::Value(ReadMixF32(operand)) : IR::Value(ReadF16AsF32(operand));
 	}
 	WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})));
-	return true;
 }
 
-bool Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto type = IR::ArgTypeOf(opcode, 0);
 	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {ReadOperand(inst.src0, type)}));
-	return true;
 }
 
-bool Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                              bool reverse) {
 	std::array<IR::Value, 2> args;
 	for (uint32_t index = 0; index < args.size(); index++) {
@@ -132,10 +126,9 @@ bool Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode o
 		args[index]        = ReadOperand(operand, IR::ArgTypeOf(opcode, index));
 	}
 	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1]}));
-	return true;
 }
 
-bool Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                               bool accumulator, bool mix) {
 	std::array<IR::Value, 3> args;
 	for (uint32_t index = 0; index < args.size(); index++) {
@@ -145,10 +138,9 @@ bool Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode
 		                                                  : ReadOperand(operand, type);
 	}
 	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1], args[2]}));
-	return true;
 }
 
-bool Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
+void Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
 	const auto bits = ReadU32(inst.src0);
 	const auto exponent =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
@@ -166,10 +158,9 @@ bool Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
 	                                 ir.Select(zero, bits, subnormal));
 	const auto result    = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite);
 	WriteOperand(DestinationOperand(inst), ir.BitCastF32(result));
-	return true;
 }
 
-bool Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
+void Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
 	auto a          = inst.src0;
 	a.op_sel        = false;
 	a.op_sel_hi     = true;
@@ -183,26 +174,25 @@ bool Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
 	const auto acc  = ReadMixF32(inst.dst);
 	const auto lo   = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc}));
 	WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo}));
-	return true;
 }
 
-bool Translator::V_CUBEID_F32(const Decoder::Instruction& inst) {
+void Translator::V_CUBEID_F32(const Decoder::Instruction& inst) {
 	return FloatCube(inst, 0u);
 }
 
-bool Translator::V_CUBESC_F32(const Decoder::Instruction& inst) {
+void Translator::V_CUBESC_F32(const Decoder::Instruction& inst) {
 	return FloatCube(inst, 1u);
 }
 
-bool Translator::V_CUBETC_F32(const Decoder::Instruction& inst) {
+void Translator::V_CUBETC_F32(const Decoder::Instruction& inst) {
 	return FloatCube(inst, 2u);
 }
 
-bool Translator::V_CUBEMA_F32(const Decoder::Instruction& inst) {
+void Translator::V_CUBEMA_F32(const Decoder::Instruction& inst) {
 	return FloatCube(inst, 3u);
 }
 
-bool Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kind) {
+void Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kind) {
 	const auto x  = ReadMixF32(inst.src0);
 	const auto y  = ReadMixF32(inst.src1);
 	const auto z  = ReadMixF32(inst.src2);
@@ -242,7 +232,6 @@ bool Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kin
 		default: EXIT("invalid cube result kind");
 	}
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -4,25 +4,23 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                 bool arithmetic) {
 	const auto value  = ReadU16AsU32(inst.src1, arithmetic);
 	const auto count  = ir.BitwiseAnd(ReadU16AsU32(inst.src0, false), IR::U32(IR::Value(15u)));
 	const auto result = IR::U32(ir.Emit(opcode, {value, count}));
 	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
-	return true;
 }
 
-bool Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                  bool sign) {
 	const auto lhs    = ReadU16AsU32(inst.src0, sign);
 	const auto rhs    = ReadU16AsU32(inst.src1, sign);
 	const auto result = IR::U32(ir.Emit(opcode, {lhs, rhs}));
 	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
-	return true;
 }
 
-bool Translator::V_MAD_I16(const Decoder::Instruction& inst) {
+void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
 	const auto lhs    = ReadU16AsU32(inst.src0, true);
 	const auto rhs    = ReadU16AsU32(inst.src1, true);
 	auto       result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, true));
@@ -31,18 +29,16 @@ bool Translator::V_MAD_I16(const Decoder::Instruction& inst) {
 		result = IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {result, IR::Value(0x7fffu)}));
 	}
 	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
-	return true;
 }
 
-bool Translator::V_MED3_I16(const Decoder::Instruction& inst) {
+void Translator::V_MED3_I16(const Decoder::Instruction& inst) {
 	const auto result = IR::U32(ir.Emit(
 	    IR::ValueOpcode::SMedTri32, {ReadU16AsU32(inst.src0, true), ReadU16AsU32(inst.src1, true),
 	                                 ReadU16AsU32(inst.src2, true)}));
 	Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
-	return true;
 }
 
-bool Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                       bool arithmetic) {
 	const auto translate_lane = [&](bool high) {
 		const auto count =
@@ -52,10 +48,9 @@ bool Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::Valu
 	};
 	WriteOperand(DestinationOperand(inst),
 	             PackU16Lanes(translate_lane(false), translate_lane(true)));
-	return true;
 }
 
-bool Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto translate_lane = [&](bool high) {
 		const auto lhs = ReadU16LaneAsU32(inst.src0, high, false);
 		const auto rhs = ReadU16LaneAsU32(inst.src1, high, false);
@@ -63,10 +58,9 @@ bool Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::Val
 	};
 	WriteOperand(DestinationOperand(inst),
 	             PackU16Lanes(translate_lane(false), translate_lane(true)));
-	return true;
 }
 
-bool Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign) {
+void Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign) {
 	const auto translate_lane = [&](bool high) {
 		const auto lhs = ReadU16LaneAsU32(inst.src0, high, false);
 		const auto rhs = ReadU16LaneAsU32(inst.src1, high, false);
@@ -74,10 +68,9 @@ bool Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign)
 	};
 	WriteOperand(DestinationOperand(inst),
 	             PackU16Lanes(translate_lane(false), translate_lane(true)));
-	return true;
 }
 
-bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                        bool sign) {
 	const auto translate_lane = [&](bool high) {
 		const auto lhs = ReadU16LaneAsU32(inst.src0, high, sign);
@@ -86,7 +79,6 @@ bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::Val
 	};
 	WriteOperand(DestinationOperand(inst),
 	             PackU16Lanes(translate_lane(false), translate_lane(true)));
-	return true;
 }
 
 IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
@@ -100,7 +92,7 @@ IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpco
 	return negate_result ? ir.LogicalNot(result) : result;
 }
 
-bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
+void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
                             IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result,
                             bool unary) {
 	const auto invocation_result =
@@ -119,7 +111,7 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
 	    (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
 		const auto mask = WriteMask(inst.dst, invocation_result, true);
 		ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))));
-		return true;
+		return;
 	}
 
 	const auto lhs        = ReadU32Pair(inst.src0);
@@ -146,10 +138,9 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
 		ir.SetScalarMaskTag(dst, mask_valid);
 	}
 	ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
-	return true;
 }
 
-bool Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                IR::Type type, bool reverse, bool mask_shift_count,
                                bool update_scc) {
 	std::array<IR::Value, 3> args;
@@ -177,10 +168,9 @@ bool Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode
 			ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u))));
 		}
 	}
-	return true;
 }
 
-bool Translator::S_ASHR_I64(const Decoder::Instruction& inst) {
+void Translator::S_ASHR_I64(const Decoder::Instruction& inst) {
 	// Signed 64-bit sources sign-extend literals; generic B64 operands zero-extend them.
 	const auto source =
 	    inst.src0.kind == Decoder::OperandKind::LiteralConstant
@@ -192,10 +182,9 @@ bool Translator::S_ASHR_I64(const Decoder::Instruction& inst) {
 	    ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {source, ReadU32(inst.src1)});
 	WriteOperand(inst.dst, result);
 	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
-	return true;
 }
 
-bool Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                                        bool negate_rhs, bool negate_result, bool update_scc) {
 	const auto lhs = ReadU32(inst.src0);
 	auto       rhs = ReadU32(inst.src1);
@@ -210,31 +199,27 @@ bool Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::Val
 	if (update_scc) {
 		ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
 	}
-	return true;
 }
 
-bool Translator::V_AND_OR_B32(const Decoder::Instruction& inst) {
+void Translator::V_AND_OR_B32(const Decoder::Instruction& inst) {
 	const auto result =
 	    ir.BitwiseOr(ir.BitwiseAnd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_OR3_B32(const Decoder::Instruction& inst) {
+void Translator::V_OR3_B32(const Decoder::Instruction& inst) {
 	const auto result =
 	    ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
+void Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
 	const auto result =
 	    ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
+void Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
 	const auto source        = ExtractU64(ReadU64(inst.src0));
 	const auto low_lsb       = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]}));
 	const auto high_lsb      = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]}));
@@ -243,10 +228,9 @@ bool Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
 	                                     ir.Select(ir.INotEqual(source[1], IR::U32(IR::Value(0u))),
 	                                               high_position, IR::U32(IR::Value(0xffffffffu))));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
+void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
 	const auto source = ReadU32(inst.src0);
 	const auto value  = sign ? ir.BitwiseXor(
 	                              source, ir.ShiftRightArithmetic(source, IR::U32(IR::Value(31u))))
@@ -256,10 +240,9 @@ bool Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
 	const auto result   = ir.Select(ir.INotEqual(value, IR::U32(IR::Value(0u))), position,
 	                                IR::U32(IR::Value(0xffffffffu)));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
+void Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
 	const auto source   = ReadU64(inst.src0);
 	const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source}));
 	const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb);
@@ -268,10 +251,9 @@ bool Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
 	                   {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
 	const auto result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu)));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
+void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
 	const auto extract24 = [&](IR::U32 value) {
 		return IR::U32(
 		    ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract,
@@ -284,10 +266,9 @@ bool Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
 		result = ir.IAdd(result, ReadU32(inst.src2));
 	}
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
+void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
 	const auto lhs       = ReadU32(inst.src0);
 	const auto rhs       = ReadU32(inst.src1);
 	const auto add       = ExtractU64(ReadU64(inst.src2));
@@ -305,36 +286,32 @@ bool Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
 	    inst.dst2.kind != Decoder::OperandKind::Unknown) {
 		WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
 	}
-	return true;
 }
 
-bool Translator::V_SAD_U32(const Decoder::Instruction& inst) {
+void Translator::V_SAD_U32(const Decoder::Instruction& inst) {
 	const auto lhs    = ReadU32(inst.src0);
 	const auto rhs    = ReadU32(inst.src1);
 	const auto lo     = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs}));
 	const auto hi     = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs}));
 	const auto result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_ADD3_U32(const Decoder::Instruction& inst) {
+void Translator::V_ADD3_U32(const Decoder::Instruction& inst) {
 	const auto result =
 	    ir.IAdd(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::S_BITSET_B32(const Decoder::Instruction& inst, bool set) {
+void Translator::S_BITSET_B32(const Decoder::Instruction& inst, bool set) {
 	const auto offset = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u)));
 	const auto bit    = ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), offset);
 	const auto old    = ReadU32(inst.dst);
 	const auto result = set ? ir.BitwiseOr(old, bit) : ir.BitwiseAnd(old, ir.BitwiseNot(bit));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::S_BITSET_B64(const Decoder::Instruction& inst, bool set) {
+void Translator::S_BITSET_B64(const Decoder::Instruction& inst, bool set) {
 	const auto offset    = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
 	const auto word_bit  = ir.BitwiseAnd(offset, IR::U32(IR::Value(31u)));
 	const auto bit       = ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), word_bit);
@@ -347,16 +324,14 @@ bool Translator::S_BITSET_B64(const Decoder::Instruction& inst, bool set) {
 	                                {offset, IR::Value(32u)}));
 	WriteU32Pair(inst.dst,
 	             {ir.Select(high, old[0], low_value), ir.Select(high, high_value, old[1])});
-	return true;
 }
 
-bool Translator::V_BCNT_U32_B32(const Decoder::Instruction& inst) {
+void Translator::V_BCNT_U32_B32(const Decoder::Instruction& inst) {
 	const auto count = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src0)}));
 	WriteOperand(DestinationOperand(inst), ir.IAdd(count, ReadU32(inst.src1)));
-	return true;
 }
 
-bool Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
+void Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
 	const auto lane  = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
 	const auto local = ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)));
 	const auto below =
@@ -368,10 +343,9 @@ bool Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
 	const auto active      = ir.BitwiseAnd(ReadU32(inst.src0), thread_mask);
 	const auto count       = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active}));
 	WriteOperand(DestinationOperand(inst), ir.IAdd(count, ReadU32(inst.src1)));
-	return true;
 }
 
-bool Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
+void Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
 	const auto replicate = [&](IR::U32 value) {
 		auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, IR::U32(IR::Value(8u))));
 		bits      = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x00ff00ffu)));
@@ -387,10 +361,9 @@ bool Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
 	const auto low    = ir.BitwiseAnd(source, IR::U32(IR::Value(0xffffu)));
 	const auto high   = ir.ShiftRightLogical(source, IR::U32(IR::Value(16u)));
 	WriteOperand(DestinationOperand(inst), ir.ConstructU64(replicate(low), replicate(high)));
-	return true;
 }
 
-bool Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
+void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
 	const auto compact = [&](IR::U32 value) {
 		auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, IR::U32(IR::Value(1u))));
 		bits      = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(2u))));
@@ -409,16 +382,14 @@ bool Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
 	const auto result = ir.ConstructU64(quads, IR::U32(IR::Value(0u)));
 	WriteOperand(DestinationOperand(inst), result);
 	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
-	return true;
 }
 
-bool Translator::BFM_B32(const Decoder::Instruction& inst) {
+void Translator::BFM_B32(const Decoder::Instruction& inst) {
 	const auto count  = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u)));
 	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto result = ir.Emit(IR::ValueOpcode::BitFieldInsert,
 	                            {IR::Value(0u), IR::Value(0xffffffffu), offset, count});
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
 IR::U32 Translator::RightMask32(IR::U32 count) {
@@ -435,15 +406,14 @@ IR::U64 Translator::RightMask64(IR::U32 count) {
 	return ir.ConstructU64(RightMask32(low_count), RightMask32(high_count));
 }
 
-bool Translator::S_BFM_B64(const Decoder::Instruction& inst) {
+void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
 	const auto count  = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
 	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
 	const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset});
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) {
+void Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) {
 	const auto source = ReadU32(inst.src0);
 	const auto field  = ReadU32(inst.src1);
 	const auto offset =
@@ -457,10 +427,9 @@ bool Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) {
 	const auto result = IR::U32(ir.Emit(opcode, {source, offset, count}));
 	WriteOperand(DestinationOperand(inst), result);
 	ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
-	return true;
 }
 
-bool Translator::S_BFE_U64(const Decoder::Instruction& inst) {
+void Translator::S_BFE_U64(const Decoder::Instruction& inst) {
 	const auto source = ReadU64(inst.src0);
 	const auto field  = ReadU32(inst.src1);
 	const auto offset =
@@ -473,10 +442,9 @@ bool Translator::S_BFE_U64(const Decoder::Instruction& inst) {
 	const auto result    = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, RightMask64(count)});
 	WriteOperand(DestinationOperand(inst), result);
 	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
-	return true;
 }
 
-bool Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) {
+void Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) {
 	const auto source    = ReadU32(inst.src0);
 	const auto offset    = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto raw_count = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
@@ -485,29 +453,26 @@ bool Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) {
 	const auto opcode =
 	    sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract;
 	WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {source, offset, count}));
-	return true;
 }
 
-bool Translator::V_BFI_B32(const Decoder::Instruction& inst) {
+void Translator::V_BFI_B32(const Decoder::Instruction& inst) {
 	const auto bits   = ReadU32(inst.src0);
 	const auto insert = ReadU32(inst.src1);
 	const auto base   = ReadU32(inst.src2);
 	const auto result =
 	    ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) {
+void Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) {
 	const auto value  = ReadU32(inst.src0);
 	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto bit =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {value, offset, IR::Value(1u)}));
 	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
-	return true;
 }
 
-bool Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
+void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
 	const auto value    = ReadU32Pair(inst.src0);
 	const auto offset   = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
 	const auto word_bit = ir.BitwiseAnd(offset, IR::U32(IR::Value(31u)));
@@ -516,10 +481,9 @@ bool Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
 	const auto bit =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {word, word_bit, IR::Value(1u)}));
 	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
-	return true;
 }
 
-bool Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
+void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
 	const auto hi      = ReadU32(inst.src0);
 	const auto lo      = ReadU32(inst.src1);
 	const auto shift   = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
@@ -530,10 +494,9 @@ bool Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
 	const auto hi_part =
 	    ir.Select(ir.INotEqual(shift, IR::U32(IR::Value(0u))), hi_part_raw, IR::U32(IR::Value(0u)));
 	WriteOperand(DestinationOperand(inst), ir.BitwiseOr(lo_part, hi_part));
-	return true;
 }
 
-bool Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
+void Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
 	const auto hi           = ReadU32(inst.src0);
 	const auto lo           = ReadU32(inst.src1);
 	const auto byte_offset  = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
@@ -546,39 +509,34 @@ bool Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
 	    IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, IR::Value(8u)}));
 	WriteOperand(DestinationOperand(inst),
 	             ir.Select(in_range, ExtractU64(shifted)[0], IR::U32(IR::Value(0u))));
-	return true;
 }
 
-bool Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
+void Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
 	const auto shift  = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_ADD_LSHL_U32(const Decoder::Instruction& inst) {
+void Translator::V_ADD_LSHL_U32(const Decoder::Instruction& inst) {
 	const auto shift  = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
 	const auto result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), shift);
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_XAD_U32(const Decoder::Instruction& inst) {
+void Translator::V_XAD_U32(const Decoder::Instruction& inst) {
 	const auto result =
 	    ir.IAdd(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_LSHL_OR_B32(const Decoder::Instruction& inst) {
+void Translator::V_LSHL_OR_B32(const Decoder::Instruction& inst) {
 	const auto shift = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto result =
 	    ir.BitwiseOr(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) {
+void Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) {
 	Decoder::Operand mask_operand;
 	mask_operand.kind = Decoder::OperandKind::VccLo;
 	if (inst.src_count >= 3u) {
@@ -594,17 +552,15 @@ bool Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) {
 		result = ir.Select(condition, ReadU32(inst.src1), ReadU32(inst.src0));
 	}
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
-bool Translator::PackB16(const Decoder::Instruction& inst, bool high0, bool high1) {
+void Translator::PackB16(const Decoder::Instruction& inst, bool high0, bool high1) {
 	const auto lo        = high0 ? ir.ShiftRightLogical(ReadU32(inst.src0), IR::U32(IR::Value(16u)))
 	                             : ReadU32(inst.src0);
 	const auto hi        = high1 ? ir.ShiftRightLogical(ReadU32(inst.src1), IR::U32(IR::Value(16u)))
 	                             : ReadU32(inst.src1);
 	const auto result = PackU16Lanes(lo, hi);
 	WriteOperand(DestinationOperand(inst), result);
-	return true;
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -1,8 +1,11 @@
+#include "common/assert.h"
+#include "common/logging/log.h"
 #include "graphics/shader/recompiler/frontend/translate/Translator.h"
 #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
 
 #include <algorithm>
 #include <array>
+#include <atomic>
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
@@ -123,6 +126,11 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
 	memory.image_r128    = decoded.image_r128;
 	memory.idxen         = decoded.idxen;
 	memory.offen         = decoded.offen;
+	// Vector loads use GLC/DLC to bypass L0/GL1; atomics use GLC only to return data.
+	const bool buffer_atomic = decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP &&
+	                           decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX;
+	memory.coherent = memory.kind == ResourceKind::Buffer && !buffer_atomic &&
+	                  (decoded.glc || decoded.dlc);
 	memory.resource      = ResourceIndexFromOperand(decoded.src1);
 	memory.sampler       = ResourceIndexFromOperand(decoded.src2);
 	if (memory.kind == ResourceKind::ScalarBuffer) {
@@ -397,7 +405,7 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) {
 	                  : ir.Emit(IR::ValueOpcode::ConvertU16U32, {value});
 }
 
-bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
+void Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	const auto resource =
 	    raw ? GetScalarAddressResource(RawScalarLoadBase(inst.src0)) : GetBufferResource(memory);
@@ -420,10 +428,9 @@ bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
 	for (uint32_t component = 0; component < memory.data_dwords; component++) {
 		WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]);
 	}
-	return true;
 }
 
-bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
+void Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
 	const auto      memory = MemoryInfoFromDecoded(inst);
 	IR::ValueOpcode opcode;
 	const auto      bits = memory.data_bits;
@@ -437,10 +444,15 @@ bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
 				case 2u: opcode = IR::ValueOpcode::LoadBufferU32x2; break;
 				case 3u: opcode = IR::ValueOpcode::LoadBufferU32x3; break;
 				case 4u: opcode = IR::ValueOpcode::LoadBufferU32x4; break;
-				default: return false;
+				default:
+					EXIT("opcode %s at pc 0x%08x has unsupported buffer load dword count %u",
+					     Decoder::InstructionToString(inst).c_str(), inst.pc,
+					     memory.data_dwords);
 			}
 			break;
-		default: return false;
+		default:
+			EXIT("opcode %s at pc 0x%08x has unsupported buffer load width %u",
+			     Decoder::InstructionToString(inst).c_str(), inst.pc, bits);
 	}
 	const auto resource = GetBufferResource(memory);
 	const auto address  = ReadBufferAddress(inst, 0);
@@ -457,10 +469,9 @@ bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
 			             ir.CompositeExtract(loaded, component));
 		}
 	}
-	return true;
 }
 
-bool Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
+void Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
 	const auto      memory   = MemoryInfoFromDecoded(inst);
 	const auto      resource = GetBufferResource(memory);
 	const auto      address  = ReadBufferAddress(inst, 1);
@@ -501,17 +512,21 @@ bool Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
 					                  ReadU32(OffsetOperand(data_src, 2u)),
 					                  ReadU32(OffsetOperand(data_src, 3u))});
 					break;
-				default: return false;
+				default:
+					EXIT("opcode %s at pc 0x%08x has unsupported buffer store dword count %u",
+					     Decoder::InstructionToString(inst).c_str(), inst.pc,
+					     memory.data_dwords);
 			}
 			break;
-		default: return false;
+		default:
+			EXIT("opcode %s at pc 0x%08x has unsupported buffer store width %u",
+			     Decoder::InstructionToString(inst).c_str(), inst.pc, memory.data_bits);
 	}
 	ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, value, ir.GetExec()},
 	        AddMemoryInfo(memory, inst.pc));
-	return true;
 }
 
-bool Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetBufferResource(memory);
 	const auto address  = ReadBufferAddress(inst, 1);
@@ -536,10 +551,9 @@ bool Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	if (inst.glc) {
 		WriteOperand(inst.dst, result);
 	}
-	return true;
 }
 
-bool Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
@@ -549,10 +563,9 @@ bool Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
 	if (inst.glc) {
 		WriteOperand(inst.dst, result);
 	}
-	return true;
 }
 
-bool Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
                            bool returns_value) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
 	const auto address = ReadU32(MemorySourceAt(inst, 1));
@@ -561,10 +574,9 @@ bool Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opc
 	if (returns_value) {
 		WriteOperand(inst.dst, result);
 	}
-	return true;
 }
 
-bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
+void Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
 	const auto      memory = MemoryInfoFromDecoded(inst);
 	IR::ValueOpcode opcode;
 	const auto      bits = memory.data_bits;
@@ -573,7 +585,9 @@ bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
 		case 8u: opcode = IR::ValueOpcode::LoadAddressU8; break;
 		case 16u: opcode = IR::ValueOpcode::LoadAddressU16; break;
 		case 32u: opcode = IR::ValueOpcode::LoadAddressU32; break;
-		default: return false;
+		default:
+			EXIT("opcode %s at pc 0x%08x has unsupported flat load width %u",
+			     Decoder::InstructionToString(inst).c_str(), inst.pc, bits);
 	}
 	const auto address = ReadAddressOperands(inst, 0);
 	const auto active  = ir.GetExec();
@@ -588,10 +602,9 @@ bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
 		WriteOperand(OffsetOperand(inst.dst, index),
 		             bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
 	}
-	return true;
 }
 
-bool Translator::FLAT_STORE(const Decoder::Instruction& inst) {
+void Translator::FLAT_STORE(const Decoder::Instruction& inst) {
 	const auto      memory  = MemoryInfoFromDecoded(inst);
 	const auto      data_op = MemorySourceAt(inst, 0);
 	const auto      address = ReadAddressOperands(inst, 1);
@@ -600,7 +613,9 @@ bool Translator::FLAT_STORE(const Decoder::Instruction& inst) {
 		case 8u: opcode = IR::ValueOpcode::StoreAddressU8; break;
 		case 16u: opcode = IR::ValueOpcode::StoreAddressU16; break;
 		case 32u: opcode = IR::ValueOpcode::StoreAddressU32; break;
-		default: return false;
+		default:
+			EXIT("opcode %s at pc 0x%08x has unsupported flat store width %u",
+			     Decoder::InstructionToString(inst).c_str(), inst.pc, memory.data_bits);
 	}
 	const auto count = memory.data_bits == 32u ? memory.data_dwords : 1u;
 	for (uint32_t index = 0; index < count; index++) {
@@ -615,20 +630,18 @@ bool Translator::FLAT_STORE(const Decoder::Instruction& inst) {
 		ir.Emit(opcode, {address.resource, address.low, address.high, value, ir.GetExec()},
 		        AddMemoryInfo(component, inst.pc));
 	}
-	return true;
 }
 
-bool Translator::IMAGE_GET_RESINFO(const Decoder::Instruction& inst) {
+void Translator::IMAGE_GET_RESINFO(const Decoder::Instruction& inst) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 0));
 	const auto result   = ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address},
 	                              AddMemoryInfo(memory, inst.pc));
 	WriteImageComponents(inst.dst, result, memory, 4u);
-	return true;
 }
 
-bool Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) {
+void Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto sampler  = GetSamplerResource(memory);
@@ -636,30 +649,27 @@ bool Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) {
 	const auto result   = ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address},
 	                              AddMemoryInfo(memory, inst.pc));
 	WriteImageComponents(inst.dst, result, memory, 2u);
-	return true;
 }
 
-bool Translator::IMAGE_LOAD(const Decoder::Instruction& inst) {
+void Translator::IMAGE_LOAD(const Decoder::Instruction& inst) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 0));
 	const auto result   = ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()},
 	                              AddMemoryInfo(memory, inst.pc));
 	WriteImageComponents(inst.dst, result, memory, 4u);
-	return true;
 }
 
-bool Translator::IMAGE_STORE(const Decoder::Instruction& inst) {
+void Translator::IMAGE_STORE(const Decoder::Instruction& inst) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto address  = MakeImageAddress(inst, MemorySourceAt(inst, 1));
 	const auto data     = ConstructU32x4(MemorySourceAt(inst, 0), memory.data_dwords);
 	ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()},
 	        AddMemoryInfo(memory, inst.pc));
-	return true;
 }
 
-bool Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) {
+void Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto sampler  = GetSamplerResource(memory);
@@ -676,10 +686,9 @@ bool Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) {
 	} else {
 		WriteImageComponents(inst.dst, result, memory, 4u);
 	}
-	return true;
 }
 
-bool Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
+void Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
 	const auto memory   = MemoryInfoFromDecoded(inst);
 	const auto resource = GetImageResource(memory);
 	const auto sampler  = GetSamplerResource(memory);
@@ -690,7 +699,6 @@ bool Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
 		WriteOperand(OffsetOperand(inst.dst, index),
 		             ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(index)}));
 	}
-	return true;
 }
 
 IR::Value Translator::LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory,
@@ -746,7 +754,7 @@ void Translator::WriteSharedU32(uint32_t width, IR::U32 address,
 	}
 }
 
-bool Translator::DS_READ(const Decoder::Instruction& inst) {
+void Translator::DS_READ(const Decoder::Instruction& inst) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
 	const auto address = ReadU32(MemorySourceAt(inst, 0));
 	if (memory.data_bits == 32u) {
@@ -755,16 +763,15 @@ bool Translator::DS_READ(const Decoder::Instruction& inst) {
 		for (uint32_t index = 0; index < width; index++) {
 			WriteOperand(OffsetOperand(inst.dst, index), ExtractSharedU32(loaded, width, index));
 		}
-		return true;
+		return;
 	}
 	const auto opcode =
 	    memory.data_bits == 8u ? IR::ValueOpcode::LoadSharedU8 : IR::ValueOpcode::LoadSharedU16;
 	const auto loaded = ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(memory, inst.pc));
 	WriteOperand(inst.dst, WidenSubdword(loaded, memory.data_bits, memory.data_signed));
-	return true;
 }
 
-bool Translator::DS_READ2(const Decoder::Instruction& inst) {
+void Translator::DS_READ2(const Decoder::Instruction& inst) {
 	const auto memory       = MemoryInfoFromDecoded(inst);
 	const auto width        = memory.data_dwords / 2u;
 	const auto address      = ReadU32(MemorySourceAt(inst, 0));
@@ -783,10 +790,9 @@ bool Translator::DS_READ2(const Decoder::Instruction& inst) {
 		WriteOperand(OffsetOperand(inst.dst, width + index),
 		             ExtractSharedU32(second_value, width, index));
 	}
-	return true;
 }
 
-bool Translator::DS_WRITE(const Decoder::Instruction& inst) {
+void Translator::DS_WRITE(const Decoder::Instruction& inst) {
 	const auto               memory       = MemoryInfoFromDecoded(inst);
 	const auto               width        = memory.data_dwords;
 	const auto               data_operand = MemorySourceAt(inst, 0);
@@ -797,16 +803,15 @@ bool Translator::DS_WRITE(const Decoder::Instruction& inst) {
 	const auto address = ReadU32(MemorySourceAt(inst, 1));
 	if (memory.data_bits == 32u) {
 		WriteSharedU32(width, address, values, memory, inst.pc);
-		return true;
+		return;
 	}
 	const auto opcode =
 	    memory.data_bits == 8u ? IR::ValueOpcode::WriteSharedU8 : IR::ValueOpcode::WriteSharedU16;
 	ir.Emit(opcode, {address, NarrowSubdword(IR::U32(values[0]), memory.data_bits), ir.GetExec()},
 	        AddMemoryInfo(memory, inst.pc));
-	return true;
 }
 
-bool Translator::DS_WRITE2(const Decoder::Instruction& inst) {
+void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
 	const auto               memory      = MemoryInfoFromDecoded(inst);
 	const auto               width       = memory.data_dwords / 2u;
 	const auto               address     = ReadU32(MemorySourceAt(inst, 1));
@@ -827,28 +832,25 @@ bool Translator::DS_WRITE2(const Decoder::Instruction& inst) {
 		second.offset = memory.secondary_offset;
 		WriteSharedU32(width, address, second_values, second, inst.pc);
 	}
-	return true;
 }
 
-bool Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	ir.Emit(opcode,
 	        {ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)),
 	         ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()},
 	        AddMemoryInfo(memory, inst.pc));
-	return true;
 }
 
-bool Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
+void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	WriteOperand(inst.dst, ir.Emit(opcode,
 	                               {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(),
 	                                ir.GetExecHi()},
 	                               AddMemoryInfo(memory, inst.pc)));
-	return true;
 }
 
-bool Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
+void Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
 	const auto memory = MemoryInfoFromDecoded(inst);
 	const auto base =
 	    ir.BitwiseAnd(ReadU32(MemorySourceAt(inst, write ? 1u : 0u)), IR::U32(IR::Value(0xffffu)));
@@ -862,24 +864,21 @@ bool Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
 		WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32, {address, ir.GetExec()},
 		                               AddMemoryInfo(memory, inst.pc)));
 	}
-	return true;
 }
 
-bool Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
+void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
 	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::SwizzleU32,
 	                               {ReadU32(MemorySourceAt(inst, 0)),
 	                                ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()}));
-	return true;
 }
 
-bool Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
+void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
 	const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset)));
 	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32,
 	                               {ReadU32(inst.src1), address, ir.GetExec()}));
-	return true;
 }
 
-bool Translator::EmitMemory(const Decoder::Instruction& inst) {
+void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
 		case Decoder::Opcode::S_LOAD_DWORD:
 		case Decoder::Opcode::S_LOAD_DWORDX2:
@@ -891,6 +890,17 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4:
 		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8:
 		case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return S_LOAD(inst, false);
+		case Decoder::Opcode::S_MEMREALTIME: {
+			static std::atomic_flag warned = ATOMIC_FLAG_INIT;
+			if (!warned.test_and_set(std::memory_order_relaxed)) {
+				Log::WriteToConsoleAndLog(
+				    "Warning: S_MEMREALTIME uses placeholder UINT64_MAX; real-time clock not implemented.\n");
+			}
+			for (uint32_t component = 0; component < 2; component++) {
+				WriteOperand(ScalarDestinationOperand(inst.dst, component), IR::Value(UINT32_MAX));
+			}
+			return;
+		}
 
 		case Decoder::Opcode::BUFFER_LOAD_UBYTE:
 		case Decoder::Opcode::BUFFER_LOAD_SBYTE:
@@ -1087,7 +1097,7 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::DS_WRITE_B64:
 		case Decoder::Opcode::DS_WRITE_B96:
 		case Decoder::Opcode::DS_WRITE_B128: return DS_WRITE(inst);
-		default: return false;
+		default: return FailMissingTranslation(inst);
 	}
 }
 

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -2,106 +2,106 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::EmitScalar(const Decoder::Instruction& inst) {
+void Translator::EmitScalar(const Decoder::Instruction& inst) {
 	using O = Decoder::Opcode;
 	switch (inst.opcode) {
 		case O::S_MOV_B32:
-		case O::S_MOVK_I32: MOV_B32(inst, false); return true;
-		case O::S_MOV_B64: S_MOV_B64(inst); return true;
-		case O::S_WQM_B32: S_WQM(inst, false); return true;
-		case O::S_WQM_B64: S_WQM(inst, true); return true;
-		case O::S_GETPC_B64: S_GETPC_B64(inst); return true;
-		case O::S_SETPC_B64: return true;
-		case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return true;
-		case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return true;
-		case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true;
-		case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return true;
-		case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return true;
-		case O::S_SETREG_B32: EmitControlNop(); return true;
-		case O::S_WAITCNT: EmitWaitcnt(); return true;
+		case O::S_MOVK_I32: MOV_B32(inst, false); return;
+		case O::S_MOV_B64: S_MOV_B64(inst); return;
+		case O::S_WQM_B32: S_WQM(inst, false); return;
+		case O::S_WQM_B64: S_WQM(inst, true); return;
+		case O::S_GETPC_B64: S_GETPC_B64(inst); return;
+		case O::S_SETPC_B64: return;
+		case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return;
+		case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return;
+		case O::S_CSELECT_B32: S_CSELECT_B32(inst); return;
+		case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return;
+		case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return;
+		case O::S_SETREG_B32: EmitControlNop(); return;
+		case O::S_WAITCNT: EmitWaitcnt(); return;
 
 		case O::S_AND_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
-			return true;
+			return;
 		case O::S_ANDN1_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, false);
-			return true;
+			return;
 		case O::S_ORN2_SAVEEXEC_B32:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, false);
-			return true;
+			return;
 		case O::S_AND_SAVEEXEC_B64:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, true);
-			return true;
+			return;
 		case O::S_ANDN1_SAVEEXEC_B64:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, true);
-			return true;
+			return;
 		case O::S_ORN2_SAVEEXEC_B64:
 			S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, true);
-			return true;
-		case O::S_ADD_U32: ADD_U32(inst, false, false); return true;
-		case O::S_ADDC_U32: ADD_U32(inst, false, true); return true;
-		case O::S_SUB_U32: SUB_U32(inst, false, false); return true;
-		case O::S_SUBB_U32: SUBB_U32(inst, false, false); return true;
-		case O::S_ABSDIFF_I32: S_ABSDIFF_I32(inst); return true;
-		case O::S_ADD_I32: S_ADD_SUB_I32(inst, false); return true;
-		case O::S_SUB_I32: S_ADD_SUB_I32(inst, true); return true;
-		case O::S_LSHL1_ADD_U32: S_LSHL_ADD_U32(inst, 1u); return true;
-		case O::S_LSHL2_ADD_U32: S_LSHL_ADD_U32(inst, 2u); return true;
-		case O::S_LSHL3_ADD_U32: S_LSHL_ADD_U32(inst, 3u); return true;
-		case O::S_LSHL4_ADD_U32: S_LSHL_ADD_U32(inst, 4u); return true;
+			return;
+		case O::S_ADD_U32: ADD_U32(inst, false, false); return;
+		case O::S_ADDC_U32: ADD_U32(inst, false, true); return;
+		case O::S_SUB_U32: SUB_U32(inst, false, false); return;
+		case O::S_SUBB_U32: SUBB_U32(inst, false, false); return;
+		case O::S_ABSDIFF_I32: S_ABSDIFF_I32(inst); return;
+		case O::S_ADD_I32: S_ADD_SUB_I32(inst, false); return;
+		case O::S_SUB_I32: S_ADD_SUB_I32(inst, true); return;
+		case O::S_LSHL1_ADD_U32: S_LSHL_ADD_U32(inst, 1u); return;
+		case O::S_LSHL2_ADD_U32: S_LSHL_ADD_U32(inst, 2u); return;
+		case O::S_LSHL3_ADD_U32: S_LSHL_ADD_U32(inst, 3u); return;
+		case O::S_LSHL4_ADD_U32: S_LSHL_ADD_U32(inst, 4u); return;
 		case O::S_MIN_I32:
 			ScalarMinMax32(inst, IR::ValueOpcode::SMin32, IR::ValueOpcode::SLessThan32);
-			return true;
+			return;
 		case O::S_MAX_I32:
 			ScalarMinMax32(inst, IR::ValueOpcode::SMax32, IR::ValueOpcode::SGreaterThan32);
-			return true;
+			return;
 		case O::S_MIN_U32:
 			ScalarMinMax32(inst, IR::ValueOpcode::UMin32, IR::ValueOpcode::ULessThan32);
-			return true;
+			return;
 		case O::S_MAX_U32:
 			ScalarMinMax32(inst, IR::ValueOpcode::UMax32, IR::ValueOpcode::UGreaterThan32);
-			return true;
+			return;
 
 		case O::S_CMP_EQ_U32:
 		case O::S_CMP_EQ_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_LG_U32:
 		case O::S_CMP_LG_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_GT_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_GE_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true,
 			                   false);
-			return true;
+			return;
 		case O::S_CMP_LT_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_LE_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_GT_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_GE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true,
 			                   false);
-			return true;
+			return;
 		case O::S_CMP_LT_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_LE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true, false);
-			return true;
+			return;
 		case O::S_CMP_EQ_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true, false);
-			return true;
+			return;
 		case O::S_CMP_LG_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true, false);
-			return true;
+			return;
 
 		case O::S_AND_B64:
 			return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32,
@@ -217,12 +217,12 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_NOP:
 		case O::S_SLEEP:
 		case O::S_SETPRIO:
-		case O::S_TRAP: EmitControlNop(); return true;
-		case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return true;
-		case O::S_BARRIER: S_BARRIER(); return true;
-		case O::S_SENDMSG: S_SENDMSG(inst); return true;
-		case O::S_TTRACEDATA: S_TTRACEDATA(); return true;
-		case O::S_INST_PREFETCH: S_INST_PREFETCH(); return true;
+		case O::S_TRAP: EmitControlNop(); return;
+		case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return;
+		case O::S_BARRIER: S_BARRIER(); return;
+		case O::S_SENDMSG: S_SENDMSG(inst); return;
+		case O::S_TTRACEDATA: S_TTRACEDATA(); return;
+		case O::S_INST_PREFETCH: S_INST_PREFETCH(); return;
 		case O::S_BRANCH:
 		case O::S_CBRANCH_SCC0:
 		case O::S_CBRANCH_SCC1:
@@ -231,8 +231,8 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_CBRANCH_EXECZ:
 		case O::S_CBRANCH_EXECNZ:
 		case O::S_CBRANCH_CDBGSYS:
-		case O::S_ENDPGM: return true;
-		default: return false;
+		case O::S_ENDPGM: return;
+		default: return FailMissingTranslation(inst);
 	}
 }
 

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -80,34 +80,34 @@ private:
 	BufferAddress ReadBufferAddress(const Decoder::Instruction& inst, uint32_t source_offset);
 	IR::U32       WidenSubdword(IR::Value value, uint32_t bits, bool sign);
 	IR::Value     NarrowSubdword(IR::U32 value, uint32_t bits);
-	bool          S_LOAD(const Decoder::Instruction& inst, bool raw);
-	bool          BUFFER_LOAD(const Decoder::Instruction& inst);
-	bool          BUFFER_STORE(const Decoder::Instruction& inst);
-	bool          BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
-	bool FLAT_LOAD(const Decoder::Instruction& inst);
-	bool FLAT_STORE(const Decoder::Instruction& inst);
-	bool IMAGE_GET_RESINFO(const Decoder::Instruction& inst);
-	bool IMAGE_GET_LOD(const Decoder::Instruction& inst);
-	bool IMAGE_LOAD(const Decoder::Instruction& inst);
-	bool IMAGE_STORE(const Decoder::Instruction& inst);
-	bool IMAGE_SAMPLE(const Decoder::Instruction& inst);
-	bool IMAGE_GATHER(const Decoder::Instruction& inst);
+	void          S_LOAD(const Decoder::Instruction& inst, bool raw);
+	void          BUFFER_LOAD(const Decoder::Instruction& inst);
+	void          BUFFER_STORE(const Decoder::Instruction& inst);
+	void          BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void          IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
+	void FLAT_LOAD(const Decoder::Instruction& inst);
+	void FLAT_STORE(const Decoder::Instruction& inst);
+	void IMAGE_GET_RESINFO(const Decoder::Instruction& inst);
+	void IMAGE_GET_LOD(const Decoder::Instruction& inst);
+	void IMAGE_LOAD(const Decoder::Instruction& inst);
+	void IMAGE_STORE(const Decoder::Instruction& inst);
+	void IMAGE_SAMPLE(const Decoder::Instruction& inst);
+	void IMAGE_GATHER(const Decoder::Instruction& inst);
 	IR::Value LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory,
 	                        uint32_t pc);
 	IR::Value ExtractSharedU32(IR::Value value, uint32_t width, uint32_t index);
 	void WriteSharedU32(uint32_t width, IR::U32 address, const std::array<IR::Value, 4>& values,
 	                    const IR::MemoryInfo& memory, uint32_t pc);
-	bool DS_READ(const Decoder::Instruction& inst);
-	bool DS_READ2(const Decoder::Instruction& inst);
-	bool DS_WRITE(const Decoder::Instruction& inst);
-	bool DS_WRITE2(const Decoder::Instruction& inst);
-	bool DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool DS_ADDTID(const Decoder::Instruction& inst, bool write);
-	bool DS_SWIZZLE_B32(const Decoder::Instruction& inst);
-	bool DS_BPERMUTE_B32(const Decoder::Instruction& inst);
+	void DS_READ(const Decoder::Instruction& inst);
+	void DS_READ2(const Decoder::Instruction& inst);
+	void DS_WRITE(const Decoder::Instruction& inst);
+	void DS_WRITE2(const Decoder::Instruction& inst);
+	void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void DS_ADDTID(const Decoder::Instruction& inst, bool write);
+	void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
+	void DS_BPERMUTE_B32(const Decoder::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -124,7 +124,7 @@ private:
 	                          bool signed_value, bool cmpx);
 	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
 	                      bool cmpx);
-	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered);
+	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx);
 	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx);
 	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
 	void V_CVT_F32_U32(const Decoder::Instruction& inst);
@@ -143,77 +143,77 @@ private:
 	void V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
 	void V_CVT_PK_U8_F32(const Decoder::Instruction& inst);
 	void V_PACK_B32_F16(const Decoder::Instruction& inst);
-	bool PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
+	void PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
 	                   bool quiet_snan);
-	bool Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	void Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
 	                  bool invalid_negative);
-	bool Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
-	bool Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
+	void Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
+	void Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
 	                    bool mix);
-	bool FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
-	bool FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
+	void FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
+	void FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
 	                  bool mix);
-	bool V_FREXP_MANT_F32(const Decoder::Instruction& inst);
-	bool V_DOT2C_F32_F16(const Decoder::Instruction& inst);
-	bool V_CUBEID_F32(const Decoder::Instruction& inst);
-	bool V_CUBESC_F32(const Decoder::Instruction& inst);
-	bool V_CUBETC_F32(const Decoder::Instruction& inst);
-	bool V_CUBEMA_F32(const Decoder::Instruction& inst);
-	bool FloatCube(const Decoder::Instruction& inst, uint32_t result_kind);
-	bool Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
-	bool Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
-	bool V_MAD_I16(const Decoder::Instruction& inst);
-	bool V_MED3_I16(const Decoder::Instruction& inst);
-	bool PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	void V_FREXP_MANT_F32(const Decoder::Instruction& inst);
+	void V_DOT2C_F32_F16(const Decoder::Instruction& inst);
+	void V_CUBEID_F32(const Decoder::Instruction& inst);
+	void V_CUBESC_F32(const Decoder::Instruction& inst);
+	void V_CUBETC_F32(const Decoder::Instruction& inst);
+	void V_CUBEMA_F32(const Decoder::Instruction& inst);
+	void FloatCube(const Decoder::Instruction& inst, uint32_t result_kind);
+	void Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
+	void Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
+	void V_MAD_I16(const Decoder::Instruction& inst);
+	void V_MED3_I16(const Decoder::Instruction& inst);
+	void PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
 	                          bool arithmetic);
-	bool PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
-	bool PackedInteger16Mad(const Decoder::Instruction& inst, bool sign);
-	bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
-	bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
+	void PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
+	void PackedInteger16Mad(const Decoder::Instruction& inst, bool sign);
+	void PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
+	void S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
 	                IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary);
 	IR::U1 U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs,
 	                     bool negate_result);
-	bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
+	void SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
 	                   bool reverse, bool mask_shift_count, bool update_scc);
-	bool S_ASHR_I64(const Decoder::Instruction& inst);
-	bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
+	void S_ASHR_I64(const Decoder::Instruction& inst);
+	void ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
 	                           bool negate_rhs, bool negate_result, bool update_scc);
-	bool V_AND_OR_B32(const Decoder::Instruction& inst);
-	bool V_OR3_B32(const Decoder::Instruction& inst);
-	bool V_XOR3_B32(const Decoder::Instruction& inst);
-	bool S_FF1_I32_B64(const Decoder::Instruction& inst);
-	bool V_FFBH_32(const Decoder::Instruction& inst, bool sign);
-	bool S_FLBIT_I32_B64(const Decoder::Instruction& inst);
-	bool Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
-	bool V_MAD_U64_U32(const Decoder::Instruction& inst);
-	bool V_SAD_U32(const Decoder::Instruction& inst);
-	bool V_ADD3_U32(const Decoder::Instruction& inst);
-	bool S_BITSET_B32(const Decoder::Instruction& inst, bool set);
-	bool S_BITSET_B64(const Decoder::Instruction& inst, bool set);
-	bool V_BCNT_U32_B32(const Decoder::Instruction& inst);
-	bool V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
-	bool S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
-	bool S_QUADMASK_B64(const Decoder::Instruction& inst);
-	bool BFM_B32(const Decoder::Instruction& inst);
+	void V_AND_OR_B32(const Decoder::Instruction& inst);
+	void V_OR3_B32(const Decoder::Instruction& inst);
+	void V_XOR3_B32(const Decoder::Instruction& inst);
+	void S_FF1_I32_B64(const Decoder::Instruction& inst);
+	void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
+	void S_FLBIT_I32_B64(const Decoder::Instruction& inst);
+	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
+	void V_MAD_U64_U32(const Decoder::Instruction& inst);
+	void V_SAD_U32(const Decoder::Instruction& inst);
+	void V_ADD3_U32(const Decoder::Instruction& inst);
+	void S_BITSET_B32(const Decoder::Instruction& inst, bool set);
+	void S_BITSET_B64(const Decoder::Instruction& inst, bool set);
+	void V_BCNT_U32_B32(const Decoder::Instruction& inst);
+	void V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
+	void S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
+	void S_QUADMASK_B64(const Decoder::Instruction& inst);
+	void BFM_B32(const Decoder::Instruction& inst);
 	IR::U32 RightMask32(IR::U32 count);
 	IR::U64 RightMask64(IR::U32 count);
-	bool    S_BFM_B64(const Decoder::Instruction& inst);
-	bool    S_BFE_U32(const Decoder::Instruction& inst, bool sign);
-	bool    S_BFE_U64(const Decoder::Instruction& inst);
-	bool    V_BFE_U32(const Decoder::Instruction& inst, bool sign);
-	bool    V_BFI_B32(const Decoder::Instruction& inst);
-	bool    S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
-	bool    S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
-	bool    V_ALIGNBIT_B32(const Decoder::Instruction& inst);
-	bool    V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
-	bool    V_LSHL_ADD_U32(const Decoder::Instruction& inst);
-	bool    V_ADD_LSHL_U32(const Decoder::Instruction& inst);
-	bool    V_XAD_U32(const Decoder::Instruction& inst);
-	bool    V_LSHL_OR_B32(const Decoder::Instruction& inst);
-	bool    V_CNDMASK_B32(const Decoder::Instruction& inst);
-	bool    PackB16(const Decoder::Instruction& inst, bool high0, bool high1);
+	void    S_BFM_B64(const Decoder::Instruction& inst);
+	void    S_BFE_U32(const Decoder::Instruction& inst, bool sign);
+	void    S_BFE_U64(const Decoder::Instruction& inst);
+	void    V_BFE_U32(const Decoder::Instruction& inst, bool sign);
+	void    V_BFI_B32(const Decoder::Instruction& inst);
+	void    S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
+	void    S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
+	void    V_ALIGNBIT_B32(const Decoder::Instruction& inst);
+	void    V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
+	void    V_LSHL_ADD_U32(const Decoder::Instruction& inst);
+	void    V_ADD_LSHL_U32(const Decoder::Instruction& inst);
+	void    V_XAD_U32(const Decoder::Instruction& inst);
+	void    V_LSHL_OR_B32(const Decoder::Instruction& inst);
+	void    V_CNDMASK_B32(const Decoder::Instruction& inst);
+	void    PackB16(const Decoder::Instruction& inst, bool high0, bool high1);
 
 	void S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin);
 	void S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec,
@@ -249,10 +249,11 @@ private:
 	void V_INTERP_MOV_F32(const Decoder::Instruction& inst);
 	void EXP(const Decoder::Instruction& inst);
 
-	bool EmitScalar(const Decoder::Instruction& inst);
-	bool EmitVector(const Decoder::Instruction& inst);
-	bool EmitInterpolation(const Decoder::Instruction& inst);
-	bool EmitMemory(const Decoder::Instruction& inst);
+	void EmitScalar(const Decoder::Instruction& inst);
+	void EmitVector(const Decoder::Instruction& inst);
+	void EmitInterpolation(const Decoder::Instruction& inst);
+	void EmitMemory(const Decoder::Instruction& inst);
+	static void FailMissingTranslation(const Decoder::Instruction& inst);
 
 	IR::Program&    program;
 	IR::IREmitter   ir;

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -2,318 +2,322 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-bool Translator::EmitVector(const Decoder::Instruction& inst) {
+void Translator::EmitVector(const Decoder::Instruction& inst) {
 	using O = Decoder::Opcode;
 	switch (inst.opcode) {
-		case O::V_NOP: return true;
-		case O::V_ADD_I32: ADD_U32(inst, true, false); return true;
-		case O::V_ADDC_U32: ADD_U32(inst, true, true); return true;
-		case O::V_SUB_I32: SUB_U32(inst, true, false); return true;
-		case O::V_SUBREV_I32: SUB_U32(inst, true, true); return true;
-		case O::V_SUB_CO_CI_U32: SUBB_U32(inst, true, false); return true;
-		case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true, true); return true;
+		case O::V_NOP: return;
+		case O::V_ADD_I32: ADD_U32(inst, true, false); return;
+		case O::V_ADDC_U32: ADD_U32(inst, true, true); return;
+		case O::V_SUB_I32: SUB_U32(inst, true, false); return;
+		case O::V_SUBREV_I32: SUB_U32(inst, true, true); return;
+		case O::V_SUB_CO_CI_U32: SUBB_U32(inst, true, false); return;
+		case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true, true); return;
 
-		case O::V_MOV_B32: MOV_B32(inst, true); return true;
-		case O::V_MOVRELS_B32: V_MOVRELS_B32(inst); return true;
-		case O::V_MOVRELD_B32: V_MOVRELD_B32(inst); return true;
-		case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return true;
-		case O::V_READLANE_B32: V_READLANE_B32(inst); return true;
-		case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return true;
-		case O::V_PERMLANE16_B32: V_PERMLANE16_B32(inst, false); return true;
-		case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return true;
+		case O::V_MOV_B32: MOV_B32(inst, true); return;
+		case O::V_MOVRELS_B32: V_MOVRELS_B32(inst); return;
+		case O::V_MOVRELD_B32: V_MOVRELD_B32(inst); return;
+		case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return;
+		case O::V_READLANE_B32: V_READLANE_B32(inst); return;
+		case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return;
+		case O::V_PERMLANE16_B32: V_PERMLANE16_B32(inst, false); return;
+		case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return;
 
 		case O::V_CMP_F_I32:
-		case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return true;
+		case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return;
 		case O::V_CMP_T_I32:
-		case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return true;
+		case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return;
 		case O::V_CMP_EQ_U32:
 		case O::V_CMP_EQ_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_EQ_U32:
 		case O::V_CMPX_EQ_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NE_U32:
 		case O::V_CMP_NE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NE_U32:
 		case O::V_CMPX_NE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GT_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_GT_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GE_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
 			                   false);
-			return true;
+			return;
 		case O::V_CMPX_GE_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
 			                   true);
-			return true;
+			return;
 		case O::V_CMP_LT_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_LT_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_LE_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false,
 			                   false);
-			return true;
+			return;
 		case O::V_CMPX_LE_U32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GT_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_GT_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
 			                   false);
-			return true;
+			return;
 		case O::V_CMPX_GE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
 			                   true);
-			return true;
+			return;
 		case O::V_CMP_LT_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_LT_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_LE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false,
 			                   false);
-			return true;
+			return;
 		case O::V_CMPX_LE_I32:
 			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true);
-			return true;
+			return;
 		case O::V_CMP_EQ_I64:
 		case O::V_CMP_EQ_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false);
-			return true;
+			return;
 		case O::V_CMP_LT_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, false);
-			return true;
+			return;
 		case O::V_CMP_GT_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
-			return true;
+			return;
 		case O::V_CMP_NE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NE_I64:
 		case O::V_CMPX_NE_U64:
 			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
-			return true;
+			return;
 
 		case O::V_CMP_EQ_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMP_EQ_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMP_NE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMP_NE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMP_GT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_GT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMP_LT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_LT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, true);
-			return true;
+			return;
+		case O::V_CMPX_EQ_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, true);
+			return;
 		case O::V_CMP_LE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMP_GT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
-			return true;
+			return;
 		case O::V_CMP_GE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMP_LT_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
-			return true;
+			return;
 		case O::V_CMP_LE_I16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
-			return true;
+			return;
 
-		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return true;
-		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return true;
+		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return;
+		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return;
 		case O::V_CMP_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_EQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_LG_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_LG_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_GT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, true);
-			return true;
+			return;
 		case O::V_CMP_GE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_GE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_LT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_LT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, true);
-			return true;
+			return;
 		case O::V_CMP_LE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_LE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NLG_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NLG_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NEQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NEQ_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NLE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NLE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NLT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NLT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NGE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NGE_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, true);
-			return true;
+			return;
 		case O::V_CMP_NGT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, false);
-			return true;
+			return;
 		case O::V_CMPX_NGT_F32:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, true);
-			return true;
+			return;
 		case O::V_CMP_EQ_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_EQ_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, true);
-			return true;
+			return;
 		case O::V_CMP_LG_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMP_GT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_GT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, true);
-			return true;
+			return;
 		case O::V_CMP_GE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_GE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, true);
-			return true;
+			return;
 		case O::V_CMP_LT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_LT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, true);
-			return true;
+			return;
 		case O::V_CMP_LE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_LE_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
-			return true;
+			return;
 		case O::V_CMP_NGT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_NGT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, true);
-			return true;
+			return;
 		case O::V_CMP_NEQ_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_NEQ_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, true);
-			return true;
+			return;
 		case O::V_CMP_NLT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, false);
-			return true;
+			return;
 		case O::V_CMPX_NLT_F16:
 			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, true);
-			return true;
-		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true); return true;
-		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false); return true;
-		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return true;
-		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return true;
+			return;
+		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true, false); return;
+		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false, false); return;
+		case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true, true); return;
+		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return;
+		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return;
 
-		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return true;
-		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return true;
-		case O::V_CVT_F32_UBYTE2: V_CVT_F32_UBYTE(inst, 2); return true;
-		case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return true;
-		case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return true;
-		case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return true;
-		case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return true;
-		case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return true;
-		case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return true;
-		case O::V_CVT_F32_F16: V_CVT_F32_F16(inst); return true;
-		case O::V_CVT_F16_U16: V_CVT_F16_16(inst, false); return true;
-		case O::V_CVT_F16_I16: V_CVT_F16_16(inst, true); return true;
-		case O::V_CVT_U16_F16: V_CVT_16_F16(inst, false); return true;
-		case O::V_CVT_I16_F16: V_CVT_16_F16(inst, true); return true;
-		case O::V_CVT_RPI_I32_F32: V_CVT_RPI_I32_F32(inst); return true;
-		case O::V_CVT_FLR_I32_F32: V_CVT_FLR_I32_F32(inst); return true;
-		case O::V_FREXP_EXP_I32_F32: V_FREXP_EXP_I32_F32(inst); return true;
-		case O::V_CVT_OFF_F32_I4: V_CVT_OFF_F32_I4(inst); return true;
-		case O::V_CVT_PKRTZ_F16_F32: V_CVT_PKRTZ_F16_F32(inst); return true;
+		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return;
+		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return;
+		case O::V_CVT_F32_UBYTE2: V_CVT_F32_UBYTE(inst, 2); return;
+		case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return;
+		case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return;
+		case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return;
+		case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return;
+		case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return;
+		case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return;
+		case O::V_CVT_F32_F16: V_CVT_F32_F16(inst); return;
+		case O::V_CVT_F16_U16: V_CVT_F16_16(inst, false); return;
+		case O::V_CVT_F16_I16: V_CVT_F16_16(inst, true); return;
+		case O::V_CVT_U16_F16: V_CVT_16_F16(inst, false); return;
+		case O::V_CVT_I16_F16: V_CVT_16_F16(inst, true); return;
+		case O::V_CVT_RPI_I32_F32: V_CVT_RPI_I32_F32(inst); return;
+		case O::V_CVT_FLR_I32_F32: V_CVT_FLR_I32_F32(inst); return;
+		case O::V_FREXP_EXP_I32_F32: V_FREXP_EXP_I32_F32(inst); return;
+		case O::V_CVT_OFF_F32_I4: V_CVT_OFF_F32_I4(inst); return;
+		case O::V_CVT_PKRTZ_F16_F32: V_CVT_PKRTZ_F16_F32(inst); return;
 		case O::V_CVT_PKNORM_I16_F32:
 			V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackSnorm2x16);
-			return true;
+			return;
 		case O::V_CVT_PKNORM_U16_F32:
 			V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackUnorm2x16);
-			return true;
-		case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return true;
-		case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return true;
+			return;
+		case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return;
+		case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return;
 		case O::V_CVT_PK_U16_U32:
 		case O::V_CVT_PK_I16_I32: return PackB16(inst, false, false);
 
@@ -534,7 +538,7 @@ bool Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_XAD_U32: return V_XAD_U32(inst);
 		case O::V_LSHL_OR_B32: return V_LSHL_OR_B32(inst);
 		case O::V_CNDMASK_B32: return V_CNDMASK_B32(inst);
-		default: return false;
+		default: return FailMissingTranslation(inst);
 	}
 }
 

src/graphics/shader/recompiler/ir/Program.cpp

@@ -454,10 +454,8 @@ void ValidateProgram(const Program& program, bool require_ssa) {
 					                        ValueOpcodeName(inst.GetOpcode())));
 				}
 				if (memory.kind == ResourceKind::IndirectBuffer &&
-				    (memory.formatted || memory.typed ||
-				     (inst.GetOpcode() != ValueOpcode::LoadBufferU32x2 &&
-				      inst.GetOpcode() != ValueOpcode::LoadBufferU32x4))) {
-					return Fail("indirect buffer requires a raw DWORD x2/x4 load");
+				    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) {
+					return Fail("indirect buffer requires a raw DWORD x2/x3/x4 load");
 				}
 				if (buffer_components > 1u &&
 				    (!vector_buffer || memory.data_bits != 32u ||

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -67,8 +67,15 @@ struct MemoryInfo {
 	bool                    image_r128                                            = false;
 	bool                    idxen                                                 = false;
 	bool                    offen                                                 = false;
+	bool                    coherent                                              = false;
 	bool                    planning_only                                         = false;
 
+	[[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const {
+		return !formatted && !typed && data_bits == 32u &&
+		       (opcode == ValueOpcode::LoadBufferU32x2 || opcode == ValueOpcode::LoadBufferU32x3 ||
+		        opcode == ValueOpcode::LoadBufferU32x4);
+	}
+
 	bool operator==(const MemoryInfo& other) const = default;
 };
 
@@ -428,7 +435,7 @@ struct BindingLayout {
 };
 
 struct ShaderInfo {
-	static constexpr uint32_t MaxBuffers      = 32;
+	static constexpr uint32_t MaxBuffers      = 64;
 	static constexpr uint32_t MaxImages       = 64;
 	static constexpr uint32_t MaxSamplers     = 32;
 	static constexpr uint32_t MaxSampledPairs = 64;
@@ -463,9 +470,9 @@ struct DescriptorSource {
 		uint32_t table_source    = 0;
 		uint32_t selector_stride = 0;
 		uint32_t selector_offset = 0;
-		uint32_t key_arg         = 0;
 		uint32_t table_offset    = 0;
-		uint32_t key_count       = 0;
+		Value    key_count;
+		Value    selector_mask;
 
 		bool operator==(const IndirectImage& other) const = default;
 	};
@@ -541,6 +548,7 @@ struct ResourcePlan {
 	std::vector<SrtRead>                srt_reads;
 	std::vector<uint8_t>                clean_flat_slots;
 	bool                                requires_specialization_memory = false;
+	bool                                has_address_writes = false;
 	bool                                srt_plan_complete          = false;
 	bool                                resource_tracking_complete = false;
 	ShaderInfo                          info;
@@ -553,6 +561,7 @@ struct ResourcePlan {
 	mutable std::vector<uint8_t>            visited_blocks;
 	mutable std::vector<uint32_t>           pending_blocks;
 	mutable std::vector<uint32_t>           material_keys;
+	mutable std::vector<std::pair<uint64_t, uint64_t>> specialization_reads;
 };
 
 struct Program: ResourcePlan {

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -200,6 +200,13 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 	switch (inst.GetOpcode()) {
 		case ValueOpcode::Phi: FoldPhi(inst); return;
 		case ValueOpcode::SelectU1:
+			if (!FoldSelect(inst) && IsImmediate(Arg(inst, 2), Type::U1) &&
+			    !Arg(inst, 2).U1()) {
+				auto result = block.PrependNewInst(instruction, ValueOpcode::LogicalAnd,
+				                                   {Arg(inst, 0), Arg(inst, 1)});
+				Replace(inst, Value(&*result));
+			}
+			return;
 		case ValueOpcode::SelectU32:
 		case ValueOpcode::SelectF32: FoldSelect(inst); return;
 		case ValueOpcode::BitFieldInsert: {
@@ -431,6 +438,10 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			return;
 		case ValueOpcode::ShiftRightLogical32:
 			if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a >> (b & 31u); })) {
+				if (IsImmediate(Arg(inst, 0), Type::U32) && Arg(inst, 0).U32() == 0u) {
+					Replace(inst, Value(0u));
+					return;
+				}
 				const auto shift = Arg(inst, 1);
 				if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) {
 					Replace(inst, Arg(inst, 0));
@@ -573,6 +584,23 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 			if (!FoldLogical(inst, [](bool a, bool b) { return a && b; })) {
 				const auto lhs = Arg(inst, 0);
 				const auto rhs = Arg(inst, 1);
+				const auto simplify = [&](Value assumption, Value expression) {
+					const auto* disjunction = expression.TryInstruction();
+					if (disjunction == nullptr || disjunction->GetOpcode() != ValueOpcode::LogicalOr) {
+						return false;
+					}
+					for (uint32_t i = 0; i < 2u; ++i) {
+						const auto* inverse = disjunction->Arg(i).Resolve().TryInstruction();
+						if (inverse != nullptr && inverse->GetOpcode() == ValueOpcode::LogicalNot &&
+						    inverse->Arg(0).Resolve() == assumption) {
+							inst.SetArg(assumption == lhs ? 1u : 0u,
+							            disjunction->Arg(i ^ 1u));
+							return true;
+						}
+					}
+					return false;
+				};
+				if (simplify(lhs, rhs) || simplify(rhs, lhs)) return;
 				if (IsImmediate(lhs, Type::U1)) {
 					Replace(inst, lhs.U1() ? rhs : lhs);
 				} else if (IsImmediate(rhs, Type::U1)) {

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -138,6 +138,50 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou
 	return true;
 }
 
+struct ReadCapture {
+	SrtRuntime                                  source;
+	std::vector<std::pair<uint64_t, uint64_t>>& ranges;
+};
+
+bool CaptureStrictRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
+	auto& capture = *static_cast<ReadCapture*>(userdata);
+	if (!capture.source.read_specialization_memory(capture.source.userdata, address, values)) {
+		return false;
+	}
+	capture.ranges.emplace_back(address, values.size_bytes());
+	return true;
+}
+
+bool CaptureOrdinaryRead(void* userdata, uint64_t address, std::span<uint32_t> values) {
+	auto& capture = *static_cast<ReadCapture*>(userdata);
+	if (!capture.source.read_memory(capture.source.userdata, address, values)) {
+		return false;
+	}
+	capture.ranges.emplace_back(address, values.size_bytes());
+	return true;
+}
+
+bool WrittenBuffersDisjoint(const ResourcePlan& program, const ResourceSnapshot& snapshot,
+                            std::span<const std::pair<uint64_t, uint64_t>> reads) {
+	for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
+		if (!program.info.buffers[i].written) continue;
+		ShaderBufferResource buffer;
+		if (!DecodeBufferDescriptor(snapshot.buffers[i], buffer)) return false;
+		const auto base = buffer.Base48();
+		const auto size = buffer.GetSize();
+		if (size == 0u) continue;
+		if (size - 1u > AddressMask - base) return false;
+		for (const auto [address, bytes]: reads) {
+			if (bytes == 0u) continue;
+			if (address > AddressMask || bytes - 1u > AddressMask - address ||
+			    (address <= base ? base - address < bytes : address - base < size)) {
+				return false;
+			}
+		}
+	}
+	return true;
+}
+
 const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
 	if (source >= program.descriptor_sources.size()) {
 		return nullptr;
@@ -146,12 +190,15 @@ const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) {
 }
 
 void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* source,
-                        std::span<uint8_t> slots) {
-	if (source == nullptr) {
+                        std::span<uint8_t> slots, Value extra = {}) {
+	if (source == nullptr && extra.IsEmpty()) {
 		return;
 	}
-	std::vector<Value>       pending(source->dwords.begin(),
-	                                 source->dwords.begin() + source->dword_count);
+	std::vector<Value>       pending;
+	if (source != nullptr) {
+		pending.assign(source->dwords.begin(), source->dwords.begin() + source->dword_count);
+	}
+	if (!extra.IsEmpty()) pending.push_back(extra);
 	std::vector<const Inst*> visited;
 	while (!pending.empty()) {
 		auto value = pending.back().Resolve();
@@ -198,7 +245,8 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
                               const DescriptorSource::IndirectImage& indirect,
                               const DescriptorValue& material_value,
                               const DescriptorValue& table_value, uint32_t image_index,
-                              const SrtRuntime& runtime, ResourceSnapshot& snapshot,
+                              const SrtRuntime& runtime, SrtWalker& clean,
+                              ResourceSnapshot& snapshot,
                               ResourceSpecialization& specialization) {
 	uint64_t table_base = 0;
 	uint64_t table_size = UINT64_MAX; // Scalar addresses have no buffer descriptor bounds.
@@ -214,12 +262,41 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
 	auto& keys = program.material_keys;
 	keys.clear();
 	if (indirect.material_source == UINT32_MAX) {
-		if (table_value.dword_count != 2u || indirect.key_count == 0u ||
-		    indirect.key_count > MaxIndirectImageProbes) {
+		uint32_t key_count = 0;
+		const bool evaluated = clean.Evaluate(indirect.key_count, key_count);
+		if (std::bit_cast<int32_t>(key_count) <= 0) key_count = 0;
+		if (table_value.dword_count != 2u || !evaluated ||
+		    key_count > MaxIndirectImageProbes ||
+		    uint64_t {indirect.table_offset} + uint64_t {key_count} * 32u > UINT32_MAX + 1ull) {
 			return false;
 		}
-		keys.resize(indirect.key_count);
+		keys.resize(key_count);
 		std::iota(keys.begin(), keys.end(), 0u);
+	} else if (!indirect.selector_mask.IsEmpty()) {
+		uint32_t mask = 0;
+		uint32_t count = 0;
+		if (material_value.dword_count != 2u || table_value.dword_count != 2u ||
+		    !clean.Evaluate(indirect.selector_mask, mask) ||
+		    !clean.Evaluate(indirect.key_count, count) || count == 0u || count > 32u) {
+			return false;
+		}
+		if (count < 32u) mask &= (1u << count) - 1u;
+		const auto material_base =
+		    (static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
+		keys.reserve(std::popcount(mask));
+		while (mask != 0u) {
+			const auto index = std::countr_zero(mask);
+			const auto offset = static_cast<uint64_t>(indirect.selector_offset) +
+			                    static_cast<uint64_t>(index) * indirect.selector_stride;
+			if (offset > UINT32_MAX) return false;
+			uint32_t key = 0;
+			if (!ReadScalarTable(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
+			                     runtime, {&key, 1})) return false;
+			keys.push_back(key);
+			mask &= mask - 1u;
+		}
+		std::ranges::sort(keys);
+		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 	} else {
 		ShaderBufferResource material;
 		if (!DecodeBufferDescriptor(material_value, material) || table_value.dword_count != 4u ||
@@ -779,6 +856,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 	plan.memory_info                = program.memory_info;
 	plan.srt_plan_complete          = program.srt_plan_complete;
 	plan.resource_tracking_complete = program.resource_tracking_complete;
+	plan.has_address_writes         = program.has_address_writes;
 
 	std::unordered_map<const Inst*, Inst*> cloned;
 	std::function<Value(Value)>            Clone = [&](Value value) -> Value {
@@ -816,6 +894,10 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		auto& target          = plan.descriptor_sources.emplace_back();
 		target.dword_count    = source.dword_count;
 		target.indirect_image = source.indirect_image;
+		if (target.indirect_image.has_value()) {
+			target.indirect_image->key_count = Clone(target.indirect_image->key_count);
+			target.indirect_image->selector_mask = Clone(target.indirect_image->selector_mask);
+		}
 		for (uint32_t dword = 0; dword < source.dword_count; dword++) {
 			target.dwords[dword] = Clone(source.dwords[dword]);
 		}
@@ -840,7 +922,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
 		}
 		plan.requires_specialization_memory = true;
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
-		                   plan.clean_flat_slots);
+		                   plan.clean_flat_slots, source->indirect_image->selector_mask);
 		MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
 		                   plan.clean_flat_slots);
 	}
@@ -853,8 +935,29 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 	    (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) {
 		return false;
 	}
-	SrtWalker clean(program, CleanRuntime(runtime));
-	SrtWalker walker(program, runtime, program.clean_flat_slots, &clean);
+	const bool masked_image = std::ranges::any_of(program.info.images, [&](const auto& image) {
+		const auto* source = Source(program, image.source);
+		return source != nullptr && source->indirect_image.has_value() &&
+		       !source->indirect_image->selector_mask.IsEmpty();
+	});
+	if (masked_image &&
+	    (program.has_address_writes ||
+	     std::ranges::any_of(program.info.images, &ImageResource::written))) {
+		return false;
+	}
+	const bool capture_reads = masked_image &&
+	    std::ranges::any_of(program.info.buffers, &BufferResource::written);
+	auto& reads = program.specialization_reads;
+	ReadCapture capture {runtime, reads};
+	SrtRuntime observed = runtime;
+	if (capture_reads) {
+		reads.clear();
+		observed.userdata = &capture;
+		observed.read_specialization_memory = CaptureStrictRead;
+		if (observed.read_memory != nullptr) observed.read_memory = CaptureOrdinaryRead;
+	}
+	SrtWalker clean(program, CleanRuntime(observed));
+	SrtWalker walker(program, observed, program.clean_flat_slots, &clean);
 	const auto active = clean.FindActiveSources();
 	if (!walker.RefreshFlatBuffer(snapshot.flattened_srt)) {
 		return false;
@@ -888,11 +991,20 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			return false;
 		}
 	}
+	if (capture_reads) {
+		for (uint32_t i = 0; i < program.info.buffers.size(); ++i) {
+			const auto& buffer = program.info.buffers[i];
+			if (!buffer.written || (!active.empty() && !active[buffer.source])) continue;
+			DescriptorValue strict;
+			if (!clean.EvaluateDescriptor(buffer.source, strict) ||
+			    strict != snapshot.buffers[i]) return false;
+		}
+	}
 	snapshot.images.resize(program.info.images.size());
-	specialization.images.clear();
-	specialization.images.reserve(program.info.images.size());
-	for (const auto& image: program.info.images) {
-		specialization.images.push_back({
+	specialization.images.resize(program.info.images.size());
+	for (uint32_t i = 0; i < program.info.images.size(); ++i) {
+		const auto& image = program.info.images[i];
+		specialization.images[i] = {
 		    .numeric_class = image.numeric_class,
 		    .dimension = image.dimension,
 		    .mip_count = image.mip_count,
@@ -902,10 +1014,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		    .indirect_mapping_offset = image.indirect_mapping_offset,
 		    .indirect_search_iterations = image.indirect_search_iterations,
 		    .cube = image.cube,
-		});
-	}
-	for (uint32_t i = 0; i < program.info.images.size(); ++i) {
-		const auto& image = program.info.images[i];
+		};
 		const auto* source = Source(program, image.source);
 		if (source == nullptr) {
 			return false;
@@ -921,7 +1030,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			if ((indirect.material_source != UINT32_MAX &&
 			     !clean.EvaluateDescriptor(indirect.material_source, material)) ||
 			    !clean.EvaluateDescriptor(indirect.table_source, table) ||
-			    !MaterializeIndirectImage(program, indirect, material, table, i, runtime, snapshot,
+			    !MaterializeIndirectImage(program, indirect, material, table, i, observed, clean, snapshot,
 			                              specialization)) {
 				return false;
 			}
@@ -940,6 +1049,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			return false;
 		}
 	}
+	if (capture_reads && !WrittenBuffersDisjoint(program, snapshot, reads)) return false;
 	snapshot.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
 	return BuildResourceSpecialization(program, snapshot, specialization);
 }

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -313,9 +313,20 @@ private:
 		for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
 			const auto& current = m_sources[candidate];
 			if (current.dword_count != descriptor.dword_count ||
-			    current.indirect_image != descriptor.indirect_image) {
+			    current.indirect_image.has_value() != descriptor.indirect_image.has_value()) {
 				continue;
 			}
+			if (current.indirect_image.has_value()) {
+				const auto& a = *current.indirect_image;
+				const auto& b = *descriptor.indirect_image;
+				if (a.material_source != b.material_source || a.table_source != b.table_source ||
+				    a.selector_stride != b.selector_stride || a.selector_offset != b.selector_offset ||
+				    a.table_offset != b.table_offset ||
+				    !EquivalentValue(m_program, a.key_count, b.key_count) ||
+				    a.selector_mask.IsEmpty() != b.selector_mask.IsEmpty() ||
+				    (!a.selector_mask.IsEmpty() &&
+				     !EquivalentValue(m_program, a.selector_mask, b.selector_mask))) continue;
+			}
 			bool same = true;
 			for (uint32_t i = 0; i < descriptor.dword_count; i++) {
 				same = same && EquivalentValue(m_program, current.dwords[i], descriptor.dwords[i]);
@@ -504,6 +515,355 @@ private:
 		}
 	}
 
+	const Block* FindBlock(uint32_t id) const {
+		const auto info = std::ranges::find(m_program.block_info, id, &BlockInfo::id);
+		return info == m_program.block_info.end()
+		           ? nullptr
+		           : m_program.blocks[info - m_program.block_info.begin()];
+	}
+
+	bool CanReach(const Block* start, const Block* target, const Block* avoid) const {
+		std::vector<const Block*> pending {start};
+		std::vector<const Block*> visited;
+		while (!pending.empty()) {
+			const auto* block = pending.back();
+			pending.pop_back();
+			if (block == nullptr || block == avoid ||
+			    std::ranges::find(visited, block) != visited.end()) continue;
+			if (block == target) return true;
+			visited.push_back(block);
+			for (const auto* next: block->ImmSuccessors()) pending.push_back(next);
+		}
+		return false;
+	}
+
+	bool ConditionalEdge(const Block* from, const Block* to, Value& condition,
+	                     bool& positive) const {
+		const auto position = std::ranges::find(m_program.blocks, from);
+		const auto target = std::ranges::find(m_program.blocks, to);
+		if (position == m_program.blocks.end() || target == m_program.blocks.end()) return false;
+		const auto& info = m_program.block_info[position - m_program.blocks.begin()];
+		const auto& term = info.terminator;
+		const auto id = m_program.block_info[target - m_program.blocks.begin()].id;
+		if (term.kind != CFG::TerminatorKind::ConditionalBranch ||
+		    (term.true_block == id) == (term.false_block == id)) return false;
+		condition = info.condition;
+		positive = term.true_block == id;
+		while (const auto* inst = condition.Resolve().TryInstruction()) {
+			if (inst->GetOpcode() != ValueOpcode::LogicalNot) break;
+			condition = inst->Arg(0);
+			positive = !positive;
+		}
+		condition = condition.Resolve();
+		return true;
+	}
+
+	Value PositiveUseGuard(const Block* use) const {
+		if (use == nullptr || use->ImmPredecessors().size() != 1u) return {};
+		Value condition;
+		bool positive = false;
+		return ConditionalEdge(use->ImmPredecessors()[0], use, condition, positive) && positive
+		           ? condition : Value {};
+	}
+
+	Value SimplifyGuard(Value guard) const {
+		const auto invariant = ResolveInvariantPhi(m_program, guard);
+		return (invariant.IsEmpty() ? guard : invariant).Resolve();
+	}
+
+	bool Implies(Value guard, Value required) const {
+		guard = SimplifyGuard(guard);
+		required = required.Resolve();
+		if (EquivalentValue(m_program, guard, required)) return true;
+		const auto* inst = guard.TryInstruction();
+		return inst != nullptr && inst->GetOpcode() == ValueOpcode::LogicalAnd &&
+		       (Implies(inst->Arg(0), required) || Implies(inst->Arg(1), required));
+	}
+
+	Value EqualLocalKey(Value guard, Value key) const {
+		guard = SimplifyGuard(guard);
+		const auto* inst = guard.TryInstruction();
+		if (inst == nullptr) return {};
+		if (inst->GetOpcode() == ValueOpcode::LogicalAnd) {
+			const auto left = EqualLocalKey(inst->Arg(0), key);
+			return left.IsEmpty() ? EqualLocalKey(inst->Arg(1), key) : left;
+		}
+		if (inst->GetOpcode() != ValueOpcode::IEqual32 || inst->NumArgs() != 2u) return {};
+		if (EquivalentValue(m_program, inst->Arg(0), key)) return inst->Arg(1).Resolve();
+		if (EquivalentValue(m_program, inst->Arg(1), key)) return inst->Arg(0).Resolve();
+		return {};
+	}
+
+	struct AffineOffset {
+		Value    index;
+		uint64_t stride = 0;
+		uint64_t offset = 0;
+	};
+
+	bool MatchAffineOffset(Value value, Value guard, AffineOffset& out,
+	                       uint32_t depth = 0) const {
+		if (depth > 16u) return false;
+		value = value.Resolve();
+		uint32_t immediate = 0;
+		if (ImmediateU32(value, immediate)) {
+			out.offset = immediate;
+			return true;
+		}
+		const auto* inst = value.TryInstruction();
+		if (inst == nullptr) return false;
+		if (inst->GetOpcode() == ValueOpcode::SelectU32 && inst->NumArgs() == 3u &&
+		    Implies(guard, inst->Arg(0))) {
+			return MatchAffineOffset(inst->Arg(1), guard, out, depth + 1u);
+		}
+		if (inst->GetOpcode() == ValueOpcode::IAdd32 && inst->NumArgs() == 2u) {
+			AffineOffset left, right;
+			if (!MatchAffineOffset(inst->Arg(0), guard, left, depth + 1u) ||
+			    !MatchAffineOffset(inst->Arg(1), guard, right, depth + 1u) ||
+			    (!left.index.IsEmpty() && !right.index.IsEmpty() &&
+			     !EquivalentValue(m_program, left.index, right.index))) return false;
+			out.index = left.index.IsEmpty() ? right.index : left.index;
+			out.stride = left.stride + right.stride;
+			out.offset = left.offset + right.offset;
+			return out.stride <= UINT32_MAX && out.offset <= UINT32_MAX;
+		}
+		if (inst->GetOpcode() == ValueOpcode::ShiftLeftLogical32 && inst->NumArgs() == 2u &&
+		    ImmediateU32(inst->Arg(1), immediate) && immediate < 32u) {
+			if (!MatchAffineOffset(inst->Arg(0), guard, out, depth + 1u)) return false;
+			out.stride <<= immediate;
+			out.offset <<= immediate;
+			return out.stride <= UINT32_MAX && out.offset <= UINT32_MAX;
+		}
+		out.index = value;
+		out.stride = 1u;
+		return value.GetType() == Type::U32;
+	}
+
+	bool ImpliesNonzero(Value guard, Value value) const {
+		guard = SimplifyGuard(guard);
+		const auto* inst = guard.TryInstruction();
+		if (inst == nullptr) return false;
+		if (inst->GetOpcode() == ValueOpcode::LogicalAnd) {
+			return ImpliesNonzero(inst->Arg(0), value) ||
+			       ImpliesNonzero(inst->Arg(1), value);
+		}
+		if (inst->GetOpcode() != ValueOpcode::INotEqual32 || inst->NumArgs() != 2u)
+			return false;
+		uint32_t zero = 1u;
+		return (ImmediateU32(inst->Arg(0), zero) && zero == 0u &&
+		        EquivalentValue(m_program, inst->Arg(1), value)) ||
+		       (ImmediateU32(inst->Arg(1), zero) && zero == 0u &&
+		        EquivalentValue(m_program, inst->Arg(0), value));
+	}
+
+	bool ImpliesIndexBelow32(Value guard, Value index) const {
+		guard = SimplifyGuard(guard);
+		const auto* inst = guard.TryInstruction();
+		if (inst == nullptr) return false;
+		if (inst->GetOpcode() == ValueOpcode::LogicalAnd) {
+			return ImpliesIndexBelow32(inst->Arg(0), index) ||
+			       ImpliesIndexBelow32(inst->Arg(1), index);
+		}
+		if (inst->NumArgs() != 2u) return false;
+		uint32_t limit = 0;
+		return (inst->GetOpcode() == ValueOpcode::SLessThan32 &&
+		        EquivalentValue(m_program, inst->Arg(0), index) &&
+		        ImmediateU32(inst->Arg(1), limit) && limit == 32u) ||
+		       (inst->GetOpcode() == ValueOpcode::SGreaterThan32 &&
+		        ImmediateU32(inst->Arg(0), limit) && limit == 32u &&
+		        EquivalentValue(m_program, inst->Arg(1), index));
+	}
+
+	bool ClearsFirstSetBit(Value value, Value mask) const {
+		const auto* update = value.Resolve().TryInstruction();
+		if (update == nullptr || update->GetOpcode() != ValueOpcode::BitwiseXor32 ||
+		    update->NumArgs() != 2u) return false;
+		Value bit;
+		if (EquivalentValue(m_program, update->Arg(0), mask)) bit = update->Arg(1);
+		else if (EquivalentValue(m_program, update->Arg(1), mask)) bit = update->Arg(0);
+		else return false;
+		const auto* shift = bit.Resolve().TryInstruction();
+		uint32_t one = 0;
+		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 ||
+		    shift->NumArgs() != 2u || !ImmediateU32(shift->Arg(0), one) || one != 1u)
+			return false;
+		Value position = shift->Arg(1).Resolve();
+		const auto* masked = position.TryInstruction();
+		uint32_t lane_mask = 0;
+		if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 &&
+		    masked->NumArgs() == 2u) {
+			if (ImmediateU32(masked->Arg(0), lane_mask) && lane_mask == 31u)
+				position = masked->Arg(1);
+			else if (ImmediateU32(masked->Arg(1), lane_mask) && lane_mask == 31u)
+				position = masked->Arg(0);
+		}
+		const auto* first = position.Resolve().TryInstruction();
+		return first != nullptr && first->GetOpcode() == ValueOpcode::FindILsb32 &&
+		       first->NumArgs() == 1u && EquivalentValue(m_program, first->Arg(0), mask);
+	}
+
+	Value InitialClearingMask(Value value, const Block* update_block) const {
+		const auto* phi = value.Resolve().TryInstruction();
+		if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
+		    phi->NumArgs() != 2u || phi->GetType() != Type::U32) return {};
+		for (uint32_t back = 0; back < 2u; ++back) {
+			if (phi->PhiBlock(back) != update_block ||
+			    !ClearsFirstSetBit(phi->Arg(back), value)) continue;
+			const auto initial = phi->Arg(back ^ 1u).Resolve();
+			return ValidateRuntimeValue(m_program, initial, RuntimeValueType::Integer)
+			           ? initial : Value {};
+		}
+		return {};
+	}
+
+	bool EdgeOn(const Block* from, const Block* to, Value predicate, bool positive) const {
+		Value condition;
+		bool taken = false;
+		return ConditionalEdge(from, to, condition, taken) && taken == positive &&
+		       EquivalentValue(m_program, condition, predicate);
+	}
+
+	Value BoundedSetBitMask(Value index, Value guard) const {
+		const auto* phi = index.Resolve().TryInstruction();
+		if (phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
+		    phi->NumArgs() != 3u || phi->GetType() != Type::U32 ||
+		    !ImpliesIndexBelow32(guard, index)) return {};
+		for (uint32_t bit_arm = 0; bit_arm < 3u; ++bit_arm) {
+			const auto bit_guard = PositiveUseGuard(phi->PhiBlock(bit_arm));
+			const auto* selected = phi->Arg(bit_arm).Resolve().TryInstruction();
+			if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 ||
+			    selected->NumArgs() != 3u ||
+			    !Implies(bit_guard, selected->Arg(0))) continue;
+			const auto* first = selected->Arg(1).Resolve().TryInstruction();
+			if (first == nullptr || first->GetOpcode() != ValueOpcode::FindILsb32 ||
+			    first->NumArgs() != 1u || !ImpliesNonzero(bit_guard, first->Arg(0)))
+				continue;
+			const auto initial_mask = InitialClearingMask(first->Arg(0), phi->PhiBlock(bit_arm));
+			if (initial_mask.IsEmpty()) continue;
+			for (uint32_t sentinel_arm = 0; sentinel_arm < 3u; ++sentinel_arm) {
+				if (sentinel_arm == bit_arm) continue;
+				const auto sentinel_guard = PositiveUseGuard(phi->PhiBlock(sentinel_arm));
+				const auto* sentinel = phi->Arg(sentinel_arm).Resolve().TryInstruction();
+				uint32_t bound = 0;
+				if (sentinel == nullptr || sentinel->GetOpcode() != ValueOpcode::SelectU32 ||
+				    sentinel->NumArgs() != 3u ||
+				    !Implies(sentinel_guard, sentinel->Arg(0)) ||
+				    !ImmediateU32(sentinel->Arg(1), bound) || bound != 32u) continue;
+				const auto loop_active = sentinel->Arg(0).Resolve();
+				const auto* active_phi = loop_active.TryInstruction();
+				if (active_phi == nullptr || active_phi->GetOpcode() != ValueOpcode::Phi ||
+				    active_phi->NumArgs() != 2u) continue;
+				bool invariant = false;
+				for (uint32_t initial = 0; initial < 2u; ++initial) {
+					invariant = Implies(guard, active_phi->Arg(initial)) &&
+					            EdgeOn(active_phi->PhiBlock(initial ^ 1u), active_phi->Parent(),
+					                   active_phi->Arg(initial ^ 1u), true);
+					if (invariant) break;
+				}
+				if (!invariant) continue;
+				const auto other_arm = 3u - bit_arm - sentinel_arm;
+				if (EdgeOn(phi->PhiBlock(other_arm), phi->Parent(), loop_active, false))
+					return initial_mask;
+			}
+		}
+		return {};
+	}
+
+	bool MatchUniformizedMaterialKey(Value key, const Inst& image,
+	                                DescriptorSource::IndirectImage& indirect,
+	                                DescriptorSource& material_source, uint32_t pc) {
+		const auto guard = PositiveUseGuard(image.Parent());
+		if (guard.IsEmpty()) return false;
+		const auto local = EqualLocalKey(guard, key);
+		const auto* selected = local.Resolve().TryInstruction();
+		if (selected == nullptr || selected->GetOpcode() != ValueOpcode::SelectU32 ||
+		    selected->NumArgs() != 3u || !Implies(guard, selected->Arg(0))) return false;
+		const auto active = selected->Arg(0).Resolve();
+		const auto* read = selected->Arg(1).Resolve().TryInstruction();
+		if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadAddressU32 ||
+		    read->NumArgs() != 4u || !EquivalentValue(m_program, read->Arg(3), active))
+			return false;
+		uint32_t high = 1u;
+		if (!ImmediateU32(read->Arg(2), high) || high != 0u) return false;
+		const auto memory_index = read->Flags<MemoryFlags>().index;
+		if (memory_index >= m_program.memory_info.size()) return false;
+		const auto& memory = m_program.memory_info[memory_index];
+		if (memory.kind != ResourceKind::Global || memory.data_bits != 32u ||
+		    memory.data_dwords != 1u || !MemoryIndexBelongsTo(memory_index, *read)) return false;
+		const auto* material_handle = read->Arg(0).Resolve().TryInstruction();
+		if (material_handle == nullptr ||
+		    material_handle->GetOpcode() != ValueOpcode::GetAddressResource ||
+		    !MakeRuntimeTableSource(*material_handle, pc, material_source)) return false;
+		AffineOffset offset;
+		if (!MatchAffineOffset(read->Arg(1), active, offset) || offset.index.IsEmpty() ||
+		    offset.stride == 0u || offset.offset + memory.offset > UINT32_MAX ||
+		    offset.offset + memory.offset + 31u * offset.stride + 4u >
+		        static_cast<uint64_t>(UINT32_MAX) + 1u) return false;
+		const auto mask = BoundedSetBitMask(offset.index, active);
+		if (mask.IsEmpty()) return false;
+		indirect.material_source = InternSource(material_source);
+		indirect.selector_stride = static_cast<uint32_t>(offset.stride);
+		indirect.selector_offset = static_cast<uint32_t>(offset.offset + memory.offset);
+		indirect.key_count = Value(32u);
+		indirect.selector_mask = mask;
+		return true;
+	}
+
+	Value BoundedLoopCount(Value key, const Block* use) const {
+		const auto* phi = key.Resolve().TryInstruction();
+		if (m_shader_writes || phi == nullptr || phi->GetOpcode() != ValueOpcode::Phi ||
+		    phi->GetType() != Type::U32 || phi->NumArgs() != 2u ||
+		    m_program.blocks.size() != m_program.block_info.size()) {
+			return {};
+		}
+		bool induction = false;
+		for (uint32_t initial = 0; initial < 2u; ++initial) {
+			const auto zero = phi->Arg(initial).Resolve();
+			const auto* step = phi->Arg(initial ^ 1u).Resolve().TryInstruction();
+			if (!zero.IsImmediate() || zero.GetType() != Type::U32 || zero.U32() != 0u ||
+			    step == nullptr || step->GetOpcode() != ValueOpcode::IAdd32 ||
+			    step->Parent() != phi->PhiBlock(initial ^ 1u)) {
+				continue;
+			}
+			uint32_t increment = 0;
+			induction = (step->Arg(0).Resolve() == key &&
+			             ImmediateU32(step->Arg(1), increment) && increment == 1u) ||
+			            (step->Arg(1).Resolve() == key &&
+			             ImmediateU32(step->Arg(0), increment) && increment == 1u);
+			if (induction) break;
+		}
+		if (!induction) return {};
+
+		const auto contains = [&](auto&& self, Value value, const Inst* comparison) -> bool {
+			value = value.Resolve();
+			if (value.TryInstruction() == comparison) return true;
+			const auto* inst = value.TryInstruction();
+			return inst != nullptr && inst->GetOpcode() == ValueOpcode::LogicalAnd &&
+			       (self(self, inst->Arg(0), comparison) || self(self, inst->Arg(1), comparison));
+		};
+		for (const auto& use_of_key: phi->Uses()) {
+			const auto* compare = use_of_key.user;
+			if (compare->GetOpcode() != ValueOpcode::SLessThan32 || use_of_key.operand != 0u ||
+			    !ValidateRuntimeValue(m_program, compare->Arg(1))) continue;
+			for (uint32_t i = 0; i < m_program.block_info.size(); ++i) {
+				const auto& info = m_program.block_info[i];
+				const auto* negated = info.condition.Resolve().TryInstruction();
+				if (info.terminator.kind != CFG::TerminatorKind::ConditionalBranch ||
+				    compare->Parent() != m_program.blocks[i] || negated == nullptr ||
+				    negated->GetOpcode() != ValueOpcode::LogicalNot ||
+				    !contains(contains, negated->Arg(0), compare) ||
+				    use == m_program.blocks[i] ||
+				    CanReach(m_program.blocks.front(), use, m_program.blocks[i]) ||
+				    CanReach(phi->Parent(), use, m_program.blocks[i]) ||
+				    CanReach(FindBlock(info.terminator.true_block), use, phi->Parent()) ||
+				    !CanReach(FindBlock(info.terminator.false_block), use, phi->Parent())) {
+					continue;
+				}
+				return compare->Arg(1);
+			}
+		}
+		return {};
+	}
+
 	bool TryMakeIndirectImage(Inst& handle, uint32_t pc, IndirectImagePlan& plan) {
 		if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) {
 			return false;
@@ -562,13 +922,20 @@ private:
 		indirect.table_offset = table_offset;
 		if (table_source.dword_count == 2u) {
 			const auto* selector = key.Resolve().TryInstruction();
-			if (m_shader_writes || selector == nullptr ||
-			    selector->GetOpcode() != ValueOpcode::FindILsb32 || selector->NumArgs() != 1u ||
-			    !NonzeroOnEntry(selector->Arg(0), handle.Parent()) ||
-			    (table_offset & 3u) != 0u || table_offset > UINT32_MAX - (32u * 32u - 1u)) {
+			const bool bitscan = selector != nullptr && selector->GetOpcode() == ValueOpcode::FindILsb32 &&
+			    selector->NumArgs() == 1u && !m_shader_writes &&
+			    NonzeroOnEntry(selector->Arg(0), handle.Parent());
+			if (bitscan) {
+				indirect.key_count = Value(32u);
+			} else {
+				indirect.key_count = BoundedLoopCount(key, handle.Parent());
+			}
+			if (indirect.key_count.IsEmpty() &&
+			    !MatchUniformizedMaterialKey(key, handle, indirect, material_source, pc)) {
 				return false;
 			}
-			indirect.key_count = 32u;
+			if ((table_offset & 3u) != 0u ||
+			    (bitscan && table_offset > UINT32_MAX - (32u * 32u - 1u))) return false;
 		} else {
 			auto* material_read = key.Resolve().TryInstruction();
 			uint32_t material_memory_index = 0;
@@ -839,11 +1206,10 @@ private:
 		if (buffer != BufferAccess::None) {
 			if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle,
 			               source)) {
-				if (memory.kind != ResourceKind::Buffer || memory.formatted || memory.typed ||
-				    (op != ValueOpcode::LoadBufferU32x2 && op != ValueOpcode::LoadBufferU32x4)) {
+				if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) {
 					Fail(flags.pc,
 					     "buffer descriptor is not a valid runtime value; GPU-selected access "
-					     "requires a raw DWORD x2/x4 load");
+					     "requires a raw DWORD x2/x3/x4 load");
 				}
 				m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer;
 				m_info.uses_dma                         = true;
@@ -873,6 +1239,9 @@ private:
 				return;
 			}
 			ValidateAddressHandle(inst.Arg(0), flags.pc);
+			if (address_info.access == AddressAccess::Write) {
+				m_program.has_address_writes = true;
+			}
 			m_info.uses_dma = true;
 			return;
 		}

src/graphics/shader/shader.cpp

@@ -388,6 +388,8 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 
 	EXIT_IF(attrib == nullptr || buffer == nullptr);
 
+	const bool debug_dump = Config::GraphicsDebugDumpEnabled();
+
 	for (uint32_t i = 0; i < num_input_semantics; i++) {
 		const auto& in = input_semantics[i];
 
@@ -396,7 +398,7 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 		uint32_t reg  = in.hardware_mapping;
 		uint32_t size = in.size_in_elements;
 
-		if (Config::GraphicsDebugDumpEnabled()) {
+		if (debug_dump) {
 			LOGF("reg = %u, size = %u, va[%u] = 0x%08" PRIx32 "\n", reg, size, i,
 			     attrib[in.semantic]);
 		}
@@ -407,16 +409,6 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 		uint32_t offset      = (attrib[in.semantic] >> 14u) & 0xfffu;
 		uint32_t fetch_index = (attrib[in.semantic] >> 26u) & 0x1u;
 
-		if (fetch_index != 0) {
-			static std::atomic<uint64_t> log_count = 0;
-			auto                         log_id    = log_count.fetch_add(1);
-			if (log_id < 64) {
-				LOGF("\t temporary: PS5 vertex attrib semantic %u uses fetch index %u, buffer "
-				     "index %zu\n",
-				     static_cast<uint32_t>(in.semantic), fetch_index, index);
-			}
-		}
-
 		EXIT_NOT_IMPLEMENTED(index >= ShaderVertexInputInfo::RES_MAX);
 
 		const auto* sharp = &buffer[index * 4];
@@ -437,14 +429,6 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
 			const auto                   format_raw    = static_cast<uint32_t>(format);
 			const auto                   buffer_format = format_raw >> 2u;
 			const auto                   channels      = (format_raw & 3u) + 1u;
-			static std::atomic<uint64_t> log_count      = 0;
-			auto                         log_id         = log_count.fetch_add(1);
-			if (log_id < 64) {
-				LOGF("\t PS5 vertex attrib semantic %u uses attrib format %u -> buffer "
-				     "format %u, offset %u, buffer index %zu\n",
-				     static_cast<uint32_t>(in.semantic), static_cast<uint32_t>(format),
-				     static_cast<uint32_t>(buffer_format), offset, index);
-			}
 			// AGC vertex formats encode the buffer format above the two channel-count bits.
 			// The fetch prolog selects X001, XY01, XYZ1, or XYZW from that count.
 			r.fields[3] = (r.fields[3] & ~((0x7fu << 12u) | 0xfffu)) |

src/kernel/memory.cpp

@@ -200,6 +200,10 @@ static bool IsPrivateCommittedRangeType(VirtualRangeType type) {
 	       type == VirtualRangeType::Runtime;
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+static bool g_test_fail_next_range_replace = false;
+#endif
+
 class VirtualRanges {
 public:
 	struct Range {
@@ -288,36 +292,22 @@ public:
 		return true;
 	}
 
-	bool ConsumeReserved(uint64_t start, uint64_t size,
-	                     VirtualRangeType type = VirtualRangeType::Reserved) {
+	bool ReplaceSpan(uint64_t start, uint64_t size, VirtualRangeType expected_type,
+	                 uint64_t offset, int protection, int memory_type, VirtualRangeType type,
+	                 const char* name, bool disallow_merge = false) {
 		Common::LockGuard lock(m_mutex);
 
-		auto end = End(start, size);
-		for (const auto& r: m_ranges) {
-			if (r.type == type && start >= r.start && end <= End(r.start, r.size)) {
-				RemoveUnlocked(start, size);
-				MergeUnlocked();
-				return true;
-			}
-		}
-
-		return false;
-	}
-
-	bool ConsumeReservedSpan(uint64_t start, uint64_t size, Range* first_range = nullptr,
-	                         VirtualRangeType type = VirtualRangeType::Reserved) {
-		Common::LockGuard lock(m_mutex);
-
-		if (size == 0) {
+		if (start == 0 || size == 0 || size > UINT64_MAX - start) {
 			return false;
 		}
 
 		auto current = start;
-		auto end     = End(start, size);
+		const auto end = start + size;
 		while (current < end) {
 			const Range* candidate = nullptr;
 			for (const auto& r: m_ranges) {
-				if (r.type == type && current >= r.start && current < End(r.start, r.size)) {
+				if (r.type == expected_type && current >= r.start &&
+				    current < End(r.start, r.size)) {
 					candidate = &r;
 					break;
 				}
@@ -325,14 +315,29 @@ public:
 			if (candidate == nullptr) {
 				return false;
 			}
-			if (current == start && first_range != nullptr) {
-				*first_range = *candidate;
-			}
 			current = std::min(end, End(candidate->start, candidate->size));
 		}
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+		if (g_test_fail_next_range_replace) {
+			g_test_fail_next_range_replace = false;
+			return false;
+		}
+#endif
+		Range replacement {};
+		replacement.start          = start;
+		replacement.size           = size;
+		replacement.offset         = offset;
+		replacement.protection     = protection;
+		replacement.memory_type    = memory_type;
+		replacement.type           = type;
+		replacement.disallow_merge = disallow_merge;
+		CopyVirtualRangeName(replacement.name, name);
 		RemoveUnlocked(start, size);
-		MergeUnlocked();
+		auto position = LowerBound(start);
+		const auto index = static_cast<size_t>(position - m_ranges.begin());
+		m_ranges.insert(position, replacement);
+		MergeAroundUnlocked(index);
 		return true;
 	}
 
@@ -2250,10 +2255,9 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le
 		EXIT("unknown prot: %d\n", prot);
 	}
 
-	auto                 in_addr              = reinterpret_cast<uint64_t>(*addr_in_out);
-	uint64_t             out_addr             = 0;
-	bool                 consumed_reservation = false;
-	VirtualRanges::Range consumed_range {};
+	auto     in_addr         = reinterpret_cast<uint64_t>(*addr_in_out);
+	uint64_t out_addr        = 0;
+	bool     reserved_target = false;
 
 	if ((flags & GUEST_MAP_FIXED) != 0) {
 		if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1)) != 0 ||
@@ -2269,16 +2273,12 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le
 			    return range.type == VirtualRangeType::Reserved;
 		    })) {
 			UnmapGpuRange(in_addr, len);
-			consumed_range = reserved_ranges.front();
-			if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len)) {
-				consumed_reservation = true;
-				out_addr             = in_addr;
-			}
+			reserved_target = true;
+			out_addr        = in_addr;
 		}
-		if (!consumed_reservation && ReplaceFixedRangeWithReserved(in_addr, len) &&
-		    g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) {
-			consumed_reservation = true;
-			out_addr             = in_addr;
+		if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
+			reserved_target = true;
+			out_addr        = in_addr;
 		}
 	} else {
 		const auto search_addr = (in_addr != 0 ? in_addr : DEFAULT_PS5_BASE);
@@ -2291,30 +2291,25 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le
 	*addr_in_out = reinterpret_cast<void*>(out_addr);
 
 	if (out_addr == 0) {
-		if (consumed_reservation) {
-			g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved,
-			                      consumed_range.name);
-		}
 		return KERNEL_ERROR_ENOMEM;
 	}
 
 	if (!g_flexible_memory->Map(out_addr, len, prot, mode, gpu_mode, name)) {
 		LOGF_COLOR(Log::Color::Red, "\t [Fail]\n");
-		if (consumed_reservation) {
-			EXIT_IF(!g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved,
-			                               consumed_range.name));
-		}
 		return KERNEL_ERROR_ENOMEM;
 	}
 
-	if (!g_virtual_ranges->Add(out_addr, len, 0, prot, 0, VirtualRangeType::Flexible, name,
-	                           (map_flags & GUEST_MAP_NO_COALESCE) != 0)) {
+	const bool published = reserved_target
+	                           ? g_virtual_ranges->ReplaceSpan(
+	                                 out_addr, len, VirtualRangeType::Reserved, 0, prot, 0,
+	                                 VirtualRangeType::Flexible, name,
+	                                 (map_flags & GUEST_MAP_NO_COALESCE) != 0)
+	                           : g_virtual_ranges->Add(out_addr, len, 0, prot, 0,
+	                                                   VirtualRangeType::Flexible, name,
+	                                                   (map_flags & GUEST_MAP_NO_COALESCE) != 0);
+	if (!published) {
 		GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess;
 		EXIT_IF(!g_flexible_memory->Unmap(out_addr, len, &rollback_gpu_mode));
-		if (consumed_reservation) {
-			EXIT_IF(!g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved,
-			                               consumed_range.name));
-		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
@@ -2946,25 +2941,17 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 		return KERNEL_ERROR_ENOMEM;
 	}
 
-	auto                 in_addr              = reinterpret_cast<uint64_t>(*addr);
-	uint64_t             out_addr             = 0;
-	bool                 shared_backing       = false;
-	bool                 consumed_reservation = false;
-	VirtualRanges::Range consumed_range {};
-	auto                 shared_failure = GuestBackingStore::FailureReason::None;
+	auto     in_addr         = reinterpret_cast<uint64_t>(*addr);
+	uint64_t out_addr        = 0;
+	bool     shared_backing  = false;
+	bool     reserved_target = false;
+	auto     shared_failure  = GuestBackingStore::FailureReason::None;
 	// Direct mappings must remain views of the single backing object. Anonymous fallbacks break
 	// aliasing and lose direct-memory contents when a range is unmapped and mapped again.
 	auto map_shared_fixed = [&](uint64_t target_addr) -> bool {
 		return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode,
 		                                         &shared_failure);
 	};
-	auto map_consumed_reserved_fixed = [&]() {
-		if (map_shared_fixed(in_addr)) {
-			out_addr       = in_addr;
-			shared_backing = true;
-		}
-	};
-
 	if (fixed) {
 		if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1u)) != 0 ||
 		    (alignment != 0 && in_addr % alignment != 0)) {
@@ -2980,20 +2967,18 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 			    return range.type == VirtualRangeType::Reserved;
 		    })) {
 			UnmapGpuRange(in_addr, len);
-			consumed_range = reserved_ranges.front();
-			if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len)) {
-				consumed_reservation = true;
-				map_consumed_reserved_fixed();
-			}
+			reserved_target = true;
 		}
-		if (!consumed_reservation && ReplaceFixedRangeWithReserved(in_addr, len) &&
-		    g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) {
-			consumed_reservation = true;
-			map_consumed_reserved_fixed();
+		if (!reserved_target && ReplaceFixedRangeWithReserved(in_addr, len)) {
+			reserved_target = true;
 		}
-		if (!consumed_reservation) {
+		if (!reserved_target) {
 			return KERNEL_ERROR_ENOMEM;
 		}
+		if (map_shared_fixed(in_addr)) {
+			out_addr       = in_addr;
+			shared_backing = true;
+		}
 	} else {
 		constexpr size_t DEFAULT_ALIGNMENT = 0x4000;
 		alignment                          = (alignment != 0 ? alignment : DEFAULT_ALIGNMENT);
@@ -3003,16 +2988,13 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 			    return range.type == VirtualRangeType::Reserved;
 		    })) {
 			UnmapGpuRange(in_addr, len);
-			consumed_range = reserved_ranges.front();
-			if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len)) {
-				consumed_reservation = true;
-				if (map_shared_fixed(in_addr)) {
-					out_addr       = in_addr;
-					shared_backing = true;
-				}
+			reserved_target = true;
+			if (map_shared_fixed(in_addr)) {
+				out_addr       = in_addr;
+				shared_backing = true;
 			}
 		}
-		if (!consumed_reservation) {
+		if (!reserved_target) {
 			out_addr = FindGuestFreeRange(in_addr, len, alignment);
 			if (out_addr != 0) {
 				UnmapGpuRange(out_addr, len);
@@ -3046,34 +3028,28 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f
 	     magic_enum::enum_name(gpu_mode), shared_backing ? "yes" : "no", shared_reason);
 
 	if (out_addr == 0) {
-		if (consumed_reservation) {
-			g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved,
-			                      consumed_range.name);
-		}
 		return KERNEL_ERROR_ENOMEM;
 	}
 
 	if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) {
 		LOGF_COLOR(Log::Color::Red, "\t [Fail]\n");
 		EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
-		if (consumed_reservation) {
-			EXIT_IF(!g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved,
-			                               consumed_range.name));
-		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
 	PhysicalMemory::AllocatedBlock mapped_block {};
 	g_physical_memory->Find(direct_memory_start, false, &mapped_block);
-	if (!g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, mapped_block.memory_type,
-	                           VirtualRangeType::Direct, "")) {
+	const bool published = reserved_target
+	                           ? g_virtual_ranges->ReplaceSpan(
+	                                 out_addr, len, VirtualRangeType::Reserved, direct_memory_start,
+	                                 prot, mapped_block.memory_type, VirtualRangeType::Direct, "")
+	                           : g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot,
+	                                                   mapped_block.memory_type,
+	                                                   VirtualRangeType::Direct, "");
+	if (!published) {
 		GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess;
 		EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode));
 		EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len));
-		if (consumed_reservation) {
-			EXIT_IF(!g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved,
-			                               consumed_range.name));
-		}
 		return KERNEL_ERROR_EBUSY;
 	}
 
@@ -3442,6 +3418,10 @@ void TestFailNextFixedReserveRangeRegistration() {
 	g_test_fail_next_fixed_reserve_range_add = true;
 }
 
+void TestFailNextVirtualRangeReplacement() {
+	g_test_fail_next_range_replace = true;
+}
+
 bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size) {
 	return g_guest_address_space->TestContainsFree(vaddr, size);
 }
@@ -3965,12 +3945,8 @@ int KYTY_SYSV_ABI KernelMemoryPoolReserve(void* addr_in, size_t len, size_t alig
 		    !IsReservedRangeType(reserved_range.type)) {
 			return KERNEL_ERROR_EBUSY;
 		}
-		g_virtual_ranges->Remove(out_vaddr, len);
-		if (!g_virtual_ranges->Add(out_vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved,
-		                           reserved_range.name)) {
-			g_virtual_ranges->Add(out_vaddr, len, reserved_range.offset, reserved_range.protection,
-			                      reserved_range.memory_type, reserved_range.type,
-			                      reserved_range.name);
+		if (!g_virtual_ranges->ReplaceSpan(out_vaddr, len, VirtualRangeType::Reserved, 0, 0, 0,
+		                                   VirtualRangeType::PoolReserved, reserved_range.name)) {
 			return KERNEL_ERROR_EBUSY;
 		}
 		*addr_out = out_addr;
@@ -4011,17 +3987,13 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p
 
 	VirtualRanges::Range old_range {};
 	if (!g_virtual_ranges->Query(vaddr, 0, &old_range) ||
-	    old_range.type != VirtualRangeType::PoolReserved) {
-		return KERNEL_ERROR_EACCES;
-	}
-
-	if (!g_virtual_ranges->ConsumeReserved(vaddr, len, VirtualRangeType::PoolReserved)) {
+	    old_range.type != VirtualRangeType::PoolReserved ||
+	    len > old_range.size - (vaddr - old_range.start)) {
 		return KERNEL_ERROR_EACCES;
 	}
 
 	std::vector<PooledMemory::Mapping> mappings;
 	if (!g_pooled_memory->Allocate(vaddr, len, gpu_mode, &mappings)) {
-		g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name);
 		return KERNEL_ERROR_ENOMEM;
 	}
 
@@ -4034,7 +4006,6 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p
 		if (!g_pooled_memory->Release(vaddr, len, &rollback_gpu_mode)) {
 			EXIT("failed to release pooled-memory rollback allocation\n");
 		}
-		g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name);
 	};
 
 	for (const auto& mapping: mappings) {
@@ -4050,8 +4021,8 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p
 		mapped.push_back(mapping);
 	}
 
-	if (!g_virtual_ranges->Add(vaddr, len, 0, prot, type, VirtualRangeType::Pooled,
-	                           old_range.name)) {
+	if (!g_virtual_ranges->ReplaceSpan(vaddr, len, VirtualRangeType::PoolReserved, 0, prot, type,
+	                                   VirtualRangeType::Pooled, old_range.name)) {
 		rollback();
 		return KERNEL_ERROR_EBUSY;
 	}

src/kernel/memory.h

@@ -190,6 +190,7 @@ void     TestFailNextPhysicalMemoryUnmap();
 void     TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
 void     TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);
 void     TestFailNextFixedReserveRangeRegistration();
+void     TestFailNextVirtualRangeReplacement();
 bool     TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
 bool     TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
 bool     TestGuestBackingOutsideAddressSpace();

src/libs/agc.cpp

@@ -2110,7 +2110,7 @@ uint32_t KYTY_SYSV_ABI AgcCbQueueEndOfPipeActionGetSize() {
 uint32_t* KYTY_SYSV_ABI AgcAcbResetQueue(CommandBuffer* buf, uint32_t op) {
 	PRINT_NAME();
 
-	LOGF("\t op    = 0x%08" PRIx32 "\n", op);
+	AgcTrace("\t op    = 0x%08" PRIx32 "\n", op);
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 	EXIT_NOT_IMPLEMENTED((op & ~0x1c2u) != 0);
@@ -2130,9 +2130,9 @@ uint32_t* KYTY_SYSV_ABI AgcAcbResetQueue(CommandBuffer* buf, uint32_t op) {
 uint32_t* KYTY_SYSV_ABI AgcDcbResetQueue(CommandBuffer* buf, uint32_t op, uint32_t state) {
 	PRINT_NAME();
 
-	LOGF("\t op    = 0x%08" PRIx32 "\n"
-	     "\t state = 0x%08" PRIx32 "\n",
-	     op, state);
+	AgcTrace("\t op    = 0x%08" PRIx32 "\n"
+	         "\t state = 0x%08" PRIx32 "\n",
+	         op, state);
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 	EXIT_NOT_IMPLEMENTED((op & ~0xfffu) != 0);
@@ -2749,9 +2749,9 @@ uint32_t* KYTY_SYSV_ABI AgcDcbSetBaseIndirectArgs(CommandBuffer* buf, uint32_t s
                                                   const volatile void* indirect_base_addr) {
 	PRINT_NAME();
 
-	LOGF("\t shader_type        = %" PRIu32 "\n"
-	     "\t indirect_base_addr = 0x%016" PRIx64 "\n",
-	     shader_type, reinterpret_cast<uint64_t>(indirect_base_addr));
+	AgcTrace("\t shader_type        = %" PRIu32 "\n"
+	         "\t indirect_base_addr = 0x%016" PRIx64 "\n",
+	         shader_type, reinterpret_cast<uint64_t>(indirect_base_addr));
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 
@@ -2941,9 +2941,9 @@ uint32_t* KYTY_SYSV_ABI AgcDcbDispatchIndirect(CommandBuffer* buf, uint32_t data
                                                uint32_t flags) {
 	PRINT_NAME();
 
-	LOGF("\t data_offset = 0x%" PRIx32 "\n"
-	     "\t flags       = 0x%08" PRIx32 "\n",
-	     data_offset_in_bytes, flags);
+	AgcTrace("\t data_offset = 0x%" PRIx32 "\n"
+	         "\t flags       = 0x%08" PRIx32 "\n",
+	         data_offset_in_bytes, flags);
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 
@@ -3294,9 +3294,9 @@ uint32_t* KYTY_SYSV_ABI AgcAcbDispatchIndirect(CommandBuffer*       buf,
                                                uint32_t             modifier) {
 	PRINT_NAME();
 
-	LOGF("\t indirect_args = 0x%016" PRIx64 "\n"
-	     "\t modifier      = 0x%08" PRIx32 "\n",
-	     reinterpret_cast<uint64_t>(indirect_args), modifier);
+	AgcTrace("\t indirect_args = 0x%016" PRIx64 "\n"
+	         "\t modifier      = 0x%08" PRIx32 "\n",
+	         reinterpret_cast<uint64_t>(indirect_args), modifier);
 
 	if (buf == nullptr) {
 		return nullptr;
@@ -3724,9 +3724,9 @@ int KYTY_SYSV_ABI AgcCondExecPatchSetCommandAddress(uint32_t*                cmd
                                                     const volatile uint32_t* command) {
 	PRINT_NAME();
 
-	LOGF("\t cmd     = 0x%016" PRIx64 "\n"
-	     "\t command = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(command));
+	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
+	         "\t command = 0x%016" PRIx64 "\n",
+	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(command));
 
 	if (cmd == nullptr || command == nullptr) {
 		return GRAPHICS5_ERROR_INVALID_PACKET;
@@ -3868,9 +3868,9 @@ static uint32_t* get_agc_wait_packet(uint32_t* cmd) {
 int KYTY_SYSV_ABI AgcWaitRegMemPatchAddress(uint32_t* cmd, const volatile void* address) {
 	PRINT_NAME();
 
-	LOGF("\t cmd     = 0x%016" PRIx64 "\n"
-	     "\t address = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
+	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
+	         "\t address = 0x%016" PRIx64 "\n",
+	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
 
 	auto* wait = get_agc_wait_packet(cmd);
 	if (wait == nullptr) {
@@ -3913,9 +3913,9 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchAddress(uint32_t*             cmd,
 
 	// Not sure
 
-	LOGF("\t cmd     = 0x%016" PRIx64 "\n"
-	     "\t address = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
+	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
+	         "\t address = 0x%016" PRIx64 "\n",
+	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
@@ -4246,7 +4246,7 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
 	}
 
 	for (uint32_t i = 0; i < std::min<uint32_t>(size_in_dwords, 8); i++) {
-		LOGF("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]);
+		AgcTrace("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]);
 	}
 
 	GraphicsDbgDumpDcb("a", size_in_dwords, acb);
@@ -4322,17 +4322,17 @@ int KYTY_SYSV_ABI AgcDriverSubmitMultiCommandBuffers(void*            queue_cont
 int KYTY_SYSV_ABI AgcDriverSubmitAcb(uint32_t queue, const Packet* packet) {
 	PRINT_NAME();
 
-	LOGF("\t queue  = 0x%08" PRIx32 "\n"
-	     "\t packet = 0x%016" PRIx64 "\n",
-	     queue, reinterpret_cast<uint64_t>(packet));
+	AgcTrace("\t queue  = 0x%08" PRIx32 "\n"
+	         "\t packet = 0x%016" PRIx64 "\n",
+	         queue, reinterpret_cast<uint64_t>(packet));
 
 	if (packet == nullptr) {
 		return OK;
 	}
-	LOGF("\t acb   = 0x%016" PRIx64 "\n"
-	     "\t size  = 0x%08" PRIx32 "\n"
-	     "\t flags = 0x%02" PRIx8 "\n",
-	     reinterpret_cast<uint64_t>(packet->addr), packet->dw_num, packet->flags);
+	AgcTrace("\t acb   = 0x%016" PRIx64 "\n"
+	         "\t size  = 0x%08" PRIx32 "\n"
+	         "\t flags = 0x%02" PRIx8 "\n",
+	         reinterpret_cast<uint64_t>(packet->addr), packet->dw_num, packet->flags);
 
 	submit_acb(queue, packet->addr, packet->dw_num);
 	return OK;

src/libs/ajm.cpp

@@ -682,16 +682,9 @@ static AjmDecodeResult AjmControlInstance(uint32_t instance, uint64_t flags,
 
 static AjmDecodeResult AjmSetGaplessDecode(uint32_t instance, const void* gapless_decode,
                                            int reset) {
-	std::scoped_lock lock(g_ajm_instances_mutex);
-	auto*            state = AjmFindInstanceLocked(instance);
-	if (state == nullptr || state->decoder == nullptr || gapless_decode == nullptr) {
-		AjmDecodeResult result {};
-		result.result = AJM_RESULT_INVALID_PARAMETER;
-		return result;
-	}
-
-	state->gapless.Set(*static_cast<const AjmSidebandGaplessDecode*>(gapless_decode), reset != 0);
-	return state->decoder->MakeResult();
+	const auto flags = AJM_FLAG_SIDEBAND_GAPLESS | (reset != 0 ? AJM_FLAG_CONTROL_RESET : 0);
+	return AjmControlInstance(instance, flags, gapless_decode, sizeof(AjmSidebandGaplessDecode),
+	                          nullptr, nullptr);
 }
 
 static AjmDecodeResult AjmGetGaplessDecode(uint32_t                  instance,
@@ -930,18 +923,8 @@ int KYTY_SYSV_ABI AjmBatchJobClearContext(AjmBatchInfo* info, uint32_t instance,
 	PRINT_NAME();
 	EXIT_NOT_IMPLEMENTED(info == nullptr);
 
-	AjmDecodeResult decode_result {};
-	{
-		std::scoped_lock lock(g_ajm_instances_mutex);
-		auto*            state = AjmFindInstanceLocked(instance);
-		if (state != nullptr && state->decoder != nullptr) {
-			state->gapless.Reset();
-			state->decoder->Reset();
-			decode_result = state->decoder->MakeResult();
-		} else {
-			decode_result.result = AJM_RESULT_INVALID_PARAMETER;
-		}
-	}
+	const auto decode_result =
+	    AjmControlInstance(instance, AJM_FLAG_CONTROL_RESET, nullptr, 0, nullptr, nullptr);
 
 	AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE, decode_result);
 	LOGF("\t instance = 0x%08" PRIx32 "\n", instance);

src/libs/audio.cpp

@@ -101,6 +101,7 @@ private:
 		uint32_t freq             = 0;
 		Format   format           = Format::Unknown;
 		uint64_t last_output_time = 0;
+		bool     queue_primed     = false;
 		int      channels_num     = 0;
 		int      volume[12]       = {};
 
@@ -360,20 +361,34 @@ bool Audio::QueueSdlAudio(PortOut* port, const void* data, bool blocking) {
 	const auto           prepared_size =
 	    BytesPerSample(port->format) * output_channels * port->samples_num;
 
+	uint32_t min_queued_size = 0;
 	if (blocking) {
 		constexpr uint64_t target_latency_us = 40000;
 		const auto buffer_us = port->freq != 0 ? (1000000ULL * port->samples_num) / port->freq : 0;
 		const auto buffers =
 		    buffer_us != 0 ? static_cast<uint32_t>((target_latency_us + buffer_us - 1) / buffer_us)
 		                   : 2u;
-		const auto min_queued_size = prepared_size * std::clamp(buffers, 2u, 16u);
+		min_queued_size           = prepared_size * std::clamp(buffers, 2u, 16u);
 		const auto wait_start      = LibKernel::KernelGetProcessTime();
-		while (SDL_GetAudioStreamQueued(port->stream) > static_cast<int>(min_queued_size)) {
+		auto queued                = SDL_GetAudioStreamQueued(port->stream);
+		if (queued < static_cast<int>(prepared_size)) {
+			port->queue_primed = false;
+		}
+		while (queued > static_cast<int>(min_queued_size)) {
 			if (LibKernel::KernelGetProcessTime() - wait_start > 200000) {
 				SDL_ClearAudioStream(port->stream);
+				port->queue_primed = false;
 				break;
 			}
 			Common::Thread::SleepMicro(1000);
+			queued = SDL_GetAudioStreamQueued(port->stream);
+		}
+		if (port->queue_primed) {
+			const auto next_time = port->last_output_time + buffer_us;
+			const auto now       = LibKernel::KernelGetProcessTime();
+			if (next_time > now) {
+				Common::Thread::SleepMicro(next_time - now);
+			}
 		}
 	}
 
@@ -381,6 +396,10 @@ bool Audio::QueueSdlAudio(PortOut* port, const void* data, bool blocking) {
 		LOGF("AudioOut: SDL_PutAudioStreamData failed: %s\n", SDL_GetError());
 		return false;
 	}
+	if (blocking && !port->queue_primed &&
+	    SDL_GetAudioStreamQueued(port->stream) >= static_cast<int>(min_queued_size)) {
+		port->queue_primed = true;
+	}
 
 	return true;
 }

src/libs/avPlayer.cpp

@@ -1719,7 +1719,7 @@ static AvPlayerInternal* create_player(const AvPlayerMemAllocator&     mem,
 	h->mem           = mem;
 	h->file          = file;
 	h->event         = event;
-	h->auto_start    = auto_start;
+	h->auto_start    = auto_start || event.event_callback == nullptr;
 	h->video_buffers = std::clamp(video_buffers <= 0 ? 2 : video_buffers, 2, 16);
 	h->post_init.demux_video_buffer_size = 4 * 1024 * 1024;
 	return h;

src/libs/libAudio2.cpp

@@ -188,7 +188,7 @@ struct AudioOut2PortStateEntry {
 	uint32_t               samples_num   = 512;
 	AudioInternal::Format  audio_format  = AudioInternal::Format::Unknown;
 	int                    audio_handle  = 0;
-	const void*            pcm_data      = nullptr;
+	std::vector<uint8_t>   pcm_data;
 };
 
 struct AudioOut2SpeakerArrayState {
@@ -341,11 +341,17 @@ static AudioOut2PortStateEntry* audioout2_find_port_locked(AudioOut2PortHandle p
 	return nullptr;
 }
 
+static size_t audioout2_pcm_size(const AudioOut2PortStateEntry& state) {
+	const auto bytes_per_sample = (state.data_format & 0x7fu) == 1 ? sizeof(int16_t) : sizeof(float);
+	return static_cast<size_t>(state.samples_num) *
+	       audioout2_data_format_channels(state.data_format) * bytes_per_sample;
+}
+
 static bool audioout2_context_has_queueable_device(AudioOut2ContextHandle ctx) {
 	Common::LockGuard lock(g_audioout2_port_mutex);
 	for (const auto& state: g_audioout2_ports) {
 		if (state.used && state.context == ctx && state.audio_handle > 0 &&
-		    state.pcm_data != nullptr && AudioInternal::AudioOutHasDevice(state.audio_handle)) {
+		    !state.pcm_data.empty() && AudioInternal::AudioOutHasDevice(state.audio_handle)) {
 			return true;
 		}
 	}
@@ -353,24 +359,23 @@ static bool audioout2_context_has_queueable_device(AudioOut2ContextHandle ctx) {
 }
 
 static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, bool blocking) {
+	// The backend consumes the buffers synchronously, but can block while pacing SDL's queue.
+	// Keep both PCM storage and port handles alive until it returns.
 	std::vector<AudioInternal::OutputParam> params;
 	params.reserve(AudioInternal::OUT_PORTS_MAX);
 
-	g_audioout2_port_mutex.Lock();
+	Common::LockGuard lock(g_audioout2_port_mutex);
 	for (const auto& state: g_audioout2_ports) {
 		if (state.used && state.context == ctx && state.audio_handle > 0 &&
-		    state.pcm_data != nullptr && params.size() < AudioInternal::OUT_PORTS_MAX) {
-			params.push_back(AudioInternal::OutputParam {state.audio_handle, state.pcm_data});
+		    !state.pcm_data.empty() && params.size() < AudioInternal::OUT_PORTS_MAX) {
+			params.push_back(AudioInternal::OutputParam {state.audio_handle, state.pcm_data.data()});
 		}
 	}
-	g_audioout2_port_mutex.Unlock();
 
-	if (params.empty()) {
-		return;
+	if (!params.empty()) {
+		(void)AudioInternal::AudioOutOutputs(params.data(), static_cast<uint32_t>(params.size()),
+		                                     blocking);
 	}
-
-	(void)AudioInternal::AudioOutOutputs(params.data(), static_cast<uint32_t>(params.size()),
-	                                     blocking);
 }
 
 static void audioout2_close_audio_handle(int audio_handle) {
@@ -679,7 +684,12 @@ int KYTY_SYSV_ABI AudioOut2PortSetAttributes(AudioOut2PortHandle       port,
 	if (has_pcm) {
 		g_audioout2_port_mutex.Lock();
 		if (auto* state = audioout2_find_port_locked(port); state != nullptr) {
-			state->pcm_data = pcm_data;
+			if (pcm_data != nullptr && state->audio_format != AudioInternal::Format::Unknown) {
+				const auto* bytes = static_cast<const uint8_t*>(pcm_data);
+				state->pcm_data.assign(bytes, bytes + audioout2_pcm_size(*state));
+			} else {
+				state->pcm_data.clear();
+			}
 		}
 		g_audioout2_port_mutex.Unlock();
 	}

src/libs/libKernel.cpp

@@ -1226,6 +1226,7 @@ static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_
 
 	auto* program = rt->FindProgramByFileName(module_path);
 	if (program != nullptr) {
+		++program->load_count;
 		if (res != nullptr) {
 			*res = OK;
 		}
@@ -1240,6 +1241,7 @@ static KYTY_SYSV_ABI KernelModule KernelLoadStartModule(const char* module_file_
 
 	rt->RelocateProgram(program);
 
+	program->load_count = 1;
 	int result = rt->StartModule(program, args, argp, nullptr);
 
 	LOGF("\tmodule_start() result = %d\n", result);
@@ -1269,6 +1271,11 @@ static int KYTY_SYSV_ABI KernelStopUnloadModule(KernelModule handle, size_t args
 
 	if (program == nullptr) {
 		LOGF("\tinvalid module handle = %" PRId32 "\n", handle);
+		return KERNEL_ERROR_ESRCH;
+	}
+
+	if (program->load_count > 1) {
+		--program->load_count;
 		return OK;
 	}
 

src/libs/libVideoOut.cpp

@@ -1,5 +1,6 @@
 #include "common/abi.h"
 #include "graphics/presentation/videoOut.h"
+#include "libs/errno.h"
 #include "libs/libs.h"
 #include "loader/symbolDatabase.h"
 
@@ -51,8 +52,14 @@ namespace LibGen5::VrrStatus {
 
 LIB_VERSION("VideoOutVrrStatus", 1, "VideoOut", 1, 1);
 
+static KYTY_SYSV_ABI int VideoOutVrrStatus_kP2L8t3j_aM() {
+	// The observed guest call passes no arguments.
+	// Return success for Kyty's fixed-refresh path.
+	return OK;
+}
+
 LIB_DEFINE(InitVideoOutVrrStatus_1) {
-	LIB_FUNC("kP2L8t3j-aM", VideoOut::VideoOutGetVrrStatus);
+	LIB_FUNC("kP2L8t3j-aM", VideoOutVrrStatus_kP2L8t3j_aM);
 }
 
 } // namespace LibGen5::VrrStatus

src/libs/libs.h

@@ -7,6 +7,8 @@
 #include "common/threads.h"
 #include "loader/timer.h" // IWYU pragma: keep
 
+#include <fmt/format.h>
+
 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
 #define PRINT_NAME_ENABLED g_print_name
 
@@ -49,12 +51,7 @@
 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
 #define PRINT_NAME()                                                                               \
 	if (PRINT_NAME_ENABLED) {                                                                      \
-		if (Log::GetDirection() != Log::Direction::Silent) {                                       \
-			const auto print_name_time = Loader::Timer::GetTime().ToString("HH24:MI:SS.FFF");      \
-			LOGF_COLOR(Log::Color::Cyan, "[%d][%s] %s::%s::%s()\n",                                \
-			           Common::Thread::GetThreadIdUnique(), print_name_time.c_str(), g_library,    \
-			           g_module, __func__);                                                        \
-		}                                                                                          \
+		Libs::PrintName(g_library, g_module, __func__);                                              \
 	}
 
 namespace Loader {
@@ -63,6 +60,19 @@ class SymbolDatabase;
 
 namespace Libs {
 
+// Keep the formatting path from inflating fiber functions' stack frames under LTO.
+[[gnu::noinline]] inline void PrintName(const char* library, const char* module, const char* function) {
+	if (Log::GetDirection() != Log::Direction::Silent) {
+		const auto elapsed_ms      = static_cast<uint64_t>(Loader::Timer::GetTimeMs());
+		const auto print_name_time = fmt::format(
+		    "{:02}:{:02}:{:02}.{:03}", elapsed_ms / 3600000, (elapsed_ms / 60000) % 60,
+		    (elapsed_ms / 1000) % 60, elapsed_ms % 1000);
+		LOGF_COLOR(Log::Color::Cyan, "[%d][%s] %s::%s::%s()\n",
+		           Common::Thread::GetThreadIdUnique(), print_name_time.c_str(), library, module,
+		           function);
+	}
+}
+
 void InitAll(Loader::SymbolDatabase* s);
 
 } // namespace Libs

src/loader/jit.h

@@ -7,53 +7,6 @@ namespace Loader::Jit {
 
 #pragma pack(1)
 
-struct JmpWithIndex {
-	void SetIndex(uint32_t index) { *reinterpret_cast<uint32_t*>(&code[1]) = index; }
-
-	void SetFunc(void* handler) {
-		auto func_addr = reinterpret_cast<int64_t>(handler);
-		auto rip_addr  = reinterpret_cast<int64_t>(&code[10]);
-		auto offset64  = func_addr - rip_addr;
-		auto offset32  = static_cast<uint32_t>(static_cast<uint64_t>(offset64) & 0xffffffffu);
-
-		*reinterpret_cast<uint32_t*>(&code[6]) = offset32;
-	}
-
-	static uint64_t GetSize() { return 16; }
-
-	// 68 00 00 00 00          push     <index>
-	// E9 E0 FF FF FF          jmp      <handler>
-	uint8_t code[16] = {0x68, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00,
-	                    0x00, 0x00, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
-};
-
-struct CallPlt {
-	explicit CallPlt(uint32_t table_size) {
-		for (uint32_t index = 0; index < table_size; index++) {
-			auto* c = new (&code[32] + JmpWithIndex::GetSize() * index) JmpWithIndex;
-			c->SetIndex(index);
-			c->SetFunc(this);
-		}
-	}
-
-	void SetPltGot(uint64_t vaddr) { *reinterpret_cast<uint64_t*>(&code[2]) = vaddr; }
-
-	uint64_t GetAddr(uint32_t index) {
-		return reinterpret_cast<uint64_t>(&code[32] + JmpWithIndex::GetSize() * index);
-	}
-
-	static uint64_t GetSize(uint32_t table_size) {
-		return 32 + JmpWithIndex::GetSize() * table_size;
-	}
-
-	// 0:  49 bb 88 77 66 55 44    movabs r11,0x1122334455667788
-	// 7:  33 22 11
-	// a:  41 ff 73 08             push   QWORD PTR [r11+0x8]
-	// e:  41 ff 63 10             jmp    QWORD PTR [r11+0x10]
-	uint8_t code[32] = {0x49, 0xBB, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x41, 0xFF,
-	                    0x73, 0x08, 0x41, 0xFF, 0x63, 0x10, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90};
-};
-
 struct JmpRax {
 	template <class Handler>
 	void SetFunc(Handler func) {

src/loader/runtimeLinker.cpp

@@ -126,19 +126,13 @@ struct StubbedImportRecord {
 	std::string program;
 };
 
-// The structure will be passed via the stack
-// since the size of an object is larger than 16 bytes
-struct RelocateHandlerStack {
-	uint64_t stack[3];
-};
-
 static std::vector<StubbedImportRecord> g_stubbed_imports;
 static std::atomic_uint32_t             g_unresolved_stub_call_log_count {0};
 static std::vector<uint64_t>            g_unresolved_stub_thunk_pages;
 static uint64_t                         g_unresolved_stub_thunk_offset = 0;
 static constexpr uint64_t               UNRESOLVED_STUB_PAGE_SIZE      = 4096;
 
-static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id);
+static KYTY_SYSV_ABI uint64_t UnresolvedImportStub(uint64_t record_id);
 
 static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
 	auto* ptr     = reinterpret_cast<uint64_t*>(vaddr);
@@ -148,7 +142,7 @@ static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
 }
 
 static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
-	constexpr uint64_t thunk_size = 165;
+	constexpr uint64_t thunk_size = 34;
 
 	if (g_unresolved_stub_thunk_pages.empty() ||
 	    g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
@@ -164,105 +158,18 @@ static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
 	                                        g_unresolved_stub_thunk_offset);
 	g_unresolved_stub_thunk_offset += thunk_size;
 
-	const auto target = reinterpret_cast<uint64_t>(ResolveImportStubWithId);
-	uint8_t    bytes[thunk_size] {};
-	size_t     i      = 0;
-	const auto emit   = [&](uint8_t b) { bytes[i++] = b; };
-	const auto emit64 = [&](uint64_t v) {
-		std::memcpy(bytes + i, &v, sizeof(v));
-		i += sizeof(v);
+	uint8_t bytes[thunk_size] = {
+	    0x48, 0x83, 0xec, 0x08,               // sub rsp, 8
+	    0x48, 0xbf, 0, 0, 0, 0, 0, 0, 0, 0, // mov rdi, record_id
+	    0x48, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, // mov rax, UnresolvedImportStub
+	    0xff, 0xd0,                           // call rax
+	    0x48, 0x83, 0xc4, 0x08,               // add rsp, 8
+	    0x0f, 0x57, 0xc0,                     // xorps xmm0, xmm0
+	    0xc3,                                 // ret
 	};
-	const auto emit32 = [&](uint32_t v) {
-		std::memcpy(bytes + i, &v, sizeof(v));
-		i += sizeof(v);
-	};
-	const auto save_xmm = [&](uint8_t reg, uint8_t offset) {
-		emit(0xf3);
-		emit(0x0f);
-		emit(0x7f);
-		if (offset == 0) {
-			emit(static_cast<uint8_t>(0x04u | (reg << 3u)));
-			emit(0x24);
-		} else {
-			emit(static_cast<uint8_t>(0x44u | (reg << 3u)));
-			emit(0x24);
-			emit(offset);
-		}
-	};
-	const auto load_xmm = [&](uint8_t reg, uint8_t offset) {
-		emit(0xf3);
-		emit(0x0f);
-		emit(0x6f);
-		if (offset == 0) {
-			emit(static_cast<uint8_t>(0x04u | (reg << 3u)));
-			emit(0x24);
-		} else {
-			emit(static_cast<uint8_t>(0x44u | (reg << 3u)));
-			emit(0x24);
-			emit(offset);
-		}
-	};
-
-	emit(0x50); // push rax; preserve AL for variadic SysV calls
-	emit(0x57); // push rdi
-	emit(0x56); // push rsi
-	emit(0x52); // push rdx
-	emit(0x51); // push rcx
-	emit(0x41);
-	emit(0x50); // push r8
-	emit(0x41);
-	emit(0x51); // push r9
-	emit(0x48);
-	emit(0x81);
-	emit(0xec);
-	emit32(0x80); // sub rsp, 0x80
-	for (uint8_t reg = 0; reg < 8; reg++) {
-		save_xmm(reg, static_cast<uint8_t>(reg * 0x10u));
-	}
-	emit(0x48);
-	emit(0xbf);
-	emit64(record_id); // mov rdi, record_id
-	emit(0x48);
-	emit(0xb8);
-	emit64(target); // mov rax, ResolveImportStubWithId
-	emit(0xff);
-	emit(0xd0); // call rax
-	emit(0x49);
-	emit(0x89);
-	emit(0xc3); // mov r11, rax
-	for (uint8_t reg = 0; reg < 8; reg++) {
-		load_xmm(reg, static_cast<uint8_t>(reg * 0x10u));
-	}
-	emit(0x48);
-	emit(0x81);
-	emit(0xc4);
-	emit32(0x80); // add rsp, 0x80
-	emit(0x41);
-	emit(0x59); // pop r9
-	emit(0x41);
-	emit(0x58); // pop r8
-	emit(0x59); // pop rcx
-	emit(0x5a); // pop rdx
-	emit(0x5e); // pop rsi
-	emit(0x5f); // pop rdi
-	emit(0x58); // pop rax
-	emit(0x4d);
-	emit(0x85);
-	emit(0xdb); // test r11, r11
-	emit(0x74);
-	emit(0x03); // jz +3
-	emit(0x41);
-	emit(0xff);
-	emit(0xe3); // jmp r11
-	// Match the integer fallback for floating-point return values.
-	emit(0x0f);
-	emit(0x57);
-	emit(0xc0); // xorps xmm0, xmm0
-	emit(0x31);
-	emit(0xc0); // xor eax, eax
-	emit(0xc3); // ret
-
-	EXIT_NOT_IMPLEMENTED(i != thunk_size);
+	const auto target = reinterpret_cast<uint64_t>(UnresolvedImportStub);
+	std::memcpy(bytes + 6, &record_id, sizeof(record_id));
+	std::memcpy(bytes + 16, &target, sizeof(target));
 	std::memcpy(code, bytes, sizeof(bytes));
 	Common::VirtualMemory::FlushInstructionCache(reinterpret_cast<uint64_t>(code), thunk_size);
 	return reinterpret_cast<uint64_t>(code);
@@ -270,15 +177,8 @@ static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
 
 static uint64_t RegisterStubbedImport(uint32_t index, const Program* program,
                                       const RelocationInfo& ri) {
-	const auto program_name = program != nullptr ? Common::PathToString(program->file_name) : "";
-
-	for (auto& record: g_stubbed_imports) {
+	for (const auto& record: g_stubbed_imports) {
 		if (record.patch_vaddr == ri.vaddr) {
-			record.index   = index;
-			record.name    = ri.name;
-			record.type    = ri.type;
-			record.bind    = ri.bind;
-			record.program = program_name;
 			return record.thunk_vaddr;
 		}
 	}
@@ -289,7 +189,7 @@ static uint64_t RegisterStubbedImport(uint32_t index, const Program* program,
 	record.name        = ri.name;
 	record.type        = ri.type;
 	record.bind        = ri.bind;
-	record.program     = program_name;
+	record.program     = Common::PathToString(program->file_name);
 	g_stubbed_imports.push_back(record);
 	const auto record_id                     = g_stubbed_imports.size() - 1;
 	const auto thunk                         = AllocateUnresolvedImportThunk(record_id);
@@ -297,31 +197,7 @@ static uint64_t RegisterStubbedImport(uint32_t index, const Program* program,
 	return thunk;
 }
 
-static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) {
-	if (record_id < g_stubbed_imports.size()) {
-		auto& record = g_stubbed_imports[record_id];
-		auto  nid    = record.name;
-		auto  pos    = nid.find('[');
-		if (pos != std::string::npos) {
-			nid.resize(pos);
-		}
-
-		SymbolRecord resolved {};
-		if (!nid.empty() &&
-		    Common::Singleton<RuntimeLinker>::Instance()->ResolveLoadedSymbolByNid(nid, record.type,
-		                                                                           &resolved) &&
-		    resolved.vaddr != 0 && resolved.vaddr != record.thunk_vaddr) {
-			LOGF("Late-resolved import: %s -> %s [0x%016" PRIx64 "]\n", record.name.c_str(),
-			     resolved.name.c_str(), resolved.vaddr);
-
-			if (record.patch_vaddr != 0) {
-				PatchGuestMemory64(record.patch_vaddr, resolved.vaddr);
-			}
-
-			return resolved.vaddr;
-		}
-	}
-
+static KYTY_SYSV_ABI uint64_t UnresolvedImportStub(uint64_t record_id) {
 	const auto log_index = g_unresolved_stub_call_log_count.fetch_add(1);
 	if (log_index < 1024) {
 		if (record_id < g_stubbed_imports.size()) {
@@ -1007,135 +883,56 @@ static RelocationInfo GetRelocationInfo(Elf64_Rela* r, Program* program) {
 	return ret;
 }
 
-static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool jmprela_table,
-                           bool imports_only, std::vector<std::string>* unresolved) {
+static bool RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool jmprela_table) {
 	KYTY_PROFILER_FUNCTION();
 
-	auto ri = GetRelocationInfo(r, program);
-
-	if (imports_only &&
-	    (ri.bind_self || (ri.bind != BindType::Global && ri.bind != BindType::Weak))) {
-		return;
-	}
-
-	[[maybe_unused]] bool patched        = false;
-	bool                  stubbed_import = false;
-	bool                  stubbed_func   = false;
-
-	// KYTY_PROFILER_BLOCK("patch");
-
-	if (ri.resolved) {
-		patched = PatchGuestMemory64(ri.vaddr, ri.value);
-	} else {
-		uint64_t value = 0;
-		bool     weak  = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved);
-		if (ri.type == SymbolType::Object && weak) {
+	const auto ri      = GetRelocationInfo(r, program);
+	auto       value   = ri.value;
+	bool       stubbed = false;
+	if (!ri.resolved) {
+		const bool weak = ri.bind == BindType::Weak || !program->fail_if_global_not_resolved;
+		if (!weak) {
+			LOGF("Stubbed: %s\n",
+			     fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value, ri.name,
+			                 magic_enum::enum_name(ri.type), magic_enum::enum_name(ri.bind),
+			                 ri.dbg_name).c_str());
+		}
+		if (ri.type == SymbolType::Object) {
 			value = g_invalid_memory;
-		} else if (ri.type == SymbolType::Func && jmprela_table && weak) {
-			value          = RegisterStubbedImport(index, program, ri);
-			stubbed_import = true;
-			stubbed_func   = true;
-		} else if (ri.type == SymbolType::Func && !jmprela_table && weak) {
-			value        = RegisterStubbedImport(index, program, ri);
-			stubbed_func = true;
-		} else if (ri.type == SymbolType::NoType && weak) {
-			value = RuntimeLinker::ReadFromElf(program, ri.vaddr) + ri.base_vaddr;
-		}
-
-		if (value != 0) {
-			patched = PatchGuestMemory64(ri.vaddr, value);
 		} else {
-			auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
-			                           ri.name.c_str(), magic_enum::enum_name(ri.type),
-			                           magic_enum::enum_name(ri.bind), ri.dbg_name.c_str());
-
-			if (unresolved != nullptr) {
-				unresolved->push_back(dbg_str);
-			} else {
-				EXIT("Can't resolve: %s\n", dbg_str.c_str());
-			}
-
-			if (ri.type == SymbolType::Object) {
-				value = g_invalid_memory;
-			} else if (ri.type == SymbolType::Func || ri.type == SymbolType::NoType) {
-				value        = RegisterStubbedImport(index, program, ri);
-				stubbed_func = true;
-				if (jmprela_table) {
-					stubbed_import = true;
-				}
-			}
-
-			if (value != 0) {
-				patched = PatchGuestMemory64(ri.vaddr, value);
-			}
+			value   = RegisterStubbedImport(index, program, ri);
+			stubbed = true;
 		}
 	}
 
-	// KYTY_PROFILER_END_BLOCK;
-
-	if (patched && stubbed_import) {
-		const auto thunk = RegisterStubbedImport(index, program, ri);
-		LOGF("Relocate: unresolved PLT import patched to stub [%u] [%016" PRIx64 "] <- %016" PRIx64
+	const bool patched = PatchGuestMemory64(ri.vaddr, value);
+	if (patched && stubbed) {
+		LOGF("Relocate: unresolved %s import patched to stub [%u] [%016" PRIx64 "] <- %016" PRIx64
 		     ", %s, %s, %s, %s\n",
-		     index, ri.vaddr, thunk, ri.name.c_str(), magic_enum::enum_name(ri.type),
-		     magic_enum::enum_name(ri.bind), Common::PathToString(program->file_name).c_str());
-	} else if (patched && stubbed_func) {
-		const auto thunk = RegisterStubbedImport(index, program, ri);
-		LOGF("Relocate: unresolved non-PLT function patched to stub [%u] [%016" PRIx64
-		     "] <- %016" PRIx64 ", %s, %s, %s, %s\n",
-		     index, ri.vaddr, thunk, ri.name.c_str(), magic_enum::enum_name(ri.type),
-		     magic_enum::enum_name(ri.bind), Common::PathToString(program->file_name).c_str());
+		     jmprela_table ? "PLT" : "non-PLT", index, ri.vaddr, value, ri.name.c_str(),
+		     magic_enum::enum_name(ri.type), magic_enum::enum_name(ri.bind),
+		     Common::PathToString(program->file_name).c_str());
 	}
 
-	if (program->dbg_print_reloc) {
-		if (patched && !ri.bind_self &&
-		    (ri.bind == BindType::Global || ri.bind == BindType::Weak ||
-		     ri.type == SymbolType::TlsModule)) {
-			auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
-			                           ri.name.c_str(), magic_enum::enum_name(ri.type),
-			                           magic_enum::enum_name(ri.bind), ri.dbg_name.c_str());
-
-			LOGF("Relocate: %s\n", dbg_str.c_str());
-		}
+	if (program->dbg_print_reloc && patched && !ri.bind_self &&
+	    (ri.bind == BindType::Global || ri.bind == BindType::Weak ||
+	     ri.type == SymbolType::TlsModule)) {
+		LOGF("Relocate: %s\n",
+		     fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value, ri.name,
+		                 magic_enum::enum_name(ri.type), magic_enum::enum_name(ri.bind),
+		                 ri.dbg_name).c_str());
 	}
+	return ri.resolved;
 }
 
-static void RelocateRecords(Elf64_Rela* records, uint64_t size, Program* program,
-                            bool jmprela_table, bool imports_only,
-                            std::vector<std::string>* unresolved) {
-	KYTY_PROFILER_FUNCTION();
-
-	uint32_t index = 0;
-	for (auto* r = records;
-	     reinterpret_cast<uint8_t*>(r) < reinterpret_cast<uint8_t*>(records) + size; r++, index++) {
-		RelocateRecord(index, r, program, jmprela_table, imports_only, unresolved);
+static void ForEachRelocation(Program* program, auto&& func) {
+	const auto& info = *program->dynamic_info;
+	for (uint32_t i = 0; i < info.rela_table_total_size / sizeof(Elf64_Rela); ++i) {
+		func(info.rela_table + i, i, false);
 	}
-}
-
-__attribute__((naked)) static KYTY_SYSV_ABI void RelocateHandlerReturnStub() {
-	asm volatile("addq $8, %rsp\n\t"
-	             "retq\n");
-}
-
-static KYTY_SYSV_ABI uint64_t RelocateHandler(RelocateHandlerStack s) {
-	auto*       stack     = s.stack;
-	auto*       program   = reinterpret_cast<Program*>(stack[-1]);
-	auto        rel_index = stack[0];
-	std::string name      = "<unknown function>";
-
-	if (program != nullptr && program->dynamic_info != nullptr &&
-	    program->dynamic_info->jmprela_table != nullptr) {
-		auto ri = GetRelocationInfo(program->dynamic_info->jmprela_table + rel_index, program);
-
-		name = ri.name.c_str();
+	for (uint32_t i = 0; i < info.jmprela_table_size / sizeof(Elf64_Rela); ++i) {
+		func(info.jmprela_table + i, i, true);
 	}
-
-	// Restore return address (for stack trace)
-	stack[-1] = reinterpret_cast<uint64_t>(RelocateHandlerReturnStub);
-
-	LOGF("=== Stubbed function, returning OK ===\n[%d]\t%s\n", Common::Thread::GetThreadIdUnique(),
-	     name.c_str());
-	return 0;
 }
 
 static KYTY_MS_ABI uint8_t* TlsMainGetAddr() {
@@ -1314,8 +1111,6 @@ void RuntimeLinker::RelocateAll() {
 	for (auto* p: m_programs) {
 		Relocate(p);
 	}
-
-	m_relocated = true;
 }
 
 void RuntimeLinker::RelocateProgram(Program* program) {
@@ -1335,17 +1130,52 @@ void RuntimeLinker::UnloadProgram(Program* program) {
 
 	Common::LockGuard lock(m_mutex);
 
-	if (auto it = std::find(m_programs.begin(), m_programs.end(), program);
-	    it != m_programs.end()) {
-		DeleteProgram(*it);
-		m_programs.erase(it);
-	} else {
-		EXIT("program not found");
-	}
+	auto it = std::find(m_programs.begin(), m_programs.end(), program);
+	EXIT_IF(it == m_programs.end());
+	m_programs.erase(it);
 
-	if (m_relocated) {
-		RelocateAll();
+	for (auto* importer: m_programs) {
+		const auto& info = *importer->dynamic_info;
+		const auto rela_count =
+		    static_cast<uint32_t>(info.rela_table_total_size / sizeof(Elf64_Rela));
+		ForEachRelocation(importer, [&](Elf64_Rela* r, uint32_t index, bool jmprela) {
+			const auto bit_index = (jmprela ? rela_count : 0) + index;
+			auto&      bits      = importer->rela_bits[bit_index >> 3u];
+			const auto mask      = 1u << (bit_index & 7u);
+			const auto type      = r->GetType();
+			if ((bits & mask) == 0 || (type != R_X86_64_64 && type != R_X86_64_GLOB_DAT &&
+			                           type != R_X86_64_JUMP_SLOT)) {
+				return;
+			}
+			const auto& symbol = info.symbol_table[r->GetSymbol()];
+			if (symbol.GetBind() != STB_GLOBAL && symbol.GetBind() != STB_WEAK) {
+				return;
+			}
+			const auto value =
+			    *reinterpret_cast<const uint64_t*>(importer->base_vaddr + r->r_offset);
+			const auto address = value - (type == R_X86_64_64 ? r->r_addend : 0);
+			if (address < program->base_vaddr ||
+			    address >= program->base_vaddr + program->mapped_size) {
+				return;
+			}
+			SymbolType symbol_type;
+			switch (symbol.GetType()) {
+				case STT_NOTYPE: symbol_type = SymbolType::NoType; break;
+				case STT_FUNC: symbol_type = SymbolType::Func; break;
+				case STT_OBJECT: symbol_type = SymbolType::Object; break;
+				default: return;
+			}
+			const std::string name = info.str_table + symbol.st_name;
+			if (program->export_symbols->FindByNid(name.substr(0, name.find('#')), symbol_type,
+			                                       address) != nullptr) {
+				bits &= ~mask;
+				if (RelocateRecord(index, r, importer, jmprela)) {
+					bits |= mask;
+				}
+			}
+		});
 	}
+	DeleteProgram(program);
 }
 
 RuntimeLinker::RuntimeLinker(): m_symbols(std::make_unique<SymbolDatabase>()) {
@@ -1499,7 +1329,6 @@ void RuntimeLinker::Clear() {
 	g_tls_cached_main_tcb     = nullptr;
 	g_desired_base_addr       = SYSTEM_RESERVED + CODE_BASE_OFFSET;
 	m_symbols.reset();
-	m_relocated = false;
 }
 
 void RuntimeLinker::Resolve(const std::string& name, SymbolType type, Program* program,
@@ -1559,10 +1388,20 @@ void RuntimeLinker::Resolve(const std::string& name, SymbolType type, Program* p
 			}
 
 			if (rec == nullptr) {
-				if (auto* p = FindProgram(*m, *l); p != nullptr && p->export_symbols != nullptr) {
+				for (auto* p: m_programs) {
+					const auto& libs = p->dynamic_info->export_libs;
+					const auto& modules = p->dynamic_info->export_modules;
+					if (p->export_symbols == nullptr ||
+					    std::find(libs.begin(), libs.end(), *l) == libs.end() ||
+					    std::find(modules.begin(), modules.end(), *m) == modules.end()) {
+						continue;
+					}
 					rec = p->export_symbols->Find(sr);
-					if (bind_self != nullptr) {
-						*bind_self = (p == program);
+					if (rec != nullptr) {
+						if (bind_self != nullptr) {
+							*bind_self = (p == program);
+						}
+						break;
 					}
 				}
 			}
@@ -1575,7 +1414,6 @@ void RuntimeLinker::Resolve(const std::string& name, SymbolType type, Program* p
 			}
 
 			if (rec != nullptr) {
-				//*out_vaddr = rec->vaddr;
 				*out_info = *rec;
 			} else {
 				out_info->vaddr    = 0;
@@ -1598,33 +1436,6 @@ void RuntimeLinker::Resolve(const std::string& name, SymbolType type, Program* p
 	}
 }
 
-bool RuntimeLinker::ResolveLoadedSymbolByNid(const std::string& nid, SymbolType type,
-                                             SymbolRecord* out_info) {
-	KYTY_PROFILER_FUNCTION();
-
-	Common::LockGuard lock(m_mutex);
-
-	EXIT_IF(out_info == nullptr);
-
-	for (auto* p: m_programs) {
-		if (p != nullptr && p->export_symbols != nullptr) {
-			if (const auto* rec = p->export_symbols->FindByNid(nid, type); rec != nullptr) {
-				*out_info = *rec;
-				return true;
-			}
-		}
-	}
-
-	if (m_symbols != nullptr) {
-		if (const auto* rec = m_symbols->FindByNid(nid, type); rec != nullptr) {
-			*out_info = *rec;
-			return true;
-		}
-	}
-
-	return false;
-}
-
 uint64_t RuntimeLinker::ReadFromElf(Program* program, uint64_t vaddr) {
 	EXIT_IF(program == nullptr);
 	EXIT_IF(program->base_vaddr == 0 || program->base_size == 0);
@@ -2208,11 +2019,6 @@ void RuntimeLinker::DeleteProgram(Program* p) {
 		EXIT_IF(
 		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
 	}
-
-	if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) {
-		const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
-		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size));
-	}
 }
 
 void RuntimeLinker::ParseProgramDynamicInfo(Program* program) {
@@ -2255,10 +2061,6 @@ void RuntimeLinker::ParseProgramDynamicInfo(Program* program) {
 	GetDynValue(elf, &program->dynamic_info->fini_array_size, DT_FINI_ARRAYSZ);
 	GetDynValue(elf, &program->dynamic_info->preinit_array_size, DT_PREINIT_ARRAYSZ);
 
-	EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_PLTGOT) && elf->HasDynValue(DT_PLTGOT));
-	GetDynPtr(elf, &program->dynamic_info->pltgot_vaddr, DT_OS_PLTGOT);
-	GetDynPtr(elf, &program->dynamic_info->pltgot_vaddr, DT_PLTGOT);
-
 	Elf64_Sxword jmprel_type = 0;
 	EXIT_NOT_IMPLEMENTED(elf->HasDynValue(DT_OS_PLTREL) && elf->HasDynValue(DT_PLTREL));
 	GetDynValue(elf, &jmprel_type, DT_OS_PLTREL);
@@ -2281,6 +2083,11 @@ void RuntimeLinker::ParseProgramDynamicInfo(Program* program) {
 	GetDynValue(elf, &program->dynamic_info->rela_table_total_size, DT_RELASZ);
 	GetDynValue(elf, &program->dynamic_info->rela_table_entry_size, DT_OS_RELAENT);
 	GetDynValue(elf, &program->dynamic_info->rela_table_entry_size, DT_RELAENT);
+	EXIT_NOT_IMPLEMENTED(program->dynamic_info->rela_table_total_size % sizeof(Elf64_Rela) != 0 ||
+	                     program->dynamic_info->jmprela_table_size % sizeof(Elf64_Rela) != 0);
+	const auto rela_count = program->dynamic_info->rela_table_total_size / sizeof(Elf64_Rela);
+	const auto jmprela_count = program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
+	program->rela_bits.resize((rela_count + jmprela_count + 7u) / 8u);
 
 	GetDynValue(elf, &program->dynamic_info->relative_count, DT_RELACOUNT);
 
@@ -2327,43 +2134,6 @@ void RuntimeLinker::ParseProgramDynamicInfo(Program* program) {
 	           DT_OS_EXPORT_LIB_1);
 }
 
-static void InstallRelocateHandler(Program* program) {
-	KYTY_PROFILER_FUNCTION();
-
-	uint64_t pltgot_vaddr = program->dynamic_info->pltgot_vaddr + program->base_vaddr;
-	uint64_t pltgot_size  = static_cast<uint64_t>(3) * 8;
-	void**   pltgot       = reinterpret_cast<void**>(pltgot_vaddr);
-
-	Common::VirtualMemory::Mode old_mode {};
-	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(
-	    pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode));
-
-	pltgot[1] = program;
-	pltgot[2] = reinterpret_cast<void*>(RelocateHandler);
-
-	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode));
-
-	if (Common::VirtualMemory::IsExecute(old_mode)) {
-		Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size);
-	}
-
-	// TODO(): check if this table already generated by compiler (sometimes it is missing)
-	if (program->custom_call_plt_vaddr == 0) {
-		program->custom_call_plt_num =
-		    program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
-		auto size                      = Jit::CallPlt::GetSize(program->custom_call_plt_num);
-		program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory(
-		    SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt");
-		EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0);
-		auto* code = new (reinterpret_cast<void*>(program->custom_call_plt_vaddr))
-		    Jit::CallPlt(program->custom_call_plt_num);
-		code->SetPltGot(pltgot_vaddr);
-		EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size,
-		                                                     Common::VirtualMemory::Mode::Execute));
-		Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size);
-	}
-}
-
 void RuntimeLinker::Relocate(Program* program) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -2383,46 +2153,23 @@ void RuntimeLinker::Relocate(Program* program) {
 	EXIT_NOT_IMPLEMENTED(program->dynamic_info->jmprela_table == nullptr);
 	EXIT_NOT_IMPLEMENTED(program->dynamic_info->rela_table == nullptr);
 	EXIT_NOT_IMPLEMENTED(program->dynamic_info->symbol_table == nullptr);
-	EXIT_NOT_IMPLEMENTED(program->dynamic_info->pltgot_vaddr == 0);
 
-	InstallRelocateHandler(program);
+	const auto rela_count = static_cast<uint32_t>(
+	    program->dynamic_info->rela_table_total_size / sizeof(Elf64_Rela));
 
-	std::vector<std::string> unresolved;
-	const bool               imports_only = program->relocated;
-
-	RelocateRecords(program->dynamic_info->rela_table, program->dynamic_info->rela_table_total_size,
-	                program, false, imports_only, &unresolved);
-	RelocateRecords(program->dynamic_info->jmprela_table, program->dynamic_info->jmprela_table_size,
-	                program, true, imports_only, &unresolved);
-	program->relocated = true;
+	ForEachRelocation(program, [&](Elf64_Rela* r, uint32_t index, bool jmprela) {
+		const auto bit_index = (jmprela ? rela_count : 0) + index;
+		if ((program->rela_bits[bit_index >> 3u] & (1u << (bit_index & 7u))) == 0 &&
+		    RelocateRecord(index, r, program, jmprela)) {
+			program->rela_bits[bit_index >> 3u] |= 1u << (bit_index & 7u);
+		}
+	});
 
 	if (program->tls.image_vaddr != 0 && program->tls.init_size != 0 &&
 	    program->tls.init_image.empty()) {
 		const auto* src = reinterpret_cast<const uint8_t*>(program->tls.image_vaddr);
 		program->tls.init_image.assign(src, src + program->tls.init_size);
 	}
-
-	if (!unresolved.empty()) {
-		LOGF("--- Stubbed unresolved imports: %zu ---\n", unresolved.size());
-		for (const auto& symbol: unresolved) {
-			LOGF("Stubbed: %s\n", symbol.c_str());
-		}
-	}
-}
-
-Program* RuntimeLinker::FindProgram(const ModuleId& m, const LibraryId& l) {
-	Common::LockGuard lock(m_mutex);
-
-	for (auto* p: m_programs) {
-		const auto& export_libs    = p->dynamic_info->export_libs;
-		const auto& export_modules = p->dynamic_info->export_modules;
-
-		if (std::find(export_libs.begin(), export_libs.end(), l) != export_libs.end() &&
-		    std::find(export_modules.begin(), export_modules.end(), m) != export_modules.end()) {
-			return p;
-		}
-	}
-	return nullptr;
 }
 
 const ModuleId* RuntimeLinker::FindModule(const Program& program, const std::string& id) {

src/loader/runtimeLinker.h

@@ -86,7 +86,6 @@ struct DynamicInfo {
 	uint64_t init_array_size     = 0;
 	uint64_t fini_array_size     = 0;
 	uint64_t preinit_array_size  = 0;
-	uint64_t pltgot_vaddr        = 0;
 
 	Elf64_Rela* jmprela_table      = nullptr;
 	uint64_t    jmprela_table_size = 0;
@@ -116,6 +115,7 @@ struct Program {
 
 	int32_t                      unique_id = -1;
 	RuntimeLinker*               rt        = nullptr;
+	uint32_t                     load_count = 0; // Successful sceKernelLoadStartModule calls.
 	std::filesystem::path        file_name;
 	std::unique_ptr<Elf64>       elf;
 	std::unique_ptr<DynamicInfo> dynamic_info;
@@ -132,10 +132,8 @@ struct Program {
 	ThreadLocalStorage              tls;
 	bool                            fail_if_global_not_resolved = true;
 	bool                            dbg_print_reloc             = false;
-	bool                            relocated                   = false;
+	std::vector<uint8_t>            rela_bits;
 	uint64_t                        proc_param_vaddr            = 0;
-	uint64_t                        custom_call_plt_vaddr       = 0;
-	uint32_t                        custom_call_plt_num         = 0;
 };
 
 class RuntimeLinker {
@@ -171,7 +169,6 @@ public:
 
 	void Resolve(const std::string& name, SymbolType type, Program* program, SymbolRecord* out_info,
 	             bool* bind_self);
-	bool ResolveLoadedSymbolByNid(const std::string& nid, SymbolType type, SymbolRecord* out_info);
 
 	SymbolDatabase* Symbols() { return m_symbols.get(); }
 
@@ -194,14 +191,11 @@ private:
 	static void SetupTlsHandler(Program* program);
 	void        PreloadAdjacentPrograms();
 
-	Program* FindProgram(const ModuleId& m, const LibraryId& l);
-
 	static const ModuleId*  FindModule(const Program& program, const std::string& id);
 	static const LibraryId* FindLibrary(const Program& program, const std::string& id);
 
 	std::vector<Program*>           m_programs;
 	std::unique_ptr<SymbolDatabase> m_symbols;
-	bool                            m_relocated = false;
 	Common::Mutex                   m_mutex;
 
 	application_heap_malloc_func_t         m_application_heap_malloc         = nullptr;

src/loader/symbolDatabase.cpp

@@ -65,12 +65,14 @@ const SymbolRecord* SymbolDatabase::Find(const SymbolResolve& s) const {
 	return &m_symbols[index];
 }
 
-const SymbolRecord* SymbolDatabase::FindByNid(const std::string& nid, SymbolType type) const {
+const SymbolRecord* SymbolDatabase::FindByNid(const std::string& nid, SymbolType type,
+                                                uint64_t vaddr) const {
 	auto prefix = nid + "[";
 	auto suffix = fmt::format("[{}]", magic_enum::enum_name(type));
 
 	for (const auto& symbol: m_symbols) {
-		if (symbol.name.starts_with(prefix) && symbol.name.ends_with(suffix)) {
+		if ((vaddr == 0 || symbol.vaddr == vaddr) && symbol.name.starts_with(prefix) &&
+		    symbol.name.ends_with(suffix)) {
 			return &symbol;
 		}
 	}

src/loader/symbolDatabase.h

@@ -43,7 +43,8 @@ public:
 	void Add(const SymbolResolve& s, uint64_t vaddr, const std::string& dbg_name);
 
 	[[nodiscard]] const SymbolRecord* Find(const SymbolResolve& s) const;
-	[[nodiscard]] const SymbolRecord* FindByNid(const std::string& nid, SymbolType type) const;
+	[[nodiscard]] const SymbolRecord* FindByNid(const std::string& nid, SymbolType type,
+	                                              uint64_t vaddr = 0) const;
 	[[nodiscard]] const SymbolRecord* FindByName(const std::string& name, SymbolType type) const;
 
 	void DbgDump(const std::string& folder, const std::string& file_name);

src/loader/timer.cpp

@@ -1,7 +1,5 @@
 #include "common/timer.h"
 
-#include "common/abi.h"
-#include "common/dateTime.h"
 #include "loader/timer.h"
 
 namespace Loader::Timer {
@@ -16,8 +14,4 @@ double GetTimeMs() {
 	return g_timer.GetTimeMs();
 }
 
-Common::Time GetTime() {
-	return Common::Time(static_cast<int>(GetTimeMs()));
-}
-
 } // namespace Loader::Timer

src/loader/timer.h

@@ -1,14 +1,10 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_LOADER_TIMER_H_
 #define EMULATOR_INCLUDE_EMULATOR_LOADER_TIMER_H_
 
-#include "common/abi.h"
-#include "common/common.h"
-#include "common/dateTime.h"
 namespace Loader::Timer {
 
-void         Start();
-double       GetTimeMs();
-Common::Time GetTime();
+void   Start();
+double GetTimeMs();
 
 struct Lifecycle {
 	static constexpr const char* name       = "Timer";

tests/AudioOut2PortTests.cpp

@@ -8,6 +8,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
 #include <mutex>
 #include <thread>
 #include <vector>
@@ -16,14 +17,16 @@ namespace {
 
 namespace AudioOut2 = Libs::Audio::AudioOut2;
 
-std::mutex              g_device_mutex;
-std::condition_variable g_device_cv;
-std::vector<int>        g_live_devices;
-std::vector<int>        g_device_backed_handles;
-std::vector<bool>       g_output_blocking;
-int                     g_next_device  = 1;
-int                     g_open_waiters = 0;
-bool                    g_block_opens  = false;
+std::mutex                        g_device_mutex;
+std::condition_variable           g_device_cv;
+std::vector<int>                  g_live_devices;
+std::vector<int>                  g_device_backed_handles;
+std::vector<bool>                 g_output_blocking;
+std::vector<std::vector<uint8_t>> g_output_pcm;
+size_t                            g_capture_bytes = 0;
+int                               g_next_device  = 1;
+int                               g_open_waiters = 0;
+bool                              g_block_opens  = false;
 
 void Check(bool value, const char* text) {
 	if (!value) {
@@ -146,6 +149,17 @@ std::vector<bool> OutputCalls() {
 	return g_output_blocking;
 }
 
+void CaptureOutputPcm(size_t bytes) {
+	std::lock_guard lock(g_device_mutex);
+	g_capture_bytes = bytes;
+	g_output_pcm.clear();
+}
+
+std::vector<std::vector<uint8_t>> OutputPcm() {
+	std::lock_guard lock(g_device_mutex);
+	return g_output_pcm;
+}
+
 void TestSlotReuse() {
 	const auto context = CreateContext();
 	const auto param   = MakeParam();
@@ -327,6 +341,41 @@ void TestHandleWithoutPcmDoesNotBypassQueue() {
 	AudioOut2::AudioOut2ContextDestroy(context);
 }
 
+void TestPcmCopiedBeforeScratchBufferReuse() {
+	const auto context = CreateContext();
+	const auto param   = MakeParam();
+	AudioOut2::AudioOut2PortHandle first = 0;
+	AudioOut2::AudioOut2PortHandle second = 0;
+	Check(AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &first) == OK,
+	      "first port create failed");
+	Check(AudioOut2::AudioOut2PortCreate(context, AsParam(&param), &second) == OK,
+	      "second port create failed");
+
+	std::vector<float> scratch(512 * 2);
+	std::vector<float> first_pcm(scratch.size(), 0.25f);
+	std::vector<float> second_pcm(scratch.size(), -0.5f);
+	std::copy(first_pcm.begin(), first_pcm.end(), scratch.begin());
+	SetPcm(first, scratch.data());
+	std::copy(second_pcm.begin(), second_pcm.end(), scratch.begin());
+	SetPcm(second, scratch.data());
+	std::fill(scratch.begin(), scratch.end(), 0.0f);
+
+	const auto pcm_bytes = scratch.size() * sizeof(float);
+	CaptureOutputPcm(pcm_bytes);
+	Check(AudioOut2::AudioOut2ContextPush(context, 1) == OK, "shared-buffer push failed");
+	const auto output = OutputPcm();
+	Check(output.size() == 2, "shared-buffer push did not output both ports");
+	Check(std::memcmp(output[0].data(), first_pcm.data(), pcm_bytes) == 0,
+	      "first port lost PCM when scratch buffer was reused");
+	Check(std::memcmp(output[1].data(), second_pcm.data(), pcm_bytes) == 0,
+	      "second port lost PCM when scratch buffer was reused");
+
+	CaptureOutputPcm(0);
+	AudioOut2::AudioOut2PortDestroy(first);
+	AudioOut2::AudioOut2PortDestroy(second);
+	AudioOut2::AudioOut2ContextDestroy(context);
+}
+
 } // namespace
 
 namespace Libs::Audio::AudioInternal {
@@ -363,9 +412,15 @@ bool AudioOutHasDevice(int handle) {
 	       g_device_backed_handles.end();
 }
 
-uint32_t AudioOutOutputs(const OutputParam* /*params*/, uint32_t /*num*/, bool blocking) {
+uint32_t AudioOutOutputs(const OutputParam* params, uint32_t num, bool blocking) {
 	std::lock_guard lock(g_device_mutex);
 	g_output_blocking.push_back(blocking);
+	if (g_capture_bytes != 0) {
+		for (uint32_t i = 0; i < num; i++) {
+			const auto* bytes = static_cast<const uint8_t*>(params[i].data);
+			g_output_pcm.emplace_back(bytes, bytes + g_capture_bytes);
+		}
+	}
 	return 0;
 }
 
@@ -389,6 +444,7 @@ int main() {
 	TestFloat12ChannelPortOutputsPcm();
 	TestAsynchronousDevicePushKeepsQueueBounded();
 	TestHandleWithoutPcmDoesNotBypassQueue();
+	TestPcmCopiedBeforeScratchBufferReuse();
 	std::printf("AudioOut2PortTests: all cases passed\n");
 	return 0;
 }

tests/Ngs2SamplerTests.cpp

@@ -0,0 +1,945 @@
+// Exercise sampler controls and rendering without a host audio device.
+#include "libs/ngs2.cpp"
+
+#include <bit>
+#include <cstdio>
+#include <cstdlib>
+#include <numbers>
+
+using namespace Libs::Audio::Ngs2;
+
+namespace {
+
+void Check(bool condition, const char* message) {
+	if (!condition) {
+		std::fprintf(stderr, "Ngs2SamplerTests: %s\n", message);
+		std::abort();
+	}
+}
+
+struct Fixture {
+	Ngs2Internal      system;
+	Ngs2RackInternal  rack {};
+	Ngs2VoiceInternal voice;
+
+	Fixture(uint32_t rate, uint32_t channels = 1, Ngs2RackType type = Ngs2RackType::Sampler) {
+		system.option.sample_rate = 48000;
+		rack.ngs                  = &system;
+		rack.type                 = type;
+		voice.rack                = &rack;
+		struct Setup {
+			Ngs2VoiceParamHeader header {40, 0, 0x10000000};
+			Ngs2WaveformFormat   format;
+			uint32_t             flags = 0, reserved = 0;
+		} setup;
+		setup.header.id = type == Ngs2RackType::Sampler ? 0x10000000 : 0x40010000;
+		setup.format    = {0x12, channels, rate};
+		Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&voice), &setup.header) == OK,
+		      "sampler setup failed");
+		voice.state = Ngs2VoicePlayState::Playing;
+	}
+
+	void Queue(const int16_t* data, uint32_t frames, uint32_t flags, uint32_t repeats = 0,
+	           uint32_t skip = 0) {
+		Ngs2WaveformBlock block {
+		    0,  size_t(frames + skip) * voice.channels * sizeof(int16_t), repeats, skip, frames, 0,
+		    123};
+		struct Blocks {
+			Ngs2VoiceParamHeader     header {32, 0, 0x10000001};
+			const void*              data;
+			uint32_t                 flags, count;
+			const Ngs2WaveformBlock* blocks;
+		} param {{32, 0, 0x10000001}, data, flags, 1, &block};
+		param.header.id = rack.type == Ngs2RackType::Sampler ? 0x10000001 : 0x40010001;
+		Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&voice), &param.header) == OK,
+		      "sampler queue failed");
+	}
+
+	void Render(uint32_t frames) {
+		voice.rendered = false;
+		Ngs2RenderVoice(voice, {}, frames);
+	}
+};
+
+void TestStreamingResample() {
+	Fixture              f(44100, 2);
+	std::vector<int16_t> pcm(441 * 2);
+	for (int i = 0; i < 441; ++i) {
+		pcm[i * 2]     = static_cast<int16_t>(i * 32);
+		pcm[i * 2 + 1] = static_cast<int16_t>(-i * 32);
+	}
+	f.Queue(pcm.data(), 200, 1);
+	f.Queue(pcm.data() + 400, 241, 0);
+	for (uint32_t grain = 0; grain < 3; ++grain) {
+		f.Render(160);
+		for (uint32_t i = 0; i < 160; ++i) {
+			const float expected = std::min((grain * 160 + i) * 44100.0 / 48000.0, 440.0) / 1024.0;
+			Check(std::abs(f.voice.samples[i] - expected) < 0.00001f,
+			      "resampling discontinuity across grain or block boundary");
+			Check(std::abs(f.voice.samples[160 + i] + expected) < 0.00001f,
+			      "stereo channels were mixed");
+		}
+	}
+	Check(f.voice.blocks.empty(), "final streaming block did not finish");
+	Check(f.voice.state == Ngs2VoicePlayState::Empty, "finished voice remained active");
+	Ngs2SamplerVoiceState state {};
+	Ngs2VoiceGetState(reinterpret_cast<uintptr_t>(&f.voice), &state.voice_state, sizeof(state));
+	Check(state.num_decoded_samples == 441 && state.decoded_data_size == pcm.size() * 2,
+	      "streaming progress does not match consumed source data");
+}
+
+void TestAtrac9ResampleAcrossFrames() {
+	const auto make_voice = [] {
+		auto f = std::make_unique<Fixture>(24000);
+		f->voice.SetupSampler({NGS2_WAVEFORM_TYPE_ATRAC9, 1, 24000, 0xfe4005f0});
+		f->voice.state = Ngs2VoicePlayState::Playing;
+		f->voice.decoded_frame[0] = 0.0f;
+		f->voice.decoded_frame[1] = 0.5f;
+		f->voice.frame_samples = 2;
+		f->voice.next_decoded_frame.resize(f->voice.decoded_frame.size());
+		f->voice.next_decoded_frame[0] = 1.0f;
+		f->voice.next_decoded_frame[1] = 1.5f;
+		f->voice.next_frame_samples = 2;
+		f->voice.accepts_blocks = false;
+		return f;
+	};
+	auto whole = make_voice();
+	whole->Render(8);
+	const std::vector<float> expected {0.0f, 0.25f, 0.5f, 0.75f,
+	                                   1.0f, 1.25f, 1.5f, 1.5f};
+	Check(whole->voice.samples == expected && whole->voice.decoded_samples == 4 &&
+	          whole->voice.state == Ngs2VoicePlayState::Empty,
+	      "24 kHz ATRAC9 interpolation or source progress failed at frame boundary");
+	auto split = make_voice();
+	split->Render(3);
+	const auto first = split->voice.samples;
+	split->Render(5);
+	std::vector<float> joined = first;
+	joined.insert(joined.end(), split->voice.samples.begin(), split->voice.samples.end());
+	Check(joined == expected && split->voice.decoded_samples == 4,
+	      "ATRAC9 resampling changed across render grains");
+	auto pitched = make_voice();
+	struct Pitch {
+		Ngs2VoiceParamHeader header {16, 0, 0x10000005};
+		float                ratio    = 2.0f;
+		uint32_t             reserved = 0;
+	} pitch;
+	Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&pitched->voice), &pitch.header) == OK,
+	      "ATRAC9 pitch control failed");
+	pitched->Render(4);
+	Check(pitched->voice.samples == std::vector<float> {0.0f, 0.5f, 1.0f, 1.5f} &&
+	          pitched->voice.decoded_samples == 4,
+	      "ATRAC9 pitch did not use source-rate phase advancement");
+}
+
+void TestAtrac9ResampleAfterStarvation() {
+	Fixture f(24000);
+	f.voice.SetupSampler({NGS2_WAVEFORM_TYPE_ATRAC9, 1, 24000, 0xfe4005f0});
+	f.voice.state            = Ngs2VoicePlayState::Playing;
+	f.voice.accepts_blocks   = true;
+	f.voice.decoded_frame[0] = 0.0f;
+	f.voice.decoded_frame[1] = 0.5f;
+	f.voice.frame_samples    = 2;
+	f.Render(4);
+	Check(f.voice.samples == std::vector<float> {0.0f, 0.25f, 0.5f, 0.5f} &&
+	          f.voice.decoded_samples == 2,
+	      "ATRAC9 resampling lost the final sample before starvation");
+	f.Render(2);
+	Check(!f.voice.has_samples && f.voice.state == Ngs2VoicePlayState::Playing &&
+	          f.voice.decoded_samples == 2,
+	      "starved ATRAC9 stream consumed source samples");
+	f.voice.next_decoded_frame.resize(f.voice.decoded_frame.size());
+	f.voice.next_decoded_frame[0] = 1.0f;
+	f.voice.next_decoded_frame[1] = 1.5f;
+	f.voice.next_frame_samples    = 2;
+	f.Render(4);
+	Check(f.voice.samples == std::vector<float> {1.0f, 1.25f, 1.5f, 1.5f} &&
+	          f.voice.decoded_samples == 4,
+	      "ATRAC9 refill lost source-rate phase");
+}
+
+void TestWaveformBlockGeometry() {
+	const Ngs2WaveformFormat format {NGS2_WAVEFORM_TYPE_ATRAC9, 1, 24000, 0xfe4005f0};
+	Ngs2WaveformBlock block {};
+	Check(Ngs2CalcWaveformBlock(&format, 0, 34534, &block) == OK &&
+	          block.data_offset == 0 && block.data_size == 13056 &&
+	          block.num_skip_samples == 0 && block.num_samples == 34534,
+	      "ATRAC9 block calculation omitted encoded superframes");
+	Check(Ngs2CalcWaveformBlock(&format, 513, 1, &block) == OK &&
+	          block.data_offset == 192 && block.data_size == 192 &&
+	          block.num_skip_samples == 1 && block.num_samples == 1,
+	      "ATRAC9 block calculation lost the source offset");
+	Check(Ngs2CalcWaveformBlock(&format, 0, 0, &block) == OK && block.data_size == 0 &&
+	          block.num_samples == 0,
+	      "empty ATRAC9 block acquired encoded data");
+	Fixture f(24000);
+	const int16_t silence = 0;
+	f.Queue(&silence, 0, 0, UINT32_MAX);
+	f.Render(1);
+	Check(f.voice.blocks.empty() && f.voice.state == Ngs2VoicePlayState::Empty,
+	      "empty repeated block kept the sampler active");
+}
+
+void TestSubmixerSetupAndRouting() {
+	Fixture f(48000, 2);
+	Ngs2RackInternal submixer_rack {};
+	submixer_rack.ngs                         = &f.system;
+	submixer_rack.type                        = Ngs2RackType::Submixer;
+	submixer_rack.option.submixer.max_channels = 8;
+	Ngs2VoiceInternal submixer;
+	submixer.rack = &submixer_rack;
+	submixer.ports.resize(1);
+	submixer.matrices.resize(1);
+	Ngs2RackInternal mastering_rack {};
+	mastering_rack.type = Ngs2RackType::Mastering;
+	Ngs2VoiceInternal mastering;
+	mastering.rack     = &mastering_rack;
+	mastering.channels = 2;
+	mastering.state    = Ngs2VoicePlayState::Playing;
+	submixer.state         = Ngs2VoicePlayState::Playing;
+	submixer.ports[0].dest = &mastering;
+	submixer.matrices[0]  = {1.0f};
+	f.voice.ports.push_back({&submixer, 0, 1.0f, -1});
+	const int16_t pcm[] = {8192, -8192, 16384, -16384};
+	f.Queue(pcm, 2, 0);
+	const std::vector<Ngs2VoiceInternal*> voices {&f.voice, &submixer, &mastering};
+	Ngs2RenderVoice(mastering, voices, 1);
+	Check(!submixer.has_samples && !mastering.has_samples && f.voice.decoded_samples == 0,
+	      "unconfigured zero-channel submixer claimed to produce samples");
+	Ngs2SubmixerVoiceSetupParam setup {{16, 0, 0x20000000}, 2, 0};
+	Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&submixer), &setup.header) == OK &&
+	          submixer.channels == 2 && submixer.state == Ngs2VoicePlayState::Empty &&
+	          submixer.ports[0].dest == nullptr && submixer.matrices[0].empty(),
+	      "submixer setup did not establish channels and clear old routing");
+	submixer.state = Ngs2VoicePlayState::Playing;
+	submixer.ports[0] = {&mastering, 0, 1.0f, 0};
+	submixer.matrices[0] = {1.0f, 0.0f, 0.0f, 1.0f};
+	for (float expected: {0.25f, 0.5f}) {
+		for (auto* voice: voices) {
+			voice->rendered = false;
+		}
+		Ngs2RenderVoice(mastering, voices, 1);
+		Check(mastering.has_samples && mastering.samples[0] == expected &&
+		          mastering.samples[1] == -expected,
+		      "submixer lost routed samples across render grains");
+	}
+}
+
+void TestPcm16RenderOutput() {
+	Ngs2Internal system;
+	system.option                   = Ngs2DefaultSystemOption();
+	system.option.num_grain_samples = 4;
+	Ngs2RackOptionUnion sampler_option {}, mastering_option {};
+	Ngs2FillDefaultRackOption(0x1000, &sampler_option);
+	Ngs2FillDefaultRackOption(0x3000, &mastering_option);
+	sampler_option.common.max_voices = 2;
+	sampler_option.common.max_ports  = 1;
+	alignas(Ngs2RackInternal) alignas(Ngs2VoiceInternal)
+	    std::byte sampler_storage[sizeof(Ngs2RackInternal) + 2 * sizeof(Ngs2VoiceInternal)];
+	alignas(Ngs2RackInternal) alignas(Ngs2VoiceInternal)
+	    std::byte mastering_storage[sizeof(Ngs2RackInternal) + sizeof(Ngs2VoiceInternal)];
+	Ngs2ContextBufferInfo sampler_buffer {sampler_storage, sizeof(sampler_storage)};
+	Ngs2ContextBufferInfo mastering_buffer {mastering_storage, sizeof(mastering_storage)};
+	uintptr_t sampler_handle = 0, mastering_handle = 0;
+	const auto system_handle = reinterpret_cast<uintptr_t>(&system);
+	Check(Ngs2RackCreate(system_handle, 0x1000, &sampler_option.common, &sampler_buffer,
+	                     &sampler_handle) == OK &&
+	          Ngs2RackCreate(system_handle, 0x3000, &mastering_option.common,
+	                         &mastering_buffer, &mastering_handle) == OK,
+	      "PCM16 render rack creation failed");
+	auto* samplers = reinterpret_cast<Ngs2VoiceInternal*>(
+	    reinterpret_cast<Ngs2RackInternal*>(sampler_handle) + 1);
+	auto& mastering = *reinterpret_cast<Ngs2VoiceInternal*>(
+	    reinterpret_cast<Ngs2RackInternal*>(mastering_handle) + 1);
+	mastering.channels = 1;
+	mastering.state    = Ngs2VoicePlayState::Playing;
+	const int16_t source[2][4] = {{16384, -16384, 32767, -32768},
+	                              {8192, -8192, 32767, -32768}};
+	for (uint32_t i = 0; i < 2; ++i) {
+		samplers[i].SetupSampler({0x12, 1, 48000});
+		samplers[i].blocks.push_back(
+		    {reinterpret_cast<const uint8_t*>(source[i]), {0, sizeof(source[i]), 0, 0, 4}});
+		samplers[i].ports[0].dest = &mastering;
+		samplers[i].SetEvent(1);
+	}
+	int16_t output_samples[4] = {};
+	Ngs2RenderBufferInfo output {output_samples, sizeof(output_samples), 0x12, 1};
+	Check(Ngs2SystemRender(system_handle, &output, 1) == OK &&
+	          output_samples[0] == 24576 && output_samples[1] == -24576 &&
+	          output_samples[2] == 32767 && output_samples[3] == -32768,
+	      "PCM16 output failed to mix or saturate mastering voices");
+	Check(Ngs2SystemRender(system_handle, &output, 1) == OK &&
+	          std::ranges::all_of(output_samples, [](int16_t sample) { return sample == 0; }),
+	      "PCM16 output retained samples from the previous render");
+	Check(Ngs2RackDestroy(mastering_handle, nullptr) == OK &&
+	          Ngs2RackDestroy(sampler_handle, nullptr) == OK,
+	      "PCM16 render rack cleanup failed");
+}
+
+void TestPitchLoopAndSkip() {
+	Fixture       f(24000);
+	const int16_t pcm[] = {-30000, 8192, 16384};
+	f.Queue(pcm, 2, 0, 1, 1);
+	struct Pitch {
+		Ngs2VoiceParamHeader header {16, 0, 0x10000005};
+		float                ratio    = 2.0f;
+		uint32_t             reserved = 0;
+	} pitch;
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f.voice), &pitch.header);
+	f.Render(5);
+	const float expected[] = {0.25f, 0.5f, 0.25f, 0.5f, 0.0f};
+	for (size_t i = 0; i < 5; ++i) {
+		Check(std::abs(f.voice.samples[i] - expected[i]) < 0.00001f,
+		      "pitch, repeat count, or skipped samples incorrect");
+	}
+}
+
+void TestPitchClamps() {
+	Fixture       f(48000);
+	const int16_t pcm[] = {8192, 0, 0, 0, -8192};
+	f.Queue(pcm, 5, 0);
+	struct Pitch {
+		Ngs2VoiceParamHeader header {16, 0, 0x10000005};
+		float                ratio    = 5.0f;
+		uint32_t             reserved = 0;
+	} pitch;
+	Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f.voice), &pitch.header) == OK,
+	      "out-of-range pitch was rejected");
+	f.Render(2);
+	Check(f.voice.samples[0] == 0.25f && f.voice.samples[1] == -0.25f,
+	      "pitch was not clamped to four times playback speed");
+}
+
+void TestStarvationAndPause() {
+	Fixture       f(48000);
+	const int16_t pcm[] = {16384, 8192};
+	f.Queue(pcm, 2, 1);
+	f.Render(4);
+	Check(f.voice.state == Ngs2VoicePlayState::Playing, "stream starvation stopped voice");
+	Check(f.voice.samples[2] == 0.0f, "starved output was not silent");
+	f.Queue(pcm, 2, 0);
+	f.voice.SetEvent(16);
+	f.Render(2);
+	Check(!f.voice.has_samples && f.voice.blocks.front().cursor == 0, "paused voice consumed data");
+	f.voice.SetEvent(32);
+	f.Render(2);
+	Check(f.voice.samples[0] == 0.5f && f.voice.samples[1] == 0.25f,
+	      "stream failed to resume after starvation/pause");
+}
+
+void TestMonoRateAndRouting() {
+	Fixture              f(22050);
+	std::vector<int16_t> pcm(441, 16384);
+	f.Queue(pcm.data(), 441, 0);
+	Ngs2RackInternal mastering_rack {};
+	mastering_rack.type = Ngs2RackType::Mastering;
+	Ngs2VoiceInternal mastering;
+	mastering.rack     = &mastering_rack;
+	mastering.channels = 2;
+	mastering.state    = Ngs2VoicePlayState::Playing;
+	f.voice.matrices   = {{0.5f, 0.25f}};
+	f.voice.ports.push_back({&mastering, 0, 1.0f, 0});
+	for (int i = 0; i < 3; ++i) {
+		mastering.rendered = false;
+		f.voice.rendered   = false;
+		Ngs2RenderVoice(mastering, {&f.voice, &mastering}, 320);
+		Check(mastering.has_samples, "standard sampler did not reach mastering voice");
+		for (uint32_t j = 0; j < 320; ++j) {
+			Check(mastering.samples[j] == 0.25f && mastering.samples[320 + j] == 0.125f,
+			      "mono PCM routing matrix was not applied");
+		}
+	}
+	Check(f.voice.decoded_samples == 441 && f.voice.blocks.empty(),
+	      "22.05 kHz effect duration was incorrect");
+}
+
+void TestMatrixOutputChannels() {
+	Fixture f(48000, 2);
+	Ngs2RackInternal mastering_rack {};
+	mastering_rack.type = Ngs2RackType::Mastering;
+	Ngs2VoiceInternal mastering;
+	mastering.rack     = &mastering_rack;
+	mastering.channels = 8;
+	mastering.state    = Ngs2VoicePlayState::Playing;
+	f.voice.matrices   = {{1.0f, 0.0f, 0.0f, 1.0f}};
+	f.voice.ports.push_back({&mastering, 0, 1.0f, 0});
+	const int16_t pcm[] = {8192, -8192};
+	f.Queue(pcm, 1, 0);
+	Ngs2RenderVoice(mastering, {&f.voice, &mastering}, 1);
+	Check(mastering.samples[0] == 0.25f && mastering.samples[1] == -0.25f,
+	      "stereo matrix did not route its output channels");
+	for (size_t channel = 2; channel < mastering.channels; ++channel) {
+		Check(mastering.samples[channel] == 0.0f,
+		      "matrix routed audio to an unspecified output channel");
+	}
+}
+
+void TestCustomPcmStillPlays() {
+	Fixture       f(48000, 1, Ngs2RackType::CustomSampler);
+	const int16_t pcm[] = {8192, -16384};
+	f.Queue(pcm, 2, 0);
+	f.Render(4);
+	Check(f.voice.samples[0] == 0.25f && f.voice.samples[1] == -0.5f &&
+	          f.voice.samples[2] == 0.0f && f.voice.blocks.empty(),
+	      "custom PCM sampler regressed");
+}
+
+void TestSamplerReuseResetsRouting() {
+	for (auto type: {Ngs2RackType::Sampler, Ngs2RackType::CustomSampler}) {
+		Fixture          f(48000, 1, type);
+		Ngs2RackInternal mastering_rack {};
+		mastering_rack.type = Ngs2RackType::Mastering;
+		Ngs2VoiceInternal mastering;
+		mastering.rack     = &mastering_rack;
+		mastering.channels = 2;
+		mastering.state    = Ngs2VoicePlayState::Playing;
+		f.voice.ports.push_back({&mastering, 1, 0.25f, 0});
+		f.voice.matrices = {{0.5f, 0.25f}};
+		f.voice.SetupSampler({0x12, 2, 48000});
+		Check(f.voice.ports.size() == 1 && f.voice.ports[0].dest == nullptr &&
+		          f.voice.ports[0].input == 0 && f.voice.ports[0].volume == 1.0f &&
+		          f.voice.ports[0].matrix == -1 && f.voice.matrices.size() == 1 &&
+		          f.voice.matrices[0].empty(),
+		      "sampler setup retained routing from the previous channel layout");
+		const int16_t pcm[] = {8192, 16384, -8192, -16384};
+		f.Queue(pcm, 2, 0);
+		f.Render(1);
+		Check(!f.voice.has_samples && f.voice.blocks.front().cursor == 0,
+		      "sampler setup started the new waveform without a play event");
+		f.voice.ports[0].dest = &mastering;
+		f.voice.SetEvent(1);
+		f.voice.rendered = false;
+		Ngs2RenderVoice(mastering, {&f.voice, &mastering}, 2);
+		Check(mastering.samples == std::vector<float> {0.25f, -0.25f, 0.5f, -0.5f},
+		      "reused stereo sampler did not restore default channel routing");
+	}
+}
+
+void TestDeferredSamplerSetup() {
+	Fixture f(48000);
+	struct Setup {
+		Ngs2VoiceParamHeader header {40, 0, 0x10000000};
+		Ngs2WaveformFormat   format;
+		uint32_t             flags = 0, reserved = 0;
+	} setup;
+	Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f.voice), &setup.header) == OK,
+	      "deferred stream setup failed");
+	Check(f.voice.state == Ngs2VoicePlayState::Empty && f.voice.channels == 0 &&
+	          f.voice.sample_rate == 0 && f.voice.decoder == nullptr,
+	      "deferred stream setup did not clear the sampler");
+	setup.format = {0x12, 1, 48000};
+	Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f.voice), &setup.header) == OK,
+	      "sampler setup after deferred stream failed");
+	const int16_t pcm = 8192;
+	f.Queue(&pcm, 1, 0);
+	f.voice.SetEvent(1);
+	f.Render(1);
+	Check(f.voice.samples[0] == 0.25f,
+	      "sampler did not play after deferred stream setup");
+}
+
+void TestPlayStateBeforeRender() {
+	Fixture       f(48000);
+	const int16_t pcm[] = {8192, -16384};
+	f.voice.state       = Ngs2VoicePlayState::Empty;
+	f.Queue(pcm, 2, 0);
+	Ngs2VoiceEventParam event {{sizeof(Ngs2VoiceEventParam), 0, 6}, 1};
+	const auto handle = reinterpret_cast<uintptr_t>(&f.voice);
+	Ngs2VoiceControl(handle, &event.header);
+	uint32_t flags = 0;
+	Ngs2VoiceGetStateFlags(handle, &flags);
+	Check(flags == 1, "play must reserve the voice before rendering starts playback");
+	Ngs2SamplerVoiceState state {};
+	Ngs2VoiceGetState(handle, &state.voice_state, sizeof(state));
+	Check(state.voice_state.state_flags == flags && state.num_decoded_samples == 0,
+	      "state queries disagreed or play consumed samples before render");
+	f.Render(4);
+	Check(f.voice.samples[0] == 0.25f && f.voice.samples[1] == -0.5f,
+	      "one-shot effect was dropped before its first render");
+}
+
+void TestStatePublication() {
+	Ngs2Internal system;
+	system.option = Ngs2DefaultSystemOption();
+	Ngs2RackOptionUnion option {};
+	Ngs2FillDefaultRackOption(0x1000, &option);
+	option.common.max_voices = 1;
+	alignas(Ngs2RackInternal) alignas(Ngs2VoiceInternal)
+	    std::byte         storage[sizeof(Ngs2RackInternal) + sizeof(Ngs2VoiceInternal)];
+	Ngs2ContextBufferInfo buffer {storage, sizeof(storage)};
+	uintptr_t             rack_handle   = 0;
+	const auto            system_handle = reinterpret_cast<uintptr_t>(&system);
+	Check(Ngs2RackCreate(system_handle, 0x1000, &option.common, &buffer, &rack_handle) == OK,
+	      "state publication rack creation failed");
+	auto& voice =
+	    *reinterpret_cast<Ngs2VoiceInternal*>(reinterpret_cast<Ngs2RackInternal*>(rack_handle) + 1);
+	voice.SetupSampler({0x12, 1, 48000});
+	const auto handle      = reinterpret_cast<uintptr_t>(&voice);
+	auto       check_flags = [&](uint32_t expected) {
+		uint32_t              flags = 0;
+		Ngs2SamplerVoiceState state {};
+		Ngs2VoiceGetStateFlags(handle, &flags);
+		Ngs2VoiceGetState(handle, &state.voice_state, sizeof(state));
+		Check(flags == expected && state.voice_state.state_flags == expected,
+		      "voice state was published at the wrong render boundary");
+	};
+	struct Transition {
+		uint32_t event, before_render, after_render;
+	};
+	const Transition     transitions[] = {{1, 1, 3}, {16, 3, 5}, {32, 5, 3}, {2, 3, 0},
+	                                      {1, 1, 3}, {16, 3, 5}, {2, 5, 0},  {1, 1, 3},
+	                                      {4, 3, 0}, {1, 1, 3},  {8, 3, 0}};
+	Ngs2RenderBufferInfo output {};
+	for (const auto& transition: transitions) {
+		Ngs2VoiceEventParam event {{sizeof(Ngs2VoiceEventParam), 0, 6}, transition.event};
+		Ngs2VoiceControl(handle, &event.header);
+		check_flags(transition.before_render);
+		Ngs2SystemRender(system_handle, &output, 1);
+		check_flags(transition.after_render);
+	}
+	Check(Ngs2RackDestroy(rack_handle, nullptr) == OK, "state publication rack cleanup failed");
+}
+
+void TestStopThenPlayBeforeRender() {
+	Fixture       f(48000);
+	const int16_t pcm[] = {16384, -8192};
+	f.Queue(pcm, 2, 0);
+	f.voice.SetEvent(2);
+	f.voice.SetEvent(1);
+	Check(Ngs2GetStateFlags(&f.voice) == 1, "stopped voice could not restart");
+	f.voice.SetEvent(8);
+	f.voice.SetEvent(1);
+	Check(Ngs2GetStateFlags(&f.voice) == 1, "kill/play commands lost their ordering");
+	f.Render(2);
+	Check(f.voice.samples[0] == 0.5f && f.voice.samples[1] == -0.25f,
+	      "restarted voice produced no audio");
+}
+
+void SetLowPass(Fixture& f, float frequency = 1000.0f, uint64_t mask = 0) {
+	struct Param {
+		Ngs2VoiceParamHeader   header {56, 0, 0x1000000a};
+		Ngs2SamplerFilterParam filter {0, 1, 1, 0, 1000.0f, 0.70710678f, 1.0f, {}};
+	} param;
+	param.filter.frequency            = frequency;
+	param.filter.channel_mask         = mask;
+	f.rack.option.sampler.max_filters = 8;
+	Check(Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f.voice), &param.header) == OK,
+	      "filter ABI command failed");
+}
+
+void TestLowPassResponse() {
+	// A second-order Butterworth LPF has unity DC gain and -3 dB at cutoff.
+	for (double frequency: {100.0, 1000.0, 10000.0}) {
+		Fixture f(48000);
+		SetLowPass(f);
+		std::vector<int16_t> pcm(48000);
+		for (size_t i = 0; i < pcm.size(); ++i) {
+			pcm[i] =
+			    static_cast<int16_t>(8192 * std::sin((2.0 * std::numbers::pi) * frequency * i / 48000));
+		}
+		f.Queue(pcm.data(), static_cast<uint32_t>(pcm.size()), 0);
+		f.Render(static_cast<uint32_t>(pcm.size()));
+		double input = 0, output = 0;
+		for (size_t i = 24000; i < pcm.size(); ++i) {
+			input += std::pow(pcm[i] / 32768.0, 2);
+			output += std::pow(f.voice.samples[i], 2);
+		}
+		const double ratio = std::sqrt(output / input);
+		if (frequency == 100) {
+			Check(std::abs(ratio - 1.0) < 0.002, "low-pass lost bass");
+		}
+		if (frequency == 1000) {
+			Check(std::abs(ratio - std::sqrt(0.5)) < 0.002, "cutoff gain incorrect");
+		}
+		if (frequency == 10000) {
+			Check(ratio < 0.01, "low-pass did not reject high frequencies");
+		}
+	}
+}
+
+void TestFilterHistoryAndChannels() {
+	Fixture whole(48000, 2), split(48000, 2);
+	SetLowPass(whole, 1000, 1);
+	SetLowPass(split, 1000, 1);
+	std::vector<int16_t> pcm(256 * 2, 0);
+	pcm[0] = pcm[1] = 16384;
+	whole.Queue(pcm.data(), 256, 0);
+	split.Queue(pcm.data(), 32, 1);
+	split.Queue(pcm.data() + 64, 224, 0);
+	whole.Render(256);
+	for (uint32_t g = 0; g < 8; ++g) {
+		// Games resend unchanged parameters; this must not reset the delay line.
+		SetLowPass(split, 1000, 1);
+		split.Render(32);
+		for (uint32_t c = 0; c < 2; ++c) {
+			for (uint32_t i = 0; i < 32; ++i) {
+				Check(std::abs(split.voice.samples[c * 32 + i] -
+				               whole.voice.samples[c * 256 + g * 32 + i]) < 1e-7,
+				      "filter history changed at a grain/block/control boundary");
+			}
+		}
+	}
+	Check(whole.voice.samples[0] == 0.5f && whole.voice.samples[1] == 0.0f &&
+	          whole.voice.samples[256] > 0.0f && whole.voice.samples[256] < 0.5f,
+	      "channel mask must bypass selected channels and filter the others");
+	Fixture stereo(48000, 2);
+	SetLowPass(stereo);
+	pcm[1] = 0;
+	stereo.Queue(pcm.data(), 256, 0);
+	stereo.Render(256);
+	for (size_t i = 256; i < 512; ++i) {
+		Check(stereo.voice.samples[i] == 0.0f, "filter leaked history between channels");
+	}
+	stereo.voice.SetupSampler({0x12, 2, 48000});
+	Check(stereo.voice.filters.empty(), "reused voice retained old filters");
+}
+
+void TestFilterUsesOutputRate() {
+	Fixture f(22050);
+	SetLowPass(f, 6833.0f);
+	const auto& filter = f.voice.filters[0];
+	// Rear filtering operates at the 48 kHz system rate, not the PCM source rate.
+	const double w        = (2.0 * std::numbers::pi) * 6833 / 48000;
+	const double expected = (1 - std::cos(w)) / (2 * (1 + std::sin(w) / (2 * double(0.70710678f))));
+	Check(std::abs(filter.b0 - expected) < 1e-10, "rear filter used source sample rate");
+	SetLowPass(f, 48000);
+	const int16_t pcm[] = {16384, 16384};
+	f.Queue(pcm, 2, 0);
+	f.Render(2);
+	Check(f.voice.samples[0] == 0.5f && f.voice.samples[1] == 0.5f,
+	      "above-Nyquist cutoff was unstable");
+}
+
+void TestFilterControlLayout() {
+	Fixture f(48000, 2);
+	f.rack.option.sampler.max_filters = 8;
+	// A 56-byte control command with nonzero trailing padding.
+	alignas(8) uint32_t words[] = {56, 0x1000000a, 0, 1, 1, 0, 0,
+	                               0,  0x45d58800, 0x3fd55555, 0x3f800000, 0, 0, 0x13f};
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&f.voice),
+	                 reinterpret_cast<const Ngs2VoiceParamHeader*>(words));
+	Check(f.voice.filters.size() == 1 && f.voice.filters[0].enabled,
+	      "FCQ filter command was not recognized");
+	const double omega    = (2.0 * std::numbers::pi) * 6833.0 / 48000.0;
+	const double q        = static_cast<double>(std::bit_cast<float>(0x3fd55555u));
+	const double expected = (1 - std::cos(omega)) / (2 * (1 + std::sin(omega) / (2 * q)));
+	Check(std::abs(f.voice.filters[0].b0 - expected) < 1e-10,
+	      "FCQ fields were read at the wrong offsets");
+	words[6] = 1; // Bypass channel 0 in the low half of the 64-bit mask.
+	words[7] = 1; // Bypass channel 32 in the high half.
+	Fixture wide(48000, 33);
+	wide.rack.option.sampler.max_filters = 8;
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&wide.voice),
+	                 reinterpret_cast<const Ngs2VoiceParamHeader*>(words));
+	const std::vector<int16_t> pcm(33, 16384);
+	wide.Queue(pcm.data(), 1, 1);
+	wide.Render(1);
+	Check(wide.voice.samples[0] == 0.5f && wide.voice.samples[32] == 0.5f &&
+	          std::abs(wide.voice.samples[1] - expected * 0.5) < 1e-7,
+	      "channel mask was not decoded as a 64-bit bypass mask");
+	words[4] = 0; // OFF ignores unused FCQ values and clears the old delay line.
+	words[8] = 0x7fc00000;
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&wide.voice),
+	                 reinterpret_cast<const Ngs2VoiceParamHeader*>(words));
+	Check(!wide.voice.filters[0].enabled && wide.voice.filters[0].history.empty() &&
+	          !wide.voice.filters[0].warned,
+	      "filter OFF was treated as an unsupported FCQ configuration");
+	wide.Queue(pcm.data(), 1, 1);
+	wide.Render(1);
+	Check(wide.voice.samples[1] == 0.5f, "filter OFF did not bypass audio");
+	SetLowPass(wide, 0);
+	wide.Queue(pcm.data(), 1, 1);
+	wide.Render(1);
+	Check(wide.voice.samples[1] == 0.0f, "zero-Hz low-pass leaked source audio");
+}
+
+struct LoopCallbacks {
+	std::vector<std::pair<uint32_t, uint32_t>> events;
+	bool                                       pause = false;
+};
+
+void KYTY_SYSV_ABI RecordLoopCallback(const Ngs2VoiceCallbackInfo* info) {
+	auto& log = *reinterpret_cast<LoopCallbacks*>(info->data);
+	log.events.emplace_back(info->flag, info->repeats);
+	Check(info->user == 123 && info->attributes == 0, "loop callback metadata changed");
+	if (log.pause && info->flag == 2) {
+		reinterpret_cast<Ngs2VoiceInternal*>(info->voice)->SetEvent(16);
+	}
+}
+
+void TestLoopCallbacksAndExit() {
+	const int16_t pcm[] = {8192, 16384, 24576};
+	Fixture       f(48000);
+	LoopCallbacks log;
+	f.voice.callback       = reinterpret_cast<uintptr_t>(&RecordLoopCallback);
+	f.voice.callback_data  = reinterpret_cast<uintptr_t>(&log);
+	f.voice.callback_flags = 3;
+	f.Queue(pcm, 2, 0, 2);
+	f.Render(6);
+	Check(log.events == std::vector<std::pair<uint32_t, uint32_t>> {{2, 1}, {2, 2}, {1, 2}},
+	      "loop and completion callbacks lost the completed repeat count");
+	Check(f.voice.blocks.empty(), "finite loop did not complete");
+
+	Fixture active(48000);
+	log.events.clear();
+	log.pause                   = true;
+	active.voice.callback       = reinterpret_cast<uintptr_t>(&RecordLoopCallback);
+	active.voice.callback_data  = reinterpret_cast<uintptr_t>(&log);
+	active.voice.callback_flags = 2;
+	active.Queue(pcm, 2, 1, UINT32_MAX);
+	active.Queue(pcm + 2, 1, 0);
+	active.Render(6);
+	Check(log.events.size() == 1 && active.voice.decoded_samples == 2 &&
+	          active.voice.blocks.front().cursor == 0 && active.voice.samples[2] == 0,
+	      "loop callback pause consumed another iteration");
+	Ngs2VoiceParamHeader exit_loop {sizeof(Ngs2VoiceParamHeader), 0, 0x10000004};
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&active.voice), &exit_loop);
+	active.voice.SetEvent(32);
+	active.Render(3);
+	Check(active.voice.samples == std::vector<float> {0.25f, 0.5f, 0.75f} &&
+	          active.voice.blocks.empty() && log.events.size() == 1,
+	      "exiting infinite loop lost its final iteration or ignored callback flags");
+
+	Fixture pending(48000);
+	pending.Queue(pcm, 1, 1);
+	pending.Queue(pcm + 1, 1, 1, UINT32_MAX);
+	pending.Queue(pcm + 2, 1, 0);
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&pending.voice), &exit_loop);
+	pending.Render(3);
+	Check(pending.voice.samples == std::vector<float> {0.25f, 0.5f, 0.75f} &&
+	          pending.voice.blocks.empty(),
+	      "exiting a pending loop skipped its first iteration");
+	Ngs2VoiceControl(reinterpret_cast<uintptr_t>(&pending.voice), &exit_loop);
+}
+
+void TestWaveformReadAddress() {
+	Fixture       f(48000, 2);
+	const int16_t pcm[] = {-100, -100, 8192, 4096, 16384, 8192};
+	f.Queue(pcm, 2, 1, 1, 1);
+	const auto next_data = [&]() {
+		Ngs2SamplerVoiceState state {};
+		Ngs2VoiceGetState(reinterpret_cast<uintptr_t>(&f.voice), &state.voice_state, sizeof(state));
+		return state.waveform_data;
+	};
+	Check(next_data() == pcm + 2, "read address ignored skipped source samples");
+	f.Render(1);
+	Check(next_data() == pcm + 4, "read address did not advance by a stereo frame");
+	f.Render(1);
+	Check(next_data() == pcm + 2, "read address did not wrap at a loop boundary");
+	f.Render(2);
+	Check(f.voice.blocks.empty() && next_data() == pcm + 6,
+	      "starved read address did not preserve the consumed waveform end");
+	f.Queue(pcm + 2, 2, 0);
+	Check(next_data() == pcm + 2, "refilled read address retained the previous waveform end");
+	f.Render(2);
+	Check(next_data() == pcm + 6, "completed read address did not preserve waveform end");
+	f.voice.SetupSampler({0x12, 2, 48000});
+	Check(next_data() == nullptr, "setup retained an old waveform address");
+}
+
+struct CallbackEvent {
+	Ngs2VoiceInternal* voice;
+	uint32_t           event;
+	uint32_t           calls = 0;
+};
+
+void KYTY_SYSV_ABI ApplyCallbackEvent(const Ngs2VoiceCallbackInfo* info) {
+	auto& context = *reinterpret_cast<CallbackEvent*>(info->data);
+	if (++context.calls == 1) {
+		context.voice->SetEvent(context.event);
+	}
+}
+
+void TestCallbackStopsConsumption() {
+	for (auto rack: {Ngs2RackType::Sampler, Ngs2RackType::CustomSampler}) {
+		for (uint32_t event: {2u, 4u, 8u, 16u}) {
+			Fixture       f(48000, 1, rack);
+			const int16_t pcm[] = {8192, 16384};
+			f.Queue(pcm, 1, 1);
+			f.Queue(pcm + 1, 1, 1);
+			CallbackEvent context {&f.voice, event};
+			f.voice.callback       = reinterpret_cast<uintptr_t>(&ApplyCallbackEvent);
+			f.voice.callback_data  = reinterpret_cast<uintptr_t>(&context);
+			f.voice.callback_flags = 1;
+			f.Render(2);
+			Check(f.voice.state != Ngs2VoicePlayState::Playing && context.calls == 1,
+			      "completion callback failed to interrupt playback");
+			Check(f.voice.samples[0] == 0.25f && f.voice.samples[1] == 0.0f &&
+			          f.voice.decoded_samples == 1 && f.voice.blocks.size() == 1 &&
+			          f.voice.blocks.front().cursor == 0,
+			      "callback pause/stop consumed the next queued block");
+			if (event == 16) {
+				f.voice.SetEvent(32);
+				f.Render(1);
+				Check(f.voice.samples[0] == 0.5f && f.voice.decoded_samples == 2,
+				      "callback pause failed to preserve queued audio for resume");
+			}
+		}
+	}
+}
+
+void TestCallbackPauseDuringLargeStep() {
+	Fixture       f(48000);
+	const int16_t pcm[] = {8192, 16384, 24576, 4096};
+	f.Queue(pcm, 1, 1);
+	f.Queue(pcm + 1, 1, 1);
+	f.Queue(pcm + 2, 2, 1);
+	f.voice.sample_step = uint64_t(120000) << 32u;
+	CallbackEvent context {&f.voice, 16};
+	f.voice.callback       = reinterpret_cast<uintptr_t>(&ApplyCallbackEvent);
+	f.voice.callback_data  = reinterpret_cast<uintptr_t>(&context);
+	f.voice.callback_flags = 1;
+	f.Render(2);
+	Check(context.calls == 1 && f.voice.decoded_samples == 1 &&
+	          f.voice.sample_phase == (uint64_t(72000) << 32u),
+	      "pause advanced across additional blocks in a large resampling step");
+	f.voice.SetEvent(32);
+	f.Render(1);
+	Check(std::abs(f.voice.samples[0] - 0.4375f) < 1e-7f && context.calls == 3,
+	      "resume lost fractional position or read past the next short block");
+}
+
+void TestUpstreamCustomSamplerControls() {
+	Fixture f(24000, 1, Ngs2RackType::CustomSampler);
+	const int16_t pcm[] = {8192, 16384, 24576, 4096};
+	f.Queue(pcm, 4, 1);
+	f.Render(1);
+	Check(f.voice.samples[0] == 0.25f, "custom sampler source rate was rejected");
+	f.Queue(pcm + 2, 2, 4);
+	f.Render(1);
+	Check(f.voice.samples[0] == 0.75f && f.voice.blocks.size() == 1,
+	      "replacement blocks retained old data or fractional position");
+	struct Pitch {
+		Ngs2VoiceParamHeader header {16, 0, 0x40010005};
+		float                ratio = 0.0f;
+		uint32_t             reserved = 0;
+	} pitch;
+	const auto handle = reinterpret_cast<uintptr_t>(&f.voice);
+	Ngs2VoiceControl(handle, &pitch.header);
+	f.Render(2);
+	Check(f.voice.samples[0] == 0.4375f && f.voice.samples[1] == 0.4375f,
+	      "zero custom sampler pitch advanced playback");
+	pitch.ratio = 4.0f;
+	Ngs2VoiceControl(handle, &pitch.header);
+	f.Render(1);
+	Check(f.voice.blocks.empty(), "custom sampler pitch did not advance playback");
+	f.rack.option.custom_sampler.custom_rack_option.state_size =
+	    sizeof(Ngs2CustomSamplerVoiceState);
+	Ngs2CustomSamplerVoiceState state {};
+	Check(Ngs2VoiceGetState(handle, &state.voice_state, sizeof(state)) == OK &&
+	          state.voice_state.state_flags == Ngs2GetStateFlags(&f.voice),
+	      "custom sampler state layout regressed");
+}
+
+void TestFilterTailDuringStarvation() {
+	Fixture reference(48000), stream(48000);
+	SetLowPass(reference);
+	SetLowPass(stream);
+	std::vector<int16_t> pcm(128, 0);
+	pcm[0] = pcm[64] = 16384;
+	reference.Queue(pcm.data(), 128, 1);
+	reference.Render(128);
+	stream.Queue(pcm.data(), 1, 1);
+	stream.Render(1);
+	Check(stream.voice.samples[0] == reference.voice.samples[0], "initial filter sample differs");
+	stream.Render(63);
+	Check(stream.voice.has_samples, "starved filter tail was not available for routing");
+	for (size_t i = 0; i < 63; ++i) {
+		Check(std::abs(stream.voice.samples[i] - reference.voice.samples[i + 1]) < 1e-7f,
+		      "filter tail changed at a starvation/grain boundary");
+	}
+	stream.Queue(pcm.data() + 64, 64, 1);
+	stream.Render(64);
+	for (size_t i = 0; i < 64; ++i) {
+		Check(std::abs(stream.voice.samples[i] - reference.voice.samples[i + 64]) < 1e-7f,
+		      "refilled stream reused stale filter history");
+	}
+	stream.Render(8192);
+	stream.Render(32);
+	Check(!stream.voice.has_samples && !stream.voice.filters[0].HasHistory(),
+	      "fully decayed filter kept a silent stream active");
+}
+
+void TestResamplingPhaseAfterStarvation() {
+	for (auto rack: {Ngs2RackType::Sampler, Ngs2RackType::CustomSampler}) {
+		for (uint32_t rate: {44100u, 96000u}) {
+			Fixture        reference(rate, 1, rack), stream(rate, 1, rack);
+			const int16_t  pcm[]  = {0, 0, 16384, 8192};
+			const uint32_t frames = rate == 44100 ? 2 : 1;
+			reference.Queue(pcm, 4, 0);
+			stream.Queue(pcm, 1, 1);
+			reference.Render(frames);
+			stream.Render(frames);
+			Check(stream.voice.blocks.empty() && stream.voice.state == Ngs2VoicePlayState::Playing,
+			      "open stream did not enter starvation");
+			stream.Render(8);
+			Check(!stream.voice.has_samples, "starved stream produced audio without a filter");
+			stream.Queue(pcm + 1, 3, 0);
+			reference.Render(1);
+			stream.Render(1);
+			Check(std::abs(stream.voice.samples[0] - reference.voice.samples[0]) < 1e-7f,
+			      "stream refill lost resampling phase");
+			Check(stream.voice.decoded_samples == reference.voice.decoded_samples,
+			      "stream refill lost pending source advancement");
+			stream.voice.SetupSampler({0x12, 1, rate});
+			Check(stream.voice.sample_phase == 0, "sampler setup retained old resampling phase");
+		}
+	}
+}
+
+void TestFiniteFilterTail() {
+	Fixture whole(48000), split(48000);
+	SetLowPass(whole);
+	SetLowPass(split);
+	const int16_t impulse = 16384;
+	whole.Queue(&impulse, 1, 0);
+	split.Queue(&impulse, 1, 0);
+	whole.Render(256);
+	for (size_t i = 0; i < whole.voice.samples.size(); ++i) {
+		split.Render(1);
+		Check(std::abs(split.voice.samples[0] - whole.voice.samples[i]) < 1e-7f,
+		      "finite filter tail changed with render grain size");
+	}
+	Check(split.voice.state == Ngs2VoicePlayState::Playing,
+	      "finite voice stopped before its filter tail decayed");
+	split.Render(8192);
+	Check(split.voice.state == Ngs2VoicePlayState::Empty,
+	      "finite voice remained active after its filter tail decayed");
+	split.Render(32);
+	Check(!split.voice.has_samples, "completed finite filter tail still produced samples");
+}
+
+} // namespace
+
+int main() {
+	TestPcm16RenderOutput();
+	TestSubmixerSetupAndRouting();
+	TestWaveformBlockGeometry();
+	TestAtrac9ResampleAfterStarvation();
+	TestAtrac9ResampleAcrossFrames();
+	TestResamplingPhaseAfterStarvation();
+	TestSamplerReuseResetsRouting();
+	TestDeferredSamplerSetup();
+	TestLoopCallbacksAndExit();
+	TestWaveformReadAddress();
+	TestFilterTailDuringStarvation();
+	TestFiniteFilterTail();
+	TestLowPassResponse();
+	TestFilterHistoryAndChannels();
+	TestFilterUsesOutputRate();
+	TestFilterControlLayout();
+	TestUpstreamCustomSamplerControls();
+	TestCallbackStopsConsumption();
+	TestCallbackPauseDuringLargeStep();
+	TestStreamingResample();
+	TestPitchLoopAndSkip();
+	TestPitchClamps();
+	TestStarvationAndPause();
+	TestMonoRateAndRouting();
+	TestMatrixOutputChannels();
+	TestCustomPcmStillPlays();
+	TestPlayStateBeforeRender();
+	TestStatePublication();
+	TestStopThenPlayBeforeRender();
+	std::puts("Ngs2SamplerTests: all cases passed");
+}

tests/ResourceTrackingTests.cpp

@@ -614,7 +614,8 @@ void TestGuardedDirectImageTable() {
     const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
     Check(indirect && indirect->material_source == UINT32_MAX &&
               indirect->selector_stride == 0u && indirect->table_offset == 344u &&
-              indirect->key_count == 32u &&
+              indirect->key_count.Resolve().IsImmediate() &&
+              indirect->key_count.Resolve().U32() == 32u &&
               fixture.program.descriptor_sources[indirect->table_source].dword_count == 2u,
           "guarded direct image table lost its pointer or proven selector range");
     return ExtractResourcePlan(fixture.program);
@@ -699,6 +700,390 @@ void TestGuardedDirectImageTable() {
         "batched descriptor read crossed the 48-bit endpoint");
 }
 
+void TestBoundedComputeImageLoop() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  enum class Variant {
+    Bounded, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep
+  };
+  const auto make_plan = [](Variant variant) {
+    Fixture fixture;
+    auto *entry = fixture.block;
+    auto *header = fixture.AddBlock();
+    auto *body = fixture.AddBlock();
+    auto *latch = fixture.AddBlock();
+    auto *exit = fixture.AddBlock();
+    entry->AddBranch(header);
+    header->AddBranch(exit);
+    header->AddBranch(body);
+    body->AddBranch(latch);
+    latch->AddBranch(header);
+    if (variant == Variant::EntryBypass) entry->AddBranch(body);
+    if (variant == Variant::ExitBypass) exit->AddBranch(body);
+    fixture.program.block_info[0].terminator = {
+        .kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
+                                               : CFG::TerminatorKind::Branch,
+        .true_block = 1u, .false_block = 2u};
+    fixture.program.block_info[1].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 4u, .false_block = 2u};
+    fixture.program.block_info[2].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 3u};
+    fixture.program.block_info[3].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+    fixture.program.block_info[4].terminator = {
+        .kind = variant == Variant::ExitBypass ? CFG::TerminatorKind::Branch
+                                              : CFG::TerminatorKind::Return,
+        .true_block = 2u};
+
+    auto &phi = header->AppendNewInst(ValueOpcode::Phi, {},
+                                      static_cast<uint64_t>(Type::U32));
+    const auto key = Value(&phi);
+    const auto count = fixture.UserData(2);
+    const auto in_range = fixture.Emit(ValueOpcode::SLessThan32,
+                                       {variant == Variant::WrongGuard
+                                            ? Value(0u) : key,
+                                        count}, 0, header);
+    const auto allowed = fixture.Emit(ValueOpcode::LogicalAnd,
+                                      {in_range, Value(true)}, 0, header);
+    fixture.program.block_info[1].condition =
+        fixture.Emit(ValueOpcode::LogicalNot, {allowed}, 0, header);
+    const auto step = fixture.Emit(ValueOpcode::IAdd32,
+                                   {key, Value(variant == Variant::WrongStep ? 2u : 1u)},
+                                   0, latch);
+    phi.AddPhiOperand(entry, Value(0u));
+    phi.AddPhiOperand(latch, step);
+
+    fixture.block = variant == Variant::GuardBlock ? header : body;
+    const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1));
+    const auto offset = fixture.Emit(ValueOpcode::IAdd32,
+        {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {key, Value(5u)}),
+         Value(0x6b0u)});
+    std::array<Value, 8> words;
+    for (uint32_t word = 0; word < words.size(); ++word) {
+      MemoryInfo memory;
+      memory.kind = ResourceKind::ScalarAddress;
+      memory.offset = word * sizeof(uint32_t);
+      words[word] = fixture.Emit(
+          ValueOpcode::LoadAddressU32,
+          {table, offset, Value(0u), Value(true)},
+          fixture.AddMemory(memory, 0x29c));
+    }
+    const auto image = fixture.Image(words, 0x29c);
+    const auto sampler = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)});
+    MemoryInfo sample;
+    sample.kind = ResourceKind::Image;
+    sample.image_dimension = Decoder::ImageDimension::Dim2D;
+    fixture.Emit(ValueOpcode::ImageSampleRaw,
+                 {image, sampler, fixture.ImageAddress()},
+                 fixture.AddMemory(sample, 0x29c));
+    fixture.PlanAndTrack();
+    const auto source = fixture.program.info.images[0].source;
+    const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
+    Check(indirect && indirect->material_source == UINT32_MAX &&
+              indirect->table_offset == 0x6b0u &&
+              indirect->key_count.Resolve() == count.Resolve(),
+          "bounded compute loop lost its runtime image count");
+    return ExtractResourcePlan(fixture.program);
+  };
+
+  auto plan = make_plan(Variant::Bounded);
+  CheckFatal([&] { make_plan(Variant::WrongGuard); },
+             "not a valid runtime value",
+             "compute image loop accepted an unrelated guard");
+  CheckFatal([&] { make_plan(Variant::EntryBypass); },
+             "not a valid runtime value",
+             "compute image loop accepted an entry bypass");
+  CheckFatal([&] { make_plan(Variant::ExitBypass); },
+             "not a valid runtime value",
+             "compute image loop accepted an exit bypass");
+  CheckFatal([&] { make_plan(Variant::GuardBlock); },
+             "not a valid runtime value",
+             "compute image loop accepted a descriptor read before the guard");
+  CheckFatal([&] { make_plan(Variant::WrongStep); },
+             "not a valid runtime value",
+             "compute image loop accepted a two-step induction");
+
+  LinearTestMemory memory;
+  const auto table = 0x1800u + 0x6b0u;
+  for (uint32_t key = 0; key < 3u; ++key) {
+    const auto word = (table - memory.base) / 4u + key * 8u;
+    memory.words[word] = 0x100u + key;
+    memory.words[word + 1u] = static_cast<uint32_t>(
+        Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
+    memory.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) |
+        (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
+  }
+  std::array<uint32_t, 3> user_data{0x1800u, 0u, 2u};
+  SrtRuntime runtime{.user_data = user_data,
+                     .read_memory = ReadLinearTestMemory,
+                     .userdata = &memory,
+                     .read_specialization_memory = ReadLinearTestMemory};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  for (const uint32_t count : {2u, 3u, 2u}) {
+    user_data[2] = count;
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.images.size() == count &&
+              specialization.images.size() == count &&
+              snapshot.flattened_srt[
+                  specialization.images[0].indirect_mapping_offset] == count &&
+              snapshot.images.back().dwords[0] == 0x100u + count - 1u,
+          "compute image table did not refresh for a changed loop bound");
+  }
+  for (const uint32_t count : {0u, UINT32_MAX}) {
+    user_data[2] = count;
+    Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+              snapshot.images.size() == 1u &&
+              specialization.images.size() == 1u &&
+              std::ranges::all_of(snapshot.images[0].dwords,
+                                  [](uint32_t word) { return word == 0u; }) &&
+              specialization.images[0].indirect_root ==
+                  ImageResource::NoIndirectImage &&
+              specialization.images[0].indirect_search_iterations == 0u,
+          "empty compute loop bound retained unreachable image candidates");
+  }
+  user_data[2] = 65537u;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
+        "oversized compute loop bound was accepted for image enumeration");
+}
+
+void TestUniformizedMaterialImageKeys() {
+  namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
+  const auto make_plan = [](bool wrong_update, bool wrong_equality) {
+    Fixture fixture;
+    auto *entry = fixture.block;
+    auto *header = fixture.AddBlock();
+    auto *inactive = fixture.AddBlock();
+    auto *sentinel = fixture.AddBlock();
+    auto *bit = fixture.AddBlock();
+    auto *merge = fixture.AddBlock();
+    auto *choose = fixture.AddBlock();
+    auto *sample = fixture.AddBlock();
+    auto *done = fixture.AddBlock();
+    entry->AddBranch(header);
+    header->AddBranch(inactive);
+    inactive->AddBranch(merge);
+    inactive->AddBranch(sentinel);
+    sentinel->AddBranch(merge);
+    sentinel->AddBranch(bit);
+    bit->AddBranch(header);
+    bit->AddBranch(merge);
+    merge->AddBranch(choose);
+    choose->AddBranch(sample);
+    choose->AddBranch(done);
+    sample->AddBranch(done);
+    fixture.program.block_info[0].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 1u};
+    fixture.program.block_info[1].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 2u};
+    const auto active_on_entry = fixture.Emit(
+        ValueOpcode::INotEqual32, {fixture.UserData(5u), Value(0u)}, 0, entry);
+    auto &active_phi = header->AppendNewInst(
+        ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
+    auto &mask_phi = header->AppendNewInst(
+        ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
+    const auto mask = Value(&mask_phi);
+    const auto active = Value(&active_phi);
+    fixture.program.block_info[2].condition = fixture.Emit(
+        ValueOpcode::LogicalNot, {active}, 0, inactive);
+    fixture.program.block_info[2].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 5u, .false_block = 3u};
+    const auto nonzero = fixture.Emit(
+        ValueOpcode::INotEqual32, {Value(0u), mask}, 0, sentinel);
+    const auto bit_guard = fixture.Emit(
+        ValueOpcode::LogicalAnd, {active, nonzero}, 0, sentinel);
+    fixture.program.block_info[3].condition = fixture.Emit(
+        ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel);
+    fixture.program.block_info[3].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 5u, .false_block = 4u};
+    const auto first = fixture.Emit(ValueOpcode::FindILsb32, {mask}, 0, bit);
+    const auto position = fixture.Emit(
+        ValueOpcode::BitwiseAnd32, {first, Value(31u)}, 0, bit);
+    const auto one_bit = fixture.Emit(
+        ValueOpcode::ShiftLeftLogical32, {Value(1u), position}, 0, bit);
+    const auto cleared = fixture.Emit(
+        wrong_update ? ValueOpcode::BitwiseOr32 : ValueOpcode::BitwiseXor32,
+        {mask, one_bit}, 0, bit);
+    const auto continuation = fixture.Emit(
+        ValueOpcode::LogicalAnd, {bit_guard, active_on_entry}, 0, bit);
+    fixture.program.block_info[4].condition = continuation;
+    fixture.program.block_info[4].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 1u, .false_block = 5u};
+    active_phi.AddPhiOperand(entry, active_on_entry);
+    active_phi.AddPhiOperand(bit, continuation);
+    mask_phi.AddPhiOperand(entry, fixture.UserData(4u));
+    mask_phi.AddPhiOperand(bit, cleared);
+    const auto arbitrary = fixture.UserData(6u);
+    const auto sentinel_index = fixture.Emit(
+        ValueOpcode::SelectU32, {active, Value(32u), arbitrary}, 0, sentinel);
+    const auto bit_index = fixture.Emit(
+        ValueOpcode::SelectU32, {bit_guard, first, sentinel_index}, 0, bit);
+    auto &index_phi = merge->AppendNewInst(
+        ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
+    index_phi.AddPhiOperand(inactive, arbitrary);
+    index_phi.AddPhiOperand(sentinel, sentinel_index);
+    index_phi.AddPhiOperand(bit, bit_index);
+    const auto index = Value(&index_phi);
+    const auto below = fixture.Emit(
+        ValueOpcode::SGreaterThan32, {Value(32u), index}, 0, merge);
+    const auto material_guard = fixture.Emit(
+        ValueOpcode::LogicalAnd, {active_on_entry, below}, 0, merge);
+    fixture.program.block_info[5].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 6u};
+    const auto scaled = fixture.Emit(
+        ValueOpcode::ShiftLeftLogical32, {index, Value(4u)}, 0, choose);
+    const auto selected_scale = fixture.Emit(
+        ValueOpcode::SelectU32, {material_guard, scaled, Value(0u)}, 0, choose);
+    const auto times_eight = fixture.Emit(
+        ValueOpcode::ShiftLeftLogical32, {selected_scale, Value(3u)}, 0, choose);
+    const auto times_nine = fixture.Emit(
+        ValueOpcode::IAdd32, {times_eight, selected_scale}, 0, choose);
+    const auto material_offset = fixture.Emit(
+        ValueOpcode::SelectU32,
+        {material_guard,
+         fixture.Emit(ValueOpcode::IAdd32,
+                      {times_nine, Value(0xc00u)}, 0, choose),
+         times_nine}, 0, choose);
+    const auto base = fixture.Address(fixture.UserData(0u), fixture.UserData(1u));
+    MemoryInfo material_memory;
+    material_memory.kind = ResourceKind::Global;
+    const auto loaded = fixture.Emit(
+        ValueOpcode::LoadAddressU32,
+        {base, material_offset, Value(0u), material_guard},
+        fixture.AddMemory(material_memory, 0x1a88u), choose);
+    const auto local = fixture.Emit(
+        ValueOpcode::SelectU32, {material_guard, loaded, arbitrary}, 0, choose);
+    const auto key = fixture.Emit(
+        ValueOpcode::ReadLane, {local, Value(0u)}, 0, choose);
+    const auto compared = fixture.Emit(
+        ValueOpcode::IEqual32,
+        {key, wrong_equality ? arbitrary : local}, 0, choose);
+    const auto sample_guard = fixture.Emit(
+        ValueOpcode::LogicalAnd, {material_guard, compared}, 0, choose);
+    fixture.program.block_info[6].condition = fixture.Emit(
+        ValueOpcode::LogicalNot, {sample_guard}, 0, choose);
+    fixture.program.block_info[6].terminator = {
+        .kind = CFG::TerminatorKind::ConditionalBranch,
+        .true_block = 8u, .false_block = 7u};
+    const auto table_offset = fixture.Emit(
+        ValueOpcode::IAdd32,
+        {fixture.Emit(ValueOpcode::ShiftLeftLogical32,
+                      {key, Value(5u)}, 0, sample), Value(0x20e0u)}, 0, sample);
+    std::array<Value, 8> words;
+    for (uint32_t word = 0; word < words.size(); ++word) {
+      MemoryInfo memory;
+      memory.kind = ResourceKind::ScalarAddress;
+      memory.offset = word * 4u;
+      words[word] = fixture.Emit(
+          ValueOpcode::LoadAddressU32,
+          {base, table_offset, Value(0u), Value(true)},
+          fixture.AddMemory(memory, 0x1accu), sample);
+    }
+    const auto image = fixture.Emit(
+        ValueOpcode::GetImageResource,
+        {words[0], words[1], words[2], words[3],
+         words[4], words[5], words[6], words[7]}, 0, sample);
+    const auto sampler = fixture.Emit(
+        ValueOpcode::GetSamplerResource,
+        {Value(0u), Value(0u), Value(0u), Value(0u)}, 0, sample);
+    const auto address = fixture.Emit(
+        ValueOpcode::MakeImageAddress,
+        {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
+         Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u),
+         Value(0u)}, 0, sample);
+    MemoryInfo image_memory;
+    image_memory.kind = ResourceKind::Image;
+    image_memory.image_dimension = Decoder::ImageDimension::Dim2D;
+    fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
+                 fixture.AddMemory(image_memory, 0x1aecu), sample);
+    fixture.program.block_info[7].terminator = {
+        .kind = CFG::TerminatorKind::Branch, .true_block = 8u};
+    fixture.program.block_info[8].terminator.kind = CFG::TerminatorKind::Return;
+    const auto output = fixture.Emit(
+        ValueOpcode::GetBufferResource,
+        {fixture.UserData(8u), fixture.UserData(9u),
+         fixture.UserData(10u), fixture.UserData(11u)}, 0, done);
+    MemoryInfo output_memory;
+    output_memory.kind = ResourceKind::Buffer;
+    fixture.Emit(ValueOpcode::StoreBufferU32,
+                 {output, Value(0u), Value(0u), Value(0u),
+                  Value(1u), Value(true)},
+                 fixture.AddMemory(output_memory, 0x1b00u), done);
+    fixture.PlanAndTrack();
+    const auto source = fixture.program.info.images[0].source;
+    const auto &indirect = fixture.program.descriptor_sources[source].indirect_image;
+    Check(indirect && indirect->selector_stride == 0x90u &&
+              indirect->selector_offset == 0xc00u &&
+              indirect->table_offset == 0x20e0u &&
+              !indirect->selector_mask.IsEmpty() &&
+              indirect->key_count.Resolve().IsImmediate() &&
+              indirect->key_count.Resolve().U32() == 32u,
+          "uniformized image key lost its finite material range");
+    return ExtractResourcePlan(fixture.program);
+  };
+  auto plan = make_plan(false, false);
+  Check(plan.requires_specialization_memory &&
+            plan.descriptor_sources[plan.info.images[0].source]
+                .indirect_image->selector_mask.Resolve().TryInstruction() != nullptr,
+        "uniformized image mask did not survive extraction");
+  LinearTestMemory memory;
+  memory.words.resize(0x23000u / 4u);
+  constexpr uint64_t base = 0x1000u;
+  constexpr uint64_t first_material = base + 0xc00u + 2u * 0x90u;
+  constexpr uint64_t second_material = base + 0xc00u + 29u * 0x90u;
+  constexpr uint64_t first_table = base + 0x20e0u + 7u * 32u;
+  constexpr uint64_t second_table = base + 0x20e0u + 4096u * 32u;
+  memory.words[(first_material - base) / 4u] = 7u;
+  memory.words[(second_material - base) / 4u] = 4096u;
+  const auto set_descriptor = [&](uint64_t address, uint32_t color) {
+    const auto word = (address - base) / 4u;
+    memory.words[word] = color;
+    memory.words[word + 1u] = static_cast<uint32_t>(
+        Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
+    memory.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) |
+        (static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
+  };
+  set_descriptor(first_table, 0x200u);
+  set_descriptor(second_table, 0x400u);
+  std::array<uint32_t, 12> user_data{};
+  user_data[0] = base;
+  user_data[4] = (1u << 2u) | (1u << 29u);
+  user_data[5] = 1u;
+  user_data[8] = 0x400000u;
+  user_data[9] = 16u << 16u;
+  user_data[10] = 1u;
+  memory.fail_address = base + 0xc00u + 3u * 0x90u;
+  const SrtRuntime runtime{
+      .user_data = user_data, .read_memory = ReadLinearTestMemory,
+      .userdata = &memory,
+      .read_specialization_memory = ReadLinearTestMemory};
+  ResourceSnapshot snapshot;
+  ResourceSpecialization specialization;
+  Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
+            snapshot.images.size() == 2u &&
+            memory.reads == 4u && memory.descriptor_reads == 2u &&
+            snapshot.flattened_srt[
+                specialization.images[0].indirect_mapping_offset] == 2u &&
+            snapshot.flattened_srt[
+                specialization.images[0].indirect_mapping_offset + 1u] == 7u &&
+            snapshot.flattened_srt[
+                specialization.images[0].indirect_mapping_offset + 3u] == 4096u,
+        "material mask did not limit sparse descriptor reads");
+  user_data[8] = first_material;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
+        "written buffer alias with a material key was accepted");
+  user_data[8] = first_table;
+  Check(!MaterializeResources(plan, runtime, snapshot, specialization),
+        "written buffer alias with an image record was accepted");
+  CheckFatal([&] { make_plan(true, false); }, "not a valid runtime value",
+             "non-clearing material mask was accepted");
+  CheckFatal([&] { make_plan(false, true); }, "not a valid runtime value",
+             "unrelated ReadLane key was accepted");
+}
+
 void TestImageDescriptorFields() {
   constexpr std::array<std::pair<uint32_t, uint32_t>, 5> reserved{
       {{1u, 0x20000000u}, {2u, 0x70003000u}, {4u, 0xe000e000u},
@@ -2430,6 +2815,31 @@ void TestGraphicsPushConstantLayout() {
 }
 
 void TestResourceLimitIsTransactional() {
+  Fixture accepted;
+  MemoryInfo accepted_memory;
+  accepted_memory.kind = ResourceKind::Buffer;
+  for (uint32_t index = 0; index < ShaderInfo::MaxBuffers; index++) {
+    const auto handle = accepted.Buffer(
+        {Value(index), Value(index + 1u), Value(index + 2u), Value(index + 3u)},
+        index * 4u);
+    accepted.Emit(ValueOpcode::LoadBufferU32,
+                  {handle, Value(0u), Value(0u), Value(0u), Value(true)},
+                  accepted.AddMemory(accepted_memory, index * 4u));
+  }
+  accepted.PlanAndTrack();
+  Check(accepted.program.info.buffers.size() == 64u &&
+            accepted.program.descriptor_sources.size() == 64u &&
+            accepted.program.memory_info.back().resource == 63u,
+        "compute shader did not retain all 64 distinct buffers");
+  ShaderComputeInputInfo compute{};
+  CollectShaderInfo(accepted.program, {.compute = &compute});
+  AllocateBindings(accepted.program);
+  const auto *binding = FindBinding(accepted.program.bindings,
+                                    DescriptorBindingKind::Buffers);
+  Check(binding != nullptr && binding->resources.size() == 64u &&
+            accepted.program.bindings.memory_offset_count == 64u,
+        "compute shader binding layout truncated the 64 buffers");
+
   Fixture fixture;
   MemoryInfo memory;
   memory.kind = ResourceKind::Buffer;
@@ -2506,6 +2916,8 @@ int main() {
     Run("dynamic storage mips", TestDynamicStorageMipTracking);
     Run("invariant indirect images", TestInvariantIndirectImageMaterialization);
     Run("guarded direct image table", TestGuardedDirectImageTable);
+    Run("bounded compute image loop", TestBoundedComputeImageLoop);
+    Run("uniformized material image keys", TestUniformizedMaterialImageKeys);
     Run("image descriptor fields", TestImageDescriptorFields);
     Run("draw-uniform scalar image", TestUniformScalarBufferImage);
     Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization);

tests/SaveDataMemoryTests.cpp

@@ -27,8 +27,8 @@ bool SystemContentParamSfoGetString(const char*, std::string* value) {
 }
 void SymbolDatabase::Add(const SymbolResolve&, uint64_t, const std::string&) {}
 namespace Timer {
-Common::Time GetTime() {
-	return Common::Time(0);
+double GetTimeMs() {
+	return 0.0;
 }
 } // namespace Timer
 } // namespace Loader

tests/ShaderRayTracingTests.inc

@@ -0,0 +1,87 @@
+void TestRayTracingDispatchDetection() {
+  namespace D = ShaderRecompiler::Decoder;
+  const auto check_decoded = [](std::span<const uint32_t> code, bool expected,
+                                const char *message) {
+    D::Program decoded;
+    D::DecodeProgram(code, decoded);
+    Check(decoded.has_bvh == expected, message);
+    if (expected) {
+      const auto &last = decoded.instructions.back();
+      Check(last.family == D::Family::MIMG &&
+                (last.opcode_id == 0xe6u || last.opcode_id == 0xe7u),
+            "decoder continued past the first BVH instruction");
+      const auto result = ShaderRecompiler::TranslateProgram(
+          code, MakeCompileOptions(ShaderType::Compute));
+      Check(result.skip_dispatch && result.program.blocks.empty() && result.cfg_dump.empty(),
+            "BVH compute shader reached IR translation");
+    }
+  };
+  const auto check = [&](std::initializer_list<uint32_t> code, bool expected,
+                         const char *message) {
+    check_decoded(std::span<const uint32_t>{code.begin(), code.size()}, expected, message);
+  };
+
+  check({0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
+        "captured contiguous BVH instruction did not skip dispatch");
+  check({0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u,
+         EncodeSopp(0x01)}, true,
+        "captured NSA BVH instruction did not skip dispatch");
+  check({0xf19c9f01u, 0x00081014u, EncodeSopp(0x01)}, true,
+        "RDNA2 BVH64 instruction did not skip dispatch");
+  check({0xbe8e210eu, 0xf1989f01u, 0x00081014u, EncodeSopp(0x01)}, true,
+        "unsupported SWAPPC before BVH prevented dispatch detection");
+  check({0xf1989f01u, 0x00081014u, 0xc0000000u, EncodeSopp(0x01)}, true,
+        "unknown instruction after BVH prevented dispatch skipping");
+
+  for (uint32_t opcode : {0xe6u, 0xe7u}) {
+    for (uint32_t nsa = 0; nsa < 4; ++nsa) {
+      for (uint32_t slc = 0; slc < 2; ++slc) {
+        std::array<uint32_t, 6> code{
+            EncodeMimg0(opcode, 0xf, true) | (nsa << 1u) | (slc << 25u),
+            EncodeMimg1(0, 0, 0, 0)};
+        code[2 + nsa] = EncodeSopp(0x01);
+        check_decoded(std::span(code).first(3 + nsa), true,
+                      "BVH cache flags or NSA length prevented dispatch detection");
+      }
+    }
+  }
+  for (uint32_t opcode : {0x66u, 0x67u, 0xe4u, 0xe5u, 0xe8u, 0xe9u}) {
+    check({EncodeMimg0(opcode, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
+          false, "non-BVH image opcode was incorrectly skipped");
+  }
+
+  check({EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)}, false,
+        "scalar literal was mistaken for a BVH instruction");
+  check({EncodeMimg0(0x20, 0xf) | 6u, EncodeMimg1(0, 0, 0, 0),
+         0xf1989f01u, 0xf19c9f01u, 0u, EncodeSopp(0x01)}, false,
+        "NSA address words were mistaken for BVH instructions");
+  check({EncodeMimg0(0x20, 0xf), 0xf1989f01u, EncodeSopp(0x01)}, false,
+        "MIMG operand word was mistaken for a BVH instruction");
+  check({EncodeSopp(0x01), 0xf1989f01u, 0x00081014u}, false,
+        "data after the shader end was mistaken for a BVH instruction");
+  check({EncodeMimg0(0x20, 0xf), EncodeMimg1(0, 0, 0, 0), EncodeSopp(0x01)},
+        false, "ordinary image shader was incorrectly skipped");
+
+  // An end instruction on one branch does not end another branch's code.
+  check({EncodeSopp(0x04, 1), EncodeSopp(0x01), 0xf1989f01u, 0x00081014u,
+         EncodeSopp(0x01)}, true,
+        "BVH at a branch target after an early end was not detected");
+
+  const std::array bvh_code{0xf1989f01u, 0x00081014u, 0xf19c9f01u, 0x00081014u,
+                            EncodeSopp(0x01)};
+  const std::array ordinary_code{EncodeSMovB32(0, 255), 0xf1989f01u, EncodeSopp(0x01)};
+  D::Program reused;
+  D::DecodeProgram(bvh_code, reused);
+  Check(reused.has_bvh && reused.instructions.size() == 1,
+        "decoder did not stop at the first BVH instruction");
+  D::DecodeProgram(ordinary_code, reused);
+  Check(!reused.has_bvh, "decoder retained stale BVH metadata");
+  for (const auto stage : {ShaderType::Compute, ShaderType::Vertex, ShaderType::Pixel}) {
+    const auto options = MakeCompileOptions(stage);
+    Check(!ShaderRecompiler::TranslateProgram(ordinary_code, options).skip_dispatch,
+          "BVH-shaped literal marked an ordinary shader for skipping");
+    const auto compiled = RecompileForTest(ordinary_code, options);
+    Check(!compiled.spirv.empty(), "ordinary shader did not compile");
+    CheckSpirvBinaryValidates(compiled.spirv);
+  }
+}

tests/VirtualMemoryAllocationTests.cpp

@@ -462,6 +462,89 @@ void TestPrtBackingReadPreservesSparseResidency() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestPrtReadDuringDirectCommit() {
+	const char*        test       = "PrtReadDuringDirectCommit";
+	constexpr uint64_t chunk_size = SceKernelMemoryPoolCommitLen;
+	constexpr uint64_t chunks     = 32;
+	constexpr uint64_t size       = chunk_size * chunks;
+	int64_t            physical   = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, chunk_size,
+	            SceKernelMtypeC, &physical),
+	        "KernelAllocateDirectMemory");
+	void* arena = reinterpret_cast<void*>(0x1000000000ull);
+	CheckOk(test, Libs::LibKernel::Memory::KernelReserveVirtualRange(&arena, size, 0, size),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(arena);
+	CheckOk(test, Libs::LibKernel::Memory::KernelSetPrtAperture(2, arena, size),
+	        "KernelSetPrtAperture");
+	const auto map_chunk = [&](uint64_t index) {
+		void* address = reinterpret_cast<void*>(base + index * chunk_size);
+		return Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+		    &address, chunk_size, SceKernelProtCpuRw, SceKernelMapFixed,
+		    physical + static_cast<int64_t>(index * chunk_size), chunk_size, "prt_direct");
+	};
+	CheckOk(test, map_chunk(0), "KernelMapNamedDirectMemory(first)");
+	CheckOk(test, map_chunk(chunks - 1), "KernelMapNamedDirectMemory(last)");
+	std::memset(reinterpret_cast<void*>(base), 0x3c, chunk_size);
+	std::memset(reinterpret_cast<void*>(base + (chunks - 1) * chunk_size), 0xa7, chunk_size);
+
+	std::vector<uint8_t> bytes(size, 0x5a);
+	Check(test, Libs::LibKernel::Memory::TryReadPrtBacking(base, bytes.data(), size),
+	      "PRT read rejected a partly committed direct mapping");
+	Check(test, bytes.front() == 0x3c && bytes[chunk_size] == 0 && bytes.back() == 0xa7,
+	      "PRT read lost resident bytes or sparse zeros");
+	Libs::LibKernel::Memory::TestFailNextVirtualRangeReplacement();
+	CheckFailed(test, map_chunk(1), "KernelMapNamedDirectMemory(injected replacement failure)");
+	ExpectRange(test, Query(test, base + chunk_size), base + chunk_size,
+	            base + (chunks - 1) * chunk_size, 0, 0, 0, 0, 0, "anon");
+	Check(test, !Libs::LibKernel::Memory::TryReadBacking(base + chunk_size, bytes.data(),
+	                                                    chunk_size) &&
+	                Libs::LibKernel::Memory::TryReadPrtBacking(base, bytes.data(), size) &&
+	                bytes[chunk_size] == 0,
+	      "failed direct publication did not restore sparse reservation and backing");
+
+	std::atomic<bool>     started {false};
+	std::atomic<bool>     stop {false};
+	std::atomic<bool>     read_failed {false};
+	std::atomic<uint32_t> reads {0};
+	std::thread reader([&] {
+		std::vector<uint8_t> snapshot(size);
+		started.store(true, std::memory_order_release);
+		while (!stop.load(std::memory_order_acquire)) {
+			if (!Libs::LibKernel::Memory::TryReadPrtBacking(base, snapshot.data(), size) ||
+			    snapshot.front() != 0x3c || snapshot.back() != 0xa7) {
+				read_failed.store(true, std::memory_order_relaxed);
+			}
+			reads.fetch_add(1, std::memory_order_relaxed);
+		}
+	});
+	while (!started.load(std::memory_order_acquire)) {
+		std::this_thread::yield();
+	}
+	int map_result = OK;
+	for (uint64_t index = 1; index + 1 < chunks; ++index) {
+		map_result = map_chunk(index);
+		if (map_result != OK) {
+			break;
+		}
+	}
+	stop.store(true, std::memory_order_release);
+	reader.join();
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelSetPrtAperture(2, nullptr, 0),
+	        "KernelSetPrtAperture(clear)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap");
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(physical, size),
+	        "KernelReleaseDirectMemory");
+	CheckOk(test, map_result, "KernelMapNamedDirectMemory(middle)");
+	Check(test, reads.load(std::memory_order_relaxed) != 0 &&
+	                !read_failed.load(std::memory_order_relaxed),
+	      "PRT read observed a gap while direct backing was committed");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestGuestAddressSpaceHasNoFixedFallback() {
 	const char* test            = "GuestAddressSpaceHasNoFixedFallback";
 	const auto  unowned_address = reinterpret_cast<void*>(0x10000);
@@ -3031,6 +3114,7 @@ int main(int argc, char** argv) {
 	RunTest(TestProsperoArgumentAndInfoSizeContracts);
 	RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
 	RunTest(TestPrtBackingReadPreservesSparseResidency);
+	RunTest(TestPrtReadDuringDirectCommit);
 	RunTest(TestGuestAddressSpaceHasNoFixedFallback);
 	RunTest(TestGuestFreeRangeSearchDoesNotUnderflow);
 	RunTest(TestFlexibleMemoryCapacityIsBootFixed);

tests/shaderCfgTests.cpp

@@ -3523,8 +3523,6 @@ void TestNewShaderRecompilerVop1SdwaNotDestination() {
   check_rejected(0x00260400u, "V_NOT_B32 SDWA accepted source absolute");
   check_rejected(0x00062400u, "V_NOT_B32 SDWA accepted clamp");
   check_rejected(0x00064400u, "V_NOT_B32 SDWA accepted output modifier");
-  check_rejected(0x00000400u,
-                 "V_NOT_B32 SDWA partial destination accepted a byte source");
 }
 
 void TestNewShaderRecompilerBootB16PackedAndSdwaOpcodes() {
@@ -6711,9 +6709,9 @@ void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() {
   Check(result.resources.buffers.size() == 1 &&
             result.resources.buffers[0].dwords[2] == 5u,
         "formatted store test did not preserve descriptor NumRecords");
-  Check((result.decoded_dump.find("buffer_store_format_x") != std::string::npos),
+  Check((result.decoded_dump.find("BUFFER_STORE_FORMAT_X") != std::string::npos),
         "formatted store regression did not decode buffer_store_format_x");
-  Check((result.ir_dump.find("typed=0 formatted=1") != std::string::npos),
+  Check((result.decoded_dump.find("typed=0 formatted=1") != std::string::npos),
         "formatted store regression did not preserve formatted metadata");
   CheckSpirvBinaryValidates(result.spirv);
 
@@ -6723,6 +6721,15 @@ void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() {
   Check(
       !SpirvSourceHasInstructionUsing(source, "OpULessThan", "%uint_5"),
       "formatted store SPIR-V baked descriptor NumRecords into a store guard");
+
+  user_data[1] = 8u << 16u;
+  user_data[3] = static_cast<uint32_t>(Prospero::BufferFormat::k16_16_16_16Float) << 12u;
+  options.user_data = user_data;
+  result = RecompileForTest(shader, options);
+  Check((DisassembleSpirvBinary(result.spirv).find("PackHalf2x16") !=
+         std::string::npos),
+        "formatted half-float store did not convert F32 to F16");
+  CheckSpirvBinaryValidates(result.spirv);
 }
 
 void TestNewShaderRecompilerTypedBufferTranslation() {
@@ -7846,6 +7853,44 @@ void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() {
   CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerCfgLoopExitSharedWithSelection() {
+  const uint32_t shader[] = {
+      EncodeSopc(0x0a, 0, 129),    // loop condition
+      EncodeSopp(0x04, 10),        // loop exit -> end
+      EncodeSopc(0x06, 1, 1),      // selection within the loop
+      EncodeSopp(0x05, 3),         // choose either arm
+      EncodeSopc(0x06, 2, 2),      // first arm
+      EncodeSopp(0x04, 6),         // first arm -> shared end
+      EncodeSopp(0x02, 3),         // first arm -> repeat
+      EncodeSopc(0x06, 3, 3),      // second arm
+      EncodeSopp(0x04, 3),         // second arm -> shared end
+      EncodeSopp(0x02, 0),         // second arm -> repeat
+      EncodeSop2(0x00, 0, 0, 129), // repeat work
+      EncodeSopp(0x02, 0xfff4u),  // backedge
+      0xbf810000u,
+  };
+
+  ShaderRecompiler::Decoder::Program decoded;
+  ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded);
+  auto graph = ShaderRecompiler::CFG::BuildGraph(decoded);
+  const auto block_count = graph.blocks.size();
+  const auto coverage = CfgInstructionCoverage(graph, decoded.instructions.size());
+  Check(block_count == 8u && graph.natural_loops.size() == 1u,
+        "shared loop-exit fixture has the wrong native CFG");
+  Check(!ShaderRecompiler::CFG::Structurize(graph) &&
+            graph.unsupported_reason.find("duplicate structured merge block") !=
+                std::string::npos,
+        "shared loop exit did not terminate at its structured merge conflict");
+  Check(graph.blocks.size() == block_count &&
+            CfgInstructionCoverage(graph, decoded.instructions.size()) == coverage,
+        "shared loop-exit fallback changed semantic instruction coverage");
+
+  auto result = RecompileForTest(shader, MakeCompileOptions(ShaderType::Compute));
+  Check(result.program.dispatcher_fallback && SpirvContainsOpcode(result.spirv, 251),
+        "shared loop exit did not emit its dispatcher fallback");
+  CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() {
   const uint32_t shader[] = {
       EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
@@ -11499,6 +11544,27 @@ void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() {
 }
 
 void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() {
+  for (uint32_t wave_size : {32u, 64u}) {
+    for (uint32_t opcode : {0x09u, 0x0au}) {
+      for (uint32_t source : {126u, 8u}) {
+        const uint32_t pixel_shader[] = {
+            EncodeSop1(0x04, 8, 126), // Save the entry live mask.
+            EncodeSop1(opcode, 126, source),
+            EncodeVop1(0x01, 0, 242), // v_mov_b32 v0, 1.0
+            EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0),
+            EncodeSopp(0x01),
+        };
+        auto pixel_options = MakeCompileOptions(ShaderType::Pixel);
+        pixel_options.wave_size = wave_size;
+        const auto pixel_result = RecompileForTest(pixel_shader, pixel_options);
+        CheckSpirvBinaryValidates(pixel_result.spirv);
+        Check(DisassembleSpirvBinary(pixel_result.spirv).find("OpGroupNonUniformBallot") ==
+                  std::string::npos,
+              "entry WQM lost the known live predicate through a scalar ballot");
+      }
+    }
+  }
+
   const uint32_t local_shader[] = {
       EncodeVopc(0xc1, 5 + 256, 8),    // v_cmp_lt_u32 vcc, v5, v8
       EncodeSop2(0x0f, 2, 126, 106),   // s_and_b64 s[2:3], exec, vcc
@@ -13487,6 +13553,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() {
   CheckSpirvPhiParents(dispatcher_result.spirv);
 }
 
+#include "ShaderRayTracingTests.inc"
+
 } // namespace
 } // namespace Libs::Graphics
 
@@ -13494,6 +13562,7 @@ int main() {
   using namespace Libs::Graphics;
 
   EnsureConfigInitialized();
+  TestRayTracingDispatchDetection();
   TestResourceDescriptorClassification();
   TestShaderBufferResourceSize();
   TestNativeShaderResourceDependencies();
@@ -13515,6 +13584,7 @@ int main() {
   TestNewShaderRecompilerNativeWideBufferIr();
   TestNewShaderRecompilerScalarB64LaneTranslation();
   TestNewShaderRecompilerMubufFormatTranslation();
+  TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly();
   TestNewShaderRecompilerTypedBufferTranslation();
   TestNewShaderRecompilerDsReadWrite2Translation();
   TestNewShaderRecompilerDsWideAndAtomicTranslation();
@@ -13552,6 +13622,7 @@ int main() {
   TestNewShaderRecompilerCfgLoopHeaderDsReadStructured();
   TestNewShaderRecompilerCfgLoopHeaderDsRead2B64Structured();
   TestNewShaderRecompilerCfgSharedOuterAndLoopMerge();
+  TestNewShaderRecompilerCfgLoopExitSharedWithSelection();
   TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection();
   TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher();
   TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection();