ngs2: let CLEAR reopen a closed waveform block list

Co-authored-by: Phr3d13 <phr3d13@gmail.com>

.github/workflows/build.yml

@@ -32,7 +32,7 @@ jobs:
   windows:
     name: Build KytyPS5 (Windows)
     needs: metadata
-    runs-on: windows-2022
+    runs-on: windows-2025
 
     steps:
       - name: Checkout repository
@@ -113,7 +113,7 @@ jobs:
         shell: cmd
         run: |
           cmake --build _Build/windows --target launcher ^
-            audio_out2_port_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
+            audio_out2_port_tests ngs2_sampler_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
             shader_recompiler_compute_tests ^
             resource_tracking_tests scalar_provenance_tests ^
             resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests --parallel
@@ -121,7 +121,7 @@ jobs:
       - name: Test
         shell: cmd
         run: |
-          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
+          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|ngs2_sampler|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
 
       - name: Install
         shell: cmd
@@ -202,7 +202,7 @@ jobs:
         shell: bash
         run: |
           cmake --build _Build/macos \
-            --target launcher audio_out2_port_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
+            --target launcher audio_out2_port_tests ngs2_sampler_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
               memory_tracker_tests page_manager_tests shader_recompiler_compute_tests \
               resource_tracking_tests scalar_provenance_tests \
               resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
@@ -212,7 +212,7 @@ jobs:
         shell: bash
         run: |
           ctest --test-dir _Build/macos --output-on-failure \
-            -R '^(audio_out2_port|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|ngs2_sampler|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash
@@ -388,7 +388,7 @@ jobs:
         run: |
           cmake --build _Build/linux \
             --target launcher page_manager_tests memory_tracker_tests \
-              audio_out2_port_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
+              audio_out2_port_tests ngs2_sampler_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
               resource_tracking_tests scalar_provenance_tests \
               resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
             --parallel
@@ -397,7 +397,7 @@ jobs:
         shell: bash
         run: |
           ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(audio_out2_port|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|ngs2_sampler|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash

3rdparty/CMakeLists.txt

@@ -42,6 +42,28 @@ if(NOT KYTY_CLANG_CL)
 endif()
 install(FILES "${openxr_SOURCE_DIR}/LICENSE" DESTINATION "licenses/openxr")
 
+# Noto Sans v2.015 (SIL Open Font License 1.1) is libFont's fallback font.
+set(KYTY_NOTO_SANS_REPOSITORY "https://raw.githubusercontent.com/notofonts/notofonts.github.io")
+set(KYTY_NOTO_SANS_COMMIT 42af67ccd12bc0e31ed85384bf73e39248265374)
+set(KYTY_NOTO_SANS_BASE_URL "${KYTY_NOTO_SANS_REPOSITORY}/${KYTY_NOTO_SANS_COMMIT}/fonts")
+set(KYTY_NOTO_SANS_DIR "${FETCHCONTENT_BASE_DIR}/noto-sans")
+function(kyty_fetch_noto_sans path sha256)
+	get_filename_component(name "${path}" NAME)
+	file(DOWNLOAD "${KYTY_NOTO_SANS_BASE_URL}/${path}" "${KYTY_NOTO_SANS_DIR}/${name}"
+		EXPECTED_HASH SHA256=${sha256})
+endfunction()
+kyty_fetch_noto_sans(LICENSE f2095b08bed08b23a6fe26112fcd679a2bee3f002eef077eb05d215ed1051bd8)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-Regular.ttf
+	478c558ea716033cd60c03438f628dfa75694dcf6b5f6d505a2f05fd2b4f3823)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-Bold.ttf
+	1df075a380fc7cb898acf64c1f7b3b4dd780de3caa860178bf929de35817a913)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-Italic.ttf
+	467e3f89eeca4108bb8710a2b9e0cf2281ac56d5b0609211a83776d0505eecb5)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-BoldItalic.ttf
+	1b602a9d6353be42c91df097a4857b69fa2696f26703d7a33b54a15d87c2622c)
+install(FILES "${KYTY_NOTO_SANS_DIR}/LICENSE" DESTINATION "licenses/noto-sans")
+set(KYTY_NOTO_SANS_DIR "${KYTY_NOTO_SANS_DIR}" PARENT_SCOPE)
+
 # The guest instruction patcher needs a decoder/encoder and a small
 # trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)

CMakeLists.txt

@@ -325,6 +325,7 @@ add_gpu_blit_shader(gpu_blit_fs_triangle gpu_blit_fs_triangle.vert)
 add_gpu_blit_shader(gpu_blit_color_to_ms_depth gpu_blit_color_to_ms_depth.frag)
 add_gpu_blit_shader(gpu_video_out_overlay gpu_video_out_overlay.frag)
 add_gpu_blit_shader(gpu_imgui_hdr gpu_video_out_overlay.frag -DIMGUI_HDR=1)
+add_gpu_blit_shader(gpu_pssr_linear gpu_pssr_linear.comp)
 list(APPEND kyty_emulator_src ${gpu_blit_shader_headers})
 
 list(APPEND kyty_emulator_src
@@ -407,6 +408,8 @@ add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
 target_link_libraries(scalar_provenance_tests fmt::fmt xbyak::xbyak)
 target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
 
+add_executable(build_flags_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/BuildFlagsTests.cpp")
+
 add_executable(page_manager_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/PageManagerTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/pageManager.cpp"
@@ -520,6 +523,13 @@ add_executable(ngs2_sampler_tests EXCLUDE_FROM_ALL
 target_link_libraries(ngs2_sampler_tests common fmt::fmt LibAtrac9)
 target_include_directories(ngs2_sampler_tests PRIVATE ${inc_headers})
 
+add_executable(ajm_resample_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/AjmResampleTests.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
+target_link_libraries(ajm_resample_tests common fmt::fmt FFmpeg::ffmpeg LibAtrac9 opus)
+target_include_directories(ajm_resample_tests PRIVATE ${inc_headers})
+
 add_executable(pad_haptics_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/PadHapticsTests.cpp"
 )
@@ -622,6 +632,19 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 	"${gpu_test_ms_depth_header}")
 
 add_kyty_full_emulator_test(virtual_memory_allocation_tests "${KYTY_TESTS_DIR}/VirtualMemoryAllocationTests.cpp")
+add_library(kyty_test_gtest STATIC EXCLUDE_FROM_ALL
+	"${KYTY_THIRD_PARTY_DIR}/fmt/test/gtest/gmock-gtest-all.cc")
+target_include_directories(kyty_test_gtest PUBLIC "${KYTY_THIRD_PARTY_DIR}/fmt/test/gtest")
+target_sources(virtual_memory_allocation_tests PRIVATE "${KYTY_TESTS_DIR}/TrinityTests.cpp")
+target_link_libraries(virtual_memory_allocation_tests kyty_test_gtest)
+
+add_executable(psml_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/PsmlTests.cpp"
+	"${KYTY_SOURCE_DIR}/libs/libPsml.cpp"
+	"${KYTY_SOURCE_DIR}/loader/symbolDatabase.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp")
+target_link_libraries(psml_tests common kyty_test_gtest)
+target_include_directories(psml_tests PRIVATE ${inc_headers})
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
 
@@ -650,6 +673,7 @@ if(BUILD_TESTING)
 	add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
 	add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
 	add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
+	add_test(NAME build_flags COMMAND $<TARGET_FILE:build_flags_tests>)
 	add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
 	add_test(NAME trophy_system COMMAND $<TARGET_FILE:trophy_system_tests>)
 	add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
@@ -665,6 +689,7 @@ if(BUILD_TESTING)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME audio_out_timing COMMAND $<TARGET_FILE:audio_out_timing_tests>)
 	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
+	add_test(NAME ajm_resample COMMAND $<TARGET_FILE:ajm_resample_tests>)
 	add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
 	add_test(NAME controller_settings COMMAND $<TARGET_FILE:controller_settings_tests>)
 	add_test(NAME host_input COMMAND $<TARGET_FILE:host_input_tests>)
@@ -681,6 +706,16 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --tessellation-only)
 	add_test(NAME shader_recompiler_float_min_max
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --float-min-max-only)
+	add_test(NAME shader_recompiler_vop3_conversions
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --vop3-conversions-only)
+	add_test(NAME shader_recompiler_quadmask
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --quadmask-only)
+	add_test(NAME shader_recompiler_integer16_ternary
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --integer16-ternary-only)
+	add_test(NAME shader_recompiler_mad_u16
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --mad-u16-only)
+	add_test(NAME shader_recompiler_mul_hi24
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --mul-hi24-only)
 	add_test(NAME shader_recompiler_alignbyte
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
 	add_test(NAME graphics_pipeline_rasterization
@@ -689,6 +724,10 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --draw-offset-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
+	add_test(NAME trinity_mode
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests> --trinity-only)
+	add_test(NAME psml COMMAND $<TARGET_FILE:psml_tests>)
+	add_test(NAME pssr COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --pssr-only)
 	add_test(NAME guest_red_zone_patcher
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests> --red-zone-patcher-only)
 	add_test(NAME command_scheduler_timeline
@@ -697,8 +736,6 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --stream-buffer-only)
 	add_test(NAME gpu_command_lane
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
-	add_test(NAME wait_until_safe_for_rendering
-		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --wait-until-safe-for-rendering-only)
 	add_test(NAME pm4_flip
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --flip-only)
 	add_test(NAME depth_bits
@@ -743,6 +780,7 @@ if(BUILD_TESTING)
 		image_page_table_tests
 		memory_tracker_tests
 		page_manager_tests
+		build_flags_tests
 		archive_file_tests
 		trophy_system_tests
 		avplayer_file_tests
@@ -760,6 +798,7 @@ if(BUILD_TESTING)
 		audio_out2_port_tests
 		audio_out_timing_tests
 		ngs2_sampler_tests
+		ajm_resample_tests
 		pad_haptics_tests
 		controller_settings_tests
 		host_input_tests
@@ -811,6 +850,19 @@ add_custom_target(kyty_touch_cursors
 add_dependencies(kyty_emulator kyty_touch_cursors)
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/assets/cursors" DESTINATION assets)
 
+set(KYTY_NOTO_SANS_FONTS
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-Regular.ttf"
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-Bold.ttf"
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-Italic.ttf"
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-BoldItalic.ttf")
+add_custom_target(kyty_noto_sans_fonts
+	COMMAND ${CMAKE_COMMAND} -E make_directory "$<TARGET_FILE_DIR:kyty_emulator>/assets/fonts"
+	COMMAND ${CMAKE_COMMAND} -E copy_if_different ${KYTY_NOTO_SANS_FONTS}
+		"$<TARGET_FILE_DIR:kyty_emulator>/assets/fonts"
+	VERBATIM)
+add_dependencies(kyty_emulator kyty_noto_sans_fonts)
+install(FILES ${KYTY_NOTO_SANS_FONTS} DESTINATION assets/fonts)
+
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)
 	target_link_libraries(kyty_emulator opengl32)

src/common/emulatorConfig.cpp

@@ -105,10 +105,18 @@ bool HideCursorEnabled() {
 	return g_config->hide_cursor_enabled;
 }
 
+bool StretchEnabled() {
+	return g_config->stretch_enabled;
+}
+
 bool VrEnabled() {
 	return g_config->vr_enabled;
 }
 
+bool Ps5ProEnabled() {
+	return g_config->pro_enabled;
+}
+
 bool AmdCpuEnabled() {
 	return g_config->amd_cpu_enabled;
 }
@@ -129,6 +137,10 @@ bool IsNetworkConnected() {
 	return g_config->network_connected;
 }
 
+bool PsnEmulationEnabled() {
+	return g_config->psn_emulation_enabled;
+}
+
 bool ShaderValidationEnabled() {
 	return g_config->shader_validation_enabled;
 }

src/common/emulatorConfig.h

@@ -64,10 +64,13 @@ struct ConfigOptions {
 	bool                   fullscreen_enabled          = false;
 	bool                   hdr_enabled                 = false;
 	bool                   hide_cursor_enabled         = false;
+	bool                   stretch_enabled             = false;
 	bool                   splash_enabled              = true;
 	bool                   vr_enabled                  = false;
+	bool                   pro_enabled                 = false;
 	bool                   amd_cpu_enabled             = false;
 	bool                   network_connected           = false;
+	bool                   psn_emulation_enabled       = false;
 	uint32_t               vblank_frequency            = 60;
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
@@ -125,9 +128,12 @@ int32_t GetGpuIndex();
 bool     FullscreenEnabled();
 bool     HdrEnabled();
 bool     HideCursorEnabled();
+bool     StretchEnabled();
 bool     VrEnabled();
+bool     Ps5ProEnabled();
 bool     AmdCpuEnabled();
 bool     IsNetworkConnected();
+bool     PsnEmulationEnabled();
 uint32_t GetVblankFrequency();
 uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();

src/emulator.cpp

@@ -99,10 +99,6 @@ static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path&
 
 	if (Common::File::IsFileExisting(param_json)) {
 		Loader::SystemContentLoadParamSfo(param_json);
-		if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
-		    flexible_memory_size != 0) {
-			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
-		}
 	}
 	if (cfg.splash_enabled) {
 		Libs::Graphics::InitializeSplashScreen(param_json.parent_path());

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -3,6 +3,7 @@
 
 #include "common/assert.h"
 #include "graphics/guest_gpu/hardwareContext.h"
+#include "graphics/guest_gpu/pssr.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 
@@ -27,6 +28,7 @@ struct IndirectDrawRegisters {
 	uint32_t vertex_offset;
 	uint32_t instance_offset;
 	uint32_t index_offset;
+	uint32_t draw_index;
 };
 
 class Pm4Execution {
@@ -91,12 +93,14 @@ public:
 	                    uint32_t mode);
 	void DispatchIndirect(uint64_t args_addr, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
+	void ProcessPssr(const Pssr::Request& request);
 	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);
 
 	void WaitForRewind(bool valid);
 
 	template <typename T>
 	void WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, uint32_t wait_op);
+	void MemSemaphore(uint64_t* counter, bool signal, bool write);
 	void WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num, uint32_t write_control);
 	void WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes);
 	void DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cache_policy,

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -202,6 +202,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_DMA_DATA]                  = CpOpDmaData;
 	g_cp_op_func[Pm4::IT_ACQUIRE_MEM]               = CpOpAcquireMem;
 	g_cp_op_func[Pm4::IT_RELEASE_MEM]               = CpOpReleaseMem;
+	g_cp_op_func[Pm4::IT_MEM_SEMAPHORE]             = CpOpMemSemaphore;
 	g_cp_op_func[Pm4::IT_REWIND]                    = CpOpRewind;
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG]           = CpOpSetContextReg;
 	g_cp_op_func[Pm4::IT_SET_SH_REG]                = CpOpSetShaderReg;
@@ -225,6 +226,7 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;
+	g_cp_op_custom_func[Pm4::R_PSSR]           = CpOpPssr;
 
 	return g_cp_op_custom_func;
 }

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -136,6 +136,7 @@ uint32_t CpOpEventWrite(CommandProcessor&, uint32_t, const uint32_t*, uint32_t,
 uint32_t CpOpDmaData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpAcquireMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpReleaseMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpMemSemaphore(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetContextReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetShaderReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetUconfigReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
@@ -150,6 +151,7 @@ uint32_t CpOpPushMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t,
 uint32_t CpOpPopMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpWaitRegMem64(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpFlip(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpPssr(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 
 } // namespace Libs::Graphics
 

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -1582,7 +1582,7 @@ KYTY_CP_OP_PARSER(CpOpDrawIndirect) {
 	// The native indexed packet enables the high-half first-index destination in word 3.
 	const IndirectDrawRegisters registers {
 	    buffer[1] & 0xffffu, buffer[2] & 0xffffu,
-	    indexed && (buffer[2] & (1u << 28u)) != 0 ? buffer[1] >> 16u : Pm4::SH_NOP};
+	    indexed && (buffer[2] & (1u << 28u)) != 0 ? buffer[1] >> 16u : Pm4::SH_NOP, Pm4::SH_NOP};
 
 	cp.DrawIndirect(data_offset, registers, draw_initiator, indexed);
 
@@ -1602,10 +1602,12 @@ KYTY_CP_OP_PARSER(CpOpDrawIndirectMulti) {
 	const auto stride_in_bytes = buffer[7];
 	const auto draw_initiator  = buffer[8];
 	const bool indexed         = (cmd_id == 0xc0083800);
-	// Multi-draw moves the first-index enable to word 4; the destinations stay in 2/3.
+	// Multi-draw moves the first-index enable to word 4; the destinations stay in 2/3. Word 4
+	// bit 31 enables the subdraw index, written to the register in its low half.
 	const IndirectDrawRegisters registers {
 	    buffer[1] & 0xffffu, buffer[2] & 0xffffu,
-	    indexed && (buffer[3] & (1u << 28u)) != 0 ? buffer[1] >> 16u : Pm4::SH_NOP};
+	    indexed && (buffer[3] & (1u << 28u)) != 0 ? buffer[1] >> 16u : Pm4::SH_NOP,
+	    (buffer[3] >> 31u) != 0 ? buffer[3] & 0xffffu : Pm4::SH_NOP};
 
 	if (count_indirect == 0) {
 		count_addr = nullptr;
@@ -1704,6 +1706,17 @@ KYTY_CP_OP_PARSER(CpOpFlip) {
 	return 5;
 }
 
+KYTY_CP_OP_PARSER(CpOpPssr) {
+	KYTY_PROFILER_FUNCTION();
+
+	const auto packet_dw = KYTY_PM4_LEN(cmd_id) - 1;
+	EXIT_NOT_IMPLEMENTED(packet_dw < Pssr::RequestDwords || packet_dw > dw);
+	Pssr::Request request {};
+	std::memcpy(&request, buffer, sizeof(request));
+	cp.ProcessPssr(request);
+	return packet_dw;
+}
+
 KYTY_CP_OP_PARSER(CpOpIndexType) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -2058,6 +2071,21 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 	return 7;
 }
 
+KYTY_CP_OP_PARSER(CpOpMemSemaphore) {
+	KYTY_PROFILER_FUNCTION();
+
+	const auto control = buffer[2];
+	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0023900 || (control & ~0xe0110000u) != 0 ||
+	                     (control >> 29u) < 6u);
+
+	auto* counter = reinterpret_cast<uint64_t*>(
+	    (buffer[0] & ~0x7u) | (static_cast<uint64_t>(buffer[1] & 0x3ffffu) << 32u));
+	// A signal updates the counter before the next packet, so its mailbox stall never waits.
+	cp.MemSemaphore(counter, (control >> 29u) == 6u, ((control >> 20u) & 0x1u) != 0);
+
+	return 3;
+}
+
 KYTY_CP_OP_PARSER(CpOpSetContextReg) {
 	KYTY_PROFILER_FUNCTION();
 

src/graphics/guest_gpu/gpu_format.cpp

@@ -46,6 +46,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
 	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
 	{BufferFormat::k10_11_11Float, 4, 0, 4, true, false},
+	{BufferFormat::k2_10_10_10UNorm, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
 	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},

src/graphics/guest_gpu/graphicsRun.cpp

@@ -30,7 +30,6 @@
 #include <mutex>
 #include <semaphore>
 #include <thread>
-#include <utility>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -92,12 +91,6 @@ void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
 	m_work_available.Signal();
 }
 
-void GuestGpu::Wake() {
-	Common::LockGuard lock(m_queue_mutex);
-	m_wake_pending = true;
-	m_work_available.Signal();
-}
-
 void GuestGpu::ProcessCommands() {
 	EXIT_IF(!IsGpuThread());
 	while (m_pending_commands.load(std::memory_order_acquire) != 0) {
@@ -281,6 +274,16 @@ template void CommandProcessor::WaitRegMem<uint32_t>(uint32_t, const uint32_t*,
 template void CommandProcessor::WaitRegMem<uint64_t>(uint32_t, const uint64_t*, uint64_t, uint64_t,
                                                      uint32_t, uint32_t);
 
+void CommandProcessor::MemSemaphore(uint64_t* counter, bool signal, bool write) {
+	if (signal) {
+		*counter = write ? 1u : *counter + 1u;
+	} else if (*counter == 0) {
+		SuspendPm4();
+	} else {
+		(*counter)--;
+	}
+}
+
 void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num,
                                  uint32_t write_control) {
 	const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
@@ -433,10 +436,7 @@ void GuestGpu::ThreadRun(void* data) {
 				}
 				if (selected_queue < 0) {
 					gpu->m_processing = false;
-					// A wake that came before this wait must not be lost.
-					if (!std::exchange(gpu->m_wake_pending, false)) {
-						gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
-					}
+					gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
 					for (auto& queue: gpu->m_queues) {
 						if (!queue.empty()) {
 							queue.front().blocked = false;
@@ -789,7 +789,7 @@ void CommandProcessor::DrawIndexOffset(uint32_t index_offset, uint32_t index_cou
 
 static void PatchIndirectDrawOffsets(CommandProcessor& cp, IndirectDrawRegisters registers,
                                      uint32_t vertex_offset, uint32_t instance_offset,
-                                     uint32_t first_index) {
+                                     uint32_t first_index, uint32_t draw_index) {
 	const auto write = [&](uint32_t location, uint32_t value) {
 		if (location == Pm4::SH_NOP) {
 			return;
@@ -801,6 +801,28 @@ static void PatchIndirectDrawOffsets(CommandProcessor& cp, IndirectDrawRegisters
 	write(registers.vertex_offset, vertex_offset);
 	write(registers.instance_offset, instance_offset);
 	write(registers.index_offset, first_index);
+	write(registers.draw_index, draw_index);
+}
+
+// Writes the GS user-data slots the packet patches; false when the shaders cannot read the
+// offsets from GPU records: a first-index or subdraw-index destination, or another stage's
+// user data.
+static bool IndirectDrawOffsets(IndirectDrawRegisters registers, ShaderIndirectOffsets& offsets) {
+	if (registers.index_offset != Pm4::SH_NOP || registers.draw_index != Pm4::SH_NOP) {
+		return false;
+	}
+	for (const auto [location, slot]: {std::pair {registers.vertex_offset, &offsets.vertex},
+	                                   std::pair {registers.instance_offset, &offsets.instance}}) {
+		if (location == Pm4::SH_NOP) {
+			continue;
+		}
+		const uint32_t index = location - Pm4::SPI_SHADER_USER_DATA_GS_0;
+		if (index >= HW::UserSgprInfo::SGPRS_MAX) {
+			return false;
+		}
+		*slot = static_cast<uint8_t>(index);
+	}
+	return true;
 }
 
 void CommandProcessor::DrawIndirect(uint32_t data_offset, IndirectDrawRegisters registers,
@@ -834,12 +856,10 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 	// NUM_INSTANCES keeps the last draw's instance count.
 	m_num_instances_addr = args_addr + static_cast<uint64_t>(draw_count - 1) * stride_in_bytes +
 	                       offsetof(vk::DrawIndirectCommand, instanceCount);
-	// With no SGPRs to patch, the GPU reads the records if the shaders allow. Shaders apply
-	// the records' offsets only through those SGPRs; Vulkan applies them, which matches
-	// while they are zero.
-	const DrawIndirectArgs records {args_addr, draw_count, stride_in_bytes};
-	if (registers.vertex_offset == Pm4::SH_NOP && registers.instance_offset == Pm4::SH_NOP &&
-	    registers.index_offset == Pm4::SH_NOP && (!indexed || m_index_buffer_size != 0) &&
+	// The GPU reads the records if the shaders allow; the VS prolog removes Vulkan's bases
+	// from the index VGPRs and copies them only to the user SGPRs the packet patches.
+	DrawIndirectArgs records {args_addr, draw_count, stride_in_bytes};
+	if (IndirectDrawOffsets(registers, records.offsets) && (!indexed || m_index_buffer_size != 0) &&
 	    (indexed ? executor.DrawIndex(m_submit_id, CurrentBuffer(),
 	                                  {.index_count         = m_index_buffer_size,
 	                                   .index_addr          = IndexAddress(0),
@@ -854,7 +874,7 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 		const auto addr = args_addr + static_cast<uint64_t>(i) * stride_in_bytes;
 		if (!indexed) {
 			const auto& args = *reinterpret_cast<const vk::DrawIndirectCommand*>(addr);
-			PatchIndirectDrawOffsets(*this, registers, args.firstVertex, args.firstInstance, 0);
+			PatchIndirectDrawOffsets(*this, registers, args.firstVertex, args.firstInstance, 0, i);
 			executor.DrawAuto(m_submit_id, CurrentBuffer(),
 			                  {.vertex_count   = args.vertexCount,
 			                   .instance_count = args.instanceCount,
@@ -866,7 +886,7 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 		const auto& args = *reinterpret_cast<const vk::DrawIndexedIndirectCommand*>(addr);
 		PatchIndirectDrawOffsets(*this, registers, static_cast<uint32_t>(args.vertexOffset),
-		                         args.firstInstance, args.firstIndex);
+		                         args.firstInstance, args.firstIndex, i);
 		executor.DrawIndex(m_submit_id, CurrentBuffer(),
 		                   {.index_count         = std::min(args.indexCount, max_indices),
 		                    .index_addr          = IndexAddress(args.firstIndex),
@@ -909,6 +929,11 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 	}
 }
 
+void CommandProcessor::ProcessPssr(const Pssr::Request& request) {
+	EXIT_NOT_IMPLEMENTED(IsAsyncComputeQueue());
+	m_renderer.GetRenderExecutor().ProcessPssr(CurrentBuffer(), request);
+}
+
 void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {
 	EXIT_NOT_IMPLEMENTED(args_addr == 0 || (args_addr & 3u) != 0);
 	if ((mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0) {
@@ -929,11 +954,10 @@ void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {
 }
 
 void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
-	// Stall only this command processor, not the host command lane or other queues.
-	if (!m_renderer.GetVideoOut().IsSafeForRendering(static_cast<int>(video_out_handle),
-	                                                 static_cast<int>(display_buffer_index))) {
-		SuspendPm4();
-	}
+	BufferFlush();
+
+	m_renderer.GetVideoOut().WaitFlipDone(static_cast<int>(video_out_handle),
+	                                      static_cast<int>(display_buffer_index));
 }
 
 void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector, void* address,
@@ -964,16 +988,28 @@ void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector
 	// Dumped occlusion counters are written before any end-of-pipe signal reaches the guest.
 	m_renderer.GetOcclusionQueries().Publish();
 
+	// A label beside GPU-written bytes goes to the backing store, so they are not downloaded.
+	// Other labels go through the guest mapping: its fault keeps a clean cached copy current and
+	// downloads GPU-written bytes under the label before the store.
+	const auto write_label = [this, address](const void* value, uint64_t size) {
+		const auto vaddr   = reinterpret_cast<uint64_t>(address);
+		auto&      buffers = m_renderer.GetBufferCache();
+		if (!GuestRange {vaddr, size}.Valid() || !buffers.IsRegionGpuModified(vaddr, size) ||
+		    buffers.HasGpuDirtyBytes(vaddr, size) ||
+		    !LibKernel::Memory::TryWriteBacking(vaddr, value, size)) {
+			std::memcpy(address, value, size);
+		}
+	};
 	// EndOfPipeData: kNone, kImmediate32b, kImmediate64b, kRefClk; 5 reads GDS.
 	switch (data_sel) {
 		case 0: break;
 		case 1: {
 			const auto value = static_cast<uint32_t>(data);
-			std::memcpy(address, &value, sizeof(value));
+			write_label(&value, sizeof(value));
 			break;
 		}
 		case 3: data = Sync::ReadReferenceClock(); [[fallthrough]];
-		case 2: std::memcpy(address, &data, sizeof(data)); break;
+		case 2: write_label(&data, sizeof(data)); break;
 		case 5: {
 			GetScheduler().Finish();
 			const auto gds = static_cast<uint32_t>(data);

src/graphics/guest_gpu/graphicsRun.h

@@ -31,8 +31,6 @@ public:
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
-	// Retries blocked submissions now instead of after the 100 ms stall timeout.
-	void               Wake();
 
 	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
 	void Submit(std::span<const uint32_t> commands);
@@ -84,7 +82,6 @@ private:
 	bool                                           m_processing        = false;
 	bool                                           m_accepting         = true;
 	bool                                           m_stopping          = false;
-	bool                                           m_wake_pending      = false;
 	bool                                           m_shutdown_complete = false;
 	// Completion callbacks can outlive GuestGpu during renderer shutdown.
 	std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =

src/graphics/guest_gpu/pm4.h

@@ -81,6 +81,7 @@ constexpr uint32_t R_WRITE_DATA     = 0x15;
 constexpr uint32_t R_FLIP           = 0x17;
 constexpr uint32_t R_DMA_DATA       = 0x19;
 constexpr uint32_t R_CONTEXT_STATE  = 0x1A;
+constexpr uint32_t R_PSSR           = 0x1B;
 
 constexpr uint32_t R_NUM = 0x3F + 1;
 

src/graphics/guest_gpu/pssr.h

@@ -0,0 +1,20 @@
+#ifndef EMULATOR_SRC_GRAPHICS_GUEST_GPU_PSSR_H_
+#define EMULATOR_SRC_GRAPHICS_GUEST_GPU_PSSR_H_
+
+#include <cstdint>
+
+namespace Libs::Graphics::Pssr {
+
+struct Request {
+	uint32_t output[8];
+	uint32_t input[8];
+	uint32_t width;
+	uint32_t height;
+};
+
+constexpr uint32_t RequestDwords = sizeof(Request) / sizeof(uint32_t);
+static_assert(sizeof(Request) == 72);
+
+} // namespace Libs::Graphics::Pssr
+
+#endif // EMULATOR_SRC_GRAPHICS_GUEST_GPU_PSSR_H_

src/graphics/host_gpu/rangeSet.h

@@ -5,6 +5,7 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <iterator>
 #include <map>
 
 namespace Libs::Graphics {
@@ -17,14 +18,19 @@ public:
 		if (it != m_ranges.begin() && std::prev(it)->second >= address) {
 			it = std::prev(it);
 		}
-		uint64_t begin = address;
-		uint64_t last  = end;
-		while (it != m_ranges.end() && it->first <= last) {
-			begin = std::min(begin, it->first);
-			last  = std::max(last, it->second);
-			it    = m_ranges.erase(it);
+		if (it == m_ranges.end() || it->first > address) {
+			it = m_ranges.emplace_hint(it, address, end);
+		} else {
+			if (it->second >= end) {
+				return;
+			}
+			it->second = end;
+		}
+		auto next = std::next(it);
+		while (next != m_ranges.end() && next->first <= it->second) {
+			it->second = std::max(it->second, next->second);
+			next       = m_ranges.erase(next);
 		}
-		m_ranges.emplace(begin, last);
 	}
 
 	void Subtract(uint64_t address, uint64_t size) {

src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h

@@ -8,6 +8,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
+#include <iterator>
 #include <memory>
 #include <type_traits>
 #include <utility>
@@ -118,6 +119,13 @@ public:
 
 	class const_iterator final {
 	public:
+		// libstdc++'s algorithms need these. Input, not forward, because operator* returns a copy.
+		using iterator_category = std::input_iterator_tag;
+		using value_type        = OwnerT;
+		using difference_type   = std::ptrdiff_t;
+		using pointer           = void;
+		using reference         = OwnerT;
+
 		[[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); }
 		const_iterator&      operator++() noexcept {
 			++m_index;

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -488,16 +488,15 @@ void TextureCache::RefreshCopySource(ImageId id) {
 }
 
 bool TextureCache::CopyD16(Image& destination, Image& source) {
-	const bool source_depth      = source.info.IsDepth();
-	const bool destination_depth = destination.info.IsDepth();
-	if (source_depth == destination_depth) {
+	// Converts guest D16 between a four-byte host depth aspect and two-byte color or D16 depth.
+	const bool source_wide = DepthAspectTransferBytes(source.backing.format) == sizeof(uint32_t);
+	if (source_wide == (DepthAspectTransferBytes(destination.backing.format) == sizeof(uint32_t))) {
 		return false;
 	}
-	auto&      depth          = source_depth ? source : destination;
-	auto&      color          = source_depth ? destination : source;
-	const auto transfer_bytes = DepthAspectTransferBytes(depth.backing.format);
+	auto& depth  = source_wide ? source : destination;
+	auto& narrow = source_wide ? destination : source;
 	if (depth.info.bytes_per_block != sizeof(uint16_t) ||
-	    color.info.bytes_per_block != sizeof(uint16_t) || transfer_bytes != sizeof(uint32_t)) {
+	    narrow.info.bytes_per_block != sizeof(uint16_t)) {
 		return false;
 	}
 	EXIT_IF(source.backing.samples != 1 || destination.backing.samples != 1 ||
@@ -507,42 +506,42 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {
 
 	const auto     layers = depth.info.resources.layers;
 	const uint64_t depth_slice =
-	    static_cast<uint64_t>(depth.info.pitch) * depth.info.extent.height * transfer_bytes;
-	const uint64_t color_slice =
-	    static_cast<uint64_t>(color.info.pitch) * color.info.extent.height * sizeof(uint16_t);
-	EXIT_IF(layers == 0 || depth_slice > UINT64_MAX / layers || color_slice > UINT64_MAX / layers);
-	const auto                       depth_size = depth_slice * layers;
-	const auto                       color_size = color_slice * layers;
-	const std::array                 depth_copies {
+	    static_cast<uint64_t>(depth.info.pitch) * depth.info.extent.height * sizeof(uint32_t);
+	const uint64_t narrow_slice =
+	    static_cast<uint64_t>(narrow.info.pitch) * narrow.info.extent.height * sizeof(uint16_t);
+	EXIT_IF(layers == 0 || depth_slice > UINT64_MAX / layers || narrow_slice > UINT64_MAX / layers);
+	const std::array depth_copies {
 	    vk::BufferImageCopy {0,
 	                         depth.info.pitch,
 	                         depth.info.extent.height,
 	                         {vk::ImageAspectFlagBits::eDepth, 0, 0, layers},
 	                         {},
 	                         depth.info.extent}};
-	const std::array color_copies {
+	const std::array narrow_copies {
 	    vk::BufferImageCopy {0,
-	                         color.info.pitch,
-	                         color.info.extent.height,
-	                         {vk::ImageAspectFlagBits::eColor, 0, 0, layers},
+	                         narrow.info.pitch,
+	                         narrow.info.extent.height,
+	                         {narrow.info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
+	                                                : vk::ImageAspectFlagBits::eColor,
+	                          0, 0, layers},
 	                         {},
-	                         color.info.extent}};
+	                         narrow.info.extent}};
 
-	auto                         depth_buffer = m_tiler.GetScratchBuffer(depth_size);
-	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer);
+	auto depth_buffer  = m_tiler.GetScratchBuffer(depth_slice * layers);
+	auto narrow_buffer = m_tiler.GetScratchBuffer(narrow_slice * layers, depth_buffer.buffer);
 	const TileManager::D16Layout promote_layout {
 	    .width               = depth.info.extent.width,
 	    .height              = depth.info.extent.height,
 	    .layers              = layers,
-	    .source_row_stride   = static_cast<uint64_t>(color.info.pitch) * sizeof(uint16_t),
-	    .target_row_stride   = static_cast<uint64_t>(depth.info.pitch) * transfer_bytes,
-	    .source_slice_stride = color_slice,
+	    .source_row_stride   = static_cast<uint64_t>(narrow.info.pitch) * sizeof(uint16_t),
+	    .target_row_stride   = static_cast<uint64_t>(depth.info.pitch) * sizeof(uint32_t),
+	    .source_slice_stride = narrow_slice,
 	    .target_slice_stride = depth_slice,
 	};
 	const bool d32 = DepthAspectTransferFormat(depth.backing.format) == vk::Format::eD32Sfloat;
-	if (source_depth) {
+	if (source_wide) {
 		source.Download(depth_copies, depth_buffer.buffer, depth_buffer.offset, depth_buffer.size);
-		m_tiler.ConvertD16(depth_buffer, color_buffer, TileManager::D16Direction::Demote, d32,
+		m_tiler.ConvertD16(depth_buffer, narrow_buffer, TileManager::D16Direction::Demote, d32,
 		                   {.width               = promote_layout.width,
 		                    .height              = promote_layout.height,
 		                    .layers              = promote_layout.layers,
@@ -550,11 +549,12 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {
 		                    .target_row_stride   = promote_layout.source_row_stride,
 		                    .source_slice_stride = promote_layout.target_slice_stride,
 		                    .target_slice_stride = promote_layout.source_slice_stride});
-		destination.Upload(color_copies, color_buffer.buffer, color_buffer.offset,
-		                   color_buffer.size);
+		destination.Upload(narrow_copies, narrow_buffer.buffer, narrow_buffer.offset,
+		                   narrow_buffer.size);
 	} else {
-		source.Download(color_copies, color_buffer.buffer, color_buffer.offset, color_buffer.size);
-		m_tiler.ConvertD16(color_buffer, depth_buffer, TileManager::D16Direction::Promote, d32,
+		source.Download(narrow_copies, narrow_buffer.buffer, narrow_buffer.offset,
+		                narrow_buffer.size);
+		m_tiler.ConvertD16(narrow_buffer, depth_buffer, TileManager::D16Direction::Promote, d32,
 		                   promote_layout);
 		destination.Upload(depth_copies, depth_buffer.buffer, depth_buffer.offset,
 		                   depth_buffer.size);
@@ -783,7 +783,15 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 	if (!cached.registered) {
 		return {merged_id};
 	}
-	if (BcAlias::Matches(requested, cached.info) && !cached.IsCpuDirty()) {
+	// PS5: image heaps keep BC images and raw block views of their layers resident together, so a
+	// raw read of a layer subset with the same levels views the BC image instead of evicting it.
+	const bool raw_view = binding == BindingType::Texture && !requested.IsBlock() &&
+	                      requested.type == cached.info.type &&
+	                      requested.data.address == cached.info.data.address &&
+	                      requested.BlockExtent() == cached.info.BlockExtent() &&
+	                      requested.resources.levels == cached.info.resources.levels &&
+	                      requested.resources.layers < cached.info.resources.layers;
+	if (!raw_view && BcAlias::Matches(requested, cached.info) && !cached.IsCpuDirty()) {
 		BcAlias::Resolve(*this, cached_id, merged_id);
 		return {merged_id};
 	}
@@ -1302,6 +1310,19 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 					view_layer = overlap.layer;
 				}
 			}
+			// PS5: image heaps hold a descriptor per dynamic-resolution extent of a render target,
+			// and only one extent at an address holds data, so heap lookups create no second one.
+			if (!result && desc.heap &&
+			    std::any_of(candidates.begin(), candidates.end(), [&](ImageId id) {
+				    const auto* image = m_slot_images.try_get(id);
+				    return image != nullptr && image->registered &&
+					       image->heap_refs.load(std::memory_order_relaxed) != 0u &&
+					       image->info.data.address == desc.info.data.address &&
+					       image->info.type == desc.info.type &&
+					       image->info.extent != desc.info.extent;
+			    })) {
+				return {};
+			}
 		} else if (block_alias) {
 			for (const auto candidate: candidates) {
 				const auto& owner = m_slot_images[candidate];

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -43,6 +43,7 @@ public:
 		ImageInfo     info;
 		ImageViewInfo view_info;
 		BindingType   type = BindingType::Texture;
+		bool          heap = false; // PS5: resolved for a resident image heap.
 	};
 
 	TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,

src/graphics/host_gpu/renderer/dynamicState.cpp

@@ -125,6 +125,14 @@ void DynamicState::Commit(const GraphicContext& graphics, vk::CommandBuffer cmdb
 			}
 		}
 	}
+	if (dirty_state.cull_mode) {
+		dirty_state.cull_mode = false;
+		cmdbuf.setCullMode(cull_mode);
+	}
+	if (dirty_state.front_face) {
+		dirty_state.front_face = false;
+		cmdbuf.setFrontFace(front_face);
+	}
 	if (dirty_state.blend_constants) {
 		dirty_state.blend_constants = false;
 		cmdbuf.setBlendConstants(blend_constants.data());

src/graphics/host_gpu/renderer/dynamicState.h

@@ -52,6 +52,9 @@ struct DynamicState {
 		bool stencil_back_write_mask    : 1;
 		bool stencil_back_compare_mask  : 1;
 
+		bool cull_mode  : 1;
+		bool front_face : 1;
+
 		bool blend_constants       : 1;
 		bool color_write_enables   : 1;
 		bool line_width            : 1;
@@ -86,6 +89,9 @@ struct DynamicState {
 	uint32_t   stencil_back_write_mask {};
 	uint32_t   stencil_back_compare_mask {};
 
+	vk::CullModeFlags cull_mode {};
+	vk::FrontFace     front_face {};
+
 	std::array<float, 4>                               blend_constants {};
 	std::array<vk::Bool32, RENDER_COLOR_ATTACHMENTS_MAX> color_write_enables {};
 	uint32_t                                           num_color_write_enables {};
@@ -221,6 +227,20 @@ struct DynamicState {
 		}
 	}
 
+	void SetCullMode(const vk::CullModeFlags cull_mode_) {
+		if (cull_mode != cull_mode_) {
+			cull_mode             = cull_mode_;
+			dirty_state.cull_mode = true;
+		}
+	}
+
+	void SetFrontFace(const vk::FrontFace front_face_) {
+		if (front_face != front_face_) {
+			front_face             = front_face_;
+			dirty_state.front_face = true;
+		}
+	}
+
 	void SetBlendConstants(const std::array<float, 4> blend_constants_) {
 		if (blend_constants != blend_constants_) {
 			blend_constants             = blend_constants_;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -425,15 +425,15 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
 	const auto destination_height =
 	    std::max(backing.extent.height >> destination_range.base_level, 1u);
 	const bool copy = source.backing.samples == 1;
-	EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height ||
-	        info.extent.width > destination_width || info.extent.height > destination_height ||
+	EXIT_IF(layers == 0 ||
 	        (copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format)
 	              : source.backing.format != backing.format));
 	auto resolved_source_range             = source_range;
 	auto resolved_destination_range        = destination_range;
 	resolved_source_range.layer_count      = layers;
 	resolved_destination_range.layer_count = layers;
-	const vk::Extent3D resolve_extent {info.extent.width, info.extent.height, 1};
+	const vk::Extent3D resolve_extent {std::min(source_width, destination_width),
+	                                   std::min(source_height, destination_height), 1};
 
 	m_scheduler.EndRendering();
 	auto command = m_scheduler.Current().Handle();

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -17,9 +17,15 @@
 
 namespace Libs::Graphics {
 
-enum class ColorTransform : uint32_t { None, SwapBgra16, Reverse10_11_11 };
+enum class ColorTransform : uint32_t { None, SwapBgra16, Reverse10_11_11, Rotate2_10_10_10 };
 
-enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported };
+enum class VideoOutCompression : uint8_t {
+	Uncompressed,
+	Dcc256_256_0,
+	Dcc256_64_64,
+	Dcc128_128_128,
+	Unsupported
+};
 
 enum class ImageMetadataKind : uint8_t { None, Htile, Dcc, Cmask };
 
@@ -99,6 +105,8 @@ struct ImageInfo {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k10_11_11Float:
 				return ColorTransform::Reverse10_11_11;
+			case Prospero::BufferFormat::k2_10_10_10UNorm:
+				return ColorTransform::Rotate2_10_10_10;
 			default: return ColorTransform::None;
 		}
 	}
@@ -287,8 +295,9 @@ inline bool ImageInfo::IsDepth() const noexcept {
 [[nodiscard]] inline VideoOutCompression
 ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t dcc_control,
                             uint64_t dcc_clear_color) noexcept {
-	constexpr uint32_t DCC_256_256_0 = 0x00000048u;
-	constexpr uint32_t DCC_256_64_64 = 0x00000208u;
+	constexpr uint32_t DCC_256_256_0   = 0x00000048u;
+	constexpr uint32_t DCC_256_64_64   = 0x00000208u;
+	constexpr uint32_t DCC_128_128_128 = 0x00100024u;
 	if (!compressed) {
 		return metadata_address == 0 && dcc_control == 0 && dcc_clear_color == 0
 		           ? VideoOutCompression::Uncompressed
@@ -300,6 +309,7 @@ ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t
 	switch (dcc_control) {
 		case DCC_256_256_0: return VideoOutCompression::Dcc256_256_0;
 		case DCC_256_64_64: return VideoOutCompression::Dcc256_64_64;
+		case DCC_128_128_128: return VideoOutCompression::Dcc128_128_128;
 		default: return VideoOutCompression::Unsupported;
 	}
 }
@@ -378,6 +388,9 @@ inline constexpr std::array<VideoOutFormatPolicy, 8> VIDEO_OUT_FORMAT_POLICIES {
 		const auto encoded = static_cast<float>(value & 0xffu) / 255.0f;
 		return encoded <= 0.04045f ? encoded / 12.92f : std::pow((encoded + 0.055f) / 1.055f, 2.4f);
 	};
+	const auto unorm16 = [](uint64_t value) {
+		return static_cast<float>(value & 0xffffu) / 65535.0f;
+	};
 	const auto float16 = [](uint32_t value) {
 		const auto sign = (value & 0x8000u) << 16u;
 		const auto exponent = (value >> 10u) & 0x1fu;
@@ -403,19 +416,49 @@ inline constexpr std::array<VideoOutFormatPolicy, 8> VIDEO_OUT_FORMAT_POLICIES {
 			next.float32[1] = float16(packed >> 16u);
 			[[fallthrough]];
 		case vk::Format::eR16Sfloat: next.float32[0] = float16(packed); break;
-		case vk::Format::eR16G16Unorm:
-			next.float32[1] = static_cast<float>(packed >> 16u) / 65535.0f;
+		case vk::Format::eR16G16B16A16Unorm:
+			next.float32[2] = unorm16(clear_words >> 32u);
+			next.float32[3] = unorm16(clear_words >> 48u);
 			[[fallthrough]];
-		case vk::Format::eR16Unorm:
-			next.float32[0] = static_cast<float>(packed & 0xffffu) / 65535.0f;
+		case vk::Format::eR16G16Unorm:
+			next.float32[1] = unorm16(packed >> 16u);
+			[[fallthrough]];
+		case vk::Format::eR16Unorm: next.float32[0] = unorm16(packed); break;
+		case vk::Format::eR16G16B16A16Uint:
+			for (uint32_t i = 0; i < 4; i++) {
+				next.uint32[i] = static_cast<uint16_t>(clear_words >> (16u * i));
+			}
 			break;
-		// Each clear register stores one raw 32-bit float channel.
+		case vk::Format::eR16G16B16A16Sint:
+			for (uint32_t i = 0; i < 4; i++) {
+				next.int32[i] = static_cast<int16_t>(clear_words >> (16u * i));
+			}
+			break;
+		// Each clear register stores one raw 32-bit channel.
 		case vk::Format::eR32G32Sfloat:
 			next.float32[1] = std::bit_cast<float>(static_cast<uint32_t>(clear_words >> 32u));
 			[[fallthrough]];
 		case vk::Format::eR32Sfloat: next.float32[0] = std::bit_cast<float>(packed); break;
-		case vk::Format::eR32Uint: next.uint32[0] = packed; break;
-		case vk::Format::eR32Sint: next.int32[0] = static_cast<int32_t>(packed); break;
+		case vk::Format::eR32G32Uint:
+		case vk::Format::eR32G32Sint:
+			next.uint32[1] = static_cast<uint32_t>(clear_words >> 32u);
+			[[fallthrough]];
+		case vk::Format::eR32Uint:
+		case vk::Format::eR32Sint: next.uint32[0] = packed; break;
+		// Narrow targets hold their channels in the low bits of the first word. UFC 6 (PPSA23566)
+		// fast-clears an R8_UINT map through CMASK every frame and then writes only part of it;
+		// without these the clear was dropped and old contents piled up.
+		case vk::Format::eR8Uint: next.uint32[0] = packed & 0xffu; break;
+		case vk::Format::eR8Unorm: next.float32[0] = unorm8(packed); break;
+		case vk::Format::eR8G8Uint:
+			next.uint32[0] = packed & 0xffu;
+			next.uint32[1] = (packed >> 8u) & 0xffu;
+			break;
+		case vk::Format::eR8G8Unorm:
+			next.float32[0] = unorm8(packed);
+			next.float32[1] = unorm8(packed >> 8u);
+			break;
+		case vk::Format::eR16Uint: next.uint32[0] = packed & 0xffffu; break;
 		case vk::Format::eR8G8B8A8Srgb:
 			next.float32[0] = srgb8(packed);
 			next.float32[1] = srgb8(packed >> 8u);
@@ -462,10 +505,16 @@ inline constexpr std::array<VideoOutFormatPolicy, 8> VIDEO_OUT_FORMAT_POLICIES {
 // Keep this separate: the first register word alone cannot describe a 64-bit color.
 [[nodiscard]] inline bool DecodeColorDwordFill(vk::Format format, uint32_t packed,
                                                vk::ClearColorValue& clear) {
-	if ((format == vk::Format::eR16Sfloat || format == vk::Format::eR16Unorm) &&
+	if ((format == vk::Format::eR16Sfloat || format == vk::Format::eR16Unorm ||
+	     format == vk::Format::eR16Uint || format == vk::Format::eR8G8Unorm ||
+	     format == vk::Format::eR8G8Uint) &&
 	    (packed & 0xffffu) != (packed >> 16u)) {
 		return false;
 	}
+	if ((format == vk::Format::eR8Unorm || format == vk::Format::eR8Uint) &&
+	    packed != (packed & 0xffu) * 0x01010101u) {
+		return false;
+	}
 	if (format == vk::Format::eR16G16B16A16Sfloat) {
 		if (packed != 0) {
 			return false;

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -32,8 +32,12 @@ struct HostFormatInfo {
 
 HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
                                  Prospero::ChannelOrder order) {
-	if (guest_format == Prospero::BufferFormat::k10_11_11Float) {
-		return {vk::Format::eB10G11R11UfloatPack32, Prospero::ColorMappingBgra};
+	switch (guest_format) {
+		case Prospero::BufferFormat::k10_11_11Float:
+			return {vk::Format::eB10G11R11UfloatPack32, Prospero::ColorMappingBgra};
+		case Prospero::BufferFormat::k2_10_10_10UNorm:
+			return {vk::Format::eA2R10G10B10UnormPack32, Prospero::ColorMappingAbgr};
+		default: break;
 	}
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {

src/graphics/host_gpu/renderer/image/tiler.cpp

@@ -104,7 +104,8 @@ TileManager::Tiling TileManager::Prepare(bool tile, uint64_t tiled_capacity,
 	    block.block_depth == 0 || block.block_size == 0 || info.tiling.texel_width == 0 ||
 	    info.tiling.texel_height == 0 || info.tiled_slice_stride == 0);
 	EXIT_IF((transform == ColorTransform::SwapBgra16 && block.bytes_per_element != 8u) ||
-	        (transform == ColorTransform::Reverse10_11_11 && block.bytes_per_element != 4u));
+	        ((transform == ColorTransform::Reverse10_11_11 ||
+	          transform == ColorTransform::Rotate2_10_10_10) && block.bytes_per_element != 4u));
 	const auto checked_multiply = [](uint64_t left, uint64_t right, uint64_t& result) {
 		return (left == 0 || right <= UINT64_MAX / left) && (result = left * right, true);
 	};
@@ -484,7 +485,7 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	                            vk::AccessFlagBits::eHostWrite |
 	                            vk::AccessFlagBits::eTransferWrite |
 	                            vk::AccessFlagBits::eShaderWrite;
-	barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
+	barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderWrite;
 	command.pipelineBarrier(
 	    vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost,
 	    vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 2, barriers, 0, nullptr);
@@ -560,7 +561,8 @@ void TileManager::TransformColor(Result input, Result output, ColorTransform tra
 	uint32_t element_bytes = 0;
 	switch (transform) {
 		case ColorTransform::SwapBgra16: element_bytes = 8; break;
-		case ColorTransform::Reverse10_11_11: element_bytes = 4; break;
+		case ColorTransform::Reverse10_11_11:
+		case ColorTransform::Rotate2_10_10_10: element_bytes = 4; break;
 		default: EXIT("invalid color transform\n");
 	}
 	EXIT_IF(input.size == 0 || input.size % element_bytes != 0 ||

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -604,8 +604,13 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
 	                                 view_levels, desc.info.resources.layers);
 	desc.view_info.base_level = view_base;
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
+	desc.heap = resource.heap;
 
-	auto       id                  = texture_cache.FindImage(desc, shader_conversion);
+	auto id = texture_cache.FindImage(desc, shader_conversion);
+	if (!id) {
+		// Another extent at the address holds it; the heap retries once an image is created.
+		return {.desc = std::move(desc), .layout = vk::ImageLayout::eGeneral};
+	}
 	auto*      image               = &texture_cache.GetImage(id);
 	const bool stencil_association = static_cast<bool>(image->depth_id);
 	if (stencil_association) {
@@ -801,6 +806,9 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 		prepared.buffers[index] = NativeStorageBuffer(m_context, prepared.buffer_sources[index],
 		                                              resource, buffer_offset);
 		pack_memory_offset(index, buffer_offset);
+		// Null bindings allow 0 bytes, not the shared null buffer's 16, so no store reaches it.
+		prepared.shader_data[layout.MemorySizeDword() + index] =
+		    static_cast<uint32_t>(prepared.buffer_sources[index].size + buffer_offset);
 	}
 	if (layout.descriptor_counts[static_cast<size_t>(BindingKind::FlattenedSrt)] != 0) {
 		prepared.flattened_srt = NativeUpload(m_context, snapshot.flattened_srt);

src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp

@@ -12,6 +12,7 @@
 #include "kernel/memory.h"
 
 #include <algorithm>
+#include <bit>
 #include <xxhash.h>
 
 namespace Libs::Graphics {
@@ -19,7 +20,7 @@ namespace {
 
 namespace IR = ShaderRecompiler::IR;
 
-// Smaller tables are cheap to materialize on each draw and are often transient.
+// Smaller tables are often written for one draw, so they enter on a second draw of a frame.
 constexpr uint64_t MinTableBytes = 64ull * 1024ull;
 constexpr uint32_t ElementMask   = (1u << IR::ImageHeapKindShift) - 1u;
 
@@ -44,6 +45,20 @@ void Widen(size_t& begin, size_t& end, size_t first, size_t last) {
 	end              = empty ? last : std::max(end, last);
 }
 
+// Probes at first + k * step whose first dword lies within a table of size bytes.
+uint64_t Probes(uint64_t size, uint64_t first, uint64_t step) {
+	return size >= first + 4u ? (size - first - 4u) / step + 1u : 0u;
+}
+
+// Tables grow a little on most frames, so their slot words get room to double.
+std::unique_ptr<Buffer> SlotBuffer(RenderContext& context, uint64_t size) {
+	return std::make_unique<Buffer>(
+	    context.GetGraphics(), context.GetCommandScheduler(), MemoryUsage::DeviceLocal, 0,
+	    vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer |
+	        vk::BufferUsageFlagBits::eShaderDeviceAddress,
+	    std::bit_ceil(size + 4u));
+}
+
 // Guest bytes of a table that its tracker pages [begin, end) cover.
 std::pair<uint64_t, uint64_t> PageRange(uint64_t base, uint64_t size, size_t begin, size_t end) {
 	const auto origin = Common::AlignDown(base, TRACKER_PAGE_SIZE);
@@ -118,26 +133,25 @@ ImageHeap::~ImageHeap() {
 bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) {
 	KYTY_PROFILER_FUNCTION();
 	const auto size = request.size & ~uint64_t {3};
-	if (size < MinTableBytes || request.count == 0u || request.step == 0u ||
+	if (request.count == 0u || request.step == 0u ||
 	    ((request.base | request.first | request.step) & 3u) != 0u) {
 		return false;
 	}
-	auto memory = std::ranges::find_if(m_memories, [&](const auto& candidate) {
-		return candidate->base == request.base && candidate->size == size;
-	});
-	if (memory == m_memories.end()) {
+	auto memory = std::ranges::find_if(
+	    m_memories, [&](const auto& candidate) { return candidate->base == request.base; });
+	if (memory == m_memories.end() && size < MinTableBytes &&
+	    m_sightings.try_emplace(request.base, m_draw).first->second == m_draw) {
+		return false;
+	}
+	// A table follows its descriptor's size, as the first program of a draw acquires it.
+	if (memory == m_memories.end() || ((*memory)->size != size && (*memory)->draw != m_draw)) {
 		if (!m_context.IsMapped(request.base, size)) return false;
-		auto created   = std::make_unique<Memory>();
-		created->base  = request.base;
-		created->size  = size;
-		created->words.resize(size / 4u);
-		created->dead.resize(size / 4u);
-		created->dirty_pages.assign((Common::AlignUp(request.base + size, TRACKER_PAGE_SIZE) -
-		                             Common::AlignDown(request.base, TRACKER_PAGE_SIZE)) /
-		                                TRACKER_PAGE_SIZE,
-		                            uint8_t {1});
-		std::scoped_lock lock {m_mutex};
-		memory = m_memories.insert(m_memories.end(), std::move(created));
+		if (memory == m_memories.end()) {
+			std::scoped_lock lock {m_mutex};
+			memory = m_memories.insert(m_memories.end(),
+			                           std::make_unique<Memory>(Memory {.base = request.base}));
+		}
+		Resize(**memory, size);
 	}
 	// A draw refreshes a table and a view takes its changes once, so all programs of the draw see
 	// one snapshot and one set of kinds.
@@ -146,27 +160,28 @@ bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView&
 		(*memory)->draw = m_draw;
 	}
 	auto&      views   = (*memory)->views;
+	const auto bytes   = (*memory)->size;
 	const auto profile = IR::ImageHeapResource(*request.root, 0u);
+	const auto probes  = Probes(size, request.first, request.step);
+	const auto limit   = request.count < probes ? request.count : UINT64_MAX;
 	auto       view    = std::ranges::find_if(views, [&](const auto& candidate) {
 		return candidate->step == request.step && candidate->first == request.first &&
-		       candidate->count == request.count && candidate->profile == profile;
+		       candidate->limit == limit && candidate->profile == profile;
 	});
 	if (view == views.end()) {
 		const auto epoch   = Image::heap_epoch.load(std::memory_order_relaxed);
+		const auto count   = std::min(limit, Probes(bytes, request.first, request.step));
 		auto       created = std::make_unique<View>(View {.memory        = **memory,
 		                                                  .step          = request.step,
 		                                                  .first         = request.first,
-		                                                  .count         = request.count,
+		                                                  .count         = count,
+		                                                  .limit         = limit,
 		                                                  .profile       = profile,
 		                                                  .resolve_epoch = epoch,
 		                                                  .layout_epoch  = epoch});
-		created->slots.resize(size / 4u + 1u);
-		created->buffer = std::make_unique<Buffer>(
-		    m_context.GetGraphics(), m_context.GetCommandScheduler(), MemoryUsage::DeviceLocal, 0,
-		    vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer |
-		        vk::BufferUsageFlagBits::eShaderDeviceAddress,
-		    size + 4u);
-		UpdateProbes(*created, 0u, size);
+		created->slots.resize(bytes / 4u + 1u);
+		created->buffer = SlotBuffer(m_context, bytes);
+		UpdateProbes(*created, 0u, bytes);
 		// The first upload also clears the slots no probe selects.
 		created->dirty_begin = 0;
 		created->dirty_end   = created->slots.size();
@@ -254,10 +269,12 @@ void ImageHeap::Unwatch(const Memory& memory) {
 void ImageHeap::InvalidateMemory(uint64_t address, uint64_t size) {
 	std::scoped_lock lock {m_mutex};
 	for (auto& memory: m_memories) {
-		const auto begin = std::max(address, memory->base);
-		const auto end   = std::min(address + size, memory->base + memory->size);
-		if (begin >= end) continue;
+		// Watches cover whole pages, so a write next to the table must unwatch its page too.
 		const auto origin = Common::AlignDown(memory->base, TRACKER_PAGE_SIZE);
+		const auto begin  = std::max(address, origin);
+		const auto end    = std::min(address + size,
+		                             origin + memory->dirty_pages.size() * TRACKER_PAGE_SIZE);
+		if (begin >= end) continue;
 		MarkDirty(*memory, (begin - origin) / TRACKER_PAGE_SIZE,
 		          (end - 1u - origin) / TRACKER_PAGE_SIZE + 1u);
 	}
@@ -381,15 +398,56 @@ uint32_t ImageHeap::Allocate() {
 }
 
 void ImageHeap::Retire(View& view) {
+	if (view.released.empty()) return;
+	// A retired index is reused once its tick ends, so no held entry may keep it.
+	std::erase_if(view.held, [&](uint32_t index) { return m_elements[index].refs == 0u; });
 	const auto tick = m_context.GetCommandScheduler().CurrentTick();
 	for (const auto index: view.released) m_retired.emplace_back(tick, index);
 	view.released.clear();
 }
 
+void ImageHeap::Resize(Memory& memory, uint64_t size) {
+	{
+		std::scoped_lock lock {m_mutex};
+		Unwatch(memory);
+		memory.size = size;
+		// The next refresh rereads the whole table.
+		memory.dirty_pages.assign((Common::AlignUp(memory.base + size, TRACKER_PAGE_SIZE) -
+		                           Common::AlignDown(memory.base, TRACKER_PAGE_SIZE)) /
+		                              TRACKER_PAGE_SIZE,
+		                          uint8_t {1});
+	}
+	memory.words.resize(size / 4u);
+	memory.dead.resize(size / 4u);
+	auto& scheduler = m_context.GetCommandScheduler();
+	for (auto& view: memory.views) {
+		const auto count = std::min(view->limit, Probes(size, view->first, view->step));
+		for (auto probe = count; probe < view->count; ++probe) {
+			Assign(*view, (view->first + probe * view->step) / 4u, 0u);
+		}
+		view->count       = count;
+		const auto slots  = view->slots.size();
+		view->slots.resize(size / 4u + 1u);
+		view->dirty_end   = std::min(view->dirty_end, view->slots.size());
+		view->dirty_begin = std::min(view->dirty_begin, view->dirty_end);
+		if (view->slots.size() * sizeof(uint32_t) > view->buffer->Size()) {
+			if (scheduler.Active()) {
+				scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
+			}
+			view->buffer = SlotBuffer(m_context, size);
+			Widen(view->dirty_begin, view->dirty_end, 0u, view->slots.size());
+		} else if (slots < view->slots.size()) {
+			// Device words past the old end may hold stale slots.
+			Widen(view->dirty_begin, view->dirty_end, slots, view->slots.size());
+		}
+	}
+}
+
 void ImageHeap::Rebind(RenderExecutor& executor) {
 	KYTY_PROFILER_FUNCTION();
 	auto&      textures = m_context.GetTextureCache();
 	const auto epoch    = Image::heap_epoch.load(std::memory_order_relaxed);
+	const auto serial   = Image::serials.load(std::memory_order_relaxed);
 	for (auto* view: m_active) {
 		bool       stale = false;
 		const auto mark  = [&](uint32_t index) { m_elements[index].stale = stale = true; };
@@ -418,6 +476,19 @@ void ImageHeap::Rebind(RenderExecutor& executor) {
 			mark(index);
 			return true;
 		});
+		// Held elements retry once an image is created, such as a render target of their extent.
+		if (view->held_serial != serial) {
+			view->held_serial = serial;
+			std::erase_if(view->held, [&](uint32_t index) {
+				const auto& element = m_elements[index];
+				if (element.refs == 0u || element.stale) return true;
+				const auto binding = executor.ResolveTexture(
+				    IR::ImageHeapResource(view->profile, element.kind), element.value);
+				if (binding.image_id) mark(index);
+				// An unmapped texture stays null.
+				return binding.image_id || binding.desc.info.data.Empty();
+			});
+		}
 		if (stale) {
 			// A changed image needs a new element; pending draws still read the old one. The new
 			// element keeps the old value, since a view takes table changes only when acquired.
@@ -446,6 +517,8 @@ void ImageHeap::Rebind(RenderExecutor& executor) {
 					view->dcc.push_back(index);
 				}
 				view->away.push_back(index);
+			} else if (!binding.desc.info.data.Empty()) {
+				view->held.push_back(index);
 			}
 			m_elements[index].binding = std::move(binding);
 			m_writes.push_back(index);
@@ -524,26 +597,42 @@ void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) {
 	} else {
 		m_unmapped.push_back({address, size});
 	}
-	auto&           scheduler = m_context.GetCommandScheduler();
 	std::scoped_lock lock {m_mutex};
 	std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
 		if (address >= memory->base + memory->size || memory->base >= address + size) {
 			return false;
 		}
-		Unwatch(*memory);
-		for (auto& view: memory->views) {
-			std::erase(m_active, view.get());
-			for (const auto& [value, index]: view->elements) {
-				Unreference(m_elements[index]);
-				view->released.push_back(index);
-			}
-			Retire(*view);
-			if (scheduler.Active()) {
-				scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
-			}
-		}
+		Drop(*memory);
 		return true;
 	});
 }
 
+void ImageHeap::RunGarbageCollector() {
+	// A smaller table rebuilt each frame moves to a new base. Larger ones keep their dead words.
+	std::scoped_lock lock {m_mutex};
+	std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
+		if (memory->size >= MinTableBytes || memory->draw > m_suspend_draw) return false;
+		Drop(*memory);
+		return true;
+	});
+	m_suspend_draw = m_draw;
+	m_sightings.clear();
+}
+
+void ImageHeap::Drop(Memory& memory) {
+	auto& scheduler = m_context.GetCommandScheduler();
+	Unwatch(memory);
+	for (auto& view: memory.views) {
+		std::erase(m_active, view.get());
+		for (const auto& [value, index]: view->elements) {
+			Unreference(m_elements[index]);
+			view->released.push_back(index);
+		}
+		Retire(*view);
+		if (scheduler.Active()) {
+			scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
+		}
+	}
+}
+
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/imageHeap.h

@@ -22,10 +22,11 @@ class RenderExecutor;
 struct ImageHeapTestAccess;
 
 // Guest image tables that shaders index by GPU-computed offsets, kept resident in one bindless
-// sampled-image array. Each view of a table owns device slot words, one per table dword, that
-// select an array element. CPU writes to a table fault in its snapshot, which each draw refreshes
-// once; a view takes the changes when a draw first acquires it. Changes to the images behind
-// elements bump Image::heap_epoch.
+// sampled-image array. A table is keyed by its base and follows its descriptor's size. Each view
+// of a table owns device slot words, one per table dword, that select an array element. CPU
+// writes to a table fault in its snapshot, which each draw refreshes once; a view takes the
+// changes when a draw first acquires it. Changes to the images behind elements bump
+// Image::heap_epoch. A smaller table leaves at a suspend point if the frame did not acquire it.
 class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource {
 public:
 	ImageHeap(RenderContext& context, PageManager& page_manager);
@@ -46,6 +47,8 @@ public:
 	// CPU write fault, on any thread.
 	void InvalidateMemory(uint64_t address, uint64_t size);
 	void UnmapMemory(uint64_t address, uint64_t size);
+	// At a suspend point.
+	void RunGarbageCollector();
 
 private:
 	using DescriptorValue = ShaderRecompiler::IR::DescriptorValue;
@@ -70,6 +73,7 @@ private:
 		uint64_t                            step  = 0;
 		uint64_t                            first = 0;
 		uint64_t                            count = 0;
+		uint64_t                            limit = 0; // Probes a key domain allows, if fewer.
 		ShaderRecompiler::IR::ImageResource profile;
 		std::vector<uint32_t>               slots; // Plus a trailing null slot.
 		std::vector<uint32_t>               kind_keys;
@@ -79,6 +83,7 @@ private:
 		std::vector<uint32_t>   released; // No slot selects them after the next upload.
 		std::vector<uint32_t>   away;     // Not yet moved to the read layout.
 		std::vector<uint32_t>   dcc;      // Color metadata clears apply on each discovery.
+		std::vector<uint32_t>   held;     // Another extent holds their address.
 		std::unique_ptr<Buffer> buffer;
 		size_t                  dirty_begin   = 0; // Slots to upload.
 		size_t                  dirty_end     = 0;
@@ -86,6 +91,7 @@ private:
 		size_t                  changed_end   = 0;
 		uint64_t                resolve_epoch = 0;
 		uint64_t                layout_epoch  = 0;
+		uint64_t                held_serial   = 0; // Image::serials when held was last retried.
 	};
 
 	struct Memory {
@@ -103,6 +109,7 @@ private:
 	template <bool track>
 	void     Watch(const Memory& memory, size_t begin, size_t end);
 	void     Unwatch(const Memory& memory);
+	void     Drop(Memory& memory);
 	void     UpdateProbes(View& view, uint64_t begin, uint64_t end);
 	void     UpdateProbe(View& view, uint64_t probe);
 	void     Assign(View& view, size_t slot, uint32_t entry);
@@ -113,6 +120,7 @@ private:
 	Image*   ImageOf(const Element& element) const;
 	uint32_t Allocate();
 	void     Retire(View& view);
+	void     Resize(Memory& memory, uint64_t size);
 	void     WriteDescriptors();
 
 	RenderContext&                       m_context;
@@ -120,9 +128,11 @@ private:
 	std::mutex                           m_mutex; // Fault-visible Memory state and m_memories.
 	std::vector<std::unique_ptr<Memory>> m_memories;
 	std::vector<View*>                   m_active; // Acquired since the last commit.
-	uint64_t                             m_draw = 1;
+	uint64_t                             m_draw         = 1;
+	uint64_t                             m_suspend_draw = 0; // m_draw at the last suspend point.
 	std::vector<Element>                 m_elements;
 	std::vector<uint32_t>                m_free;
+	std::unordered_map<uint64_t, uint64_t>    m_sightings; // Smaller tables: base, first draw.
 	std::deque<std::pair<uint64_t, uint32_t>> m_retired; // Tick, element.
 	std::vector<std::pair<size_t, size_t>>    m_runs;
 	std::vector<uint32_t>                     m_scratch;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -687,6 +687,10 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 		vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]);
 	}
 	const bool mesh_active = vertex_info[0].logical_stage == ShaderType::Mesh;
+	if (!tess_active && !mesh_active) {
+		vertex_info[0].indirect_offsets = state.indirect_offsets;
+		vertex_info[0].indirect_draw    = state.indirect_draw;
+	}
 	if (object_primitive_id) {
 		// CxObjectIdMode: register source plus a 32-bit pipelined primitive-ID addition.
 		EXIT_NOT_IMPLEMENTED(sh.m_paClObjprimIdCntl != 2u || mesh_active || tess_active ||
@@ -711,8 +715,13 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 		    mesh.max_vertices > limits.maxMeshOutputVertices ||
 		    mesh.max_primitives > limits.maxMeshOutputPrimitives ||
 		    mesh.lds_size_dwords * sizeof(uint32_t) > limits.maxMeshSharedMemorySize) {
-			EXIT("mesh shader exceeds host limits: threads=%u vertices=%u primitives=%u LDS=%u\n",
-			     host_threads, mesh.max_vertices, mesh.max_primitives, mesh.lds_size_dwords);
+			static std::atomic_bool warned = false;
+			if (!warned.exchange(true, std::memory_order_relaxed)) {
+				Log::WriteToConsoleAndLog(fmt::format(
+				    "Warning: dropping mesh draws the host does not support (shader 0x{:016x}).\n",
+				    vertex_params[0].hash));
+			}
+			return;
 		}
 	}
 	ShaderParams pixel_params;
@@ -736,6 +745,11 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 			const auto& rt = context.GetRenderTarget(slot);
 			const auto target_mask = render_target_mask_slot(context.GetRenderTargetMask(), slot);
 			if (rt.base.addr == 0 || target_mask == 0u) continue;
+			if (rt.info.channel_type == Prospero::ChannelType::kUInt) {
+				pixel_info.target_uint_mask |= 1u << export_index;
+			} else if (rt.info.channel_type == Prospero::ChannelType::kSInt) {
+				pixel_info.target_sint_mask |= 1u << export_index;
+			}
 			const auto format = Prospero::ResolveRenderTargetFormat(rt.info.format, rt.info.channel_type);
 			if (format.buffer_format != Prospero::BufferFormat::k11_11_10UNorm) continue;
 			const auto mapping = pixel_info.target_export_mapping[export_index];
@@ -767,9 +781,11 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 		     (blend.separate_alpha_blend && (BlendFactorIsDualSource(blend.alpha_srcblend) ||
 		                                     BlendFactorIsDualSource(blend.alpha_destblend))));
 		if (pixel_info.dual_source_blending) {
-			// MRT1 supplies the second blend source for target 0.
+			// MRT1 supplies the second blend source for target 0; blended targets are not integer.
 			pixel_info.target_output_mode[1]    = pixel_info.target_output_mode[0];
 			pixel_info.target_export_mapping[1] = pixel_info.target_export_mapping[0];
+			pixel_info.target_uint_mask &= ~2u;
+			pixel_info.target_sint_mask &= ~2u;
 		} else if (blend.enable && !context.GetRenderTarget(0).info.blend_bypass &&
 		           pixel_info.target_output_mode[0] != 0 && pixel_info.target_output_mode[0] != 7 &&
 		           std::all_of(std::begin(pixel_info.target_output_mode) + 1,
@@ -991,15 +1007,11 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer&
 	}
 	static_params.alpha_to_coverage_enable = ps_active && ctx.GetShaderRegisters().AlphaToMask();
 	const bool rect_list = Prospero::IsRectList(command.GetUserConfig().GetPrimType());
-	static_params.cull_back  = !rect_list && mc.cull_back;
-	static_params.cull_front = !rect_list && mc.cull_front;
-	static_params.face       = mc.face;
 	static_params.provoking_vtx_last = mc.provoking_vtx_last;
 	static_params.polygon_mode =
-	    ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back);
+	    ResolvePolygonMode(mc, !rect_list && mc.cull_front, !rect_list && mc.cull_back);
 	if (tessellation) {
-		static_params.patch_control_points =
-		    static_cast<uint8_t>(vs_input_info.tess.input_control_points);
+		static_params.patch_control_points = vs_input_info.tess.input_control_points;
 	} else if (!mesh && static_params.topology == vk::PrimitiveTopology::ePatchList) {
 		// Rectangle lists expand through generated tessellation stages.
 		static_params.patch_control_points = 3;

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -54,15 +54,12 @@ struct PipelineStaticParameters {
 	bool                       depth_clip_enable        = true;
 	vk::PrimitiveTopology      topology                 = vk::PrimitiveTopology::ePointList;
 	// 0 without tessellation, 3 for a rectangle list, else the input control points.
-	uint8_t                    patch_control_points     = 0;
+	uint32_t                   patch_control_points     = 0;
 	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
 	bool                       alpha_to_coverage_enable = false;
 	uint32_t                   color_mask[RENDER_COLOR_ATTACHMENTS_MAX]           = {};
-	bool                       cull_front                                         = false;
-	bool                       cull_back                                          = false;
-	bool                       face                                               = false;
 	bool                       provoking_vtx_last                                 = false;
 	vk::PolygonMode            polygon_mode                                       = vk::PolygonMode::eFill;
 	uint8_t                    color_srcblend[RENDER_COLOR_ATTACHMENTS_MAX]       = {};
@@ -140,6 +137,9 @@ struct DrawRenderState {
 	bool                                 ps_active                                = false;
 	std::array<ShaderVertexInputInfo, 3> vertex_info;
 	ShaderPixelInputInfo                 ps_input_info;
+	// Input: the user-data slots a GPU-read indirect draw patches.
+	ShaderIndirectOffsets                indirect_offsets;
+	bool                                 indirect_draw = false;
 
 	[[nodiscard]] std::span<const ShaderVertexInputInfo> VertexStages() const {
 		return {vertex_info.data(), vertex_info[1].stage ? 3u : 1u};

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -219,17 +219,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	vk::PipelineViewportStateCreateInfo viewport_state {};
 	viewport_state.pNext = &depth_clip_control;
 
-	vk::CullModeFlags cull_mode = vk::CullModeFlagBits::eNone;
-	if (static_params.cull_back) {
-		cull_mode |= vk::CullModeFlagBits::eBack;
-	}
-	if (static_params.cull_front) {
-		cull_mode |= vk::CullModeFlagBits::eFront;
-	}
-
-	vk::FrontFace front_face =
-	    (static_params.face ? vk::FrontFace::eClockwise : vk::FrontFace::eCounterClockwise);
-
 	vk::PipelineRasterizationDepthClipStateCreateInfoEXT clip_ext {};
 	clip_ext.depthClipEnable = static_params.depth_clip_enable ? VK_TRUE : VK_FALSE;
 
@@ -250,8 +239,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		provoking_vertex.pNext = rasterizer.pNext;
 		rasterizer.pNext = &provoking_vertex;
 	}
-	rasterizer.cullMode  = cull_mode;
-	rasterizer.frontFace = front_face;
 	rasterizer.polygonMode = static_params.polygon_mode;
 	rasterizer.lineWidth = 1.0f;
 
@@ -347,6 +334,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	    vk::DynamicState::eViewportWithCount,
 	    vk::DynamicState::eScissorWithCount,
 	    vk::DynamicState::eLineWidth,
+	    vk::DynamicState::eCullMode,
+	    vk::DynamicState::eFrontFace,
 	    vk::DynamicState::eDepthTestEnable,
 	    vk::DynamicState::eDepthWriteEnable,
 	    vk::DynamicState::eDepthCompareOp,

src/graphics/host_gpu/renderer/render.h

@@ -45,9 +45,10 @@ enum class DrawOffsetSource : uint8_t {
 
 // Nonzero address: the GPU reads count Vulkan indirect draw records, stride bytes apart.
 struct DrawIndirectArgs {
-	uint64_t address = 0;
-	uint32_t count   = 0;
-	uint32_t stride  = 0;
+	uint64_t              address = 0;
+	uint32_t              count   = 0;
+	uint32_t              stride  = 0;
+	ShaderIndirectOffsets offsets;
 };
 
 struct DrawIndexArgs {
@@ -149,15 +150,21 @@ private:
 	friend class CommandScheduler;
 };
 
+namespace Pssr {
+struct Request;
+}
+
 class RenderExecutor {
 public:
 	explicit RenderExecutor(RenderContext& context): m_context(context) {}
+	~RenderExecutor();
 	KYTY_CLASS_NO_COPY(RenderExecutor);
 
 	void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x,
 	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode, bool async_compute);
 	void DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, uint64_t args_addr,
 	                      uint32_t mode, bool async_compute);
+	void ProcessPssr(CommandBuffer& buffer, const Pssr::Request& request);
 
 	void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared);
 	void                           FindBuffers(std::span<PreparedBindings* const> stages);
@@ -218,6 +225,9 @@ private:
 	std::vector<ImageId>                  m_bound_images;
 	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;
 	std::vector<vk::WriteDescriptorSet>   m_descriptor_writes;
+	vk::DescriptorSetLayout               m_pssr_descriptor_layout = nullptr;
+	vk::PipelineLayout                    m_pssr_pipeline_layout   = nullptr;
+	vk::Pipeline                          m_pssr_pipeline          = nullptr;
 
 	friend class CommandProcessor;
 	friend struct RenderExecutorTestAccess;

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -147,6 +147,7 @@ void RenderContext::RunGarbageCollector() {
 		m_fault_process_pending = false;
 		m_buffer_cache.ProcessFaultBuffer();
 	}
+	m_image_heap.RunGarbageCollector();
 	m_texture_cache.ProcessDownloadImages();
 	m_texture_cache.RunGarbageCollector();
 	m_buffer_cache.RunGarbageCollector();

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -52,7 +52,7 @@ std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
 	                                           const ShaderVertexInputInfo& vs_input_info) {
 	auto     vertex_offset   = static_cast<int32_t>(index_offset);
 	uint32_t instance_offset = 0;
-	if (!vs_input_info.fetch_embedded) {
+	if (!vs_input_info.has_dynamic_vertex_fetch) {
 		return {vertex_offset, instance_offset};
 	}
 
@@ -403,6 +403,20 @@ static void UpdateGraphicsDynamicState(const CommandBuffer& buffer, DynamicState
 		dynamic_state.SetDepthBias(constant_factor, poly_offset.clamp, slope_factor);
 	}
 
+	// Rect lists are drawn without culling.
+	vk::CullModeFlags cull_mode {};
+	if (!Prospero::IsRectList(buffer.GetUserConfig().GetPrimType())) {
+		if (mode.cull_front) {
+			cull_mode |= vk::CullModeFlagBits::eFront;
+		}
+		if (mode.cull_back) {
+			cull_mode |= vk::CullModeFlagBits::eBack;
+		}
+	}
+	dynamic_state.SetCullMode(cull_mode);
+	dynamic_state.SetFrontFace(mode.face ? vk::FrontFace::eClockwise
+	                                     : vk::FrontFace::eCounterClockwise);
+
 	dynamic_state.SetStencilTestEnabled(depth.stencil_test_enable);
 	if (depth.stencil_test_enable) {
 		const auto& front = depth.stencil_front;
@@ -843,6 +857,11 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
 		LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
 	}
 	m_context.GetPipelineCache().GetGraphicsPrograms(buffer, state);
+	// Vertex programs beyond the host mesh limits or with unsupported call targets are not
+	// compiled.
+	if (state.vertex_info[0].stage.module == nullptr) {
+		return false;
+	}
 	if (draw.IsIndexed()) {
 		LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
 	}
@@ -1156,7 +1175,6 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 		if (mesh_active) {
 			draw_data = {draw.index_count,
 			             static_cast<uint32_t>(emit.vertex_offset),
-			             emit.first_instance,
 			             index_source.guest_element_size,
 			             static_cast<uint32_t>(index_source.address),
 			             static_cast<uint32_t>(index_source.address >> 32u)};
@@ -1232,24 +1250,36 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	}
 }
 
-// Vulkan offsets for a prepared draw, or nullopt when its indirect records need CPU counts, as
-// mesh and quad-list draws do. Indirect records carry their own offsets.
+// Vulkan offsets for a prepared draw, or nullopt when its indirect records need the CPU: mesh
+// and quad-list draws for counts, tessellation draws for the index/SGPR prolog. Indirect records
+// carry their own offsets.
 static std::optional<DrawEmitInfo> ResolveDrawEmit(const HW::UserConfig&        ucfg,
                                                    const ShaderVertexInputInfo& vs,
                                                    DrawOffsetSource             source,
                                                    int32_t                      first_vertex,
                                                    uint32_t                     first_instance,
                                                    const DrawIndirectArgs&      indirect) {
+	if ((indirect.address != 0 || source == DrawOffsetSource::IndirectArgs) &&
+	    ucfg.GetIndexOffset() != 0) {
+		static std::once_flag warning_once;
+		std::call_once(warning_once, [&] {
+			Log::WriteToConsoleAndLog(fmt::format(
+			    "GPU: indirect draw ignores GE_INDX_OFFSET=0x{:08x}; geometry may be incorrect.\n",
+			    ucfg.GetIndexOffset()));
+		});
+	}
 	if (indirect.address != 0) {
-		if (vs.stage.module->info.stage == ShaderType::Mesh ||
-		    ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy) {
+		if (!vs.indirect_draw ||
+		    vs.stage.module->info.stage == ShaderType::Mesh ||
+		    ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy ||
+		    vs.indirect_offsets != indirect.offsets) {
 			return std::nullopt;
 		}
 		return DrawEmitInfo {.indirect = indirect};
 	}
 	if (source == DrawOffsetSource::IndirectArgs) {
 		// Native fetches consume the patched SGPRs; rewritten fetches need Vulkan offsets.
-		return vs.fetch_embedded ? DrawEmitInfo {first_vertex, first_instance} : DrawEmitInfo {};
+		return vs.has_dynamic_vertex_fetch ? DrawEmitInfo {first_vertex, first_instance} : DrawEmitInfo {};
 	}
 	const auto [vertex_offset, instance_offset] = ResolveDrawOffsets(ucfg.GetIndexOffset(), vs);
 	return DrawEmitInfo {vertex_offset + first_vertex, instance_offset};
@@ -1306,7 +1336,8 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
 
 	hw_check(buffer);
 
-	DrawRenderState state {};
+	DrawRenderState state {.indirect_offsets = args.indirect.offsets,
+	                       .indirect_draw    = args.indirect.address != 0};
 	if (!GetDrawTopology(ucfg, state.topology)) {
 		return true;
 	}
@@ -1411,7 +1442,8 @@ bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndexAuto,
 	                         args.vertex_count, args.instance_count, args.first_instance};
 
-	DrawRenderState state {};
+	DrawRenderState state {.indirect_offsets = args.indirect.offsets,
+	                       .indirect_draw    = args.indirect.address != 0};
 	if (!GetDrawTopology(ucfg, state.topology)) {
 		ResetBindings();
 		return true;
@@ -1475,8 +1507,9 @@ bool RenderExecutor::ResolveColorTargets(CommandBuffer& buffer, uint32_t render_
 	cache.MarkGpuWritten(dst.image_id);
 	auto& source      = cache.GetImage(src.image_id);
 	auto& destination = cache.GetImage(dst.image_id);
-	destination.Resolve(source, {src.guest_mip_level, 1, src.guest_array_layer, 1},
-	                    {dst.guest_mip_level, 1, dst.guest_array_layer, 1});
+	destination.Resolve(source,
+	                    {src.desc.view_info.base_level, 1, src.desc.view_info.base_layer, 1},
+	                    {dst.desc.view_info.base_level, 1, dst.desc.view_info.base_layer, 1});
 	return true;
 }
 

src/graphics/host_gpu/renderer/renderPssr.cpp

@@ -0,0 +1,184 @@
+#include "common/assert.h"
+#include "common/threads.h"
+#include "gpu_blit_shaders/gpu_pssr_linear_spv.h"
+#include "graphics/guest_gpu/pssr.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/image/image.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
+#include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/renderContext.h"
+#include "graphics/shader/shaderBindings.h"
+
+#include <algorithm>
+#include <array>
+
+namespace Libs::Graphics {
+namespace {
+
+struct PssrParameters {
+	uint32_t input_size[2];
+	uint32_t output_size[2];
+	uint32_t output_channels[4];
+};
+
+bool IsInputFormat(Prospero::BufferFormat format) {
+	return format == Prospero::BufferFormat::k11_11_10Float ||
+	       format == Prospero::BufferFormat::k10_11_11Float ||
+	       format == Prospero::BufferFormat::k16_16_16_16Float;
+}
+
+} // namespace
+
+RenderExecutor::~RenderExecutor() {
+	const auto device = m_context.GetGraphics().device;
+	device.destroyPipeline(m_pssr_pipeline, nullptr);
+	device.destroyPipelineLayout(m_pssr_pipeline_layout, nullptr);
+	device.destroyDescriptorSetLayout(m_pssr_descriptor_layout, nullptr);
+}
+
+void RenderExecutor::ProcessPssr(CommandBuffer& buffer, const Pssr::Request& request) {
+	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
+	Common::LockGuard lock(m_context.GetMutex());
+	ShaderTextureResource input_descriptor {}, output_descriptor {};
+	std::copy_n(request.input, 8, input_descriptor.fields);
+	std::copy_n(request.output, 8, output_descriptor.fields);
+	const auto is_2d = [](const ShaderTextureResource& descriptor) {
+		return !descriptor.IsNull() &&
+		       (descriptor.Type() == Prospero::ImageType::kColor2D ||
+		        descriptor.Type() == Prospero::ImageType::kColor2DArray) &&
+		       descriptor.TileMode() != Prospero::TileMode::kDepth;
+	};
+	EXIT_IF(!is_2d(input_descriptor) || !is_2d(output_descriptor) ||
+	        !IsInputFormat(input_descriptor.Format()) ||
+	        output_descriptor.Format() != Prospero::BufferFormat::k11_11_10Float ||
+	        output_descriptor.TileMode() != Prospero::TileMode::kStandard256B ||
+	        output_descriptor.BaseLevel() != 0 || output_descriptor.LastLevel() != 0 ||
+	        output_descriptor.MaxMip() != 0 || output_descriptor.BaseArray5() != 0 ||
+	        output_descriptor.Depth() != 0 || request.width == 0 || request.height == 0);
+	const auto input_width = std::max(1u, (input_descriptor.Width5() + 1u) >>
+	                                         input_descriptor.BaseLevel());
+	const auto input_height = std::max(1u, (input_descriptor.Height5() + 1u) >>
+	                                          input_descriptor.BaseLevel());
+	EXIT_IF(request.width > input_width || request.height > input_height);
+	PssrParameters parameters {{request.width, request.height},
+	                           {output_descriptor.Width5() + 1u, output_descriptor.Height5() + 1u},
+	                           {4, 4, 4, 4}};
+	for (uint32_t component = 0; component < 3; ++component) {
+		for (uint32_t source = 0; source < 4; ++source) {
+			if (GetDstSel(output_descriptor.DstSelXYZW(), source) == 4u + component) {
+				parameters.output_channels[component] = source;
+				break;
+			}
+		}
+	}
+	const auto resolve = [&](const uint32_t* words, bool output) {
+		ShaderRecompiler::IR::ImageResource resource {};
+		resource.resource_class = output ? ShaderRecompiler::IR::ImageResourceClass::Storage
+		                                 : ShaderRecompiler::IR::ImageResourceClass::Sampled;
+		resource.numeric_class = Prospero::TextureNumericClass::Float;
+		resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
+		resource.read = !output;
+		resource.written = output;
+		ShaderRecompiler::IR::DescriptorValue value {};
+		std::copy_n(words, 8, value.dwords.begin());
+		value.dword_count = 8;
+		auto binding = ResolveTexture(resource, value);
+		binding.desc.view_info.level_count = 1;
+		binding.desc.view_info.min_lod = 0;
+		return binding;
+	};
+	auto& cache = m_context.GetTextureCache();
+	auto input = resolve(request.input, false);
+	BindImage(input.image_id, false);
+	auto output = resolve(request.output, true);
+	BindImage(output.image_id, true);
+	EXIT_IF(input.desc.info.data.address < output.desc.info.data.End() &&
+	        output.desc.info.data.address < input.desc.info.data.End());
+	const auto acquire = [&](TextureBinding& binding, const uint32_t* words, bool storage) {
+		auto& previous = cache.GetImage(binding.image_id);
+		if (!previous.registered || previous.binding.needs_rebind) {
+			previous.binding = {};
+			binding = resolve(words, storage);
+			BindImage(binding.image_id, storage);
+		}
+		binding.image_view = cache.FindTexture(binding.image_id, binding.desc);
+	};
+	acquire(input, request.input, false);
+	acquire(output, request.output, true);
+	auto& source = cache.GetImage(input.image_id);
+	auto& destination = cache.GetImage(output.image_id);
+	const auto& source_view = input.desc.view_info;
+	const auto& destination_view = output.desc.view_info;
+	EXIT_IF(source.info.IsDepth() || destination.info.IsDepth() ||
+	        source.backing.samples != 1 || destination.backing.samples != 1 ||
+	        (input.image_id == output.image_id &&
+	         source_view.base_level == destination_view.base_level &&
+	         source_view.base_layer == destination_view.base_layer));
+
+	auto& graphics = m_context.GetGraphics();
+	if (m_pssr_pipeline == nullptr) {
+		const std::array bindings {
+		    vk::DescriptorSetLayoutBinding {0, vk::DescriptorType::eSampledImage, 1,
+		                                    vk::ShaderStageFlagBits::eCompute},
+		    vk::DescriptorSetLayoutBinding {1, vk::DescriptorType::eStorageImage, 1,
+		                                    vk::ShaderStageFlagBits::eCompute}};
+		vk::DescriptorSetLayoutCreateInfo descriptor_info {};
+		descriptor_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
+		descriptor_info.bindingCount = bindings.size();
+		descriptor_info.pBindings = bindings.data();
+		RequireVulkanSuccess(graphics.device.createDescriptorSetLayout(
+		                         &descriptor_info, nullptr, &m_pssr_descriptor_layout),
+		                     "create upscaling descriptor layout");
+		const vk::PushConstantRange push {vk::ShaderStageFlagBits::eCompute, 0,
+		                                  sizeof(PssrParameters)};
+		vk::PipelineLayoutCreateInfo layout_info {};
+		layout_info.setLayoutCount = 1;
+		layout_info.pSetLayouts = &m_pssr_descriptor_layout;
+		layout_info.pushConstantRangeCount = 1;
+		layout_info.pPushConstantRanges = &push;
+		RequireVulkanSuccess(graphics.device.createPipelineLayout(
+		                         &layout_info, nullptr, &m_pssr_pipeline_layout),
+		                     "create upscaling pipeline layout");
+		const auto shader = CompileSPV(GPU_PSSR_LINEAR_SPV, graphics.device);
+		vk::ComputePipelineCreateInfo create {};
+		create.stage.stage = vk::ShaderStageFlagBits::eCompute;
+		create.stage.module = shader;
+		create.stage.pName = "main";
+		create.layout = m_pssr_pipeline_layout;
+		const auto result = graphics.device.createComputePipelines(
+		    nullptr, 1, &create, nullptr, &m_pssr_pipeline);
+		graphics.device.destroyShaderModule(shader, nullptr);
+		RequireVulkanSuccess(result, "create upscaling pipeline");
+	}
+	buffer.EndRendering();
+	const auto command = buffer.Handle();
+	source.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderRead,
+	               ImageSubresourceRange {source_view.base_level, 1, source_view.base_layer, 1},
+	               command);
+	destination.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite,
+	                    ImageSubresourceRange {destination_view.base_level, 1,
+	                                           destination_view.base_layer, 1}, command);
+	const std::array images {
+	    vk::DescriptorImageInfo {nullptr, input.image_view, vk::ImageLayout::eGeneral},
+	    vk::DescriptorImageInfo {nullptr, output.image_view, vk::ImageLayout::eGeneral}};
+	std::array<vk::WriteDescriptorSet, 2> writes {};
+	for (uint32_t i = 0; i < writes.size(); ++i) {
+		writes[i].dstBinding = i;
+		writes[i].descriptorCount = 1;
+		writes[i].descriptorType = i == 0 ? vk::DescriptorType::eSampledImage
+		                                 : vk::DescriptorType::eStorageImage;
+		writes[i].pImageInfo = &images[i];
+	}
+	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pssr_pipeline_layout,
+	                             0, writes.size(), writes.data());
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_pssr_pipeline);
+	command.pushConstants(m_pssr_pipeline_layout, vk::ShaderStageFlagBits::eCompute,
+	                      0, sizeof(parameters), &parameters);
+	command.dispatch((parameters.output_size[0] + 7) / 8,
+	                 (parameters.output_size[1] + 7) / 8, 1);
+	ShaderAccessBarrier(command, vk::PipelineStageFlagBits::eComputeShader);
+	ResetBindings();
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/shaders/gpu_pssr_linear.comp

@@ -0,0 +1,33 @@
+#version 450 core
+#extension GL_EXT_samplerless_texture_functions : require
+
+layout(local_size_x = 8, local_size_y = 8) in;
+layout(binding = 0) uniform texture2D source;
+layout(binding = 1) writeonly uniform image2D destination;
+layout(push_constant) uniform Parameters {
+	uvec2 input_size;
+	uvec2 output_size;
+	uvec4 output_channels;
+} parameters;
+
+vec4 fetch_color(ivec2 coordinate) {
+	return texelFetch(source, clamp(coordinate, ivec2(0), ivec2(parameters.input_size) - 1), 0);
+}
+
+void main() {
+	uvec2 output_coordinate = gl_GlobalInvocationID.xy;
+	if (any(greaterThanEqual(output_coordinate, parameters.output_size))) return;
+	vec2 input_coordinate = (vec2(output_coordinate) + 0.5) *
+	                        vec2(parameters.input_size) / vec2(parameters.output_size) - 0.5;
+	ivec2 lower = ivec2(floor(input_coordinate));
+	vec2 weight = fract(input_coordinate);
+	vec4 color = mix(mix(fetch_color(lower), fetch_color(lower + ivec2(1, 0)), weight.x),
+	                 mix(fetch_color(lower + ivec2(0, 1)), fetch_color(lower + ivec2(1, 1)), weight.x),
+	                 weight.y);
+	vec4 stored = vec4(0.0);
+	for (uint channel = 0; channel < 3; ++channel) {
+		uint selected = parameters.output_channels[channel];
+		if (selected < 4) stored[channel] = color[selected];
+	}
+	imageStore(destination, ivec2(output_coordinate), stored);
+}

src/graphics/host_gpu/shaders/gpu_tiler_color.inc

@@ -1,4 +1,4 @@
-// ColorTransform values match imageInfo.h. Preserve the floating-point bit fields exactly.
+// ColorTransform values match imageInfo.h. Preserve the component bit fields exactly.
 uvec2 transform_color(uvec2 value, uint transform, bool to_host) {
 	switch (transform) {
 		case 1u:
@@ -15,6 +15,13 @@ uvec2 transform_color(uvec2 value, uint transform, bool to_host) {
 			}
 			break;
 		}
+		case 3u:
+			if (to_host) {
+				value.x = (value.x >> 2u) | (value.x << 30u);
+			} else {
+				value.x = (value.x << 2u) | (value.x >> 30u);
+			}
+			break;
 	}
 	return value;
 }

src/graphics/host_gpu/shaders/gpu_tiler_demote_d16.comp

@@ -1,10 +1,11 @@
 #version 450
+#extension GL_EXT_shader_16bit_storage : require
 
 layout(constant_id = 0) const uint D32 = 0u;
 layout(local_size_x = 64) in;
 
 layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer;
-layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer;
+layout(set = 0, binding = 1, std430) buffer Output { uint16_t data[]; } output_buffer;
 
 layout(push_constant, std430) uniform Conversion {
 	uint src_base;
@@ -32,9 +33,5 @@ void main() {
 	}
 	uint src = params.src_base + y * params.source_pitch + x * 4u;
 	uint dst = params.dst_base + y * params.target_pitch + x * 2u;
-	uint shift = (dst & 2u) * 8u;
-	uint mask = 0xffffu << shift;
-	uint word = dst >> 2u;
-	atomicAnd(output_buffer.data[word], ~mask);
-	atomicOr(output_buffer.data[word], decode(input_buffer.data[src >> 2u]) << shift);
+	output_buffer.data[dst >> 1u] = uint16_t(decode(input_buffer.data[src >> 2u]));
 }

src/graphics/host_gpu/vma.cpp

@@ -151,7 +151,8 @@ bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanIm
 	EXIT_IF(allocator == nullptr || image.image != nullptr || image.allocation != nullptr);
 
 	VmaAllocationCreateInfo alloc_info {};
-	alloc_info.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+	alloc_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
+	alloc_info.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
 
 	vk::Image::CType native_image = VK_NULL_HANDLE;
 	const auto        result       = static_cast<vk::Result>(

src/graphics/presentation/performanceOverlay.cpp

@@ -533,16 +533,16 @@ void DrawMemoryAndIo(const Snapshot& shown) {
 		    Bytes(guest.pooled_allocated) + " (" + Bytes(guest.pool_committed) + " committed)");
 		Row("Flexible", Bytes(guest.flexible_used) + " / " + Bytes(guest.flexible_total));
 		Row("Extended", Bytes(guest.automatic_allocated));
-		// Past the pool size the guest sees no page-table entries available.
-		const auto entries = std::max(guest.cpu_page_entries, guest.gpu_page_entries);
+		// Past its total a side sees no page-table entries available.
+		const auto percent = std::max(guest.cpu_page_entries * 100 / guest.cpu_page_entries_total,
+		                              guest.gpu_page_entries * 100 / guest.gpu_page_entries_total);
 		Row("Page table (2 MB)",
-		    Fmt("CPU %llu  GPU %llu  / %llu",
+		    Fmt("CPU %llu / %llu  GPU %llu / %llu",
 		        static_cast<unsigned long long>(guest.cpu_page_entries),
+		        static_cast<unsigned long long>(guest.cpu_page_entries_total),
 		        static_cast<unsigned long long>(guest.gpu_page_entries),
-		        static_cast<unsigned long long>(guest.page_entries_total)),
-		    entries > guest.page_entries_total            ? BAD
-		    : entries * 10 > guest.page_entries_total * 9 ? MEDIUM
-		                                                  : GOOD);
+		        static_cast<unsigned long long>(guest.gpu_page_entries_total)),
+		    LevelColor(static_cast<double>(percent), 90, 100));
 		ImGui::EndTable();
 	}
 	if (BeginSection("Emulator process")) {

src/graphics/presentation/videoOut.cpp

@@ -221,7 +221,7 @@ public:
 	void WaitForSubmitSlot(VideoOutConfig& cfg);
 	bool Flip(uint32_t micros);
 	void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out);
-	bool IsSafeForRendering(VideoOutConfig& cfg, int index);
+	void Wait(VideoOutConfig& cfg, int index);
 
 private:
 	enum class RequestState { Reserved, Recording, Ready, Presenting };
@@ -240,7 +240,6 @@ private:
 		Graphics::Presenter::Frame* frame;
 		bool                        premultiplied_alpha;
 	};
-	void SignalDoneLocked();
 
 	Graphics::Presenter& m_presenter;
 	Common::Mutex        m_mutex;
@@ -251,7 +250,6 @@ private:
 	std::list<Request>   m_cpu_requests;
 	std::list<Request>   m_cancelled_requests;
 	bool                 m_processing      = false;
-	bool                 m_gpu_waiting     = false;
 	uint64_t             m_next_request_id = 1;
 };
 
@@ -955,7 +953,7 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe
 				}
 			}
 		}
-		SignalDoneLocked();
+		m_done_cond_var.SignalAll();
 	}
 	if (cfg.flip_status.flipPendingNum >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
 		Common::DebugCounters::Add(Common::DebugCounters::Counter::FlipsRejected);
@@ -1035,7 +1033,7 @@ void FlipQueue::Cancel(VideoOutConfig& cfg) {
 			it = queue->erase(it);
 		}
 	}
-	SignalDoneLocked();
+	m_done_cond_var.SignalAll();
 	m_submit_slot_cond_var.SignalAll();
 	m_submit_cond_var.SignalAll();
 	m_mutex.Unlock();
@@ -1207,24 +1205,17 @@ void FlipQueue::WaitForSubmitSlot(VideoOutConfig& cfg) {
 	}
 }
 
-void FlipQueue::SignalDoneLocked() {
-	m_done_cond_var.SignalAll();
-	// A stalled GPU cannot drain and shut down without re-checking under m_mutex, so it is alive here.
-	if (m_gpu_waiting) {
-		m_gpu_waiting = false;
-		m_presenter.Renderer().GetGpu().Wake();
-	}
-}
-
-// A display buffer is safe for rendering once it is neither displayed nor waiting to be.
-bool FlipQueue::IsSafeForRendering(VideoOutConfig& cfg, int index) {
+void FlipQueue::Wait(VideoOutConfig& cfg, int index) {
 	Common::LockGuard lock(m_mutex);
-	auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
-	const bool safe = cfg.flip_status.currentBuffer != index &&
-	                  std::none_of(m_requests.begin(), m_requests.end(), matches) &&
-	                  std::none_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
-	m_gpu_waiting = !safe;
-	return safe;
+
+	auto has_request = [this, &cfg, index] {
+		auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
+		return std::any_of(m_requests.begin(), m_requests.end(), matches) ||
+		       std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
+	};
+	while (has_request()) {
+		m_done_cond_var.Wait(&m_mutex);
+	}
 }
 
 bool FlipQueue::Flip(uint32_t micros) {
@@ -1320,7 +1311,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 		it = m_requests.erase(it);
 	}
 	m_processing = false;
-	SignalDoneLocked();
+	m_done_cond_var.SignalAll();
 	m_submit_slot_cond_var.SignalAll();
 	m_mutex.Unlock();
 	for (size_t i = count; i != 0; i--) {
@@ -1703,12 +1694,12 @@ void VideoOutDriver::WaitForSubmitSlot(int handle) {
 	m_impl->GetFlipQueue().WaitForSubmitSlot(*cfg);
 }
 
-bool VideoOutDriver::IsSafeForRendering(int handle, int index) {
+void VideoOutDriver::WaitFlipDone(int handle, int index) {
 	auto* ctx = m_impl->Get(handle);
 	EXIT_IF(ctx == nullptr);
 
 	EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index));
-	return m_impl->GetFlipQueue().IsSafeForRendering(*ctx, index);
+	m_impl->GetFlipQueue().Wait(*ctx, index);
 }
 
 KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) {

src/graphics/presentation/videoOut.h

@@ -38,7 +38,7 @@ public:
 	void CompleteFlip(uint64_t request_id);
 	void SubmitFlipPreparation(uint64_t request_id);
 	void WaitForSubmitSlot(int handle);
-	[[nodiscard]] bool IsSafeForRendering(int handle, int index);
+	void WaitFlipDone(int handle, int index);
 
 	[[nodiscard]] Impl& State() noexcept;
 

src/graphics/presentation/window/swapchain.cpp

@@ -290,7 +290,8 @@ public:
 private:
 	vk::SurfaceFormatKHR SelectFormat();
 	void Destroy();
-	void DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer);
+	void DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer,
+	                 const vk::Rect2D& rect);
 
 	WindowContext&   m_window;
 	vk::SwapchainKHR m_handle = nullptr;
@@ -637,7 +638,23 @@ bool Swapchain::PrepareSystemOverlay() {
 	return m_system_overlay->PrepareFrame(m_extent, m_format, ImageCount());
 }
 
-void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer) {
+static vk::Rect2D FitAspect(const vk::Extent3D& source, const vk::Extent2D& target) {
+	const uint64_t src_w = std::max(source.width, 1u);
+	const uint64_t src_h = std::max(source.height, 1u);
+	uint32_t       w     = target.width;
+	uint32_t       h     = target.height;
+	if (src_w * h > src_h * w) {
+		h = std::max<uint32_t>(1, static_cast<uint32_t>(src_h * w / src_w));
+	} else {
+		w = std::max<uint32_t>(1, static_cast<uint32_t>(src_w * h / src_h));
+	}
+	return {{static_cast<int32_t>((target.width - w) / 2),
+	         static_cast<int32_t>((target.height - h) / 2)},
+	        {w, h}};
+}
+
+void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer,
+                            const vk::Rect2D& rect) {
 	auto device = m_window.graphic_ctx.device;
 	if (m_overlay_pipeline == nullptr) {
 		vk::SamplerCreateInfo sampler {};
@@ -756,12 +773,14 @@ void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& l
 	rendering.colorAttachmentCount = 1;
 	rendering.pColorAttachments    = &attachment;
 	command.beginRendering(&rendering);
-	const vk::Viewport viewport {
-	    0.0f, 0.0f, static_cast<float>(m_extent.width), static_cast<float>(m_extent.height),
-	    0.0f, 1.0f};
-	const vk::Rect2D scissor {{0, 0}, m_extent};
+	const vk::Viewport viewport {static_cast<float>(rect.offset.x),
+	                             static_cast<float>(rect.offset.y),
+	                             static_cast<float>(rect.extent.width),
+	                             static_cast<float>(rect.extent.height),
+	                             0.0f,
+	                             1.0f};
 	command.setViewport(0, 1, &viewport);
-	command.setScissor(0, 1, &scissor);
+	command.setScissor(0, 1, &rect);
 	command.draw(3, 1, 0, 0);
 	command.endRendering();
 }
@@ -799,6 +818,24 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame*
 	                           vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags {}, 0,
 	                           nullptr, 0, nullptr, 1, &to_transfer);
 
+	const auto* aspect_source = source != nullptr ? source : overlay.frame;
+	const auto  game_rect     = !Config::StretchEnabled() && aspect_source != nullptr
+	                               ? FitAspect(aspect_source->image.extent, m_extent)
+	                               : vk::Rect2D {{0, 0}, m_extent};
+	if (source == nullptr || game_rect.extent != m_extent) {
+		const vk::ClearColorValue black {};
+		vk_command.clearColorImage(m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal,
+		                           &black, 1, &to_transfer.subresourceRange);
+		if (source != nullptr) {
+			vk::ImageMemoryBarrier after_clear = to_transfer;
+			after_clear.srcAccessMask          = vk::AccessFlagBits::eTransferWrite;
+			after_clear.oldLayout              = vk::ImageLayout::eTransferDstOptimal;
+			vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+			                           vk::PipelineStageFlagBits::eTransfer,
+			                           vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1,
+			                           &after_clear);
+		}
+	}
 	if (source != nullptr) {
 		vk::ImageBlit region {};
 		region.srcSubresource.aspectMask     = vk::ImageAspectFlagBits::eColor;
@@ -812,16 +849,14 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame*
 		region.dstSubresource.mipLevel       = 0;
 		region.dstSubresource.baseArrayLayer = 0;
 		region.dstSubresource.layerCount     = 1;
-		region.dstOffsets[1].x               = static_cast<int>(m_extent.width);
-		region.dstOffsets[1].y               = static_cast<int>(m_extent.height);
+		region.dstOffsets[0].x = game_rect.offset.x;
+		region.dstOffsets[0].y = game_rect.offset.y;
+		region.dstOffsets[1].x = game_rect.offset.x + static_cast<int>(game_rect.extent.width);
+		region.dstOffsets[1].y = game_rect.offset.y + static_cast<int>(game_rect.extent.height);
 		region.dstOffsets[1].z               = 1;
 		vk_command.blitImage(source->image.image, vk::ImageLayout::eTransferSrcOptimal,
 		                     m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal, 1,
 		                     &region, vk::Filter::eLinear);
-	} else {
-		const vk::ClearColorValue black {};
-		vk_command.clearColorImage(m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal,
-		                           &black, 1, &to_transfer.subresourceRange);
 	}
 
 	vk::ImageMemoryBarrier to_present {};
@@ -848,7 +883,7 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame*
 	                           &to_present);
 	if (draw_attachment) {
 		if (draw_overlay) {
-			DrawOverlay(vk_command, overlay);
+			DrawOverlay(vk_command, overlay, game_rect);
 			if (draw_system_overlay) {
 				to_present.srcAccessMask = vk::AccessFlagBits::eColorAttachmentWrite;
 				to_present.oldLayout     = vk::ImageLayout::eColorAttachmentOptimal;

src/graphics/presentation/window/vulkanWindow.cpp

@@ -51,6 +51,7 @@ struct VulkanExtensions {
 vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
 	vk::PhysicalDeviceVulkan11Features features {};
 	features.storageBuffer16BitAccess = VK_TRUE;
+	features.shaderDrawParameters     = VK_TRUE;
 	return features;
 }
 
@@ -278,6 +279,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		              required_features12.samplerMirrorClampToEdge);
 		check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
 		              WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
+		check_feature(features11.shaderDrawParameters, "shaderDrawParameters",
+		              WindowContext::RequiredVulkan11Features().shaderDrawParameters);
 		check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
 		              required_features12.storageBuffer8BitAccess);
 		check_feature(features12.timelineSemaphore, "timelineSemaphore",
@@ -337,6 +340,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		check_feature(device_features2.features.shaderImageGatherExtended,
 		              "shaderImageGatherExtended");
+		check_feature(device_features2.features.shaderResourceMinLod, "shaderResourceMinLod");
 
 		check_feature(device_features2.features.independentBlend, "independentBlend");
 		check_feature(device_features2.features.tessellationShader, "tessellationShader");
@@ -636,6 +640,7 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 	device_features.shaderStorageImageMultisample        =
 	    supported_features2.features.shaderStorageImageMultisample;
 	device_features.shaderImageGatherExtended            = VK_TRUE;
+	device_features.shaderResourceMinLod                 = VK_TRUE;
 	device_features.independentBlend                     = VK_TRUE;
 	device_features.dualSrcBlend                         = VK_TRUE;
 	device_features.tessellationShader                   = VK_TRUE;
@@ -733,6 +738,15 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
 		conditional_rendering.pNext = const_cast<void*>(create_info.pNext);
 		create_info.pNext = &conditional_rendering;
 	}
+	// The renderer only uses the Vulkan 1.3 core dynamic states, but RADV (Mesa 26.3) creates its
+	// PS epilog cache only when this EXT feature is enabled and crashes in vkCmdDraw when a
+	// depth- or stencil-writing fragment shader needs an epilog.
+	vk::PhysicalDeviceExtendedDynamicStateFeaturesEXT extended_dynamic_state {};
+	extended_dynamic_state.extendedDynamicState = VK_TRUE;
+	if (HasExtension(device_extensions, VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME)) {
+		extended_dynamic_state.pNext = const_cast<void*>(create_info.pNext);
+		create_info.pNext = &extended_dynamic_state;
+	}
 	create_info.pQueueCreateInfos       = &queue_create_info;
 	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
@@ -1111,7 +1125,8 @@ void WindowContext::CreateVulkan() {
 		                             VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME,
 		                             VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME,
 		                             VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME,
-		                             VK_EXT_CONDITIONAL_RENDERING_EXTENSION_NAME}) {
+		                             VK_EXT_CONDITIONAL_RENDERING_EXTENSION_NAME,
+		                             VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME}) {
 			if (HasExtension(available_extensions, extension)) {
 				device_extensions.push_back(extension);
 			}

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -467,6 +467,7 @@ Decoder::Program DecodeFusedProgram(std::span<const uint32_t> front, std::span<c
 void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
 	using namespace IR;
 	EXIT_NOT_IMPLEMENTED(options.stage != ShaderType::Compute);
+	source.wave_size = options.wave_size;
 	for (uint32_t i = 0; i < source.decoded.instructions.size(); ++i) {
 		if (source.decoded.instructions[i].opcode != Decoder::Opcode::S_SWAPPC_B64) continue;
 		source.calls.push_back({.instruction = i});
@@ -503,7 +504,8 @@ void PrepareCallTargets(ShaderSource& source, const CompileOptions& options) {
 	};
 	Register = [&](uint32_t before, uint32_t code) -> Value {
 		std::vector<uint32_t> crossed_calls;
-		const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code, crossed_calls);
+		const auto index = CFG::FindScalarDefinition(source.decoded, graph, before, code,
+		                                             source.wave_size, crossed_calls);
 		for (const auto crossed: crossed_calls) {
 			std::ranges::find(source.calls, crossed, &ShaderSource::Call::instruction)->preserved_registers.set(code);
 		}
@@ -702,10 +704,11 @@ const Decoder::Program& RefreshShaderSource(ShaderSource& source, const IR::SrtR
 		EXIT_IF(function_pc > UINT32_MAX - function_size);
 		const auto function_offset = static_cast<uint32_t>(linked.code.size());
 		for (auto inst: function) {
-			EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg) ||
-			        CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u));
+			EXIT_IF(CFG::MayWriteScalarRegister(inst, call.dst.reg, source.wave_size) ||
+			        CFG::MayWriteScalarRegister(inst, call.dst.reg + 1u, source.wave_size));
 			for (uint32_t code = 0; code < site.preserved_registers.size(); ++code) {
-				if (site.preserved_registers[code] && CFG::MayWriteScalarRegister(inst, code))
+				if (site.preserved_registers[code] &&
+				    CFG::MayWriteScalarRegister(inst, code, source.wave_size))
 					EXIT("shader call at pc 0x%08x clobbers scalar target dependency s%u", call.pc, code);
 			}
 			linked.code.insert(linked.code.end(), inst.raw, inst.raw + inst.word_count);
@@ -811,7 +814,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 	const auto& cfg = *selected_cfg;
 	Frontend::EmbeddedFetchPlan embedded_fetch;
 	if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
-	    options.input_info.vertex->fetch_embedded) {
+	    options.input_info.vertex->has_dynamic_vertex_fetch) {
 		embedded_fetch = DetectEmbeddedVertexFetch(
 		    decoded, *options.input_info.vertex, options.user_data_base,
 		    static_cast<uint32_t>(options.user_data.size()), options.wave_size);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -62,6 +62,7 @@ struct ShaderSource {
 	IR::ResourcePlan call_targets;
 	std::vector<Call> calls;
 	uint32_t unsupported_call_pc = UINT32_MAX;
+	uint32_t wave_size = 64;
 	std::optional<Linked> linked;
 	uint64_t revision = 0;
 	std::vector<std::pair<uint64_t, uint64_t>> reads;

src/graphics/shader/recompiler/Tessellation.cpp

@@ -156,11 +156,14 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 						if (rhs.constant == 0u && third.constant == 8u) value = constant(0u);
 					}
 					break;
-				case Opcode::V_AND_B32:
-					if ((lhs.kind == Kind::PackedControlPoint && rhs == constant(0xffu)) ||
-					    (rhs.kind == Kind::PackedControlPoint && lhs == constant(0xffu)))
-						value = constant(0u);
+				case Opcode::V_AND_B32: {
+					const auto mask = lhs.kind == Kind::PackedControlPoint ? rhs
+					                : rhs.kind == Kind::PackedControlPoint ? lhs
+					                                                       : Address {};
+					if (mask == constant(0xffu)) value = constant(0u);
+					if (mask == constant(0x1f00u)) value = {Kind::Affine, 256u, 0u};
 					break;
+				}
 				case Opcode::V_ADD_NC_U32: value = add(lhs, rhs); break;
 				case Opcode::V_MUL_LO_U32: value = multiply(lhs, rhs); break;
 				case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -30,6 +30,13 @@ bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
 	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
 }
 
+uint32_t MrtOutputType(EmitterState& state, uint32_t index) {
+	const auto& pixel = *state.input_info.pixel;
+	if ((pixel.target_sint_mask & (1u << index)) != 0u) return TypeI32Vector(state, 4);
+	if ((pixel.target_uint_mask & (1u << index)) != 0u) return TypeU32Vector(state, 4);
+	return TypeF32Vector(state, 4);
+}
+
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
 	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
 		return state.per_vertex_variable;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp

@@ -30,16 +30,7 @@ uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
 uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
                   Vec3 origin, Vec3 direction) {
 	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
-	const auto zero = ConstantF32Value(s, 0.0f);
-	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
-		const auto value = Binary(s, code, f, x, y);
-		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
-		return value;
-	};
 	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
-	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
-	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
-	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
 	const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); };
 	// The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0).
 	const std::array vertices {
@@ -56,6 +47,28 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
 			    NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis))));
 	}
 	const auto flag = NodeWord(s, block, ConstantU32(s, 15));
+	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
+	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
+	return EmitBvhTriangle(s, positions, origin, direction, remap,
+	    Binary(s, spv::OpIAdd, u, flag, kind), bary);
+}
+
+} // namespace
+
+uint32_t EmitBvhTriangle(EmitterState& s, const std::array<Vec3, 3>& positions,
+                         Vec3 origin, Vec3 direction, uint32_t remap,
+                         uint32_t triangle_id, uint32_t bary) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
+		const auto value = Binary(s, code, f, x, y);
+		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
+		return value;
+	};
+	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
+	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
+	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
+	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
 	const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1]));
 	const auto z_largest = cmp(spv::OpFOrdLessThan,
 	    Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2]));
@@ -112,8 +125,6 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
 		    cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right);
 		missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded));
 	}
-	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
-	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
 	const auto coordinate = [&](uint32_t shift) {
 		const auto index = Binary(s, spv::OpBitwiseAnd, u,
 		    Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3));
@@ -124,11 +135,13 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
 	const std::array result {
 		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)),
 		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)),
-		Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)),
+		Select(s, u, bary, coordinate(0), triangle_id),
 		Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) };
 	return Vector(s, TypeU32Vector(s, 4), result);
 }
 
+namespace {
+
 std::array<uint32_t, 6> Box(EmitterState& s, uint32_t first, uint32_t second,
                            uint32_t half, uint32_t child) {
 	const auto f = TypeF32(s);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -63,7 +63,8 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
 	    kind == IR::StageInputKind::InvocationId || kind == IR::StageInputKind::PrimitiveId ||
-	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
+	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId ||
+	    kind == IR::StageInputKind::BaseVertex || kind == IR::StageInputKind::BaseInstance) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
 		state.builder.AddFunction(spv::OpLoad, TypeI32(state), value, variable);
@@ -291,9 +292,10 @@ uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t com
 }
 
 uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp,
-                      bool uint_output) {
-	auto& state = ctx.state;
-	if (exp.compr && !uint_output) {
+                      uint32_t vector_type) {
+	auto&      state          = ctx.state;
+	const bool integer_output = vector_type != TypeF32Vector(state, 4);
+	if (exp.compr && !integer_output) {
 		auto unpack = GLSLstd450UnpackHalf2x16;
 		switch (MrtOutputMode(state, exp)) {
 			case 5: unpack = GLSLstd450UnpackUnorm2x16; break;
@@ -330,9 +332,11 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 		    ConstantU32(state, 0),
 		    ConstantU32(state, 0),
 		    ConstantU32(state, 0),
-		    ConstantU32(state, uint_output ? 1u : 0x3f800000u),
+		    ConstantU32(state, integer_output ? 1u : 0x3f800000u),
 		};
 		if (exp.compr) {
+			const auto extract = vector_type == TypeU32Vector(state, 4) ? spv::OpBitFieldUExtract
+			                                                            : spv::OpBitFieldSExtract;
 			for (uint32_t pair = 0; pair < 2u; pair++) {
 				if ((exp.en & (3u << (pair * 2u))) == 0u) {
 					continue;
@@ -344,8 +348,8 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 						continue;
 					}
 					raw[component] = state.builder.AllocateId();
-					state.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), raw[component],
-					                          packed, ConstantU32(state, lane * 16u),
+					state.builder.AddFunction(extract, TypeU32(state), raw[component], packed,
+					                          ConstantU32(state, lane * 16u),
 					                          ConstantU32(state, 16));
 				}
 			}
@@ -360,12 +364,10 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 		state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), data,
 		                          raw[0], raw[1], raw[2], raw[3]);
 	}
-	if (uint_output) {
+	if (vector_type == TypeU32Vector(state, 4)) {
 		return data;
 	}
-	const auto vector = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpBitcast, TypeF32Vector(state, 4), vector, data);
-	return vector;
+	return Unary(state, spv::OpBitcast, vector_type, data);
 }
 
 uint32_t DistancePointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) {
@@ -563,11 +565,11 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (state.program.stage != ShaderType::Mesh && variable == 0) {
 			return;
 		}
-		const bool uint_output = MrtOutputMode(state, exp) == 7u;
-		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
-		auto       value       = ExportVector(ctx, data, exp, uint_output);
+		const auto vector_type = MrtOutputMode(state, exp) != 0u ? MrtOutputType(state, exp.index)
+		                                                         : TypeF32Vector(state, 4);
+		auto       value       = ExportVector(ctx, data, exp, vector_type);
 		if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-		    exp.index == 0 && !uint_output &&
+		    exp.index == 0 && vector_type == TypeF32Vector(state, 4) &&
 		    state.input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None) {
 			// Broadcast logical alpha before swizzling the primary output.
 			const auto blend_output =
@@ -711,9 +713,16 @@ void EmitBarrier(EmitterState& state) {
 		return;
 	}
 	const auto memory_scope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup;
-	const auto memory = state.lds_storage_class == spv::StorageClassStorageBuffer
-	                        ? spv::MemorySemanticsUniformMemoryMask
-	                        : spv::MemorySemanticsWorkgroupMemoryMask;
+	auto memory = state.lds_storage_class == spv::StorageClassStorageBuffer
+	                  ? spv::MemorySemanticsUniformMemoryMask
+	                  : spv::MemorySemanticsWorkgroupMemoryMask;
+	// Shaders read other waves' buffer and image stores with plain loads right after S_BARRIER.
+	const auto written = [](const auto& resource) { return resource.written; };
+	if (std::ranges::any_of(state.program.info.buffers, written) ||
+	    std::ranges::any_of(state.program.info.images, written)) {
+		memory = memory | spv::MemorySemanticsUniformMemoryMask |
+		         spv::MemorySemanticsImageMemoryMask;
+	}
 	const auto semantics = tessellation ? spv::MemorySemanticsMaskNone
 	                                    : spv::MemorySemanticsAcquireReleaseMask | memory;
 	state.builder.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
@@ -811,6 +820,13 @@ uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
 	return ctx.Ballot(predicate);
 }
 
+uint32_t EmitLiveMask(EmitterState& state) {
+	const auto helper =
+	    EmitNative<spv::OpLoad, IR::Type::U1>(state, state.helper_invocation_variable);
+	return EmitNative<spv::OpGroupNonUniformBallot, IR::Type::U32x4>(
+	    state, ConstantU32(state, spv::ScopeSubgroup), EmitLogicalNotBool(state, helper));
+}
+
 uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto ballot = ctx.Ballot(inst.Arg(1));
 	const auto lane   = ctx.FirstLane(ballot);
@@ -821,18 +837,28 @@ uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return ctx.Shuffle(inst, 0, ctx.Arg(inst, 1));
 }
 
-uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+template <spv::Op Reduce, auto Combine>
+static uint32_t EmitSubgroupReduceU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	auto value = ctx.Arg(inst, 0);
 	if (ctx.other_half != nullptr) {
-		value = EmitUMin32(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1));
+		value = Combine(state, ctx.HalfArg(inst, 0, 0), ctx.HalfArg(inst, 0, 1));
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpGroupNonUniformUMin, TypeU32(state), result,
-	                          ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce, value);
+	state.builder.AddFunction(Reduce, TypeU32(state), result,
+	                          ConstantU32(state, spv::ScopeSubgroup), spv::GroupOperationReduce,
+	                          value);
 	return result;
 }
 
+uint32_t EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitSubgroupReduceU32<spv::OpGroupNonUniformUMin, EmitUMin32>(ctx, inst);
+}
+
+uint32_t EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitSubgroupReduceU32<spv::OpGroupNonUniformUMax, EmitUMax32>(ctx, inst);
+}
+
 uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&      state = ctx.state;
 	const auto hit   = state.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -38,7 +38,8 @@ bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {
 
 uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
                     uint32_t component) {
-	const auto layout = Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component);
+	const auto layout = Decoder::ImageAddressComponentLayout(mem.image_sample_flags,
+	                                                         mem.image_dimension, component);
 	const auto packed = layout.bit_offset / 32u;
 	if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0);
 	auto value = ctx.Def(address.Arg(packed));
@@ -55,10 +56,11 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 
 uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
                     uint32_t component) {
-	const auto value = AddressU32(ctx, mem, address, component);
-	return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u
-	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
+	const auto value  = AddressU32(ctx, mem, address, component);
+	const auto layout = Decoder::ImageAddressComponentLayout(mem.image_sample_flags,
+	                                                         mem.image_dimension, component);
+	return layout.bit_width == 16u ? EmitF16BitsToF32(ctx.state, value)
+	                               : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
 }
 
 ImageSampleLayout Layout(const IR::MemoryInfo& mem) {
@@ -77,6 +79,7 @@ ImageSampleLayout Layout(const IR::MemoryInfo& mem) {
 	layout.coord = cursor;
 	cursor += info.coordinate_components;
 	if (HasFlag(mem, Decoder::ImageSampleFlagLod)) layout.lod = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagLodClamp)) layout.clamp = cursor++;
 	return layout;
 }
 
@@ -810,6 +813,25 @@ uint32_t EmitImageResult(ValueEmitContext& ctx, const IR::Inst& inst, const Emit
 	return mixed ? value : result_bits(mem.resource, value);
 }
 
+// Ordinals outside the table select candidates[0]. Past ordinal 0, a consecutive table is an
+// offset of ordinal - 1, which wraps at 0 so that one unsigned compare checks both ends.
+uint32_t SamplerArrayIndex(EmitterState& state, uint32_t ordinal,
+                           const std::vector<uint32_t>& candidates) {
+	const auto count = static_cast<uint32_t>(candidates.size());
+	const auto gap = std::adjacent_find(candidates.begin() + 1, candidates.end(),
+	    [](uint32_t lhs, uint32_t rhs) { return rhs != lhs + 1u; });
+	if (count == 1u || gap != candidates.end()) {
+		return EmitIndexSwitch(state, ordinal, count, TypeU32(state),
+		                       [&](uint32_t item) { return ConstantU32(state, candidates[item]); });
+	}
+	const auto offset = Binary(state, spv::OpISub, TypeU32(state), ordinal, ConstantU32(state, 1u));
+	const auto in_table = Binary(state, spv::OpULessThan, TypeBool(state), offset,
+	                             ConstantU32(state, count - 1u));
+	const auto index = Binary(state, spv::OpIAdd, TypeU32(state), offset,
+	                          ConstantU32(state, candidates[1]));
+	return Select(state, TypeU32(state), in_table, index, ConstantU32(state, candidates[0]));
+}
+
 struct ImageSampler {
 	uint32_t id = 0;
 	bool dynamic = false;
@@ -845,9 +867,7 @@ std::array<ImageSampler, 2> LoadImageSamplers(ValueEmitContext& ctx, const IR::I
 				selected = EmitIndirectResourceIndex(state, selected,
 				    samplers[index].indirect_mapping_offset,
 				    samplers[index].indirect_search_iterations, 0u);
-			array_index = EmitIndexSwitch(
-			    state, selected, static_cast<uint32_t>(candidates.size()),
-			    TypeU32(state), [&](uint32_t ordinal) { return ConstantU32(state, candidates[ordinal]); });
+			array_index = SamplerArrayIndex(state, selected, candidates);
 		}
 		sampler = {LoadSamplerDescriptor(state, index, array_index), array_index != 0u};
 	};
@@ -1069,7 +1089,12 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				dref_value = AddressF32(ctx, mem, *address, layout.dref);
 			}
 		}
-		std::array<uint32_t, 4> operands {};
+		// _cl clamps the computed LOD from below.
+		uint32_t min_lod = 0;
+		if (layout.clamp != NoImageComponent) {
+			min_lod = AddressF32(ctx, mem, *address, layout.clamp);
+		}
+		std::array<uint32_t, 5> operands {};
 		size_t operand_count = 0;
 		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
 			operands[0] = spv::ImageOperandsGradMask;
@@ -1081,6 +1106,11 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) {
 				lod = AddressF32(ctx, mem, *address, layout.lod);
 			}
+			// SPIR-V takes MinLod only with implicit LOD or Grad.
+			if (min_lod != 0) {
+				lod     = EmitGlsl<GLSLstd450FMax, IR::Type::F32>(state, lod, min_lod);
+				min_lod = 0;
+			}
 			operands[0] = spv::ImageOperandsLodMask;
 			operands[1] = lod;
 			operand_count = 2;
@@ -1104,6 +1134,13 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				sample_operands[sample_operand_count++] = PackedOffset(
 				    ctx, offset_value, candidate.dimension, sample_operands[0]);
 			}
+			// MinLod follows the offset in operand order.
+			if (min_lod != 0) {
+				state.builder.RequireCapability(spv::CapabilityMinLod);
+				sample_operand_count = std::max(size_t{1}, sample_operand_count);
+				sample_operands[0] |= spv::ImageOperandsMinLodMask;
+				sample_operands[sample_operand_count++] = min_lod;
+			}
 			const auto coord =
 			    CoordF32(ctx, mem, *address, layout.coord,
 			             ImageDimensionInfoFor(candidate.dimension).coordinate_components,

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h

@@ -230,10 +230,12 @@ uint32_t              EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in
 uint32_t              EmitWqmU64(EmitterState& state, uint32_t value);
 uint32_t              EmitLaneId(EmitterState& state);
 uint32_t              EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
+uint32_t              EmitLiveMask(EmitterState& state);
 uint32_t              EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -289,6 +291,7 @@ inline constexpr auto EmitBufferAtomicIAdd32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicISub32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicSMin32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicUMin64    = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
@@ -335,6 +338,9 @@ inline constexpr auto EmitDataConsume           = EmitAppendConsume;
 uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvh8Intersect(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvh8Extract(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst);
 inline constexpr auto EmitImageQueryDimensions = EmitImage;
 inline constexpr auto EmitImageQueryLod        = EmitImage;
 inline constexpr auto EmitImageRead            = EmitImage;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -112,13 +112,17 @@ struct EmitterState {
 	uint32_t                                         lane_half               = 0;
 	std::array<BufferDefinition, 4>                  storage_buffers {};
 	std::vector<uint32_t>                           memory_byte_offsets;
+	std::vector<uint32_t>                           memory_byte_sizes;
 	uint32_t                                         bda_pagetable_variable  = 0;
 	uint32_t                                         fault_buffer_variable   = 0;
 	uint32_t                                         bda_pointer_function    = 0;
 	uint32_t                                         indirect_format_function = 0;
 	uint32_t                                         indirect_format_geometry_function = 0;
-	std::map<std::array<uint32_t, 3>, uint32_t>        indirect_buffer_load_functions;
+	std::map<std::array<uint32_t, 4>, uint32_t>        indirect_buffer_load_functions;
 	uint32_t                                         bvh_intersect_function  = 0;
+	uint32_t                                         bvh8_result_type       = 0;
+	uint32_t                                         bvh8_intersect_function = 0;
+	uint32_t                                         bvh_stack_function      = 0;
 	uint32_t                                         gds_variable            = 0;
 	uint32_t                                         gds_length              = 0;
 	uint32_t                                         push_constant_variable  = 0;
@@ -145,6 +149,7 @@ struct EmitterState {
 	const IR::Block*           current_block                         = nullptr;
 	uint32_t                   pixel_valid_mask_variable             = 0;
 	uint32_t                   subgroup_local_invocation_id_variable = 0;
+	uint32_t                   helper_invocation_variable            = 0;
 	uint32_t                   per_vertex_variable                   = 0;
 	uint32_t                   point_size_variable                   = 0;
 	uint32_t                   clip_distance_variable                = 0;
@@ -167,6 +172,7 @@ uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
 uint32_t TypeU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
+uint32_t TypeBvh8Result(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
 uint32_t TypeI32Pair(EmitterState& state);
 uint32_t TypeF32(EmitterState& state);
@@ -282,6 +288,7 @@ struct ImageSampleLayout {
 	uint32_t bias   = NoImageComponent;
 	uint32_t coord  = 0;
 	uint32_t lod    = NoImageComponent;
+	uint32_t clamp  = NoImageComponent;
 	uint32_t grad_x = NoImageComponent;
 	uint32_t grad_y = NoImageComponent;
 };
@@ -305,6 +312,7 @@ uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind ki
 
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
+uint32_t MrtOutputType(EmitterState& state, uint32_t index);
 
 uint32_t ConstantU32(EmitterState& state, uint32_t value);
 
@@ -414,7 +422,7 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
 
 uint32_t SharedIndirectBufferResource(const EmitterState& state, uint32_t resource);
 
-void EmitMemoryOffsets(EmitterState& state);
+void EmitMemoryBindings(EmitterState& state);
 
 uint32_t LdsDwordCount(const EmitterState& state);
 void EnsureLdsStorage(EmitterState& state);
@@ -536,9 +544,13 @@ uint32_t EmitF32BitsToF16RtzBits(EmitterState& state, uint32_t bits);
 void EmitProgram(EmitterState& state);
 
 void DefineGetBdaPointer(EmitterState& state);
-void DefineIndirectBufferFormats(ValueEmitContext& ctx);
 void DefineIndirectBufferLoads(ValueEmitContext& ctx);
 void DefineBvhIntersect(EmitterState& state);
+void DefineTrinityBvhIntersect(EmitterState& state);
+uint32_t EmitBvhTriangle(EmitterState& state, const std::array<std::array<uint32_t, 3>, 3>& positions,
+                         std::array<uint32_t, 3> origin, std::array<uint32_t, 3> direction,
+                         uint32_t remap, uint32_t triangle_id, uint32_t bary);
+void DefineTrinityFunctions(EmitterState& state);
 uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
 
 // These templates accept local lambdas from several emitter translation units.

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -1026,53 +1026,50 @@ uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IndirectBufferAccess& b
 	return ConstructU32Composite(state, components, values);
 }
 
-uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) {
+uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IndirectBufferAccess& buffer) {
 	auto& state = ctx.state;
-	return EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
-		const auto buffer = PrepareIndirectBuffer(state, IndirectBufferArgs(ctx, inst), true);
-		const auto geometry = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpFunctionCall, TypeU32Composite(state, 4), geometry,
-		                          state.indirect_format_geometry_function, buffer.format);
-		const auto field = [&](uint32_t index) {
-			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, geometry, index);
-			return value;
-		};
-		const auto bytes = field(0);
-		const auto component = Binary(
-		    state, spv::OpUMod, TypeU32(state),
-		    Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), field(1));
-		const auto alignment = Unary(state, spv::OpUConvert, TypeU64(state), field(3));
-		const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
-		                         Unary(state, spv::OpNot, TypeU64(state), alignment));
-		const auto address = Binary(
-		    state, spv::OpIAdd, TypeU64(state), base,
-		    Unary(state, spv::OpUConvert, TypeU64(state),
-		          Binary(state, spv::OpIMul, TypeU32(state), component, field(2))));
-		const auto memory = AndCondition(
-		    state, Binary(state, spv::OpINotEqual, TypeBool(state), bytes, ConstantU32(state, 0)),
-		    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector,
-		           ConstantU32(state, 4)));
-		const auto raw = EmitValueOrZeroIfCondition(
-		    state, AndCondition(state, memory, IndirectBufferInBounds(state, buffer, 0u, bytes, true)),
-		    [&]() {
-			    // Format alignment and component spacing keep every selected component within one DWORD.
-			    const auto word = LoadBdaDword(
-			        ctx, Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
-			                    ConstantU64(state, ~uint64_t {3})));
-			    const auto byte = Binary(
-			        state, spv::OpBitwiseAnd, TypeU32(state),
-			        Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3));
-			    return Binary(state, spv::OpShiftRightLogical, TypeU32(state), word,
-			                  Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte,
-			                         ConstantU32(state, 3)));
-		    });
-		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpFunctionCall, TypeU32(state), result,
-		                          state.indirect_format_function, buffer.format, buffer.selector,
-		                          component, raw);
-		return result;
-	});
+	const auto geometry = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeU32Composite(state, 4), geometry,
+	                          state.indirect_format_geometry_function, buffer.format);
+	const auto field = [&](uint32_t index) {
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, geometry, index);
+		return value;
+	};
+	const auto bytes = field(0);
+	const auto component = Binary(
+	    state, spv::OpUMod, TypeU32(state),
+	    Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), field(1));
+	const auto alignment = Unary(state, spv::OpUConvert, TypeU64(state), field(3));
+	const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
+	                         Unary(state, spv::OpNot, TypeU64(state), alignment));
+	const auto address = Binary(
+	    state, spv::OpIAdd, TypeU64(state), base,
+	    Unary(state, spv::OpUConvert, TypeU64(state),
+	          Binary(state, spv::OpIMul, TypeU32(state), component, field(2))));
+	const auto memory = AndCondition(
+	    state, Binary(state, spv::OpINotEqual, TypeBool(state), bytes, ConstantU32(state, 0)),
+	    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector,
+	           ConstantU32(state, 4)));
+	const auto raw = EmitValueOrZeroIfCondition(
+	    state, AndCondition(state, memory, IndirectBufferInBounds(state, buffer, 0u, bytes, true)),
+	    [&]() {
+		    // Format alignment and component spacing keep every selected component within one DWORD.
+		    const auto word = LoadBdaDword(
+		        ctx, Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
+		                    ConstantU64(state, ~uint64_t {3})));
+		    const auto byte = Binary(
+		        state, spv::OpBitwiseAnd, TypeU32(state),
+		        Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3));
+		    return Binary(state, spv::OpShiftRightLogical, TypeU32(state), word,
+		                  Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte,
+		                         ConstantU32(state, 3)));
+	    });
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeU32(state), result,
+	                          state.indirect_format_function, buffer.format, buffer.selector,
+	                          component, raw);
+	return result;
 }
 
 uint32_t LoadRawBuffer(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
@@ -1135,15 +1132,12 @@ uint32_t LoadRawBuffer(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Me
 uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
 	auto& state = ctx.state;
 	const auto mem = ctx.Memory(inst);
-	if (mem.kind == IR::ResourceKind::IndirectBuffer && mem.formatted) {
-		return LoadIndirectFormattedX(ctx, inst);
-	}
 	return EmitValueOrDefaultIfCondition(
 	    state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
 	    ConstantU32CompositeZero(state, components), [&]() {
 		    if (mem.kind == IR::ResourceKind::IndirectBuffer) {
 			    const auto function = state.indirect_buffer_load_functions.at(
-			        {mem.data_bits, components, uint32_t(mem.coherent)});
+			        {mem.data_bits, components, uint32_t(mem.coherent), uint32_t(mem.formatted)});
 			    const auto args = IndirectBufferArgs(ctx, inst);
 			    const auto result = state.builder.AllocateId();
 			    state.builder.AddFunction(spv::OpFunctionCall, TypeU32Composite(state, components),
@@ -1231,62 +1225,8 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 	});
 }
 
-} // namespace
-
-uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result,
-	                          state.bda_pointer_function, address);
-	return result;
-}
-
-void DefineIndirectBufferLoads(ValueEmitContext& ctx) {
-	auto& state = ctx.state;
-	for (const auto* block: state.program.blocks) {
-		for (const auto& inst: *block) {
-			const auto components = IR::BufferComponentCount(inst.GetOpcode());
-			if (components == 0u || inst.GetOpcode() == IR::ValueOpcode::ReadConstBuffer) continue;
-			const auto& memory = ctx.Memory(inst);
-			if (memory.kind != IR::ResourceKind::IndirectBuffer || memory.formatted ||
-			    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) continue;
-			auto [entry, inserted] = state.indirect_buffer_load_functions.try_emplace(
-			    std::array {memory.data_bits, components, uint32_t(memory.coherent)}, 0u);
-			if (!inserted) continue;
-			const auto function = entry->second = state.builder.AllocateId();
-			const auto type = TypeU32(state);
-			const auto result_type = TypeU32Composite(state, components);
-			const auto signature = state.builder.Type(
-			    spv::OpTypeFunction, result_type, type, type, type, type, type, type, type, type);
-			const auto name = "load_indirect_buffer_" + std::to_string(memory.data_bits) + "x" +
-			                  std::to_string(components) + (memory.coherent ? "_coherent" : "");
-			state.builder.AddName(function, name.c_str());
-			state.builder.AddFunction(spv::OpFunction, result_type, function,
-			                          spv::FunctionControlMaskNone, signature);
-			std::array<uint32_t, 8> args;
-			for (auto& arg: args) {
-				arg = state.builder.AllocateId();
-				state.builder.AddFunction(spv::OpFunctionParameter, type, arg);
-			}
-			EmitLabel(state, state.builder.AllocateId());
-			const auto result = LoadIndirectBuffer(ctx, PrepareIndirectBuffer(state, args, false),
-			                                       memory.data_bits, components, memory.coherent);
-			state.builder.AddFunction(spv::OpReturnValue, result);
-			state.builder.AddFunction(spv::OpFunctionEnd);
-		}
-	}
-}
-
 void DefineIndirectBufferFormats(ValueEmitContext& ctx) {
 	auto& state = ctx.state;
-	const auto needed = std::ranges::any_of(state.program.blocks, [&](const IR::Block* block) {
-		return std::ranges::any_of(*block, [&](const IR::Inst& inst) {
-			if (inst.GetOpcode() != IR::ValueOpcode::LoadBufferU32) return false;
-			const auto& memory = ctx.Memory(inst);
-			return memory.kind == IR::ResourceKind::IndirectBuffer && memory.formatted;
-		});
-	});
-	if (!needed) return;
-
 	const auto type = TypeU32(state);
 	constexpr auto format_count =
 	    static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32Float) + 1u;
@@ -1358,6 +1298,57 @@ void DefineIndirectBufferFormats(ValueEmitContext& ctx) {
 	state.builder.AddFunction(spv::OpFunctionEnd);
 }
 
+} // namespace
+
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result,
+	                          state.bda_pointer_function, address);
+	return result;
+}
+
+void DefineIndirectBufferLoads(ValueEmitContext& ctx) {
+	auto& state = ctx.state;
+	for (const auto* block: state.program.blocks) {
+		for (const auto& inst: *block) {
+			const auto components = IR::BufferComponentCount(inst.GetOpcode());
+			if (components == 0u || inst.GetOpcode() == IR::ValueOpcode::ReadConstBuffer) continue;
+			const auto& memory = ctx.Memory(inst);
+			if (memory.kind != IR::ResourceKind::IndirectBuffer ||
+			    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) continue;
+			auto [entry, inserted] = state.indirect_buffer_load_functions.try_emplace(
+			    std::array {memory.data_bits, components, uint32_t(memory.coherent),
+			                uint32_t(memory.formatted)}, 0u);
+			if (!inserted) continue;
+			if (memory.formatted && state.indirect_format_function == 0u) {
+				DefineIndirectBufferFormats(ctx);
+			}
+			const auto function = entry->second = state.builder.AllocateId();
+			const auto type = TypeU32(state);
+			const auto result_type = TypeU32Composite(state, components);
+			const auto signature = state.builder.Type(
+			    spv::OpTypeFunction, result_type, type, type, type, type, type, type, type, type);
+			const auto name = "load_indirect_buffer_" + std::to_string(memory.data_bits) + "x" +
+			                  std::to_string(components) + (memory.coherent ? "_coherent" : "") +
+			                  (memory.formatted ? "_formatted" : "");
+			state.builder.AddName(function, name.c_str());
+			state.builder.AddFunction(spv::OpFunction, result_type, function,
+			                          spv::FunctionControlMaskNone, signature);
+			std::array<uint32_t, 8> args;
+			for (auto& arg: args) {
+				arg = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpFunctionParameter, type, arg);
+			}
+			EmitLabel(state, state.builder.AllocateId());
+			const auto buffer = PrepareIndirectBuffer(state, args, memory.formatted);
+			const auto result = memory.formatted ? LoadIndirectFormattedX(ctx, buffer)
+			    : LoadIndirectBuffer(ctx, buffer, memory.data_bits, components, memory.coherent);
+			state.builder.AddFunction(spv::OpReturnValue, result);
+			state.builder.AddFunction(spv::OpFunctionEnd);
+		}
+	}
+}
+
 void DefineGetBdaPointer(EmitterState& state) {
 	if (!state.program.info.uses_dma) {
 		return;
@@ -1541,16 +1532,16 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto index =
 	    Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
 	const auto access = PrepareMemoryResourceAccess(state, mem);
-	const auto exec   = ctx.Arg(inst, 1);
-	const auto ballot = ctx.Ballot(inst.Arg(1));
-	const auto low    = state.builder.AllocateId();
-	const auto high   = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), low, ballot, 0);
-	state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, ballot, 1);
+	// Elect and count from the guest EXEC words, which V_MBCNT also ranks lanes by.
+	const auto low   = ctx.Arg(inst, 1);
+	const auto high  = ctx.Arg(inst, 2);
 	const auto count = Binary(state, spv::OpIAdd, TypeU32(state),
 	                          Unary(state, spv::OpBitCount, TypeU32(state), low),
 	                          Unary(state, spv::OpBitCount, TypeU32(state), high));
-	const auto first = ctx.FirstLane(ballot);
+	const auto mask  = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), mask, low, high,
+	                          ConstantU32(state, 0), ConstantU32(state, 0));
+	const auto first = ctx.FirstLane(mask);
 	const auto source_lane =
 	    state.lane_count == 2
 	        ? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31))
@@ -1564,9 +1555,7 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto condition = AndCondition(
 	    state, is_first,
 	    AndCondition(state,
-	                 state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
-	                                                ConstantU32(state, 0))
-	                                       : exec,
+	                 Binary(state, spv::OpINotEqual, TypeBool(state), count, ConstantU32(state, 0)),
 	                 bounds));
 	const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
 		const auto value = state.builder.AllocateId();

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -154,9 +154,10 @@ uint32_t SharedIndirectBufferResource(const EmitterState& state, uint32_t resour
 	return representative;
 }
 
-void EmitMemoryOffsets(EmitterState& state) {
+void EmitMemoryBindings(EmitterState& state) {
 	const auto count = state.program.bindings.descriptor_counts[static_cast<size_t>(IR::DescriptorBindingKind::Buffers)];
 	state.memory_byte_offsets.resize(count);
+	state.memory_byte_sizes.resize(count);
 	std::vector<bool> dynamic_offsets(count);
 	for (uint32_t resource = 0; resource < state.program.info.buffers.size(); ++resource) {
 		if (SharedIndirectBufferResource(state, resource) == UINT32_MAX) continue;
@@ -174,6 +175,8 @@ void EmitMemoryOffsets(EmitterState& state) {
 			if (dynamic_offsets[i + lane]) continue;
 			state.memory_byte_offsets[i + lane] = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
 			    state, word, ConstantU32(state, lane * 8u), ConstantU32(state, 8u));
+			state.memory_byte_sizes[i + lane] =
+			    EmitShaderDataDwordLoad(state, state.program.bindings.MemorySizeDword() + i + lane);
 		}
 	}
 }
@@ -257,22 +260,28 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
 		state.builder.RequireCapability(spv::CapabilityShaderNonUniform);
 		state.builder.RequireCapability(spv::CapabilityStorageBufferArrayNonUniformIndexing);
 		state.builder.AddAnnotation(spv::OpDecorate, access.object_pointer, spv::DecorationNonUniform);
-		const auto offsets_start = state.program.bindings.memory_offset_dword +
-		    (state.program.bindings.UsesPushData() ? state.program.bindings.push_data_start_dword : 0u);
+		const auto& bindings   = state.program.bindings;
+		const auto  data_start = bindings.UsesPushData() ? bindings.push_data_start_dword : 0u;
 		const auto word = EmitShaderDataDwordLoadAt(state, EmitAddU32(
-		    state, ConstantU32(state, offsets_start),
+		    state, ConstantU32(state, bindings.memory_offset_dword + data_start),
 		    Binary(state, spv::OpShiftRightLogical, TypeU32(state), array_index, ConstantU32(state, 2))));
 		const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
 		    Binary(state, spv::OpBitwiseAnd, TypeU32(state), array_index, ConstantU32(state, 3)),
 		    ConstantU32(state, 3));
 		access.byte_offset = EmitNative<spv::OpBitFieldUExtract, IR::Type::U32>(
 		    state, word, shift, ConstantU32(state, 8));
+		access.length = EmitShaderDataDwordLoadAt(state, EmitAddU32(
+		    state, ConstantU32(state, bindings.MemorySizeDword() + data_start), array_index));
 	} else {
 		access.byte_offset = state.memory_byte_offsets[static_index];
+		access.length      = state.memory_byte_sizes[static_index];
+	}
+	// storage_buffers[i] views the binding as 1 << i byte elements.
+	const auto shift = static_cast<uint32_t>(&buffer - state.storage_buffers.data());
+	if (shift != 0u) {
+		access.length = Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.length,
+		                       ConstantU32(state, shift));
 	}
-	access.length      = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
-	                          access.object_pointer, 0);
 	return access;
 }
 
@@ -576,6 +585,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicUMin32:
 		case IR::ValueOpcode::ImageAtomicUMin64:
 		case IR::ValueOpcode::BufferAtomicUMin32:
+		case IR::ValueOpcode::BufferAtomicUMin64:
 		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
 		case IR::ValueOpcode::ImageAtomicSMax32:
 		case IR::ValueOpcode::BufferAtomicSMax32:

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -476,6 +476,8 @@ uint32_t BuiltInForInput(IR::StageInputKind kind) {
 		case IR::StageInputKind::LocalInvocationId: return spv::BuiltInLocalInvocationId;
 		case IR::StageInputKind::LocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
 		case IR::StageInputKind::GlobalInvocationId: return spv::BuiltInGlobalInvocationId;
+		case IR::StageInputKind::BaseVertex: return spv::BuiltInBaseVertex;
+		case IR::StageInputKind::BaseInstance: return spv::BuiltInBaseInstance;
 		default: return UINT32_MAX;
 	}
 }
@@ -526,7 +528,9 @@ void DefineInputs(EmitterState& state) {
 			case IR::StageInputKind::PrimitiveId:
 			case IR::StageInputKind::InstanceIndex:
 			case IR::StageInputKind::Layer:
-			case IR::StageInputKind::SampleId: type = TypeI32(state); break;
+			case IR::StageInputKind::SampleId:
+			case IR::StageInputKind::BaseVertex:
+			case IR::StageInputKind::BaseInstance: type = TypeI32(state); break;
 			case IR::StageInputKind::WorkgroupId:
 			case IR::StageInputKind::NumWorkgroups:
 			case IR::StageInputKind::LocalInvocationId:
@@ -595,6 +599,13 @@ void DefineInputs(EmitterState& state) {
 			state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationFlat);
 		}
 	}
+	if (state.program.info.helper_invocation) {
+		const auto variable = DefineInterfaceVariable(state, TypeBool(state), spv::StorageClassInput,
+		                                              "gl_HelperInvocation");
+		state.helper_invocation_variable = variable;
+		state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBuiltIn,
+		                            spv::BuiltInHelperInvocation);
+	}
 }
 
 void DefineOutputs(EmitterState& state) {
@@ -691,13 +702,15 @@ void DefineOutputs(EmitterState& state) {
 				break;
 			case IR::StageOutputKind::Parameter:
 			case IR::StageOutputKind::Mrt: {
-				const bool uint_output =
-				    binding.kind == IR::StageOutputKind::Mrt &&
-				    state.program.stage == ShaderType::Pixel &&
-				    binding.index < std::size(state.input_info.pixel->target_output_mode) &&
-				    (state.input_info.pixel->target_output_mode[binding.index] == 7u ||
-				     (state.input_info.pixel->target_packed_unorm_mask & (1u << binding.index)) != 0u);
-				const auto type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
+				auto type = TypeF32Vector(state, 4);
+				if (binding.kind == IR::StageOutputKind::Mrt &&
+				    state.program.stage == ShaderType::Pixel && binding.index < 8u) {
+					// Packed UNorm stores its encoded bits.
+					const auto bit = 1u << binding.index;
+					type = (state.input_info.pixel->target_packed_unorm_mask & bit) != 0u
+					           ? TypeU32Vector(state, 4)
+					           : MrtOutputType(state, binding.index);
+				}
 				binding.variable_id = DefineInterfaceVariable(state, type, spv::StorageClassOutput,
 				                                              binding.debug_name.c_str());
 				const bool dual_source = binding.kind == IR::StageOutputKind::Mrt &&
@@ -793,6 +806,10 @@ void DefineModule(EmitterState& state) {
 	if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) {
 		state.builder.RequireCapability(spv::CapabilitySampleRateShading);
 	}
+	if (InputVariableForKind(state, IR::StageInputKind::BaseVertex) != 0 ||
+	    InputVariableForKind(state, IR::StageInputKind::BaseInstance) != 0) {
+		state.builder.RequireCapability(spv::CapabilityDrawParameters);
+	}
 	if (state.program.info.image_gather_extended) {
 		state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -235,7 +235,7 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto op = inst.GetOpcode();
 	if (ctx.half != 0 &&
 	    (op == Op::Ballot || op == Op::ReadFirstLane || op == Op::SubgroupUMinU32 ||
-	     op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock ||
+	     op == Op::SubgroupUMaxU32 || op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock ||
 	     (op == Op::LoadAddressU32 && ctx.Memory(inst).kind == IR::ResourceKind::ScalarAddress))) {
 		// Whole-wave results, clocks and SMEM loads (SGPR operands, no EXEC) match in both halves.
 		ctx.Define(inst, ctx.other_half->Result(inst));
@@ -414,6 +414,7 @@ uint32_t TypeId(EmitterState& state, IR::Type type) {
 		case IR::Type::U32:
 		case IR::Type::F16: return TypeU32(state);
 		case IR::Type::U64: return TypeU64(state);
+		case IR::Type::Bvh8Result: return TypeBvh8Result(state);
 		case IR::Type::U32x2: return TypeU32Pair(state);
 		case IR::Type::F32: return TypeF32(state);
 		case IR::Type::F64: return TypeF64(state);
@@ -675,8 +676,8 @@ void EmitProgram(EmitterState& state) {
 	}
 	DefineGetBdaPointer(state);
 	DefineIndirectBufferLoads(ctx);
-	DefineIndirectBufferFormats(ctx);
 	DefineBvhIntersect(state);
+	DefineTrinityFunctions(state);
 	if (program.info.uses_swizzle) {
 		ctx.scratch_u32_variable = state.builder.AllocateId();
 		if (state.lane_count == 2) {
@@ -746,7 +747,7 @@ void EmitProgram(EmitterState& state) {
 		state.lds_base_dwords = EmitBinaryU32(state, spv::OpIMul, index,
 		                                     ConstantU32(state, LdsDwordCount(state)));
 	}
-	EmitMemoryOffsets(state);
+	EmitMemoryBindings(state);
 	if (program.blocks.empty()) {
 		EmitReturn(ctx);
 	} else if (state.program.dispatcher_fallback) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp

@@ -29,12 +29,14 @@ uint32_t TessellationPointer(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto pointer = state.builder.AllocateId();
 	if (kind == Attribute::Factor) {
 		EXIT_NOT_IMPLEMENTED(!inst.Arg(1).IsImmediate());
-		const auto index = inst.Arg(1).U32() / 4u;
-		const auto outer = index < 3u;
-		EXIT_NOT_IMPLEMENTED(index >= 4u);
+		const auto index       = inst.Arg(1).U32() / 4u;
+		const bool quad        = tess.domain == 2u;
+		const auto outer_count = quad ? 4u : 3u;
+		const bool outer       = index < outer_count;
+		EXIT_NOT_IMPLEMENTED(index >= (quad ? 6u : 4u));
 		state.builder.AddFunction(spv::OpAccessChain, TypePointer(state, storage, TypeF32(state)),
 		                          pointer, outer ? variable : state.tess_inner_variable,
-		                          ConstantU32(state, outer ? index : index - 3u));
+		                          ConstantU32(state, outer ? index : index - outer_count));
 		return pointer;
 	}
 	auto address = ctx.Arg(inst, 1);
@@ -128,20 +130,25 @@ void DefineTessellationInterfaces(EmitterState& state) {
 
 void DefineTessellationExecutionModes(EmitterState& state) {
 	const auto& tess = state.input_info.vertex->tess;
-	// CxVgtTfParam: triangles, integer or fractional-odd partitioning, clockwise output.
-	EXIT_NOT_IMPLEMENTED(tess.domain != 1u ||
-	                     (tess.partitioning != 0u && tess.partitioning != 2u) ||
-	                     tess.output_topology != 2u);
+	// Triangle or quad domain, integer or fractional partitioning, triangle output.
+	EXIT_NOT_IMPLEMENTED((tess.domain != 1u && tess.domain != 2u) ||
+	                     (tess.partitioning != 0u && tess.partitioning != 2u && tess.partitioning != 3u) ||
+	                     (tess.output_topology != 2u && tess.output_topology != 3u));
 	state.builder.RequireCapability(spv::CapabilityTessellation);
 	if (state.program.stage == ShaderType::TessellationControl) {
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeOutputVertices,
 		                               tess.output_control_points);
 	} else {
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeTriangles);
+		state.builder.AddExecutionMode(state.main_func, tess.domain == 1u
+		                                                   ? spv::ExecutionModeTriangles
+		                                                   : spv::ExecutionModeQuads);
 		state.builder.AddExecutionMode(
 		    state.main_func, tess.partitioning == 0u ? spv::ExecutionModeSpacingEqual
-		                                             : spv::ExecutionModeSpacingFractionalOdd);
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeVertexOrderCw);
+		                     : tess.partitioning == 2u ? spv::ExecutionModeSpacingFractionalOdd
+		                                                : spv::ExecutionModeSpacingFractionalEven);
+		state.builder.AddExecutionMode(state.main_func, tess.output_topology == 2u
+		                                                   ? spv::ExecutionModeVertexOrderCw
+		                                                   : spv::ExecutionModeVertexOrderCcw);
 	}
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp

@@ -0,0 +1,208 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t Bits(EmitterState& s, uint32_t value, uint32_t offset, uint32_t count) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(s), result, value,
+	                      ConstantU32(s, offset), ConstantU32(s, count));
+	return result;
+}
+
+uint32_t SetBits(EmitterState& s, uint32_t value, uint32_t field, uint32_t offset, uint32_t count) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpBitFieldInsert, TypeU32(s), result, value, field,
+	                      ConstantU32(s, offset), ConstantU32(s, count));
+	return result;
+}
+
+uint32_t Pair(EmitterState& s, uint32_t first, uint32_t second) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Pair(s), result, first, second);
+	return result;
+}
+
+uint32_t StackIndex(EmitterState& s, uint32_t control, uint32_t ring) {
+	return EmitAddU32(s, Bits(s, control, 15, 14),
+	    Binary(s, spv::OpShiftLeftLogical, TypeU32(s), ring, ConstantU32(s, 5)));
+}
+
+uint32_t StackPush(EmitterState& s, const MemoryResourceAccess& memory,
+                   uint32_t control, uint32_t node, uint32_t capacity) {
+	const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
+	const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5);
+	const auto index = StackIndex(s, control, ring);
+	EmitIfCondition(s, EmitMemoryElementInBounds(s, memory, index), [&] {
+		s.builder.AddFunction(spv::OpStore, EmitMemoryElementPointer(s, memory, index), node);
+	});
+	control = Binary(s, spv::OpBitwiseOr, u, control,
+	    Select(s, u, Binary(s, spv::OpIEqual, b, count, capacity), ConstantU32(s, 1u << 30), zero));
+	control = Select(s, u, Binary(s, spv::OpIEqual, b, parent, capacity),
+	    Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control);
+	control = SetBits(s, control, EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+	    EmitAddU32(s, count, one), capacity), 0, 5);
+	control = SetBits(s, control, EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+	    EmitAddU32(s, parent, one), capacity), 5, 5);
+	const auto next = EmitAddU32(s, ring, one);
+	return SetBits(s, control, Select(s, u, Binary(s, spv::OpIEqual, b, next, capacity), zero, next), 10, 5);
+}
+
+uint32_t StackPop(EmitterState& s, const MemoryResourceAccess& memory,
+                  uint32_t control, uint32_t capacity) {
+	const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
+	const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5);
+	const auto has_parent = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero);
+	const auto returning = Binary(s, spv::OpLogicalAnd, b, has_parent,
+	    Binary(s, spv::OpIEqual, b, parent, zero));
+	control = SetBits(s, control, Select(s, u, returning, one, zero), 31, 1);
+	control = Select(s, u, returning,
+	    Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control);
+	const auto overflow = Binary(s, spv::OpINotEqual, b, Bits(s, control, 30, 1), zero);
+	const auto empty = Pair(s,
+	    Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, 0x5fffffff)),
+	    Select(s, u, overflow, ConstantU32(s, 0xffffffff), ConstantU32(s, 0xfffffffe)));
+	return EmitValueOrDefaultIfCondition(s, Binary(s, spv::OpINotEqual, b, count, zero),
+	    TypeU32Pair(s), empty, [&] {
+		control = SetBits(s, control, Binary(s, spv::OpISub, u, count, one), 0, 5);
+		control = SetBits(s, control, Binary(s, spv::OpISub, u, parent, one), 5, 5);
+		const auto previous = Select(s, u, Binary(s, spv::OpIEqual, b, ring, zero),
+		    Binary(s, spv::OpISub, u, capacity, one), Binary(s, spv::OpISub, u, ring, one));
+		control = SetBits(s, control, previous, 10, 5);
+		const auto index = StackIndex(s, control, previous);
+		const auto node = EmitValueOrDefaultIfCondition(s,
+		    EmitMemoryElementInBounds(s, memory, index), u, zero, [&] {
+			const auto value = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpLoad, u, value, EmitMemoryElementPointer(s, memory, index));
+			return value;
+		});
+		const auto instance = Binary(s, spv::OpIEqual, b, Bits(s, node, 0, 3), ConstantU32(s, 6));
+		control = EmitValueOrDefaultIfCondition(s, instance, u, control, [&] {
+			const auto nested = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero);
+			const auto discarded = Binary(s, spv::OpBitwiseOr, u,
+			    SetBits(s, control, Bits(s, control, 5, 5), 0, 5), ConstantU32(s, 1u << 30));
+			return Binary(s, spv::OpBitwiseOr, u,
+			    SetBits(s, Select(s, u, nested, discarded, control), zero, 5, 5), ConstantU32(s, 1u << 29));
+		});
+		return Pair(s, control, node);
+	});
+}
+
+void DefineBvhStack(EmitterState& s) {
+	if (!s.program.info.bvh_stack) return;
+	const auto u = TypeU32(s), b = TypeBool(s), vec4 = TypeU32Vector(s, 4), pair = TypeU32Pair(s);
+	const auto zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
+	const bool local = s.lds_storage_class == spv::StorageClassFunction;
+	const bool buffer = s.lds_storage_class == spv::StorageClassStorageBuffer;
+	std::vector<uint32_t> types {u, u, vec4, vec4, u};
+	if (local) types.push_back(TypeU32ArrayPointer(s, spv::StorageClassFunction, LdsDwordCount(s)));
+	if (buffer) types.push_back(u);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, pair, types);
+	s.bvh_stack_function = s.builder.AllocateId();
+	s.builder.AddName(s.bvh_stack_function, "trinity_bvh_stack");
+	s.builder.AddFunction(spv::OpFunction, pair, s.bvh_stack_function, spv::FunctionControlMaskNone, signature);
+	std::vector<uint32_t> args;
+	for (const auto type: types) {
+		args.push_back(s.builder.AllocateId());
+		s.builder.AddFunction(spv::OpFunctionParameter, type, args.back());
+	}
+	const auto old_lds = s.lds_variable, old_base = s.lds_base_dwords;
+	if (local) s.lds_variable = args.back();
+	if (buffer) s.lds_base_dwords = args.back();
+	const auto memory = PrepareMemoryResourceAccess(s, {.kind = IR::ResourceKind::Lds});
+	const auto entry = s.builder.AllocateId();
+	EmitLabel(s, entry);
+	const auto last = args[1], capacity = args[4];
+	const auto walkback = Binary(s, spv::OpINotEqual, b, Bits(s, last, 3, 29), ConstantU32(s, 0x1fffffff));
+	const auto last4 = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, vec4, last4, last, last, last, last);
+	const auto found = [&](uint32_t values) {
+		return Unary(s, spv::OpAny, b, Binary(s, spv::OpIEqual, TypeBoolVector(s, 4), values, last4));
+	};
+	const auto initial_valid = Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, walkback),
+	    Binary(s, spv::OpLogicalOr, b, found(args[2]), found(args[3])));
+	const auto header = s.builder.AllocateId(), body = s.builder.AllocateId();
+	const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpBranch, header);
+	EmitLabel(s, header);
+	const auto index = s.builder.AllocateId(), control = s.builder.AllocateId();
+	const auto valid = s.builder.AllocateId(), skip_low = s.builder.AllocateId(), skip_high = s.builder.AllocateId();
+	const auto index_phi = s.builder.AddDeferredPhi(u, index, 2);
+	const auto control_phi = s.builder.AddDeferredPhi(u, control, 2);
+	const auto valid_phi = s.builder.AddDeferredPhi(b, valid, 2);
+	const auto low_phi = s.builder.AddDeferredPhi(b, skip_low, 2);
+	const auto high_phi = s.builder.AddDeferredPhi(b, skip_high, 2);
+	const auto more = Binary(s, spv::OpLogicalAnd, b, valid,
+	    Binary(s, spv::OpULessThan, b, index, ConstantU32(s, 8)));
+	s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask);
+	s.builder.AddFunction(spv::OpBranchConditional, more, body, merge);
+	EmitLabel(s, body);
+	const auto high = Binary(s, spv::OpUGreaterThanEqual, b, index, ConstantU32(s, 4));
+	const auto high4 = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, TypeBoolVector(s, 4), high4, high, high, high, high);
+	const auto values = Select(s, vec4, high4, args[3], args[2]);
+	const auto node = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpVectorExtractDynamic, u, node, values, Bits(s, index, 0, 2));
+	const auto sentinel = Binary(s, spv::OpIEqual, b, Bits(s, node, 3, 29), ConstantU32(s, 0x1fffffff));
+	const auto enabled = Unary(s, spv::OpLogicalNot, b, Select(s, b, high, skip_high, skip_low));
+	const auto marker = Binary(s, spv::OpLogicalAnd, b, sentinel, enabled);
+	const auto next_low = Binary(s, spv::OpLogicalOr, b, skip_low,
+	    Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 1, 1), zero)));
+	const auto next_high = Binary(s, spv::OpLogicalOr, b, skip_high,
+	    Binary(s, spv::OpLogicalAnd, b, high,
+	        Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 2, 1), zero))));
+	const auto next_valid = Unary(s, spv::OpLogicalNot, b,
+	    Binary(s, spv::OpLogicalAnd, b, walkback, Binary(s, spv::OpIEqual, b, node, last)));
+	const auto push = Binary(s, spv::OpLogicalAnd, b, next_valid,
+	    Unary(s, spv::OpLogicalNot, b,
+	        Binary(s, spv::OpLogicalOr, b, sentinel, Select(s, b, high, next_high, next_low))));
+	const auto next_control = EmitValueOrDefaultIfCondition(s, push, u, control, [&] {
+		return StackPush(s, memory, control, node, capacity);
+	});
+	s.builder.AddFunction(spv::OpBranch, cont);
+	EmitLabel(s, cont);
+	const auto next_index = Binary(s, spv::OpISub, u, index, one);
+	s.builder.AddFunction(spv::OpBranch, header);
+	s.builder.PatchDeferredPhi(index_phi, 0, ConstantU32(s, 7), entry);
+	s.builder.PatchDeferredPhi(index_phi, 1, next_index, cont);
+	s.builder.PatchDeferredPhi(control_phi, 0, args[0], entry);
+	s.builder.PatchDeferredPhi(control_phi, 1, next_control, cont);
+	s.builder.PatchDeferredPhi(valid_phi, 0, initial_valid, entry);
+	s.builder.PatchDeferredPhi(valid_phi, 1, next_valid, cont);
+	s.builder.PatchDeferredPhi(low_phi, 0, ConstantBool(s, false), entry);
+	s.builder.PatchDeferredPhi(low_phi, 1, next_low, cont);
+	s.builder.PatchDeferredPhi(high_phi, 0, ConstantBool(s, false), entry);
+	s.builder.PatchDeferredPhi(high_phi, 1, next_high, cont);
+	EmitLabel(s, merge);
+	s.builder.AddFunction(spv::OpReturnValue, StackPop(s, memory, control, capacity));
+	s.builder.AddFunction(spv::OpFunctionEnd);
+	if (local) s.lds_variable = old_lds;
+	if (buffer) s.lds_base_dwords = old_base;
+}
+
+} // namespace
+
+void DefineTrinityFunctions(EmitterState& s) {
+	DefineBvhStack(s);
+	DefineTrinityBvhIntersect(s);
+}
+
+uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto capacity = ctx.Memory(inst).offset;
+	if (s.bvh_stack_function == 0 || capacity < 8 || capacity > 31)
+		ctx.Fail(inst, "invalid BVH stack function or capacity");
+	const std::array values {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)};
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 4), TypeU32Pair(s),
+	    ConstantU32CompositeZero(s, 2), [&] {
+		const auto result = s.builder.AllocateId();
+		std::vector<uint32_t> call {spv::OpFunctionCall, TypeU32Pair(s), result,
+		    s.bvh_stack_function, values[0], values[1], values[2], values[3], ConstantU32(s, capacity)};
+		if (s.lds_storage_class == spv::StorageClassFunction) call.push_back(s.lds_variable);
+		if (s.lds_storage_class == spv::StorageClassStorageBuffer) call.push_back(s.lds_base_dwords);
+		s.builder.AddFunction(call);
+		return result;
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinityBvh.cpp

@@ -0,0 +1,511 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t TypeBvh8Result(EmitterState& s) {
+	if (s.bvh8_result_type == 0) {
+		const auto v = TypeU32Vector(s, 4);
+		s.bvh8_result_type = s.builder.Type(spv::OpTypeStruct, v, v, v, v, TypeU32(s));
+	}
+	return s.bvh8_result_type;
+}
+
+namespace {
+using Vec3 = std::array<uint32_t, 3>;
+
+struct Bvh8Emitter {
+	EmitterState& s;
+	const uint32_t u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), w = TypeU64(s);
+	const uint32_t v4 = TypeU32Vector(s, 4), v3 = TypeF32Vector(s, 3);
+	uint32_t triangle_function = 0, boxes_function = 0, decompress_function = 0;
+
+	uint32_t C(uint32_t x) { return ConstantU32(s, x); }
+	uint32_t And(uint32_t x, uint32_t y) { return Binary(s, spv::OpBitwiseAnd, u, x, y); }
+	uint32_t Or(uint32_t x, uint32_t y) { return Binary(s, spv::OpBitwiseOr, u, x, y); }
+	uint32_t Add(uint32_t x, uint32_t y) { return EmitAddU32(s, x, y); }
+	uint32_t Sub(uint32_t x, uint32_t y) { return Binary(s, spv::OpISub, u, x, y); }
+	uint32_t Shl(uint32_t x, uint32_t n) { return Binary(s, spv::OpShiftLeftLogical, u, x, C(n)); }
+	uint32_t Shr(uint32_t x, uint32_t n) { return Binary(s, spv::OpShiftRightLogical, u, x, C(n)); }
+	uint32_t Eq(uint32_t x, uint32_t y) { return Binary(s, spv::OpIEqual, b, x, y); }
+	uint32_t Both(uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); }
+	uint32_t Either(uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); }
+	uint32_t Not(uint32_t x) { return Unary(s, spv::OpLogicalNot, b, x); }
+	uint32_t Bit(uint32_t x, uint32_t n) { return Not(Eq(And(x, C(1u << n)), C(0))); }
+	uint32_t Choose(uint32_t condition, uint32_t x, uint32_t y) { return Select(s, u, condition, x, y); }
+	uint32_t Float(uint32_t x) { return Unary(s, spv::OpBitcast, f, x); }
+	uint32_t Raw(uint32_t x) { return Unary(s, spv::OpBitcast, u, x); }
+	uint32_t Fop(spv::Op op, uint32_t x, uint32_t y) { return Binary(s, op, f, x, y); }
+	uint32_t Fcmp(spv::Op op, uint32_t x, uint32_t y) { return Binary(s, op, b, x, y); }
+	uint32_t Min(uint32_t x, uint32_t y) { return EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s, x, y); }
+	uint32_t Max(uint32_t x, uint32_t y) { return EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s, x, y); }
+	uint32_t Extract(uint32_t type, uint32_t value, uint32_t index) {
+		const auto id = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpCompositeExtract, type, id, value, index);
+		return id;
+	}
+	uint32_t Make(uint32_t type, std::span<const uint32_t> values) {
+		const auto id = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpCompositeConstruct, type, id, values);
+		return id;
+	}
+	uint32_t Make4(uint32_t x, uint32_t y, uint32_t z, uint32_t t) {
+		return Make(v4, std::array{x, y, z, t});
+	}
+	Vec3 Unpack3(uint32_t value) {
+		return {Extract(f, value, 0), Extract(f, value, 1), Extract(f, value, 2)};
+	}
+	uint32_t Word(uint32_t block, uint32_t index) {
+		const auto offset = Unary(s, spv::OpUConvert, w, Shl(index, 2));
+		const auto address = Binary(s, spv::OpIAdd, w, block, offset);
+		const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
+		const auto id = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpLoad, u, id, pointer, spv::MemoryAccessAlignedMask, 4u);
+		return id;
+	}
+	uint32_t Call(uint32_t type, uint32_t function, std::span<const uint32_t> args) {
+		const auto id = s.builder.AllocateId();
+		std::vector<uint32_t> call {spv::OpFunctionCall, type, id, function};
+		call.insert(call.end(), args.begin(), args.end());
+		s.builder.AddFunction(call);
+		return id;
+	}
+	std::vector<uint32_t> Begin(uint32_t& function, const char* name, uint32_t result,
+	                            std::span<const uint32_t> types) {
+		function = s.builder.AllocateId();
+		s.builder.AddName(function, name);
+		s.builder.AddFunction(spv::OpFunction, result, function, spv::FunctionControlMaskNone,
+		    s.builder.Type(spv::OpTypeFunction, result, types));
+		std::vector<uint32_t> args;
+		for (auto type: types) {
+			args.push_back(s.builder.AllocateId());
+			s.builder.AddFunction(spv::OpFunctionParameter, type, args.back());
+		}
+		EmitLabel(s, s.builder.AllocateId());
+		return args;
+	}
+	void End(uint32_t result) {
+		s.builder.AddFunction(spv::OpReturnValue, result);
+		s.builder.AddFunction(spv::OpFunctionEnd);
+	}
+	uint32_t Result(const std::array<uint32_t, 10>& dst, const Vec3& org, const Vec3& dir,
+	                uint32_t instance) {
+		const std::array groups {Make4(dst[0], dst[1], dst[2], dst[3]),
+		    Make4(dst[4], dst[5], dst[6], dst[7]), Make4(dst[8], dst[9], Raw(org[0]), Raw(org[1])),
+		    Make4(Raw(org[2]), Raw(dir[0]), Raw(dir[1]), Raw(dir[2])), instance};
+		return Make(TypeBvh8Result(s), groups);
+	}
+	uint32_t NormalizeFlags(uint32_t flags) {
+		flags = Choose(Bit(flags, 2), And(flags, C(~0x30u)), flags);
+		return Or(flags, Choose(Eq(And(flags, C(0x30)), C(0x30)), C(0x100), C(0)));
+	}
+
+	void DefineTriangle() {
+		const auto a = Begin(triangle_function, "trinity_triangle", v4,
+		    std::array{w, u, u, u, v3, v3});
+		const auto block = a[0], slot = a[1], flags = a[2], descriptor = a[3];
+		const auto origin = Unpack3(a[4]), direction = Unpack3(a[5]);
+		const auto flag_word = Word(block, C(15));
+		const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag_word, Shl(slot, 3));
+		const auto bary = Bit(descriptor, 24);
+		const auto pointer_flags = Both(bary, Bit(descriptor, 23));
+		const auto tri_flags = Choose(pointer_flags, remap, C(0));
+		const auto procedural = Bit(tri_flags, 6);
+		const auto raw = EmitValueOrDefaultIfCondition(s, Not(procedural), v4,
+		    Make4(C(0), C(0x3f800000), C(0x3f800000), C(0x3f800000)), [&] {
+			std::array<Vec3, 3> positions;
+			const std::array vertices {slot, Choose(Eq(slot, C(0)), C(1), C(3)), C(2)};
+			for (uint32_t vertex = 0; vertex < 3; ++vertex)
+				for (uint32_t axis = 0; axis < 3; ++axis)
+					positions[vertex][axis] = Float(Word(block,
+					    Add(Binary(s, spv::OpIMul, u, vertices[vertex], C(3)), C(axis))));
+			return EmitBvhTriangle(s, positions, origin, direction, remap, Add(flag_word, slot), bary);
+		});
+		std::array<uint32_t, 4> result;
+		for (uint32_t i = 0; i < 4; ++i) result[i] = Extract(u, raw, i);
+		auto missed = Eq(result[0], C(0x7f800000));
+		const auto flip = Both(Both(Not(procedural), Not(missed)),
+		    Binary(s, spv::OpLogicalNotEqual, b, Bit(flags, 3), Bit(tri_flags, 5)));
+		for (uint32_t i = 0; i < 4; ++i)
+			result[i] = Binary(s, spv::OpBitwiseXor, u, result[i],
+			    Choose(i < 2 ? flip : Both(flip, bary), C(0x80000000), C(0)));
+		const auto front = Fcmp(spv::OpFOrdGreaterThan, Float(result[1]), ConstantF32Value(s, 0.0f));
+		const auto opaque = Both(Either(Bit(tri_flags, 7), Bit(flags, 0)), Not(Bit(flags, 1)));
+		missed = Either(missed, Either(Both(opaque, Bit(flags, 6)), Both(Not(opaque), Bit(flags, 7))));
+		missed = Either(missed, Both(Both(Not(Bit(tri_flags, 4)), Not(procedural)),
+		    Select(s, b, front, Bit(flags, 5), Bit(flags, 4))));
+		missed = Either(missed, Select(s, b, procedural, Bit(flags, 9), Bit(flags, 8)));
+		const auto sign = [&](uint32_t value, uint32_t enabled) {
+			return Choose(pointer_flags, Or(And(value, C(0x7fffffff)),
+			    Choose(enabled, C(0x80000000), C(0))), value);
+		};
+		result[2] = sign(result[2], Either(Not(opaque), procedural));
+		result[3] = sign(result[3], Not(opaque));
+		const std::array miss {C(0x7f800000), C(0x3f800000), Choose(bary, C(0), C(~0u)), C(0)};
+		for (uint32_t i = 0; i < 4; ++i) result[i] = Choose(missed, miss[i], result[i]);
+		End(Make(v4, result));
+	}
+
+	uint32_t Instance(uint32_t block, uint32_t flags, uint32_t mask, Vec3 org, Vec3 dir) {
+		const auto lo = Word(block, C(12)), hi = Word(block, C(13));
+		const auto inst_flags = Shr(hi, 22);
+		auto merged = Or(And(Or(flags, inst_flags), C(0x3f0)), And(inst_flags, C(4)));
+		merged = Or(merged, And(Binary(s, spv::OpBitwiseXor, u, flags, inst_flags), C(8)));
+		merged = Or(merged, Choose(Either(Bit(flags, 0), Both(Bit(inst_flags, 0), Not(Bit(flags, 1)))), C(1), C(0)));
+		merged = Or(merged, Choose(Either(Bit(flags, 1), Both(Bit(inst_flags, 1), Not(Bit(flags, 0)))), C(2), C(0)));
+		const auto user = Word(block, C(15));
+		std::array<uint32_t, 10> dst;
+		dst.fill(C(~0u));
+		dst[2] = lo;
+		dst[3] = Or(And(hi, C(0x3fffff)), Shl(merged, 22));
+		dst[6] = And(user, C(0xffffff));
+		dst[7] = Choose(Eq(And(mask, Shr(user, 24)), C(0)), C(~0u), Word(block, C(14)));
+		Vec3 transformed_org, transformed_dir;
+		for (uint32_t axis = 0; axis < 3; ++axis) {
+			Vec3 row;
+			for (uint32_t i = 0; i < 3; ++i) row[i] = Float(Word(block, C(axis * 4 + i)));
+			const auto dot = [&](const Vec3& value) {
+				return Fop(spv::OpFAdd, Fop(spv::OpFAdd, Fop(spv::OpFMul, row[0], value[0]),
+				    Fop(spv::OpFMul, row[1], value[1])), Fop(spv::OpFMul, row[2], value[2]));
+			};
+			transformed_org[axis] = Fop(spv::OpFAdd, dot(org), Float(Word(block, C(axis * 4 + 3))));
+			transformed_dir[axis] = dot(dir);
+		}
+		return Result(dst, transformed_org, transformed_dir, C(1));
+	}
+
+	void DefineDecompress();
+	void DefineBoxes();
+	uint32_t Boxes(uint32_t first, uint32_t second, uint32_t full, uint32_t compressed,
+	               uint32_t flags, uint32_t descriptor, uint32_t extent, uint32_t org, uint32_t dir);
+	void DefineIntersect();
+};
+
+void Bvh8Emitter::DefineDecompress() {
+	const auto a = Begin(decompress_function, "trinity_shared_exponent", f, std::array{u, u, b});
+	const auto exponent = a[0], packed = a[1], minimum = a[2];
+	const auto negative = Bit(packed, 17), mantissa = And(packed, C(0x1ffff));
+	const auto outward = Eq(Choose(negative, C(1), C(0)), Choose(minimum, C(1), C(0)));
+	const auto nonzero = Not(Eq(mantissa, C(0)));
+	const auto highest = EmitGlsl<GLSLstd450FindUMsb, IR::Type::U32>(s, mantissa);
+	const auto shift = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, Sub(C(16), highest), exponent);
+	const auto decoded_exp = Choose(nonzero, Sub(exponent, shift),
+	    Choose(Both(outward, Binary(s, spv::OpUGreaterThan, b, exponent, C(17))), Sub(exponent, C(17)), C(0)));
+	// Zero mantissas take a separate result, but their unused shifts must still be defined.
+	const auto amount = Add(C(7), Choose(nonzero, shift, C(0)));
+	const auto trail = Choose(Both(outward, Not(Eq(decoded_exp, C(255)))),
+	    Sub(Binary(s, spv::OpShiftLeftLogical, u, C(1), amount), C(1)), C(0));
+	const auto fraction = Choose(nonzero,
+	    And(Or(Binary(s, spv::OpShiftLeftLogical, u, mantissa, amount), trail), C(0x7fffff)),
+	    Choose(outward, C(0x7fffff), C(0)));
+	End(Float(Or(Or(fraction, Shl(decoded_exp, 23)), Choose(negative, C(0x80000000), C(0)))));
+}
+
+void Bvh8Emitter::DefineBoxes() {
+	const auto result_type = s.builder.Type(spv::OpTypeStruct, v4, v4);
+	const auto a = Begin(boxes_function, "trinity_box_slot", result_type,
+	    std::array{w, w, b, b, u, u, f, v3, v3});
+	const auto first = a[0], second = a[1], full = a[2], compressed = a[3];
+	const auto flags = a[4], descriptor = a[5], extent = a[6];
+	const auto org = Unpack3(a[7]), dir = Unpack3(a[8]);
+	const auto zero = ConstantF32Value(s, 0.0f), infinity = ConstantF32(s, 0x7f800000);
+	Vec3 inverse;
+	for (uint32_t i = 0; i < 3; ++i) inverse[i] = Fop(spv::OpFDiv, ConstantF32Value(s, 1.0f), dir[i]);
+	const auto packed = [&](uint32_t offset, uint32_t width) {
+		const auto word = Shr(offset, 5), shift = And(offset, C(31));
+		const auto lo = Binary(s, spv::OpShiftRightLogical, u, Word(first, word), shift);
+		const auto next = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, Add(word, C(1)), C(15));
+		const auto hi = Binary(s, spv::OpShiftLeftLogical, u, Word(first, next), And(Sub(C(32), shift), C(31)));
+		return And(Or(lo, Choose(Eq(shift, C(0)), C(0), hi)), C((1u << width) - 1));
+	};
+	const auto format = Choose(compressed, C(0), Choose(full, C(1), C(2)));
+	const auto entry_label = s.current_label;
+	const auto header = s.builder.AllocateId(), body = s.builder.AllocateId();
+	const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId();
+	const auto initial_nodes = Make4(C(~0u), C(~0u), C(~0u), C(~0u));
+	const auto initial_keys = Make4(C(0x7f800000), C(0x7f800000), C(0x7f800000), C(0x7f800000));
+	const auto base = And(Word(first, C(0)), C(0x1fffffff));
+	s.builder.AddFunction(spv::OpBranch, header);
+	EmitLabel(s, header);
+	const auto child = s.builder.AllocateId(), prefix = s.builder.AllocateId();
+	const auto nodes = s.builder.AllocateId(), keys = s.builder.AllocateId();
+	const auto child_phi = s.builder.AddDeferredPhi(u, child, 2);
+	const auto prefix_phi = s.builder.AddDeferredPhi(u, prefix, 2);
+	const auto nodes_phi = s.builder.AddDeferredPhi(v4, nodes, 2);
+	const auto keys_phi = s.builder.AddDeferredPhi(v4, keys, 2);
+	const auto more = Binary(s, spv::OpULessThan, b, child, C(4));
+	s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask);
+	s.builder.AddFunction(spv::OpBranchConditional, more, body, merge);
+	EmitLabel(s, body);
+	const auto kind = packed(Add(C(29), Binary(s, spv::OpIMul, u, child, C(3))), 3);
+	const auto direct_node = Word(first, child);
+	const auto node = Choose(compressed, Or(Shl(prefix, 3), kind), direct_node);
+	const auto child_flags = Choose(compressed,
+	    And(packed(Add(C(41), Shl(child, 2)), 4), Choose(Eq(child, C(3)), C(7), C(15))), C(0));
+	const auto step = Choose(Eq(kind, C(2)), C(4),
+	    Choose(Either(Either(Eq(kind, C(3)), Eq(kind, C(5))), Eq(kind, C(6))), C(2), C(1)));
+	const auto next_prefix = Add(prefix, step);
+	const auto bounds_type = s.builder.Type(spv::OpTypeStruct, v3, v3);
+	const auto box = EmitIndexSwitch(s, format, 3, bounds_type, [&](uint32_t encoding) {
+		std::array<uint32_t, 6> bounds;
+		if (encoding == 0) {
+			const auto yz = Word(first, C(2));
+			const std::array exponents {Shr(Word(first, C(1)), 24), And(yz, C(255)), And(Shr(yz, 8), C(255))};
+			for (uint32_t component = 0; component < 6; ++component) {
+				const auto offset = Add(C(80 + component * 18), Binary(s, spv::OpIMul, u, child, C(108)));
+				bounds[component] = Call(f, decompress_function,
+				    std::array{exponents[component % 3], packed(offset, 18), ConstantBool(s, component < 3)});
+			}
+		} else if (encoding == 1) {
+			for (uint32_t component = 0; component < 6; ++component) {
+				const auto word = Add(C(4 + component), Binary(s, spv::OpIMul, u, child, C(6)));
+				const auto block = Select(s, w, Binary(s, spv::OpULessThan, b, word, C(16)), first, second);
+				bounds[component] = Float(Word(block, And(word, C(15))));
+			}
+		} else {
+			for (uint32_t pair = 0; pair < 3; ++pair) {
+				const auto word = Add(C(4 + pair), Binary(s, spv::OpIMul, u, child, C(3)));
+				const auto value = EmitGlsl<GLSLstd450UnpackHalf2x16, IR::Type::F32x2>(s, Word(first, word));
+				bounds[pair * 2] = Extract(f, value, 0);
+				bounds[pair * 2 + 1] = Extract(f, value, 1);
+			}
+		}
+		return Make(bounds_type, std::array{Make(v3, std::span(bounds).first(3)),
+		    Make(v3, std::span(bounds).last(3))});
+	});
+	const auto lo_bound = Unpack3(Extract(v3, box, 0)), hi_bound = Unpack3(Extract(v3, box, 1));
+	Vec3 near, far;
+	auto valid = Not(Eq(And(child_flags, C(3)), C(3)));
+	for (uint32_t axis = 0; axis < 3; ++axis) {
+		valid = Both(valid, Fcmp(spv::OpFOrdLessThanEqual, lo_bound[axis], hi_bound[axis]));
+		const auto lo = Fop(spv::OpFMul, Fop(spv::OpFSub, lo_bound[axis], org[axis]), inverse[axis]);
+		const auto hi = Fop(spv::OpFMul, Fop(spv::OpFSub, hi_bound[axis], org[axis]), inverse[axis]);
+		const auto forward = Fcmp(spv::OpFOrdGreaterThanEqual, inverse[axis], zero);
+		near[axis] = Select(s, f, forward, lo, hi);
+		far[axis] = Select(s, f, forward, hi, lo);
+	}
+	const auto entry = Max(Max(near[0], near[1]), near[2]);
+	const auto exit = Min(Min(far[0], far[1]), far[2]);
+	const auto exit_bits = Raw(Select(s, f, Fcmp(spv::OpFOrdEqual, exit, zero), zero, exit));
+	const auto grown = Float(EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+	    Add(exit_bits, And(Shr(descriptor, 23), C(255))), C(0x7f800000)));
+	valid = Both(valid, Both(Fcmp(spv::OpFOrdGreaterThanEqual, exit, zero),
+	    Both(Fcmp(spv::OpFOrdLessThan, entry, extent), Fcmp(spv::OpFOrdLessThanEqual, entry, grown))));
+	const auto opaque = Both(Either(Bit(child_flags, 0), Bit(flags, 0)), Not(Bit(flags, 1)));
+	const auto nonopaque = Both(Either(Bit(child_flags, 1), Bit(flags, 1)), Not(Bit(flags, 0)));
+	const auto culled = Either(Either(Both(opaque, Bit(flags, 6)), Both(nonopaque, Bit(flags, 7))),
+	    Either(Both(Bit(child_flags, 2), Bit(flags, 8)), Both(Bit(child_flags, 3), Bit(flags, 9))));
+	valid = Both(valid, Not(culled));
+	const auto heuristic = And(Shr(descriptor, 21), C(3));
+	auto key = Max(entry, zero);
+	key = Select(s, f, Eq(heuristic, C(1)), Unary(s, spv::OpFNegate, f, Fop(spv::OpFSub, exit, entry)), key);
+	key = Select(s, f, Eq(heuristic, C(2)), Fop(spv::OpFAdd, entry, exit), key);
+	const auto next_nodes = s.builder.AllocateId(), next_keys = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpVectorInsertDynamic, v4, next_nodes, nodes, Choose(valid, node, C(~0u)), child);
+	s.builder.AddFunction(spv::OpVectorInsertDynamic, v4, next_keys, keys, Raw(Select(s, f, valid, key, infinity)), child);
+	s.builder.AddFunction(spv::OpBranch, cont);
+	EmitLabel(s, cont);
+	const auto next_child = Add(child, C(1));
+	s.builder.AddFunction(spv::OpBranch, header);
+	s.builder.PatchDeferredPhi(child_phi, 0, C(0), entry_label);
+	s.builder.PatchDeferredPhi(child_phi, 1, next_child, cont);
+	s.builder.PatchDeferredPhi(prefix_phi, 0, base, entry_label);
+	s.builder.PatchDeferredPhi(prefix_phi, 1, next_prefix, cont);
+	s.builder.PatchDeferredPhi(nodes_phi, 0, initial_nodes, entry_label);
+	s.builder.PatchDeferredPhi(nodes_phi, 1, next_nodes, cont);
+	s.builder.PatchDeferredPhi(keys_phi, 0, initial_keys, entry_label);
+	s.builder.PatchDeferredPhi(keys_phi, 1, next_keys, cont);
+	EmitLabel(s, merge);
+	End(Make(result_type, std::array{nodes, keys}));
+}
+
+uint32_t Bvh8Emitter::Boxes(uint32_t first, uint32_t second, uint32_t full, uint32_t compressed,
+                            uint32_t flags, uint32_t descriptor, uint32_t extent,
+                            uint32_t org, uint32_t dir) {
+	return Call(s.builder.Type(spv::OpTypeStruct, v4, v4), boxes_function,
+	    std::array{first, second, full, compressed, flags, descriptor, extent, org, dir});
+}
+
+void Bvh8Emitter::DefineIntersect() {
+	const auto result_type = TypeBvh8Result(s);
+	const auto a = Begin(s.bvh8_intersect_function, "trinity_bvh_intersect", result_type,
+	    std::array{v4, w, u, f, u, v3, v3});
+	const auto descriptor = a[0], encoded_base = a[1], node = a[2], extent = a[3], mask = a[4];
+	const auto org = Unpack3(a[5]), dir = Unpack3(a[6]);
+	std::array<uint32_t, 4> d;
+	for (uint32_t i = 0; i < 4; ++i) d[i] = Extract(u, descriptor, i);
+	const auto kind = And(node, C(7));
+	const auto triangle = Binary(s, spv::OpULessThan, b, kind, C(2));
+	const auto instance = Eq(kind, C(6));
+	const auto full = Either(Eq(kind, C(2)), Eq(kind, C(5)));
+	const auto big = Either(Eq(kind, C(2)), Eq(kind, C(3)));
+	const auto compressed = Both(Not(full), Bit(d[3], 20));
+	const auto flags = NormalizeFlags(Choose(Bit(d[3], 23),
+	    Unary(s, spv::OpUConvert, u, Binary(s, spv::OpShiftRightLogical, w, encoded_base, C(54))), C(0)));
+	const auto base = Binary(s, spv::OpShiftLeftLogical, w, PackU64(s, d[0], And(d[1], C(255))), C(8));
+	const auto lane_base = Binary(s, spv::OpShiftLeftLogical, w,
+	    Binary(s, spv::OpBitwiseAnd, w, encoded_base, ConstantU64(s, (uint64_t{1} << 54) - 1)), C(3));
+	const auto offset = Binary(s, spv::OpShiftLeftLogical, w, Unary(s, spv::OpUConvert, w, Shr(node, 3)), C(6));
+	const auto address = Binary(s, spv::OpIAdd, w, base, Binary(s, spv::OpIAdd, w, lane_base, offset));
+	const auto capacity = Binary(s, spv::OpIAdd, w,
+	    PackU64(s, d[2], And(d[3], C(0x3ff))), ConstantU64(s, 1));
+	const auto limit = Binary(s, spv::OpIAdd, w, base,
+	    Binary(s, spv::OpShiftLeftLogical, w, capacity, C(6)));
+	auto valid = Both(Eq(Shr(d[3], 28), C(8)),
+	    Eq(And(Unary(s, spv::OpUConvert, u, encoded_base), C(7)), C(0)));
+	valid = Both(valid, Both(Not(Eq(kind, C(7))), Either(Not(instance), Bit(d[3], 22))));
+	valid = Both(valid, Not(Unary(s, spv::OpIsNan, b, extent)));
+	for (uint32_t i = 0; i < 3; ++i) {
+		valid = Both(valid, Not(Either(Unary(s, spv::OpIsNan, b, org[i]), Unary(s, spv::OpIsInf, b, org[i]))));
+		valid = Both(valid, Not(Unary(s, spv::OpIsNan, b, dir[i])));
+	}
+	const auto opaque = Bit(flags, 0), nonopaque = Bit(flags, 1);
+	valid = Both(valid, Not(Either(Either(Both(opaque, Bit(flags, 6)), Both(nonopaque, Bit(flags, 7))),
+	    Both(Bit(flags, 6), Bit(flags, 7)))));
+	const auto fetch_size = Select(s, w, full, ConstantU64(s, 128), ConstantU64(s, 64));
+	const auto fetch = [&](uint32_t at, uint32_t enabled) {
+		const auto end = Binary(s, spv::OpIAdd, w, at, fetch_size);
+		const auto in_bounds = Both(Binary(s, spv::OpULessThanEqual, b, end, limit),
+		    Binary(s, spv::OpULessThanEqual, b, end, ConstantU64(s, (uint64_t{1} << 48) - 1)));
+		return EmitValueOrDefaultIfCondition(s, Both(Both(valid, enabled), in_bounds), w,
+		    ConstantU64(s, 0), [&] { return GetBdaPointer(s, at); });
+	};
+	const auto first = fetch(address, ConstantBool(s, true));
+	const auto present = [&](uint32_t pointer) { return Not(Binary(s, spv::OpIEqual, b, pointer, ConstantU64(s, 0))); };
+	const auto tail = [&](uint32_t at, uint32_t pointer) {
+		return EmitValueOrDefaultIfCondition(s, Both(full, present(pointer)), w, ConstantU64(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, w, at, ConstantU64(s, 64)));
+		});
+	};
+	const auto second = tail(address, first);
+	const auto available = Both(present(first), Either(Not(full), present(second)));
+	std::array<uint32_t, 10> invalid;
+	invalid.fill(C(~0u));
+	for (uint32_t slot = 0; slot < 2; ++slot) {
+		invalid[slot * 4] = Choose(triangle, C(0x7f800000), C(~0u));
+		invalid[slot * 4 + 1] = Choose(triangle, C(0x3f800000), C(~0u));
+		invalid[slot * 4 + 3] = Choose(triangle, C(0), C(~0u));
+	}
+	const auto default_result = Result(invalid, org, dir, C(0));
+	const auto value = EmitValueOrDefaultIfCondition(s, Both(instance, available), result_type, default_result, [&] {
+		return Instance(first, flags, mask, org, dir);
+	});
+	const auto result = EmitValueOrDefaultIfCondition(s, Both(Not(instance), valid), result_type, value, [&] {
+		const auto second_address = Binary(s, spv::OpIAdd, w, address, fetch_size);
+		const auto next_first = fetch(second_address, big);
+		const auto next_second = tail(second_address, next_first);
+		const auto next_available = Both(present(next_first), Either(Not(full), present(next_second)));
+		const auto triangles = EmitValueOrDefaultIfCondition(s, triangle, result_type, default_result, [&] {
+			auto dst = invalid;
+			for (uint32_t slot = 0; slot < 2; ++slot) {
+				const auto active = Both(available, slot == 0 ? ConstantBool(s, true) : Eq(kind, C(1)));
+				const auto absent = Both(available, Both(Eq(kind, C(0)), Bit(d[3], 24)));
+				const auto miss = Make4(C(0x7f800000), C(0x3f800000),
+				    slot == 1 ? Choose(absent, C(0), C(~0u)) : C(~0u), C(0));
+				const auto hit = EmitValueOrDefaultIfCondition(s, active, v4, miss, [&] {
+					return Call(v4, triangle_function, std::array{first, C(slot), flags, d[3], a[5], a[6]});
+				});
+				for (uint32_t i = 0; i < 4; ++i) dst[slot * 4 + i] = Extract(u, hit, i);
+				dst[8 + slot] = EmitValueOrDefaultIfCondition(s, active, u, C(~0u), [&] { return Word(first, C(14)); });
+			}
+			return Result(dst, org, dir, C(0));
+		});
+		return EmitValueOrDefaultIfCondition(s, Not(triangle), result_type, triangles, [&] {
+			auto dst = invalid;
+			const auto slot_type = s.builder.Type(spv::OpTypeStruct, v4, v4);
+			const auto miss = Make(slot_type, std::array{Make4(C(~0u), C(~0u), C(~0u), C(~0u)),
+			    Make4(C(0x7f800000), C(0x7f800000), C(0x7f800000), C(0x7f800000))});
+			std::array<uint32_t, 8> keys;
+			for (uint32_t slot = 0; slot < 2; ++slot) {
+				const auto hit = EmitValueOrDefaultIfCondition(s, slot == 0 ? available : next_available,
+				    slot_type, miss, [&] {
+					return Boxes(slot == 0 ? first : next_first, slot == 0 ? second : next_second,
+					    full, compressed, flags, d[1], extent, a[5], a[6]);
+				});
+				const auto indices = Extract(v4, hit, 0), depths = Extract(v4, hit, 1);
+				for (uint32_t i = 0; i < 4; ++i) {
+					dst[slot * 4 + i] = Extract(u, indices, i);
+					keys[slot * 4 + i] = Extract(u, depths, i);
+				}
+			}
+			const auto sort = Bit(d[1], 31), wide = Both(sort, Bit(d[3], 21)), tri_first = Bit(d[1], 20);
+			constexpr std::array<std::pair<uint32_t, uint32_t>, 19> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2},
+			    {4,5}, {6,7}, {4,6}, {5,7}, {5,6}, {0,4}, {1,5}, {1,4}, {2,6}, {3,7}, {3,6}, {2,4}, {3,5}, {3,4}}};
+			const auto unsorted = Make(slot_type, std::array{Make(v4, std::span(dst).first(4)),
+			    Make(v4, std::span(dst).subspan(4, 4))});
+			const auto sorted_result = EmitValueOrDefaultIfCondition(s, sort, slot_type, unsorted, [&] {
+				auto sorted_dst = dst;
+				for (size_t step = 0; step < network.size(); ++step) {
+					const auto [x, y] = network[step];
+					const auto is_tri = [&](uint32_t index) { return Binary(s, spv::OpULessThan, b, And(index, C(7)), C(2)); };
+					const auto xt = is_tri(sorted_dst[x]), yt = is_tri(sorted_dst[y]);
+					auto swap = Fcmp(spv::OpFOrdGreaterThan, Float(keys[x]), Float(keys[y]));
+					swap = Select(s, b, Both(tri_first, Binary(s, spv::OpLogicalNotEqual, b, xt, yt)), yt, swap);
+					if (step >= 10) swap = Both(wide, swap);
+					const auto key = keys[x], index = sorted_dst[x];
+					const auto later = [&](uint32_t position) {
+						return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) {
+							return pair.first == position || pair.second == position;
+						});
+					};
+					if (later(x)) keys[x] = Choose(swap, keys[y], key);
+					if (later(y)) keys[y] = Choose(swap, key, keys[y]);
+					sorted_dst[x] = Choose(swap, sorted_dst[y], index);
+					sorted_dst[y] = Choose(swap, index, sorted_dst[y]);
+				}
+				return Make(slot_type, std::array{Make(v4, std::span(sorted_dst).first(4)),
+				    Make(v4, std::span(sorted_dst).subspan(4, 4))});
+			});
+			for (uint32_t slot = 0; slot < 2; ++slot) {
+				const auto indices = Extract(v4, sorted_result, slot);
+				for (uint32_t i = 0; i < 4; ++i) dst[slot * 4 + i] = Extract(u, indices, i);
+			}
+			return Result(dst, org, dir, C(0));
+		});
+	});
+	End(result);
+}
+
+} // namespace
+
+void DefineTrinityBvhIntersect(EmitterState& s) {
+	if (!s.program.info.bvh8) return;
+	Bvh8Emitter e {s};
+	e.DefineDecompress();
+	e.DefineTriangle();
+	e.DefineBoxes();
+	e.DefineIntersect();
+}
+
+uint32_t EmitBvh8Extract(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto index = inst.Arg(1).U32();
+	if (index > 16) ctx.Fail(inst, "invalid BVH8 result component");
+	const auto result = ctx.state.builder.AllocateId();
+	if (index == 16)
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), result,
+		    ctx.Arg(inst, 0), 4u);
+	else
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), result,
+		    ctx.Arg(inst, 0), index / 4, index % 4);
+	return result;
+}
+
+uint32_t EmitBvh8Intersect(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto* ray = ctx.ImageAddress(inst.Arg(1));
+	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh8_intersect_function == 0)
+		ctx.Fail(inst, "BVH8 intersection requires the complete ray and shared function");
+	Bvh8Emitter e {s};
+	const auto vector = [&](uint32_t first) {
+		return e.Make(e.v3, Vec3{e.Float(ctx.Arg(*ray, first)), e.Float(ctx.Arg(*ray, first + 1)),
+		    e.Float(ctx.Arg(*ray, first + 2))});
+	};
+	const std::array args {ctx.Arg(inst, 0), PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1)),
+	    ctx.Arg(*ray, 10), e.Float(ctx.Arg(*ray, 2)), ctx.Arg(*ray, 3), vector(4), vector(7)};
+	const auto type = TypeBvh8Result(s);
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), type,
+	    s.builder.Constant(spv::OpConstantNull, type), [&] {
+		return e.Call(type, s.bvh8_intersect_function, args);
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -1710,21 +1710,25 @@ Graph Structurize(const Graph& graph) {
 	return GotoStructurizer(graph).Run();
 }
 
-bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code) {
-	// Vector comparisons and carry results can write a scalar pair in wave64.
+bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code, uint32_t wave_size) {
+	// A vector lane mask (comparison or carry) spans wave_size / 32 SGPRs; a lane read writes one.
 	if (inst.family == Decoder::Family::VOP1 || inst.family == Decoder::Family::VOP2 ||
 	    inst.family == Decoder::Family::VOPC || inst.family == Decoder::Family::VOP3 ||
 	    inst.family == Decoder::Family::VOP3P) {
+		const bool lane_read =
+		    inst.opcode == Opcode::V_READLANE_B32 || inst.opcode == Opcode::V_READFIRSTLANE_B32;
+		const uint32_t size = lane_read ? 1u : wave_size / 32u;
 		uint32_t destination = 0;
 		for (const auto* operand: {&inst.dst, &inst.dst2})
-			if (ScalarOperandCode(*operand, destination) && code >= destination && code <= destination + 1u)
+			if (ScalarOperandCode(*operand, destination) && code >= destination && code < destination + size)
 				return true;
 	}
 	return InstructionWritesScalarCode(inst, code);
 }
 
 uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
-                              uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls) {
+                              uint32_t before, uint32_t code, uint32_t wave_size,
+                              std::vector<uint32_t>& crossed_calls) {
 	EXIT_IF(before >= program.instructions.size());
 	const auto found = std::ranges::find_if(graph.blocks, [before](const auto& block) {
 		return block.inst_begin <= before && before < block.inst_end;
@@ -1743,11 +1747,11 @@ uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& grap
 		for (uint32_t i = point.second; i > block->inst_begin;) {
 			const auto& inst = program.instructions[--i];
 			if (inst.opcode == Opcode::S_SWAPPC_B64) {
-				EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code));
+				EXIT_NOT_IMPLEMENTED(MayWriteScalarRegister(inst, code, wave_size));
 				crossed_calls.push_back(i);
 				continue;
 			}
-			if (MayWriteScalarRegister(inst, code)) { index = i; break; }
+			if (MayWriteScalarRegister(inst, code, wave_size)) { index = i; break; }
 		}
 		if (index != UINT32_MAX || block->id == graph.entry_block) {
 			EXIT_IF(definition && *definition != index);

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -113,9 +113,10 @@ struct Graph {
 };
 
 Graph       BuildGraph(const Decoder::Program& program);
-bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code);
+bool MayWriteScalarRegister(const Decoder::Instruction& inst, uint32_t code, uint32_t wave_size);
 uint32_t FindScalarDefinition(const Decoder::Program& program, const Graph& graph,
-                              uint32_t before, uint32_t code, std::vector<uint32_t>& crossed_calls);
+                              uint32_t before, uint32_t code, uint32_t wave_size,
+                              std::vector<uint32_t>& crossed_calls);
 // Returns structured control flow or failure diagnostics without changing the native graph.
 // On failure, failure_block is an original block ID or UINT32_MAX.
 Graph       Structurize(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -14,12 +14,6 @@ struct MimgSampleInfo {
 	uint32_t    flags    = 0;
 };
 
-struct MimgGatherInfo {
-	uint32_t encoding = 0;
-	Opcode   decoded  = Opcode::UNSUPPORTED;
-	uint32_t flags    = 0;
-};
-
 struct MimgAtomicInfo {
 	uint32_t encoding    = 0;
 	Opcode   decoded     = Opcode::UNSUPPORTED;
@@ -73,7 +67,7 @@ constexpr uint32_t ImageSampleAddressComponents(uint32_t flags, ImageDimension d
 	if ((flags & ImageSampleFlagBias) != 0) {
 		components++;
 	}
-	if ((flags & ImageSampleFlagLod) != 0) {
+	if ((flags & (ImageSampleFlagLod | ImageSampleFlagLodClamp)) != 0) {
 		components++;
 	}
 	if ((flags & ImageSampleFlagDerivative) != 0) {
@@ -181,23 +175,72 @@ constexpr MimgSampleInfo MIMG_SAMPLE_OPCODE_LIST[] = {
          ImageSampleFlagOffset | ImageSampleFlagAdjust},
 };
 
-constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
-    {0x44u, Opcode::IMAGE_GATHER4_L, ImageSampleFlagLod},
-    {0x47u, Opcode::IMAGE_GATHER4_LZ, ImageSampleFlagLevelZero},
-    {0x48u, Opcode::IMAGE_GATHER4_C, ImageSampleFlagCompare},
-    {0x4cu, Opcode::IMAGE_GATHER4_C_L, ImageSampleFlagCompare | ImageSampleFlagLod},
-    {0x4fu, Opcode::IMAGE_GATHER4_C_LZ,
-     ImageSampleFlagCompare | ImageSampleFlagLevelZero},
-    {0x57u, Opcode::IMAGE_GATHER4_LZ_O,
-     ImageSampleFlagLevelZero | ImageSampleFlagOffset},
-    {0x58u, Opcode::IMAGE_GATHER4_C_O,
-     ImageSampleFlagCompare | ImageSampleFlagOffset},
-    {0x5fu, Opcode::IMAGE_GATHER4_C_LZ_O,
+constexpr MimgSampleInfo MIMG_GATHER_OPCODE_LIST[] = {
+    {0x40u, "image_gather4", 0},
+    {0x41u, "image_gather4_cl", ImageSampleFlagLodClamp},
+    {0x44u, "image_gather4_l", ImageSampleFlagLod},
+    {0x45u, "image_gather4_b", ImageSampleFlagBias},
+    {0x46u, "image_gather4_b_cl", ImageSampleFlagBias | ImageSampleFlagLodClamp},
+    {0x47u, "image_gather4_lz", ImageSampleFlagLevelZero},
+    {0x48u, "image_gather4_c", ImageSampleFlagCompare},
+    {0x49u, "image_gather4_c_cl", ImageSampleFlagCompare | ImageSampleFlagLodClamp},
+    {0x4cu, "image_gather4_c_l", ImageSampleFlagCompare | ImageSampleFlagLod},
+    {0x4du, "image_gather4_c_b", ImageSampleFlagCompare | ImageSampleFlagBias},
+    {0x4eu, "image_gather4_c_b_cl",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp},
+    {0x4fu, "image_gather4_c_lz", ImageSampleFlagCompare | ImageSampleFlagLevelZero},
+    {0x50u, "image_gather4_o", ImageSampleFlagOffset},
+    {0x51u, "image_gather4_cl_o", ImageSampleFlagLodClamp | ImageSampleFlagOffset},
+    {0x54u, "image_gather4_l_o", ImageSampleFlagLod | ImageSampleFlagOffset},
+    {0x55u, "image_gather4_b_o", ImageSampleFlagBias | ImageSampleFlagOffset},
+    {0x56u, "image_gather4_b_cl_o",
+     ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset},
+    {0x57u, "image_gather4_lz_o", ImageSampleFlagLevelZero | ImageSampleFlagOffset},
+    {0x58u, "image_gather4_c_o", ImageSampleFlagCompare | ImageSampleFlagOffset},
+    {0x59u, "image_gather4_c_cl_o",
+     ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset},
+    {0x5cu, "image_gather4_c_l_o",
+     ImageSampleFlagCompare | ImageSampleFlagLod | ImageSampleFlagOffset},
+    {0x5du, "image_gather4_c_b_o",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset},
+    {0x5eu, "image_gather4_c_b_cl_o",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+         ImageSampleFlagOffset},
+    {0x5fu, "image_gather4_c_lz_o",
      ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset},
-    {0x61u, Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal},
-    {0xc8u, Opcode::IMAGE_GATHER4_C, ImageSampleFlagCompare | ImageSampleFlagAdjust},
-    {0xd8u, Opcode::IMAGE_GATHER4_C_O,
+    {0x61u, "image_gather4h", ImageSampleFlagGatherHorizontal},
+    {0xc0u, "image_gather4_a", ImageSampleFlagAdjust},
+    {0xc1u, "image_gather4_cl_a", ImageSampleFlagLodClamp | ImageSampleFlagAdjust},
+    {0xc5u, "image_gather4_b_a", ImageSampleFlagBias | ImageSampleFlagAdjust},
+    {0xc6u, "image_gather4_b_cl_a",
+     ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagAdjust},
+    {0xc8u, "image_gather4_c_a", ImageSampleFlagCompare | ImageSampleFlagAdjust},
+    {0xc9u, "image_gather4_c_cl_a",
+     ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagAdjust},
+    {0xcdu, "image_gather4_c_b_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagAdjust},
+    {0xceu, "image_gather4_c_b_cl_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+         ImageSampleFlagAdjust},
+    {0xd0u, "image_gather4_o_a", ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd1u, "image_gather4_cl_o_a",
+     ImageSampleFlagLodClamp | ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd5u, "image_gather4_b_o_a",
+     ImageSampleFlagBias | ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd6u, "image_gather4_b_cl_o_a",
+     ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+         ImageSampleFlagAdjust},
+    {0xd8u, "image_gather4_c_o_a",
      ImageSampleFlagCompare | ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd9u, "image_gather4_c_cl_o_a",
+     ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+         ImageSampleFlagAdjust},
+    {0xddu, "image_gather4_c_b_o_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset |
+         ImageSampleFlagAdjust},
+    {0xdeu, "image_gather4_c_b_cl_o_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+         ImageSampleFlagOffset | ImageSampleFlagAdjust},
 };
 
 constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = {
@@ -219,13 +262,13 @@ constexpr auto MIMG_SAMPLE_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_SAMPLE_OPCO
 constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCODE_LIST);
 constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST);
 
-Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather,
+Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgSampleInfo* gather,
                         const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return Opcode::IMAGE_SAMPLE;
 	}
 	if (gather != nullptr) {
-		return gather->decoded;
+		return Opcode::IMAGE_GATHER4;
 	}
 	if (atomic != nullptr) {
 		return atomic->decoded;
@@ -238,36 +281,24 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
+		case 0xe5u: return Opcode::IMAGE_BVH8_INTERSECT_RAY;
 		case 0xe6u:
 		case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }
 
-uint32_t DecodeMimgSampleFlags(const MimgSampleInfo* sample, const MimgGatherInfo* gather) {
-	if (sample != nullptr) {
-		return sample->flags;
-	}
-	if (gather != nullptr) {
-		return gather->flags;
-	}
-	return 0;
-}
-
 uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
-                                     const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                                     const MimgAtomicInfo* atomic) {
-	if (sample != nullptr) {
-		return ImageSampleAddressComponents(sample->flags, dimension);
-	}
-	if (gather != nullptr) {
-		return ImageSampleAddressComponents(gather->flags, dimension);
+                                     const MimgSampleInfo* sampled, const MimgAtomicInfo* atomic) {
+	if (sampled != nullptr) {
+		return ImageSampleAddressComponents(sampled->flags, dimension);
 	}
 	if (atomic != nullptr) {
 		return ImageCoordComponents(dimension);
 	}
 
 	switch (opcode) {
+		case 0xe5u:
 		case 0xe6u: return 11u;
 		case 0xe7u: return 12u;
 		case 0x0eu: return 1u;
@@ -290,33 +321,31 @@ uint32_t CountDmaskComponents(uint32_t dmask) {
 
 } // namespace
 
-ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, uint32_t component) {
-	const auto width = [flags](uint32_t index) {
-		if ((flags & ImageSampleFlagA16) == 0u) return 32u;
-		uint32_t cursor = 0;
-		if ((flags & ImageSampleFlagOffset) != 0u) {
-			if (index == cursor++) return 32u;
-		}
-		if ((flags & ImageSampleFlagBias) != 0u) {
-			if (index == cursor++) return 16u;
-		}
-		if ((flags & ImageSampleFlagCompare) != 0u && index == cursor) return 32u;
-		return 16u;
-	};
-	uint32_t offset = 0;
-	for (uint32_t index = 0; index < component; index++) {
-		const auto bits = width(index);
-		if (bits == 32u) offset = (offset + 31u) & ~31u;
-		offset += bits;
+ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, ImageDimension dimension,
+                                                  uint32_t component) {
+	if ((flags & ImageSampleFlagA16) == 0u) return {component * 32u, 32u};
+	// A16 offsets, bias and PCF reference each take a whole VGPR, the bias as a half in its low
+	// 16 bits. The other components pack two per VGPR, except that each 1D derivative and the last
+	// two 3D ones take the low half of their own VGPR.
+	const bool     has_bias    = (flags & ImageSampleFlagBias) != 0u;
+	const uint32_t bias_index  = (flags & ImageSampleFlagOffset) != 0u ? 1u : 0u;
+	const uint32_t whole_vgprs = std::popcount(
+	    flags & (ImageSampleFlagOffset | ImageSampleFlagBias | ImageSampleFlagCompare));
+	if (component < whole_vgprs) {
+		return {component * 32u, has_bias && component == bias_index ? 16u : 32u};
 	}
-	const auto bits = width(component);
-	if (bits == 32u) offset = (offset + 31u) & ~31u;
-	return {offset, bits};
+	const uint32_t axes =
+	    (flags & ImageSampleFlagDerivative) != 0u ? ImageGradientComponents(dimension) : 0u;
+	const uint32_t unpaired = axes % 2u * 2u;
+	const uint32_t paired   = axes * 2u - unpaired;
+	const uint32_t index    = component - whole_vgprs;
+	const uint32_t half     = index + std::clamp(index, paired, paired + unpaired) - paired;
+	return {whole_vgprs * 32u + half * 16u, 16u};
 }
 
-uint32_t ImageAddressDwordCount(uint32_t flags, uint32_t components) {
+uint32_t ImageAddressDwordCount(uint32_t flags, ImageDimension dimension, uint32_t components) {
 	if (components == 0u) return 0u;
-	const auto last = ImageAddressComponentLayout(flags, components - 1u);
+	const auto last = ImageAddressComponentLayout(flags, dimension, components - 1u);
 	return (last.bit_offset + last.bit_width + 31u) / 32u;
 }
 
@@ -329,16 +358,17 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const auto     dimension  = DecodeImageDimension((word0 >> 3u) & 0x7u);
 	const uint32_t word_count = 2u + nsa_dwords;
 
-	const uint32_t vdata  = (word1 >> 8u) & 0xffu;
-	const uint32_t vaddr  = word1 & 0xffu;
-	const uint32_t srsrc  = (word1 >> 16u) & 0x1fu;
-	const uint32_t ssamp  = (word1 >> 21u) & 0x1fu;
-	const bool     r128   = ((word0 >> 15u) & 0x1u) != 0u;
-	const bool     a16    = ((word1 >> 30u) & 0x1u) != 0u;
-	const bool     d16    = ((word1 >> 31u) & 0x1u) != 0u;
-	const auto*    sample = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
-	const auto*    gather = Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
-	const auto*    atomic = Detail::FindOpcode(MIMG_ATOMIC_OPS, opcode);
+	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
+	const uint32_t vaddr   = word1 & 0xffu;
+	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
+	const uint32_t ssamp   = (word1 >> 21u) & 0x1fu;
+	const bool     r128    = ((word0 >> 15u) & 0x1u) != 0u;
+	const bool     a16     = ((word1 >> 30u) & 0x1u) != 0u;
+	const bool     d16     = ((word1 >> 31u) & 0x1u) != 0u;
+	const auto*    sample  = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
+	const auto*    gather  = Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
+	const auto*    atomic  = Detail::FindOpcode(MIMG_ATOMIC_OPS, opcode);
+	const auto*    sampled = sample != nullptr ? sample : gather;
 
 	inst.pc                 = pc;
 	inst.word_count         = word_count;
@@ -351,11 +381,11 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.data_dwords        = d16 ? (inst.data_components + 1u) / 2u : inst.data_components;
 	inst.glc                = ((word0 >> 13u) & 1u) != 0;
 	inst.slc                = ((word0 >> 25u) & 1u) != 0;
-	inst.image_sample_flags = DecodeMimgSampleFlags(sample, gather);
+	inst.image_sample_flags = sampled != nullptr ? sampled->flags : 0u;
 	if (a16) {
 		inst.image_sample_flags |= ImageSampleFlagA16;
 	}
-	if ((sample != nullptr || gather != nullptr) && (word0 & (1u << 12u)) != 0u) {
+	if (sampled != nullptr && (word0 & (1u << 12u)) != 0u) {
 		inst.image_sample_flags |= ImageSampleFlagUnnormalized;
 	}
 	inst.image_dimension  = dimension;
@@ -364,8 +394,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	for (uint32_t i = 0; i < nsa_dwords * 4u; i++) {
 		inst.image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
 	}
-	inst.image_address_components =
-	    DecodeMimgAddressComponents(opcode, dimension, sample, gather, atomic);
+	inst.image_address_components = DecodeMimgAddressComponents(opcode, dimension, sampled, atomic);
 	SetRawWords(inst, code, word_index, word_count);
 
 	if (inst.opcode == Opcode::UNSUPPORTED) {
@@ -389,8 +418,8 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			}
 		}
 	}
-	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
-	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
+	const bool supports_d16 = sampled != nullptr || opcode == 0x00u || opcode == 0x01u ||
+	                          opcode == 0x08u || opcode == 0x09u;
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
@@ -400,6 +429,36 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		               "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf");
 	}
 
+	if (opcode == 0xe5u) {
+		inst.data_components = inst.data_dwords = 10;
+		if (a16 || d16 || !r128 || inst.dmask != 0xfu ||
+		    (nsa_dwords != 0u && nsa_dwords != 2u) || vdata > 246u ||
+		    (nsa_dwords == 0u && vaddr > 245u)) {
+			SetUnsupported(inst, Family::MIMG, opcode, "invalid BVH8 ray or destination encoding");
+		}
+		if (nsa_dwords == 2u) {
+			const auto origin = inst.image_nsa_addr[3], direction = inst.image_nsa_addr[4];
+			const auto node = inst.image_nsa_addr[5];
+			if (origin > 253u || direction > 253u)
+				SetUnsupported(inst, Family::MIMG, opcode, "BVH8 ray registers exceed VGPR range");
+			for (uint32_t i = 0; i < 3; ++i) {
+				inst.image_nsa_addr[3 + i] = origin + i;
+				inst.image_nsa_addr[6 + i] = direction + i;
+			}
+			inst.image_nsa_addr[9] = node;
+		}
+		const auto origin = nsa_dwords != 0 ? inst.image_nsa_addr[3] : vaddr + 4;
+		const auto direction = nsa_dwords != 0 ? inst.image_nsa_addr[6] : vaddr + 7;
+		bool overlapping = origin < direction + 3 && direction < origin + 3;
+		for (const auto output : {2u, 3u, 6u, 7u}) {
+			const auto reg = vdata + output;
+			overlapping |= (reg >= origin && reg < origin + 3) ||
+			               (reg >= direction && reg < direction + 3);
+		}
+		if (overlapping)
+			SetUnsupported(inst, Family::MIMG, opcode, "overlapping BVH8 instance outputs are unsupported");
+	}
+
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);
 	DecodeScalarSource(srsrc * 4u, pc, inst.src1);
@@ -407,9 +466,10 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.src_count = 3;
 }
 
-const char* MimgSampleOpcodeName(uint32_t opcode) {
-	const auto* sample = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
-	return sample != nullptr ? sample->name : nullptr;
+const char* MimgSampledOpcodeName(uint32_t opcode) {
+	const auto* sample  = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
+	const auto* sampled = sample != nullptr ? sample : Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
+	return sampled != nullptr ? sampled->name : nullptr;
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/frontend/decode/ImageOps.h

@@ -8,15 +8,16 @@ namespace Libs::Graphics::ShaderRecompiler::Decoder {
 void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                 Instruction& inst);
 
-const char* MimgSampleOpcodeName(uint32_t opcode);
+const char* MimgSampledOpcodeName(uint32_t opcode);
 
 struct ImageAddressComponent {
 	uint32_t bit_offset;
 	uint32_t bit_width;
 };
 
-ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, uint32_t component);
-uint32_t ImageAddressDwordCount(uint32_t flags, uint32_t components);
+ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, ImageDimension dimension,
+                                                  uint32_t component);
+uint32_t ImageAddressDwordCount(uint32_t flags, ImageDimension dimension, uint32_t components);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder
 

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -22,7 +22,7 @@ constexpr MemoryOpcodeInfo SMEM_OPCODE_LIST[] = {
     {0x04u, Opcode::S_LOAD_DWORDX16, 16, 32},      {0x08u, Opcode::S_BUFFER_LOAD_DWORD, 1, 32},
     {0x09u, Opcode::S_BUFFER_LOAD_DWORDX2, 2, 32}, {0x0au, Opcode::S_BUFFER_LOAD_DWORDX4, 4, 32},
     {0x0bu, Opcode::S_BUFFER_LOAD_DWORDX8, 8, 32}, {0x0cu, Opcode::S_BUFFER_LOAD_DWORDX16, 16, 32},
-    {0x25u, Opcode::S_MEMREALTIME, 2, 32},
+    {0x20u, Opcode::S_DCACHE_INV, 0, 0},           {0x25u, Opcode::S_MEMREALTIME, 2, 32},
 };
 
 constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
@@ -69,6 +69,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
     {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
     {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
     {0x52u, Opcode::BUFFER_ATOMIC_ADD_X2, 2, 32},
+    {0x56u, Opcode::BUFFER_ATOMIC_UMIN_X2, 2, 32},
     {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
     {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
     {0x71u, Opcode::BUFFER_GL0_INV, 0, 0},
@@ -95,14 +96,14 @@ constexpr MemoryOpcodeInfo MTBUF_OPCODE_LIST[] = {
 };
 
 constexpr MemoryOpcodeInfo FLAT_OPCODE_LIST[] = {
-    {0x08u, Opcode::FLAT_LOAD_UBYTE, 1, 8},     {0x09u, Opcode::FLAT_LOAD_SBYTE, 1, 8, true},
-    {0x0au, Opcode::FLAT_LOAD_USHORT, 1, 16},   {0x0bu, Opcode::FLAT_LOAD_SSHORT, 1, 16, true},
-    {0x0cu, Opcode::FLAT_LOAD_DWORD, 1, 32},    {0x0du, Opcode::FLAT_LOAD_DWORDX2, 2, 32},
-    {0x0eu, Opcode::FLAT_LOAD_DWORDX4, 4, 32},  {0x0fu, Opcode::FLAT_LOAD_DWORDX3, 3, 32},
-    {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},     {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
-    {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},   {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
-    {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32}, {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
-    {0x24u, Opcode::FLAT_LOAD_SHORT_D16, 1, 16},
+    {0x08u, Opcode::FLAT_LOAD_UBYTE, 1, 8},      {0x09u, Opcode::FLAT_LOAD_SBYTE, 1, 8, true},
+    {0x0au, Opcode::FLAT_LOAD_USHORT, 1, 16},    {0x0bu, Opcode::FLAT_LOAD_SSHORT, 1, 16, true},
+    {0x0cu, Opcode::FLAT_LOAD_DWORD, 1, 32},     {0x0du, Opcode::FLAT_LOAD_DWORDX2, 2, 32},
+    {0x0eu, Opcode::FLAT_LOAD_DWORDX4, 4, 32},   {0x0fu, Opcode::FLAT_LOAD_DWORDX3, 3, 32},
+    {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},      {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
+    {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},    {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
+    {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32},  {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
+    {0x24u, Opcode::FLAT_LOAD_SHORT_D16, 1, 16}, {0x25u, Opcode::FLAT_LOAD_SHORT_D16_HI, 1, 16},
 };
 
 constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
@@ -141,6 +142,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
     {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32},     {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
     {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
+    {0xe1u, Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32, 1, 32},
     {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
 };
 
@@ -231,6 +233,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
 
 uint32_t DsSourceCount(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return 3u;
 		case Opcode::DS_WRITE2_B32:
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
@@ -285,6 +288,9 @@ void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (inst.opcode == Opcode::UNSUPPORTED) {
 		SetUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
 	}
+	if (inst.opcode == Opcode::S_DCACHE_INV) {
+		return;
+	}
 
 	DecodeScalarDestination(sdst, pc, inst.dst);
 	if (inst.opcode == Opcode::S_MEMREALTIME) {
@@ -419,9 +425,10 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 
 	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst);
-	if (inst.opcode == Opcode::FLAT_LOAD_SHORT_D16) {
-		// D16 loads reuse partial destinations to preserve the untouched high half.
-		inst.dst.sdwa_sel = 4u;
+	if (inst.opcode == Opcode::FLAT_LOAD_SHORT_D16 ||
+	    inst.opcode == Opcode::FLAT_LOAD_SHORT_D16_HI) {
+		// D16 loads reuse partial destinations to preserve the untouched half.
+		inst.dst.sdwa_sel = inst.opcode == Opcode::FLAT_LOAD_SHORT_D16_HI ? 5u : 4u;
 	}
 	DecodeVectorGpr(addr, inst.src0);
 	inst.src_count = 1;
@@ -468,6 +475,13 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
+	if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 &&
+	    (inst.gds || inst.offset < 8u || inst.offset > 31u || data1 > 248u)) {
+		SetUnsupported(inst, Family::DS, opcode, "invalid BVH stack operands or capacity");
+	}
+	if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 && vdst == addr) {
+		SetUnsupported(inst, Family::DS, opcode, "overlapping BVH stack outputs are not implemented");
+	}
 	if (inst.gds && inst.opcode == Opcode::DS_CMPST_RTN_B32) {
 		SetUnsupported(inst, Family::DS, opcode, "DS compare-store return B32 GDS variant is not implemented");
 	}

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -70,6 +70,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x21u, Opcode::S_SWAPPC_B64},
     {0x24u, Opcode::S_AND_SAVEEXEC_B64},
     {0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
+    {0x2cu, Opcode::S_QUADMASK_B32},
     {0x2du, Opcode::S_QUADMASK_B64},
     {0x34u, Opcode::S_ABS_I32},
     {0x37u, Opcode::S_ANDN1_SAVEEXEC_B64},

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -123,10 +123,9 @@ std::string ImageSampleFlagsToString(uint32_t flags) {
 }
 
 std::string FormatMimg(const Instruction& inst) {
-	const char* sample_name =
-	    inst.opcode == Opcode::IMAGE_SAMPLE ? MimgSampleOpcodeName(inst.opcode_id) : nullptr;
+	const char*            sampled_name = MimgSampledOpcodeName(inst.opcode_id);
 	const std::string_view name =
-	    sample_name != nullptr ? sample_name : magic_enum::enum_name(inst.opcode);
+	    sampled_name != nullptr ? sampled_name : magic_enum::enum_name(inst.opcode);
 	std::string text =
 	    fmt::format("0x{:08x}: {} {}, {}, {}, {} ; dmask=0x{:x} image_dim={}", inst.pc, name,
 	                OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(),
@@ -138,22 +137,10 @@ std::string FormatMimg(const Instruction& inst) {
 	if (inst.image_r128) {
 		text += " r128=1";
 	}
-	switch (inst.opcode) {
-		case Opcode::IMAGE_SAMPLE:
-		case Opcode::IMAGE_GATHER4_L:
-		case Opcode::IMAGE_GATHER4_LZ:
-		case Opcode::IMAGE_GATHER4_C:
-		case Opcode::IMAGE_GATHER4_C_L:
-		case Opcode::IMAGE_GATHER4_C_LZ:
-		case Opcode::IMAGE_GATHER4_LZ_O:
-		case Opcode::IMAGE_GATHER4_C_O:
-		case Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Opcode::IMAGE_GATHER4H:
-			text += fmt::format(" sample_flags={} addr_components={}",
-			                    ImageSampleFlagsToString(inst.image_sample_flags).c_str(),
-			                    inst.image_address_components);
-			break;
-		default: break;
+	if (sampled_name != nullptr) {
+		text += fmt::format(" sample_flags={} addr_components={}",
+		                    ImageSampleFlagsToString(inst.image_sample_flags).c_str(),
+		                    inst.image_address_components);
 	}
 	if (inst.image_nsa_dwords != 0) {
 		text += fmt::format(" nsa_dwords={} nsa_addr=", inst.image_nsa_dwords);
@@ -258,7 +245,9 @@ void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) {
 		case 126u: operand.kind = OperandKind::ExecLo; return;
 		case 127u: operand.kind = OperandKind::ExecHi; return;
 		case 235u: operand.kind = OperandKind::SharedBase; return;
+		case 236u: operand.kind = OperandKind::SharedLimit; return;
 		case 237u: operand.kind = OperandKind::PrivateBase; return;
+		case 238u: operand.kind = OperandKind::PrivateLimit; return;
 		case 239u: operand.kind = OperandKind::PopsExitingWaveId; return;
 		case 248u:
 			operand.kind      = OperandKind::FloatInlineConstant;
@@ -459,7 +448,9 @@ std::string OperandToString(const Operand& operand) {
 		case OperandKind::M0: text = "m0"; break;
 		case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break;
 		case OperandKind::SharedBase: text = "shared_base"; break;
+		case OperandKind::SharedLimit: text = "shared_limit"; break;
 		case OperandKind::PrivateBase: text = "private_base"; break;
+		case OperandKind::PrivateLimit: text = "private_limit"; break;
 		case OperandKind::Null: text = "null"; break;
 		default: text = "unknown"; break;
 	}
@@ -526,6 +517,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_NOT_B64:
 		case Opcode::S_WQM_B32:
 		case Opcode::S_WQM_B64:
+		case Opcode::S_QUADMASK_B32:
 		case Opcode::S_QUADMASK_B64:
 		case Opcode::S_AND_SAVEEXEC_B32:
 		case Opcode::S_ORN2_SAVEEXEC_B32:
@@ -566,6 +558,7 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_BARRIER:
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_barrier", inst.pc));
+		case Opcode::S_DCACHE_INV:
 		case Opcode::BUFFER_GL0_INV:
 		case Opcode::BUFFER_GL1_INV:
 			return fmt::format("0x{:08x}: {}", inst.pc, magic_enum::enum_name(inst.opcode));
@@ -612,15 +605,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::IMAGE_LOAD_MIP:
 		case Opcode::IMAGE_GET_RESINFO:
 		case Opcode::IMAGE_GET_LOD:
-		case Opcode::IMAGE_GATHER4_L:
-		case Opcode::IMAGE_GATHER4_LZ:
-		case Opcode::IMAGE_GATHER4_C:
-		case Opcode::IMAGE_GATHER4_C_L:
-		case Opcode::IMAGE_GATHER4_C_LZ:
-		case Opcode::IMAGE_GATHER4_LZ_O:
-		case Opcode::IMAGE_GATHER4_C_O:
-		case Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Opcode::IMAGE_GATHER4H: return WithUnsupportedReason(inst, FormatMimg(inst));
+		case Opcode::IMAGE_GATHER4: return WithUnsupportedReason(inst, FormatMimg(inst));
 		case Opcode::S_LOAD_DWORD:
 		case Opcode::S_LOAD_DWORDX2:
 		case Opcode::S_LOAD_DWORDX4:
@@ -681,6 +666,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_SUB:
 		case Opcode::BUFFER_ATOMIC_SMIN:
 		case Opcode::BUFFER_ATOMIC_UMIN:
+		case Opcode::BUFFER_ATOMIC_UMIN_X2:
 		case Opcode::BUFFER_ATOMIC_SMAX:
 		case Opcode::BUFFER_ATOMIC_UMAX:
 		case Opcode::BUFFER_ATOMIC_AND:
@@ -697,6 +683,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_LOAD_USHORT:
 		case Opcode::FLAT_LOAD_SSHORT:
 		case Opcode::FLAT_LOAD_SHORT_D16:
+		case Opcode::FLAT_LOAD_SHORT_D16_HI:
 		case Opcode::FLAT_LOAD_DWORD:
 		case Opcode::FLAT_LOAD_DWORDX2:
 		case Opcode::FLAT_LOAD_DWORDX3:
@@ -753,6 +740,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_READ2ST64_B64:
 		case Opcode::DS_READ_B96:
 		case Opcode::DS_READ_B128:
+		case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32:
 		case Opcode::DS_WRITE_B8:
 		case Opcode::DS_WRITE_B16:
 		case Opcode::DS_WRITE_B8_D16_HI:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -63,6 +63,7 @@ enum class Opcode {
 	S_FLBIT_I32,
 	S_FLBIT_I32_I64,
 	S_BITREPLICATE_B64_B32,
+	S_QUADMASK_B32,
 	S_QUADMASK_B64,
 	S_GETPC_B64,
 	S_SETPC_B64,
@@ -248,7 +249,9 @@ enum class Opcode {
 	V_MIN_I32,
 	V_MAX_I32,
 	V_MUL_I32_I24,
+	V_MUL_HI_I32_I24,
 	V_MUL_U32_U24,
+	V_MUL_HI_U32_U24,
 	V_MUL_LO_U32,
 	V_MUL_HI_U32,
 	V_MUL_LO_I32,
@@ -257,6 +260,7 @@ enum class Opcode {
 	V_CVT_PK_U8_F32,
 	V_MAD_F32,
 	V_MAD_I16,
+	V_MAD_U16,
 	V_MAD_I32_I24,
 	V_MAD_U32_U24,
 	V_MAD_U64_U32,
@@ -277,16 +281,20 @@ enum class Opcode {
 	V_MIN3_I32,
 	V_MIN3_U32,
 	V_MIN3_F16,
+	V_MIN3_I16,
 	V_MIN3_U16,
 	V_MAX3_F32,
 	V_MAX3_I32,
 	V_MAX3_U32,
 	V_MAX3_F16,
+	V_MAX3_I16,
+	V_MAX3_U16,
 	V_MED3_F32,
 	V_MED3_I32,
 	V_MED3_U32,
 	V_MED3_F16,
 	V_MED3_I16,
+	V_MED3_U16,
 	V_SAD_U32,
 	V_ADD3_U32,
 	V_LSHL_ADD_U32,
@@ -354,6 +362,7 @@ enum class Opcode {
 	V_ASHRREV_I32,
 	V_LSHLREV_B64,
 	V_LSHRREV_B64,
+	V_ASHRREV_I64,
 	V_LSHLREV_B16,
 	V_LSHRREV_B16,
 	V_ASHRREV_I16,
@@ -544,6 +553,7 @@ enum class Opcode {
 	S_BUFFER_LOAD_DWORDX8,
 	S_BUFFER_LOAD_DWORDX16,
 	S_MEMREALTIME,
+	S_DCACHE_INV,
 	BUFFER_LOAD_FORMAT_X,
 	BUFFER_LOAD_FORMAT_XY,
 	BUFFER_LOAD_FORMAT_XYZ,
@@ -596,6 +606,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_SUB,
 	BUFFER_ATOMIC_SMIN,
 	BUFFER_ATOMIC_UMIN,
+	BUFFER_ATOMIC_UMIN_X2,
 	BUFFER_ATOMIC_SMAX,
 	BUFFER_ATOMIC_UMAX,
 	BUFFER_ATOMIC_AND,
@@ -612,6 +623,7 @@ enum class Opcode {
 	FLAT_LOAD_USHORT,
 	FLAT_LOAD_SSHORT,
 	FLAT_LOAD_SHORT_D16,
+	FLAT_LOAD_SHORT_D16_HI,
 	FLAT_LOAD_DWORD,
 	FLAT_LOAD_DWORDX2,
 	FLAT_LOAD_DWORDX3,
@@ -673,6 +685,7 @@ enum class Opcode {
 	DS_READ2ST64_B64,
 	DS_READ_B96,
 	DS_READ_B128,
+	DS_BVH_STACK_PUSH8_POP1_RTN_B32,
 	DS_WRITE_B8,
 	DS_WRITE_B16,
 	DS_WRITE_B8_D16_HI,
@@ -688,6 +701,7 @@ enum class Opcode {
 	DS_WRITE_ADDTID_B32,
 	DS_READ_ADDTID_B32,
 	IMAGE_BVH_INTERSECT_RAY,
+	IMAGE_BVH8_INTERSECT_RAY,
 	IMAGE_GET_RESINFO,
 	IMAGE_GET_LOD,
 	IMAGE_LOAD,
@@ -707,15 +721,7 @@ enum class Opcode {
 	IMAGE_ATOMIC_FMIN,
 	IMAGE_ATOMIC_FMAX,
 	IMAGE_SAMPLE,
-	IMAGE_GATHER4_L,
-	IMAGE_GATHER4_LZ,
-	IMAGE_GATHER4_C,
-	IMAGE_GATHER4_C_L,
-	IMAGE_GATHER4_C_LZ,
-	IMAGE_GATHER4_LZ_O,
-	IMAGE_GATHER4_C_O,
-	IMAGE_GATHER4_C_LZ_O,
-	IMAGE_GATHER4H,
+	IMAGE_GATHER4,
 	V_INTERP_P1_F32,
 	V_INTERP_P2_F32,
 	V_INTERP_MOV_F32,
@@ -766,7 +772,9 @@ enum class OperandKind {
 	M0,
 	PopsExitingWaveId,
 	SharedBase,
+	SharedLimit,
 	PrivateBase,
+	PrivateLimit,
 	Null,
 	Vgpr,
 };

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -35,7 +35,9 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
     {0x05u, Opcode::V_SUBREV_F32},
     {0x08u, Opcode::V_MUL_F32, Vop2SdwaProfile::Float32},
     {0x09u, Opcode::V_MUL_I32_I24, Vop2SdwaProfile::IntegerPartialDestination},
+    {0x0au, Opcode::V_MUL_HI_I32_I24, Vop2SdwaProfile::IntegerPartialDestination},
     {0x0bu, Opcode::V_MUL_U32_U24, Vop2SdwaProfile::IntegerPartialDestination},
+    {0x0cu, Opcode::V_MUL_HI_U32_U24, Vop2SdwaProfile::IntegerPartialDestination},
     {0x0fu, Opcode::V_MIN_F32},
     {0x10u, Opcode::V_MAX_F32},
     {0x11u, Opcode::V_MIN_I32, Vop2SdwaProfile::IntegerFullDestination},
@@ -157,6 +159,10 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x04u, Opcode::V_CVT_F64_I32},
     {0x0fu, Opcode::V_CVT_F32_F64},
     {0x10u, Opcode::V_CVT_F64_F32},
+    {0x11u, Opcode::V_CVT_F32_UBYTE0},
+    {0x12u, Opcode::V_CVT_F32_UBYTE1},
+    {0x13u, Opcode::V_CVT_F32_UBYTE2},
+    {0x14u, Opcode::V_CVT_F32_UBYTE3},
     {0x16u, Opcode::V_CVT_F64_U32},
     {0x17u, Opcode::V_TRUNC_F64},
     {0x18u, Opcode::V_CEIL_F64},
@@ -167,6 +173,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
     {0x07u, Opcode::V_CVT_U32_F32},
     {0x08u, Opcode::V_CVT_I32_F32},
     {0x0au, Opcode::V_CVT_F16_F32},
+    {0x0bu, Opcode::V_CVT_F32_F16},
     {0x0cu, Opcode::V_CVT_RPI_I32_F32},
     {0x0du, Opcode::V_CVT_FLR_I32_F32},
     {0x0eu, Opcode::V_CVT_OFF_F32_I4},
@@ -334,16 +341,20 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x152u, Opcode::V_MIN3_I32},
     {0x153u, Opcode::V_MIN3_U32},
     {0x351u, Opcode::V_MIN3_F16},
+    {0x352u, Opcode::V_MIN3_I16},
     {0x353u, Opcode::V_MIN3_U16},
     {0x154u, Opcode::V_MAX3_F32},
     {0x155u, Opcode::V_MAX3_I32},
     {0x156u, Opcode::V_MAX3_U32},
     {0x354u, Opcode::V_MAX3_F16},
+    {0x355u, Opcode::V_MAX3_I16},
+    {0x356u, Opcode::V_MAX3_U16},
     {0x157u, Opcode::V_MED3_F32},
     {0x158u, Opcode::V_MED3_I32},
     {0x159u, Opcode::V_MED3_U32},
     {0x357u, Opcode::V_MED3_F16},
     {0x358u, Opcode::V_MED3_I16},
+    {0x359u, Opcode::V_MED3_U16},
     {0x15du, Opcode::V_SAD_U32},
     {0x15eu, Opcode::V_CVT_PK_U8_F32},
     {0x178u, Opcode::V_XOR3_B32},
@@ -354,6 +365,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x16cu, Opcode::V_MUL_HI_I32},
     {0x2ffu, Opcode::V_LSHLREV_B64},
     {0x300u, Opcode::V_LSHRREV_B64},
+    {0x301u, Opcode::V_ASHRREV_I64},
     {0x303u, Opcode::V_ADD_NC_U16},
     {0x304u, Opcode::V_SUB_NC_U16},
     {0x305u, Opcode::V_MUL_LO_U16},
@@ -370,6 +382,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
     {0x311u, Opcode::V_PACK_B32_F16},
     {0x314u, Opcode::V_LSHLREV_B16},
     {0x319u, Opcode::V_SUBREV_I32},
+    {0x340u, Opcode::V_MAD_U16},
     {0x345u, Opcode::V_XAD_U32},
     {0x346u, Opcode::V_LSHL_ADD_U32},
     {0x347u, Opcode::V_ADD_LSHL_U32},
@@ -488,8 +501,10 @@ bool IsNativeVop3F16TernaryOpcode(Opcode opcode) {
 }
 
 bool IsNativeVop3B16TernaryOpcode(Opcode opcode) {
-	return opcode == Opcode::V_MIN3_U16 || opcode == Opcode::V_MED3_I16 ||
-	       opcode == Opcode::V_MAD_I16;
+	return opcode == Opcode::V_MIN3_I16 || opcode == Opcode::V_MIN3_U16 ||
+	       opcode == Opcode::V_MAX3_I16 || opcode == Opcode::V_MAX3_U16 ||
+	       opcode == Opcode::V_MED3_I16 || opcode == Opcode::V_MED3_U16 ||
+	       opcode == Opcode::V_MAD_I16 || opcode == Opcode::V_MAD_U16;
 }
 
 bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
@@ -1335,6 +1350,7 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 		case Opcode::V_SUB_NC_I16:
 		case Opcode::V_LSHLREV_B64:
 		case Opcode::V_LSHRREV_B64:
+		case Opcode::V_ASHRREV_I64:
 		case Opcode::V_LSHLREV_B16:
 		case Opcode::V_LSHRREV_B16:
 		case Opcode::V_ASHRREV_I16:
@@ -1523,7 +1539,10 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
 
 bool SupportsNativeVop3Clamp(Opcode opcode) {
 	return SupportsNativeVop3ResultModifiers(opcode) || UsesInexactClampControl(opcode) ||
-	       opcode == Opcode::V_MAD_I16;
+	       opcode == Opcode::V_MAD_I16 || opcode == Opcode::V_MAD_U16 ||
+	       opcode == Opcode::V_MIN3_I16 || opcode == Opcode::V_MAX3_I16 ||
+	       opcode == Opcode::V_MAX3_U16 || opcode == Opcode::V_MED3_U16 ||
+	       opcode == Opcode::V_MUL_HI_I32_I24 || opcode == Opcode::V_MUL_HI_U32_U24;
 }
 
 bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix,
@@ -1554,7 +1573,17 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
 		return clamp != 0u || omod != 0u || neg != 0u;
 	}
 	switch (opcode) {
+		case Opcode::V_MUL_HI_I32_I24:
+		case Opcode::V_MUL_HI_U32_U24:
+			// The high product fits in 32 bits, so clamp has no effect.
+			return abs != 0u || op_sel != 0u || omod != 0u || neg != 0u;
 		case Opcode::V_LDEXP_F32: return (abs & ~1u) != 0u || op_sel != 0u || (neg & ~1u) != 0u;
+		case Opcode::V_CVT_F32_UBYTE0:
+		case Opcode::V_CVT_F32_UBYTE1:
+		case Opcode::V_CVT_F32_UBYTE2:
+		case Opcode::V_CVT_F32_UBYTE3:
+			// Byte conversions ignore input modifiers and support all output modifiers.
+			return op_sel != 0u;
 		case Opcode::V_INTERP_P1LL_F16:
 		case Opcode::V_INTERP_P2_F16:
 			return abs != 0u || (op_sel & 0x7u) != 0u || clamp != 0u || omod != 0u || neg != 0u;

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -5,30 +5,20 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend {
 
-void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar,
-                                   bool cmpx) {
+void Translator::EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar) {
 	if (scalar) {
 		ir.SetScc(value);
 		return;
 	}
-	const auto masked = ir.LogicalAnd(ir.GetExec(), value);
-	if (cmpx) {
-		const auto mask = BallotMask(masked);
-		ir.SetExec(masked);
-		ir.SetExecLo(mask[0]);
-		ir.SetExecHi(mask[1]);
-		return;
-	}
-	WriteMask(inst.dst, masked);
+	WriteVectorMask(inst.dst, value);
 }
 
-void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar,
-                                     bool cmpx) {
-	EmitCompareResult(inst, IR::U1(IR::Value(value)), scalar, cmpx);
+void Translator::EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar) {
+	EmitCompareResult(inst, IR::U1(IR::Value(value)), scalar);
 }
 
 void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
-                                    IR::Type type, bool scalar, bool cmpx, bool swap_operands) {
+                                    IR::Type type, bool scalar, bool swap_operands) {
 	// The equality compares share their IR opcode with the unsigned ones.
 	const bool signed_64 = opcode == IR::ValueOpcode::SLessThan64 ||
 	                       opcode == IR::ValueOpcode::SLessThanEqual64 ||
@@ -49,18 +39,18 @@ void Translator::EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueO
 	if (swap_operands) {
 		std::swap(lhs, rhs);
 	}
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar, cmpx);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), scalar);
 }
 
 void Translator::EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
-                                      bool signed_value, bool cmpx) {
+                                      bool signed_value) {
 	const auto lhs = ReadU16AsU32(inst.src0, signed_value);
 	const auto rhs = ReadU16AsU32(inst.src1, signed_value);
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false);
 }
 
 void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
-                                  bool half, bool cmpx) {
+                                  bool half) {
 	const auto type = IR::ArgTypeOf(opcode, 0);
 	const auto lhs =
 	    half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, type);
@@ -68,10 +58,10 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc
 	    half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, type);
 	const IR::FPCompareFlags flags{
 	    .flush_input_denorms = !half && type == IR::Type::F32 && flush_f32_inputs};
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false);
 }
 
-void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx,
+void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered,
                                          bool is64) {
 	const auto type      = is64 ? IR::Type::F64 : IR::Type::F32;
 	const auto is_nan_op = is64 ? IR::ValueOpcode::FPIsNan64 : IR::ValueOpcode::FPIsNan32;
@@ -79,16 +69,16 @@ void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool
 	const auto rhs       = ReadOperand(inst.src1, type);
 	const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(is_nan_op, {lhs})),
 	                                    IR::U1(ir.Emit(is_nan_op, {rhs})));
-	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, cmpx);
+	EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false);
 }
 
-void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half) {
+void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool half) {
 	const auto value = half ? IR::Value(Read16LaneBits(inst.src0, false))
 	                        : ReadOperand(inst.src0, IR::Type::F32);
 	const auto mask = half ? IR::Value(Read16LaneBits(inst.src1, false))
 	                      : ReadOperand(inst.src1, IR::Type::U32);
 	const auto opcode = half ? IR::ValueOpcode::FPCmpClass16 : IR::ValueOpcode::FPCmpClass32;
-	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {value, mask})), false, cmpx);
+	EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {value, mask})), false);
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -51,33 +51,33 @@ void Translator::S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin)
 
 void Translator::S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation,
                             bool negate_exec, bool negate_source, bool write_64) {
-	if (!write_64) {
-		// Read the encoded scalar word and preserve EXEC_HI, including in wave32.
-		const auto old = ir.GetExecLo();
-		const auto src = ReadU32(inst.src0);
+	const auto combine = [&](IR::U32 old, IR::U32 src) {
 		const auto lhs = negate_exec ? ir.BitwiseNot(old) : old;
 		const auto rhs = negate_source ? ir.BitwiseNot(src) : src;
-		IR::U32 result;
-		switch (operation) {
-			case IR::ValueOpcode::LogicalAnd: result = ir.BitwiseAnd(lhs, rhs); break;
-			case IR::ValueOpcode::LogicalOr: result = ir.BitwiseOr(lhs, rhs); break;
-			default: EXIT("unsupported SAVEEXEC operation");
-		}
+		return operation == IR::ValueOpcode::LogicalAnd ? ir.BitwiseAnd(lhs, rhs)
+		                                                : ir.BitwiseOr(lhs, rhs);
+	};
+	if (!write_64) {
+		// Read the encoded scalar word and preserve EXEC_HI, including in wave32.
+		const auto old    = ir.GetExecLo();
+		const auto result = combine(old, ReadU32(inst.src0));
 		WriteRawU32(inst.dst, old);
 		WriteRawU32(ConditionOperand(Decoder::OperandKind::ExecLo), result);
 		ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
 		return;
 	}
-	const auto old    = ir.GetExec();
-	const auto src    = ReadMask(inst.src0);
-	const auto lhs    = negate_exec ? ir.LogicalNot(old) : old;
-	const auto rhs    = negate_source ? ir.LogicalNot(src) : src;
-	const auto result = IR::U1(ir.Emit(operation, {lhs, rhs}));
-	WriteMask(inst.dst, old, true);
-	const auto mask = BallotMask(result);
-	ir.SetExec(result);
-	ir.SetExecLo(mask[0]);
-	ir.SetExecHi(mask[1]);
+	// EXEC words combine bitwise; the per-lane EXEC combines the per-lane masks, so helper lanes
+	// stay enabled.
+	const auto old      = ir.GetExec();
+	const auto src      = ReadMask(inst.src0);
+	const auto lhs      = negate_exec ? ir.LogicalNot(old) : old;
+	const auto rhs      = negate_source ? ir.LogicalNot(src) : src;
+	const auto src_mask = ReadU32Pair(inst.src0);
+	const std::array old_mask {ir.GetExecLo(), ir.GetExecHi()};
+	const std::array mask {combine(old_mask[0], src_mask[0]), combine(old_mask[1], src_mask[1])};
+	WriteMask(inst.dst, old, old_mask, true);
+	WriteMask(ConditionOperand(Decoder::OperandKind::ExecLo), IR::U1(ir.Emit(operation, {lhs, rhs})),
+	          mask);
 	ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))));
 }
 
@@ -95,7 +95,7 @@ void Translator::ADD_U32(const Decoder::Instruction& inst, bool vector, bool use
 	}
 	WriteOperand(DestinationOperand(inst), result);
 	if (vector) {
-		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry));
+		WriteVectorMask(inst.dst2, carry);
 	} else {
 		ir.SetScc(carry);
 	}
@@ -108,7 +108,7 @@ void Translator::SUB_U32(const Decoder::Instruction& inst, bool vector, bool rev
 	const auto borrow = ir.UGreaterThan(rhs, lhs);
 	WriteOperand(DestinationOperand(inst), result);
 	if (vector) {
-		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
+		WriteVectorMask(inst.dst2, borrow);
 	} else {
 		ir.SetScc(borrow);
 	}
@@ -126,7 +126,7 @@ void Translator::SUBB_U32(const Decoder::Instruction& inst, bool vector, bool re
 	const auto borrow    = ir.LogicalOr(borrow0, borrow1);
 	WriteOperand(DestinationOperand(inst), result);
 	if (vector) {
-		WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
+		WriteVectorMask(inst.dst2, borrow);
 	} else {
 		ir.SetScc(borrow);
 	}

src/graphics/shader/recompiler/frontend/translate/Convert.cpp

@@ -135,6 +135,8 @@ void Translator::V_PACK_B32_F16(const Decoder::Instruction& inst) {
 }
 
 IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
+	// A zero high lane zero-extends; descriptor tracking bounds a mask, not a bitfield insert.
+	if (high == IR::Value(0u)) return ir.BitwiseAnd(low, IR::U32(IR::Value(0xffffu)));
 	return IR::U32(
 	    ir.Emit(IR::ValueOpcode::BitFieldInsert, {low, high, IR::Value(16u), IR::Value(16u)}));
 }

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -20,12 +20,12 @@ void Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpco
 	Write16Bits(DestinationOperand(inst), result);
 }
 
-void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
-	const auto lhs    = ReadU16AsU32(inst.src0, true);
-	const auto rhs    = ReadU16AsU32(inst.src1, true);
-	auto       result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, true));
+void Translator::Integer16Mad(const Decoder::Instruction& inst, bool sign) {
+	const auto lhs    = ReadU16AsU32(inst.src0, sign);
+	const auto rhs    = ReadU16AsU32(inst.src1, sign);
+	auto       result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, sign));
 	if (inst.dst.clamp) {
-		result = Saturate16(result, true);
+		result = Saturate16(result, sign);
 	}
 	Write16Bits(DestinationOperand(inst), result);
 }
@@ -107,19 +107,7 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
 			default: return false;
 		}
 	};
-	if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) ||
-	    (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
-		const auto mask = WriteMask(inst.dst, invocation_result, true);
-		ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))));
-		return;
-	}
-
-	// Descriptor tracking reasons about each scalar word independently.
-	const auto lhs        = ReadU32Pair(inst.src0);
-	auto       mask_valid = ReadMaskValid(inst.src0);
-	if (inst.src_count > 1u) {
-		mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1));
-	}
+	const auto lhs = ReadU32Pair(inst.src0);
 	std::array<IR::U32, 2> result;
 	if (unary) {
 		result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
@@ -132,11 +120,21 @@ void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
 			result[component] = negate_result ? ir.BitwiseNot(IR::U32(value)) : IR::U32(value);
 		}
 	}
-	WriteU32Pair(inst.dst, result);
-	if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
-		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
-		ir.SetThreadBitScalarReg(dst, invocation_result);
-		ir.SetScalarMaskTag(dst, mask_valid);
+	if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src0) ||
+	    (inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
+		WriteMask(inst.dst, invocation_result, result, true);
+	} else {
+		// Descriptor tracking reasons about each scalar word independently.
+		auto mask_valid = ReadMaskValid(inst.src0);
+		if (inst.src_count > 1u) {
+			mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src1));
+		}
+		WriteU32Pair(inst.dst, result);
+		if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
+			const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
+			ir.SetThreadBitScalarReg(dst, invocation_result);
+			ir.SetScalarMaskTag(dst, mask_valid);
+		}
 	}
 	ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
 }
@@ -149,6 +147,12 @@ void Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode
 		const auto arg_type = IR::ArgTypeOf(opcode, index);
 		const auto& operand = SourceAt(inst, reverse && index < 2u ? 1u - index : index);
 		args[index]         = ReadOperand(operand, arg_type == IR::Type::Void ? type : arg_type);
+		if (opcode == IR::ValueOpcode::ShiftRightArithmetic64 && index == 0u &&
+		    operand.kind == Decoder::OperandKind::LiteralConstant) {
+			args[index] = ir.ConstructU64(
+			    ReadU32(operand),
+			    IR::U32(IR::Value((operand.value & 0x80000000u) ? 0xffffffffu : 0u)));
+		}
 		if (mask_shift_count && index == 1u) {
 			args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u)));
 		}
@@ -265,7 +269,7 @@ void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
 	WriteOperand(DestinationOperand(inst), result);
 }
 
-void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
+void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend, bool high) {
 	const auto extract24 = [&](IR::U32 value) {
 		return IR::U32(
 		    ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract,
@@ -273,7 +277,9 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
 	};
 	const auto lhs    = extract24(ReadU32(inst.src0));
 	const auto rhs    = extract24(ReadU32(inst.src1));
-	auto       result = ir.IMul(lhs, rhs);
+	const auto multiply = high ? (sign ? IR::ValueOpcode::SMulHi : IR::ValueOpcode::UMulHi)
+	                           : IR::ValueOpcode::IMul32;
+	auto result = IR::U32(ir.Emit(multiply, {lhs, rhs}));
 	if (addend) {
 		result = ir.IAdd(result, ReadU32(inst.src2));
 	}
@@ -292,7 +298,8 @@ void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
 	if (inst.dst2.kind != Decoder::OperandKind::Null &&
 	    inst.dst2.kind != Decoder::OperandKind::Unknown) {
 		// The 32x32 product fits in 64 bits, so only the addition can carry.
-		WriteMask(inst.dst2, IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product})));
+		const auto carry = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product}));
+		WriteVectorMask(inst.dst2, carry);
 	}
 }
 
@@ -370,7 +377,7 @@ void Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), ir.ConstructU64(replicate(low), replicate(high)));
 }
 
-void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
+void Translator::S_QUADMASK(const Decoder::Instruction& inst, bool is64) {
 	const auto compact = [&](IR::U32 value) {
 		auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, IR::U32(IR::Value(1u))));
 		bits      = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(2u))));
@@ -383,12 +390,17 @@ void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
 		    ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(12u)))),
 		    IR::U32(IR::Value(0xffu)));
 	};
-	const auto source = ReadU32Pair(inst.src0);
-	const auto quads  = ir.BitwiseOr(
-	    compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), IR::U32(IR::Value(8u))));
-	const auto result = ir.ConstructU64(quads, IR::U32(IR::Value(0u)));
-	WriteOperand(DestinationOperand(inst), result);
-	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+	IR::U32 quads;
+	if (is64) {
+		const auto source = ReadU32Pair(inst.src0);
+		quads = ir.BitwiseOr(
+		    compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), IR::U32(IR::Value(8u))));
+		WriteOperand(DestinationOperand(inst), ir.ConstructU64(quads, IR::U32(IR::Value(0u))));
+	} else {
+		quads = compact(ReadU32(inst.src0));
+		WriteOperand(DestinationOperand(inst), quads);
+	}
+	ir.SetScc(ir.INotEqual(quads, IR::U32(IR::Value(0u))));
 }
 
 void Translator::BFM_B32(const Decoder::Instruction& inst) {
@@ -472,7 +484,7 @@ void Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) {
 	const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
 	const auto bit =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {value, offset, IR::Value(1u)}));
-	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
+	ir.SetScc(ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
 }
 
 void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
@@ -483,7 +495,7 @@ void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
 	    ir.Select(ir.ULessThan(offset, IR::U32(IR::Value(32u))), value[0], value[1]);
 	const auto bit =
 	    IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {word, word_bit, IR::Value(1u)}));
-	WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
+	ir.SetScc(ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
 }
 
 void Translator::V_ALIGN_B32(const Decoder::Instruction& inst, bool byte_offset) {

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -322,11 +322,12 @@ IR::Value Translator::MakeImageAddress(const Decoder::Instruction& inst,
                                        const Decoder::Operand&     base) {
 	std::array<IR::Value, 13> components {};
 	components.fill(IR::Value(0u));
-	const auto count =
-	    Decoder::ImageAddressDwordCount(inst.image_sample_flags, inst.image_address_components);
+	const auto count = Decoder::ImageAddressDwordCount(
+	    inst.image_sample_flags, inst.image_dimension, inst.image_address_components);
 	EXIT_IF(count > components.size());
 	const auto nsa_components =
-	    std::min(inst.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
+	    inst.opcode == Decoder::Opcode::IMAGE_BVH8_INTERSECT_RAY && inst.image_nsa_dwords != 0u
+	        ? 10u : std::min(inst.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
 	for (uint32_t index = 0; index < count; index++) {
 		if (index != 0u && index - 1u < nsa_components) {
 			components[index] =
@@ -781,6 +782,15 @@ void Translator::WriteSharedU32(uint32_t width, IR::U32 address,
 	}
 }
 
+void Translator::DS_BVH_STACK(const Decoder::Instruction& inst) {
+	const auto result = ir.Emit(IR::ValueOpcode::BvhStack,
+	    {ReadU32(inst.src0), ReadU32(inst.src1), ConstructU32x4(inst.src2, 4),
+	     ConstructU32x4(OffsetOperand(inst.src2, 4), 4), ir.GetExec()},
+	    AddMemoryInfo(MemoryInfoFromDecoded(inst), inst.pc));
+	WriteOperand(inst.src0, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(0u)}));
+	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(1u)}));
+}
+
 void Translator::DS_READ(const Decoder::Instruction& inst) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
 	const auto address = ReadU32(MemorySourceAt(inst, 0));
@@ -863,9 +873,10 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
 
 void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
 	const auto memory = MemoryInfoFromDecoded(inst);
-	WriteOperand(inst.dst, ir.Emit(opcode,
-	                               {ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
-	                               AddMemoryInfo(memory, inst.pc)));
+	const auto exec   = ExecMask();
+	WriteOperand(inst.dst,
+	             ir.Emit(opcode, {ReadU32(MemorySourceAt(inst, 0)), exec[0], exec[1], ir.GetExec()},
+	                     AddMemoryInfo(memory, inst.pc)));
 }
 
 void Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
@@ -901,10 +912,7 @@ void Translator::DS_ORDERED_COUNT(const Decoder::Instruction& inst) {
 		const auto packer = ir.BitwiseAnd(m0, IR::U32(IR::Value(0xfu)));
 		address = ir.IAdd(address, ir.ShiftLeftLogical(packer, IR::U32(IR::Value(2u))));
 	}
-	const auto exec   = ir.GetExec();
-	const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {exec});
-	const auto low = ir.CompositeExtract(ballot, 0u);
-	const auto high = ir.CompositeExtract(ballot, 1u);
+	const auto [low, high] = ExecMask();
 	const auto active = ir.INotEqual(ir.BitwiseOr(low, high), IR::U32(IR::Value(0u)));
 	const auto first = ir.Select(
 	    ir.INotEqual(low, IR::U32(IR::Value(0u))),
@@ -941,12 +949,40 @@ void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) {
 	                               {ReadU32(inst.src1), address, ir.GetExec()}));
 }
 
+void Translator::IMAGE_BVH8(const Decoder::Instruction& inst) {
+	std::array<IR::U32, 10> previous;
+	for (uint32_t i = 0; i < previous.size(); ++i)
+		previous[i] = ReadU32(OffsetOperand(inst.dst, i));
+	const auto result = ir.Emit(IR::ValueOpcode::Bvh8Intersect,
+	    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+	const auto get = [&](uint32_t i) {
+		return IR::U32(ir.Emit(IR::ValueOpcode::Bvh8Extract, {result, IR::Value(i)}));
+	};
+	const auto instance = ir.INotEqual(get(16), IR::U32(IR::Value(0u)));
+	for (uint32_t i = 0; i < 10; ++i) {
+		auto value = get(i);
+		if (i == 0 || i == 1 || i == 4 || i == 5 || i == 8 || i == 9)
+			value = ir.Select(instance, previous[i], value);
+		WriteOperand(OffsetOperand(inst.dst, i), value);
+	}
+	for (uint32_t i = 0; i < 6; ++i) {
+		auto reg = inst.image_nsa_dwords != 0 ? inst.image_nsa_addr[3 + i] : inst.src0.reg + 4 + i;
+		auto target = inst.src0;
+		target.reg = reg;
+		// Non-instance results may alias the ray inputs.
+		WriteOperand(target, ir.Select(instance, get(10 + i), ReadU32(target)));
+	}
+}
+
 void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
+		// Translated scalar loads have no cache of their own to invalidate.
+		case Decoder::Opcode::S_DCACHE_INV: return;
 		case Decoder::Opcode::BUFFER_GL0_INV:
 		case Decoder::Opcode::BUFFER_GL1_INV:
 			ir.Emit(IR::ValueOpcode::DataCacheInvalidate);
 			return;
+		case Decoder::Opcode::IMAGE_BVH8_INTERSECT_RAY: return IMAGE_BVH8(inst);
 		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
 			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
 			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()},
@@ -1031,6 +1067,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSMin32);
 		case Decoder::Opcode::BUFFER_ATOMIC_UMIN:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMin32);
+		case Decoder::Opcode::BUFFER_ATOMIC_UMIN_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMin64);
 		case Decoder::Opcode::BUFFER_ATOMIC_SMAX:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSMax32);
 		case Decoder::Opcode::BUFFER_ATOMIC_UMAX:
@@ -1148,6 +1186,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::FLAT_LOAD_USHORT:
 		case Decoder::Opcode::FLAT_LOAD_SSHORT:
 		case Decoder::Opcode::FLAT_LOAD_SHORT_D16:
+		case Decoder::Opcode::FLAT_LOAD_SHORT_D16_HI:
 		case Decoder::Opcode::FLAT_LOAD_DWORD:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX2:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX3:
@@ -1167,15 +1206,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::IMAGE_STORE:
 		case Decoder::Opcode::IMAGE_STORE_MIP: return IMAGE_STORE(inst);
 		case Decoder::Opcode::IMAGE_SAMPLE: return IMAGE_SAMPLE(inst);
-		case Decoder::Opcode::IMAGE_GATHER4_L:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_C:
-		case Decoder::Opcode::IMAGE_GATHER4_C_L:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst);
+		case Decoder::Opcode::IMAGE_GATHER4: return IMAGE_GATHER(inst);
 
 		case Decoder::Opcode::DS_MIN_F32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false);
@@ -1191,6 +1222,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataAppend);
 		case Decoder::Opcode::DS_WRITE_ADDTID_B32: return DS_ADDTID(inst, true);
 		case Decoder::Opcode::DS_READ_ADDTID_B32: return DS_ADDTID(inst, false);
+		case Decoder::Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return DS_BVH_STACK(inst);
 		case Decoder::Opcode::DS_READ2_B32:
 		case Decoder::Opcode::DS_READ2ST64_B32:
 		case Decoder::Opcode::DS_READ2_B64:

src/graphics/shader/recompiler/frontend/translate/Scalar.cpp

@@ -73,43 +73,41 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 
 		case O::S_CMP_EQ_U32:
 		case O::S_CMP_EQ_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_LG_U32:
 		case O::S_CMP_LG_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_GT_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_GE_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true,
-			                   false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_LT_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_LE_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_GT_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_GE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true,
-			                   false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_LT_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_LE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true);
 			return;
 		case O::S_CMP_EQ_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true);
 			return;
 		case O::S_CMP_LG_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true, false);
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true);
 			return;
 
 		case O::S_AND_B64:
@@ -250,7 +248,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BITSET0_B64: return S_BITSET_B64(inst, false);
 		case O::S_BITSET1_B64: return S_BITSET_B64(inst, true);
 		case O::S_BITREPLICATE_B64_B32: return S_BITREPLICATE_B64_B32(inst);
-		case O::S_QUADMASK_B64: return S_QUADMASK_B64(inst);
+		case O::S_QUADMASK_B32: return S_QUADMASK(inst, false);
+		case O::S_QUADMASK_B64: return S_QUADMASK(inst, true);
 		case O::S_BFM_B32: return BFM_B32(inst);
 		case O::S_BFM_B64: return S_BFM_B64(inst);
 		case O::S_BFE_U32: return S_BFE_U32(inst, false);

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -129,6 +129,20 @@ std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
 	        program.wave_size == 64u ? ir.CompositeExtract(mask, 1) : IR::U32(IR::Value(0u))};
 }
 
+std::array<IR::U32, 2> Translator::ExecMask() {
+	return {ir.GetExecLo(), program.wave_size == 64u ? ir.GetExecHi() : IR::U32(IR::Value(0u))};
+}
+
+// Pixel EXEC words exclude the helper invocations that the per-lane EXEC keeps enabled.
+std::array<IR::U32, 2> Translator::VectorMask(IR::U1 value) {
+	auto mask = BallotMask(value);
+	if (program.stage == ShaderType::Pixel) {
+		const auto exec = ExecMask();
+		mask = {ir.BitwiseAnd(mask[0], exec[0]), ir.BitwiseAnd(mask[1], exec[1])};
+	}
+	return mask;
+}
+
 IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
 	switch (operand.kind) {
 		case Decoder::OperandKind::LiteralConstant:
@@ -138,6 +152,8 @@ IR::U32 Translator::ReadRawU32(const Decoder::Operand& operand) {
 		case Decoder::OperandKind::SharedBase:
 		case Decoder::OperandKind::PrivateBase:
 		case Decoder::OperandKind::PopsExitingWaveId: return IR::U32(IR::Value(0u));
+		case Decoder::OperandKind::SharedLimit:
+		case Decoder::OperandKind::PrivateLimit: return IR::U32(IR::Value(0xffffffffu));
 		case Decoder::OperandKind::Sgpr:
 			return ir.GetScalarReg(static_cast<IR::ScalarReg>(operand.reg));
 		case Decoder::OperandKind::Vgpr:
@@ -388,21 +404,9 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
 		switch (operand.kind) {
 			case Decoder::OperandKind::Scc: ir.SetScc(IR::U1(value)); return;
 			case Decoder::OperandKind::ExecLo:
-			case Decoder::OperandKind::ExecHi: {
-				const auto mask = BallotMask(IR::U1(value));
-				ir.SetExec(IR::U1(value));
-				ir.SetExecLo(mask[0]);
-				ir.SetExecHi(mask[1]);
-				return;
-			}
+			case Decoder::OperandKind::ExecHi:
 			case Decoder::OperandKind::VccLo:
-			case Decoder::OperandKind::VccHi: {
-				const auto mask = BallotMask(IR::U1(value));
-				ir.SetVcc(IR::U1(value));
-				ir.SetVccLo(mask[0]);
-				ir.SetVccHi(mask[1]);
-				return;
-			}
+			case Decoder::OperandKind::VccHi: EXIT("lane masks are written through WriteMask");
 			default:
 				WriteRawU32(operand, ir.Select(IR::U1(value), IR::U32(IR::Value(1u)),
 				                               IR::U32(IR::Value(0u))));
@@ -471,17 +475,17 @@ IR::U32 Translator::ReadU32(const Decoder::Operand& operand) {
 }
 
 std::array<IR::U32, 2> Translator::ReadU32Pair(const Decoder::Operand& operand) {
-	if (operand.kind == Decoder::OperandKind::PrivateBase ||
-	    operand.kind == Decoder::OperandKind::SharedBase) {
-		return {IR::U32(IR::Value(0u)), IR::U32(IR::Value(
-		    operand.kind == Decoder::OperandKind::PrivateBase ? Decoder::PrivateApertureHigh
-		                                                      : Decoder::SharedApertureHigh))};
-	}
-	if (operand.kind == Decoder::OperandKind::ExecLo) {
-		return {ir.GetExecLo(), ir.GetExecHi()};
-	}
-	if (operand.kind == Decoder::OperandKind::VccLo) {
-		return {ir.GetVccLo(), ir.GetVccHi()};
+	// Each aperture limit is its base plus 0xffffffff.
+	switch (operand.kind) {
+		case Decoder::OperandKind::SharedBase:
+		case Decoder::OperandKind::SharedLimit:
+			return {ReadRawU32(operand), IR::U32(IR::Value(Decoder::SharedApertureHigh))};
+		case Decoder::OperandKind::PrivateBase:
+		case Decoder::OperandKind::PrivateLimit:
+			return {ReadRawU32(operand), IR::U32(IR::Value(Decoder::PrivateApertureHigh))};
+		case Decoder::OperandKind::ExecLo: return {ir.GetExecLo(), ir.GetExecHi()};
+		case Decoder::OperandKind::VccLo: return {ir.GetVccLo(), ir.GetVccHi()};
+		default: break;
 	}
 	if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
 		const auto bits =
@@ -639,7 +643,7 @@ IR::U1 Translator::ReadMask(const Decoder::Operand& operand) {
 		}
 		case Decoder::OperandKind::ExecLo:
 		case Decoder::OperandKind::ExecHi: return ir.GetExec();
-		case Decoder::OperandKind::VccLo:
+		case Decoder::OperandKind::VccLo: return ir.GetVcc();
 		case Decoder::OperandKind::VccHi:
 			return program.wave_size == 32u
 			           ? ThreadBit({ReadRawU32(operand), IR::U32(IR::Value(0u))})
@@ -677,9 +681,8 @@ IR::U1 Translator::ReadMaskValid(const Decoder::Operand& operand) {
 	}
 }
 
-std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value,
-                                             bool write_64) {
-	const auto mask = BallotMask(value);
+void Translator::WriteMask(const Decoder::Operand& operand, IR::U1 value,
+                           const std::array<IR::U32, 2>& mask, bool write_64) {
 	switch (operand.kind) {
 		case Decoder::OperandKind::Sgpr: {
 			const auto reg  = static_cast<IR::ScalarReg>(operand.reg);
@@ -700,43 +703,43 @@ std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR
 				ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
 				ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
 			}
-			return mask;
+			return;
 		}
 		case Decoder::OperandKind::ExecLo:
 		case Decoder::OperandKind::ExecHi: {
 			ir.SetExec(value);
 			ir.SetExecLo(mask[0]);
 			ir.SetExecHi(mask[1]);
-			return mask;
+			return;
 		}
 		case Decoder::OperandKind::VccLo:
 		case Decoder::OperandKind::VccHi: {
-			if (!write_64 && program.wave_size == 32u) {
+			if (operand.kind == Decoder::OperandKind::VccHi && !write_64 &&
+			    program.wave_size == 32u) {
 				WriteRawU32(operand, mask[0]);
-				return mask;
+				return;
 			}
 			ir.SetVcc(value);
 			ir.SetVccLo(mask[0]);
-			ir.SetVccHi(mask[1]);
-			return mask;
+			if (write_64 || program.wave_size == 64u) {
+				ir.SetVccHi(mask[1]);
+			}
+			return;
 		}
-		case Decoder::OperandKind::Scc: ir.SetScc(value); return mask;
+		case Decoder::OperandKind::Scc: ir.SetScc(value); return;
 		default:
 			WriteRawU32(operand, mask[0]);
 			WriteRawU32(OffsetOperand(operand, 1), mask[1]);
-			return mask;
+			return;
 	}
 }
 
-void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 value) {
-	if (operand.kind == Decoder::OperandKind::Scc) {
-		WriteOperand(operand, value);
-		return;
-	}
+void Translator::WriteVectorMask(const Decoder::Operand& operand, IR::U1 value) {
 	// VALU instructions only write lanes enabled by the incoming EXEC mask. This is
 	// especially important for V_CMPX: replacing EXEC with the ungated comparison would
 	// reactivate lanes disabled by an enclosing divergent region.
-	WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
+	const auto active = ir.LogicalAnd(ir.GetExec(), value);
+	WriteMask(operand, active, VectorMask(active));
 }
 
 void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
@@ -1153,12 +1156,23 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			}
 		}
 		entry_ir.SetExec(initial_exec);
-		const auto initial_mask = entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec});
+		// Pixel EXEC words start with the live (non-helper) lanes; helpers keep the per-lane EXEC.
+		const auto initial_mask = options.stage == ShaderType::Pixel
+		                              ? entry_ir.Emit(IR::ValueOpcode::LiveMask)
+		                              : entry_ir.Emit(IR::ValueOpcode::Ballot, {initial_exec});
 		const auto initial_lo = entry_ir.CompositeExtract(initial_mask, 0);
 		const auto initial_hi = options.wave_size == 64u ? entry_ir.CompositeExtract(initial_mask, 1)
 		                                               : IR::U32(IR::Value(0u));
 		entry_ir.SetExecLo(initial_lo);
 		entry_ir.SetExecHi(initial_hi);
+		const auto active_lane_count = [&] {
+			auto count = IR::U32(entry_ir.Emit(IR::ValueOpcode::BitCount32, {initial_lo}));
+			if (options.wave_size == 64u) {
+				count = entry_ir.IAdd(count, IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::BitCount32, {initial_hi})));
+			}
+			return count;
+		};
 		if (options.stage == ShaderType::Compute) {
 			const auto* cs = options.input_info.compute;
 			const auto  thread_ids =
@@ -1238,12 +1252,12 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 			    entry_ir.BitwiseOr(wave_info, entry_ir.BitwiseOr(entry_ir.ShiftLeftLogical(
 			                                                         primitive_count, u32(8)),
 			                                                     vertex_count)));
+			// Instance IDs exclude the start instance, which shaders add from user data.
 			if (mesh.fast_launch) {
 				// Fast launch broadcasts the subgroup's base vertex and instance to every lane.
 				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5), entry_ir.IAdd(draw(1), group));
-				entry_ir.SetVectorReg(
-				    static_cast<IR::VectorReg>(6),
-				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(6),
+				                      builtin(IR::StageInputKind::WorkgroupId, 1));
 			} else {
 				// GS adjacency addresses local ES records in LDS. Fans retain the draw's
 				// center in every subgroup; strip winding follows the global primitive.
@@ -1271,14 +1285,14 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				                                         entry_ir.ShiftLeftLogical(second, u32(18))));
 				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(1),
 				                      entry_ir.ShiftLeftLogical(third, u32(2)));
-				const auto index_bytes  = draw(3);
+				const auto index_bytes  = draw(2);
 				const auto indexed      = entry_ir.INotEqual(index_bytes, u32(0));
-				const auto index_low    = draw(4);
+				const auto index_low    = draw(3);
 				const auto byte_offset  = entry_ir.IAdd(entry_ir.BitwiseAnd(index_low, u32(3)),
 				                                           entry_ir.IMul(input_vertex, index_bytes));
 				const auto index_resource = entry_ir.Emit(
 				    IR::ValueOpcode::GetAddressResource,
-				    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(5)});
+				    {entry_ir.BitwiseAnd(index_low, u32(~3u)), draw(4)});
 				const auto memory_index = static_cast<uint32_t>(result.memory_info.size());
 				result.memory_info.push_back({.kind = IR::ResourceKind::Global});
 				const auto packed_index = entry_ir.Emit(
@@ -1292,12 +1306,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				     entry_ir.IMul(index_bytes, u32(8))}));
 				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
 				                      entry_ir.IAdd(draw(1), entry_ir.Select(indexed, index, input_vertex)));
-				entry_ir.SetVectorReg(
-				    static_cast<IR::VectorReg>(8),
-				    entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(8),
+				                      builtin(IR::StageInputKind::WorkgroupId, 1));
 			}
 		} else if (options.stage == ShaderType::Local) {
-			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
+			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), active_lane_count());
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(2),
 			                      builtin(IR::StageInputKind::VertexIndex));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(3), IR::U32(IR::Value(0u)));
@@ -1375,19 +1388,32 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 		} else if (options.stage == ShaderType::Vertex) {
 			// Vulkan owns primitive assembly; each vertex subgroup is one NGG wave.
 			// ESVertCount describes input vertices, independently of lane ordinals.
-			auto vertex_count = IR::U32(entry_ir.Emit(IR::ValueOpcode::BitCount32, {initial_lo}));
-			if (options.wave_size == 64u) {
-				vertex_count = entry_ir.IAdd(vertex_count, IR::U32(entry_ir.Emit(
-				    IR::ValueOpcode::BitCount32, {initial_hi})));
-			}
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(2),
 			                      IR::U32(IR::Value(options.wave_size << 12u)));
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3),
-			                      entry_ir.BitwiseOr(vertex_count, IR::U32(IR::Value(1u << 28u))));
+			                      entry_ir.BitwiseOr(active_lane_count(), IR::U32(IR::Value(1u << 28u))));
+			// Vulkan adds an indirect record's bases to the indices; the hardware passes them
+			// only in the user SGPRs the packet patches. SH_NOP leaves those SGPRs unchanged;
+			// it does not make the record's bases zero.
+			const auto& offsets = options.input_info.vertex->indirect_offsets;
+			const auto  index   = [&](uint8_t slot, IR::StageInputKind kind,
+			                          IR::StageInputKind base_kind) {
+				if (!options.input_info.vertex->indirect_draw) {
+					return builtin(kind);
+				}
+				const auto base = builtin(base_kind);
+				if (slot < options.user_data_count) {
+					entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(options.user_data_base + slot),
+					                      base);
+				}
+				return entry_ir.ISub(builtin(kind), base);
+			};
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
-			                      builtin(IR::StageInputKind::VertexIndex));
+			                      index(offsets.vertex, IR::StageInputKind::VertexIndex,
+			                            IR::StageInputKind::BaseVertex));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(8),
-			                      builtin(IR::StageInputKind::InstanceIndex));
+			                      index(offsets.instance, IR::StageInputKind::InstanceIndex,
+			                            IR::StageInputKind::BaseInstance));
 		}
 	}
 	const bool flush_f32_inputs = options.stage == ShaderType::Compute &&

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -27,6 +27,8 @@ private:
 	Decoder::Operand ScalarDestinationOperand(const Decoder::Operand& operand, uint32_t offset);
 	Decoder::Operand PlainOperand(const Decoder::Operand& operand);
 	std::array<IR::U32, 2> BallotMask(IR::U1 value);
+	std::array<IR::U32, 2> ExecMask();
+	std::array<IR::U32, 2> VectorMask(IR::U1 value);
 	IR::U32                ReadRawU32(const Decoder::Operand& operand);
 	IR::U32                ReadScalarCode(uint32_t code);
 	IR::U32                ApplyBitSourceModifiers(const Decoder::Operand& operand, IR::U32 value);
@@ -52,9 +54,9 @@ private:
 	IR::U32 ConditionBit(const Decoder::Operand& operand);
 	IR::U1  ReadMask(const Decoder::Operand& operand);
 	IR::U1  ReadMaskValid(const Decoder::Operand& operand);
-	std::array<IR::U32, 2> WriteMask(const Decoder::Operand& operand, IR::U1 value,
-	                                 bool write_64 = false);
-	void    WriteCompareResult(const Decoder::Operand& operand, IR::U1 value);
+	void    WriteMask(const Decoder::Operand& operand, IR::U1 value,
+	                  const std::array<IR::U32, 2>& mask, bool write_64 = false);
+	void    WriteVectorMask(const Decoder::Operand& operand, IR::U1 value);
 
 	IR::MemoryFlags AddMemoryInfo(const IR::MemoryInfo& memory, uint32_t pc);
 	IR::ExportFlags AddExportInfo(const Decoder::Instruction& inst);
@@ -103,6 +105,8 @@ private:
 	void WriteSharedU32(uint32_t width, IR::U32 address, const std::array<IR::Value, 4>& values,
 	                    const IR::MemoryInfo& memory, uint32_t pc);
 	void DS_READ(const Decoder::Instruction& inst);
+	void IMAGE_BVH8(const Decoder::Instruction& inst);
+	void DS_BVH_STACK(const Decoder::Instruction& inst);
 	void DS_READ2(const Decoder::Instruction& inst);
 	void DS_WRITE(const Decoder::Instruction& inst);
 	void DS_WRITE2(const Decoder::Instruction& inst);
@@ -115,17 +119,15 @@ private:
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
 	IR::U32 Saturate16(IR::U32 value, bool signed_value);
-	void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar, bool cmpx);
-	void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar, bool cmpx);
+	void EmitCompareResult(const Decoder::Instruction& inst, IR::U1 value, bool scalar);
+	void EmitCompareConstant(const Decoder::Instruction& inst, bool value, bool scalar);
 	void EmitIntegerCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
-	                        bool scalar, bool cmpx, bool swap_operands = false);
+	                        bool scalar, bool swap_operands = false);
 	void EmitInteger16Compare(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
-	                          bool signed_value, bool cmpx);
-	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
-	                      bool cmpx);
-	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx,
-	                             bool is64 = false);
-	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx, bool half = false);
+	                          bool signed_value);
+	void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half);
+	void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool is64 = false);
+	void EmitFloatClassCompare(const Decoder::Instruction& inst, bool half = false);
 	void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
 	void V_CVT_F32_U32(const Decoder::Instruction& inst);
 	void V_CVT_F32_I32(const Decoder::Instruction& inst);
@@ -167,7 +169,7 @@ private:
 	void Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
 	void Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
 	void Integer16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
-	void V_MAD_I16(const Decoder::Instruction& inst);
+	void Integer16Mad(const Decoder::Instruction& inst, bool sign);
 	void PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
 	                          bool arithmetic);
 	void PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
@@ -187,7 +189,7 @@ private:
 	void V_XOR3_B32(const Decoder::Instruction& inst);
 	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb, bool sign);
 	void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
-	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
+	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend, bool high = false);
 	void V_MAD_U64_U32(const Decoder::Instruction& inst);
 	void V_SAD_U32(const Decoder::Instruction& inst);
 	void V_ADD3_U32(const Decoder::Instruction& inst);
@@ -196,7 +198,7 @@ private:
 	void V_BCNT_U32_B32(const Decoder::Instruction& inst);
 	void V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
 	void S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
-	void S_QUADMASK_B64(const Decoder::Instruction& inst);
+	void S_QUADMASK(const Decoder::Instruction& inst, bool is64);
 	void BFM_B32(const Decoder::Instruction& inst);
 	IR::U32 RightMask32(IR::U32 count);
 	void    S_BFM_B64(const Decoder::Instruction& inst);

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -25,444 +25,310 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return;
 
 		case O::V_CMP_F_I32:
-		case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return;
+		case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false); return;
 		case O::V_CMP_T_I32:
-		case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return;
+		case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false); return;
 		case O::V_CMP_EQ_U32:
 		case O::V_CMP_EQ_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, false);
-			return;
 		case O::V_CMPX_EQ_U32:
 		case O::V_CMPX_EQ_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_NE_U32:
 		case O::V_CMP_NE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, false);
-			return;
 		case O::V_CMPX_NE_U32:
 		case O::V_CMPX_NE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_GT_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, false);
-			return;
 		case O::V_CMPX_GT_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_GE_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
-			                   false);
-			return;
 		case O::V_CMPX_GE_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
-			                   true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_LT_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, false);
-			return;
 		case O::V_CMPX_LT_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_LE_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false,
-			                   false);
-			return;
 		case O::V_CMPX_LE_U32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_GT_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, false);
-			return;
 		case O::V_CMPX_GT_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_GE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
-			                   false);
-			return;
 		case O::V_CMPX_GE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
-			                   true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_LT_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, false);
-			return;
 		case O::V_CMPX_LT_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_LE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false,
-			                   false);
-			return;
 		case O::V_CMPX_LE_I32:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false);
 			return;
 		case O::V_CMP_F_I64:
-		case O::V_CMP_F_U64: EmitCompareConstant(inst, false, false, false); return;
-		case O::V_CMP_T_I64:
-		case O::V_CMP_T_U64: EmitCompareConstant(inst, true, false, false); return;
+		case O::V_CMP_F_U64:
 		case O::V_CMPX_F_I64:
-		case O::V_CMPX_F_U64: EmitCompareConstant(inst, false, false, true); return;
+		case O::V_CMPX_F_U64: EmitCompareConstant(inst, false, false); return;
+		case O::V_CMP_T_I64:
+		case O::V_CMP_T_U64:
 		case O::V_CMPX_T_I64:
-		case O::V_CMPX_T_U64: EmitCompareConstant(inst, true, false, true); return;
+		case O::V_CMPX_T_U64: EmitCompareConstant(inst, true, false); return;
 		case O::V_CMP_EQ_I64:
 		case O::V_CMP_EQ_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_EQ_I64:
 		case O::V_CMPX_EQ_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false);
 			return;
 		case O::V_CMP_LT_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_LT_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false);
 			return;
 		case O::V_CMP_GT_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_GT_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false);
 			return;
 		case O::V_CMP_NE_I64:
 		case O::V_CMP_NE_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_NE_I64:
 		case O::V_CMPX_NE_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false);
 			return;
 
 		case O::V_CMP_LT_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_LT_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false);
 			return;
 		case O::V_CMP_LE_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_LE_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false);
 			return;
 		// The IR has no signed 64-bit greater-than: a > b is b < a, and a >= b is b <= a.
 		case O::V_CMP_GT_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, false,
-			                   true);
-			return;
 		case O::V_CMPX_GT_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true,
-			                   true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan64, IR::Type::U64, false, true);
 			return;
 		case O::V_CMP_GE_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, false,
-			                   true);
-			return;
 		case O::V_CMPX_GE_I64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true,
-			                   true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual64, IR::Type::U64, false, true);
 			return;
 		case O::V_CMP_LE_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_LE_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual64, IR::Type::U64, false);
 			return;
 		case O::V_CMP_GE_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, false);
-			return;
 		case O::V_CMPX_GE_U64:
-			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false, true);
+			EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual64, IR::Type::U64, false);
 			return;
 
 		case O::V_CMP_EQ_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
+		case O::V_CMPX_EQ_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false);
 			return;
 		case O::V_CMP_EQ_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
-			return;
 		case O::V_CMPX_EQ_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true);
 			return;
 		case O::V_CMP_NE_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
-			return;
 		case O::V_CMPX_NE_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false);
 			return;
 		case O::V_CMP_NE_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
-			return;
 		case O::V_CMPX_NE_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true);
 			return;
 		case O::V_CMP_GT_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
-			return;
 		case O::V_CMPX_GT_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false);
 			return;
 		case O::V_CMP_GE_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false, false);
+			EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false);
 			return;
 		case O::V_CMP_LT_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, false);
-			return;
 		case O::V_CMPX_LT_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, true);
-			return;
-		case O::V_CMPX_EQ_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false);
 			return;
 		case O::V_CMP_LE_U16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false);
+			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false);
 			return;
 		case O::V_CMP_GT_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
-			return;
 		case O::V_CMPX_GT_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true);
 			return;
 		case O::V_CMP_GE_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
-			return;
 		case O::V_CMPX_GE_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true);
 			return;
 		case O::V_CMP_LT_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
-			return;
 		case O::V_CMPX_LT_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true);
 			return;
 		case O::V_CMP_LE_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
-			return;
 		case O::V_CMPX_LE_I16:
-			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, true);
+			EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true);
 			return;
 
-		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return;
-		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return;
+		case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false); return;
+		case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false); return;
 		case O::V_CMP_EQ_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
+		case O::V_CMPX_EQ_F32:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false);
 			return;
 		case O::V_CMP_EQ_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, false);
+		case O::V_CMPX_EQ_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false);
 			return;
 		case O::V_CMP_LE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, false);
-			return;
 		case O::V_CMPX_LE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false, true);
-			return;
-		case O::V_CMPX_GE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, true);
-			return;
-		case O::V_CMP_F_F64: EmitCompareConstant(inst, false, false, false); return;
-		case O::V_CMP_TRU_F64: EmitCompareConstant(inst, true, false, false); return;
-		case O::V_CMPX_F_F64: EmitCompareConstant(inst, false, false, true); return;
-		case O::V_CMPX_TRU_F64: EmitCompareConstant(inst, true, false, true); return;
-		case O::V_CMP_LT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan64, false, false);
-			return;
-		case O::V_CMPX_LT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan64, false, true);
-			return;
-		case O::V_CMPX_EQ_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual64, false, true);
-			return;
-		case O::V_CMP_GT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan64, false, false);
-			return;
-		case O::V_CMPX_GT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan64, false, true);
-			return;
-		case O::V_CMP_LG_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual64, false, false);
-			return;
-		case O::V_CMPX_LG_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual64, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual64, false);
 			return;
 		case O::V_CMP_GE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false, false);
+		case O::V_CMPX_GE_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual64, false);
+			return;
+		case O::V_CMP_F_F64:
+		case O::V_CMPX_F_F64: EmitCompareConstant(inst, false, false); return;
+		case O::V_CMP_TRU_F64:
+		case O::V_CMPX_TRU_F64: EmitCompareConstant(inst, true, false); return;
+		case O::V_CMP_LT_F64:
+		case O::V_CMPX_LT_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan64, false);
+			return;
+		case O::V_CMP_GT_F64:
+		case O::V_CMPX_GT_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan64, false);
+			return;
+		case O::V_CMP_LG_F64:
+		case O::V_CMPX_LG_F64:
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual64, false);
 			return;
 		case O::V_CMP_NLG_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual64, false, false);
-			return;
 		case O::V_CMPX_NLG_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual64, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual64, false);
 			return;
 		case O::V_CMP_NEQ_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual64, false, false);
-			return;
 		case O::V_CMPX_NEQ_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual64, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual64, false);
 			return;
 		case O::V_CMP_NLE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan64, false, false);
-			return;
 		case O::V_CMPX_NLE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan64, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan64, false);
 			return;
 		case O::V_CMP_NLT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual64, false, false);
-			return;
 		case O::V_CMPX_NLT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual64, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual64, false);
 			return;
 		case O::V_CMP_NGE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan64, false, false);
-			return;
 		case O::V_CMPX_NGE_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan64, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan64, false);
 			return;
 		case O::V_CMP_NGT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual64, false, false);
-			return;
 		case O::V_CMPX_NGT_F64:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual64, false, true);
-			return;
-		case O::V_CMP_O_F64: EmitFloatOrderedCompare(inst, true, false, true); return;
-		case O::V_CMP_U_F64: EmitFloatOrderedCompare(inst, false, false, true); return;
-		case O::V_CMPX_O_F64: EmitFloatOrderedCompare(inst, true, true, true); return;
-		case O::V_CMPX_U_F64: EmitFloatOrderedCompare(inst, false, true, true); return;
-		case O::V_CMPX_EQ_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual64, false);
 			return;
+		case O::V_CMP_O_F64:
+		case O::V_CMPX_O_F64: EmitFloatOrderedCompare(inst, true, true); return;
+		case O::V_CMP_U_F64:
+		case O::V_CMPX_U_F64: EmitFloatOrderedCompare(inst, false, true); return;
 		case O::V_CMP_LG_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, false);
-			return;
 		case O::V_CMPX_LG_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false);
 			return;
 		case O::V_CMP_GT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, false);
-			return;
 		case O::V_CMPX_GT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false);
 			return;
 		case O::V_CMP_GE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, false);
-			return;
 		case O::V_CMPX_GE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false);
 			return;
 		case O::V_CMP_LT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, false);
-			return;
 		case O::V_CMPX_LT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false);
 			return;
 		case O::V_CMP_LE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, false);
-			return;
 		case O::V_CMPX_LE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false);
 			return;
 		case O::V_CMP_NLG_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, false);
-			return;
 		case O::V_CMPX_NLG_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false);
 			return;
 		case O::V_CMP_NEQ_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, false);
-			return;
 		case O::V_CMPX_NEQ_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false);
 			return;
 		case O::V_CMP_NLE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, false);
-			return;
 		case O::V_CMPX_NLE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false);
 			return;
 		case O::V_CMP_NLT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, false);
-			return;
 		case O::V_CMPX_NLT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false);
 			return;
 		case O::V_CMP_NGE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, false);
-			return;
 		case O::V_CMPX_NGE_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false);
 			return;
 		case O::V_CMP_NGT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, false);
-			return;
 		case O::V_CMPX_NGT_F32:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false);
 			return;
 		case O::V_CMP_EQ_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, false);
-			return;
 		case O::V_CMPX_EQ_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true);
 			return;
 		case O::V_CMP_LG_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true, false);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true);
 			return;
 		case O::V_CMP_GT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, false);
-			return;
 		case O::V_CMPX_GT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true);
 			return;
 		case O::V_CMP_GE_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, false);
-			return;
 		case O::V_CMPX_GE_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true);
 			return;
 		case O::V_CMP_LT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, false);
-			return;
 		case O::V_CMPX_LT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true);
 			return;
 		case O::V_CMP_LE_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, false);
-			return;
 		case O::V_CMPX_LE_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true);
 			return;
 		case O::V_CMP_NGE_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true, false);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, true);
 			return;
 		case O::V_CMP_NGT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false);
-			return;
 		case O::V_CMPX_NGT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true);
 			return;
 		case O::V_CMPX_NLE_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, true);
 			return;
 		case O::V_CMP_NEQ_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, false);
-			return;
 		case O::V_CMPX_NEQ_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true);
 			return;
 		case O::V_CMP_NLT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, false);
-			return;
 		case O::V_CMPX_NLT_F16:
-			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, true);
+			EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true);
 			return;
-		case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true, false); return;
-		case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false, false); return;
-		case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true, true); return;
-		case O::V_CMPX_U_F32: EmitFloatOrderedCompare(inst, false, true); return;
-		case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return;
-		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return;
-		case O::V_CMP_CLASS_F16: EmitFloatClassCompare(inst, false, true); return;
-		case O::V_CMPX_CLASS_F16: EmitFloatClassCompare(inst, true, true); return;
+		case O::V_CMP_O_F32:
+		case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true); return;
+		case O::V_CMP_U_F32:
+		case O::V_CMPX_U_F32: EmitFloatOrderedCompare(inst, false); return;
+		case O::V_CMP_CLASS_F32:
+		case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst); return;
+		case O::V_CMP_CLASS_F16:
+		case O::V_CMPX_CLASS_F16: EmitFloatClassCompare(inst, true); return;
 
 		case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return;
 		case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return;
@@ -509,9 +375,14 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_SUB_NC_U16:
 		case O::V_SUB_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::ISub32, false);
 		case O::V_MUL_LO_U16: return Integer16Binary(inst, IR::ValueOpcode::IMul32, false);
-		case O::V_MAD_I16: return V_MAD_I16(inst);
+		case O::V_MAD_I16: return Integer16Mad(inst, true);
+		case O::V_MAD_U16: return Integer16Mad(inst, false);
 		case O::V_MED3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMedTri32, true);
 		case O::V_MIN3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMinTri32, false);
+		case O::V_MIN3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMinTri32, true);
+		case O::V_MAX3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMaxTri32, true);
+		case O::V_MAX3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMaxTri32, false);
+		case O::V_MED3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMedTri32, false);
 		case O::V_MIN_I16: return Integer16Binary(inst, IR::ValueOpcode::SMin32, true);
 		case O::V_MAX_I16: return Integer16Binary(inst, IR::ValueOpcode::SMax32, true);
 		case O::V_MIN_U16: return Integer16Binary(inst, IR::ValueOpcode::UMin32, false);
@@ -699,6 +570,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_LSHRREV_B64:
 			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, true,
 			                     false, false);
+		case O::V_ASHRREV_I64:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic64, IR::Type::U64, true,
+			                     false, false);
 
 		case O::V_XNOR_B32:
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, false);
@@ -712,6 +586,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_MAD_U32_U24: return Integer24(inst, false, true);
 		case O::V_MUL_I32_I24: return Integer24(inst, true, false);
 		case O::V_MUL_U32_U24: return Integer24(inst, false, false);
+		case O::V_MUL_HI_I32_I24: return Integer24(inst, true, false, true);
+		case O::V_MUL_HI_U32_U24: return Integer24(inst, false, false, true);
 		case O::V_MAD_U64_U32: return V_MAD_U64_U32(inst);
 		case O::V_SAD_U32: return V_SAD_U32(inst);
 		case O::V_ADD3_U32: return V_ADD3_U32(inst);

src/graphics/shader/recompiler/ir/Program.cpp

@@ -118,15 +118,14 @@ ResourcePlan& ResourcePlan::operator=(ResourcePlan&& other) noexcept {
 }
 
 Program::~Program() {
-	// Planning expressions can refer to block values but outlive block storage in the base class.
+	// Planning expressions can refer to block values but outlive block storage in the base class,
+	// and values cross block boundaries: detach everything before any storage is destroyed.
 	for (auto& inst: value_storage) {
-		inst.Invalidate();
+		inst.DropArgs();
 	}
-	// Values may cross block boundaries. Detach all arguments before any block starts destroying
-	// its instruction storage so reverse-use links always point to live definitions.
 	for (auto* block: blocks) {
 		for (auto& inst: *block) {
-			inst.Invalidate();
+			inst.DropArgs();
 		}
 	}
 }
@@ -169,6 +168,8 @@ bool EquivalentSelectorDomain(const ResourcePlan& program,
                               const DescriptorSource::IndirectDescriptor& right) {
 	return left.selector == right.selector && left.signed_count == right.signed_count &&
 	       left.workgroup_axis == right.workgroup_axis &&
+	       left.workgroup_multiplier == right.workgroup_multiplier &&
+	       left.workgroup_shift == right.workgroup_shift &&
 	       EquivalentValue(program, left.key_count, right.key_count) &&
 	       EquivalentValue(program, left.selector_first, right.selector_first) &&
 	       EquivalentValue(program, left.selector_mask, right.selector_mask);

src/graphics/shader/recompiler/ir/Serialize.h

@@ -155,14 +155,17 @@ KYTY_IO_FIELDS(SampledResourcePair, a, b, c)
 KYTY_IO_FIELDS(StageInput, a, b, c, d, e)
 KYTY_IO_FIELDS(StageOutput, a, b, c, d)
 KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
-               A, B, C, D, E, F, G, H, I, J)
+               A, B, C, D, E, F, G, H, I, J, K, L, M)
 KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b)
-KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks, a, b, c)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j, k,
+               l)
 KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::PointerTable, a, b, c, d)
-KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p,
+               q, r)
 KYTY_IO_FIELDS(DescriptorSource, a, b, c)
 KYTY_IO_FIELDS(SrtRead, a, b)
-KYTY_IO_FIELDS(ResourceBlock, a, b, c, d)
+KYTY_IO_FIELDS(ResourceBlock, a, b, c, d, e)
 KYTY_IO_FIELDS(UniformFill, a, b, c, d, e, f)
 KYTY_IO_FIELDS(UniformFillPlan, a, b, c, d)
 KYTY_IO_FIELDS(ResourceSpecialization::Buffer, a, b, c, d, e, f, g, h)

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -251,6 +251,8 @@ enum class StageInputKind {
 	LocalInvocationId,
 	LocalInvocationIndex,
 	GlobalInvocationId,
+	BaseVertex,
+	BaseInstance,
 	Parameter,
 };
 
@@ -365,7 +367,7 @@ static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 58u);
 
 struct PushData {
 	static constexpr uint32_t DwordCount = 32;
-	static constexpr uint32_t MeshDrawDwordCount = 6;
+	static constexpr uint32_t MeshDrawDwordCount = 5;
 	static constexpr uint32_t NoStart    = UINT32_MAX;
 	std::array<uint32_t, DwordCount> dwords {};
 
@@ -495,10 +497,16 @@ struct Bindings {
 	uint32_t dispatch_thread_dword = PushData::NoStart;
 	uint32_t memory_offset_dword = 0;
 
-	[[nodiscard]] uint32_t ShaderDataDwords() const {
+	// Packed 8-bit binding offsets, then each binding's bound range in bytes: OpArrayLength
+	// reads 0 for ranges of 2 GiB or more on NVIDIA.
+	[[nodiscard]] uint32_t MemorySizeDword() const {
 		return memory_offset_dword +
 		       (descriptor_counts[static_cast<size_t>(DescriptorBindingKind::Buffers)] + 3u) / 4u;
 	}
+	[[nodiscard]] uint32_t ShaderDataDwords() const {
+		return MemorySizeDword() +
+		       descriptor_counts[static_cast<size_t>(DescriptorBindingKind::Buffers)];
+	}
 	[[nodiscard]] bool UsesPushData() const {
 		return push_data_start_dword != PushData::NoStart;
 	}
@@ -535,12 +543,15 @@ struct ShaderInfo {
 	bool                            uses_image_heap               = false;
 	bool                            uses_dispatch_threads         = false;
 	bool                            bvh                           = false;
+	bool                            bvh8                          = false;
+	bool                            bvh_stack                     = false;
 	bool                            subgroup_ballot               = false;
 	bool                            subgroup_vote                 = false;
 	bool                            subgroup_barrier              = false;
 	bool                            subgroup_shuffle              = false;
 	bool                            subgroup_arithmetic           = false;
 	bool                            subgroup_local_invocation_id  = false;
+	bool                            helper_invocation             = false;
 	bool                            compute_derivatives           = false;
 	bool                            image_gather_extended         = false;
 	bool                            function_lds                  = false;
@@ -565,6 +576,13 @@ struct DescriptorSource {
 
 				bool operator==(const CountedRows& other) const = default;
 			};
+			struct MaskRanks {
+				uint32_t source = UINT32_MAX;
+				uint32_t stride = 0; // Byte spacing of the mask words its load can address.
+				uint32_t offset = 0;
+
+				bool operator==(const MaskRanks& other) const = default;
+			};
 			uint32_t source = UINT32_MAX;
 			uint32_t stride = 0;
 			uint32_t offset = 0;
@@ -572,10 +590,12 @@ struct DescriptorSource {
 			uint32_t bit_offset = 0;
 			uint32_t bit_mask = UINT32_MAX;
 			bool global = false;
-			// Four DWORD bitmap words per record; set-bit ranks subtract from the last key.
+			// Four or eight DWORD bitmap words per record; set ranks subtract from the last key.
 			bool bitset = false;
 			std::optional<CountedRows> rows;
 			std::vector<std::pair<uint32_t, uint32_t>> byte_ranges;
+			std::optional<uint32_t> excluded; // A guard keeps this key from the read.
+			std::optional<MaskRanks> ranks;   // Records are the bits some mask word sets.
 
 			bool operator==(const SelectorRead& other) const = default;
 		};
@@ -599,8 +619,12 @@ struct DescriptorSource {
 		bool     indexed = false;
 		bool     signed_count = false;
 		uint32_t workgroup_axis  = UINT32_MAX;
+		// Nonzero shift: workgroup keys are (selector_first + ID) * multiplier >> shift.
+		uint32_t workgroup_multiplier = 0;
+		uint32_t workgroup_shift      = 0;
 		Value    key_count;
-		// First selector index, or first descriptor key for a direct or bitmap range.
+		// First selector index, workgroup key base, or first descriptor key for a direct or bitmap
+		// range.
 		Value                 selector_first;
 		Value    selector_mask;
 		std::vector<uint32_t> sources;
@@ -627,6 +651,7 @@ struct ResourceBlock {
 	Value                 condition;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> sources;
+	std::vector<uint32_t> images;
 	std::vector<uint32_t> srt_reads;
 };
 

src/graphics/shader/recompiler/ir/Type.cpp

@@ -31,6 +31,7 @@ std::string TypeName(Type type) {
 	    "SamplerResource",
 	    "ImageAddress",
 	    "F64",
+	    "Bvh8Result",
 	};
 	const auto bits = static_cast<uint32_t>(type);
 	if (bits == 0) {

src/graphics/shader/recompiler/ir/Type.h

@@ -28,6 +28,7 @@ enum class Type : uint32_t {
 	SamplerResource = 1u << 18u,
 	ImageAddress    = 1u << 19u,
 	F64             = 1u << 20u,
+	Bvh8Result      = 1u << 21u,
 };
 
 constexpr Type operator|(Type lhs, Type rhs) {

src/graphics/shader/recompiler/ir/Value.cpp

@@ -25,22 +25,6 @@ Value Value::F32(float value) {
 	return Value(Type::F32, std::bit_cast<uint32_t>(value));
 }
 
-bool Value::IsEmpty() const {
-	return type == Type::Void;
-}
-
-bool Value::IsImmediate() const {
-	return type != Type::Opaque;
-}
-
-bool Value::IsIdentity() const {
-	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Identity;
-}
-
-bool Value::IsPhi() const {
-	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Phi;
-}
-
 Type Value::GetType() const {
 	if (IsPhi()) {
 		return inst->Flags<Type>();
@@ -56,19 +40,6 @@ Inst* Value::Instruction() const {
 	return inst;
 }
 
-Inst* Value::TryInstruction() const {
-	return type == Type::Opaque ? inst : nullptr;
-}
-
-Inst* Value::ResolveInstruction() const {
-	EXIT_IF(type != Type::Opaque);
-	return IsIdentity() ? inst->Arg(0).ResolveInstruction() : inst;
-}
-
-Value Value::Resolve() const {
-	return IsIdentity() ? inst->Arg(0).Resolve() : *this;
-}
-
 ScalarReg Value::ScalarRegister() const {
 	EXIT_IF(type != Type::ScalarReg);
 	return scalar_reg;
@@ -94,11 +65,6 @@ uint16_t Value::U16() const {
 	return imm_u16;
 }
 
-uint32_t Value::U32() const {
-	EXIT_IF(type != Type::U32);
-	return imm_u32;
-}
-
 uint64_t Value::U64() const {
 	EXIT_IF(type != Type::U64);
 	return imm_u64;
@@ -152,10 +118,6 @@ Inst::~Inst() {
 	ClearArgs();
 }
 
-ValueOpcode Inst::GetOpcode() const {
-	return opcode;
-}
-
 Type Inst::GetType() const {
 	if (opcode == ValueOpcode::Phi) {
 		return static_cast<Type>(flags);
@@ -178,51 +140,25 @@ size_t Inst::UseCount() const {
 	return uses.size();
 }
 
-size_t Inst::NumArgs() const {
-	return num_args == PhiArity ? phi_args.size() : num_args;
-}
-
 size_t Inst::NumPhiBlocks() const {
 	return num_args == PhiArity ? phi_args.size() : 0;
 }
 
-Value Inst::Arg(size_t index) const {
-	EXIT_IF(index >= NumArgs());
-	if (num_args <= InlineArity) {
-		return fixed_args[index];
-	}
-	return num_args == PhiArity ? phi_args[index].second : large_args[index];
-}
-
 Block* Inst::PhiBlock(size_t index) const {
 	EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_args.size());
 	return phi_args[index].first;
 }
 
-Block* Inst::Parent() const {
-	return parent;
-}
-
-const std::vector<Use>& Inst::Uses() const {
-	return uses;
-}
-
 void Inst::SetParent(Block* block) {
 	parent = block;
 }
 
 void Inst::SetArg(size_t index, Value value) {
-	const auto old = Arg(index);
-	if (auto* old_inst = old.TryInstruction(); old_inst != nullptr) {
+	auto& arg = ArgSlot(*this, index);
+	if (auto* old_inst = arg.TryInstruction(); old_inst != nullptr) {
 		RemoveUse(old_inst, index);
 	}
-	if (num_args <= InlineArity) {
-		fixed_args[index] = value;
-	} else if (num_args == PhiArity) {
-		phi_args[index].second = value;
-	} else {
-		large_args[index] = value;
-	}
+	arg = value;
 	if (auto* new_inst = value.TryInstruction(); new_inst != nullptr) {
 		AddUse(new_inst, index);
 	}
@@ -238,11 +174,15 @@ void Inst::AddPhiOperand(Block* predecessor, Value value) {
 }
 
 void Inst::ReplaceUsesWith(Value replacement, bool preserve) {
-	const auto old_uses = uses;
-	for (const auto& use: old_uses) {
-		use.user->SetArg(use.operand, replacement);
-	}
+	// Detach the arguments first (a phi can use itself), then move every use over in order.
 	Invalidate();
+	const auto old_uses = std::exchange(uses, {});
+	for (const auto& use: old_uses) {
+		ArgSlot(*use.user, use.operand) = replacement;
+	}
+	if (auto* target = replacement.TryInstruction(); target != nullptr) {
+		target->uses.insert(target->uses.end(), old_uses.begin(), old_uses.end());
+	}
 	if (preserve) {
 		opcode = ValueOpcode::Identity;
 		num_args = 1;
@@ -256,9 +196,6 @@ void Inst::Invalidate() {
 }
 
 void Inst::AddUse(Inst* used, size_t operand) {
-	const auto found = std::ranges::find_if(
-	    used->uses, [&](const Use& use) { return use.user == this && use.operand == operand; });
-	EXIT_IF(found != used->uses.end());
 	used->uses.push_back({this, operand});
 }
 
@@ -275,6 +212,10 @@ void Inst::ClearArgs() {
 			RemoveUse(value_inst, index);
 		}
 	}
+	DropArgs();
+}
+
+void Inst::DropArgs() {
 	if (num_args == PhiArity) {
 		std::destroy_at(&phi_args);
 		std::construct_at(&fixed_args);

src/graphics/shader/recompiler/ir/Value.h

@@ -33,14 +33,14 @@ public:
 	static Value F16(uint16_t bits);
 	static Value F32(float value);
 
-	[[nodiscard]] bool IsEmpty() const;
-	[[nodiscard]] bool IsImmediate() const;
+	[[nodiscard]] bool IsEmpty() const { return type == Type::Void; }
+	[[nodiscard]] bool IsImmediate() const { return type != Type::Opaque; }
 	[[nodiscard]] bool IsIdentity() const;
 	[[nodiscard]] bool IsPhi() const;
 	[[nodiscard]] Type GetType() const;
 
 	[[nodiscard]] Inst*     Instruction() const;
-	[[nodiscard]] Inst*     TryInstruction() const;
+	[[nodiscard]] Inst*     TryInstruction() const { return type == Type::Opaque ? inst : nullptr; }
 	[[nodiscard]] Inst*     ResolveInstruction() const;
 	[[nodiscard]] Value     Resolve() const;
 	[[nodiscard]] ScalarReg ScalarRegister() const;
@@ -112,7 +112,7 @@ public:
 	Inst(Inst&&)                 = delete;
 	Inst& operator=(Inst&&)      = delete;
 
-	[[nodiscard]] ValueOpcode             GetOpcode() const;
+	[[nodiscard]] ValueOpcode             GetOpcode() const { return opcode; }
 	[[nodiscard]] Type                    GetType() const;
 	[[nodiscard]] bool                    MayHaveSideEffects() const;
 	[[nodiscard]] bool                    HasUses() const;
@@ -121,8 +121,8 @@ public:
 	[[nodiscard]] size_t                  NumPhiBlocks() const;
 	[[nodiscard]] Value                   Arg(size_t index) const;
 	[[nodiscard]] Block*                  PhiBlock(size_t index) const;
-	[[nodiscard]] Block*                  Parent() const;
-	[[nodiscard]] const std::vector<Use>& Uses() const;
+	[[nodiscard]] Block*                  Parent() const { return parent; }
+	[[nodiscard]] const std::vector<Use>& Uses() const { return uses; }
 	// SRT walker output index, assigned by the resource plan that owns this instruction.
 	[[nodiscard]] uint32_t WalkOutput() const { return walk_output; }
 	void SetWalkOutput(uint32_t index) const { walk_output = index; }
@@ -138,6 +138,8 @@ public:
 	void AddPhiOperand(Block* predecessor, Value value);
 	void ReplaceUsesWith(Value replacement, bool preserve = true);
 	void Invalidate();
+	// Drops the arguments without updating their use lists, for teardown that destroys both sides.
+	void DropArgs();
 
 	template <typename T>
 	requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v<T>)
@@ -157,6 +159,8 @@ public:
 private:
 	friend void EliminateDeadCode(const std::vector<Block*>& blocks);
 
+	template <typename Self>
+	static auto& ArgSlot(Self& self, size_t index);
 	void AddUse(Inst* used, size_t operand);
 	void RemoveUse(Inst* used, size_t operand);
 	void ClearArgs();
@@ -181,4 +185,44 @@ private:
 
 static_assert(sizeof(Inst) <= 120, "Inst operand and backend storage unintentionally increased");
 
+// Called per operand by the IR passes and the resource walker; inline, as the build has no LTO.
+inline bool Value::IsIdentity() const {
+	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Identity;
+}
+
+inline bool Value::IsPhi() const {
+	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Phi;
+}
+
+inline Inst* Value::ResolveInstruction() const {
+	EXIT_IF(type != Type::Opaque);
+	return IsIdentity() ? inst->Arg(0).ResolveInstruction() : inst;
+}
+
+inline Value Value::Resolve() const {
+	return IsIdentity() ? inst->Arg(0).Resolve() : *this;
+}
+
+inline uint32_t Value::U32() const {
+	EXIT_IF(type != Type::U32);
+	return imm_u32;
+}
+
+inline size_t Inst::NumArgs() const {
+	return num_args == PhiArity ? phi_args.size() : num_args;
+}
+
+template <typename Self>
+auto& Inst::ArgSlot(Self& self, size_t index) {
+	EXIT_IF(index >= self.NumArgs());
+	if (self.num_args <= InlineArity) {
+		return self.fixed_args[index];
+	}
+	return self.num_args == PhiArity ? self.phi_args[index].second : self.large_args[index];
+}
+
+inline Value Inst::Arg(size_t index) const {
+	return ArgSlot(*this, index);
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp

@@ -44,6 +44,7 @@ constexpr Type AddressResource = Type::AddressResource;
 constexpr Type ImageResource   = Type::ImageResource;
 constexpr Type SamplerResource = Type::SamplerResource;
 constexpr Type ImageAddress    = Type::ImageAddress;
+constexpr Type Bvh8Result      = Type::Bvh8Result;
 
 constexpr std::array<OpcodeMeta, static_cast<size_t>(ValueOpcode::Count)> MetaTable = {{
 #define VALUE_OPCODE(name, ...) MakeMeta(#name __VA_OPT__(, ) __VA_ARGS__),
@@ -76,7 +77,8 @@ bool HasSideEffects(ValueOpcode opcode) {
 		return true;
 	}
 	const auto shared_access = SharedAccessOf(opcode);
-	if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic ||
+	if (shared_access == SharedAccess::Write || shared_access == SharedAccess::ReadWrite ||
+	    shared_access == SharedAccess::Atomic ||
 	    shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) {
 		return true;
 	}
@@ -123,6 +125,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicISub32:
 		case ValueOpcode::BufferAtomicSMin32:
 		case ValueOpcode::BufferAtomicUMin32:
+		case ValueOpcode::BufferAtomicUMin64:
 		case ValueOpcode::BufferAtomicSMax32:
 		case ValueOpcode::BufferAtomicUMax32:
 		case ValueOpcode::BufferAtomicAnd32:
@@ -140,6 +143,7 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
 		case ValueOpcode::BufferAtomicIAdd64:
+		case ValueOpcode::BufferAtomicUMin64:
 		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
@@ -154,6 +158,7 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {
 
 SharedAccess SharedAccessOf(ValueOpcode opcode) {
 	switch (opcode) {
+		case ValueOpcode::BvhStack: return SharedAccess::ReadWrite;
 		case ValueOpcode::LoadSharedU8:
 		case ValueOpcode::LoadSharedU16:
 		case ValueOpcode::LoadSharedU32:

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h

@@ -17,7 +17,7 @@ enum class ValueOpcode : uint16_t {
 };
 
 enum class BufferAccess { None, Read, Write, Atomic };
-enum class SharedAccess { None, Read, Write, Atomic, Append, Consume };
+enum class SharedAccess { None, Read, Write, ReadWrite, Atomic, Append, Consume };
 enum class AddressAccess { None, Read, Write };
 enum class ImageAccess { None, Read, Write, Atomic };
 enum class ImageResourceClass { None, Sampled, Storage };

src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc

@@ -219,9 +219,11 @@ VALUE_OPCODE(FPFract64, F64, F64)
 
 VALUE_OPCODE(LaneId, U32)
 VALUE_OPCODE(Ballot, U32x4, U1)
+VALUE_OPCODE(LiveMask, U32x4)
 VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
 VALUE_OPCODE(ReadLane, U32, U32, U32)
 VALUE_OPCODE(SubgroupUMinU32, U32, U32)
+VALUE_OPCODE(SubgroupUMaxU32, U32, U32)
 VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
 VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1, U32)
 VALUE_OPCODE(PermuteU32, U32, U32, U32, U1)
@@ -267,6 +269,7 @@ VALUE_OPCODE(BufferAtomicIAdd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicUMin64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -309,12 +312,15 @@ VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1)
 VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
 VALUE_OPCODE(SharedAtomicMaskedOr32, Void, U32, U32, U32, U1)
-VALUE_OPCODE(DataAppend, U32, U32, U1)
-VALUE_OPCODE(DataConsume, U32, U32, U1)
+VALUE_OPCODE(DataAppend, U32, U32, U32, U32, U1)
+VALUE_OPCODE(DataConsume, U32, U32, U32, U32, U1)
 VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
 
 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
 VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
+VALUE_OPCODE(Bvh8Intersect, Bvh8Result, U32x4, ImageAddress, U1)
+VALUE_OPCODE(Bvh8Extract, U32, Bvh8Result, U32)
+VALUE_OPCODE(BvhStack, U32x2, U32, U32, U32x4, U32x4, U1)
 VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
 VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)

src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp

@@ -167,6 +167,16 @@ bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) {
 	return false;
 }
 
+// Whether one operand is the other's negation.
+bool IsComplementPair(const Inst& inst, ValueOpcode negation) {
+	for (uint32_t arg = 0; arg < 2u; ++arg) {
+		const auto* negate = Arg(inst, arg).TryInstruction();
+		if (negate != nullptr && negate->GetOpcode() == negation &&
+		    Arg(*negate, 0) == Arg(inst, arg ^ 1u)) return true;
+	}
+	return false;
+}
+
 bool FoldSelect(Inst& inst) {
 	const auto condition   = Arg(inst, 0);
 	const auto true_value  = Arg(inst, 1);
@@ -626,8 +636,9 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 		case ValueOpcode::BitwiseXor32:
 			if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
 				    return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b;
-			    })) {
-				ReplaceBinaryIdentity(inst, Type::U32, 0u);
+			    }) && !ReplaceBinaryIdentity(inst, Type::U32, 0u) &&
+			    IsComplementPair(inst, ValueOpcode::BitwiseNot32)) {
+				Replace(inst, Value(UINT32_MAX));
 			}
 			return;
 		case ValueOpcode::BitwiseNot32: {
@@ -802,6 +813,8 @@ void FoldInstruction(Block& block, Block::iterator instruction,
 					Replace(inst, lhs.U1() ? lhs : rhs);
 				} else if (IsImmediate(rhs, Type::U1)) {
 					Replace(inst, rhs.U1() ? rhs : lhs);
+				} else if (IsComplementPair(inst, ValueOpcode::LogicalNot)) {
+					Replace(inst, Value(true));
 				} else {
 					const auto* left = lhs.TryInstruction();
 					const auto* right = rhs.TryInstruction();
@@ -892,8 +905,10 @@ private:
 		if (inst.GetOpcode() != ValueOpcode::CompositeExtractU32x4 ||
 		    !Immediate(Arg(inst, 1), 0u)) return {};
 		const auto* source = Arg(inst, 0).TryInstruction();
-		return source != nullptr && source->GetOpcode() == ValueOpcode::Ballot
-		           ? Arg(*source, 0) : Value {};
+		if (source == nullptr) return {};
+		// Helper lanes keep the per-lane EXEC, so the live mask projects to true.
+		if (source->GetOpcode() == ValueOpcode::LiveMask) return Value(true);
+		return source->GetOpcode() == ValueOpcode::Ballot ? Arg(*source, 0) : Value {};
 	}
 
 	bool CanProject(Value value) {

src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp

@@ -99,89 +99,66 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s
 	return Value(&phi);
 }
 
-// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction.
+// A full-EXEC prefix scan with both half-wave endpoint reads is a reduction. Wave64 first joins
+// the two rows of each half; a wave32 half is one row.
 // Fragment subgroups may have no physical invocation at either endpoint.
-class UMinReductionMatcher {
+class WaveReductionMatcher {
 public:
-	explicit UMinReductionMatcher(Block* block): block(block) {}
+	WaveReductionMatcher(const Block* block, ValueOpcode operation)
+	    : block(block), operation(operation) {}
 
-	Value Match(const Inst& minimum) {
-		const auto* low  = Get(minimum.Arg(0), ValueOpcode::ReadLane);
-		const auto* high = Get(minimum.Arg(1), ValueOpcode::ReadLane);
+	Value Match(const Inst& reduction, uint32_t wave_size) const {
+		const auto* low  = Get(reduction.Arg(0), ValueOpcode::ReadLane);
+		const auto* high = Get(reduction.Arg(1), ValueOpcode::ReadLane);
 		if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) {
 			return {};
 		}
-		const auto lane = [](Value value, uint32_t expected) {
+		const auto lane = [&](Value value, uint32_t expected) {
 			value = value.Resolve();
 			return value.IsImmediate() && value.GetType() == Type::U32 &&
-			       value.U32() % 64u == expected;
+			       value.U32() % wave_size == expected;
 		};
-		if (!((lane(low->Arg(1), 31) && lane(high->Arg(1), 63)) ||
-		      (lane(low->Arg(1), 63) && lane(high->Arg(1), 31)))) {
+		const auto half = wave_size / 2u;
+		if (!((lane(low->Arg(1), half - 1u) && lane(high->Arg(1), wave_size - 1u)) ||
+		      (lane(low->Arg(1), wave_size - 1u) && lane(high->Arg(1), half - 1u)))) {
 			return {};
 		}
-		const auto* joined = Get(Unwrap(low->Arg(0)), ValueOpcode::UMin32);
+		if (wave_size == 32u) return MatchScan(low->Arg(0));
+		const auto* joined = Get(low->Arg(0), operation);
 		if (joined == nullptr) return {};
 		for (size_t index = 0; index < 2; index++) {
-			auto attempt = *this;
-			const auto* cross = attempt.Get(attempt.Unwrap(joined->Arg(index)), ValueOpcode::Permlane16U32);
+			const auto* cross = Get(joined->Arg(index), ValueOpcode::Permlane16U32);
 			if (cross == nullptr) continue;
-			const auto flags = cross->Flags<PermlaneFlags>();
-			auto prefix = attempt.Unwrap(joined->Arg(1 - index));
+			const auto flags  = cross->Flags<PermlaneFlags>();
+			const auto prefix = joined->Arg(1 - index).Resolve();
 			if (!flags.x16 || flags.fetch_inactive || !flags.bound_control ||
 			    cross->Arg(1).Resolve() != Value(UINT32_MAX) ||
-			    cross->Arg(2).Resolve() != Value(UINT32_MAX) ||
-			    !attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) {
+			    cross->Arg(2).Resolve() != Value(UINT32_MAX) || !FullExec(cross->Arg(3)) ||
+			    cross->Arg(0).Resolve() != prefix) {
 				continue;
 			}
-			for (const uint32_t shift: {8u, 4u, 2u, 1u}) {
-				prefix = attempt.MatchRow(prefix, shift);
-			}
-			prefix = attempt.Unwrap(prefix);
-			if (prefix.IsEmpty() || attempt.saved_exec.IsEmpty()) return prefix;
-			const auto* seed = attempt.Get(prefix, ValueOpcode::SelectU32);
-			if (seed != nullptr && seed->Arg(2).Resolve() == Value(UINT32_MAX)) {
-				const auto predicate = seed->Arg(0).Resolve();
-				const auto* subset = attempt.Get(predicate, ValueOpcode::LogicalAnd);
-				if (predicate == attempt.saved_exec ||
-				    (subset != nullptr && (subset->Arg(0).Resolve() == attempt.saved_exec ||
-				                           subset->Arg(1).Resolve() == attempt.saved_exec))) {
-					return prefix;
-				}
-			}
+			if (const auto seed = MatchScan(prefix); !seed.IsEmpty()) return seed;
 		}
 		return {};
 	}
 
 private:
+	// The row scan's input.
+	Value MatchScan(Value prefix) const {
+		for (const uint32_t shift: {8u, 4u, 2u, 1u}) {
+			prefix = MatchRow(prefix, shift);
+		}
+		return prefix;
+	}
+
 	Inst* Get(Value value, ValueOpcode opcode) const {
 		auto* inst = value.Resolve().TryInstruction();
 		return inst != nullptr && inst->Parent() == block && inst->GetOpcode() == opcode
 		           ? inst : nullptr;
 	}
 
-	bool FullExec(Value predicate) {
-		predicate = predicate.Resolve();
-		if (predicate == Value(true)) return true;
-		const auto* either = Get(predicate, ValueOpcode::LogicalOr);
-		if (either == nullptr) return false;
-		for (size_t index = 0; index < 2; index++) {
-			const auto* negate = Get(either->Arg(index), ValueOpcode::LogicalNot);
-			if (negate == nullptr || negate->Arg(0).Resolve() != either->Arg(1 - index).Resolve()) continue;
-			const auto original = negate->Arg(0).Resolve();
-			if (saved_exec.IsEmpty()) saved_exec = original;
-			return saved_exec == original;
-		}
-		return false;
-	}
-
-	Value Unwrap(Value value) {
-		while (const auto* select = Get(value, ValueOpcode::SelectU32)) {
-			if (!FullExec(select->Arg(0))) break;
-			value = select->Arg(1);
-		}
-		return value.Resolve();
-	}
+	// Constant propagation folds the x | ~x EXEC of s_orn2_saveexec to true.
+	static bool FullExec(Value predicate) { return predicate.Resolve() == Value(true); }
 
 	static bool IsRowShift(const Inst& inst, uint32_t shift) {
 		const auto flags = inst.Flags<DppMoveFlags>();
@@ -189,22 +166,24 @@ private:
 		       flags.bank_mask == 0xf && !flags.fetch_inactive && !flags.bound_control && !flags.dpp8;
 	}
 
-	Value MatchRow(Value value, uint32_t shift) {
-		const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32);
+	Value MatchRow(Value value, uint32_t shift) const {
+		const auto* update = Get(value, ValueOpcode::DppUpdateU32);
 		if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {};
-		const auto* minimum = Get(update->Arg(0), ValueOpcode::UMin32);
-		if (minimum == nullptr) return {};
+		const auto* combined = Get(update->Arg(0), operation);
+		if (combined == nullptr) return {};
 		for (size_t index = 0; index < 2; index++) {
-			const auto* move = Get(minimum->Arg(index), ValueOpcode::DppMoveU32);
+			const auto* move = Get(combined->Arg(index), ValueOpcode::DppMoveU32);
 			if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue;
-			const auto previous = Unwrap(minimum->Arg(1 - index));
-			if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous;
+			const auto previous = combined->Arg(1 - index).Resolve();
+			if (move->Arg(0).Resolve() == previous && update->Arg(1).Resolve() == previous) {
+				return previous;
+			}
 		}
 		return {};
 	}
 
-	Block* block;
-	Value  saved_exec;
+	const Block* block;
+	ValueOpcode  operation;
 };
 
 } // namespace
@@ -217,18 +196,22 @@ ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) {
 
 	for (auto* block: program.blocks) {
 		for (auto& inst: *block) {
-			if (wave_size == 64u && inst.GetOpcode() == ValueOpcode::UMin32) {
-				const auto seed = UMinReductionMatcher(block).Match(inst);
+			const auto opcode = inst.GetOpcode();
+			if (opcode == ValueOpcode::UMin32 || opcode == ValueOpcode::UMax32) {
+				const auto seed = WaveReductionMatcher(block, opcode).Match(inst, wave_size);
 				if (!seed.IsEmpty()) {
 					const auto position = std::find_if(block->begin(), block->end(),
 					                                  [&](const Inst& candidate) { return &candidate == &inst; });
-					auto& reduction = *block->PrependNewInst(position, ValueOpcode::SubgroupUMinU32, {seed});
+					const auto reduce    = opcode == ValueOpcode::UMin32
+					                           ? ValueOpcode::SubgroupUMinU32
+					                           : ValueOpcode::SubgroupUMaxU32;
+					auto&      reduction = *block->PrependNewInst(position, reduce, {seed});
 					inst.ReplaceUsesWith(Value(&reduction));
 					stats.rewritten_reductions++;
 				}
 				continue;
 			}
-			if (inst.GetOpcode() != ValueOpcode::ReadLane) {
+			if (opcode != ValueOpcode::ReadLane) {
 				continue;
 			}
 			const auto selector = inst.Arg(1).Resolve();

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -331,6 +331,18 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		keys.resize(count);
 		return true;
 	};
+	// Reads the key of each record whose rank is set.
+	const auto read_ranks = [&](uint64_t base, uint64_t size, uint64_t first, uint64_t step,
+	                            uint32_t ranks) {
+		keys.reserve(std::popcount(ranks));
+		for (; ranks != 0u; ranks &= ranks - 1u) {
+			const auto offset = first + std::countr_zero(ranks) * step;
+			uint32_t key = 0;
+			if (offset > UINT32_MAX || !read_selector(base, size, offset, {&key, 1})) return false;
+			keys.push_back(key);
+		}
+		return true;
+	};
 	if (selector->rows) {
 		const auto& rows = *selector->rows;
 		DescriptorValue count_value;
@@ -377,19 +389,21 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		}
 	} else if (selector->bitset) {
 		ShaderBufferResource material;
-		std::array<uint32_t, 4> masks {};
+		std::array<uint32_t, 8> storage {};
+		const auto record_words = std::min<size_t>(selector->stride / 4u, storage.size());
+		const auto masks = std::span(storage).first(record_words);
 		uint32_t first = 0, count = 0;
 		if (!DecodeBufferDescriptor(material_value, material) ||
 		    !WalkedValue(program, indirect.selector_first, first) ||
 		    !EvaluateIndirectCount(program, indirect, runtime, count) || count != masks.size() * 32u ||
-		    selector->stride != sizeof(masks) || selector->offset != 0u ||
-		    material.Type() != 0u || material.Stride() != sizeof(masks) ||
+		    selector->stride != masks.size_bytes() || selector->offset != 0u ||
+		    material.Type() != 0u || material.Stride() != selector->stride ||
 		    material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
 		    (material.Base48() & 3u) != 0u) return false;
 		if (material.Format() != Prospero::BufferFormat::kInvalid) {
 			const auto words = material.GetSize() / sizeof(uint32_t);
-			// Bitmap storage can grow with resolution while its candidate domain stays
-			// at 128 keys. Bound scratch memory, preserving each coherent read's capture.
+			// Bitmap storage can grow with resolution while its candidate domain stays at 32 keys
+			// per record word. Bound scratch memory, preserving each coherent read's capture.
 			for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) {
 				keys.resize(std::min<uint64_t>(words - first_word, MaxIndirectDescriptorProbes));
 				if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u,
@@ -445,12 +459,35 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		     uint64_t {first} + count > material.NumRecords())) return false;
 		const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u;
 		if (last > UINT32_MAX) return false;
+		// Out-of-bounds scalar reads return zero.
 		const auto end = count != 0u ? last * step + selector->offset + 4u : 0u;
-		if (end > uint64_t {UINT32_MAX} + 1u ||
-		    (indirect.workgroup_axis == UINT32_MAX && end > size))
-			return false;
-		if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count))
+		if (end > uint64_t {UINT32_MAX} + 1u) return false;
+		const auto start = uint64_t {first} * step + selector->offset;
+		if (selector->ranks) {
+			// A scan selects the ranks some word of its mask's address class sets.
+			const auto& mask = *selector->ranks;
+			DescriptorValue mask_value;
+			ShaderBufferResource masks;
+			if (count > 32u || !WalkedDescriptor(program, mask.source, mask_value) ||
+			    !DecodeBufferDescriptor(mask_value, masks)) return false;
+			// Each chunk holds whole strides of the power-of-two class, or one of its words.
+			const auto words = mask.stride / 4u;
+			const auto chunk =
+			    words <= MaxIndirectDescriptorProbes ? MaxIndirectDescriptorProbes : 1u;
+			uint32_t ranks = 0;
+			for (auto at = uint64_t {mask.offset}; at + 4u <= masks.GetSize();
+			     at += uint64_t {std::max(chunk, words)} * 4u) {
+				keys.resize(std::min<uint64_t>((masks.GetSize() - at) / 4u, chunk));
+				if (!ReadScalarTable(masks.Base48(), masks.GetSize(), at, runtime, keys))
+					return false;
+				for (size_t word = 0; word < keys.size(); word += words) ranks |= keys[word];
+			}
+			keys.clear();
+			if (count < 32u) ranks &= (1u << count) - 1u;
+			if (!read_ranks(base, size, start, step, ranks)) return false;
+		} else if (!read_keys(base, size, start, step, count)) {
 			return false;
+		}
 	} else if (!indirect.selector_mask.IsEmpty()) {
 		uint32_t mask = 0;
 		uint32_t count = 0;
@@ -462,18 +499,8 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		if (count < 32u) mask &= (1u << count) - 1u;
 		const auto material_base =
 		    (static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
-		keys.reserve(std::popcount(mask));
-		while (mask != 0u) {
-			const auto index = std::countr_zero(mask);
-			const auto offset = static_cast<uint64_t>(selector->offset) +
-			                    static_cast<uint64_t>(index) * selector->stride;
-			if (offset > UINT32_MAX) return false;
-			uint32_t key = 0;
-			if (!read_selector(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
-			                   {&key, 1})) return false;
-			keys.push_back(key);
-			mask &= mask - 1u;
-		}
+		if (!read_ranks(material_base, UINT64_MAX, selector->offset, selector->stride, mask))
+			return false;
 	} else if (selector->stride == 0u) {
 		ShaderBufferResource material;
 		if (!DecodeBufferDescriptor(material_value, material) ||
@@ -503,6 +530,7 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX)
 			for (auto& key: keys)
 				key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum);
+		if (selector->excluded) std::erase(keys, *selector->excluded);
 		std::ranges::sort(keys);
 		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 	}
@@ -756,6 +784,12 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 			std::iota(keys.begin(), keys.end(), first_key);
 			if (!keys.empty() && keys.back() < keys.front())
 				std::rotate(keys.begin(), keys.begin() + (UINT32_MAX - first_key + 1ull), keys.end());
+			if (indirect.workgroup_shift != 0u) {
+				for (auto& key: keys)
+					key = static_cast<uint32_t>(uint64_t {key} * indirect.workgroup_multiplier >>
+					                            indirect.workgroup_shift);
+				keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
+			}
 		} else {
 			if (!indirect.selector_mask.IsEmpty() && table_value.dword_count != 2u) return false;
 			if (!selector_keys->resolved) {
@@ -1041,6 +1075,26 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			return dimension == Decoder::ImageDimension::Dim1D ||
 			       dimension == Decoder::ImageDimension::Dim2D;
 		};
+		// The descriptor's type selects the address layout. A 2D or 3D candidate reads the leading
+		// coordinates of an equal or larger dimension, but an array with more than one slice reads
+		// the next as its slice. Derivative, LOD, clamp and mip operands shift with the dimension.
+		const auto leading = [&](Decoder::ImageDimension dimension, const DescriptorValue& value) {
+			const bool sliced =
+			    static_cast<Prospero::ImageType>((value.dwords[3] >> 28u) & 0xfu) ==
+			        Prospero::ImageType::kColor2DArray &&
+			    (value.dwords[4] & 0x1fffu) != ((value.dwords[4] >> 16u) & 0x1fffu);
+			return !sliced &&
+			       (dimension == Decoder::ImageDimension::Dim2D || dimension == base.dimension);
+		};
+		const bool leading_coordinates = (base.dimension == Decoder::ImageDimension::Dim2D ||
+		                                  base.dimension == Decoder::ImageDimension::Dim3D) &&
+		    leading(image_class.dimension, snapshot.images[exemplar]) &&
+		    std::ranges::none_of(program.memory_info, [&](const MemoryInfo& memory) {
+			    return memory.kind == ResourceKind::Image && memory.resource == root_index &&
+			           (memory.image_has_mip || (memory.image_sample_flags &
+			               (Decoder::ImageSampleFlagLod | Decoder::ImageSampleFlagDerivative |
+			                Decoder::ImageSampleFlagLodClamp)) != 0u);
+		    });
 		for (uint32_t candidate = 0; candidate < specialization.images.size(); candidate++) {
 			auto& image = specialization.images[candidate];
 			if (image.indirect_root != root_index) {
@@ -1060,8 +1114,10 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			    base.dimension == Decoder::ImageDimension::Dim2D &&
 			    !image.cube && !image_class.cube && is_1d_or_2d(image.dimension) &&
 			    is_1d_or_2d(image_class.dimension);
+			const bool prefix_coordinates =
+			    leading_coordinates && leading(image.dimension, snapshot.images[candidate]);
 			if ((!same_coordinates && !(is_2d(image.dimension) && is_2d(image_class.dimension)) &&
-			     !unfiltered_coordinates) ||
+			     !unfiltered_coordinates && !prefix_coordinates) ||
 			    image.conversion_format != image_class.conversion_format ||
 			    (!storage_swizzle && image.shader_swizzle != image_class.shader_swizzle)) {
 				return SpecializationFail(
@@ -1164,14 +1220,16 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 				block.sources.push_back(program.info.buffers.at(memory.resource).source);
 			} else {
 				block.sources.push_back(program.info.images.at(memory.resource).source);
+				block.images.push_back(memory.resource);
 				if (image.needs_sampler) {
 					block.sources.push_back(program.info.samplers.at(memory.sampler).source);
 				}
 			}
 		}
-		std::ranges::sort(block.sources);
-		block.sources.erase(std::unique(block.sources.begin(), block.sources.end()),
-		                    block.sources.end());
+		for (auto* list: {&block.sources, &block.images}) {
+			std::ranges::sort(*list);
+			list->erase(std::unique(list->begin(), list->end()), list->end());
+		}
 	}
 	for (auto& block: blocks) block.condition = predicate(block.condition);
 	if (std::ranges::none_of(
@@ -1645,7 +1703,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		}
 		if (source->indirect_descriptor.has_value()) {
 			snapshot.images[i] = {.dword_count = 8u};
-			if (!WalkedActive(program, image.source)) {
+			if (!WalkedActive(program, image.source) || !WalkedImageActive(program, i)) {
 				continue;
 			}
 			const auto& indirect = *source->indirect_descriptor;
@@ -1665,7 +1723,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			if (!evaluate(image.source, snapshot.images[i])) {
 				return false;
 			}
-			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
+			// An image listed only by unreached blocks reads zero, not its source's descriptor.
+			if (!WalkedImageActive(program, i) ||
+			    !ValidImageDescriptor(snapshot.images[i], image.r128)) {
 				snapshot.images[i].dwords.fill(0);
 			}
 		}
@@ -1711,8 +1771,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 }
 
 uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value) {
-	// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, and
-	// entries that need shader-side conversion, read as null instead of failing the draw.
+	// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, entries of
+	// another dimension and entries that need shader-side conversion read as null instead of
+	// failing the draw.
 	const auto format = static_cast<Prospero::BufferFormat>((value.dwords[1] >> 20u) & 0x1ffu);
 	const auto tile = static_cast<Prospero::TileMode>((value.dwords[3] >> 20u) & 0x1fu);
 	const auto numeric_class =

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -7,6 +7,8 @@
 
 #include <algorithm>
 #include <bit>
+#include <cmath>
+#include <cstdio>
 #include <fmt/format.h>
 #include <map>
 #include <numeric>
@@ -1014,7 +1016,9 @@ private:
 				    (!shared_image_heap && (a.table_offset != b.table_offset ||
 				                           a.table_immediate != b.table_immediate ||
 				                           a.table_stride != b.table_stride)) || a.indexed != b.indexed ||
-				    a.workgroup_axis != b.workgroup_axis || a.sources != b.sources ||
+				    a.workgroup_axis != b.workgroup_axis ||
+				    a.workgroup_multiplier != b.workgroup_multiplier ||
+				    a.workgroup_shift != b.workgroup_shift || a.sources != b.sources ||
 				    a.signed_count != b.signed_count ||
 				    !EquivalentValue(m_program, a.key_count, b.key_count) ||
 				    a.selector_first.IsEmpty() != b.selector_first.IsEmpty() ||
@@ -1136,31 +1140,63 @@ private:
 		LaneQuantifier lanes = LaneQuantifier::All;
 	};
 
+	// The immediate that edge proves value differs from on every lane, if any.
+	std::optional<uint32_t> ExcludedBy(const EdgePredicate& edge, Value value) const {
+		if (edge.lanes != LaneQuantifier::All) return {};
+		const auto* test = edge.condition.TryInstruction();
+		if (test != nullptr && test->NumArgs() == 2u &&
+		    ((test->GetOpcode() == ValueOpcode::INotEqual32 && edge.positive) ||
+		     (test->GetOpcode() == ValueOpcode::IEqual32 && !edge.positive))) {
+			for (uint32_t arg = 0; arg < 2u; ++arg) {
+				uint32_t immediate;
+				if (ImmediateU32(test->Arg(arg), immediate) &&
+				    EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) return immediate;
+			}
+		}
+		return {};
+	}
+
 	bool NonzeroOnEntry(Value value, const Block* block) const {
 		if (block == nullptr) return false;
 		bool witnessed = false;
 		const bool guarded = GuardedOnEntry(block, [](const Block* current) {
 			return current->ImmPredecessors().size() != 1u;
 		}, [&](const EdgePredicate& edge) {
-			if (edge.lanes != LaneQuantifier::All) return false;
-			const auto* test = edge.condition.TryInstruction();
-			if (test != nullptr && test->NumArgs() == 2u &&
-			    ((test->GetOpcode() == ValueOpcode::INotEqual32 && edge.positive) ||
-			     (test->GetOpcode() == ValueOpcode::IEqual32 && !edge.positive))) {
-				for (uint32_t arg = 0; arg < 2u; ++arg) {
-					uint32_t immediate;
-					if (ImmediateU32(test->Arg(arg), immediate) && immediate == 0u &&
-					    EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) return witnessed = true;
-				}
-			}
-			return false;
+			return ExcludedBy(edge, value) == 0u && (witnessed = true);
 		});
 		return guarded && witnessed;
 	}
 
-	uint32_t WorkgroupAxis(Value key) const {
+	// The key that every path from its definition to block rules out, if any.
+	std::optional<uint32_t> ExcludedKeyOnEntry(Value key, const Block* block) const {
+		const auto* definition = key.Resolve().TryInstruction();
+		if (definition == nullptr) return {};
+		std::optional<uint32_t> excluded;
+		const auto defined = [&](const Block* current) { return current == definition->Parent(); };
+		const bool guarded = GuardedOnEntry(block, defined, [&](const EdgePredicate& edge) {
+			const auto immediate = ExcludedBy(edge, key);
+			if (!immediate || (excluded && excluded != immediate)) return false;
+			excluded = immediate;
+			return true;
+		});
+		return guarded ? excluded : std::nullopt;
+	}
+
+	// The workgroup ID axis of key; with first, also of a runtime first + workgroup ID sum.
+	uint32_t WorkgroupAxis(Value key, Value* first = nullptr) const {
 		const auto* builtin = key.Resolve().TryInstruction();
 		uint32_t axis = UINT32_MAX;
+		if (first != nullptr && builtin != nullptr && builtin->GetOpcode() == ValueOpcode::IAdd32) {
+			for (uint32_t arm = 0; arm < 2u; ++arm) {
+				axis = WorkgroupAxis(builtin->Arg(arm));
+				const auto base = builtin->Arg(arm ^ 1u);
+				if (axis == UINT32_MAX ||
+				    !ValidateRuntimeValue(m_program, base, RuntimeValueType::Integer)) continue;
+				*first = base;
+				return axis;
+			}
+			return UINT32_MAX;
+		}
 		if (m_program.stage != ShaderType::Compute || builtin == nullptr ||
 		    builtin->GetOpcode() != ValueOpcode::GetBuiltin || builtin->NumArgs() != 2u ||
 		    builtin->Arg(0) != Value(static_cast<uint32_t>(StageInputKind::WorkgroupId)) ||
@@ -1168,6 +1204,27 @@ private:
 		return axis;
 	}
 
+	// Matches key = (first + workgroup ID) * multiplier >> shift with shift >= 32; first and the
+	// division are optional.
+	bool MatchWorkgroupKey(Value key, DescriptorSource::IndirectDescriptor& indirect) const {
+		uint32_t bits = 0, multiplier = 0;
+		const auto* shift = key.Resolve().TryInstruction();
+		if (shift != nullptr && shift->GetOpcode() == ValueOpcode::ShiftRightLogical32) {
+			const auto* high = shift->Arg(0).Resolve().TryInstruction();
+			if (high != nullptr && high->GetOpcode() == ValueOpcode::UMulHi &&
+			    ImmediateU32(shift->Arg(1), bits) && bits < 32u &&
+			    ImmediateU32(high->Arg(1), multiplier)) {
+				key = high->Arg(0);
+				bits += 32u;
+			}
+		}
+		indirect.workgroup_axis = WorkgroupAxis(key, &indirect.selector_first);
+		if (indirect.workgroup_axis == UINT32_MAX) return false;
+		indirect.workgroup_multiplier = multiplier;
+		indirect.workgroup_shift = bits;
+		return true;
+	}
+
 	bool MatchTableOffset(Value value, Value& key, uint32_t& offset, uint32_t& stride) const {
 		offset = 0;
 		for (;;) {
@@ -1672,6 +1729,8 @@ private:
 		uint32_t initial_arm;
 	};
 
+	static bool ZeroBased(uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; }
+
 	std::optional<Induction> MatchInduction(Value key, const auto& accepts_step) const {
 		key = key.Resolve();
 		const auto* phi = key.TryInstruction();
@@ -1733,8 +1792,7 @@ private:
 	}
 
 	uint32_t PostTestLoopMaximum(Value key) const {
-		const auto induction = MatchInduction(key,
-		    [](uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; });
+		const auto induction = MatchInduction(key, ZeroBased);
 		if (!induction) return UINT32_MAX;
 		const auto [phi, step, initial_arm] = *induction;
 		const auto* incoming = phi->PhiBlock(initial_arm ^ 1u);
@@ -1767,8 +1825,7 @@ private:
 
 	const Inst* BoundedLoop(Value key, const Block* use, const auto& accepts_bound,
 	                       LaneQuantifier lanes = LaneQuantifier::Any) const {
-		return BoundedInduction(key, use,
-		    [](uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; },
+		return BoundedInduction(key, use, ZeroBased,
 		    [&](const Inst& compare, uint32_t initial) {
 			    return (compare.GetOpcode() == ValueOpcode::SLessThan32 ||
 			            compare.GetOpcode() == ValueOpcode::ULessThan32) &&
@@ -1793,6 +1850,29 @@ private:
 		});
 	}
 
+	// A zero-based key that only per-lane data bounds takes the count all uniform zero-based
+	// loops in the program share, assuming each lane's count stays within it.
+	bool ProgramLoopCount(Value key, DescriptorSource::IndirectDescriptor& indirect) const {
+		if (!MatchInduction(key, ZeroBased) || BoundedLoop(key, nullptr) != nullptr) return false;
+		const Inst* count = nullptr;
+		for (auto* block: m_program.blocks) {
+			for (auto& phi: *block) {
+				if (phi.GetOpcode() != ValueOpcode::Phi) break;
+				const auto* compare = BoundedLoop(Value(&phi), nullptr);
+				if (compare == nullptr) continue;
+				if (count != nullptr && !EquivalentValue(m_program, compare->Arg(1), count->Arg(1)))
+					return false;
+				count = compare;
+			}
+		}
+		if (count == nullptr) return false;
+		std::printf("Warning: shader 0x%016llx bounds a per-lane table loop by its uniform loops' "
+		            "count\n", static_cast<unsigned long long>(m_program.shader_hash));
+		indirect.key_count    = count->Arg(1);
+		indirect.signed_count = count->GetOpcode() == ValueOpcode::SLessThan32;
+		return true;
+	}
+
 	Value ActiveValue(Value value, Value guard) const {
 		value = value.Resolve();
 		while (const auto* select = value.TryInstruction()) {
@@ -1829,6 +1909,10 @@ private:
 	}
 
 	bool Dominates(const Block* initial, const Block* use, Value witness = {}) const {
+		// The proven local-zero invocation fails every all-lane bypass condition that excludes it,
+		// such as an all-inactive or an all-nonzero bypass.
+		const auto* negated = SimplifyGuard(witness).TryInstruction();
+		const bool nonzero = negated != nullptr && negated->GetOpcode() == ValueOpcode::LogicalNot;
 		std::vector<const Block*> pending {use};
 		for (size_t index = 0; index < pending.size(); ++index) {
 			const auto* current = pending[index];
@@ -1837,9 +1921,9 @@ private:
 			for (const auto* previous: current->ImmPredecessors()) {
 				if (!witness.IsEmpty()) {
 					const auto edge = ConditionalEdge(previous, current);
-					// The proven local-zero invocation makes an all-inactive bypass impossible.
-					if (edge && edge->lanes == LaneQuantifier::All && !edge->positive &&
-					    Implies(witness, edge->condition)) continue;
+					if (edge && edge->lanes == LaneQuantifier::All &&
+					    (edge->positive ? nonzero && Implies(edge->condition, negated->Arg(0))
+					                    : Implies(witness, edge->condition))) continue;
 				}
 				if (std::ranges::find(pending, previous) == pending.end()) pending.push_back(previous);
 			}
@@ -1948,7 +2032,10 @@ private:
 
 	bool LocalZeroWitness(Value guard) const {
 		const auto* equal = SimplifyGuard(guard).TryInstruction();
-		if (equal == nullptr || equal->GetOpcode() != ValueOpcode::IEqual32) return false;
+		const bool negated = equal != nullptr && equal->GetOpcode() == ValueOpcode::LogicalNot;
+		if (negated) equal = SimplifyGuard(equal->Arg(0)).TryInstruction();
+		const auto expected = negated ? ValueOpcode::INotEqual32 : ValueOpcode::IEqual32;
+		if (equal == nullptr || equal->GetOpcode() != expected) return false;
 		for (uint32_t arg = 0; arg < 2u; ++arg) {
 			if (equal->Arg(arg).Resolve() != Value(0u)) continue;
 			Value induction;
@@ -1982,6 +2069,10 @@ private:
 		    EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1);
 		if ((positive ? op == ValueOpcode::UGreaterThan32 : op == ValueOpcode::ULessThanEqual32) &&
 		    EquivalentValue(m_program, test->Arg(1), value)) return test->Arg(0);
+		uint32_t limit = 0;
+		if ((positive ? op == ValueOpcode::ULessThanEqual32 : op == ValueOpcode::UGreaterThan32) &&
+		    ImmediateU32(test->Arg(1), limit) && limit != UINT32_MAX &&
+		    EquivalentValue(m_program, test->Arg(0), value)) return Value(limit + 1u);
 		const auto* scan = value.Resolve().TryInstruction();
 		if (finite_bitscan && scan != nullptr && scan->GetOpcode() == ValueOpcode::FindILsb32 &&
 		    (positive ? op == ValueOpcode::INotEqual32 : op == ValueOpcode::IEqual32)) {
@@ -2056,7 +2147,7 @@ private:
 		const auto* read = ActiveValue(value, active).TryInstruction();
 		if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadSharedU32 ||
 		    m_program.stage != ShaderType::Compute ||
-		    read->Arg(1).Resolve() != Value(true)) return {};
+		    !Implies(active, read->Arg(1))) return {};
 		uint64_t group_size = 1u;
 		for (const auto dimension: m_input.compute->threads_num) {
 			if (dimension == 0u || dimension > m_input.compute->wave_size) return {};
@@ -2087,10 +2178,16 @@ private:
 		const auto* limit = BoundedLoop(read_offset.index, read->Parent(),
 		    [](const Inst& compare, uint32_t) { return compare.GetOpcode() == ValueOpcode::ULessThan32; });
 		if (limit == nullptr) return {};
+		auto counter_active = active;
 		const auto* counter_read = ActiveValue(limit->Arg(1), active).TryInstruction();
+		// Every lane reads the same counter word, so a selected lane carries it.
+		if (counter_read != nullptr && counter_read->GetOpcode() == ValueOpcode::ReadFirstLane) {
+			counter_active = counter_read->Arg(1);
+			counter_read = ActiveValue(counter_read->Arg(0), counter_active).TryInstruction();
+		}
 		uint32_t consumed_counter = 0;
 		if (counter_read == nullptr || counter_read->GetOpcode() != ValueOpcode::LoadSharedU32 ||
-		    counter_read->Arg(1).Resolve() != Value(true) ||
+		    !Implies(counter_active, counter_read->Arg(1)) ||
 		    !constant_address(*counter_read, consumed_counter)) return {};
 
 		struct Initializer { const Inst* inst; uint32_t address; uint32_t words; };
@@ -2304,25 +2401,23 @@ private:
 		    ((offset | stride | memory->offset) & 3u) == 0u &&
 		    uint64_t {offset} + memory->offset <= UINT32_MAX;
 		if (affine) {
-			workgroup_axis = WorkgroupAxis(index);
-			if (workgroup_axis == UINT32_MAX) {
-				const auto* sum = index.Resolve().TryInstruction();
-				if (sum != nullptr && sum->GetOpcode() == ValueOpcode::IAdd32) {
-					for (uint32_t arm = 0; arm < 2u; ++arm) {
-						const auto axis = WorkgroupAxis(sum->Arg(arm));
-						const auto base = sum->Arg(arm ^ 1u);
-						if (axis == UINT32_MAX ||
-						    !ValidateRuntimeValue(m_program, base, RuntimeValueType::Integer)) continue;
-						workgroup_axis = axis;
-						first = base;
-						break;
-					}
-				}
-			}
+			workgroup_axis = WorkgroupAxis(index, &first);
 			const auto* bound = BoundedLoop(index, read->Parent());
 			if (bound != nullptr && bound->GetOpcode() == ValueOpcode::ULessThan32) {
 				count = bound->Arg(1);
+			} else if (workgroup_axis == UINT32_MAX && !require_bound) {
+				// Other bounded indices, such as bit scans, read the records up to their maximum;
+				// a bit scan of a nonzero mask load reads only the ranks its words set.
+				const auto maximum = SelectorMaximum(index);
+				if (maximum == 31u) selector.ranks = BitScanRanks(index);
+				if (maximum < MaxIndirectDescriptorProbes) count = Value(maximum + 1u);
 			}
+			// A record selected from a shared append list is below the list's bound.
+			const auto* lane = index.Resolve().TryInstruction();
+			if (count.IsEmpty() && lane != nullptr &&
+			    lane->GetOpcode() == ValueOpcode::ReadFirstLane &&
+			    HasActiveLane(lane->Arg(1), lane->Parent()))
+				count = SharedLoadBound(lane->Arg(0), lane->Arg(1));
 		}
 		// Raw pointers have no V# bounds for enumerating an unbounded selector domain.
 		if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty() &&
@@ -2498,7 +2593,8 @@ private:
 	// Each equation is an inductive subset obligation. Phis require every incoming
 	// value; AND needs either operand. Grounding rejects cycles without an initial
 	// value, then the greatest fixed point checks every initial and carried edge.
-	bool BitsetMaskSubset(Value mask, Value active, Value word, const Inst& read) const {
+	bool BitsetMaskSubset(Value mask, Value active, Value word,
+	                      std::span<const Inst* const> reads) const {
 		struct State {
 			Value value;
 			Value guard;
@@ -2510,7 +2606,10 @@ private:
 			bool all = true;
 			std::vector<size_t> terms;
 		};
-		const auto load_guard = read.Arg(4).Resolve();
+		// A scalar load reads one word without EXEC; four-word loads take their guard.
+		const bool scalar = reads[0]->GetOpcode() == ValueOpcode::ReadConstBuffer;
+		const auto load_guard = scalar ? Value(true) : reads[0]->Arg(4).Resolve();
+		const auto words = static_cast<uint32_t>(reads.size()) * (scalar ? 1u : 4u);
 		std::vector<State> states(2);
 		std::vector<Equation> equations(2);
 		const auto add = [&](State state) {
@@ -2521,9 +2620,9 @@ private:
 			equations.emplace_back();
 			return states.size() - 1u;
 		};
-		std::array<size_t, 5> roots {add({.value = active})};
-		for (uint32_t component = 0; component < 4u; ++component)
-			roots[component + 1u] = add({mask, active, component, component});
+		std::vector<size_t> roots {add({.value = active})};
+		for (uint32_t component = 0; component < words; ++component)
+			roots.push_back(add({mask, active, component, component}));
 		for (size_t index = 2; index < states.size(); ++index) {
 			const auto state = states[index];
 			const auto* inst = state.value.TryInstruction();
@@ -2573,11 +2672,14 @@ private:
 				switch (inst->GetOpcode()) {
 					case ValueOpcode::CompositeExtractU32x4: {
 						uint32_t component;
-						if (inst->Arg(0).Resolve().TryInstruction() == &read &&
-						    ImmediateU32(inst->Arg(1), component) && component == state.component)
+						if (inst->Arg(0).Resolve().TryInstruction() == reads[state.component / 4u] &&
+						    ImmediateU32(inst->Arg(1), component) && component == state.component % 4u)
 							equation.terms = {1u};
 						break;
 					}
+					case ValueOpcode::ReadConstBuffer:
+						if (scalar && inst == reads[state.component]) equation.terms = {1u};
+						break;
 					case ValueOpcode::Phi:
 						// The predecessor may be an earlier outer iteration. Establish
 						// this register's invariant without the current word or EXEC.
@@ -2598,17 +2700,17 @@ private:
 								const auto* masked = key.TryInstruction();
 								uint32_t bound = 0;
 								if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 &&
-								    DescendingLoop(word, masked->Parent(), bound) && bound == 3u) {
+								    DescendingLoop(word, masked->Parent(), bound) && bound == words - 1u) {
 									for (uint32_t side = 0; side < 2u; ++side) {
 										uint32_t bits;
-										if (ImmediateU32(masked->Arg(side), bits) && (bits & 3u) == 3u)
+										if (ImmediateU32(masked->Arg(side), bits) && (bits & bound) == bound)
 											key = masked->Arg(side ^ 1u).Resolve();
 									}
 								}
 								return EquivalentValue(m_program, key, word);
 							});
 							uint32_t literal;
-							if (ImmediateU32(equality, literal) && literal < 4u) selected_word = literal;
+							if (ImmediateU32(equality, literal) && literal < words) selected_word = literal;
 							equation.terms.push_back(term(inst->Arg(1), inst->Arg(0), selected_word));
 						}
 						if (!selected || !*selected)
@@ -2646,7 +2748,53 @@ private:
 		return std::ranges::all_of(roots, [&](size_t root) { return valid[root]; });
 	}
 
-	bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds,
+	// The mask words whose set bits are every record a bit scan of index selects: the scanned
+	// value keeps only bits of one scalar load, which reads a word its address can reach.
+	std::optional<DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks>
+	BitScanRanks(Value index) {
+		const auto* scan = index.Resolve().TryInstruction();
+		if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindILsb32) return {};
+		const auto* masked = scan->Arg(0).Resolve().TryInstruction();
+		if (masked == nullptr) return {};
+		std::vector<const Inst*> values {masked};
+		const Inst* load = nullptr;
+		for (size_t i = 0; i < values.size() && load == nullptr; ++i) {
+			const auto* value = values[i];
+			if (value->GetOpcode() == ValueOpcode::ReadConstBuffer) load = value;
+			if (value->GetOpcode() != ValueOpcode::Phi &&
+			    value->GetOpcode() != ValueOpcode::BitwiseAnd32) continue;
+			for (size_t arg = 0; arg < value->NumArgs(); ++arg) {
+				const auto* next = value->Arg(arg).Resolve().TryInstruction();
+				if (next != nullptr && std::ranges::find(values, next) == values.end())
+					values.push_back(next);
+			}
+		}
+		uint32_t memory_index = 0;
+		const auto* memory = load != nullptr ? ScalarReadMemory(*load, memory_index) : nullptr;
+		if (memory == nullptr || memory->offset > INT32_MAX ||
+		    !MemoryIndexBelongsTo(memory_index, *load) ||
+		    !BitsetMaskSubset(scan->Arg(0), Value(true), Value(0u), std::array {load})) return {};
+		auto address = load->Arg(1);
+		auto active = Value(true);
+		const auto* lane = address.Resolve().TryInstruction();
+		if (lane != nullptr && lane->GetOpcode() == ValueOpcode::ReadFirstLane) {
+			if (!HasActiveLane(lane->Arg(1), lane->Parent())) return {};
+			active = lane->Arg(1);
+			address = lane->Arg(0);
+		}
+		// A left shift keeps the address a multiple of its scale; other addresses read any word.
+		const auto* shift = ActiveValue(address, active).TryInstruction();
+		uint32_t bits = 0;
+		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 ||
+		    !ImmediateU32(shift->Arg(1), bits) || bits >= 32u) bits = 0;
+		DescriptorSource source;
+		if (!MakeRuntimeTableSource(*load, source)) return {};
+		return DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks {
+		    .source = InternSource(source), .stride = std::max(1u << bits, 4u),
+		    .offset = memory->offset & ~3u};
+	}
+
+	bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds, uint32_t words,
 	                         DescriptorSource::IndirectDescriptor& indirect) {
 		const auto selected_value = [&](Value value) {
 			for (;;) {
@@ -2670,7 +2818,7 @@ private:
 		}
 		const auto* word_inst = word.TryInstruction();
 		if (word_inst == nullptr || std::ranges::none_of(bounds, [&](const auto& bound) {
-			return bound.first == word_inst && bound.second == 3u;
+			return bound.first == word_inst && bound.second == words - 1u;
 		})) return false;
 		// FFBH's leading-zero count is inverted again to recover the set-bit rank.
 		for (uint32_t inverse = 0; inverse < 2u; ++inverse) {
@@ -2682,21 +2830,34 @@ private:
 		const auto* scan = bit.TryInstruction();
 		if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindUMsb32 ||
 		    !ImpliesNonzero(active, scan->Arg(0))) return false;
+		const auto bitmap_load = [&](const Inst& read, uint32_t offset) {
+			const auto index = read.Flags<MemoryFlags>().index;
+			if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(read.Parent()) ||
+			    index >= m_program.memory_info.size()) return false;
+			const auto& memory = m_program.memory_info[index];
+			return memory.kind == ResourceKind::Buffer && !memory.typed && !memory.formatted &&
+			       memory.idxen && !memory.offen && memory.offset == offset &&
+			       memory.data_bits == 32u && memory.data_dwords == 4u &&
+			       read.Arg(2).Resolve() == Value(0u) && read.Arg(3).Resolve() == Value(0u);
+		};
 		for (const auto* block: m_program.blocks) {
 			for (const auto& read: *block) {
-				if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(block)) continue;
-				const auto index = read.Flags<MemoryFlags>().index;
-				if (index >= m_program.memory_info.size()) continue;
-				const auto& memory = m_program.memory_info[index];
-				if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
-				    !memory.idxen || memory.offen || memory.offset != 0u || memory.data_bits != 32u ||
-				    memory.data_dwords != 4u || read.Arg(2).Resolve() != Value(0u) ||
-				    read.Arg(3).Resolve() != Value(0u) ||
-				    !BitsetMaskSubset(scan->Arg(0), active, word, read)) continue;
+				if (!bitmap_load(read, 0u)) continue;
+				// An eight-word record continues in a second four-word load of the same buffer and EXEC.
+				std::array<const Inst*, 2> reads {&read};
+				if (words == 8u)
+					for (const auto* other: m_program.blocks)
+						for (const auto& load: *other)
+							if (reads[1] == nullptr && bitmap_load(load, 16u) &&
+							    EquivalentValue(m_program, load.Arg(0), read.Arg(0)) &&
+							    EquivalentValue(m_program, load.Arg(4), read.Arg(4))) reads[1] = &load;
+				const auto parts = std::span(reads).first(words / 4u);
 				DescriptorSource source;
-				if (!MakeRuntimeTableSource(read, source)) continue;
+				if (std::ranges::find(parts, nullptr) != parts.end() ||
+				    !BitsetMaskSubset(scan->Arg(0), active, word, parts) ||
+				    !MakeRuntimeTableSource(read, source)) continue;
 				indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead {
-				    .source = InternSource(source), .stride = 16u, .offset = 0u, .bitset = true});
+				    .source = InternSource(source), .stride = words * 4u, .offset = 0u, .bitset = true});
 				return true;
 			}
 		}
@@ -2726,7 +2887,8 @@ private:
 		first.SetArg(1, Value(maximum));
 		indirect.selector_first = Value(&first);
 		indirect.key_count = Value(maximum + 1u);
-		if (maximum == 127u) MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, indirect);
+		if (maximum == 127u || maximum == 255u)
+			MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, (maximum + 1u) / 32u, indirect);
 		return true;
 	}
 
@@ -2993,6 +3155,47 @@ private:
 		return true;
 	}
 
+	// The EXEC predicate of the image access through use; empty for any other use.
+	Value AccessPredicate(const Use& use, bool sampler) const {
+		const auto& access = *use.user;
+		if (use.operand != (sampler ? 1u : 0u) ||
+		    ImageOpcodeInfoOf(access.GetOpcode()).access == ImageAccess::None) return {};
+		if (sampler && (access.GetOpcode() != ValueOpcode::ImageSampleRaw ||
+		    (m_program.memory_info[access.Flags<MemoryFlags>().index].image_sample_flags &
+		     Decoder::ImageSampleFlagAdjust) != 0u)) return {};
+		const auto mask = access.Arg(access.NumArgs() - 1u);
+		return mask.GetType() == Type::U1 ? mask : Value {};
+	}
+
+	// A key no bound covers takes its maximum from a signed clamp to a constant, on the key or on
+	// the key plus a constant in an earlier-tracked sibling read of the same table. Negative keys
+	// and keys past that maximum read the first row.
+	uint32_t ClampedTableMaximum(Value key, const DescriptorSource& table,
+	                             const DescriptorSource::IndirectDescriptor& indirect) const {
+		uint32_t row = 0, offset = 0;
+		const auto clamped = [&](Value value) {
+			const auto* min = value.Resolve().TryInstruction();
+			if (min != nullptr && min->GetOpcode() == ValueOpcode::SMin32)
+				for (uint32_t arg = 0; arg < 2u; ++arg)
+					if (ImmediateU32(min->Arg(arg), row)) return min->Arg(arg ^ 1u);
+			return Value {};
+		};
+		if (!clamped(key).IsEmpty()) return row;
+		for (const auto& sibling: m_indirect_descriptors) {
+			const auto& read = *m_sources[sibling.source].indirect_descriptor;
+			const auto* add = clamped(sibling.key).Resolve().TryInstruction();
+			if (add == nullptr || add->GetOpcode() != ValueOpcode::IAdd32 ||
+			    m_sources[read.table_source] != table ||
+			    read.table_offset != indirect.table_offset ||
+			    read.table_stride != indirect.table_stride ||
+			    read.table_immediate != indirect.table_immediate) continue;
+			for (uint32_t arg = 0; arg < 2u; ++arg)
+				if (ImmediateU32(add->Arg(arg), offset) &&
+				    EquivalentValue(m_program, add->Arg(arg ^ 1u), key)) return row;
+		}
+		return UINT32_MAX;
+	}
+
 	bool TryMakeTableDescriptor(Inst& handle, const DescriptorSource& descriptor,
 	                            IndirectDescriptorPlan& plan) {
 		const bool sampler = handle.GetOpcode() == ValueOpcode::GetSamplerResource;
@@ -3019,6 +3222,7 @@ private:
 		} else if (!plan.table_indexed &&
 		           MatchScalarMaterialKey(key, indirect, material_source,
 		                                  !sampler && table_source.dword_count == 4u, sampler)) {
+			indirect.selector->excluded = ExcludedKeyOnEntry(key, handle.Parent());
 			// Keep live descriptor payloads, but project descriptor-only SMEM onto
 			// the GPU key using the same plan as other indirect image reads.
 			plan.retain_reads |= table_source.dword_count == 4u;
@@ -3080,36 +3284,42 @@ private:
 			if (indirect.key_count.IsEmpty() &&
 			    !MatchUniformizedMaterialKey(key, handle, indirect, material_source, false) &&
 			    !MatchUniformizedMaterialKey(key, handle, indirect, material_source, true) &&
-			    !MatchReversedImageRange(key, indirect)) {
-				const auto maximum = SelectorMaximum(key);
-				if (maximum >= MaxIndirectDescriptorProbes) return false;
-				indirect.key_count = Value(maximum + 1u);
+			    !MatchReversedImageRange(key, indirect) && !MatchWorkgroupKey(key, indirect)) {
+				auto maximum = SelectorMaximum(key);
+				if (maximum >= MaxIndirectDescriptorProbes) {
+					// Each image access bounds the key on the lanes it executes.
+					maximum = 0u;
+					for (const auto& use: handle.Uses()) {
+						const auto active = AccessPredicate(use, false);
+						if (active.IsEmpty()) {
+							maximum = UINT32_MAX;
+							break;
+						}
+						maximum = std::max(maximum, SelectorMaximum(key, {}, 0, active));
+					}
+				}
+				if (maximum >= MaxIndirectDescriptorProbes)
+					maximum = ClampedTableMaximum(key, table_source, indirect);
+				if (maximum < MaxIndirectDescriptorProbes)
+					indirect.key_count = Value(maximum + 1u);
+				else if (!ProgramLoopCount(key, indirect))
+					return false;
 			}
 			if ((table_offset & 3u) != 0u) return false;
 		} else {
 			// Keep the GPU selector and descriptor reads, and preserve the proven scalar
 			// offset domain within the bounded image table.
 			if (sampler || plan.split_offsets) return false;
-			const auto active = [&](const Use& use, bool sampler) -> Value {
-				const auto& access = *use.user;
-				if (use.operand != (sampler ? 1u : 0u) ||
-				    ImageOpcodeInfoOf(access.GetOpcode()).access == ImageAccess::None) return {};
-				if (sampler && (access.GetOpcode() != ValueOpcode::ImageSampleRaw ||
-				    (m_program.memory_info[access.Flags<MemoryFlags>().index].image_sample_flags &
-				     Decoder::ImageSampleFlagAdjust) != 0u)) return {};
-				const auto mask = access.Arg(access.NumArgs() - 1u);
-				return mask.GetType() == Type::U1 ? mask : Value {};
-			};
 			bool shared_selector = false;
 			if (!plan.table_indexed) {
 				for (const auto& owner: m_indirect_descriptors) {
 					const auto& selected = *m_sources[owner.source].indirect_descriptor;
 					if (!selected.selector || !EquivalentValue(m_program, key, owner.key) ||
 					    !std::ranges::all_of(handle.Uses(), [&](const Use& target) {
-						    const auto target_active = active(target, false);
+						    const auto target_active = AccessPredicate(target, false);
 						    return !target_active.IsEmpty() &&
 						        std::ranges::any_of(owner.handle->Uses(), [&](const Use& witness) {
-							        const auto owner_active = active(witness,
+							        const auto owner_active = AccessPredicate(witness,
 							            owner.handle->GetOpcode() == ValueOpcode::GetSamplerResource);
 							        return !owner_active.IsEmpty() && AvailableBefore(*witness.user, *target.user) &&
 							            Implies(target_active, owner_active);
@@ -3430,6 +3640,42 @@ private:
 		return result;
 	}
 
+	// Bounds of an F32 value on the lanes where guard holds; NaN when unknown.
+	std::pair<float, float> FloatRange(Value value, Value guard, uint32_t depth = 0) const {
+		constexpr std::pair unknown {NAN, NAN};
+		value = ActiveValue(value, guard);
+		if (value.IsImmediate() && value.GetType() == Type::F32)
+			return {value.F32Value(), value.F32Value()};
+		const auto* inst = value.TryInstruction();
+		// Each multiply-add evaluates three operands; keep the search shallow.
+		if (depth > 12u || inst == nullptr) return unknown;
+		const auto range = [&](uint32_t arg) {
+			return FloatRange(inst->Arg(arg), guard, depth + 1u);
+		};
+		switch (inst->GetOpcode()) {
+			case ValueOpcode::BitCastF32U32:
+			case ValueOpcode::BitCastU32F32: return range(0);
+			// Clamping maps NaN to zero.
+			case ValueOpcode::FPSaturate32: return {0.0f, 1.0f};
+			case ValueOpcode::FPFloor32: {
+				const auto [low, high] = range(0);
+				return {std::floor(low), std::floor(high)};
+			}
+			case ValueOpcode::FPMad32: {
+				// The rounded multiply and add are monotonic, so the bounds lie at the corners.
+				const auto a = range(0), b = range(1), c = range(2);
+				if (!std::isfinite(a.first) || !std::isfinite(a.second) ||
+				    !std::isfinite(b.first) || !std::isfinite(b.second)) return unknown;
+				const auto [low, high] = std::minmax({a.first * b.first, a.first * b.second,
+				                                      a.second * b.first, a.second * b.second});
+				const std::pair bounds {low + c.first, high + c.second};
+				if (!std::isfinite(bounds.first) || !std::isfinite(bounds.second)) return unknown;
+				return bounds;
+			}
+			default: return unknown;
+		}
+	}
+
 	uint32_t SelectorMaximum(Value value, SelectorBounds bounds = {}, uint32_t depth = 0,
 	                         Value guard = {}) const {
 		if (depth > 32u) return UINT32_MAX;
@@ -3445,9 +3691,13 @@ private:
 		switch (inst->GetOpcode()) {
 			case ValueOpcode::ReadFirstLane:
 				return SelectedSelectorDomain(*inst, guard).maximum;
-			case ValueOpcode::ReadLane:
-				// ReadLane ignores EXEC; only context-free bounds on every source lane apply.
-				return PossibleU32Bits(inst->Arg(0));
+			case ValueOpcode::ReadLane: {
+				// ReadLane ignores EXEC; only context-free bounds on every source lane apply,
+				// unless guard proves a waterfall key equal to the executing lanes' local value.
+				const auto local = EqualLocalKey(guard, value);
+				return local.IsEmpty() ? PossibleU32Bits(inst->Arg(0))
+				                       : SelectorMaximum(local, bounds, depth + 1u, guard);
+			}
 			case ValueOpcode::Phi: {
 				const auto initial = InitialCandidateMask(value);
 				if (!initial.IsEmpty()) return SelectorMaximum(initial, bounds, depth + 1u, guard);
@@ -3474,6 +3724,18 @@ private:
 				               NonzeroOnEntry(inst->Arg(0), inst->Parent()) ? 31u : UINT32_MAX;
 			case ValueOpcode::UMin32:
 			case ValueOpcode::BitwiseAnd32: return std::min(maximum(0), maximum(1));
+			case ValueOpcode::SMin32: {
+				// Signed and unsigned minimums agree on nonnegative operands.
+				const auto left = maximum(0), right = maximum(1);
+				if (std::max(left, right) <= INT32_MAX) return std::min(left, right);
+				break;
+			}
+			case ValueOpcode::ConvertS32F32: {
+				// The conversion truncates toward zero.
+				const auto [low, high] = FloatRange(inst->Arg(0), guard);
+				if (low >= 0.0f && high < 0x1p31f) return static_cast<uint32_t>(high);
+				break;
+			}
 			case ValueOpcode::SelectU32:
 				if (!guard.IsEmpty() && Implies(guard, inst->Arg(0))) return maximum(1);
 				return SelectorPredicateFalse(inst->Arg(0), bounds, depth + 1u, guard)
@@ -3920,7 +4182,7 @@ private:
 
 	void Collect(Inst& inst) {
 		const auto op           = inst.GetOpcode();
-		if (op == ValueOpcode::BvhIntersect) {
+		if (op == ValueOpcode::BvhIntersect || op == ValueOpcode::Bvh8Intersect) {
 			m_info.uses_dma = true;
 			return;
 		}

src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp

@@ -175,6 +175,8 @@ void AddBuiltinInput(ShaderInfo& info, StageInputKind kind) {
 		case StageInputKind::GlobalInvocationId:
 			AddInput(info, kind, 0, 3, "gl_GlobalInvocationID");
 			break;
+		case StageInputKind::BaseVertex: AddInput(info, kind, 0, 1, "gl_BaseVertex"); break;
+		case StageInputKind::BaseInstance: AddInput(info, kind, 0, 1, "gl_BaseInstance"); break;
 		case StageInputKind::PackedAncillary:
 		case StageInputKind::Parameter: break;
 	}
@@ -366,7 +368,6 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 			info.function_lds |= kind == ResourceKind::Lds &&
 			                     program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh;
 			if (shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) {
-				info.subgroup_ballot = true;
 				info.subgroup_shuffle = true;
 				info.subgroup_local_invocation_id = true;
 			}
@@ -445,6 +446,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 				case StageInputKind::InstanceIndex:
 				case StageInputKind::FrontFacing:
 				case StageInputKind::LocalInvocationIndex:
+				case StageInputKind::BaseVertex:
+				case StageInputKind::BaseInstance:
 					if (component != 0u) {
 						return Fail("typed scalar builtin component is out of range");
 					}
@@ -512,12 +515,16 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 		case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break;
 		case ValueOpcode::DataCacheInvalidate: info.coherent_buffers = true; break;
 		case ValueOpcode::BvhIntersect: info.bvh = true; break;
+		case ValueOpcode::Bvh8Intersect: info.bvh8 = true; break;
+		case ValueOpcode::BvhStack: info.bvh_stack = true; break;
 		case ValueOpcode::ConditionRef:
 			info.subgroup_vote |=
 			    inst.Flags<CFG::BranchCondition>() != CFG::BranchCondition::ScalarInstruction;
 			break;
 		case ValueOpcode::Ballot: info.subgroup_ballot = true; break;
-		case ValueOpcode::SubgroupUMinU32: info.subgroup_arithmetic = true; break;
+		case ValueOpcode::LiveMask: info.subgroup_ballot = info.helper_invocation = true; break;
+		case ValueOpcode::SubgroupUMinU32:
+		case ValueOpcode::SubgroupUMaxU32: info.subgroup_arithmetic = true; break;
 		case ValueOpcode::DppMoveU32:
 		case ValueOpcode::ReadFirstLane:
 		case ValueOpcode::ReadLane: {
@@ -576,7 +583,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
 	info.has_bitwise_xor = info.uses_lds = info.uses_gds = info.uses_flattened_srt =
 	    info.uses_dispatch_threads = info.uses_swizzle = false;
 	info.bvh = info.subgroup_ballot = info.subgroup_vote = info.subgroup_barrier =
-	    info.subgroup_shuffle = info.subgroup_arithmetic =
+	    info.subgroup_shuffle = info.subgroup_arithmetic = info.helper_invocation =
 	    info.subgroup_local_invocation_id = info.compute_derivatives = info.image_gather_extended =
 	    info.function_lds = info.function_scratch = info.pixel_valid_mask = info.buffer_int64_atomics =
 	    info.buffer_u8 = info.buffer_u16 = info.shared_int64_atomics = info.coherent_buffers = info.float64 = false;

src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp

@@ -82,6 +82,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::BitCastU32F32:
 		case ValueOpcode::BitCastF32U32:
 		case ValueOpcode::ConvertU32F32:
+		case ValueOpcode::ConvertS32F32:
 		case ValueOpcode::ConvertF32U32:
 		case ValueOpcode::CompositeConstructU64:
 		case ValueOpcode::CompositeExtractU64:
@@ -147,8 +148,8 @@ bool ImmediateBits(Value value, uint64_t& bits) {
 }
 
 // Scratch layout in bytes: per descriptor source its eight DWORDs and a known DWORD, one
-// activity byte per source, then one qword per walked value (all ones when unknown). The
-// walker's value slots, captured addresses, reachability and states follow.
+// activity byte per source, then per image, then one qword per walked value (all ones when
+// unknown). The walker's value slots, captured addresses, reachability and states follow.
 constexpr uint32_t DescriptorStride = 9u * sizeof(uint32_t);
 
 uint32_t ActiveOffset(const ResourcePlan& plan) {
@@ -156,7 +157,8 @@ uint32_t ActiveOffset(const ResourcePlan& plan) {
 }
 
 uint32_t OutputOffset(const ResourcePlan& plan) {
-	return (ActiveOffset(plan) + static_cast<uint32_t>(plan.descriptor_sources.size()) + 7u) & ~7u;
+	const auto active = plan.descriptor_sources.size() + plan.info.images.size();
+	return (ActiveOffset(plan) + static_cast<uint32_t>(active) + 7u) & ~7u;
 }
 
 const uint8_t* Scratch(const ResourcePlan& plan) {
@@ -1038,6 +1040,23 @@ void WalkerCompiler::EmitStrict(const Node& node) {
 			c.L(done);
 			break;
 		}
+		case ValueOpcode::ConvertS32F32: {
+			// The conversion saturates and maps NaN to zero. cvttss2si's out-of-range result,
+			// INT32_MIN, is already right for negative overflow.
+			Xbyak::Label done;
+			c.movd(c.xmm0, c.eax);
+			c.xor_(c.eax, c.eax);
+			c.ucomiss(c.xmm0, c.xmm0);
+			c.jp(done, Near);
+			c.mov(c.ecx, 0x4f000000u);
+			c.movd(c.xmm1, c.ecx);
+			c.mov(c.eax, INT32_MAX);
+			c.ucomiss(c.xmm0, c.xmm1);
+			c.jae(done, Near);
+			c.cvttss2si(c.eax, c.xmm0);
+			c.L(done);
+			break;
+		}
 		case ValueOpcode::FPMul32:
 			c.movd(c.xmm0, c.eax);
 			c.movd(c.xmm1, c.ecx);
@@ -1272,11 +1291,14 @@ void WalkerCompiler::Compile(std::span<const Value> values) {
 			for (const auto slot: blocks[block].srt_reads) flat(slot);
 			c.L(skip);
 		}
-		// A source no block lists is always active.
-		for (uint32_t source = 0; source < sources.size(); ++source) {
+		// A source or image no block lists is always active.
+		for (uint32_t index = 0; index < sources.size() + m_plan.info.images.size(); ++index) {
+			const bool     image = index >= sources.size();
+			const uint32_t id    = image ? index - static_cast<uint32_t>(sources.size()) : index;
 			bool listed = false;
 			for (uint32_t block = 0; block < blocks.size(); ++block) {
-				if (std::ranges::find(blocks[block].sources, source) == blocks[block].sources.end()) continue;
+				const auto& list = image ? blocks[block].images : blocks[block].sources;
+				if (std::ranges::find(list, id) == list.end()) continue;
 				if (listed) {
 					c.or_(c.al, c.byte[c.rdx + m_reach + block]);
 				} else {
@@ -1284,7 +1306,7 @@ void WalkerCompiler::Compile(std::span<const Value> values) {
 				}
 				listed = true;
 			}
-			const auto active = c.byte[c.rdx + ActiveOffset(m_plan) + source];
+			const auto active = c.byte[c.rdx + ActiveOffset(m_plan) + index];
 			if (listed) {
 				c.mov(active, c.al);
 			} else {
@@ -1391,6 +1413,10 @@ bool WalkedActive(const ResourcePlan& plan, uint32_t source) {
 	return Scratch(plan)[ActiveOffset(plan) + source] != 0u;
 }
 
+bool WalkedImageActive(const ResourcePlan& plan, uint32_t image) {
+	return Scratch(plan)[ActiveOffset(plan) + plan.descriptor_sources.size() + image] != 0u;
+}
+
 bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValueType type) {
 	return RuntimeValidator(program, type).Run(value);
 }

src/graphics/shader/recompiler/ir/passes/SrtWalker.h

@@ -78,6 +78,7 @@ bool WalkSrt(const ResourcePlan& plan, const SrtRuntime& runtime, std::vector<ui
 bool WalkedValue(const ResourcePlan& plan, Value value, uint32_t& result);
 bool WalkedDescriptor(const ResourcePlan& plan, uint32_t source, DescriptorValue& result);
 bool WalkedActive(const ResourcePlan& plan, uint32_t source);
+bool WalkedImageActive(const ResourcePlan& plan, uint32_t image);
 // Length of the guest load at rip if it belongs to the walker running on this thread, else 0.
 // The exception handler completes such a load as a zero read.
 uint32_t SrtWalkerLoadLength(uint64_t rip);

src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp

@@ -348,14 +348,20 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
 } // namespace
 
 void RewriteToSsa(const BlockList& blocks) {
-	Pass pass;
+	Pass                               pass;
+	std::unordered_map<Block*, size_t> filled_predecessors;
 	for (auto* block: blocks) {
+		if (block->ImmPredecessors().empty()) {
+			pass.Seal(block);
+		}
 		for (auto& inst: *block) {
 			VisitInstruction(pass, block, inst);
 		}
-	}
-	for (auto* block: blocks) {
-		pass.Seal(block);
+		for (auto* successor: block->ImmSuccessors()) {
+			if (++filled_predecessors[successor] == successor->ImmPredecessors().size()) {
+				pass.Seal(successor);
+			}
+		}
 	}
 }
 

src/graphics/shader/shader.cpp

@@ -291,10 +291,9 @@ static bool ShaderGetStaticVertexInputInfo(uint64_t shader_addr, const HW::UserS
 	const auto& metadata = data.vertex;
 
 	if (metadata.vertex_attrib_reg >= 0) {
-		info.fetch_external   = false;
-		info.fetch_embedded   = true;
-		info.fetch_attrib_reg = metadata.vertex_attrib_reg;
-		info.fetch_buffer_reg = metadata.vertex_buffer_reg;
+		info.has_dynamic_vertex_fetch = true;
+		info.fetch_attrib_reg         = metadata.vertex_attrib_reg;
+		info.fetch_buffer_reg         = metadata.vertex_buffer_reg;
 
 		const auto* attrib = reinterpret_cast<const uint32_t*>(
 		    static_cast<uint64_t>(user_sgpr.value[metadata.vertex_attrib_reg]) |
@@ -413,7 +412,7 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
 void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t>& key) {
 	EXIT_IF(info.resources_num < 0 || info.resources_num > ShaderVertexInputInfo::RES_MAX);
 	key.clear();
-	key.push_back(static_cast<uint32_t>(info.fetch_embedded));
+	key.push_back(static_cast<uint32_t>(info.has_dynamic_vertex_fetch));
 	key.push_back(static_cast<uint32_t>(info.fetch_attrib_reg));
 	key.push_back(static_cast<uint32_t>(info.fetch_buffer_reg));
 	key.push_back(info.resources_num);
@@ -422,6 +421,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 	key.push_back(info.lds_size_dwords);
 	key.push_back(info.pa_cl_vs_out_cntl);
 	key.push_back(info.user_clip_planes);
+	key.push_back(info.indirect_offsets.vertex | (uint32_t {info.indirect_offsets.instance} << 8u) |
+	              (uint32_t {info.indirect_draw} << 16u));
 	key.push_back(static_cast<uint32_t>(info.clip_space.enabled));
 	if (info.clip_space.enabled) {
 		for (const float value: info.clip_space.scale) {
@@ -490,7 +491,9 @@ void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector<uint32_t>
 	key.push_back(static_cast<uint32_t>(info.object_primitive_id));
 	key.push_back(static_cast<uint32_t>(info.alpha_blend_source));
 	key.insert(key.end(), std::begin(info.target_output_mode), std::end(info.target_output_mode));
-	key.push_back(info.target_packed_unorm_mask | (uint32_t {info.target_round_mode} << 8u));
+	key.push_back(info.target_packed_unorm_mask | (uint32_t {info.target_round_mode} << 8u) |
+	              (uint32_t {info.target_uint_mask} << 16u) |
+	              (uint32_t {info.target_sint_mask} << 24u));
 	const auto mask = info.target_shader_mask;
 	key.push_back((mask | (mask >> 1u) | (mask >> 2u) | (mask >> 3u)) & 0x11111111u);
 	for (uint32_t base = 0; base < info.target_export_mapping.size(); base += 4u) {
@@ -690,9 +693,7 @@ void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo& info) {
 
 	LOGF("ShaderDbgDumpInputInfo()\n");
 
-	LOGF("\t fetch_external = %s\n"
-	     "\t fetch_embedded = %s\n",
-	     info.fetch_external ? "true" : "false", info.fetch_embedded ? "true" : "false");
+	LOGF("\t has_dynamic_vertex_fetch = %s\n", info.has_dynamic_vertex_fetch ? "true" : "false");
 
 	for (int i = 0; i < info.resources_num; i++) {
 		LOGF("\t input %d\n", i);

src/graphics/shader/shader.h

@@ -124,6 +124,17 @@ struct ShaderTessellationInputInfo {
 	uint32_t output_topology       = 0;
 };
 
+// GS user-data slots an indirect draw packet writes its record's base vertex and first instance
+// to. A packet can leave both slots unused even when its record contains nonzero offsets.
+struct ShaderIndirectOffsets {
+	static constexpr uint8_t None = UINT8_MAX;
+
+	uint8_t vertex   = None;
+	uint8_t instance = None;
+
+	bool operator==(const ShaderIndirectOffsets&) const = default;
+};
+
 struct ShaderVertexInputInfo {
 	static constexpr int RES_MAX = 32;
 
@@ -146,8 +157,11 @@ struct ShaderVertexInputInfo {
 	ShaderClipSpaceTransform clip_space;
 	ShaderMeshInputInfo      mesh;
 	ShaderTessellationInputInfo tess;
-	bool                    fetch_external      = false;
-	bool                    fetch_embedded      = false;
+	ShaderIndirectOffsets   indirect_offsets;
+	// GPU-read records add Vulkan bases even when the packet patches no user SGPRs.
+	bool                    indirect_draw       = false;
+	// Attribute-table metadata enables dynamic fetch detection; loads are matched later.
+	bool                    has_dynamic_vertex_fetch = false;
 };
 
 struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo {
@@ -181,6 +195,9 @@ struct ShaderPixelInputInfo {
 	uint32_t                                       target_shader_mask           = UINT32_MAX;
 	std::array<Prospero::ColorComponentMapping, 8> target_export_mapping        = {};
 	uint8_t                                        target_packed_unorm_mask      = 0;
+	// Exports to UINT and SINT targets, which take integer outputs of the same signedness.
+	uint8_t                                        target_uint_mask             = 0;
+	uint8_t                                        target_sint_mask             = 0;
 	// CB round mode per converted export: zero adds half an LSB, one truncates.
 	uint8_t                                        target_round_mode            = 0;
 	uint32_t                                       scratch_size_dwords          = 0;

src/kernel/memory.h

@@ -26,9 +26,10 @@ struct DebugStats {
 	uint64_t flexible_total      = 0;
 	uint64_t flexible_used       = 0;
 	uint64_t pool_committed      = 0;
-	uint64_t cpu_page_entries    = 0; // 2 MiB page-table entries in use.
-	uint64_t gpu_page_entries    = 0;
-	uint64_t page_entries_total  = 0; // Per side.
+	uint64_t cpu_page_entries       = 0; // 2 MiB page-table entries in use.
+	uint64_t gpu_page_entries       = 0;
+	uint64_t cpu_page_entries_total = 0;
+	uint64_t gpu_page_entries_total = 0;
 };
 [[nodiscard]] DebugStats GetDebugStats();
 
@@ -125,7 +126,6 @@ static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
               "KernelMemoryPoolBlockStats struct size is incorrect");
 
 void                   RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
-void                   SetFlexibleMemorySize(uint64_t size);
 int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size, size_t alignment,
                          int memory_type, int64_t* phys_addr_out, bool automatic = false);
 int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
@@ -197,6 +197,9 @@ int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* outp
 
 uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
                                Common::VirtualMemory::Mode mode, const char* name);
+// Loaded modules' segments use flexible memory without taking its backing.
+void ChargeFlexibleMemory(uint64_t size);
+void ReleaseFlexibleMemory(uint64_t size);
 void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
 uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
                                Common::VirtualMemory::Mode mode, const char* name,
@@ -222,6 +225,11 @@ bool     TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
 bool     TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
 bool     TestGuestBackingOutsideAddressSpace();
 uint64_t TestGuestBackingSize();
+uint64_t TestFlexibleBackingHighWater();
+uint64_t TestGuestFreeRangeVisits();
+uint64_t TestDirectMappingCount();
+uint64_t TestFlexibleMappingCount();
+uint64_t TestBackingBytesZeroed();
 bool     TestGuestFreeRangeBounds();
 #endif