ngs2: size 16-bit PCM blocks in Ngs2CalcWaveformBlock (The Binding of Isaac: Repentance) (#1968 )

Ngs2CalcWaveformBlock returned NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT with a zeroed block for 16-bit PCM (0x12). The Binding of Isaac: Repentance (PPSA03311) ignores the error and queues the zeroed blocks, which the sampler skips, so the game had no audio at all. PCM frames have a fixed size, so the block now starts at sample_pos frames and spans num_samples frames. Fixes #667 Co-authored-by: gl-ungus <notasecondemail63@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
docs: update the README status, Cyberpunk 2077 screenshot and notable forks
2026-10-30 09:08:19 +00:00 · 2026-10-30 05:39:49 +01:00 · 2026-10-30 05:17:52 +01:00 · 2026-10-30 05:03:22 +01:00 · 2026-10-30 04:31:24 +01:00 · 2026-10-30 04:19:05 +01:00
155 changed files with 11319 additions and 3356 deletions
@@ -113,7 +113,7 @@ jobs:
        shell: cmd
        run: |
          cmake --build _Build/windows --target launcher ^
-            audio_out2_port_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
+            audio_out2_port_tests ngs2_sampler_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
            shader_recompiler_compute_tests ^
            resource_tracking_tests scalar_provenance_tests ^
            resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests --parallel
@@ -121,7 +121,7 @@ jobs:
      - name: Test
        shell: cmd
        run: |
-          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
+          ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|ngs2_sampler|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"

      - name: Install
        shell: cmd
@@ -202,7 +202,7 @@ jobs:
        shell: bash
        run: |
          cmake --build _Build/macos \
-            --target launcher audio_out2_port_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
+            --target launcher audio_out2_port_tests ngs2_sampler_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
              memory_tracker_tests page_manager_tests shader_recompiler_compute_tests \
              resource_tracking_tests scalar_provenance_tests \
              resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
@@ -212,7 +212,7 @@ jobs:
        shell: bash
        run: |
          ctest --test-dir _Build/macos --output-on-failure \
-            -R '^(audio_out2_port|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|ngs2_sampler|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'

      - name: Install
        shell: bash
@@ -388,7 +388,7 @@ jobs:
        run: |
          cmake --build _Build/linux \
            --target launcher page_manager_tests memory_tracker_tests \
-              audio_out2_port_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
+              audio_out2_port_tests ngs2_sampler_tests pad_haptics_tests controller_settings_tests host_input_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
              resource_tracking_tests scalar_provenance_tests \
              resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
            --parallel
@@ -397,7 +397,7 @@ jobs:
        shell: bash
        run: |
          ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(audio_out2_port|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
+            -R '^(audio_out2_port|ngs2_sampler|pad_haptics|controller_settings|host_input|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'

      - name: Install
        shell: bash
@@ -42,6 +42,28 @@ if(NOT KYTY_CLANG_CL)
 endif()
 install(FILES "${openxr_SOURCE_DIR}/LICENSE" DESTINATION "licenses/openxr")

+# Noto Sans v2.015 (SIL Open Font License 1.1) is libFont's fallback font.
+set(KYTY_NOTO_SANS_REPOSITORY "https://raw.githubusercontent.com/notofonts/notofonts.github.io")
+set(KYTY_NOTO_SANS_COMMIT 42af67ccd12bc0e31ed85384bf73e39248265374)
+set(KYTY_NOTO_SANS_BASE_URL "${KYTY_NOTO_SANS_REPOSITORY}/${KYTY_NOTO_SANS_COMMIT}/fonts")
+set(KYTY_NOTO_SANS_DIR "${FETCHCONTENT_BASE_DIR}/noto-sans")
+function(kyty_fetch_noto_sans path sha256)
+	get_filename_component(name "${path}" NAME)
+	file(DOWNLOAD "${KYTY_NOTO_SANS_BASE_URL}/${path}" "${KYTY_NOTO_SANS_DIR}/${name}"
+		EXPECTED_HASH SHA256=${sha256})
+endfunction()
+kyty_fetch_noto_sans(LICENSE f2095b08bed08b23a6fe26112fcd679a2bee3f002eef077eb05d215ed1051bd8)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-Regular.ttf
+	478c558ea716033cd60c03438f628dfa75694dcf6b5f6d505a2f05fd2b4f3823)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-Bold.ttf
+	1df075a380fc7cb898acf64c1f7b3b4dd780de3caa860178bf929de35817a913)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-Italic.ttf
+	467e3f89eeca4108bb8710a2b9e0cf2281ac56d5b0609211a83776d0505eecb5)
+kyty_fetch_noto_sans(NotoSans/hinted/ttf/NotoSans-BoldItalic.ttf
+	1b602a9d6353be42c91df097a4857b69fa2696f26703d7a33b54a15d87c2622c)
+install(FILES "${KYTY_NOTO_SANS_DIR}/LICENSE" DESTINATION "licenses/noto-sans")
+set(KYTY_NOTO_SANS_DIR "${KYTY_NOTO_SANS_DIR}" PARENT_SCOPE)
+
 # The guest instruction patcher needs a decoder/encoder and a small
 # trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
@@ -325,6 +325,7 @@ add_gpu_blit_shader(gpu_blit_fs_triangle gpu_blit_fs_triangle.vert)
 add_gpu_blit_shader(gpu_blit_color_to_ms_depth gpu_blit_color_to_ms_depth.frag)
 add_gpu_blit_shader(gpu_video_out_overlay gpu_video_out_overlay.frag)
 add_gpu_blit_shader(gpu_imgui_hdr gpu_video_out_overlay.frag -DIMGUI_HDR=1)
+add_gpu_blit_shader(gpu_pssr_linear gpu_pssr_linear.comp)
 list(APPEND kyty_emulator_src ${gpu_blit_shader_headers})

 list(APPEND kyty_emulator_src
@@ -407,6 +408,8 @@ add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
 target_link_libraries(scalar_provenance_tests fmt::fmt xbyak::xbyak)
 target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})

+add_executable(build_flags_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/BuildFlagsTests.cpp")
+
 add_executable(page_manager_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/PageManagerTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/pageManager.cpp"
@@ -520,6 +523,13 @@ add_executable(ngs2_sampler_tests EXCLUDE_FROM_ALL
 target_link_libraries(ngs2_sampler_tests common fmt::fmt LibAtrac9)
 target_include_directories(ngs2_sampler_tests PRIVATE ${inc_headers})

+add_executable(ajm_resample_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/AjmResampleTests.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp"
+)
+target_link_libraries(ajm_resample_tests common fmt::fmt FFmpeg::ffmpeg LibAtrac9 opus)
+target_include_directories(ajm_resample_tests PRIVATE ${inc_headers})
+
 add_executable(pad_haptics_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/PadHapticsTests.cpp"
 )
@@ -622,6 +632,19 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 	"${gpu_test_ms_depth_header}")

 add_kyty_full_emulator_test(virtual_memory_allocation_tests "${KYTY_TESTS_DIR}/VirtualMemoryAllocationTests.cpp")
+add_library(kyty_test_gtest STATIC EXCLUDE_FROM_ALL
+	"${KYTY_THIRD_PARTY_DIR}/fmt/test/gtest/gmock-gtest-all.cc")
+target_include_directories(kyty_test_gtest PUBLIC "${KYTY_THIRD_PARTY_DIR}/fmt/test/gtest")
+target_sources(virtual_memory_allocation_tests PRIVATE "${KYTY_TESTS_DIR}/TrinityTests.cpp")
+target_link_libraries(virtual_memory_allocation_tests kyty_test_gtest)
+
+add_executable(psml_tests EXCLUDE_FROM_ALL
+	"${KYTY_TESTS_DIR}/PsmlTests.cpp"
+	"${KYTY_SOURCE_DIR}/libs/libPsml.cpp"
+	"${KYTY_SOURCE_DIR}/loader/symbolDatabase.cpp"
+	"${KYTY_SOURCE_DIR}/loader/timer.cpp")
+target_link_libraries(psml_tests common kyty_test_gtest)
+target_include_directories(psml_tests PRIVATE ${inc_headers})
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)

@@ -650,6 +673,7 @@ if(BUILD_TESTING)
 	add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
 	add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
 	add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
+	add_test(NAME build_flags COMMAND $<TARGET_FILE:build_flags_tests>)
 	add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
 	add_test(NAME trophy_system COMMAND $<TARGET_FILE:trophy_system_tests>)
 	add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
@@ -665,6 +689,7 @@ if(BUILD_TESTING)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME audio_out_timing COMMAND $<TARGET_FILE:audio_out_timing_tests>)
 	add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
+	add_test(NAME ajm_resample COMMAND $<TARGET_FILE:ajm_resample_tests>)
 	add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
 	add_test(NAME controller_settings COMMAND $<TARGET_FILE:controller_settings_tests>)
 	add_test(NAME host_input COMMAND $<TARGET_FILE:host_input_tests>)
@@ -681,6 +706,16 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --tessellation-only)
 	add_test(NAME shader_recompiler_float_min_max
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --float-min-max-only)
+	add_test(NAME shader_recompiler_vop3_conversions
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --vop3-conversions-only)
+	add_test(NAME shader_recompiler_quadmask
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --quadmask-only)
+	add_test(NAME shader_recompiler_integer16_ternary
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --integer16-ternary-only)
+	add_test(NAME shader_recompiler_mad_u16
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --mad-u16-only)
+	add_test(NAME shader_recompiler_mul_hi24
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --mul-hi24-only)
 	add_test(NAME shader_recompiler_alignbyte
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
 	add_test(NAME graphics_pipeline_rasterization
@@ -689,6 +724,10 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --draw-offset-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
+	add_test(NAME trinity_mode
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests> --trinity-only)
+	add_test(NAME psml COMMAND $<TARGET_FILE:psml_tests>)
+	add_test(NAME pssr COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --pssr-only)
 	add_test(NAME guest_red_zone_patcher
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests> --red-zone-patcher-only)
 	add_test(NAME command_scheduler_timeline
@@ -697,8 +736,6 @@ if(BUILD_TESTING)
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --stream-buffer-only)
 	add_test(NAME gpu_command_lane
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
-	add_test(NAME wait_until_safe_for_rendering
-		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --wait-until-safe-for-rendering-only)
 	add_test(NAME pm4_flip
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --flip-only)
 	add_test(NAME depth_bits
@@ -743,6 +780,7 @@ if(BUILD_TESTING)
 		image_page_table_tests
 		memory_tracker_tests
 		page_manager_tests
+		build_flags_tests
 		archive_file_tests
 		trophy_system_tests
 		avplayer_file_tests
@@ -760,6 +798,7 @@ if(BUILD_TESTING)
 		audio_out2_port_tests
 		audio_out_timing_tests
 		ngs2_sampler_tests
+		ajm_resample_tests
 		pad_haptics_tests
 		controller_settings_tests
 		host_input_tests
@@ -811,6 +850,19 @@ add_custom_target(kyty_touch_cursors
 add_dependencies(kyty_emulator kyty_touch_cursors)
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/assets/cursors" DESTINATION assets)

+set(KYTY_NOTO_SANS_FONTS
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-Regular.ttf"
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-Bold.ttf"
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-Italic.ttf"
+	"${KYTY_NOTO_SANS_DIR}/NotoSans-BoldItalic.ttf")
+add_custom_target(kyty_noto_sans_fonts
+	COMMAND ${CMAKE_COMMAND} -E make_directory "$<TARGET_FILE_DIR:kyty_emulator>/assets/fonts"
+	COMMAND ${CMAKE_COMMAND} -E copy_if_different ${KYTY_NOTO_SANS_FONTS}
+		"$<TARGET_FILE_DIR:kyty_emulator>/assets/fonts"
+	VERBATIM)
+add_dependencies(kyty_emulator kyty_noto_sans_fonts)
+install(FILES ${KYTY_NOTO_SANS_FONTS} DESTINATION assets/fonts)
+
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)
 	target_link_libraries(kyty_emulator opengl32)
@@ -23,11 +23,8 @@ can change significantly between builds.

 ## Current Status

-KytyPS5 can boot 2D and 3D games, including titles built with Unreal Engine
-4/5, Unity, and custom engines. External low-level emulation modules are neither required nor
-planned.
-
 Development is currently focused on expanding game compatibility and improving boot reliability.
+External low-level emulation modules are neither required nor planned.

 Windows and Linux are the primary target platforms and receive the most testing.

@@ -90,6 +87,13 @@ or graphical glitches, so please include the version you tested when reporting a

 <p align="center"><em>And many more...</em></p>

+## Notable Forks
+
+These community forks bring KytyPS5 to other platforms.
+
+- [Cosmo/KytyPS5](https://github.com/Cosmo/KytyPS5) — Xbox Series X: Direct3D 12 renderer, sleek
+  controller-first interface, and local multiplayer for up to four players.
+
 ## Contributing

 Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
@@ -1,15 +1,18 @@
 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><defs>
-<linearGradient id="body" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="#ffffff"/><stop offset="1" stop-color="#e3e9f2"/></linearGradient>
+<linearGradient id="tile" x2="0" y2="1"><stop stop-color="#3d7fe6"/><stop offset="1" stop-color="#1c47b0"/></linearGradient>
+<linearGradient id="body" x2="0" y2="1"><stop stop-color="#fff"/><stop offset="1" stop-color="#e3e9f2"/></linearGradient>
+<linearGradient id="gloss" x2="0" y2="1"><stop stop-color="#fff" stop-opacity="0.28"/><stop offset="1" stop-color="#fff" stop-opacity="0"/></linearGradient>
 </defs>
-<g transform="translate(256 268) scale(1.16) translate(-256 -268)" stroke-linejoin="round" stroke-linecap="round">
-<path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z" fill="url(#body)"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z" fill="url(#body)"/>
+<rect x="16" y="16" width="480" height="480" rx="112" fill="url(#tile)"/>
+<path d="M16 128 A112 112 0 0 1 128 16 L384 16 A112 112 0 0 1 496 128 L496 200 Q256 150 16 236 Z" fill="url(#gloss)"/>
+<g transform="translate(256 268) scale(0.86) translate(-256 -268)" stroke-linejoin="round" stroke-linecap="round">
+<g transform="translate(0 18)" fill-opacity="0.22" stroke="#000" stroke-opacity="0.22" stroke-width="16"><path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z"/><path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z"/></g>
+<g fill="url(#body)" stroke="#0b0d12" stroke-width="16"><path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z"/></g>
 <path d="M128 164 L137 128 L174 156 Z" fill="#0b0d12"/><path d="M384 164 L375 128 L338 156 Z" fill="#0b0d12"/>
-<g fill="none" stroke="#0b0d12" stroke-width="16"><path d="M100 196 L120 104 Q124 92 134 99 L214 160 Z"/><path d="M412 196 L392 104 Q388 92 378 99 L298 160 Z"/></g>
-<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="url(#body)"/>
-<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="none" stroke="#0b0d12" stroke-width="16"/>
-<g fill="none" stroke="#0b0d12" stroke-width="9"><path d="M82 260 C100 274 118 288 128 304 C134 316 121 350 108 388"/><path d="M430 260 C412 274 394 288 384 304 C378 316 391 350 404 388"/></g>
-<path d="M64 322 L82 340 M62 340 L76 354 M448 322 L430 340 M450 340 L436 354" fill="none" stroke="#0b0d12" stroke-width="7"/>
-<path d="M168 230 Q194 202 220 230" fill="none" stroke="#0b0d12" stroke-width="15"/>
+<path d="M256 154 C330 154 380 160 406 178 C432 196 447 244 459 302 C471 358 466 398 430 408 C400 416 380 398 360 368 L326 318 C316 304 306 300 290 300 L222 300 C206 300 196 304 186 318 L152 368 C132 398 112 416 82 408 C46 398 41 358 53 302 C65 244 80 196 106 178 C132 160 182 154 256 154 Z" fill="url(#body)" stroke="#0b0d12" stroke-width="16"/>
+<g fill="none" stroke="#0b0d12"><g stroke-width="9"><path d="M82 260 C100 274 118 288 128 304 C134 316 121 350 108 388"/><path d="M430 260 C412 274 394 288 384 304 C378 316 391 350 404 388"/></g>
+<path d="M64 322 L82 340 M62 340 L76 354 M448 322 L430 340 M450 340 L436 354" stroke-width="7"/>
+<path d="M168 230 Q194 202 220 230" stroke-width="15"/></g>
 <circle cx="322" cy="220" r="22" fill="#0b0d12"/>
 <path d="M238 252 L274 252 L256 276 Z" fill="#0b0d12" stroke="#0b0d12" stroke-width="8"/>
 </g></svg>
@@ -105,10 +105,18 @@ bool HideCursorEnabled() {
 	return g_config->hide_cursor_enabled;
 }

+bool StretchEnabled() {
+	return g_config->stretch_enabled;
+}
+
 bool VrEnabled() {
 	return g_config->vr_enabled;
 }

+bool Ps5ProEnabled() {
+	return g_config->pro_enabled;
+}
+
 bool AmdCpuEnabled() {
 	return g_config->amd_cpu_enabled;
 }
@@ -129,6 +137,10 @@ bool IsNetworkConnected() {
 	return g_config->network_connected;
 }

+bool PsnEmulationEnabled() {
+	return g_config->psn_emulation_enabled;
+}
+
 bool ShaderValidationEnabled() {
 	return g_config->shader_validation_enabled;
 }
@@ -64,10 +64,13 @@ struct ConfigOptions {
 	bool                   fullscreen_enabled          = false;
 	bool                   hdr_enabled                 = false;
 	bool                   hide_cursor_enabled         = false;
+	bool                   stretch_enabled             = false;
 	bool                   splash_enabled              = true;
 	bool                   vr_enabled                  = false;
+	bool                   pro_enabled                 = false;
 	bool                   amd_cpu_enabled             = false;
 	bool                   network_connected           = false;
+	bool                   psn_emulation_enabled       = false;
 	uint32_t               vblank_frequency            = 60;
 	uint32_t               console_language            = DEFAULT_CONSOLE_LANGUAGE;
 	bool                   vulkan_validation_enabled   = false;
@@ -125,9 +128,12 @@ int32_t GetGpuIndex();
 bool     FullscreenEnabled();
 bool     HdrEnabled();
 bool     HideCursorEnabled();
+bool     StretchEnabled();
 bool     VrEnabled();
+bool     Ps5ProEnabled();
 bool     AmdCpuEnabled();
 bool     IsNetworkConnected();
+bool     PsnEmulationEnabled();
 uint32_t GetVblankFrequency();
 uint32_t GetConsoleLanguage();
 bool     VulkanValidationEnabled();
@@ -99,10 +99,6 @@ static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path&

 	if (Common::File::IsFileExisting(param_json)) {
 		Loader::SystemContentLoadParamSfo(param_json);
-		if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
-		    flexible_memory_size != 0) {
-			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
-		}
 	}
 	if (cfg.splash_enabled) {
 		Libs::Graphics::InitializeSplashScreen(param_json.parent_path());
@@ -3,6 +3,7 @@

 #include "common/assert.h"
 #include "graphics/guest_gpu/hardwareContext.h"
+#include "graphics/guest_gpu/pssr.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"

@@ -92,12 +93,14 @@ public:
 	                    uint32_t mode);
 	void DispatchIndirect(uint64_t args_addr, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
+	void ProcessPssr(const Pssr::Request& request);
 	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);

 	void WaitForRewind(bool valid);

 	template <typename T>
 	void WaitRegMem(uint32_t func, const T* addr, T ref, T mask, uint32_t poll, uint32_t wait_op);
+	void MemSemaphore(uint64_t* counter, bool signal, bool write);
 	void WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num, uint32_t write_control);
 	void WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes);
 	void DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cache_policy,
@@ -202,6 +202,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_DMA_DATA]                  = CpOpDmaData;
 	g_cp_op_func[Pm4::IT_ACQUIRE_MEM]               = CpOpAcquireMem;
 	g_cp_op_func[Pm4::IT_RELEASE_MEM]               = CpOpReleaseMem;
+	g_cp_op_func[Pm4::IT_MEM_SEMAPHORE]             = CpOpMemSemaphore;
 	g_cp_op_func[Pm4::IT_REWIND]                    = CpOpRewind;
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG]           = CpOpSetContextReg;
 	g_cp_op_func[Pm4::IT_SET_SH_REG]                = CpOpSetShaderReg;
@@ -225,6 +226,7 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;
+	g_cp_op_custom_func[Pm4::R_PSSR]           = CpOpPssr;

 	return g_cp_op_custom_func;
 }
@@ -136,6 +136,7 @@ uint32_t CpOpEventWrite(CommandProcessor&, uint32_t, const uint32_t*, uint32_t,
 uint32_t CpOpDmaData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpAcquireMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpReleaseMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpMemSemaphore(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetContextReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetShaderReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetUconfigReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
@@ -150,6 +151,7 @@ uint32_t CpOpPushMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t,
 uint32_t CpOpPopMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpWaitRegMem64(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpFlip(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpPssr(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);

 } // namespace Libs::Graphics

@@ -1706,6 +1706,17 @@ KYTY_CP_OP_PARSER(CpOpFlip) {
 	return 5;
 }

+KYTY_CP_OP_PARSER(CpOpPssr) {
+	KYTY_PROFILER_FUNCTION();
+
+	const auto packet_dw = KYTY_PM4_LEN(cmd_id) - 1;
+	EXIT_NOT_IMPLEMENTED(packet_dw < Pssr::RequestDwords || packet_dw > dw);
+	Pssr::Request request {};
+	std::memcpy(&request, buffer, sizeof(request));
+	cp.ProcessPssr(request);
+	return packet_dw;
+}
+
 KYTY_CP_OP_PARSER(CpOpIndexType) {
 	KYTY_PROFILER_FUNCTION();

@@ -2060,6 +2071,21 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 	return 7;
 }

+KYTY_CP_OP_PARSER(CpOpMemSemaphore) {
+	KYTY_PROFILER_FUNCTION();
+
+	const auto control = buffer[2];
+	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0023900 || (control & ~0xe0110000u) != 0 ||
+	                     (control >> 29u) < 6u);
+
+	auto* counter = reinterpret_cast<uint64_t*>(
+	    (buffer[0] & ~0x7u) | (static_cast<uint64_t>(buffer[1] & 0x3ffffu) << 32u));
+	// A signal updates the counter before the next packet, so its mailbox stall never waits.
+	cp.MemSemaphore(counter, (control >> 29u) == 6u, ((control >> 20u) & 0x1u) != 0);
+
+	return 3;
+}
+
 KYTY_CP_OP_PARSER(CpOpSetContextReg) {
 	KYTY_PROFILER_FUNCTION();

@@ -46,6 +46,7 @@ constexpr FormatInfo kFormatInfo[] = {
 	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
 	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
 	{BufferFormat::k10_11_11Float, 4, 0, 4, true, false},
+	{BufferFormat::k2_10_10_10UNorm, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
 	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
 	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
@@ -30,7 +30,6 @@
 #include <mutex>
 #include <semaphore>
 #include <thread>
-#include <utility>
 #include <vector>

 namespace Libs::Graphics {
@@ -92,12 +91,6 @@ void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
 	m_work_available.Signal();
 }

-void GuestGpu::Wake() {
-	Common::LockGuard lock(m_queue_mutex);
-	m_wake_pending = true;
-	m_work_available.Signal();
-}
-
 void GuestGpu::ProcessCommands() {
 	EXIT_IF(!IsGpuThread());
 	while (m_pending_commands.load(std::memory_order_acquire) != 0) {
@@ -281,6 +274,16 @@ template void CommandProcessor::WaitRegMem<uint32_t>(uint32_t, const uint32_t*,
 template void CommandProcessor::WaitRegMem<uint64_t>(uint32_t, const uint64_t*, uint64_t, uint64_t,
                                                     uint32_t, uint32_t);

+void CommandProcessor::MemSemaphore(uint64_t* counter, bool signal, bool write) {
+	if (signal) {
+		*counter = write ? 1u : *counter + 1u;
+	} else if (*counter == 0) {
+		SuspendPm4();
+	} else {
+		(*counter)--;
+	}
+}
+
 void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num,
                                 uint32_t write_control) {
 	const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
@@ -433,10 +436,7 @@ void GuestGpu::ThreadRun(void* data) {
 				}
 				if (selected_queue < 0) {
 					gpu->m_processing = false;
-					// A wake that came before this wait must not be lost.
-					if (!std::exchange(gpu->m_wake_pending, false)) {
-						gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
-					}
+					gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
 					for (auto& queue: gpu->m_queues) {
 						if (!queue.empty()) {
 							queue.front().blocked = false;
@@ -804,6 +804,27 @@ static void PatchIndirectDrawOffsets(CommandProcessor& cp, IndirectDrawRegisters
 	write(registers.draw_index, draw_index);
 }

+// Writes the GS user-data slots the packet patches; false when the shaders cannot read the
+// offsets from GPU records: a first-index or subdraw-index destination, or another stage's
+// user data.
+static bool IndirectDrawOffsets(IndirectDrawRegisters registers, ShaderIndirectOffsets& offsets) {
+	if (registers.index_offset != Pm4::SH_NOP || registers.draw_index != Pm4::SH_NOP) {
+		return false;
+	}
+	for (const auto [location, slot]: {std::pair {registers.vertex_offset, &offsets.vertex},
+	                                   std::pair {registers.instance_offset, &offsets.instance}}) {
+		if (location == Pm4::SH_NOP) {
+			continue;
+		}
+		const uint32_t index = location - Pm4::SPI_SHADER_USER_DATA_GS_0;
+		if (index >= HW::UserSgprInfo::SGPRS_MAX) {
+			return false;
+		}
+		*slot = static_cast<uint8_t>(index);
+	}
+	return true;
+}
+
 void CommandProcessor::DrawIndirect(uint32_t data_offset, IndirectDrawRegisters registers,
                                    uint32_t draw_initiator, bool indexed) {
 	const auto args_size =
@@ -835,13 +856,10 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 	// NUM_INSTANCES keeps the last draw's instance count.
 	m_num_instances_addr = args_addr + static_cast<uint64_t>(draw_count - 1) * stride_in_bytes +
 	                       offsetof(vk::DrawIndirectCommand, instanceCount);
-	// With no SGPRs to patch, the GPU reads the records if the shaders allow. Shaders apply
-	// the records' offsets only through those SGPRs; Vulkan applies them, which matches
-	// while they are zero.
-	const DrawIndirectArgs records {args_addr, draw_count, stride_in_bytes};
-	if (registers.vertex_offset == Pm4::SH_NOP && registers.instance_offset == Pm4::SH_NOP &&
-	    registers.index_offset == Pm4::SH_NOP && registers.draw_index == Pm4::SH_NOP &&
-	    (!indexed || m_index_buffer_size != 0) &&
+	// The GPU reads the records if the shaders allow; the VS prolog removes Vulkan's bases
+	// from the index VGPRs and copies them only to the user SGPRs the packet patches.
+	DrawIndirectArgs records {args_addr, draw_count, stride_in_bytes};
+	if (IndirectDrawOffsets(registers, records.offsets) && (!indexed || m_index_buffer_size != 0) &&
 	    (indexed ? executor.DrawIndex(m_submit_id, CurrentBuffer(),
 	                                  {.index_count         = m_index_buffer_size,
 	                                   .index_addr          = IndexAddress(0),
@@ -911,6 +929,11 @@ void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_g
 	}
 }

+void CommandProcessor::ProcessPssr(const Pssr::Request& request) {
+	EXIT_NOT_IMPLEMENTED(IsAsyncComputeQueue());
+	m_renderer.GetRenderExecutor().ProcessPssr(CurrentBuffer(), request);
+}
+
 void CommandProcessor::DispatchIndirect(uint64_t args_addr, uint32_t mode) {
 	EXIT_NOT_IMPLEMENTED(args_addr == 0 || (args_addr & 3u) != 0);
 	if ((mode & Pm4::COMPUTE_DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0) {
@@ -931,11 +954,10 @@ void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {
 }

 void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
-	// Stall only this command processor, not the host command lane or other queues.
-	if (!m_renderer.GetVideoOut().IsSafeForRendering(static_cast<int>(video_out_handle),
-	                                                 static_cast<int>(display_buffer_index))) {
-		SuspendPm4();
-	}
+	BufferFlush();
+
+	m_renderer.GetVideoOut().WaitFlipDone(static_cast<int>(video_out_handle),
+	                                      static_cast<int>(display_buffer_index));
 }

 void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector, void* address,
@@ -966,16 +988,28 @@ void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector
 	// Dumped occlusion counters are written before any end-of-pipe signal reaches the guest.
 	m_renderer.GetOcclusionQueries().Publish();

+	// A label beside GPU-written bytes goes to the backing store, so they are not downloaded.
+	// Other labels go through the guest mapping: its fault keeps a clean cached copy current and
+	// downloads GPU-written bytes under the label before the store.
+	const auto write_label = [this, address](const void* value, uint64_t size) {
+		const auto vaddr   = reinterpret_cast<uint64_t>(address);
+		auto&      buffers = m_renderer.GetBufferCache();
+		if (!GuestRange {vaddr, size}.Valid() || !buffers.IsRegionGpuModified(vaddr, size) ||
+		    buffers.HasGpuDirtyBytes(vaddr, size) ||
+		    !LibKernel::Memory::TryWriteBacking(vaddr, value, size)) {
+			std::memcpy(address, value, size);
+		}
+	};
 	// EndOfPipeData: kNone, kImmediate32b, kImmediate64b, kRefClk; 5 reads GDS.
 	switch (data_sel) {
 		case 0: break;
 		case 1: {
 			const auto value = static_cast<uint32_t>(data);
-			std::memcpy(address, &value, sizeof(value));
+			write_label(&value, sizeof(value));
 			break;
 		}
 		case 3: data = Sync::ReadReferenceClock(); [[fallthrough]];
-		case 2: std::memcpy(address, &data, sizeof(data)); break;
+		case 2: write_label(&data, sizeof(data)); break;
 		case 5: {
 			GetScheduler().Finish();
 			const auto gds = static_cast<uint32_t>(data);
@@ -31,8 +31,6 @@ public:
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
-	// Retries blocked submissions now instead of after the 100 ms stall timeout.
-	void               Wake();

 	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
 	void Submit(std::span<const uint32_t> commands);
@@ -84,7 +82,6 @@ private:
 	bool                                           m_processing        = false;
 	bool                                           m_accepting         = true;
 	bool                                           m_stopping          = false;
-	bool                                           m_wake_pending      = false;
 	bool                                           m_shutdown_complete = false;
 	// Completion callbacks can outlive GuestGpu during renderer shutdown.
 	std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =
@@ -81,6 +81,7 @@ constexpr uint32_t R_WRITE_DATA     = 0x15;
 constexpr uint32_t R_FLIP           = 0x17;
 constexpr uint32_t R_DMA_DATA       = 0x19;
 constexpr uint32_t R_CONTEXT_STATE  = 0x1A;
+constexpr uint32_t R_PSSR           = 0x1B;

 constexpr uint32_t R_NUM = 0x3F + 1;

@@ -0,0 +1,20 @@
+#ifndef EMULATOR_SRC_GRAPHICS_GUEST_GPU_PSSR_H_
+#define EMULATOR_SRC_GRAPHICS_GUEST_GPU_PSSR_H_
+
+#include <cstdint>
+
+namespace Libs::Graphics::Pssr {
+
+struct Request {
+	uint32_t output[8];
+	uint32_t input[8];
+	uint32_t width;
+	uint32_t height;
+};
+
+constexpr uint32_t RequestDwords = sizeof(Request) / sizeof(uint32_t);
+static_assert(sizeof(Request) == 72);
+
+} // namespace Libs::Graphics::Pssr
+
+#endif // EMULATOR_SRC_GRAPHICS_GUEST_GPU_PSSR_H_
@@ -259,6 +259,17 @@ static bool GetMipTailLayout(const TileBlockLayout& block, MipTailLayout& out) {
 	return false;
 }

+static uint32_t FirstTailLevel(const MipTailLayout& tail, uint32_t width, uint32_t height,
+                               uint32_t levels) {
+	for (uint32_t level = 0; levels > 1 && level < levels; ++level) {
+		if (ShiftCeil(width, level) <= tail.width_limit &&
+		    ShiftCeil(height, level) <= tail.height_limit && levels - level <= tail.max_levels) {
+			return level;
+		}
+	}
+	return levels;
+}
+
 bool TileGetTextureElementLayout(Prospero::BufferFormat format, TileTextureElementLayout& out) {
 	if (const auto bytes = Prospero::NumBytesPerElement(format); std::has_single_bit(bytes)) {
 		out = {bytes, 1, 1};
@@ -336,23 +347,14 @@ bool TileGetTiledTextureLayout(const TileSurfaceDescription& description, TileSu
 	TileSurfaceLayout result {};
 	result.description      = description;
 	result.texture          = texture;
-	result.first_tail_level = description.levels;
 	const uint32_t block_slices =
 	    volume ? ShiftCeil(description.depth, std::countr_zero(block.block_depth))
 	           : description.layers;

 	MipTailLayout tail {};
-	const bool    has_tail = GetMipTailLayout(block, tail);
-	if (has_tail && description.levels > 1) {
-		for (uint32_t level = 0; level < description.levels; ++level) {
-			if (ShiftCeil(width0, level) <= tail.width_limit &&
-			    ShiftCeil(height0, level) <= tail.height_limit &&
-			    description.levels - level <= tail.max_levels) {
-				result.first_tail_level = level;
-				break;
-			}
-		}
-	}
+	result.first_tail_level = GetMipTailLayout(block, tail)
+	                              ? FirstTailLevel(tail, width0, height0, description.levels)
+	                              : description.levels;

 	for (uint32_t level = 0; level < result.first_tail_level; ++level) {
 		auto& mip = result.mips[level];
@@ -1213,44 +1215,52 @@ bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
 	return true;
 }

-static bool TileGetColorMetadataSize(uint32_t width, uint32_t height, uint32_t slices,
-                                      uint32_t block_width, uint32_t block_height,
-                                      TileSizeAlign& total_size) {
-	if (width == 0 || height == 0 || slices == 0) {
-		return false;
-	}
-	const uint64_t blocks_x = (static_cast<uint64_t>(width) + block_width - 1u) / block_width;
-	const uint64_t blocks_y = (static_cast<uint64_t>(height) + block_height - 1u) / block_height;
-	const uint64_t blocks   = blocks_x * blocks_y;
-	if (blocks > UINT32_MAX / 4096u / slices) {
-		return false;
-	}
-	total_size.size  = static_cast<uint32_t>(blocks * slices * 4096u);
-	total_size.align = 4096;
-	return true;
-}
-
 bool TileGetDccSize(uint32_t width, uint32_t height, uint32_t slices,
                    uint32_t bytes_per_element, uint32_t levels, Prospero::TileMode tile,
-                    TileSizeAlign& total_size, uint32_t num_fragments_log2) {
+                    TileSizeAlign& total_size, uint32_t num_fragments_log2, TileSizeOffset* mips) {
 	total_size = {};
-	if (levels != 1 || num_fragments_log2 != 0 ||
-	    !std::has_single_bit(bytes_per_element) || bytes_per_element > 16 ||
-	    (tile != Prospero::TileMode::kRenderTarget && tile != Prospero::TileMode::kDepth)) {
+	TileBlockLayout block {};
+	MipTailLayout   tail {};
+	if (slices == 0 || levels > 16 || num_fragments_log2 != 0 ||
+	    (tile != Prospero::TileMode::kRenderTarget && tile != Prospero::TileMode::kDepth) ||
+	    !TileGetBlockLayout(tile == Prospero::TileMode::kDepth ? TileBlockFamily::Depth64KB
+	                                                           : TileBlockFamily::RenderTarget64KB,
+	                        bytes_per_element, block) ||
+	    !GetMipTailLayout(block, tail)) {
 		return false;
 	}
-	// Gen5 DCC uses 4 KiB blocks, with one byte per 256 bytes of color data.
-	// Metadata stays thin for volume surfaces, so every depth slice has its own block raster.
+	// Gen5 DCC uses 4 KiB blocks, with one byte per 256 bytes of color data. Each slice stores
+	// its mips backwards through the first tail mip, which fits one block; later tail mips have
+	// none. Metadata stays thin for volume surfaces, so every depth slice has its own block raster.
 	const uint32_t coverage_bits = 20u - std::countr_zero(bytes_per_element);
-	return TileGetColorMetadataSize(width, height, slices, 1u << ((coverage_bits + 1u) / 2u),
-	                                1u << (coverage_bits / 2u), total_size);
+	uint64_t       slice_size    = 0;
+	for (uint32_t level = std::min(FirstTailLevel(tail, width, height, levels) + 1u, levels);
+	     level-- > 0;) {
+		const uint64_t size = uint64_t {ShiftCeil(width, level + (coverage_bits + 1u) / 2u)} *
+		                      ShiftCeil(height, level + coverage_bits / 2u) * 4096u;
+		if (mips != nullptr) {
+			mips[level] = {.size   = static_cast<uint32_t>(size),
+			               .offset = static_cast<uint32_t>(slice_size)};
+		}
+		slice_size += size;
+	}
+	if (slice_size == 0 || slice_size > UINT32_MAX / slices) {
+		return false;
+	}
+	total_size = {static_cast<uint32_t>(slice_size * slices), 4096};
+	return true;
 }

 bool TileGetCmaskSize(uint32_t width, uint32_t height, uint32_t slices, uint32_t levels,
                      TileSizeAlign& total_size) {
 	total_size = {};
 	// Gen5 CMASK stores four bits per 8x8 tile in 4 KiB blocks covering 1024x512 pixels.
-	return levels == 1 && TileGetColorMetadataSize(width, height, slices, 1024, 512, total_size);
+	const uint64_t blocks = uint64_t {ShiftCeil(width, 10)} * ShiftCeil(height, 9);
+	if (levels != 1 || blocks == 0 || slices == 0 || blocks > UINT32_MAX / 4096u / slices) {
+		return false;
+	}
+	total_size = {static_cast<uint32_t>(blocks * slices * 4096u), 4096};
+	return true;
 }

 void TileGetTextureSize(Prospero::BufferFormat format, uint32_t width, uint32_t height,
@@ -123,9 +123,11 @@ uint32_t TileGetDepthPitch(uint32_t width, uint32_t bytes_per_element,
 bool     TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
                                 uint32_t bytes_per_element, TileSizeAlign& total_size,
                                 uint32_t num_fragments_log2 = 0);
+// mips, when given, receives the size and offset within a slice of each level that has metadata.
 bool     TileGetDccSize(uint32_t width, uint32_t height, uint32_t slices,
                        uint32_t bytes_per_element, uint32_t levels, Prospero::TileMode tile,
-                        TileSizeAlign& total_size, uint32_t num_fragments_log2 = 0);
+                        TileSizeAlign& total_size, uint32_t num_fragments_log2 = 0,
+                        TileSizeOffset* mips = nullptr);
 bool     TileGetCmaskSize(uint32_t width, uint32_t height, uint32_t slices, uint32_t levels,
                          TileSizeAlign& total_size);
 void     TileGetTextureSize(Prospero::BufferFormat format, uint32_t width, uint32_t height,
@@ -5,6 +5,7 @@

 #include <algorithm>
 #include <cstdint>
+#include <iterator>
 #include <map>

 namespace Libs::Graphics {
@@ -17,14 +18,19 @@ public:
 		if (it != m_ranges.begin() && std::prev(it)->second >= address) {
 			it = std::prev(it);
 		}
-		uint64_t begin = address;
-		uint64_t last  = end;
-		while (it != m_ranges.end() && it->first <= last) {
-			begin = std::min(begin, it->first);
-			last  = std::max(last, it->second);
-			it    = m_ranges.erase(it);
+		if (it == m_ranges.end() || it->first > address) {
+			it = m_ranges.emplace_hint(it, address, end);
+		} else {
+			if (it->second >= end) {
+				return;
+			}
+			it->second = end;
+		}
+		auto next = std::next(it);
+		while (next != m_ranges.end() && next->first <= it->second) {
+			it->second = std::max(it->second, next->second);
+			next       = m_ranges.erase(next);
 		}
-		m_ranges.emplace(begin, last);
 	}

 	void Subtract(uint64_t address, uint64_t size) {
@@ -8,6 +8,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
+#include <iterator>
 #include <memory>
 #include <type_traits>
 #include <utility>
@@ -118,6 +119,13 @@ public:

 	class const_iterator final {
 	public:
+		// libstdc++'s algorithms need these. Input, not forward, because operator* returns a copy.
+		using iterator_category = std::input_iterator_tag;
+		using value_type        = OwnerT;
+		using difference_type   = std::ptrdiff_t;
+		using pointer           = void;
+		using reference         = OwnerT;
+
 		[[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); }
 		const_iterator&      operator++() noexcept {
 			++m_index;
@@ -488,16 +488,15 @@ void TextureCache::RefreshCopySource(ImageId id) {
 }

 bool TextureCache::CopyD16(Image& destination, Image& source) {
-	const bool source_depth      = source.info.IsDepth();
-	const bool destination_depth = destination.info.IsDepth();
-	if (source_depth == destination_depth) {
+	// Converts guest D16 between a four-byte host depth aspect and two-byte color or D16 depth.
+	const bool source_wide = DepthAspectTransferBytes(source.backing.format) == sizeof(uint32_t);
+	if (source_wide == (DepthAspectTransferBytes(destination.backing.format) == sizeof(uint32_t))) {
 		return false;
 	}
-	auto&      depth          = source_depth ? source : destination;
-	auto&      color          = source_depth ? destination : source;
-	const auto transfer_bytes = DepthAspectTransferBytes(depth.backing.format);
+	auto& depth  = source_wide ? source : destination;
+	auto& narrow = source_wide ? destination : source;
 	if (depth.info.bytes_per_block != sizeof(uint16_t) ||
-	    color.info.bytes_per_block != sizeof(uint16_t) || transfer_bytes != sizeof(uint32_t)) {
+	    narrow.info.bytes_per_block != sizeof(uint16_t)) {
 		return false;
 	}
 	EXIT_IF(source.backing.samples != 1 || destination.backing.samples != 1 ||
@@ -507,42 +506,42 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {

 	const auto     layers = depth.info.resources.layers;
 	const uint64_t depth_slice =
-	    static_cast<uint64_t>(depth.info.pitch) * depth.info.extent.height * transfer_bytes;
-	const uint64_t color_slice =
-	    static_cast<uint64_t>(color.info.pitch) * color.info.extent.height * sizeof(uint16_t);
-	EXIT_IF(layers == 0 || depth_slice > UINT64_MAX / layers || color_slice > UINT64_MAX / layers);
-	const auto                       depth_size = depth_slice * layers;
-	const auto                       color_size = color_slice * layers;
-	const std::array                 depth_copies {
+	    static_cast<uint64_t>(depth.info.pitch) * depth.info.extent.height * sizeof(uint32_t);
+	const uint64_t narrow_slice =
+	    static_cast<uint64_t>(narrow.info.pitch) * narrow.info.extent.height * sizeof(uint16_t);
+	EXIT_IF(layers == 0 || depth_slice > UINT64_MAX / layers || narrow_slice > UINT64_MAX / layers);
+	const std::array depth_copies {
 	    vk::BufferImageCopy {0,
 	                         depth.info.pitch,
 	                         depth.info.extent.height,
 	                         {vk::ImageAspectFlagBits::eDepth, 0, 0, layers},
 	                         {},
 	                         depth.info.extent}};
-	const std::array color_copies {
+	const std::array narrow_copies {
 	    vk::BufferImageCopy {0,
-	                         color.info.pitch,
-	                         color.info.extent.height,
-	                         {vk::ImageAspectFlagBits::eColor, 0, 0, layers},
+	                         narrow.info.pitch,
+	                         narrow.info.extent.height,
+	                         {narrow.info.IsDepth() ? vk::ImageAspectFlagBits::eDepth
+	                                                : vk::ImageAspectFlagBits::eColor,
+	                          0, 0, layers},
 	                         {},
-	                         color.info.extent}};
+	                         narrow.info.extent}};

-	auto                         depth_buffer = m_tiler.GetScratchBuffer(depth_size);
-	auto                         color_buffer = m_tiler.GetScratchBuffer(color_size, depth_buffer.buffer);
+	auto depth_buffer  = m_tiler.GetScratchBuffer(depth_slice * layers);
+	auto narrow_buffer = m_tiler.GetScratchBuffer(narrow_slice * layers, depth_buffer.buffer);
 	const TileManager::D16Layout promote_layout {
 	    .width               = depth.info.extent.width,
 	    .height              = depth.info.extent.height,
 	    .layers              = layers,
-	    .source_row_stride   = static_cast<uint64_t>(color.info.pitch) * sizeof(uint16_t),
-	    .target_row_stride   = static_cast<uint64_t>(depth.info.pitch) * transfer_bytes,
-	    .source_slice_stride = color_slice,
+	    .source_row_stride   = static_cast<uint64_t>(narrow.info.pitch) * sizeof(uint16_t),
+	    .target_row_stride   = static_cast<uint64_t>(depth.info.pitch) * sizeof(uint32_t),
+	    .source_slice_stride = narrow_slice,
 	    .target_slice_stride = depth_slice,
 	};
 	const bool d32 = DepthAspectTransferFormat(depth.backing.format) == vk::Format::eD32Sfloat;
-	if (source_depth) {
+	if (source_wide) {
 		source.Download(depth_copies, depth_buffer.buffer, depth_buffer.offset, depth_buffer.size);
-		m_tiler.ConvertD16(depth_buffer, color_buffer, TileManager::D16Direction::Demote, d32,
+		m_tiler.ConvertD16(depth_buffer, narrow_buffer, TileManager::D16Direction::Demote, d32,
 		                   {.width               = promote_layout.width,
 		                    .height              = promote_layout.height,
 		                    .layers              = promote_layout.layers,
@@ -550,11 +549,12 @@ bool TextureCache::CopyD16(Image& destination, Image& source) {
 		                    .target_row_stride   = promote_layout.source_row_stride,
 		                    .source_slice_stride = promote_layout.target_slice_stride,
 		                    .target_slice_stride = promote_layout.source_slice_stride});
-		destination.Upload(color_copies, color_buffer.buffer, color_buffer.offset,
-		                   color_buffer.size);
+		destination.Upload(narrow_copies, narrow_buffer.buffer, narrow_buffer.offset,
+		                   narrow_buffer.size);
 	} else {
-		source.Download(color_copies, color_buffer.buffer, color_buffer.offset, color_buffer.size);
-		m_tiler.ConvertD16(color_buffer, depth_buffer, TileManager::D16Direction::Promote, d32,
+		source.Download(narrow_copies, narrow_buffer.buffer, narrow_buffer.offset,
+		                narrow_buffer.size);
+		m_tiler.ConvertD16(narrow_buffer, depth_buffer, TileManager::D16Direction::Promote, d32,
 		                   promote_layout);
 		destination.Upload(depth_copies, depth_buffer.buffer, depth_buffer.offset,
 		                   depth_buffer.size);
@@ -1043,34 +1043,51 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 	    desc.info.metadata.kind != ImageMetadataKind::Cmask) {
 		return;
 	}
-	const auto range = desc.info.metadata.range;
+	const auto  range = desc.info.metadata.range;
+	const auto& view  = desc.view_info;
 	{
 		std::scoped_lock lock {m_lock};
 		auto& image         = m_slot_images[id];
 		image.info.metadata = desc.info.metadata;
 		// Native color metadata must not retain a reused HTile clear flag.
 		m_surface_metas.erase(range.address);
-		if (range.size == 0 || desc.info.resources.levels != 1) {
+		if (range.size == 0 || view.level_count != 1) {
 			return;
 		}
 	}
 	const auto layers = desc.info.TransferLayers();
-	// These one-mip surfaces use complete 4 KiB color metadata blocks.
+	// Color metadata slices consist of complete 4 KiB blocks.
 	constexpr uint64_t MetadataBlockSize = 0x1000;
 	if (!range.Valid() || range.address % MetadataBlockSize != 0 || layers == 0 ||
 	    range.size % layers != 0 || (range.size / layers) % MetadataBlockSize != 0) {
 		EXIT("TextureCache: color metadata slices must contain aligned 4 KiB blocks\n");
 	}
-	const auto& view           = desc.view_info;
-	const bool  volume_texture = desc.info.IsVolume() && view.type == vk::ImageViewType::e3D;
-	const auto  first          = volume_texture ? 0u : metadata_base_layer;
-	const auto  image_first    = volume_texture ? 0u : view.base_layer;
-	const auto  count          = volume_texture ? desc.info.extent.depth : view.layer_count;
+	const bool volume_texture = desc.info.IsVolume() && view.type == vk::ImageViewType::e3D;
+	const auto first          = volume_texture ? 0u : metadata_base_layer;
+	const auto image_first    = volume_texture ? 0u : view.base_layer;
+	const auto count =
+	    volume_texture ? std::max(desc.info.extent.depth >> view.base_level, 1u) : view.layer_count;
 	if (first >= layers || count > layers - first) {
 		EXIT("TextureCache: color view exceeds its native metadata slices\n");
 	}
-	const auto       slice_size = range.size / layers;
-	const GuestRange viewed {range.address + slice_size * first, slice_size * count};
+	const auto slice_size = range.size / layers;
+	// Each slice holds the viewed mip's metadata in one run; DCC mips past the first tail mip
+	// have none. FindImage moves base_level only for one-mip descriptors, which keep the slice.
+	TileSizeOffset mip {.size = static_cast<uint32_t>(slice_size)};
+	if (desc.info.metadata.kind == ImageMetadataKind::Dcc && desc.info.resources.levels > 1) {
+		std::array<TileSizeOffset, 16> mips {};
+		TileSizeAlign                  total {};
+		(void)TileGetDccSize(desc.info.extent.width, desc.info.extent.height, layers,
+		                     desc.info.bytes_per_block, desc.info.resources.levels,
+		                     desc.info.tile_mode, total, 0, mips.data());
+		mip = mips[view.base_level];
+	}
+	if (mip.size == 0) {
+		return;
+	}
+	const GuestRange viewed {range.address + slice_size * first + mip.offset,
+	                         slice_size * (count - 1) + mip.size};
+	const auto run = [&](uint32_t slice) { return viewed.address + slice_size * slice; };
 	if (m_buffer_cache.IsRegionGpuModified(viewed.address, viewed.size)) {
 		std::array<vk::ClearAttachment, TileManager::ColorClearSlots> clears {};
 		uint32_t keys = 0;
@@ -1082,7 +1099,11 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 			}
 		}
 		// No selected key to clear, or none left since the last check.
-		if (m_buffer_cache.IsColorMetadataChecked(viewed.address, viewed.size, keys)) {
+		bool checked = true;
+		for (uint32_t slice = 0; slice < count && checked; slice++) {
+			checked = m_buffer_cache.IsColorMetadataChecked(run(slice), mip.size, keys);
+		}
+		if (checked) {
 			return;
 		}
 		{
@@ -1107,9 +1128,9 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 				const auto [buffer, offset] =
 				    m_buffer_cache.ObtainBuffer(viewed.address, viewed.size, expand);
 				const auto predicates = m_tiler.CheckColorClear(
-				    {buffer->Handle(), offset, viewed.size}, count, keys, expand);
-				if (expand) {
-					m_buffer_cache.MarkColorMetadataChecked(viewed.address, viewed.size, keys);
+				    {buffer->Handle(), offset, viewed.size}, slice_size, count, keys, expand);
+				for (uint32_t slice = 0; slice < count && expand; slice++) {
+					m_buffer_cache.MarkColorMetadataChecked(run(slice), mip.size, keys);
 				}
 				const auto command = m_scheduler.Current().Handle();
 				const auto area    = BeginColorRendering(image, view.format, view.base_level,
@@ -1142,7 +1163,7 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 		m_buffer_cache.ReadMemory(viewed.address, viewed.size, false);
 	}
 	for (uint32_t slice = 0; slice < count; slice++) {
-		const auto address = range.address + slice_size * (first + slice);
+		const auto address = run(slice);
 		uint8_t code = 0;
 		if (!LibKernel::Memory::TryReadBacking(address, &code, sizeof(code))) {
 			EXIT("TextureCache: failed to read color metadata backing\n");
@@ -1151,7 +1172,7 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 		if (!DecodeColorClear(desc, code, clear.color)) {
 			continue;
 		}
-		std::vector<uint8_t> bytes(slice_size);
+		std::vector<uint8_t> bytes(mip.size);
 		if (!LibKernel::Memory::TryReadBacking(address, bytes.data(), bytes.size())) {
 			EXIT("TextureCache: failed to read color metadata slice\n");
 		}
@@ -1175,7 +1196,7 @@ void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
 		// to overlapping image data. Invalidate the buffer before updating its backing.
 		if (desc.type != BindingType::VideoOut) {
 			std::fill(bytes.begin(), bytes.end(), uint8_t {0xff});
-			m_buffer_cache.InvalidateMemory(address, slice_size);
+			m_buffer_cache.InvalidateMemory(address, bytes.size());
 			LibKernel::Memory::WriteBacking(address, bytes.data(), bytes.size());
 		}
 	}
@@ -125,6 +125,14 @@ void DynamicState::Commit(const GraphicContext& graphics, vk::CommandBuffer cmdb
 			}
 		}
 	}
+	if (dirty_state.cull_mode) {
+		dirty_state.cull_mode = false;
+		cmdbuf.setCullMode(cull_mode);
+	}
+	if (dirty_state.front_face) {
+		dirty_state.front_face = false;
+		cmdbuf.setFrontFace(front_face);
+	}
 	if (dirty_state.blend_constants) {
 		dirty_state.blend_constants = false;
 		cmdbuf.setBlendConstants(blend_constants.data());
@@ -52,6 +52,9 @@ struct DynamicState {
 		bool stencil_back_write_mask    : 1;
 		bool stencil_back_compare_mask  : 1;

+		bool cull_mode  : 1;
+		bool front_face : 1;
+
 		bool blend_constants       : 1;
 		bool color_write_enables   : 1;
 		bool line_width            : 1;
@@ -86,6 +89,9 @@ struct DynamicState {
 	uint32_t   stencil_back_write_mask {};
 	uint32_t   stencil_back_compare_mask {};

+	vk::CullModeFlags cull_mode {};
+	vk::FrontFace     front_face {};
+
 	std::array<float, 4>                               blend_constants {};
 	std::array<vk::Bool32, RENDER_COLOR_ATTACHMENTS_MAX> color_write_enables {};
 	uint32_t                                           num_color_write_enables {};
@@ -221,6 +227,20 @@ struct DynamicState {
 		}
 	}

+	void SetCullMode(const vk::CullModeFlags cull_mode_) {
+		if (cull_mode != cull_mode_) {
+			cull_mode             = cull_mode_;
+			dirty_state.cull_mode = true;
+		}
+	}
+
+	void SetFrontFace(const vk::FrontFace front_face_) {
+		if (front_face != front_face_) {
+			front_face             = front_face_;
+			dirty_state.front_face = true;
+		}
+	}
+
 	void SetBlendConstants(const std::array<float, 4> blend_constants_) {
 		if (blend_constants != blend_constants_) {
 			blend_constants             = blend_constants_;
@@ -17,9 +17,15 @@

 namespace Libs::Graphics {

-enum class ColorTransform : uint32_t { None, SwapBgra16, Reverse10_11_11 };
+enum class ColorTransform : uint32_t { None, SwapBgra16, Reverse10_11_11, Rotate2_10_10_10 };

-enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported };
+enum class VideoOutCompression : uint8_t {
+	Uncompressed,
+	Dcc256_256_0,
+	Dcc256_64_64,
+	Dcc128_128_128,
+	Unsupported
+};

 enum class ImageMetadataKind : uint8_t { None, Htile, Dcc, Cmask };

@@ -99,6 +105,8 @@ struct ImageInfo {
 		switch (guest_format) {
 			case Prospero::BufferFormat::k10_11_11Float:
 				return ColorTransform::Reverse10_11_11;
+			case Prospero::BufferFormat::k2_10_10_10UNorm:
+				return ColorTransform::Rotate2_10_10_10;
 			default: return ColorTransform::None;
 		}
 	}
@@ -287,8 +295,9 @@ inline bool ImageInfo::IsDepth() const noexcept {
 [[nodiscard]] inline VideoOutCompression
 ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t dcc_control,
                            uint64_t dcc_clear_color) noexcept {
-	constexpr uint32_t DCC_256_256_0 = 0x00000048u;
-	constexpr uint32_t DCC_256_64_64 = 0x00000208u;
+	constexpr uint32_t DCC_256_256_0   = 0x00000048u;
+	constexpr uint32_t DCC_256_64_64   = 0x00000208u;
+	constexpr uint32_t DCC_128_128_128 = 0x00100024u;
 	if (!compressed) {
 		return metadata_address == 0 && dcc_control == 0 && dcc_clear_color == 0
 		           ? VideoOutCompression::Uncompressed
@@ -300,6 +309,7 @@ ClassifyVideoOutCompression(bool compressed, uint64_t metadata_address, uint32_t
 	switch (dcc_control) {
 		case DCC_256_256_0: return VideoOutCompression::Dcc256_256_0;
 		case DCC_256_64_64: return VideoOutCompression::Dcc256_64_64;
+		case DCC_128_128_128: return VideoOutCompression::Dcc128_128_128;
 		default: return VideoOutCompression::Unsupported;
 	}
 }
@@ -32,8 +32,12 @@ struct HostFormatInfo {

 HostFormatInfo ResolveHostFormat(Prospero::BufferFormat guest_format,
                                 Prospero::ChannelOrder order) {
-	if (guest_format == Prospero::BufferFormat::k10_11_11Float) {
-		return {vk::Format::eB10G11R11UfloatPack32, Prospero::ColorMappingBgra};
+	switch (guest_format) {
+		case Prospero::BufferFormat::k10_11_11Float:
+			return {vk::Format::eB10G11R11UfloatPack32, Prospero::ColorMappingBgra};
+		case Prospero::BufferFormat::k2_10_10_10UNorm:
+			return {vk::Format::eA2R10G10B10UnormPack32, Prospero::ColorMappingAbgr};
+		default: break;
 	}
 	if (order == Prospero::ChannelOrder::kAlt) {
 		switch (guest_format) {
@@ -104,7 +104,8 @@ TileManager::Tiling TileManager::Prepare(bool tile, uint64_t tiled_capacity,
 	    block.block_depth == 0 || block.block_size == 0 || info.tiling.texel_width == 0 ||
 	    info.tiling.texel_height == 0 || info.tiled_slice_stride == 0);
 	EXIT_IF((transform == ColorTransform::SwapBgra16 && block.bytes_per_element != 8u) ||
-	        (transform == ColorTransform::Reverse10_11_11 && block.bytes_per_element != 4u));
+	        ((transform == ColorTransform::Reverse10_11_11 ||
+	          transform == ColorTransform::Rotate2_10_10_10) && block.bytes_per_element != 4u));
 	const auto checked_multiply = [](uint64_t left, uint64_t right, uint64_t& result) {
 		return (left == 0 || right <= UINT64_MAX / left) && (result = left * right, true);
 	};
@@ -484,7 +485,7 @@ void TileManager::ConvertD16(Result source, Result target, D16Direction directio
 	                            vk::AccessFlagBits::eHostWrite |
 	                            vk::AccessFlagBits::eTransferWrite |
 	                            vk::AccessFlagBits::eShaderWrite;
-	barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite;
+	barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderWrite;
 	command.pipelineBarrier(
 	    vk::PipelineStageFlagBits::eAllCommands | vk::PipelineStageFlagBits::eHost,
 	    vk::PipelineStageFlagBits::eComputeShader, {}, 0, nullptr, 2, barriers, 0, nullptr);
@@ -560,7 +561,8 @@ void TileManager::TransformColor(Result input, Result output, ColorTransform tra
 	uint32_t element_bytes = 0;
 	switch (transform) {
 		case ColorTransform::SwapBgra16: element_bytes = 8; break;
-		case ColorTransform::Reverse10_11_11: element_bytes = 4; break;
+		case ColorTransform::Reverse10_11_11:
+		case ColorTransform::Rotate2_10_10_10: element_bytes = 4; break;
 		default: EXIT("invalid color transform\n");
 	}
 	EXIT_IF(input.size == 0 || input.size % element_bytes != 0 ||
@@ -628,19 +630,20 @@ TileManager::Result TileManager::TransformColor(Result input, ColorTransform tra
 	return result;
 }

-vk::Buffer TileManager::CheckColorClear(Result metadata, uint32_t slices, uint32_t keys,
-                                        bool expand) {
+vk::Buffer TileManager::CheckColorClear(Result metadata, uint64_t stride, uint32_t slices,
+                                        uint32_t keys, bool expand) {
 	struct Check {
 		uint32_t metadata_base;
 		uint32_t words;
+		uint32_t stride;
 		uint32_t keys;
 		uint32_t expand;
 	};
 	static_assert(sizeof(Check) <= sizeof(Conversion));
 	const uint32_t max_slices =
 	    m_graphics.GetPhysicalDeviceProperties().limits.maxFramebufferLayers;
-	EXIT_IF(slices == 0 || slices > max_slices || metadata.size % (4ull * slices) != 0 ||
-	        metadata.size / 4 > UINT32_MAX);
+	EXIT_IF(slices == 0 || slices > max_slices || metadata.size % 4 != 0 || stride % 4 != 0 ||
+	        metadata.size / 4 > UINT32_MAX || stride * (slices - 1) >= metadata.size);
 	if (m_color_clear == nullptr) {
 		m_color_clear      = CreatePipeline(GPU_TILER_COLOR_CLEAR_SPV, nullptr);
 		m_clear_predicates = std::make_unique<Buffer>(
@@ -671,8 +674,9 @@ vk::Buffer TileManager::CheckColorClear(Result metadata, uint32_t slices, uint32
 	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_color_clear);
 	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pipeline_layout, 0,
 	                             static_cast<uint32_t>(writes.size()), writes.data());
-	const Check push {source.base, static_cast<uint32_t>(metadata.size / 4 / slices), keys,
-	                  expand};
+	const Check push {source.base,
+	                  static_cast<uint32_t>((metadata.size - stride * (slices - 1)) / 4),
+	                  static_cast<uint32_t>(stride / 4), keys, expand};
 	command.pushConstants(m_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, sizeof(push),
 	                      &push);
 	command.dispatch(slices, 1, 1);
@@ -56,11 +56,11 @@ public:
 	void TransformColor(Result input, Result output, ColorTransform transform, bool to_host);
 	// Color-clear keys are multiples of 0x20, so key >> 5 selects one of these slots.
 	static constexpr uint32_t ColorClearSlots = 8;
-	// Returns the buffer whose dword slice * ColorClearSlots + (key >> 5) is set when every byte
-	// of that metadata slice equals a key whose bit `keys` selects; a hit expands to 0xff when
-	// `expand` is set.
-	[[nodiscard]] vk::Buffer CheckColorClear(Result metadata, uint32_t slices, uint32_t keys,
-	                                         bool expand);
+	// metadata spans `slices` equal runs `stride` bytes apart. Returns the buffer whose dword
+	// slice * ColorClearSlots + (key >> 5) is set when every byte of that run equals a key whose
+	// bit `keys` selects; a hit expands to 0xff when `expand` is set.
+	[[nodiscard]] vk::Buffer CheckColorClear(Result metadata, uint64_t stride, uint32_t slices,
+	                                         uint32_t keys, bool expand);

 private:
 	friend struct TileManagerTestAccess;
@@ -934,43 +934,6 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
 		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
-	for (const auto* reader: prepared_bindings) {
-		const auto& reads = reader->runtime->resources->specialization_reads;
-		if (reads.empty()) continue;
-		for (const auto* writer: prepared_bindings) {
-			if (writer->runtime->module->info.has_address_writes) {
-				EXIT("scalar resource reads cannot be proven disjoint from shader address writes\n");
-			}
-		}
-		for (const auto [address, size]: reads) {
-			for (const auto id: m_bound_images) {
-				const auto& image = m_context.GetTextureCache().m_slot_images[id];
-				if (!image.binding.shader_write && !image.binding.is_target) continue;
-				for (const auto written: {image.info.data, image.info.stencil,
-				                          image.info.metadata.range}) {
-					if (written.size != 0 && ImageRangeOverlaps(address, size,
-					                                          written.address, written.size)) {
-						EXIT("scalar resource reads overlap an image or attachment write\n");
-					}
-				}
-			}
-			for (const auto* writer: prepared_bindings) {
-				const auto& program = writer->runtime->module->info;
-				for (const auto& resource: program.info.buffers) {
-					if (!resource.written || resource.descriptor_index == UINT32_MAX) continue;
-					const auto& written = writer->buffer_sources[resource.descriptor_index];
-					// ADD_TID and swizzle move fixed-offset writes past write_extent.
-					const bool linear = (resource.packed_stride & ((1u << 14u) | (1u << 20u))) == 0u;
-					const auto written_size =
-					    linear ? std::min<uint64_t>(written.size, resource.write_extent) : written.size;
-					if (written_size != 0 && ImageRangeOverlaps(address, size,
-					                                          written.address, written_size)) {
-						EXIT("scalar resource reads overlap a shader buffer write\n");
-					}
-				}
-			}
-		}
-	}
 	if (pipeline.uses_image_heap) {
 		m_context.GetImageHeap().Commit(buffer, pipeline_bind_point, pipeline.pipeline_layout);
 	}
@@ -20,7 +20,7 @@ namespace {

 namespace IR = ShaderRecompiler::IR;

-// Smaller tables are cheap to materialize on each draw and are often transient.
+// Smaller tables are often written for one draw, so they enter on a second draw of a frame.
 constexpr uint64_t MinTableBytes = 64ull * 1024ull;
 constexpr uint32_t ElementMask   = (1u << IR::ImageHeapKindShift) - 1u;

@@ -133,12 +133,16 @@ ImageHeap::~ImageHeap() {
 bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) {
 	KYTY_PROFILER_FUNCTION();
 	const auto size = request.size & ~uint64_t {3};
-	if (size < MinTableBytes || request.count == 0u || request.step == 0u ||
+	if (request.count == 0u || request.step == 0u ||
 	    ((request.base | request.first | request.step) & 3u) != 0u) {
 		return false;
 	}
 	auto memory = std::ranges::find_if(
 	    m_memories, [&](const auto& candidate) { return candidate->base == request.base; });
+	if (memory == m_memories.end() && size < MinTableBytes &&
+	    m_sightings.try_emplace(request.base, m_draw).first->second == m_draw) {
+		return false;
+	}
 	// A table follows its descriptor's size, as the first program of a draw acquires it.
 	if (memory == m_memories.end() || ((*memory)->size != size && (*memory)->draw != m_draw)) {
 		if (!m_context.IsMapped(request.base, size)) return false;
@@ -593,26 +597,42 @@ void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) {
 	} else {
 		m_unmapped.push_back({address, size});
 	}
-	auto&           scheduler = m_context.GetCommandScheduler();
 	std::scoped_lock lock {m_mutex};
 	std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
 		if (address >= memory->base + memory->size || memory->base >= address + size) {
 			return false;
 		}
-		Unwatch(*memory);
-		for (auto& view: memory->views) {
-			std::erase(m_active, view.get());
-			for (const auto& [value, index]: view->elements) {
-				Unreference(m_elements[index]);
-				view->released.push_back(index);
-			}
-			Retire(*view);
-			if (scheduler.Active()) {
-				scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
-			}
-		}
+		Drop(*memory);
 		return true;
 	});
 }

+void ImageHeap::RunGarbageCollector() {
+	// A smaller table rebuilt each frame moves to a new base. Larger ones keep their dead words.
+	std::scoped_lock lock {m_mutex};
+	std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
+		if (memory->size >= MinTableBytes || memory->draw > m_suspend_draw) return false;
+		Drop(*memory);
+		return true;
+	});
+	m_suspend_draw = m_draw;
+	m_sightings.clear();
+}
+
+void ImageHeap::Drop(Memory& memory) {
+	auto& scheduler = m_context.GetCommandScheduler();
+	Unwatch(memory);
+	for (auto& view: memory.views) {
+		std::erase(m_active, view.get());
+		for (const auto& [value, index]: view->elements) {
+			Unreference(m_elements[index]);
+			view->released.push_back(index);
+		}
+		Retire(*view);
+		if (scheduler.Active()) {
+			scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
+		}
+	}
+}
+
 } // namespace Libs::Graphics
@@ -26,7 +26,7 @@ struct ImageHeapTestAccess;
 // of a table owns device slot words, one per table dword, that select an array element. CPU
 // writes to a table fault in its snapshot, which each draw refreshes once; a view takes the
 // changes when a draw first acquires it. Changes to the images behind elements bump
-// Image::heap_epoch.
+// Image::heap_epoch. A smaller table leaves at a suspend point if the frame did not acquire it.
 class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource {
 public:
 	ImageHeap(RenderContext& context, PageManager& page_manager);
@@ -47,6 +47,8 @@ public:
 	// CPU write fault, on any thread.
 	void InvalidateMemory(uint64_t address, uint64_t size);
 	void UnmapMemory(uint64_t address, uint64_t size);
+	// At a suspend point.
+	void RunGarbageCollector();

 private:
 	using DescriptorValue = ShaderRecompiler::IR::DescriptorValue;
@@ -107,6 +109,7 @@ private:
 	template <bool track>
 	void     Watch(const Memory& memory, size_t begin, size_t end);
 	void     Unwatch(const Memory& memory);
+	void     Drop(Memory& memory);
 	void     UpdateProbes(View& view, uint64_t begin, uint64_t end);
 	void     UpdateProbe(View& view, uint64_t probe);
 	void     Assign(View& view, size_t slot, uint32_t entry);
@@ -125,9 +128,11 @@ private:
 	std::mutex                           m_mutex; // Fault-visible Memory state and m_memories.
 	std::vector<std::unique_ptr<Memory>> m_memories;
 	std::vector<View*>                   m_active; // Acquired since the last commit.
-	uint64_t                             m_draw = 1;
+	uint64_t                             m_draw         = 1;
+	uint64_t                             m_suspend_draw = 0; // m_draw at the last suspend point.
 	std::vector<Element>                 m_elements;
 	std::vector<uint32_t>                m_free;
+	std::unordered_map<uint64_t, uint64_t>    m_sightings; // Smaller tables: base, first draw.
 	std::deque<std::pair<uint64_t, uint32_t>> m_retired; // Tick, element.
 	std::vector<std::pair<size_t, size_t>>    m_runs;
 	std::vector<uint32_t>                     m_scratch;
@@ -687,6 +687,10 @@ void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRender
 		vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]);
 	}
 	const bool mesh_active = vertex_info[0].logical_stage == ShaderType::Mesh;
+	if (!tess_active && !mesh_active) {
+		vertex_info[0].indirect_offsets = state.indirect_offsets;
+		vertex_info[0].indirect_draw    = state.indirect_draw;
+	}
 	if (object_primitive_id) {
 		// CxObjectIdMode: register source plus a 32-bit pipelined primitive-ID addition.
 		EXIT_NOT_IMPLEMENTED(sh.m_paClObjprimIdCntl != 2u || mesh_active || tess_active ||
@@ -1003,15 +1007,11 @@ PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer&
 	}
 	static_params.alpha_to_coverage_enable = ps_active && ctx.GetShaderRegisters().AlphaToMask();
 	const bool rect_list = Prospero::IsRectList(command.GetUserConfig().GetPrimType());
-	static_params.cull_back  = !rect_list && mc.cull_back;
-	static_params.cull_front = !rect_list && mc.cull_front;
-	static_params.face       = mc.face;
 	static_params.provoking_vtx_last = mc.provoking_vtx_last;
 	static_params.polygon_mode =
-	    ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back);
+	    ResolvePolygonMode(mc, !rect_list && mc.cull_front, !rect_list && mc.cull_back);
 	if (tessellation) {
-		static_params.patch_control_points =
-		    static_cast<uint8_t>(vs_input_info.tess.input_control_points);
+		static_params.patch_control_points = vs_input_info.tess.input_control_points;
 	} else if (!mesh && static_params.topology == vk::PrimitiveTopology::ePatchList) {
 		// Rectangle lists expand through generated tessellation stages.
 		static_params.patch_control_points = 3;
@@ -54,15 +54,12 @@ struct PipelineStaticParameters {
 	bool                       depth_clip_enable        = true;
 	vk::PrimitiveTopology      topology                 = vk::PrimitiveTopology::ePointList;
 	// 0 without tessellation, 3 for a rectangle list, else the input control points.
-	uint8_t                    patch_control_points     = 0;
+	uint32_t                   patch_control_points     = 0;
 	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
 	bool                       alpha_to_coverage_enable = false;
 	uint32_t                   color_mask[RENDER_COLOR_ATTACHMENTS_MAX]           = {};
-	bool                       cull_front                                         = false;
-	bool                       cull_back                                          = false;
-	bool                       face                                               = false;
 	bool                       provoking_vtx_last                                 = false;
 	vk::PolygonMode            polygon_mode                                       = vk::PolygonMode::eFill;
 	uint8_t                    color_srcblend[RENDER_COLOR_ATTACHMENTS_MAX]       = {};
@@ -140,6 +137,9 @@ struct DrawRenderState {
 	bool                                 ps_active                                = false;
 	std::array<ShaderVertexInputInfo, 3> vertex_info;
 	ShaderPixelInputInfo                 ps_input_info;
+	// Input: the user-data slots a GPU-read indirect draw patches.
+	ShaderIndirectOffsets                indirect_offsets;
+	bool                                 indirect_draw = false;

 	[[nodiscard]] std::span<const ShaderVertexInputInfo> VertexStages() const {
 		return {vertex_info.data(), vertex_info[1].stage ? 3u : 1u};
@@ -219,17 +219,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	vk::PipelineViewportStateCreateInfo viewport_state {};
 	viewport_state.pNext = &depth_clip_control;

-	vk::CullModeFlags cull_mode = vk::CullModeFlagBits::eNone;
-	if (static_params.cull_back) {
-		cull_mode |= vk::CullModeFlagBits::eBack;
-	}
-	if (static_params.cull_front) {
-		cull_mode |= vk::CullModeFlagBits::eFront;
-	}
-
-	vk::FrontFace front_face =
-	    (static_params.face ? vk::FrontFace::eClockwise : vk::FrontFace::eCounterClockwise);
-
 	vk::PipelineRasterizationDepthClipStateCreateInfoEXT clip_ext {};
 	clip_ext.depthClipEnable = static_params.depth_clip_enable ? VK_TRUE : VK_FALSE;

@@ -250,8 +239,6 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 		provoking_vertex.pNext = rasterizer.pNext;
 		rasterizer.pNext = &provoking_vertex;
 	}
-	rasterizer.cullMode  = cull_mode;
-	rasterizer.frontFace = front_face;
 	rasterizer.polygonMode = static_params.polygon_mode;
 	rasterizer.lineWidth = 1.0f;

@@ -347,6 +334,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
 	    vk::DynamicState::eViewportWithCount,
 	    vk::DynamicState::eScissorWithCount,
 	    vk::DynamicState::eLineWidth,
+	    vk::DynamicState::eCullMode,
+	    vk::DynamicState::eFrontFace,
 	    vk::DynamicState::eDepthTestEnable,
 	    vk::DynamicState::eDepthWriteEnable,
 	    vk::DynamicState::eDepthCompareOp,
@@ -45,9 +45,10 @@ enum class DrawOffsetSource : uint8_t {

 // Nonzero address: the GPU reads count Vulkan indirect draw records, stride bytes apart.
 struct DrawIndirectArgs {
-	uint64_t address = 0;
-	uint32_t count   = 0;
-	uint32_t stride  = 0;
+	uint64_t              address = 0;
+	uint32_t              count   = 0;
+	uint32_t              stride  = 0;
+	ShaderIndirectOffsets offsets;
 };

 struct DrawIndexArgs {
@@ -149,15 +150,21 @@ private:
 	friend class CommandScheduler;
 };

+namespace Pssr {
+struct Request;
+}
+
 class RenderExecutor {
 public:
 	explicit RenderExecutor(RenderContext& context): m_context(context) {}
+	~RenderExecutor();
 	KYTY_CLASS_NO_COPY(RenderExecutor);

 	void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x,
 	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode, bool async_compute);
 	void DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, uint64_t args_addr,
 	                      uint32_t mode, bool async_compute);
+	void ProcessPssr(CommandBuffer& buffer, const Pssr::Request& request);

 	void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared);
 	void                           FindBuffers(std::span<PreparedBindings* const> stages);
@@ -218,6 +225,9 @@ private:
 	std::vector<ImageId>                  m_bound_images;
 	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;
 	std::vector<vk::WriteDescriptorSet>   m_descriptor_writes;
+	vk::DescriptorSetLayout               m_pssr_descriptor_layout = nullptr;
+	vk::PipelineLayout                    m_pssr_pipeline_layout   = nullptr;
+	vk::Pipeline                          m_pssr_pipeline          = nullptr;

 	friend class CommandProcessor;
 	friend struct RenderExecutorTestAccess;
@@ -147,6 +147,7 @@ void RenderContext::RunGarbageCollector() {
 		m_fault_process_pending = false;
 		m_buffer_cache.ProcessFaultBuffer();
 	}
+	m_image_heap.RunGarbageCollector();
 	m_texture_cache.ProcessDownloadImages();
 	m_texture_cache.RunGarbageCollector();
 	m_buffer_cache.RunGarbageCollector();
@@ -52,7 +52,7 @@ std::pair<int32_t, uint32_t> ResolveDrawOffsets(uint32_t index_offset,
 	                                           const ShaderVertexInputInfo& vs_input_info) {
 	auto     vertex_offset   = static_cast<int32_t>(index_offset);
 	uint32_t instance_offset = 0;
-	if (!vs_input_info.fetch_embedded) {
+	if (!vs_input_info.has_dynamic_vertex_fetch) {
 		return {vertex_offset, instance_offset};
 	}

@@ -403,6 +403,20 @@ static void UpdateGraphicsDynamicState(const CommandBuffer& buffer, DynamicState
 		dynamic_state.SetDepthBias(constant_factor, poly_offset.clamp, slope_factor);
 	}

+	// Rect lists are drawn without culling.
+	vk::CullModeFlags cull_mode {};
+	if (!Prospero::IsRectList(buffer.GetUserConfig().GetPrimType())) {
+		if (mode.cull_front) {
+			cull_mode |= vk::CullModeFlagBits::eFront;
+		}
+		if (mode.cull_back) {
+			cull_mode |= vk::CullModeFlagBits::eBack;
+		}
+	}
+	dynamic_state.SetCullMode(cull_mode);
+	dynamic_state.SetFrontFace(mode.face ? vk::FrontFace::eClockwise
+	                                     : vk::FrontFace::eCounterClockwise);
+
 	dynamic_state.SetStencilTestEnabled(depth.stencil_test_enable);
 	if (depth.stencil_test_enable) {
 		const auto& front = depth.stencil_front;
@@ -1236,24 +1250,36 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff
 	}
 }

-// Vulkan offsets for a prepared draw, or nullopt when its indirect records need CPU counts, as
-// mesh and quad-list draws do. Indirect records carry their own offsets.
+// Vulkan offsets for a prepared draw, or nullopt when its indirect records need the CPU: mesh
+// and quad-list draws for counts, tessellation draws for the index/SGPR prolog. Indirect records
+// carry their own offsets.
 static std::optional<DrawEmitInfo> ResolveDrawEmit(const HW::UserConfig&        ucfg,
                                                   const ShaderVertexInputInfo& vs,
                                                   DrawOffsetSource             source,
                                                   int32_t                      first_vertex,
                                                   uint32_t                     first_instance,
                                                   const DrawIndirectArgs&      indirect) {
+	if ((indirect.address != 0 || source == DrawOffsetSource::IndirectArgs) &&
+	    ucfg.GetIndexOffset() != 0) {
+		static std::once_flag warning_once;
+		std::call_once(warning_once, [&] {
+			Log::WriteToConsoleAndLog(fmt::format(
+			    "GPU: indirect draw ignores GE_INDX_OFFSET=0x{:08x}; geometry may be incorrect.\n",
+			    ucfg.GetIndexOffset()));
+		});
+	}
 	if (indirect.address != 0) {
-		if (vs.stage.module->info.stage == ShaderType::Mesh ||
-		    ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy) {
+		if (!vs.indirect_draw ||
+		    vs.stage.module->info.stage == ShaderType::Mesh ||
+		    ucfg.GetPrimType() == Prospero::PrimitiveType::kQuadListLegacy ||
+		    vs.indirect_offsets != indirect.offsets) {
 			return std::nullopt;
 		}
 		return DrawEmitInfo {.indirect = indirect};
 	}
 	if (source == DrawOffsetSource::IndirectArgs) {
 		// Native fetches consume the patched SGPRs; rewritten fetches need Vulkan offsets.
-		return vs.fetch_embedded ? DrawEmitInfo {first_vertex, first_instance} : DrawEmitInfo {};
+		return vs.has_dynamic_vertex_fetch ? DrawEmitInfo {first_vertex, first_instance} : DrawEmitInfo {};
 	}
 	const auto [vertex_offset, instance_offset] = ResolveDrawOffsets(ucfg.GetIndexOffset(), vs);
 	return DrawEmitInfo {vertex_offset + first_vertex, instance_offset};
@@ -1310,7 +1336,8 @@ bool RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,

 	hw_check(buffer);

-	DrawRenderState state {};
+	DrawRenderState state {.indirect_offsets = args.indirect.offsets,
+	                       .indirect_draw    = args.indirect.address != 0};
 	if (!GetDrawTopology(ucfg, state.topology)) {
 		return true;
 	}
@@ -1415,7 +1442,8 @@ bool RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, const D
 	const DrawCallInfo draw {CommandBufferDebugOp::DrawIndexAuto,
 	                         args.vertex_count, args.instance_count, args.first_instance};

-	DrawRenderState state {};
+	DrawRenderState state {.indirect_offsets = args.indirect.offsets,
+	                       .indirect_draw    = args.indirect.address != 0};
 	if (!GetDrawTopology(ucfg, state.topology)) {
 		ResetBindings();
 		return true;
@@ -0,0 +1,184 @@
+#include "common/assert.h"
+#include "common/threads.h"
+#include "gpu_blit_shaders/gpu_pssr_linear_spv.h"
+#include "graphics/guest_gpu/pssr.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/image/image.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
+#include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/renderContext.h"
+#include "graphics/shader/shaderBindings.h"
+
+#include <algorithm>
+#include <array>
+
+namespace Libs::Graphics {
+namespace {
+
+struct PssrParameters {
+	uint32_t input_size[2];
+	uint32_t output_size[2];
+	uint32_t output_channels[4];
+};
+
+bool IsInputFormat(Prospero::BufferFormat format) {
+	return format == Prospero::BufferFormat::k11_11_10Float ||
+	       format == Prospero::BufferFormat::k10_11_11Float ||
+	       format == Prospero::BufferFormat::k16_16_16_16Float;
+}
+
+} // namespace
+
+RenderExecutor::~RenderExecutor() {
+	const auto device = m_context.GetGraphics().device;
+	device.destroyPipeline(m_pssr_pipeline, nullptr);
+	device.destroyPipelineLayout(m_pssr_pipeline_layout, nullptr);
+	device.destroyDescriptorSetLayout(m_pssr_descriptor_layout, nullptr);
+}
+
+void RenderExecutor::ProcessPssr(CommandBuffer& buffer, const Pssr::Request& request) {
+	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
+	Common::LockGuard lock(m_context.GetMutex());
+	ShaderTextureResource input_descriptor {}, output_descriptor {};
+	std::copy_n(request.input, 8, input_descriptor.fields);
+	std::copy_n(request.output, 8, output_descriptor.fields);
+	const auto is_2d = [](const ShaderTextureResource& descriptor) {
+		return !descriptor.IsNull() &&
+		       (descriptor.Type() == Prospero::ImageType::kColor2D ||
+		        descriptor.Type() == Prospero::ImageType::kColor2DArray) &&
+		       descriptor.TileMode() != Prospero::TileMode::kDepth;
+	};
+	EXIT_IF(!is_2d(input_descriptor) || !is_2d(output_descriptor) ||
+	        !IsInputFormat(input_descriptor.Format()) ||
+	        output_descriptor.Format() != Prospero::BufferFormat::k11_11_10Float ||
+	        output_descriptor.TileMode() != Prospero::TileMode::kStandard256B ||
+	        output_descriptor.BaseLevel() != 0 || output_descriptor.LastLevel() != 0 ||
+	        output_descriptor.MaxMip() != 0 || output_descriptor.BaseArray5() != 0 ||
+	        output_descriptor.Depth() != 0 || request.width == 0 || request.height == 0);
+	const auto input_width = std::max(1u, (input_descriptor.Width5() + 1u) >>
+	                                         input_descriptor.BaseLevel());
+	const auto input_height = std::max(1u, (input_descriptor.Height5() + 1u) >>
+	                                          input_descriptor.BaseLevel());
+	EXIT_IF(request.width > input_width || request.height > input_height);
+	PssrParameters parameters {{request.width, request.height},
+	                           {output_descriptor.Width5() + 1u, output_descriptor.Height5() + 1u},
+	                           {4, 4, 4, 4}};
+	for (uint32_t component = 0; component < 3; ++component) {
+		for (uint32_t source = 0; source < 4; ++source) {
+			if (GetDstSel(output_descriptor.DstSelXYZW(), source) == 4u + component) {
+				parameters.output_channels[component] = source;
+				break;
+			}
+		}
+	}
+	const auto resolve = [&](const uint32_t* words, bool output) {
+		ShaderRecompiler::IR::ImageResource resource {};
+		resource.resource_class = output ? ShaderRecompiler::IR::ImageResourceClass::Storage
+		                                 : ShaderRecompiler::IR::ImageResourceClass::Sampled;
+		resource.numeric_class = Prospero::TextureNumericClass::Float;
+		resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
+		resource.read = !output;
+		resource.written = output;
+		ShaderRecompiler::IR::DescriptorValue value {};
+		std::copy_n(words, 8, value.dwords.begin());
+		value.dword_count = 8;
+		auto binding = ResolveTexture(resource, value);
+		binding.desc.view_info.level_count = 1;
+		binding.desc.view_info.min_lod = 0;
+		return binding;
+	};
+	auto& cache = m_context.GetTextureCache();
+	auto input = resolve(request.input, false);
+	BindImage(input.image_id, false);
+	auto output = resolve(request.output, true);
+	BindImage(output.image_id, true);
+	EXIT_IF(input.desc.info.data.address < output.desc.info.data.End() &&
+	        output.desc.info.data.address < input.desc.info.data.End());
+	const auto acquire = [&](TextureBinding& binding, const uint32_t* words, bool storage) {
+		auto& previous = cache.GetImage(binding.image_id);
+		if (!previous.registered || previous.binding.needs_rebind) {
+			previous.binding = {};
+			binding = resolve(words, storage);
+			BindImage(binding.image_id, storage);
+		}
+		binding.image_view = cache.FindTexture(binding.image_id, binding.desc);
+	};
+	acquire(input, request.input, false);
+	acquire(output, request.output, true);
+	auto& source = cache.GetImage(input.image_id);
+	auto& destination = cache.GetImage(output.image_id);
+	const auto& source_view = input.desc.view_info;
+	const auto& destination_view = output.desc.view_info;
+	EXIT_IF(source.info.IsDepth() || destination.info.IsDepth() ||
+	        source.backing.samples != 1 || destination.backing.samples != 1 ||
+	        (input.image_id == output.image_id &&
+	         source_view.base_level == destination_view.base_level &&
+	         source_view.base_layer == destination_view.base_layer));
+
+	auto& graphics = m_context.GetGraphics();
+	if (m_pssr_pipeline == nullptr) {
+		const std::array bindings {
+		    vk::DescriptorSetLayoutBinding {0, vk::DescriptorType::eSampledImage, 1,
+		                                    vk::ShaderStageFlagBits::eCompute},
+		    vk::DescriptorSetLayoutBinding {1, vk::DescriptorType::eStorageImage, 1,
+		                                    vk::ShaderStageFlagBits::eCompute}};
+		vk::DescriptorSetLayoutCreateInfo descriptor_info {};
+		descriptor_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
+		descriptor_info.bindingCount = bindings.size();
+		descriptor_info.pBindings = bindings.data();
+		RequireVulkanSuccess(graphics.device.createDescriptorSetLayout(
+		                         &descriptor_info, nullptr, &m_pssr_descriptor_layout),
+		                     "create upscaling descriptor layout");
+		const vk::PushConstantRange push {vk::ShaderStageFlagBits::eCompute, 0,
+		                                  sizeof(PssrParameters)};
+		vk::PipelineLayoutCreateInfo layout_info {};
+		layout_info.setLayoutCount = 1;
+		layout_info.pSetLayouts = &m_pssr_descriptor_layout;
+		layout_info.pushConstantRangeCount = 1;
+		layout_info.pPushConstantRanges = &push;
+		RequireVulkanSuccess(graphics.device.createPipelineLayout(
+		                         &layout_info, nullptr, &m_pssr_pipeline_layout),
+		                     "create upscaling pipeline layout");
+		const auto shader = CompileSPV(GPU_PSSR_LINEAR_SPV, graphics.device);
+		vk::ComputePipelineCreateInfo create {};
+		create.stage.stage = vk::ShaderStageFlagBits::eCompute;
+		create.stage.module = shader;
+		create.stage.pName = "main";
+		create.layout = m_pssr_pipeline_layout;
+		const auto result = graphics.device.createComputePipelines(
+		    nullptr, 1, &create, nullptr, &m_pssr_pipeline);
+		graphics.device.destroyShaderModule(shader, nullptr);
+		RequireVulkanSuccess(result, "create upscaling pipeline");
+	}
+	buffer.EndRendering();
+	const auto command = buffer.Handle();
+	source.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderRead,
+	               ImageSubresourceRange {source_view.base_level, 1, source_view.base_layer, 1},
+	               command);
+	destination.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite,
+	                    ImageSubresourceRange {destination_view.base_level, 1,
+	                                           destination_view.base_layer, 1}, command);
+	const std::array images {
+	    vk::DescriptorImageInfo {nullptr, input.image_view, vk::ImageLayout::eGeneral},
+	    vk::DescriptorImageInfo {nullptr, output.image_view, vk::ImageLayout::eGeneral}};
+	std::array<vk::WriteDescriptorSet, 2> writes {};
+	for (uint32_t i = 0; i < writes.size(); ++i) {
+		writes[i].dstBinding = i;
+		writes[i].descriptorCount = 1;
+		writes[i].descriptorType = i == 0 ? vk::DescriptorType::eSampledImage
+		                                 : vk::DescriptorType::eStorageImage;
+		writes[i].pImageInfo = &images[i];
+	}
+	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, m_pssr_pipeline_layout,
+	                             0, writes.size(), writes.data());
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_pssr_pipeline);
+	command.pushConstants(m_pssr_pipeline_layout, vk::ShaderStageFlagBits::eCompute,
+	                      0, sizeof(parameters), &parameters);
+	command.dispatch((parameters.output_size[0] + 7) / 8,
+	                 (parameters.output_size[1] + 7) / 8, 1);
+	ShaderAccessBarrier(command, vk::PipelineStageFlagBits::eComputeShader);
+	ResetBindings();
+}
+
+} // namespace Libs::Graphics
@@ -0,0 +1,33 @@
+#version 450 core
+#extension GL_EXT_samplerless_texture_functions : require
+
+layout(local_size_x = 8, local_size_y = 8) in;
+layout(binding = 0) uniform texture2D source;
+layout(binding = 1) writeonly uniform image2D destination;
+layout(push_constant) uniform Parameters {
+	uvec2 input_size;
+	uvec2 output_size;
+	uvec4 output_channels;
+} parameters;
+
+vec4 fetch_color(ivec2 coordinate) {
+	return texelFetch(source, clamp(coordinate, ivec2(0), ivec2(parameters.input_size) - 1), 0);
+}
+
+void main() {
+	uvec2 output_coordinate = gl_GlobalInvocationID.xy;
+	if (any(greaterThanEqual(output_coordinate, parameters.output_size))) return;
+	vec2 input_coordinate = (vec2(output_coordinate) + 0.5) *
+	                        vec2(parameters.input_size) / vec2(parameters.output_size) - 0.5;
+	ivec2 lower = ivec2(floor(input_coordinate));
+	vec2 weight = fract(input_coordinate);
+	vec4 color = mix(mix(fetch_color(lower), fetch_color(lower + ivec2(1, 0)), weight.x),
+	                 mix(fetch_color(lower + ivec2(0, 1)), fetch_color(lower + ivec2(1, 1)), weight.x),
+	                 weight.y);
+	vec4 stored = vec4(0.0);
+	for (uint channel = 0; channel < 3; ++channel) {
+		uint selected = parameters.output_channels[channel];
+		if (selected < 4) stored[channel] = color[selected];
+	}
+	imageStore(destination, ivec2(output_coordinate), stored);
+}
@@ -1,4 +1,4 @@
-// ColorTransform values match imageInfo.h. Preserve the floating-point bit fields exactly.
+// ColorTransform values match imageInfo.h. Preserve the component bit fields exactly.
 uvec2 transform_color(uvec2 value, uint transform, bool to_host) {
 	switch (transform) {
 		case 1u:
@@ -15,6 +15,13 @@ uvec2 transform_color(uvec2 value, uint transform, bool to_host) {
 			}
 			break;
 		}
+		case 3u:
+			if (to_host) {
+				value.x = (value.x >> 2u) | (value.x << 30u);
+			} else {
+				value.x = (value.x << 2u) | (value.x >> 30u);
+			}
+			break;
 	}
 	return value;
 }
@@ -9,6 +9,7 @@ layout(set = 0, binding = 1, std430) writeonly buffer Predicates { uint data[];
 layout(push_constant, std430) uniform Check {
 	uint metadata_base;
 	uint words;
+	uint stride;
 	uint keys;
 	uint expand;
 } params;
@@ -20,7 +21,7 @@ shared uint mismatch;

 void main() {
 	const uint slice = gl_WorkGroupID.x;
-	const uint base  = (params.metadata_base >> 2u) + slice * params.words;
+	const uint base  = (params.metadata_base >> 2u) + slice * params.stride;
 	// Each invocation reads, and may rewrite, only its own words.
 	if (gl_LocalInvocationIndex == 0u) {
 		key      = metadata.data[base] & 0xffu;
@@ -1,10 +1,11 @@
 #version 450
+#extension GL_EXT_shader_16bit_storage : require

 layout(constant_id = 0) const uint D32 = 0u;
 layout(local_size_x = 64) in;

 layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } input_buffer;
-layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } output_buffer;
+layout(set = 0, binding = 1, std430) buffer Output { uint16_t data[]; } output_buffer;

 layout(push_constant, std430) uniform Conversion {
 	uint src_base;
@@ -32,9 +33,5 @@ void main() {
 	}
 	uint src = params.src_base + y * params.source_pitch + x * 4u;
 	uint dst = params.dst_base + y * params.target_pitch + x * 2u;
-	uint shift = (dst & 2u) * 8u;
-	uint mask = 0xffffu << shift;
-	uint word = dst >> 2u;
-	atomicAnd(output_buffer.data[word], ~mask);
-	atomicOr(output_buffer.data[word], decode(input_buffer.data[src >> 2u]) << shift);
+	output_buffer.data[dst >> 1u] = uint16_t(decode(input_buffer.data[src >> 2u]));
 }
@@ -151,7 +151,8 @@ bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanIm
 	EXIT_IF(allocator == nullptr || image.image != nullptr || image.allocation != nullptr);

 	VmaAllocationCreateInfo alloc_info {};
-	alloc_info.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+	alloc_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
+	alloc_info.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;

 	vk::Image::CType native_image = VK_NULL_HANDLE;
 	const auto        result       = static_cast<vk::Result>(
@@ -533,16 +533,16 @@ void DrawMemoryAndIo(const Snapshot& shown) {
 		    Bytes(guest.pooled_allocated) + " (" + Bytes(guest.pool_committed) + " committed)");
 		Row("Flexible", Bytes(guest.flexible_used) + " / " + Bytes(guest.flexible_total));
 		Row("Extended", Bytes(guest.automatic_allocated));
-		// Past the pool size the guest sees no page-table entries available.
-		const auto entries = std::max(guest.cpu_page_entries, guest.gpu_page_entries);
+		// Past its total a side sees no page-table entries available.
+		const auto percent = std::max(guest.cpu_page_entries * 100 / guest.cpu_page_entries_total,
+		                              guest.gpu_page_entries * 100 / guest.gpu_page_entries_total);
 		Row("Page table (2 MB)",
-		    Fmt("CPU %llu  GPU %llu  / %llu",
+		    Fmt("CPU %llu / %llu  GPU %llu / %llu",
 		        static_cast<unsigned long long>(guest.cpu_page_entries),
+		        static_cast<unsigned long long>(guest.cpu_page_entries_total),
 		        static_cast<unsigned long long>(guest.gpu_page_entries),
-		        static_cast<unsigned long long>(guest.page_entries_total)),
-		    entries > guest.page_entries_total            ? BAD
-		    : entries * 10 > guest.page_entries_total * 9 ? MEDIUM
-		                                                  : GOOD);
+		        static_cast<unsigned long long>(guest.gpu_page_entries_total)),
+		    LevelColor(static_cast<double>(percent), 90, 100));
 		ImGui::EndTable();
 	}
 	if (BeginSection("Emulator process")) {
@@ -221,7 +221,7 @@ public:
 	void WaitForSubmitSlot(VideoOutConfig& cfg);
 	bool Flip(uint32_t micros);
 	void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out);
-	bool IsSafeForRendering(VideoOutConfig& cfg, int index);
+	void Wait(VideoOutConfig& cfg, int index);

 private:
 	enum class RequestState { Reserved, Recording, Ready, Presenting };
@@ -240,7 +240,6 @@ private:
 		Graphics::Presenter::Frame* frame;
 		bool                        premultiplied_alpha;
 	};
-	void SignalDoneLocked();

 	Graphics::Presenter& m_presenter;
 	Common::Mutex        m_mutex;
@@ -251,7 +250,6 @@ private:
 	std::list<Request>   m_cpu_requests;
 	std::list<Request>   m_cancelled_requests;
 	bool                 m_processing      = false;
-	bool                 m_gpu_waiting     = false;
 	uint64_t             m_next_request_id = 1;
 };

@@ -955,7 +953,7 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe
 				}
 			}
 		}
-		SignalDoneLocked();
+		m_done_cond_var.SignalAll();
 	}
 	if (cfg.flip_status.flipPendingNum >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
 		Common::DebugCounters::Add(Common::DebugCounters::Counter::FlipsRejected);
@@ -1035,7 +1033,7 @@ void FlipQueue::Cancel(VideoOutConfig& cfg) {
 			it = queue->erase(it);
 		}
 	}
-	SignalDoneLocked();
+	m_done_cond_var.SignalAll();
 	m_submit_slot_cond_var.SignalAll();
 	m_submit_cond_var.SignalAll();
 	m_mutex.Unlock();
@@ -1207,24 +1205,17 @@ void FlipQueue::WaitForSubmitSlot(VideoOutConfig& cfg) {
 	}
 }

-void FlipQueue::SignalDoneLocked() {
-	m_done_cond_var.SignalAll();
-	// A stalled GPU cannot drain and shut down without re-checking under m_mutex, so it is alive here.
-	if (m_gpu_waiting) {
-		m_gpu_waiting = false;
-		m_presenter.Renderer().GetGpu().Wake();
-	}
-}
-
-// A display buffer is safe for rendering once it is neither displayed nor waiting to be.
-bool FlipQueue::IsSafeForRendering(VideoOutConfig& cfg, int index) {
+void FlipQueue::Wait(VideoOutConfig& cfg, int index) {
 	Common::LockGuard lock(m_mutex);
-	auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
-	const bool safe = cfg.flip_status.currentBuffer != index &&
-	                  std::none_of(m_requests.begin(), m_requests.end(), matches) &&
-	                  std::none_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
-	m_gpu_waiting = !safe;
-	return safe;
+
+	auto has_request = [this, &cfg, index] {
+		auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
+		return std::any_of(m_requests.begin(), m_requests.end(), matches) ||
+		       std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
+	};
+	while (has_request()) {
+		m_done_cond_var.Wait(&m_mutex);
+	}
 }

 bool FlipQueue::Flip(uint32_t micros) {
@@ -1320,7 +1311,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 		it = m_requests.erase(it);
 	}
 	m_processing = false;
-	SignalDoneLocked();
+	m_done_cond_var.SignalAll();
 	m_submit_slot_cond_var.SignalAll();
 	m_mutex.Unlock();
 	for (size_t i = count; i != 0; i--) {
@@ -1703,12 +1694,12 @@ void VideoOutDriver::WaitForSubmitSlot(int handle) {
 	m_impl->GetFlipQueue().WaitForSubmitSlot(*cfg);
 }

-bool VideoOutDriver::IsSafeForRendering(int handle, int index) {
+void VideoOutDriver::WaitFlipDone(int handle, int index) {
 	auto* ctx = m_impl->Get(handle);
 	EXIT_IF(ctx == nullptr);

 	EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index));
-	return m_impl->GetFlipQueue().IsSafeForRendering(*ctx, index);
+	m_impl->GetFlipQueue().Wait(*ctx, index);
 }

 KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) {
@@ -38,7 +38,7 @@ public:
 	void CompleteFlip(uint64_t request_id);
 	void SubmitFlipPreparation(uint64_t request_id);
 	void WaitForSubmitSlot(int handle);
-	[[nodiscard]] bool IsSafeForRendering(int handle, int index);
+	void WaitFlipDone(int handle, int index);

 	[[nodiscard]] Impl& State() noexcept;

@@ -290,7 +290,8 @@ public:
 private:
 	vk::SurfaceFormatKHR SelectFormat();
 	void Destroy();
-	void DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer);
+	void DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer,
+	                 const vk::Rect2D& rect);

 	WindowContext&   m_window;
 	vk::SwapchainKHR m_handle = nullptr;
@@ -637,7 +638,23 @@ bool Swapchain::PrepareSystemOverlay() {
 	return m_system_overlay->PrepareFrame(m_extent, m_format, ImageCount());
 }

-void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer) {
+static vk::Rect2D FitAspect(const vk::Extent3D& source, const vk::Extent2D& target) {
+	const uint64_t src_w = std::max(source.width, 1u);
+	const uint64_t src_h = std::max(source.height, 1u);
+	uint32_t       w     = target.width;
+	uint32_t       h     = target.height;
+	if (src_w * h > src_h * w) {
+		h = std::max<uint32_t>(1, static_cast<uint32_t>(src_h * w / src_w));
+	} else {
+		w = std::max<uint32_t>(1, static_cast<uint32_t>(src_w * h / src_h));
+	}
+	return {{static_cast<int32_t>((target.width - w) / 2),
+	         static_cast<int32_t>((target.height - h) / 2)},
+	        {w, h}};
+}
+
+void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer,
+                            const vk::Rect2D& rect) {
 	auto device = m_window.graphic_ctx.device;
 	if (m_overlay_pipeline == nullptr) {
 		vk::SamplerCreateInfo sampler {};
@@ -756,12 +773,14 @@ void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& l
 	rendering.colorAttachmentCount = 1;
 	rendering.pColorAttachments    = &attachment;
 	command.beginRendering(&rendering);
-	const vk::Viewport viewport {
-	    0.0f, 0.0f, static_cast<float>(m_extent.width), static_cast<float>(m_extent.height),
-	    0.0f, 1.0f};
-	const vk::Rect2D scissor {{0, 0}, m_extent};
+	const vk::Viewport viewport {static_cast<float>(rect.offset.x),
+	                             static_cast<float>(rect.offset.y),
+	                             static_cast<float>(rect.extent.width),
+	                             static_cast<float>(rect.extent.height),
+	                             0.0f,
+	                             1.0f};
 	command.setViewport(0, 1, &viewport);
-	command.setScissor(0, 1, &scissor);
+	command.setScissor(0, 1, &rect);
 	command.draw(3, 1, 0, 0);
 	command.endRendering();
 }
@@ -799,6 +818,24 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame*
 	                           vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags {}, 0,
 	                           nullptr, 0, nullptr, 1, &to_transfer);

+	const auto* aspect_source = source != nullptr ? source : overlay.frame;
+	const auto  game_rect     = !Config::StretchEnabled() && aspect_source != nullptr
+	                               ? FitAspect(aspect_source->image.extent, m_extent)
+	                               : vk::Rect2D {{0, 0}, m_extent};
+	if (source == nullptr || game_rect.extent != m_extent) {
+		const vk::ClearColorValue black {};
+		vk_command.clearColorImage(m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal,
+		                           &black, 1, &to_transfer.subresourceRange);
+		if (source != nullptr) {
+			vk::ImageMemoryBarrier after_clear = to_transfer;
+			after_clear.srcAccessMask          = vk::AccessFlagBits::eTransferWrite;
+			after_clear.oldLayout              = vk::ImageLayout::eTransferDstOptimal;
+			vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+			                           vk::PipelineStageFlagBits::eTransfer,
+			                           vk::DependencyFlags {}, 0, nullptr, 0, nullptr, 1,
+			                           &after_clear);
+		}
+	}
 	if (source != nullptr) {
 		vk::ImageBlit region {};
 		region.srcSubresource.aspectMask     = vk::ImageAspectFlagBits::eColor;
@@ -812,16 +849,14 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame*
 		region.dstSubresource.mipLevel       = 0;
 		region.dstSubresource.baseArrayLayer = 0;
 		region.dstSubresource.layerCount     = 1;
-		region.dstOffsets[1].x               = static_cast<int>(m_extent.width);
-		region.dstOffsets[1].y               = static_cast<int>(m_extent.height);
+		region.dstOffsets[0].x = game_rect.offset.x;
+		region.dstOffsets[0].y = game_rect.offset.y;
+		region.dstOffsets[1].x = game_rect.offset.x + static_cast<int>(game_rect.extent.width);
+		region.dstOffsets[1].y = game_rect.offset.y + static_cast<int>(game_rect.extent.height);
 		region.dstOffsets[1].z               = 1;
 		vk_command.blitImage(source->image.image, vk::ImageLayout::eTransferSrcOptimal,
 		                     m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal, 1,
 		                     &region, vk::Filter::eLinear);
-	} else {
-		const vk::ClearColorValue black {};
-		vk_command.clearColorImage(m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal,
-		                           &black, 1, &to_transfer.subresourceRange);
 	}

 	vk::ImageMemoryBarrier to_present {};
@@ -848,7 +883,7 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame*
 	                           &to_present);
 	if (draw_attachment) {
 		if (draw_overlay) {
-			DrawOverlay(vk_command, overlay);
+			DrawOverlay(vk_command, overlay, game_rect);
 			if (draw_system_overlay) {
 				to_present.srcAccessMask = vk::AccessFlagBits::eColorAttachmentWrite;
 				to_present.oldLayout     = vk::ImageLayout::eColorAttachmentOptimal;
@@ -51,6 +51,7 @@ struct VulkanExtensions {
 vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
 	vk::PhysicalDeviceVulkan11Features features {};
 	features.storageBuffer16BitAccess = VK_TRUE;
+	features.shaderDrawParameters     = VK_TRUE;
 	return features;
 }

@@ -278,6 +279,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 		              required_features12.samplerMirrorClampToEdge);
 		check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
 		              WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
+		check_feature(features11.shaderDrawParameters, "shaderDrawParameters",
+		              WindowContext::RequiredVulkan11Features().shaderDrawParameters);
 		check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
 		              required_features12.storageBuffer8BitAccess);
 		check_feature(features12.timelineSemaphore, "timelineSemaphore",
@@ -814,7 +814,7 @@ TranslateResult TranslateProgram(const Decoder::Program& decoded, const CompileO
 	const auto& cfg = *selected_cfg;
 	Frontend::EmbeddedFetchPlan embedded_fetch;
 	if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
-	    options.input_info.vertex->fetch_embedded) {
+	    options.input_info.vertex->has_dynamic_vertex_fetch) {
 		embedded_fetch = DetectEmbeddedVertexFetch(
 		    decoded, *options.input_info.vertex, options.user_data_base,
 		    static_cast<uint32_t>(options.user_data.size()), options.wave_size);
@@ -156,11 +156,14 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 						if (rhs.constant == 0u && third.constant == 8u) value = constant(0u);
 					}
 					break;
-				case Opcode::V_AND_B32:
-					if ((lhs.kind == Kind::PackedControlPoint && rhs == constant(0xffu)) ||
-					    (rhs.kind == Kind::PackedControlPoint && lhs == constant(0xffu)))
-						value = constant(0u);
+				case Opcode::V_AND_B32: {
+					const auto mask = lhs.kind == Kind::PackedControlPoint ? rhs
+					                : rhs.kind == Kind::PackedControlPoint ? lhs
+					                                                       : Address {};
+					if (mask == constant(0xffu)) value = constant(0u);
+					if (mask == constant(0x1f00u)) value = {Kind::Affine, 256u, 0u};
 					break;
+				}
 				case Opcode::V_ADD_NC_U32: value = add(lhs, rhs); break;
 				case Opcode::V_MUL_LO_U32: value = multiply(lhs, rhs); break;
 				case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break;
@@ -30,16 +30,7 @@ uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) {
 uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
                  Vec3 origin, Vec3 direction) {
 	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
-	const auto zero = ConstantF32Value(s, 0.0f);
-	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
-		const auto value = Binary(s, code, f, x, y);
-		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
-		return value;
-	};
 	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
-	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
-	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
-	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
 	const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); };
 	// The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0).
 	const std::array vertices {
@@ -56,6 +47,28 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
 			    NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis))));
 	}
 	const auto flag = NodeWord(s, block, ConstantU32(s, 15));
+	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
+	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
+	return EmitBvhTriangle(s, positions, origin, direction, remap,
+	    Binary(s, spv::OpIAdd, u, flag, kind), bary);
+}
+
+} // namespace
+
+uint32_t EmitBvhTriangle(EmitterState& s, const std::array<Vec3, 3>& positions,
+                         Vec3 origin, Vec3 direction, uint32_t remap,
+                         uint32_t triangle_id, uint32_t bary) {
+	const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s);
+	const auto zero = ConstantF32Value(s, 0.0f);
+	const auto op = [&](spv::Op code, uint32_t x, uint32_t y) {
+		const auto value = Binary(s, code, f, x, y);
+		s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction);
+		return value;
+	};
+	const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); };
+	const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); };
+	const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); };
+	const auto abs = [&](uint32_t x) { return EmitGlsl<GLSLstd450FAbs, IR::Type::F32>(s, x); };
 	const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1]));
 	const auto z_largest = cmp(spv::OpFOrdLessThan,
 	    Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2]));
@@ -112,8 +125,6 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
 		    cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right);
 		missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded));
 	}
-	const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag,
-	    Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8)));
 	const auto coordinate = [&](uint32_t shift) {
 		const auto index = Binary(s, spv::OpBitwiseAnd, u,
 		    Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3));
@@ -124,11 +135,13 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary,
 	const std::array result {
 		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)),
 		Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)),
-		Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)),
+		Select(s, u, bary, coordinate(0), triangle_id),
 		Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) };
 	return Vector(s, TypeU32Vector(s, 4), result);
 }

+namespace {
+
 std::array<uint32_t, 6> Box(EmitterState& s, uint32_t first, uint32_t second,
                           uint32_t half, uint32_t child) {
 	const auto f = TypeF32(s);
@@ -63,7 +63,8 @@ uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t c
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
 	    kind == IR::StageInputKind::InvocationId || kind == IR::StageInputKind::PrimitiveId ||
-	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
+	    kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId ||
+	    kind == IR::StageInputKind::BaseVertex || kind == IR::StageInputKind::BaseInstance) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
 		state.builder.AddFunction(spv::OpLoad, TypeI32(state), value, variable);
@@ -858,6 +859,10 @@ uint32_t EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst) {
 	return EmitSubgroupReduceU32<spv::OpGroupNonUniformUMax, EmitUMax32>(ctx, inst);
 }

+uint32_t EmitSubgroupOrU32(ValueEmitContext& ctx, const IR::Inst& inst) {
+	return EmitSubgroupReduceU32<spv::OpGroupNonUniformBitwiseOr, EmitOrU32>(ctx, inst);
+}
+
 uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&      state = ctx.state;
 	const auto hit   = state.builder.AllocateId();
@@ -173,7 +173,7 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 		state.builder.AddFunction(spv::OpLoad, scalar_type, raw, pointer);
 	}

-	if (kind == VertexInputScalarKind::Uint) {
+	if (kind == Format::ComponentType::Uint) {
 		return raw;
 	}

@@ -38,7 +38,8 @@ bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {

 uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
                    uint32_t component) {
-	const auto layout = Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component);
+	const auto layout = Decoder::ImageAddressComponentLayout(mem.image_sample_flags,
+	                                                         mem.image_dimension, component);
 	const auto packed = layout.bit_offset / 32u;
 	if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0);
 	auto value = ctx.Def(address.Arg(packed));
@@ -55,10 +56,11 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::

 uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
                    uint32_t component) {
-	const auto value = AddressU32(ctx, mem, address, component);
-	return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u
-	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
+	const auto value  = AddressU32(ctx, mem, address, component);
+	const auto layout = Decoder::ImageAddressComponentLayout(mem.image_sample_flags,
+	                                                         mem.image_dimension, component);
+	return layout.bit_width == 16u ? EmitF16BitsToF32(ctx.state, value)
+	                               : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
 }

 ImageSampleLayout Layout(const IR::MemoryInfo& mem) {
@@ -811,6 +813,25 @@ uint32_t EmitImageResult(ValueEmitContext& ctx, const IR::Inst& inst, const Emit
 	return mixed ? value : result_bits(mem.resource, value);
 }

+// Ordinals outside the table select candidates[0]. Past ordinal 0, a consecutive table is an
+// offset of ordinal - 1, which wraps at 0 so that one unsigned compare checks both ends.
+uint32_t SamplerArrayIndex(EmitterState& state, uint32_t ordinal,
+                           const std::vector<uint32_t>& candidates) {
+	const auto count = static_cast<uint32_t>(candidates.size());
+	const auto gap = std::adjacent_find(candidates.begin() + 1, candidates.end(),
+	    [](uint32_t lhs, uint32_t rhs) { return rhs != lhs + 1u; });
+	if (count == 1u || gap != candidates.end()) {
+		return EmitIndexSwitch(state, ordinal, count, TypeU32(state),
+		                       [&](uint32_t item) { return ConstantU32(state, candidates[item]); });
+	}
+	const auto offset = Binary(state, spv::OpISub, TypeU32(state), ordinal, ConstantU32(state, 1u));
+	const auto in_table = Binary(state, spv::OpULessThan, TypeBool(state), offset,
+	                             ConstantU32(state, count - 1u));
+	const auto index = Binary(state, spv::OpIAdd, TypeU32(state), offset,
+	                          ConstantU32(state, candidates[1]));
+	return Select(state, TypeU32(state), in_table, index, ConstantU32(state, candidates[0]));
+}
+
 struct ImageSampler {
 	uint32_t id = 0;
 	bool dynamic = false;
@@ -846,9 +867,7 @@ std::array<ImageSampler, 2> LoadImageSamplers(ValueEmitContext& ctx, const IR::I
 				selected = EmitIndirectResourceIndex(state, selected,
 				    samplers[index].indirect_mapping_offset,
 				    samplers[index].indirect_search_iterations, 0u);
-			array_index = EmitIndexSwitch(
-			    state, selected, static_cast<uint32_t>(candidates.size()),
-			    TypeU32(state), [&](uint32_t ordinal) { return ConstantU32(state, candidates[ordinal]); });
+			array_index = SamplerArrayIndex(state, selected, candidates);
 		}
 		sampler = {LoadSamplerDescriptor(state, index, array_index), array_index != 0u};
 	};
@@ -236,6 +236,7 @@ uint32_t              EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& i
 uint32_t              EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSubgroupUMinU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitSubgroupUMaxU32(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitSubgroupOrU32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -291,6 +292,7 @@ inline constexpr auto EmitBufferAtomicIAdd32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicISub32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicSMin32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMin32    = EmitAtomic32;
+inline constexpr auto EmitBufferAtomicUMin64    = EmitBufferAtomic64;
 inline constexpr auto EmitBufferAtomicSMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicUMax32    = EmitAtomic32;
 inline constexpr auto EmitBufferAtomicAnd32     = EmitAtomic32;
@@ -337,6 +339,9 @@ inline constexpr auto EmitDataConsume           = EmitAppendConsume;
 uint32_t              EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
 void                  EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
 uint32_t              EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvh8Intersect(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvh8Extract(ValueEmitContext& ctx, const IR::Inst& inst);
+uint32_t              EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst);
 inline constexpr auto EmitImageQueryDimensions = EmitImage;
 inline constexpr auto EmitImageQueryLod        = EmitImage;
 inline constexpr auto EmitImageRead            = EmitImage;
@@ -118,8 +118,11 @@ struct EmitterState {
 	uint32_t                                         bda_pointer_function    = 0;
 	uint32_t                                         indirect_format_function = 0;
 	uint32_t                                         indirect_format_geometry_function = 0;
-	std::map<std::array<uint32_t, 3>, uint32_t>        indirect_buffer_load_functions;
+	std::map<std::array<uint32_t, 4>, uint32_t>        indirect_buffer_load_functions;
 	uint32_t                                         bvh_intersect_function  = 0;
+	uint32_t                                         bvh8_result_type       = 0;
+	uint32_t                                         bvh8_intersect_function = 0;
+	uint32_t                                         bvh_stack_function      = 0;
 	uint32_t                                         gds_variable            = 0;
 	uint32_t                                         gds_length              = 0;
 	uint32_t                                         push_constant_variable  = 0;
@@ -169,6 +172,7 @@ uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
 uint32_t TypeU32(EmitterState& state);
 uint32_t TypeU64(EmitterState& state);
 uint32_t TypeU32Pair(EmitterState& state);
+uint32_t TypeBvh8Result(EmitterState& state);
 uint32_t TypeI32(EmitterState& state);
 uint32_t TypeI32Pair(EmitterState& state);
 uint32_t TypeF32(EmitterState& state);
@@ -269,8 +273,6 @@ struct ValueEmitContext {
 	uint32_t                                                           half       = 0;
 };

-enum class VertexInputScalarKind { Float, Sint, Uint };
-
 constexpr uint32_t NoImageComponent = 0xffffffffu;

 struct DppTargetLane {
@@ -300,12 +302,11 @@ bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr);

 bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr);

-VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location);
+Format::ComponentType VertexParameterScalarKind(const EmitterState& state, uint32_t location);

 uint32_t VertexParameterComponentCount(const InputBinding& input);

-uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind);
-
+uint32_t VertexParameterScalarType(EmitterState& state, Format::ComponentType kind);

 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
 uint32_t MrtOutputType(EmitterState& state, uint32_t index);
@@ -540,9 +541,13 @@ uint32_t EmitF32BitsToF16RtzBits(EmitterState& state, uint32_t bits);
 void EmitProgram(EmitterState& state);

 void DefineGetBdaPointer(EmitterState& state);
-void DefineIndirectBufferFormats(ValueEmitContext& ctx);
 void DefineIndirectBufferLoads(ValueEmitContext& ctx);
 void DefineBvhIntersect(EmitterState& state);
+void DefineTrinityBvhIntersect(EmitterState& state);
+uint32_t EmitBvhTriangle(EmitterState& state, const std::array<std::array<uint32_t, 3>, 3>& positions,
+                         std::array<uint32_t, 3> origin, std::array<uint32_t, 3> direction,
+                         uint32_t remap, uint32_t triangle_id, uint32_t bary);
+void DefineTrinityFunctions(EmitterState& state);
 uint32_t GetBdaPointer(EmitterState& state, uint32_t address);

 // These templates accept local lambdas from several emitter translation units.
@@ -1026,53 +1026,50 @@ uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IndirectBufferAccess& b
 	return ConstructU32Composite(state, components, values);
 }

-uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) {
+uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IndirectBufferAccess& buffer) {
 	auto& state = ctx.state;
-	return EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
-		const auto buffer = PrepareIndirectBuffer(state, IndirectBufferArgs(ctx, inst), true);
-		const auto geometry = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpFunctionCall, TypeU32Composite(state, 4), geometry,
-		                          state.indirect_format_geometry_function, buffer.format);
-		const auto field = [&](uint32_t index) {
-			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, geometry, index);
-			return value;
-		};
-		const auto bytes = field(0);
-		const auto component = Binary(
-		    state, spv::OpUMod, TypeU32(state),
-		    Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), field(1));
-		const auto alignment = Unary(state, spv::OpUConvert, TypeU64(state), field(3));
-		const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
-		                         Unary(state, spv::OpNot, TypeU64(state), alignment));
-		const auto address = Binary(
-		    state, spv::OpIAdd, TypeU64(state), base,
-		    Unary(state, spv::OpUConvert, TypeU64(state),
-		          Binary(state, spv::OpIMul, TypeU32(state), component, field(2))));
-		const auto memory = AndCondition(
-		    state, Binary(state, spv::OpINotEqual, TypeBool(state), bytes, ConstantU32(state, 0)),
-		    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector,
-		           ConstantU32(state, 4)));
-		const auto raw = EmitValueOrZeroIfCondition(
-		    state, AndCondition(state, memory, IndirectBufferInBounds(state, buffer, 0u, bytes, true)),
-		    [&]() {
-			    // Format alignment and component spacing keep every selected component within one DWORD.
-			    const auto word = LoadBdaDword(
-			        ctx, Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
-			                    ConstantU64(state, ~uint64_t {3})));
-			    const auto byte = Binary(
-			        state, spv::OpBitwiseAnd, TypeU32(state),
-			        Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3));
-			    return Binary(state, spv::OpShiftRightLogical, TypeU32(state), word,
-			                  Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte,
-			                         ConstantU32(state, 3)));
-		    });
-		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction(spv::OpFunctionCall, TypeU32(state), result,
-		                          state.indirect_format_function, buffer.format, buffer.selector,
-		                          component, raw);
-		return result;
-	});
+	const auto geometry = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeU32Composite(state, 4), geometry,
+	                          state.indirect_format_geometry_function, buffer.format);
+	const auto field = [&](uint32_t index) {
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, geometry, index);
+		return value;
+	};
+	const auto bytes = field(0);
+	const auto component = Binary(
+	    state, spv::OpUMod, TypeU32(state),
+	    Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), field(1));
+	const auto alignment = Unary(state, spv::OpUConvert, TypeU64(state), field(3));
+	const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address,
+	                         Unary(state, spv::OpNot, TypeU64(state), alignment));
+	const auto address = Binary(
+	    state, spv::OpIAdd, TypeU64(state), base,
+	    Unary(state, spv::OpUConvert, TypeU64(state),
+	          Binary(state, spv::OpIMul, TypeU32(state), component, field(2))));
+	const auto memory = AndCondition(
+	    state, Binary(state, spv::OpINotEqual, TypeBool(state), bytes, ConstantU32(state, 0)),
+	    Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector,
+	           ConstantU32(state, 4)));
+	const auto raw = EmitValueOrZeroIfCondition(
+	    state, AndCondition(state, memory, IndirectBufferInBounds(state, buffer, 0u, bytes, true)),
+	    [&]() {
+		    // Format alignment and component spacing keep every selected component within one DWORD.
+		    const auto word = LoadBdaDword(
+		        ctx, Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
+		                    ConstantU64(state, ~uint64_t {3})));
+		    const auto byte = Binary(
+		        state, spv::OpBitwiseAnd, TypeU32(state),
+		        Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3));
+		    return Binary(state, spv::OpShiftRightLogical, TypeU32(state), word,
+		                  Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte,
+		                         ConstantU32(state, 3)));
+	    });
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeU32(state), result,
+	                          state.indirect_format_function, buffer.format, buffer.selector,
+	                          component, raw);
+	return result;
 }

 uint32_t LoadRawBuffer(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
@@ -1135,15 +1132,12 @@ uint32_t LoadRawBuffer(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Me
 uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
 	auto& state = ctx.state;
 	const auto mem = ctx.Memory(inst);
-	if (mem.kind == IR::ResourceKind::IndirectBuffer && mem.formatted) {
-		return LoadIndirectFormattedX(ctx, inst);
-	}
 	return EmitValueOrDefaultIfCondition(
 	    state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU32Composite(state, components),
 	    ConstantU32CompositeZero(state, components), [&]() {
 		    if (mem.kind == IR::ResourceKind::IndirectBuffer) {
 			    const auto function = state.indirect_buffer_load_functions.at(
-			        {mem.data_bits, components, uint32_t(mem.coherent)});
+			        {mem.data_bits, components, uint32_t(mem.coherent), uint32_t(mem.formatted)});
 			    const auto args = IndirectBufferArgs(ctx, inst);
 			    const auto result = state.builder.AllocateId();
 			    state.builder.AddFunction(spv::OpFunctionCall, TypeU32Composite(state, components),
@@ -1231,62 +1225,8 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
 	});
 }

-} // namespace
-
-uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
-	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result,
-	                          state.bda_pointer_function, address);
-	return result;
-}
-
-void DefineIndirectBufferLoads(ValueEmitContext& ctx) {
-	auto& state = ctx.state;
-	for (const auto* block: state.program.blocks) {
-		for (const auto& inst: *block) {
-			const auto components = IR::BufferComponentCount(inst.GetOpcode());
-			if (components == 0u || inst.GetOpcode() == IR::ValueOpcode::ReadConstBuffer) continue;
-			const auto& memory = ctx.Memory(inst);
-			if (memory.kind != IR::ResourceKind::IndirectBuffer || memory.formatted ||
-			    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) continue;
-			auto [entry, inserted] = state.indirect_buffer_load_functions.try_emplace(
-			    std::array {memory.data_bits, components, uint32_t(memory.coherent)}, 0u);
-			if (!inserted) continue;
-			const auto function = entry->second = state.builder.AllocateId();
-			const auto type = TypeU32(state);
-			const auto result_type = TypeU32Composite(state, components);
-			const auto signature = state.builder.Type(
-			    spv::OpTypeFunction, result_type, type, type, type, type, type, type, type, type);
-			const auto name = "load_indirect_buffer_" + std::to_string(memory.data_bits) + "x" +
-			                  std::to_string(components) + (memory.coherent ? "_coherent" : "");
-			state.builder.AddName(function, name.c_str());
-			state.builder.AddFunction(spv::OpFunction, result_type, function,
-			                          spv::FunctionControlMaskNone, signature);
-			std::array<uint32_t, 8> args;
-			for (auto& arg: args) {
-				arg = state.builder.AllocateId();
-				state.builder.AddFunction(spv::OpFunctionParameter, type, arg);
-			}
-			EmitLabel(state, state.builder.AllocateId());
-			const auto result = LoadIndirectBuffer(ctx, PrepareIndirectBuffer(state, args, false),
-			                                       memory.data_bits, components, memory.coherent);
-			state.builder.AddFunction(spv::OpReturnValue, result);
-			state.builder.AddFunction(spv::OpFunctionEnd);
-		}
-	}
-}
-
 void DefineIndirectBufferFormats(ValueEmitContext& ctx) {
 	auto& state = ctx.state;
-	const auto needed = std::ranges::any_of(state.program.blocks, [&](const IR::Block* block) {
-		return std::ranges::any_of(*block, [&](const IR::Inst& inst) {
-			if (inst.GetOpcode() != IR::ValueOpcode::LoadBufferU32) return false;
-			const auto& memory = ctx.Memory(inst);
-			return memory.kind == IR::ResourceKind::IndirectBuffer && memory.formatted;
-		});
-	});
-	if (!needed) return;
-
 	const auto type = TypeU32(state);
 	constexpr auto format_count =
 	    static_cast<uint32_t>(Prospero::BufferFormat::k32_32_32_32Float) + 1u;
@@ -1358,6 +1298,57 @@ void DefineIndirectBufferFormats(ValueEmitContext& ctx) {
 	state.builder.AddFunction(spv::OpFunctionEnd);
 }

+} // namespace
+
+uint32_t GetBdaPointer(EmitterState& state, uint32_t address) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(spv::OpFunctionCall, TypeU64(state), result,
+	                          state.bda_pointer_function, address);
+	return result;
+}
+
+void DefineIndirectBufferLoads(ValueEmitContext& ctx) {
+	auto& state = ctx.state;
+	for (const auto* block: state.program.blocks) {
+		for (const auto& inst: *block) {
+			const auto components = IR::BufferComponentCount(inst.GetOpcode());
+			if (components == 0u || inst.GetOpcode() == IR::ValueOpcode::ReadConstBuffer) continue;
+			const auto& memory = ctx.Memory(inst);
+			if (memory.kind != IR::ResourceKind::IndirectBuffer ||
+			    !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) continue;
+			auto [entry, inserted] = state.indirect_buffer_load_functions.try_emplace(
+			    std::array {memory.data_bits, components, uint32_t(memory.coherent),
+			                uint32_t(memory.formatted)}, 0u);
+			if (!inserted) continue;
+			if (memory.formatted && state.indirect_format_function == 0u) {
+				DefineIndirectBufferFormats(ctx);
+			}
+			const auto function = entry->second = state.builder.AllocateId();
+			const auto type = TypeU32(state);
+			const auto result_type = TypeU32Composite(state, components);
+			const auto signature = state.builder.Type(
+			    spv::OpTypeFunction, result_type, type, type, type, type, type, type, type, type);
+			const auto name = "load_indirect_buffer_" + std::to_string(memory.data_bits) + "x" +
+			                  std::to_string(components) + (memory.coherent ? "_coherent" : "") +
+			                  (memory.formatted ? "_formatted" : "");
+			state.builder.AddName(function, name.c_str());
+			state.builder.AddFunction(spv::OpFunction, result_type, function,
+			                          spv::FunctionControlMaskNone, signature);
+			std::array<uint32_t, 8> args;
+			for (auto& arg: args) {
+				arg = state.builder.AllocateId();
+				state.builder.AddFunction(spv::OpFunctionParameter, type, arg);
+			}
+			EmitLabel(state, state.builder.AllocateId());
+			const auto buffer = PrepareIndirectBuffer(state, args, memory.formatted);
+			const auto result = memory.formatted ? LoadIndirectFormattedX(ctx, buffer)
+			    : LoadIndirectBuffer(ctx, buffer, memory.data_bits, components, memory.coherent);
+			state.builder.AddFunction(spv::OpReturnValue, result);
+			state.builder.AddFunction(spv::OpFunctionEnd);
+		}
+	}
+}
+
 void DefineGetBdaPointer(EmitterState& state) {
 	if (!state.program.info.uses_dma) {
 		return;
@@ -585,6 +585,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) {
 		case IR::ValueOpcode::ImageAtomicUMin32:
 		case IR::ValueOpcode::ImageAtomicUMin64:
 		case IR::ValueOpcode::BufferAtomicUMin32:
+		case IR::ValueOpcode::BufferAtomicUMin64:
 		case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
 		case IR::ValueOpcode::ImageAtomicSMax32:
 		case IR::ValueOpcode::BufferAtomicSMax32:
@@ -402,47 +402,23 @@ uint32_t GlslStd450(EmitterState& state) {
 	return state.builder.Import("GLSL.std.450");
 }

-VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
+Format::ComponentType VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
 	if ((state.program.stage != ShaderType::Vertex && state.program.stage != ShaderType::Local) ||
 	    location >= ShaderVertexInputInfo::RES_MAX ||
 	    location >= static_cast<uint32_t>(state.input_info.vertex->resources_num)) {
-		return VertexInputScalarKind::Float;
-	}
-
-	switch (state.input_info.vertex->resources[location].Format()) {
-		case Prospero::BufferFormat::k8UInt:
-		case Prospero::BufferFormat::k16UInt:
-		case Prospero::BufferFormat::k8_8UInt:
-		case Prospero::BufferFormat::k32UInt:
-		case Prospero::BufferFormat::k16_16UInt:
-		case Prospero::BufferFormat::k8_8_8_8UInt:
-		case Prospero::BufferFormat::k32_32UInt:
-		case Prospero::BufferFormat::k16_16_16_16UInt:
-		case Prospero::BufferFormat::k32_32_32UInt:
-		case Prospero::BufferFormat::k32_32_32_32UInt: return VertexInputScalarKind::Uint;
-		case Prospero::BufferFormat::k8SInt:
-		case Prospero::BufferFormat::k16SInt:
-		case Prospero::BufferFormat::k8_8SInt:
-		case Prospero::BufferFormat::k32SInt:
-		case Prospero::BufferFormat::k16_16SInt:
-		case Prospero::BufferFormat::k8_8_8_8SInt:
-		case Prospero::BufferFormat::k32_32SInt:
-		case Prospero::BufferFormat::k16_16_16_16SInt:
-		case Prospero::BufferFormat::k32_32_32SInt:
-		case Prospero::BufferFormat::k32_32_32_32SInt: return VertexInputScalarKind::Sint;
-		default: return VertexInputScalarKind::Float;
+		return Format::ComponentType::Float;
 	}
+	return Format::GetFormatComponentType(state.input_info.vertex->resources[location].Format());
 }

 uint32_t VertexParameterComponentCount(const InputBinding& input) {
 	return std::clamp(input.component_count, 1u, 4u);
 }

-uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind) {
+uint32_t VertexParameterScalarType(EmitterState& state, Format::ComponentType kind) {
 	switch (kind) {
-		case VertexInputScalarKind::Sint: return TypeI32(state);
-		case VertexInputScalarKind::Uint: return TypeU32(state);
-		case VertexInputScalarKind::Float:
+		case Format::ComponentType::Sint: return TypeI32(state);
+		case Format::ComponentType::Uint: return TypeU32(state);
 		default: return TypeF32(state);
 	}
 }
@@ -476,6 +452,8 @@ uint32_t BuiltInForInput(IR::StageInputKind kind) {
 		case IR::StageInputKind::LocalInvocationId: return spv::BuiltInLocalInvocationId;
 		case IR::StageInputKind::LocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
 		case IR::StageInputKind::GlobalInvocationId: return spv::BuiltInGlobalInvocationId;
+		case IR::StageInputKind::BaseVertex: return spv::BuiltInBaseVertex;
+		case IR::StageInputKind::BaseInstance: return spv::BuiltInBaseInstance;
 		default: return UINT32_MAX;
 	}
 }
@@ -526,7 +504,9 @@ void DefineInputs(EmitterState& state) {
 			case IR::StageInputKind::PrimitiveId:
 			case IR::StageInputKind::InstanceIndex:
 			case IR::StageInputKind::Layer:
-			case IR::StageInputKind::SampleId: type = TypeI32(state); break;
+			case IR::StageInputKind::SampleId:
+			case IR::StageInputKind::BaseVertex:
+			case IR::StageInputKind::BaseInstance: type = TypeI32(state); break;
 			case IR::StageInputKind::WorkgroupId:
 			case IR::StageInputKind::NumWorkgroups:
 			case IR::StageInputKind::LocalInvocationId:
@@ -802,6 +782,10 @@ void DefineModule(EmitterState& state) {
 	if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) {
 		state.builder.RequireCapability(spv::CapabilitySampleRateShading);
 	}
+	if (InputVariableForKind(state, IR::StageInputKind::BaseVertex) != 0 ||
+	    InputVariableForKind(state, IR::StageInputKind::BaseInstance) != 0) {
+		state.builder.RequireCapability(spv::CapabilityDrawParameters);
+	}
 	if (state.program.info.image_gather_extended) {
 		state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
 	}
@@ -235,7 +235,8 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto op = inst.GetOpcode();
 	if (ctx.half != 0 &&
 	    (op == Op::Ballot || op == Op::ReadFirstLane || op == Op::SubgroupUMinU32 ||
-	     op == Op::SubgroupUMaxU32 || op == Op::ReadConstBuffer || op == Op::ReadRealtimeClock ||
+	     op == Op::SubgroupUMaxU32 || op == Op::SubgroupOrU32 || op == Op::ReadConstBuffer ||
+	     op == Op::ReadRealtimeClock ||
 	     (op == Op::LoadAddressU32 && ctx.Memory(inst).kind == IR::ResourceKind::ScalarAddress))) {
 		// Whole-wave results, clocks and SMEM loads (SGPR operands, no EXEC) match in both halves.
 		ctx.Define(inst, ctx.other_half->Result(inst));
@@ -414,6 +415,7 @@ uint32_t TypeId(EmitterState& state, IR::Type type) {
 		case IR::Type::U32:
 		case IR::Type::F16: return TypeU32(state);
 		case IR::Type::U64: return TypeU64(state);
+		case IR::Type::Bvh8Result: return TypeBvh8Result(state);
 		case IR::Type::U32x2: return TypeU32Pair(state);
 		case IR::Type::F32: return TypeF32(state);
 		case IR::Type::F64: return TypeF64(state);
@@ -675,8 +677,8 @@ void EmitProgram(EmitterState& state) {
 	}
 	DefineGetBdaPointer(state);
 	DefineIndirectBufferLoads(ctx);
-	DefineIndirectBufferFormats(ctx);
 	DefineBvhIntersect(state);
+	DefineTrinityFunctions(state);
 	if (program.info.uses_swizzle) {
 		ctx.scratch_u32_variable = state.builder.AllocateId();
 		if (state.lane_count == 2) {
@@ -29,12 +29,14 @@ uint32_t TessellationPointer(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto pointer = state.builder.AllocateId();
 	if (kind == Attribute::Factor) {
 		EXIT_NOT_IMPLEMENTED(!inst.Arg(1).IsImmediate());
-		const auto index = inst.Arg(1).U32() / 4u;
-		const auto outer = index < 3u;
-		EXIT_NOT_IMPLEMENTED(index >= 4u);
+		const auto index       = inst.Arg(1).U32() / 4u;
+		const bool quad        = tess.domain == 2u;
+		const auto outer_count = quad ? 4u : 3u;
+		const bool outer       = index < outer_count;
+		EXIT_NOT_IMPLEMENTED(index >= (quad ? 6u : 4u));
 		state.builder.AddFunction(spv::OpAccessChain, TypePointer(state, storage, TypeF32(state)),
 		                          pointer, outer ? variable : state.tess_inner_variable,
-		                          ConstantU32(state, outer ? index : index - 3u));
+		                          ConstantU32(state, outer ? index : index - outer_count));
 		return pointer;
 	}
 	auto address = ctx.Arg(inst, 1);
@@ -128,20 +130,25 @@ void DefineTessellationInterfaces(EmitterState& state) {

 void DefineTessellationExecutionModes(EmitterState& state) {
 	const auto& tess = state.input_info.vertex->tess;
-	// CxVgtTfParam: triangles, integer or fractional-odd partitioning, clockwise output.
-	EXIT_NOT_IMPLEMENTED(tess.domain != 1u ||
-	                     (tess.partitioning != 0u && tess.partitioning != 2u) ||
-	                     tess.output_topology != 2u);
+	// Triangle or quad domain, integer or fractional partitioning, triangle output.
+	EXIT_NOT_IMPLEMENTED((tess.domain != 1u && tess.domain != 2u) ||
+	                     (tess.partitioning != 0u && tess.partitioning != 2u && tess.partitioning != 3u) ||
+	                     (tess.output_topology != 2u && tess.output_topology != 3u));
 	state.builder.RequireCapability(spv::CapabilityTessellation);
 	if (state.program.stage == ShaderType::TessellationControl) {
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeOutputVertices,
 		                               tess.output_control_points);
 	} else {
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeTriangles);
+		state.builder.AddExecutionMode(state.main_func, tess.domain == 1u
+		                                                   ? spv::ExecutionModeTriangles
+		                                                   : spv::ExecutionModeQuads);
 		state.builder.AddExecutionMode(
 		    state.main_func, tess.partitioning == 0u ? spv::ExecutionModeSpacingEqual
-		                                             : spv::ExecutionModeSpacingFractionalOdd);
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeVertexOrderCw);
+		                     : tess.partitioning == 2u ? spv::ExecutionModeSpacingFractionalOdd
+		                                                : spv::ExecutionModeSpacingFractionalEven);
+		state.builder.AddExecutionMode(state.main_func, tess.output_topology == 2u
+		                                                   ? spv::ExecutionModeVertexOrderCw
+		                                                   : spv::ExecutionModeVertexOrderCcw);
 	}
 }

@@ -0,0 +1,208 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t Bits(EmitterState& s, uint32_t value, uint32_t offset, uint32_t count) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(s), result, value,
+	                      ConstantU32(s, offset), ConstantU32(s, count));
+	return result;
+}
+
+uint32_t SetBits(EmitterState& s, uint32_t value, uint32_t field, uint32_t offset, uint32_t count) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpBitFieldInsert, TypeU32(s), result, value, field,
+	                      ConstantU32(s, offset), ConstantU32(s, count));
+	return result;
+}
+
+uint32_t Pair(EmitterState& s, uint32_t first, uint32_t second) {
+	const auto result = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Pair(s), result, first, second);
+	return result;
+}
+
+uint32_t StackIndex(EmitterState& s, uint32_t control, uint32_t ring) {
+	return EmitAddU32(s, Bits(s, control, 15, 14),
+	    Binary(s, spv::OpShiftLeftLogical, TypeU32(s), ring, ConstantU32(s, 5)));
+}
+
+uint32_t StackPush(EmitterState& s, const MemoryResourceAccess& memory,
+                   uint32_t control, uint32_t node, uint32_t capacity) {
+	const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
+	const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5);
+	const auto index = StackIndex(s, control, ring);
+	EmitIfCondition(s, EmitMemoryElementInBounds(s, memory, index), [&] {
+		s.builder.AddFunction(spv::OpStore, EmitMemoryElementPointer(s, memory, index), node);
+	});
+	control = Binary(s, spv::OpBitwiseOr, u, control,
+	    Select(s, u, Binary(s, spv::OpIEqual, b, count, capacity), ConstantU32(s, 1u << 30), zero));
+	control = Select(s, u, Binary(s, spv::OpIEqual, b, parent, capacity),
+	    Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control);
+	control = SetBits(s, control, EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+	    EmitAddU32(s, count, one), capacity), 0, 5);
+	control = SetBits(s, control, EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+	    EmitAddU32(s, parent, one), capacity), 5, 5);
+	const auto next = EmitAddU32(s, ring, one);
+	return SetBits(s, control, Select(s, u, Binary(s, spv::OpIEqual, b, next, capacity), zero, next), 10, 5);
+}
+
+uint32_t StackPop(EmitterState& s, const MemoryResourceAccess& memory,
+                  uint32_t control, uint32_t capacity) {
+	const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
+	const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5);
+	const auto has_parent = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero);
+	const auto returning = Binary(s, spv::OpLogicalAnd, b, has_parent,
+	    Binary(s, spv::OpIEqual, b, parent, zero));
+	control = SetBits(s, control, Select(s, u, returning, one, zero), 31, 1);
+	control = Select(s, u, returning,
+	    Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control);
+	const auto overflow = Binary(s, spv::OpINotEqual, b, Bits(s, control, 30, 1), zero);
+	const auto empty = Pair(s,
+	    Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, 0x5fffffff)),
+	    Select(s, u, overflow, ConstantU32(s, 0xffffffff), ConstantU32(s, 0xfffffffe)));
+	return EmitValueOrDefaultIfCondition(s, Binary(s, spv::OpINotEqual, b, count, zero),
+	    TypeU32Pair(s), empty, [&] {
+		control = SetBits(s, control, Binary(s, spv::OpISub, u, count, one), 0, 5);
+		control = SetBits(s, control, Binary(s, spv::OpISub, u, parent, one), 5, 5);
+		const auto previous = Select(s, u, Binary(s, spv::OpIEqual, b, ring, zero),
+		    Binary(s, spv::OpISub, u, capacity, one), Binary(s, spv::OpISub, u, ring, one));
+		control = SetBits(s, control, previous, 10, 5);
+		const auto index = StackIndex(s, control, previous);
+		const auto node = EmitValueOrDefaultIfCondition(s,
+		    EmitMemoryElementInBounds(s, memory, index), u, zero, [&] {
+			const auto value = s.builder.AllocateId();
+			s.builder.AddFunction(spv::OpLoad, u, value, EmitMemoryElementPointer(s, memory, index));
+			return value;
+		});
+		const auto instance = Binary(s, spv::OpIEqual, b, Bits(s, node, 0, 3), ConstantU32(s, 6));
+		control = EmitValueOrDefaultIfCondition(s, instance, u, control, [&] {
+			const auto nested = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero);
+			const auto discarded = Binary(s, spv::OpBitwiseOr, u,
+			    SetBits(s, control, Bits(s, control, 5, 5), 0, 5), ConstantU32(s, 1u << 30));
+			return Binary(s, spv::OpBitwiseOr, u,
+			    SetBits(s, Select(s, u, nested, discarded, control), zero, 5, 5), ConstantU32(s, 1u << 29));
+		});
+		return Pair(s, control, node);
+	});
+}
+
+void DefineBvhStack(EmitterState& s) {
+	if (!s.program.info.bvh_stack) return;
+	const auto u = TypeU32(s), b = TypeBool(s), vec4 = TypeU32Vector(s, 4), pair = TypeU32Pair(s);
+	const auto zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
+	const bool local = s.lds_storage_class == spv::StorageClassFunction;
+	const bool buffer = s.lds_storage_class == spv::StorageClassStorageBuffer;
+	std::vector<uint32_t> types {u, u, vec4, vec4, u};
+	if (local) types.push_back(TypeU32ArrayPointer(s, spv::StorageClassFunction, LdsDwordCount(s)));
+	if (buffer) types.push_back(u);
+	const auto signature = s.builder.Type(spv::OpTypeFunction, pair, types);
+	s.bvh_stack_function = s.builder.AllocateId();
+	s.builder.AddName(s.bvh_stack_function, "trinity_bvh_stack");
+	s.builder.AddFunction(spv::OpFunction, pair, s.bvh_stack_function, spv::FunctionControlMaskNone, signature);
+	std::vector<uint32_t> args;
+	for (const auto type: types) {
+		args.push_back(s.builder.AllocateId());
+		s.builder.AddFunction(spv::OpFunctionParameter, type, args.back());
+	}
+	const auto old_lds = s.lds_variable, old_base = s.lds_base_dwords;
+	if (local) s.lds_variable = args.back();
+	if (buffer) s.lds_base_dwords = args.back();
+	const auto memory = PrepareMemoryResourceAccess(s, {.kind = IR::ResourceKind::Lds});
+	const auto entry = s.builder.AllocateId();
+	EmitLabel(s, entry);
+	const auto last = args[1], capacity = args[4];
+	const auto walkback = Binary(s, spv::OpINotEqual, b, Bits(s, last, 3, 29), ConstantU32(s, 0x1fffffff));
+	const auto last4 = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, vec4, last4, last, last, last, last);
+	const auto found = [&](uint32_t values) {
+		return Unary(s, spv::OpAny, b, Binary(s, spv::OpIEqual, TypeBoolVector(s, 4), values, last4));
+	};
+	const auto initial_valid = Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, walkback),
+	    Binary(s, spv::OpLogicalOr, b, found(args[2]), found(args[3])));
+	const auto header = s.builder.AllocateId(), body = s.builder.AllocateId();
+	const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpBranch, header);
+	EmitLabel(s, header);
+	const auto index = s.builder.AllocateId(), control = s.builder.AllocateId();
+	const auto valid = s.builder.AllocateId(), skip_low = s.builder.AllocateId(), skip_high = s.builder.AllocateId();
+	const auto index_phi = s.builder.AddDeferredPhi(u, index, 2);
+	const auto control_phi = s.builder.AddDeferredPhi(u, control, 2);
+	const auto valid_phi = s.builder.AddDeferredPhi(b, valid, 2);
+	const auto low_phi = s.builder.AddDeferredPhi(b, skip_low, 2);
+	const auto high_phi = s.builder.AddDeferredPhi(b, skip_high, 2);
+	const auto more = Binary(s, spv::OpLogicalAnd, b, valid,
+	    Binary(s, spv::OpULessThan, b, index, ConstantU32(s, 8)));
+	s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask);
+	s.builder.AddFunction(spv::OpBranchConditional, more, body, merge);
+	EmitLabel(s, body);
+	const auto high = Binary(s, spv::OpUGreaterThanEqual, b, index, ConstantU32(s, 4));
+	const auto high4 = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpCompositeConstruct, TypeBoolVector(s, 4), high4, high, high, high, high);
+	const auto values = Select(s, vec4, high4, args[3], args[2]);
+	const auto node = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpVectorExtractDynamic, u, node, values, Bits(s, index, 0, 2));
+	const auto sentinel = Binary(s, spv::OpIEqual, b, Bits(s, node, 3, 29), ConstantU32(s, 0x1fffffff));
+	const auto enabled = Unary(s, spv::OpLogicalNot, b, Select(s, b, high, skip_high, skip_low));
+	const auto marker = Binary(s, spv::OpLogicalAnd, b, sentinel, enabled);
+	const auto next_low = Binary(s, spv::OpLogicalOr, b, skip_low,
+	    Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 1, 1), zero)));
+	const auto next_high = Binary(s, spv::OpLogicalOr, b, skip_high,
+	    Binary(s, spv::OpLogicalAnd, b, high,
+	        Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 2, 1), zero))));
+	const auto next_valid = Unary(s, spv::OpLogicalNot, b,
+	    Binary(s, spv::OpLogicalAnd, b, walkback, Binary(s, spv::OpIEqual, b, node, last)));
+	const auto push = Binary(s, spv::OpLogicalAnd, b, next_valid,
+	    Unary(s, spv::OpLogicalNot, b,
+	        Binary(s, spv::OpLogicalOr, b, sentinel, Select(s, b, high, next_high, next_low))));
+	const auto next_control = EmitValueOrDefaultIfCondition(s, push, u, control, [&] {
+		return StackPush(s, memory, control, node, capacity);
+	});
+	s.builder.AddFunction(spv::OpBranch, cont);
+	EmitLabel(s, cont);
+	const auto next_index = Binary(s, spv::OpISub, u, index, one);
+	s.builder.AddFunction(spv::OpBranch, header);
+	s.builder.PatchDeferredPhi(index_phi, 0, ConstantU32(s, 7), entry);
+	s.builder.PatchDeferredPhi(index_phi, 1, next_index, cont);
+	s.builder.PatchDeferredPhi(control_phi, 0, args[0], entry);
+	s.builder.PatchDeferredPhi(control_phi, 1, next_control, cont);
+	s.builder.PatchDeferredPhi(valid_phi, 0, initial_valid, entry);
+	s.builder.PatchDeferredPhi(valid_phi, 1, next_valid, cont);
+	s.builder.PatchDeferredPhi(low_phi, 0, ConstantBool(s, false), entry);
+	s.builder.PatchDeferredPhi(low_phi, 1, next_low, cont);
+	s.builder.PatchDeferredPhi(high_phi, 0, ConstantBool(s, false), entry);
+	s.builder.PatchDeferredPhi(high_phi, 1, next_high, cont);
+	EmitLabel(s, merge);
+	s.builder.AddFunction(spv::OpReturnValue, StackPop(s, memory, control, capacity));
+	s.builder.AddFunction(spv::OpFunctionEnd);
+	if (local) s.lds_variable = old_lds;
+	if (buffer) s.lds_base_dwords = old_base;
+}
+
+} // namespace
+
+void DefineTrinityFunctions(EmitterState& s) {
+	DefineBvhStack(s);
+	DefineTrinityBvhIntersect(s);
+}
+
+uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto capacity = ctx.Memory(inst).offset;
+	if (s.bvh_stack_function == 0 || capacity < 8 || capacity > 31)
+		ctx.Fail(inst, "invalid BVH stack function or capacity");
+	const std::array values {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)};
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 4), TypeU32Pair(s),
+	    ConstantU32CompositeZero(s, 2), [&] {
+		const auto result = s.builder.AllocateId();
+		std::vector<uint32_t> call {spv::OpFunctionCall, TypeU32Pair(s), result,
+		    s.bvh_stack_function, values[0], values[1], values[2], values[3], ConstantU32(s, capacity)};
+		if (s.lds_storage_class == spv::StorageClassFunction) call.push_back(s.lds_variable);
+		if (s.lds_storage_class == spv::StorageClassStorageBuffer) call.push_back(s.lds_base_dwords);
+		s.builder.AddFunction(call);
+		return result;
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
@@ -0,0 +1,511 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t TypeBvh8Result(EmitterState& s) {
+	if (s.bvh8_result_type == 0) {
+		const auto v = TypeU32Vector(s, 4);
+		s.bvh8_result_type = s.builder.Type(spv::OpTypeStruct, v, v, v, v, TypeU32(s));
+	}
+	return s.bvh8_result_type;
+}
+
+namespace {
+using Vec3 = std::array<uint32_t, 3>;
+
+struct Bvh8Emitter {
+	EmitterState& s;
+	const uint32_t u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), w = TypeU64(s);
+	const uint32_t v4 = TypeU32Vector(s, 4), v3 = TypeF32Vector(s, 3);
+	uint32_t triangle_function = 0, boxes_function = 0, decompress_function = 0;
+
+	uint32_t C(uint32_t x) { return ConstantU32(s, x); }
+	uint32_t And(uint32_t x, uint32_t y) { return Binary(s, spv::OpBitwiseAnd, u, x, y); }
+	uint32_t Or(uint32_t x, uint32_t y) { return Binary(s, spv::OpBitwiseOr, u, x, y); }
+	uint32_t Add(uint32_t x, uint32_t y) { return EmitAddU32(s, x, y); }
+	uint32_t Sub(uint32_t x, uint32_t y) { return Binary(s, spv::OpISub, u, x, y); }
+	uint32_t Shl(uint32_t x, uint32_t n) { return Binary(s, spv::OpShiftLeftLogical, u, x, C(n)); }
+	uint32_t Shr(uint32_t x, uint32_t n) { return Binary(s, spv::OpShiftRightLogical, u, x, C(n)); }
+	uint32_t Eq(uint32_t x, uint32_t y) { return Binary(s, spv::OpIEqual, b, x, y); }
+	uint32_t Both(uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); }
+	uint32_t Either(uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); }
+	uint32_t Not(uint32_t x) { return Unary(s, spv::OpLogicalNot, b, x); }
+	uint32_t Bit(uint32_t x, uint32_t n) { return Not(Eq(And(x, C(1u << n)), C(0))); }
+	uint32_t Choose(uint32_t condition, uint32_t x, uint32_t y) { return Select(s, u, condition, x, y); }
+	uint32_t Float(uint32_t x) { return Unary(s, spv::OpBitcast, f, x); }
+	uint32_t Raw(uint32_t x) { return Unary(s, spv::OpBitcast, u, x); }
+	uint32_t Fop(spv::Op op, uint32_t x, uint32_t y) { return Binary(s, op, f, x, y); }
+	uint32_t Fcmp(spv::Op op, uint32_t x, uint32_t y) { return Binary(s, op, b, x, y); }
+	uint32_t Min(uint32_t x, uint32_t y) { return EmitGlsl<GLSLstd450FMin, IR::Type::F32>(s, x, y); }
+	uint32_t Max(uint32_t x, uint32_t y) { return EmitGlsl<GLSLstd450FMax, IR::Type::F32>(s, x, y); }
+	uint32_t Extract(uint32_t type, uint32_t value, uint32_t index) {
+		const auto id = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpCompositeExtract, type, id, value, index);
+		return id;
+	}
+	uint32_t Make(uint32_t type, std::span<const uint32_t> values) {
+		const auto id = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpCompositeConstruct, type, id, values);
+		return id;
+	}
+	uint32_t Make4(uint32_t x, uint32_t y, uint32_t z, uint32_t t) {
+		return Make(v4, std::array{x, y, z, t});
+	}
+	Vec3 Unpack3(uint32_t value) {
+		return {Extract(f, value, 0), Extract(f, value, 1), Extract(f, value, 2)};
+	}
+	uint32_t Word(uint32_t block, uint32_t index) {
+		const auto offset = Unary(s, spv::OpUConvert, w, Shl(index, 2));
+		const auto address = Binary(s, spv::OpIAdd, w, block, offset);
+		const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address);
+		const auto id = s.builder.AllocateId();
+		s.builder.AddFunction(spv::OpLoad, u, id, pointer, spv::MemoryAccessAlignedMask, 4u);
+		return id;
+	}
+	uint32_t Call(uint32_t type, uint32_t function, std::span<const uint32_t> args) {
+		const auto id = s.builder.AllocateId();
+		std::vector<uint32_t> call {spv::OpFunctionCall, type, id, function};
+		call.insert(call.end(), args.begin(), args.end());
+		s.builder.AddFunction(call);
+		return id;
+	}
+	std::vector<uint32_t> Begin(uint32_t& function, const char* name, uint32_t result,
+	                            std::span<const uint32_t> types) {
+		function = s.builder.AllocateId();
+		s.builder.AddName(function, name);
+		s.builder.AddFunction(spv::OpFunction, result, function, spv::FunctionControlMaskNone,
+		    s.builder.Type(spv::OpTypeFunction, result, types));
+		std::vector<uint32_t> args;
+		for (auto type: types) {
+			args.push_back(s.builder.AllocateId());
+			s.builder.AddFunction(spv::OpFunctionParameter, type, args.back());
+		}
+		EmitLabel(s, s.builder.AllocateId());
+		return args;
+	}
+	void End(uint32_t result) {
+		s.builder.AddFunction(spv::OpReturnValue, result);
+		s.builder.AddFunction(spv::OpFunctionEnd);
+	}
+	uint32_t Result(const std::array<uint32_t, 10>& dst, const Vec3& org, const Vec3& dir,
+	                uint32_t instance) {
+		const std::array groups {Make4(dst[0], dst[1], dst[2], dst[3]),
+		    Make4(dst[4], dst[5], dst[6], dst[7]), Make4(dst[8], dst[9], Raw(org[0]), Raw(org[1])),
+		    Make4(Raw(org[2]), Raw(dir[0]), Raw(dir[1]), Raw(dir[2])), instance};
+		return Make(TypeBvh8Result(s), groups);
+	}
+	uint32_t NormalizeFlags(uint32_t flags) {
+		flags = Choose(Bit(flags, 2), And(flags, C(~0x30u)), flags);
+		return Or(flags, Choose(Eq(And(flags, C(0x30)), C(0x30)), C(0x100), C(0)));
+	}
+
+	void DefineTriangle() {
+		const auto a = Begin(triangle_function, "trinity_triangle", v4,
+		    std::array{w, u, u, u, v3, v3});
+		const auto block = a[0], slot = a[1], flags = a[2], descriptor = a[3];
+		const auto origin = Unpack3(a[4]), direction = Unpack3(a[5]);
+		const auto flag_word = Word(block, C(15));
+		const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag_word, Shl(slot, 3));
+		const auto bary = Bit(descriptor, 24);
+		const auto pointer_flags = Both(bary, Bit(descriptor, 23));
+		const auto tri_flags = Choose(pointer_flags, remap, C(0));
+		const auto procedural = Bit(tri_flags, 6);
+		const auto raw = EmitValueOrDefaultIfCondition(s, Not(procedural), v4,
+		    Make4(C(0), C(0x3f800000), C(0x3f800000), C(0x3f800000)), [&] {
+			std::array<Vec3, 3> positions;
+			const std::array vertices {slot, Choose(Eq(slot, C(0)), C(1), C(3)), C(2)};
+			for (uint32_t vertex = 0; vertex < 3; ++vertex)
+				for (uint32_t axis = 0; axis < 3; ++axis)
+					positions[vertex][axis] = Float(Word(block,
+					    Add(Binary(s, spv::OpIMul, u, vertices[vertex], C(3)), C(axis))));
+			return EmitBvhTriangle(s, positions, origin, direction, remap, Add(flag_word, slot), bary);
+		});
+		std::array<uint32_t, 4> result;
+		for (uint32_t i = 0; i < 4; ++i) result[i] = Extract(u, raw, i);
+		auto missed = Eq(result[0], C(0x7f800000));
+		const auto flip = Both(Both(Not(procedural), Not(missed)),
+		    Binary(s, spv::OpLogicalNotEqual, b, Bit(flags, 3), Bit(tri_flags, 5)));
+		for (uint32_t i = 0; i < 4; ++i)
+			result[i] = Binary(s, spv::OpBitwiseXor, u, result[i],
+			    Choose(i < 2 ? flip : Both(flip, bary), C(0x80000000), C(0)));
+		const auto front = Fcmp(spv::OpFOrdGreaterThan, Float(result[1]), ConstantF32Value(s, 0.0f));
+		const auto opaque = Both(Either(Bit(tri_flags, 7), Bit(flags, 0)), Not(Bit(flags, 1)));
+		missed = Either(missed, Either(Both(opaque, Bit(flags, 6)), Both(Not(opaque), Bit(flags, 7))));
+		missed = Either(missed, Both(Both(Not(Bit(tri_flags, 4)), Not(procedural)),
+		    Select(s, b, front, Bit(flags, 5), Bit(flags, 4))));
+		missed = Either(missed, Select(s, b, procedural, Bit(flags, 9), Bit(flags, 8)));
+		const auto sign = [&](uint32_t value, uint32_t enabled) {
+			return Choose(pointer_flags, Or(And(value, C(0x7fffffff)),
+			    Choose(enabled, C(0x80000000), C(0))), value);
+		};
+		result[2] = sign(result[2], Either(Not(opaque), procedural));
+		result[3] = sign(result[3], Not(opaque));
+		const std::array miss {C(0x7f800000), C(0x3f800000), Choose(bary, C(0), C(~0u)), C(0)};
+		for (uint32_t i = 0; i < 4; ++i) result[i] = Choose(missed, miss[i], result[i]);
+		End(Make(v4, result));
+	}
+
+	uint32_t Instance(uint32_t block, uint32_t flags, uint32_t mask, Vec3 org, Vec3 dir) {
+		const auto lo = Word(block, C(12)), hi = Word(block, C(13));
+		const auto inst_flags = Shr(hi, 22);
+		auto merged = Or(And(Or(flags, inst_flags), C(0x3f0)), And(inst_flags, C(4)));
+		merged = Or(merged, And(Binary(s, spv::OpBitwiseXor, u, flags, inst_flags), C(8)));
+		merged = Or(merged, Choose(Either(Bit(flags, 0), Both(Bit(inst_flags, 0), Not(Bit(flags, 1)))), C(1), C(0)));
+		merged = Or(merged, Choose(Either(Bit(flags, 1), Both(Bit(inst_flags, 1), Not(Bit(flags, 0)))), C(2), C(0)));
+		const auto user = Word(block, C(15));
+		std::array<uint32_t, 10> dst;
+		dst.fill(C(~0u));
+		dst[2] = lo;
+		dst[3] = Or(And(hi, C(0x3fffff)), Shl(merged, 22));
+		dst[6] = And(user, C(0xffffff));
+		dst[7] = Choose(Eq(And(mask, Shr(user, 24)), C(0)), C(~0u), Word(block, C(14)));
+		Vec3 transformed_org, transformed_dir;
+		for (uint32_t axis = 0; axis < 3; ++axis) {
+			Vec3 row;
+			for (uint32_t i = 0; i < 3; ++i) row[i] = Float(Word(block, C(axis * 4 + i)));
+			const auto dot = [&](const Vec3& value) {
+				return Fop(spv::OpFAdd, Fop(spv::OpFAdd, Fop(spv::OpFMul, row[0], value[0]),
+				    Fop(spv::OpFMul, row[1], value[1])), Fop(spv::OpFMul, row[2], value[2]));
+			};
+			transformed_org[axis] = Fop(spv::OpFAdd, dot(org), Float(Word(block, C(axis * 4 + 3))));
+			transformed_dir[axis] = dot(dir);
+		}
+		return Result(dst, transformed_org, transformed_dir, C(1));
+	}
+
+	void DefineDecompress();
+	void DefineBoxes();
+	uint32_t Boxes(uint32_t first, uint32_t second, uint32_t full, uint32_t compressed,
+	               uint32_t flags, uint32_t descriptor, uint32_t extent, uint32_t org, uint32_t dir);
+	void DefineIntersect();
+};
+
+void Bvh8Emitter::DefineDecompress() {
+	const auto a = Begin(decompress_function, "trinity_shared_exponent", f, std::array{u, u, b});
+	const auto exponent = a[0], packed = a[1], minimum = a[2];
+	const auto negative = Bit(packed, 17), mantissa = And(packed, C(0x1ffff));
+	const auto outward = Eq(Choose(negative, C(1), C(0)), Choose(minimum, C(1), C(0)));
+	const auto nonzero = Not(Eq(mantissa, C(0)));
+	const auto highest = EmitGlsl<GLSLstd450FindUMsb, IR::Type::U32>(s, mantissa);
+	const auto shift = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, Sub(C(16), highest), exponent);
+	const auto decoded_exp = Choose(nonzero, Sub(exponent, shift),
+	    Choose(Both(outward, Binary(s, spv::OpUGreaterThan, b, exponent, C(17))), Sub(exponent, C(17)), C(0)));
+	// Zero mantissas take a separate result, but their unused shifts must still be defined.
+	const auto amount = Add(C(7), Choose(nonzero, shift, C(0)));
+	const auto trail = Choose(Both(outward, Not(Eq(decoded_exp, C(255)))),
+	    Sub(Binary(s, spv::OpShiftLeftLogical, u, C(1), amount), C(1)), C(0));
+	const auto fraction = Choose(nonzero,
+	    And(Or(Binary(s, spv::OpShiftLeftLogical, u, mantissa, amount), trail), C(0x7fffff)),
+	    Choose(outward, C(0x7fffff), C(0)));
+	End(Float(Or(Or(fraction, Shl(decoded_exp, 23)), Choose(negative, C(0x80000000), C(0)))));
+}
+
+void Bvh8Emitter::DefineBoxes() {
+	const auto result_type = s.builder.Type(spv::OpTypeStruct, v4, v4);
+	const auto a = Begin(boxes_function, "trinity_box_slot", result_type,
+	    std::array{w, w, b, b, u, u, f, v3, v3});
+	const auto first = a[0], second = a[1], full = a[2], compressed = a[3];
+	const auto flags = a[4], descriptor = a[5], extent = a[6];
+	const auto org = Unpack3(a[7]), dir = Unpack3(a[8]);
+	const auto zero = ConstantF32Value(s, 0.0f), infinity = ConstantF32(s, 0x7f800000);
+	Vec3 inverse;
+	for (uint32_t i = 0; i < 3; ++i) inverse[i] = Fop(spv::OpFDiv, ConstantF32Value(s, 1.0f), dir[i]);
+	const auto packed = [&](uint32_t offset, uint32_t width) {
+		const auto word = Shr(offset, 5), shift = And(offset, C(31));
+		const auto lo = Binary(s, spv::OpShiftRightLogical, u, Word(first, word), shift);
+		const auto next = EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s, Add(word, C(1)), C(15));
+		const auto hi = Binary(s, spv::OpShiftLeftLogical, u, Word(first, next), And(Sub(C(32), shift), C(31)));
+		return And(Or(lo, Choose(Eq(shift, C(0)), C(0), hi)), C((1u << width) - 1));
+	};
+	const auto format = Choose(compressed, C(0), Choose(full, C(1), C(2)));
+	const auto entry_label = s.current_label;
+	const auto header = s.builder.AllocateId(), body = s.builder.AllocateId();
+	const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId();
+	const auto initial_nodes = Make4(C(~0u), C(~0u), C(~0u), C(~0u));
+	const auto initial_keys = Make4(C(0x7f800000), C(0x7f800000), C(0x7f800000), C(0x7f800000));
+	const auto base = And(Word(first, C(0)), C(0x1fffffff));
+	s.builder.AddFunction(spv::OpBranch, header);
+	EmitLabel(s, header);
+	const auto child = s.builder.AllocateId(), prefix = s.builder.AllocateId();
+	const auto nodes = s.builder.AllocateId(), keys = s.builder.AllocateId();
+	const auto child_phi = s.builder.AddDeferredPhi(u, child, 2);
+	const auto prefix_phi = s.builder.AddDeferredPhi(u, prefix, 2);
+	const auto nodes_phi = s.builder.AddDeferredPhi(v4, nodes, 2);
+	const auto keys_phi = s.builder.AddDeferredPhi(v4, keys, 2);
+	const auto more = Binary(s, spv::OpULessThan, b, child, C(4));
+	s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask);
+	s.builder.AddFunction(spv::OpBranchConditional, more, body, merge);
+	EmitLabel(s, body);
+	const auto kind = packed(Add(C(29), Binary(s, spv::OpIMul, u, child, C(3))), 3);
+	const auto direct_node = Word(first, child);
+	const auto node = Choose(compressed, Or(Shl(prefix, 3), kind), direct_node);
+	const auto child_flags = Choose(compressed,
+	    And(packed(Add(C(41), Shl(child, 2)), 4), Choose(Eq(child, C(3)), C(7), C(15))), C(0));
+	const auto step = Choose(Eq(kind, C(2)), C(4),
+	    Choose(Either(Either(Eq(kind, C(3)), Eq(kind, C(5))), Eq(kind, C(6))), C(2), C(1)));
+	const auto next_prefix = Add(prefix, step);
+	const auto bounds_type = s.builder.Type(spv::OpTypeStruct, v3, v3);
+	const auto box = EmitIndexSwitch(s, format, 3, bounds_type, [&](uint32_t encoding) {
+		std::array<uint32_t, 6> bounds;
+		if (encoding == 0) {
+			const auto yz = Word(first, C(2));
+			const std::array exponents {Shr(Word(first, C(1)), 24), And(yz, C(255)), And(Shr(yz, 8), C(255))};
+			for (uint32_t component = 0; component < 6; ++component) {
+				const auto offset = Add(C(80 + component * 18), Binary(s, spv::OpIMul, u, child, C(108)));
+				bounds[component] = Call(f, decompress_function,
+				    std::array{exponents[component % 3], packed(offset, 18), ConstantBool(s, component < 3)});
+			}
+		} else if (encoding == 1) {
+			for (uint32_t component = 0; component < 6; ++component) {
+				const auto word = Add(C(4 + component), Binary(s, spv::OpIMul, u, child, C(6)));
+				const auto block = Select(s, w, Binary(s, spv::OpULessThan, b, word, C(16)), first, second);
+				bounds[component] = Float(Word(block, And(word, C(15))));
+			}
+		} else {
+			for (uint32_t pair = 0; pair < 3; ++pair) {
+				const auto word = Add(C(4 + pair), Binary(s, spv::OpIMul, u, child, C(3)));
+				const auto value = EmitGlsl<GLSLstd450UnpackHalf2x16, IR::Type::F32x2>(s, Word(first, word));
+				bounds[pair * 2] = Extract(f, value, 0);
+				bounds[pair * 2 + 1] = Extract(f, value, 1);
+			}
+		}
+		return Make(bounds_type, std::array{Make(v3, std::span(bounds).first(3)),
+		    Make(v3, std::span(bounds).last(3))});
+	});
+	const auto lo_bound = Unpack3(Extract(v3, box, 0)), hi_bound = Unpack3(Extract(v3, box, 1));
+	Vec3 near, far;
+	auto valid = Not(Eq(And(child_flags, C(3)), C(3)));
+	for (uint32_t axis = 0; axis < 3; ++axis) {
+		valid = Both(valid, Fcmp(spv::OpFOrdLessThanEqual, lo_bound[axis], hi_bound[axis]));
+		const auto lo = Fop(spv::OpFMul, Fop(spv::OpFSub, lo_bound[axis], org[axis]), inverse[axis]);
+		const auto hi = Fop(spv::OpFMul, Fop(spv::OpFSub, hi_bound[axis], org[axis]), inverse[axis]);
+		const auto forward = Fcmp(spv::OpFOrdGreaterThanEqual, inverse[axis], zero);
+		near[axis] = Select(s, f, forward, lo, hi);
+		far[axis] = Select(s, f, forward, hi, lo);
+	}
+	const auto entry = Max(Max(near[0], near[1]), near[2]);
+	const auto exit = Min(Min(far[0], far[1]), far[2]);
+	const auto exit_bits = Raw(Select(s, f, Fcmp(spv::OpFOrdEqual, exit, zero), zero, exit));
+	const auto grown = Float(EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
+	    Add(exit_bits, And(Shr(descriptor, 23), C(255))), C(0x7f800000)));
+	valid = Both(valid, Both(Fcmp(spv::OpFOrdGreaterThanEqual, exit, zero),
+	    Both(Fcmp(spv::OpFOrdLessThan, entry, extent), Fcmp(spv::OpFOrdLessThanEqual, entry, grown))));
+	const auto opaque = Both(Either(Bit(child_flags, 0), Bit(flags, 0)), Not(Bit(flags, 1)));
+	const auto nonopaque = Both(Either(Bit(child_flags, 1), Bit(flags, 1)), Not(Bit(flags, 0)));
+	const auto culled = Either(Either(Both(opaque, Bit(flags, 6)), Both(nonopaque, Bit(flags, 7))),
+	    Either(Both(Bit(child_flags, 2), Bit(flags, 8)), Both(Bit(child_flags, 3), Bit(flags, 9))));
+	valid = Both(valid, Not(culled));
+	const auto heuristic = And(Shr(descriptor, 21), C(3));
+	auto key = Max(entry, zero);
+	key = Select(s, f, Eq(heuristic, C(1)), Unary(s, spv::OpFNegate, f, Fop(spv::OpFSub, exit, entry)), key);
+	key = Select(s, f, Eq(heuristic, C(2)), Fop(spv::OpFAdd, entry, exit), key);
+	const auto next_nodes = s.builder.AllocateId(), next_keys = s.builder.AllocateId();
+	s.builder.AddFunction(spv::OpVectorInsertDynamic, v4, next_nodes, nodes, Choose(valid, node, C(~0u)), child);
+	s.builder.AddFunction(spv::OpVectorInsertDynamic, v4, next_keys, keys, Raw(Select(s, f, valid, key, infinity)), child);
+	s.builder.AddFunction(spv::OpBranch, cont);
+	EmitLabel(s, cont);
+	const auto next_child = Add(child, C(1));
+	s.builder.AddFunction(spv::OpBranch, header);
+	s.builder.PatchDeferredPhi(child_phi, 0, C(0), entry_label);
+	s.builder.PatchDeferredPhi(child_phi, 1, next_child, cont);
+	s.builder.PatchDeferredPhi(prefix_phi, 0, base, entry_label);
+	s.builder.PatchDeferredPhi(prefix_phi, 1, next_prefix, cont);
+	s.builder.PatchDeferredPhi(nodes_phi, 0, initial_nodes, entry_label);
+	s.builder.PatchDeferredPhi(nodes_phi, 1, next_nodes, cont);
+	s.builder.PatchDeferredPhi(keys_phi, 0, initial_keys, entry_label);
+	s.builder.PatchDeferredPhi(keys_phi, 1, next_keys, cont);
+	EmitLabel(s, merge);
+	End(Make(result_type, std::array{nodes, keys}));
+}
+
+uint32_t Bvh8Emitter::Boxes(uint32_t first, uint32_t second, uint32_t full, uint32_t compressed,
+                            uint32_t flags, uint32_t descriptor, uint32_t extent,
+                            uint32_t org, uint32_t dir) {
+	return Call(s.builder.Type(spv::OpTypeStruct, v4, v4), boxes_function,
+	    std::array{first, second, full, compressed, flags, descriptor, extent, org, dir});
+}
+
+void Bvh8Emitter::DefineIntersect() {
+	const auto result_type = TypeBvh8Result(s);
+	const auto a = Begin(s.bvh8_intersect_function, "trinity_bvh_intersect", result_type,
+	    std::array{v4, w, u, f, u, v3, v3});
+	const auto descriptor = a[0], encoded_base = a[1], node = a[2], extent = a[3], mask = a[4];
+	const auto org = Unpack3(a[5]), dir = Unpack3(a[6]);
+	std::array<uint32_t, 4> d;
+	for (uint32_t i = 0; i < 4; ++i) d[i] = Extract(u, descriptor, i);
+	const auto kind = And(node, C(7));
+	const auto triangle = Binary(s, spv::OpULessThan, b, kind, C(2));
+	const auto instance = Eq(kind, C(6));
+	const auto full = Either(Eq(kind, C(2)), Eq(kind, C(5)));
+	const auto big = Either(Eq(kind, C(2)), Eq(kind, C(3)));
+	const auto compressed = Both(Not(full), Bit(d[3], 20));
+	const auto flags = NormalizeFlags(Choose(Bit(d[3], 23),
+	    Unary(s, spv::OpUConvert, u, Binary(s, spv::OpShiftRightLogical, w, encoded_base, C(54))), C(0)));
+	const auto base = Binary(s, spv::OpShiftLeftLogical, w, PackU64(s, d[0], And(d[1], C(255))), C(8));
+	const auto lane_base = Binary(s, spv::OpShiftLeftLogical, w,
+	    Binary(s, spv::OpBitwiseAnd, w, encoded_base, ConstantU64(s, (uint64_t{1} << 54) - 1)), C(3));
+	const auto offset = Binary(s, spv::OpShiftLeftLogical, w, Unary(s, spv::OpUConvert, w, Shr(node, 3)), C(6));
+	const auto address = Binary(s, spv::OpIAdd, w, base, Binary(s, spv::OpIAdd, w, lane_base, offset));
+	const auto capacity = Binary(s, spv::OpIAdd, w,
+	    PackU64(s, d[2], And(d[3], C(0x3ff))), ConstantU64(s, 1));
+	const auto limit = Binary(s, spv::OpIAdd, w, base,
+	    Binary(s, spv::OpShiftLeftLogical, w, capacity, C(6)));
+	auto valid = Both(Eq(Shr(d[3], 28), C(8)),
+	    Eq(And(Unary(s, spv::OpUConvert, u, encoded_base), C(7)), C(0)));
+	valid = Both(valid, Both(Not(Eq(kind, C(7))), Either(Not(instance), Bit(d[3], 22))));
+	valid = Both(valid, Not(Unary(s, spv::OpIsNan, b, extent)));
+	for (uint32_t i = 0; i < 3; ++i) {
+		valid = Both(valid, Not(Either(Unary(s, spv::OpIsNan, b, org[i]), Unary(s, spv::OpIsInf, b, org[i]))));
+		valid = Both(valid, Not(Unary(s, spv::OpIsNan, b, dir[i])));
+	}
+	const auto opaque = Bit(flags, 0), nonopaque = Bit(flags, 1);
+	valid = Both(valid, Not(Either(Either(Both(opaque, Bit(flags, 6)), Both(nonopaque, Bit(flags, 7))),
+	    Both(Bit(flags, 6), Bit(flags, 7)))));
+	const auto fetch_size = Select(s, w, full, ConstantU64(s, 128), ConstantU64(s, 64));
+	const auto fetch = [&](uint32_t at, uint32_t enabled) {
+		const auto end = Binary(s, spv::OpIAdd, w, at, fetch_size);
+		const auto in_bounds = Both(Binary(s, spv::OpULessThanEqual, b, end, limit),
+		    Binary(s, spv::OpULessThanEqual, b, end, ConstantU64(s, (uint64_t{1} << 48) - 1)));
+		return EmitValueOrDefaultIfCondition(s, Both(Both(valid, enabled), in_bounds), w,
+		    ConstantU64(s, 0), [&] { return GetBdaPointer(s, at); });
+	};
+	const auto first = fetch(address, ConstantBool(s, true));
+	const auto present = [&](uint32_t pointer) { return Not(Binary(s, spv::OpIEqual, b, pointer, ConstantU64(s, 0))); };
+	const auto tail = [&](uint32_t at, uint32_t pointer) {
+		return EmitValueOrDefaultIfCondition(s, Both(full, present(pointer)), w, ConstantU64(s, 0), [&] {
+			return GetBdaPointer(s, Binary(s, spv::OpIAdd, w, at, ConstantU64(s, 64)));
+		});
+	};
+	const auto second = tail(address, first);
+	const auto available = Both(present(first), Either(Not(full), present(second)));
+	std::array<uint32_t, 10> invalid;
+	invalid.fill(C(~0u));
+	for (uint32_t slot = 0; slot < 2; ++slot) {
+		invalid[slot * 4] = Choose(triangle, C(0x7f800000), C(~0u));
+		invalid[slot * 4 + 1] = Choose(triangle, C(0x3f800000), C(~0u));
+		invalid[slot * 4 + 3] = Choose(triangle, C(0), C(~0u));
+	}
+	const auto default_result = Result(invalid, org, dir, C(0));
+	const auto value = EmitValueOrDefaultIfCondition(s, Both(instance, available), result_type, default_result, [&] {
+		return Instance(first, flags, mask, org, dir);
+	});
+	const auto result = EmitValueOrDefaultIfCondition(s, Both(Not(instance), valid), result_type, value, [&] {
+		const auto second_address = Binary(s, spv::OpIAdd, w, address, fetch_size);
+		const auto next_first = fetch(second_address, big);
+		const auto next_second = tail(second_address, next_first);
+		const auto next_available = Both(present(next_first), Either(Not(full), present(next_second)));
+		const auto triangles = EmitValueOrDefaultIfCondition(s, triangle, result_type, default_result, [&] {
+			auto dst = invalid;
+			for (uint32_t slot = 0; slot < 2; ++slot) {
+				const auto active = Both(available, slot == 0 ? ConstantBool(s, true) : Eq(kind, C(1)));
+				const auto absent = Both(available, Both(Eq(kind, C(0)), Bit(d[3], 24)));
+				const auto miss = Make4(C(0x7f800000), C(0x3f800000),
+				    slot == 1 ? Choose(absent, C(0), C(~0u)) : C(~0u), C(0));
+				const auto hit = EmitValueOrDefaultIfCondition(s, active, v4, miss, [&] {
+					return Call(v4, triangle_function, std::array{first, C(slot), flags, d[3], a[5], a[6]});
+				});
+				for (uint32_t i = 0; i < 4; ++i) dst[slot * 4 + i] = Extract(u, hit, i);
+				dst[8 + slot] = EmitValueOrDefaultIfCondition(s, active, u, C(~0u), [&] { return Word(first, C(14)); });
+			}
+			return Result(dst, org, dir, C(0));
+		});
+		return EmitValueOrDefaultIfCondition(s, Not(triangle), result_type, triangles, [&] {
+			auto dst = invalid;
+			const auto slot_type = s.builder.Type(spv::OpTypeStruct, v4, v4);
+			const auto miss = Make(slot_type, std::array{Make4(C(~0u), C(~0u), C(~0u), C(~0u)),
+			    Make4(C(0x7f800000), C(0x7f800000), C(0x7f800000), C(0x7f800000))});
+			std::array<uint32_t, 8> keys;
+			for (uint32_t slot = 0; slot < 2; ++slot) {
+				const auto hit = EmitValueOrDefaultIfCondition(s, slot == 0 ? available : next_available,
+				    slot_type, miss, [&] {
+					return Boxes(slot == 0 ? first : next_first, slot == 0 ? second : next_second,
+					    full, compressed, flags, d[1], extent, a[5], a[6]);
+				});
+				const auto indices = Extract(v4, hit, 0), depths = Extract(v4, hit, 1);
+				for (uint32_t i = 0; i < 4; ++i) {
+					dst[slot * 4 + i] = Extract(u, indices, i);
+					keys[slot * 4 + i] = Extract(u, depths, i);
+				}
+			}
+			const auto sort = Bit(d[1], 31), wide = Both(sort, Bit(d[3], 21)), tri_first = Bit(d[1], 20);
+			constexpr std::array<std::pair<uint32_t, uint32_t>, 19> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2},
+			    {4,5}, {6,7}, {4,6}, {5,7}, {5,6}, {0,4}, {1,5}, {1,4}, {2,6}, {3,7}, {3,6}, {2,4}, {3,5}, {3,4}}};
+			const auto unsorted = Make(slot_type, std::array{Make(v4, std::span(dst).first(4)),
+			    Make(v4, std::span(dst).subspan(4, 4))});
+			const auto sorted_result = EmitValueOrDefaultIfCondition(s, sort, slot_type, unsorted, [&] {
+				auto sorted_dst = dst;
+				for (size_t step = 0; step < network.size(); ++step) {
+					const auto [x, y] = network[step];
+					const auto is_tri = [&](uint32_t index) { return Binary(s, spv::OpULessThan, b, And(index, C(7)), C(2)); };
+					const auto xt = is_tri(sorted_dst[x]), yt = is_tri(sorted_dst[y]);
+					auto swap = Fcmp(spv::OpFOrdGreaterThan, Float(keys[x]), Float(keys[y]));
+					swap = Select(s, b, Both(tri_first, Binary(s, spv::OpLogicalNotEqual, b, xt, yt)), yt, swap);
+					if (step >= 10) swap = Both(wide, swap);
+					const auto key = keys[x], index = sorted_dst[x];
+					const auto later = [&](uint32_t position) {
+						return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) {
+							return pair.first == position || pair.second == position;
+						});
+					};
+					if (later(x)) keys[x] = Choose(swap, keys[y], key);
+					if (later(y)) keys[y] = Choose(swap, key, keys[y]);
+					sorted_dst[x] = Choose(swap, sorted_dst[y], index);
+					sorted_dst[y] = Choose(swap, index, sorted_dst[y]);
+				}
+				return Make(slot_type, std::array{Make(v4, std::span(sorted_dst).first(4)),
+				    Make(v4, std::span(sorted_dst).subspan(4, 4))});
+			});
+			for (uint32_t slot = 0; slot < 2; ++slot) {
+				const auto indices = Extract(v4, sorted_result, slot);
+				for (uint32_t i = 0; i < 4; ++i) dst[slot * 4 + i] = Extract(u, indices, i);
+			}
+			return Result(dst, org, dir, C(0));
+		});
+	});
+	End(result);
+}
+
+} // namespace
+
+void DefineTrinityBvhIntersect(EmitterState& s) {
+	if (!s.program.info.bvh8) return;
+	Bvh8Emitter e {s};
+	e.DefineDecompress();
+	e.DefineTriangle();
+	e.DefineBoxes();
+	e.DefineIntersect();
+}
+
+uint32_t EmitBvh8Extract(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto index = inst.Arg(1).U32();
+	if (index > 16) ctx.Fail(inst, "invalid BVH8 result component");
+	const auto result = ctx.state.builder.AllocateId();
+	if (index == 16)
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), result,
+		    ctx.Arg(inst, 0), 4u);
+	else
+		ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), result,
+		    ctx.Arg(inst, 0), index / 4, index % 4);
+	return result;
+}
+
+uint32_t EmitBvh8Intersect(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& s = ctx.state;
+	const auto* ray = ctx.ImageAddress(inst.Arg(1));
+	if (ray == nullptr || ray->NumArgs() < 11 || s.bvh8_intersect_function == 0)
+		ctx.Fail(inst, "BVH8 intersection requires the complete ray and shared function");
+	Bvh8Emitter e {s};
+	const auto vector = [&](uint32_t first) {
+		return e.Make(e.v3, Vec3{e.Float(ctx.Arg(*ray, first)), e.Float(ctx.Arg(*ray, first + 1)),
+		    e.Float(ctx.Arg(*ray, first + 2))});
+	};
+	const std::array args {ctx.Arg(inst, 0), PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1)),
+	    ctx.Arg(*ray, 10), e.Float(ctx.Arg(*ray, 2)), ctx.Arg(*ray, 3), vector(4), vector(7)};
+	const auto type = TypeBvh8Result(s);
+	return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), type,
+	    s.builder.Constant(spv::OpConstantNull, type), [&] {
+		return e.Call(type, s.bvh8_intersect_function, args);
+	});
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
@@ -14,12 +14,6 @@ struct MimgSampleInfo {
 	uint32_t    flags    = 0;
 };

-struct MimgGatherInfo {
-	uint32_t encoding = 0;
-	Opcode   decoded  = Opcode::UNSUPPORTED;
-	uint32_t flags    = 0;
-};
-
 struct MimgAtomicInfo {
 	uint32_t encoding    = 0;
 	Opcode   decoded     = Opcode::UNSUPPORTED;
@@ -181,23 +175,72 @@ constexpr MimgSampleInfo MIMG_SAMPLE_OPCODE_LIST[] = {
         ImageSampleFlagOffset | ImageSampleFlagAdjust},
 };

-constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
-    {0x44u, Opcode::IMAGE_GATHER4_L, ImageSampleFlagLod},
-    {0x47u, Opcode::IMAGE_GATHER4_LZ, ImageSampleFlagLevelZero},
-    {0x48u, Opcode::IMAGE_GATHER4_C, ImageSampleFlagCompare},
-    {0x4cu, Opcode::IMAGE_GATHER4_C_L, ImageSampleFlagCompare | ImageSampleFlagLod},
-    {0x4fu, Opcode::IMAGE_GATHER4_C_LZ,
-     ImageSampleFlagCompare | ImageSampleFlagLevelZero},
-    {0x57u, Opcode::IMAGE_GATHER4_LZ_O,
-     ImageSampleFlagLevelZero | ImageSampleFlagOffset},
-    {0x58u, Opcode::IMAGE_GATHER4_C_O,
-     ImageSampleFlagCompare | ImageSampleFlagOffset},
-    {0x5fu, Opcode::IMAGE_GATHER4_C_LZ_O,
+constexpr MimgSampleInfo MIMG_GATHER_OPCODE_LIST[] = {
+    {0x40u, "image_gather4", 0},
+    {0x41u, "image_gather4_cl", ImageSampleFlagLodClamp},
+    {0x44u, "image_gather4_l", ImageSampleFlagLod},
+    {0x45u, "image_gather4_b", ImageSampleFlagBias},
+    {0x46u, "image_gather4_b_cl", ImageSampleFlagBias | ImageSampleFlagLodClamp},
+    {0x47u, "image_gather4_lz", ImageSampleFlagLevelZero},
+    {0x48u, "image_gather4_c", ImageSampleFlagCompare},
+    {0x49u, "image_gather4_c_cl", ImageSampleFlagCompare | ImageSampleFlagLodClamp},
+    {0x4cu, "image_gather4_c_l", ImageSampleFlagCompare | ImageSampleFlagLod},
+    {0x4du, "image_gather4_c_b", ImageSampleFlagCompare | ImageSampleFlagBias},
+    {0x4eu, "image_gather4_c_b_cl",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp},
+    {0x4fu, "image_gather4_c_lz", ImageSampleFlagCompare | ImageSampleFlagLevelZero},
+    {0x50u, "image_gather4_o", ImageSampleFlagOffset},
+    {0x51u, "image_gather4_cl_o", ImageSampleFlagLodClamp | ImageSampleFlagOffset},
+    {0x54u, "image_gather4_l_o", ImageSampleFlagLod | ImageSampleFlagOffset},
+    {0x55u, "image_gather4_b_o", ImageSampleFlagBias | ImageSampleFlagOffset},
+    {0x56u, "image_gather4_b_cl_o",
+     ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset},
+    {0x57u, "image_gather4_lz_o", ImageSampleFlagLevelZero | ImageSampleFlagOffset},
+    {0x58u, "image_gather4_c_o", ImageSampleFlagCompare | ImageSampleFlagOffset},
+    {0x59u, "image_gather4_c_cl_o",
+     ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset},
+    {0x5cu, "image_gather4_c_l_o",
+     ImageSampleFlagCompare | ImageSampleFlagLod | ImageSampleFlagOffset},
+    {0x5du, "image_gather4_c_b_o",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset},
+    {0x5eu, "image_gather4_c_b_cl_o",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+         ImageSampleFlagOffset},
+    {0x5fu, "image_gather4_c_lz_o",
     ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset},
-    {0x61u, Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal},
-    {0xc8u, Opcode::IMAGE_GATHER4_C, ImageSampleFlagCompare | ImageSampleFlagAdjust},
-    {0xd8u, Opcode::IMAGE_GATHER4_C_O,
+    {0x61u, "image_gather4h", ImageSampleFlagGatherHorizontal},
+    {0xc0u, "image_gather4_a", ImageSampleFlagAdjust},
+    {0xc1u, "image_gather4_cl_a", ImageSampleFlagLodClamp | ImageSampleFlagAdjust},
+    {0xc5u, "image_gather4_b_a", ImageSampleFlagBias | ImageSampleFlagAdjust},
+    {0xc6u, "image_gather4_b_cl_a",
+     ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagAdjust},
+    {0xc8u, "image_gather4_c_a", ImageSampleFlagCompare | ImageSampleFlagAdjust},
+    {0xc9u, "image_gather4_c_cl_a",
+     ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagAdjust},
+    {0xcdu, "image_gather4_c_b_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagAdjust},
+    {0xceu, "image_gather4_c_b_cl_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+         ImageSampleFlagAdjust},
+    {0xd0u, "image_gather4_o_a", ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd1u, "image_gather4_cl_o_a",
+     ImageSampleFlagLodClamp | ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd5u, "image_gather4_b_o_a",
+     ImageSampleFlagBias | ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd6u, "image_gather4_b_cl_o_a",
+     ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+         ImageSampleFlagAdjust},
+    {0xd8u, "image_gather4_c_o_a",
     ImageSampleFlagCompare | ImageSampleFlagOffset | ImageSampleFlagAdjust},
+    {0xd9u, "image_gather4_c_cl_o_a",
+     ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+         ImageSampleFlagAdjust},
+    {0xddu, "image_gather4_c_b_o_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset |
+         ImageSampleFlagAdjust},
+    {0xdeu, "image_gather4_c_b_cl_o_a",
+     ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+         ImageSampleFlagOffset | ImageSampleFlagAdjust},
 };

 constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = {
@@ -219,13 +262,13 @@ constexpr auto MIMG_SAMPLE_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_SAMPLE_OPCO
 constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCODE_LIST);
 constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST);

-Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather,
+Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgSampleInfo* gather,
                        const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
 		return Opcode::IMAGE_SAMPLE;
 	}
 	if (gather != nullptr) {
-		return gather->decoded;
+		return Opcode::IMAGE_GATHER4;
 	}
 	if (atomic != nullptr) {
 		return atomic->decoded;
@@ -238,36 +281,24 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 		case 0x09u: return Opcode::IMAGE_STORE_MIP;
 		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
 		case 0x60u: return Opcode::IMAGE_GET_LOD;
+		case 0xe5u: return Opcode::IMAGE_BVH8_INTERSECT_RAY;
 		case 0xe6u:
 		case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY;
 		default: return Opcode::UNSUPPORTED;
 	}
 }

-uint32_t DecodeMimgSampleFlags(const MimgSampleInfo* sample, const MimgGatherInfo* gather) {
-	if (sample != nullptr) {
-		return sample->flags;
-	}
-	if (gather != nullptr) {
-		return gather->flags;
-	}
-	return 0;
-}
-
 uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension,
-                                     const MimgSampleInfo* sample, const MimgGatherInfo* gather,
-                                     const MimgAtomicInfo* atomic) {
-	if (sample != nullptr) {
-		return ImageSampleAddressComponents(sample->flags, dimension);
-	}
-	if (gather != nullptr) {
-		return ImageSampleAddressComponents(gather->flags, dimension);
+                                     const MimgSampleInfo* sampled, const MimgAtomicInfo* atomic) {
+	if (sampled != nullptr) {
+		return ImageSampleAddressComponents(sampled->flags, dimension);
 	}
 	if (atomic != nullptr) {
 		return ImageCoordComponents(dimension);
 	}

 	switch (opcode) {
+		case 0xe5u:
 		case 0xe6u: return 11u;
 		case 0xe7u: return 12u;
 		case 0x0eu: return 1u;
@@ -290,33 +321,31 @@ uint32_t CountDmaskComponents(uint32_t dmask) {

 } // namespace

-ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, uint32_t component) {
-	const auto width = [flags](uint32_t index) {
-		if ((flags & ImageSampleFlagA16) == 0u) return 32u;
-		uint32_t cursor = 0;
-		if ((flags & ImageSampleFlagOffset) != 0u) {
-			if (index == cursor++) return 32u;
-		}
-		if ((flags & ImageSampleFlagBias) != 0u) {
-			if (index == cursor++) return 16u;
-		}
-		if ((flags & ImageSampleFlagCompare) != 0u && index == cursor) return 32u;
-		return 16u;
-	};
-	uint32_t offset = 0;
-	for (uint32_t index = 0; index < component; index++) {
-		const auto bits = width(index);
-		if (bits == 32u) offset = (offset + 31u) & ~31u;
-		offset += bits;
+ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, ImageDimension dimension,
+                                                  uint32_t component) {
+	if ((flags & ImageSampleFlagA16) == 0u) return {component * 32u, 32u};
+	// A16 offsets, bias and PCF reference each take a whole VGPR, the bias as a half in its low
+	// 16 bits. The other components pack two per VGPR, except that each 1D derivative and the last
+	// two 3D ones take the low half of their own VGPR.
+	const bool     has_bias    = (flags & ImageSampleFlagBias) != 0u;
+	const uint32_t bias_index  = (flags & ImageSampleFlagOffset) != 0u ? 1u : 0u;
+	const uint32_t whole_vgprs = std::popcount(
+	    flags & (ImageSampleFlagOffset | ImageSampleFlagBias | ImageSampleFlagCompare));
+	if (component < whole_vgprs) {
+		return {component * 32u, has_bias && component == bias_index ? 16u : 32u};
 	}
-	const auto bits = width(component);
-	if (bits == 32u) offset = (offset + 31u) & ~31u;
-	return {offset, bits};
+	const uint32_t axes =
+	    (flags & ImageSampleFlagDerivative) != 0u ? ImageGradientComponents(dimension) : 0u;
+	const uint32_t unpaired = axes % 2u * 2u;
+	const uint32_t paired   = axes * 2u - unpaired;
+	const uint32_t index    = component - whole_vgprs;
+	const uint32_t half     = index + std::clamp(index, paired, paired + unpaired) - paired;
+	return {whole_vgprs * 32u + half * 16u, 16u};
 }

-uint32_t ImageAddressDwordCount(uint32_t flags, uint32_t components) {
+uint32_t ImageAddressDwordCount(uint32_t flags, ImageDimension dimension, uint32_t components) {
 	if (components == 0u) return 0u;
-	const auto last = ImageAddressComponentLayout(flags, components - 1u);
+	const auto last = ImageAddressComponentLayout(flags, dimension, components - 1u);
 	return (last.bit_offset + last.bit_width + 31u) / 32u;
 }

@@ -329,16 +358,17 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const auto     dimension  = DecodeImageDimension((word0 >> 3u) & 0x7u);
 	const uint32_t word_count = 2u + nsa_dwords;

-	const uint32_t vdata  = (word1 >> 8u) & 0xffu;
-	const uint32_t vaddr  = word1 & 0xffu;
-	const uint32_t srsrc  = (word1 >> 16u) & 0x1fu;
-	const uint32_t ssamp  = (word1 >> 21u) & 0x1fu;
-	const bool     r128   = ((word0 >> 15u) & 0x1u) != 0u;
-	const bool     a16    = ((word1 >> 30u) & 0x1u) != 0u;
-	const bool     d16    = ((word1 >> 31u) & 0x1u) != 0u;
-	const auto*    sample = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
-	const auto*    gather = Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
-	const auto*    atomic = Detail::FindOpcode(MIMG_ATOMIC_OPS, opcode);
+	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
+	const uint32_t vaddr   = word1 & 0xffu;
+	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
+	const uint32_t ssamp   = (word1 >> 21u) & 0x1fu;
+	const bool     r128    = ((word0 >> 15u) & 0x1u) != 0u;
+	const bool     a16     = ((word1 >> 30u) & 0x1u) != 0u;
+	const bool     d16     = ((word1 >> 31u) & 0x1u) != 0u;
+	const auto*    sample  = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
+	const auto*    gather  = Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
+	const auto*    atomic  = Detail::FindOpcode(MIMG_ATOMIC_OPS, opcode);
+	const auto*    sampled = sample != nullptr ? sample : gather;

 	inst.pc                 = pc;
 	inst.word_count         = word_count;
@@ -351,11 +381,11 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.data_dwords        = d16 ? (inst.data_components + 1u) / 2u : inst.data_components;
 	inst.glc                = ((word0 >> 13u) & 1u) != 0;
 	inst.slc                = ((word0 >> 25u) & 1u) != 0;
-	inst.image_sample_flags = DecodeMimgSampleFlags(sample, gather);
+	inst.image_sample_flags = sampled != nullptr ? sampled->flags : 0u;
 	if (a16) {
 		inst.image_sample_flags |= ImageSampleFlagA16;
 	}
-	if ((sample != nullptr || gather != nullptr) && (word0 & (1u << 12u)) != 0u) {
+	if (sampled != nullptr && (word0 & (1u << 12u)) != 0u) {
 		inst.image_sample_flags |= ImageSampleFlagUnnormalized;
 	}
 	inst.image_dimension  = dimension;
@@ -364,8 +394,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	for (uint32_t i = 0; i < nsa_dwords * 4u; i++) {
 		inst.image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
 	}
-	inst.image_address_components =
-	    DecodeMimgAddressComponents(opcode, dimension, sample, gather, atomic);
+	inst.image_address_components = DecodeMimgAddressComponents(opcode, dimension, sampled, atomic);
 	SetRawWords(inst, code, word_index, word_count);

 	if (inst.opcode == Opcode::UNSUPPORTED) {
@@ -389,8 +418,8 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 			}
 		}
 	}
-	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
-	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
+	const bool supports_d16 = sampled != nullptr || opcode == 0x00u || opcode == 0x01u ||
+	                          opcode == 0x08u || opcode == 0x09u;
 	if (d16 && !supports_d16) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
 	}
@@ -400,6 +429,36 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		               "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf");
 	}

+	if (opcode == 0xe5u) {
+		inst.data_components = inst.data_dwords = 10;
+		if (a16 || d16 || !r128 || inst.dmask != 0xfu ||
+		    (nsa_dwords != 0u && nsa_dwords != 2u) || vdata > 246u ||
+		    (nsa_dwords == 0u && vaddr > 245u)) {
+			SetUnsupported(inst, Family::MIMG, opcode, "invalid BVH8 ray or destination encoding");
+		}
+		if (nsa_dwords == 2u) {
+			const auto origin = inst.image_nsa_addr[3], direction = inst.image_nsa_addr[4];
+			const auto node = inst.image_nsa_addr[5];
+			if (origin > 253u || direction > 253u)
+				SetUnsupported(inst, Family::MIMG, opcode, "BVH8 ray registers exceed VGPR range");
+			for (uint32_t i = 0; i < 3; ++i) {
+				inst.image_nsa_addr[3 + i] = origin + i;
+				inst.image_nsa_addr[6 + i] = direction + i;
+			}
+			inst.image_nsa_addr[9] = node;
+		}
+		const auto origin = nsa_dwords != 0 ? inst.image_nsa_addr[3] : vaddr + 4;
+		const auto direction = nsa_dwords != 0 ? inst.image_nsa_addr[6] : vaddr + 7;
+		bool overlapping = origin < direction + 3 && direction < origin + 3;
+		for (const auto output : {2u, 3u, 6u, 7u}) {
+			const auto reg = vdata + output;
+			overlapping |= (reg >= origin && reg < origin + 3) ||
+			               (reg >= direction && reg < direction + 3);
+		}
+		if (overlapping)
+			SetUnsupported(inst, Family::MIMG, opcode, "overlapping BVH8 instance outputs are unsupported");
+	}
+
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);
 	DecodeScalarSource(srsrc * 4u, pc, inst.src1);
@@ -407,9 +466,10 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.src_count = 3;
 }

-const char* MimgSampleOpcodeName(uint32_t opcode) {
-	const auto* sample = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
-	return sample != nullptr ? sample->name : nullptr;
+const char* MimgSampledOpcodeName(uint32_t opcode) {
+	const auto* sample  = Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
+	const auto* sampled = sample != nullptr ? sample : Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
+	return sampled != nullptr ? sampled->name : nullptr;
 }

 } // namespace Libs::Graphics::ShaderRecompiler::Decoder
@@ -8,15 +8,16 @@ namespace Libs::Graphics::ShaderRecompiler::Decoder {
 void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                Instruction& inst);

-const char* MimgSampleOpcodeName(uint32_t opcode);
+const char* MimgSampledOpcodeName(uint32_t opcode);

 struct ImageAddressComponent {
 	uint32_t bit_offset;
 	uint32_t bit_width;
 };

-ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, uint32_t component);
-uint32_t ImageAddressDwordCount(uint32_t flags, uint32_t components);
+ImageAddressComponent ImageAddressComponentLayout(uint32_t flags, ImageDimension dimension,
+                                                  uint32_t component);
+uint32_t ImageAddressDwordCount(uint32_t flags, ImageDimension dimension, uint32_t components);

 } // namespace Libs::Graphics::ShaderRecompiler::Decoder

@@ -22,7 +22,7 @@ constexpr MemoryOpcodeInfo SMEM_OPCODE_LIST[] = {
    {0x04u, Opcode::S_LOAD_DWORDX16, 16, 32},      {0x08u, Opcode::S_BUFFER_LOAD_DWORD, 1, 32},
    {0x09u, Opcode::S_BUFFER_LOAD_DWORDX2, 2, 32}, {0x0au, Opcode::S_BUFFER_LOAD_DWORDX4, 4, 32},
    {0x0bu, Opcode::S_BUFFER_LOAD_DWORDX8, 8, 32}, {0x0cu, Opcode::S_BUFFER_LOAD_DWORDX16, 16, 32},
-    {0x25u, Opcode::S_MEMREALTIME, 2, 32},
+    {0x20u, Opcode::S_DCACHE_INV, 0, 0},           {0x25u, Opcode::S_MEMREALTIME, 2, 32},
 };

 constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
@@ -69,6 +69,7 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
    {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
    {0x50u, Opcode::BUFFER_ATOMIC_SWAP_X2, 2, 32},
    {0x52u, Opcode::BUFFER_ATOMIC_ADD_X2, 2, 32},
+    {0x56u, Opcode::BUFFER_ATOMIC_UMIN_X2, 2, 32},
    {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32},
    {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32},
    {0x71u, Opcode::BUFFER_GL0_INV, 0, 0},
@@ -95,14 +96,14 @@ constexpr MemoryOpcodeInfo MTBUF_OPCODE_LIST[] = {
 };

 constexpr MemoryOpcodeInfo FLAT_OPCODE_LIST[] = {
-    {0x08u, Opcode::FLAT_LOAD_UBYTE, 1, 8},     {0x09u, Opcode::FLAT_LOAD_SBYTE, 1, 8, true},
-    {0x0au, Opcode::FLAT_LOAD_USHORT, 1, 16},   {0x0bu, Opcode::FLAT_LOAD_SSHORT, 1, 16, true},
-    {0x0cu, Opcode::FLAT_LOAD_DWORD, 1, 32},    {0x0du, Opcode::FLAT_LOAD_DWORDX2, 2, 32},
-    {0x0eu, Opcode::FLAT_LOAD_DWORDX4, 4, 32},  {0x0fu, Opcode::FLAT_LOAD_DWORDX3, 3, 32},
-    {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},     {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
-    {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},   {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
-    {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32}, {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
-    {0x24u, Opcode::FLAT_LOAD_SHORT_D16, 1, 16},
+    {0x08u, Opcode::FLAT_LOAD_UBYTE, 1, 8},      {0x09u, Opcode::FLAT_LOAD_SBYTE, 1, 8, true},
+    {0x0au, Opcode::FLAT_LOAD_USHORT, 1, 16},    {0x0bu, Opcode::FLAT_LOAD_SSHORT, 1, 16, true},
+    {0x0cu, Opcode::FLAT_LOAD_DWORD, 1, 32},     {0x0du, Opcode::FLAT_LOAD_DWORDX2, 2, 32},
+    {0x0eu, Opcode::FLAT_LOAD_DWORDX4, 4, 32},   {0x0fu, Opcode::FLAT_LOAD_DWORDX3, 3, 32},
+    {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},      {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
+    {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},    {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
+    {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32},  {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
+    {0x24u, Opcode::FLAT_LOAD_SHORT_D16, 1, 16}, {0x25u, Opcode::FLAT_LOAD_SHORT_D16_HI, 1, 16},
 };

 constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
@@ -141,6 +142,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
    {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
    {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32},     {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
    {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
+    {0xe1u, Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32, 1, 32},
    {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
 };

@@ -231,6 +233,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {

 uint32_t DsSourceCount(Opcode opcode) {
 	switch (opcode) {
+		case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return 3u;
 		case Opcode::DS_WRITE2_B32:
 		case Opcode::DS_WRITE2ST64_B32:
 		case Opcode::DS_WRITE2_B64:
@@ -285,6 +288,9 @@ void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	if (inst.opcode == Opcode::UNSUPPORTED) {
 		SetUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
 	}
+	if (inst.opcode == Opcode::S_DCACHE_INV) {
+		return;
+	}

 	DecodeScalarDestination(sdst, pc, inst.dst);
 	if (inst.opcode == Opcode::S_MEMREALTIME) {
@@ -419,9 +425,10 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}

 	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst);
-	if (inst.opcode == Opcode::FLAT_LOAD_SHORT_D16) {
-		// D16 loads reuse partial destinations to preserve the untouched high half.
-		inst.dst.sdwa_sel = 4u;
+	if (inst.opcode == Opcode::FLAT_LOAD_SHORT_D16 ||
+	    inst.opcode == Opcode::FLAT_LOAD_SHORT_D16_HI) {
+		// D16 loads reuse partial destinations to preserve the untouched half.
+		inst.dst.sdwa_sel = inst.opcode == Opcode::FLAT_LOAD_SHORT_D16_HI ? 5u : 4u;
 	}
 	DecodeVectorGpr(addr, inst.src0);
 	inst.src_count = 1;
@@ -468,6 +475,13 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}
+	if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 &&
+	    (inst.gds || inst.offset < 8u || inst.offset > 31u || data1 > 248u)) {
+		SetUnsupported(inst, Family::DS, opcode, "invalid BVH stack operands or capacity");
+	}
+	if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 && vdst == addr) {
+		SetUnsupported(inst, Family::DS, opcode, "overlapping BVH stack outputs are not implemented");
+	}
 	if (inst.gds && inst.opcode == Opcode::DS_CMPST_RTN_B32) {
 		SetUnsupported(inst, Family::DS, opcode, "DS compare-store return B32 GDS variant is not implemented");
 	}
@@ -70,6 +70,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
    {0x21u, Opcode::S_SWAPPC_B64},
    {0x24u, Opcode::S_AND_SAVEEXEC_B64},
    {0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
+    {0x2cu, Opcode::S_QUADMASK_B32},
    {0x2du, Opcode::S_QUADMASK_B64},
    {0x34u, Opcode::S_ABS_I32},
    {0x37u, Opcode::S_ANDN1_SAVEEXEC_B64},
@@ -123,10 +123,9 @@ std::string ImageSampleFlagsToString(uint32_t flags) {
 }

 std::string FormatMimg(const Instruction& inst) {
-	const char* sample_name =
-	    inst.opcode == Opcode::IMAGE_SAMPLE ? MimgSampleOpcodeName(inst.opcode_id) : nullptr;
+	const char*            sampled_name = MimgSampledOpcodeName(inst.opcode_id);
 	const std::string_view name =
-	    sample_name != nullptr ? sample_name : magic_enum::enum_name(inst.opcode);
+	    sampled_name != nullptr ? sampled_name : magic_enum::enum_name(inst.opcode);
 	std::string text =
 	    fmt::format("0x{:08x}: {} {}, {}, {}, {} ; dmask=0x{:x} image_dim={}", inst.pc, name,
 	                OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(),
@@ -138,22 +137,10 @@ std::string FormatMimg(const Instruction& inst) {
 	if (inst.image_r128) {
 		text += " r128=1";
 	}
-	switch (inst.opcode) {
-		case Opcode::IMAGE_SAMPLE:
-		case Opcode::IMAGE_GATHER4_L:
-		case Opcode::IMAGE_GATHER4_LZ:
-		case Opcode::IMAGE_GATHER4_C:
-		case Opcode::IMAGE_GATHER4_C_L:
-		case Opcode::IMAGE_GATHER4_C_LZ:
-		case Opcode::IMAGE_GATHER4_LZ_O:
-		case Opcode::IMAGE_GATHER4_C_O:
-		case Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Opcode::IMAGE_GATHER4H:
-			text += fmt::format(" sample_flags={} addr_components={}",
-			                    ImageSampleFlagsToString(inst.image_sample_flags).c_str(),
-			                    inst.image_address_components);
-			break;
-		default: break;
+	if (sampled_name != nullptr) {
+		text += fmt::format(" sample_flags={} addr_components={}",
+		                    ImageSampleFlagsToString(inst.image_sample_flags).c_str(),
+		                    inst.image_address_components);
 	}
 	if (inst.image_nsa_dwords != 0) {
 		text += fmt::format(" nsa_dwords={} nsa_addr=", inst.image_nsa_dwords);
@@ -530,6 +517,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_NOT_B64:
 		case Opcode::S_WQM_B32:
 		case Opcode::S_WQM_B64:
+		case Opcode::S_QUADMASK_B32:
 		case Opcode::S_QUADMASK_B64:
 		case Opcode::S_AND_SAVEEXEC_B32:
 		case Opcode::S_ORN2_SAVEEXEC_B32:
@@ -570,6 +558,7 @@ std::string InstructionToString(const Instruction& inst) {
 			                                               OperandToString(inst.src0).c_str()));
 		case Opcode::S_BARRIER:
 			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_barrier", inst.pc));
+		case Opcode::S_DCACHE_INV:
 		case Opcode::BUFFER_GL0_INV:
 		case Opcode::BUFFER_GL1_INV:
 			return fmt::format("0x{:08x}: {}", inst.pc, magic_enum::enum_name(inst.opcode));
@@ -616,15 +605,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::IMAGE_LOAD_MIP:
 		case Opcode::IMAGE_GET_RESINFO:
 		case Opcode::IMAGE_GET_LOD:
-		case Opcode::IMAGE_GATHER4_L:
-		case Opcode::IMAGE_GATHER4_LZ:
-		case Opcode::IMAGE_GATHER4_C:
-		case Opcode::IMAGE_GATHER4_C_L:
-		case Opcode::IMAGE_GATHER4_C_LZ:
-		case Opcode::IMAGE_GATHER4_LZ_O:
-		case Opcode::IMAGE_GATHER4_C_O:
-		case Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Opcode::IMAGE_GATHER4H: return WithUnsupportedReason(inst, FormatMimg(inst));
+		case Opcode::IMAGE_GATHER4: return WithUnsupportedReason(inst, FormatMimg(inst));
 		case Opcode::S_LOAD_DWORD:
 		case Opcode::S_LOAD_DWORDX2:
 		case Opcode::S_LOAD_DWORDX4:
@@ -685,6 +666,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::BUFFER_ATOMIC_SUB:
 		case Opcode::BUFFER_ATOMIC_SMIN:
 		case Opcode::BUFFER_ATOMIC_UMIN:
+		case Opcode::BUFFER_ATOMIC_UMIN_X2:
 		case Opcode::BUFFER_ATOMIC_SMAX:
 		case Opcode::BUFFER_ATOMIC_UMAX:
 		case Opcode::BUFFER_ATOMIC_AND:
@@ -701,6 +683,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::FLAT_LOAD_USHORT:
 		case Opcode::FLAT_LOAD_SSHORT:
 		case Opcode::FLAT_LOAD_SHORT_D16:
+		case Opcode::FLAT_LOAD_SHORT_D16_HI:
 		case Opcode::FLAT_LOAD_DWORD:
 		case Opcode::FLAT_LOAD_DWORDX2:
 		case Opcode::FLAT_LOAD_DWORDX3:
@@ -757,6 +740,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_READ2ST64_B64:
 		case Opcode::DS_READ_B96:
 		case Opcode::DS_READ_B128:
+		case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32:
 		case Opcode::DS_WRITE_B8:
 		case Opcode::DS_WRITE_B16:
 		case Opcode::DS_WRITE_B8_D16_HI:
@@ -63,6 +63,7 @@ enum class Opcode {
 	S_FLBIT_I32,
 	S_FLBIT_I32_I64,
 	S_BITREPLICATE_B64_B32,
+	S_QUADMASK_B32,
 	S_QUADMASK_B64,
 	S_GETPC_B64,
 	S_SETPC_B64,
@@ -248,7 +249,9 @@ enum class Opcode {
 	V_MIN_I32,
 	V_MAX_I32,
 	V_MUL_I32_I24,
+	V_MUL_HI_I32_I24,
 	V_MUL_U32_U24,
+	V_MUL_HI_U32_U24,
 	V_MUL_LO_U32,
 	V_MUL_HI_U32,
 	V_MUL_LO_I32,
@@ -257,6 +260,7 @@ enum class Opcode {
 	V_CVT_PK_U8_F32,
 	V_MAD_F32,
 	V_MAD_I16,
+	V_MAD_U16,
 	V_MAD_I32_I24,
 	V_MAD_U32_U24,
 	V_MAD_U64_U32,
@@ -277,16 +281,20 @@ enum class Opcode {
 	V_MIN3_I32,
 	V_MIN3_U32,
 	V_MIN3_F16,
+	V_MIN3_I16,
 	V_MIN3_U16,
 	V_MAX3_F32,
 	V_MAX3_I32,
 	V_MAX3_U32,
 	V_MAX3_F16,
+	V_MAX3_I16,
+	V_MAX3_U16,
 	V_MED3_F32,
 	V_MED3_I32,
 	V_MED3_U32,
 	V_MED3_F16,
 	V_MED3_I16,
+	V_MED3_U16,
 	V_SAD_U32,
 	V_ADD3_U32,
 	V_LSHL_ADD_U32,
@@ -354,6 +362,7 @@ enum class Opcode {
 	V_ASHRREV_I32,
 	V_LSHLREV_B64,
 	V_LSHRREV_B64,
+	V_ASHRREV_I64,
 	V_LSHLREV_B16,
 	V_LSHRREV_B16,
 	V_ASHRREV_I16,
@@ -544,6 +553,7 @@ enum class Opcode {
 	S_BUFFER_LOAD_DWORDX8,
 	S_BUFFER_LOAD_DWORDX16,
 	S_MEMREALTIME,
+	S_DCACHE_INV,
 	BUFFER_LOAD_FORMAT_X,
 	BUFFER_LOAD_FORMAT_XY,
 	BUFFER_LOAD_FORMAT_XYZ,
@@ -596,6 +606,7 @@ enum class Opcode {
 	BUFFER_ATOMIC_SUB,
 	BUFFER_ATOMIC_SMIN,
 	BUFFER_ATOMIC_UMIN,
+	BUFFER_ATOMIC_UMIN_X2,
 	BUFFER_ATOMIC_SMAX,
 	BUFFER_ATOMIC_UMAX,
 	BUFFER_ATOMIC_AND,
@@ -612,6 +623,7 @@ enum class Opcode {
 	FLAT_LOAD_USHORT,
 	FLAT_LOAD_SSHORT,
 	FLAT_LOAD_SHORT_D16,
+	FLAT_LOAD_SHORT_D16_HI,
 	FLAT_LOAD_DWORD,
 	FLAT_LOAD_DWORDX2,
 	FLAT_LOAD_DWORDX3,
@@ -673,6 +685,7 @@ enum class Opcode {
 	DS_READ2ST64_B64,
 	DS_READ_B96,
 	DS_READ_B128,
+	DS_BVH_STACK_PUSH8_POP1_RTN_B32,
 	DS_WRITE_B8,
 	DS_WRITE_B16,
 	DS_WRITE_B8_D16_HI,
@@ -688,6 +701,7 @@ enum class Opcode {
 	DS_WRITE_ADDTID_B32,
 	DS_READ_ADDTID_B32,
 	IMAGE_BVH_INTERSECT_RAY,
+	IMAGE_BVH8_INTERSECT_RAY,
 	IMAGE_GET_RESINFO,
 	IMAGE_GET_LOD,
 	IMAGE_LOAD,
@@ -707,15 +721,7 @@ enum class Opcode {
 	IMAGE_ATOMIC_FMIN,
 	IMAGE_ATOMIC_FMAX,
 	IMAGE_SAMPLE,
-	IMAGE_GATHER4_L,
-	IMAGE_GATHER4_LZ,
-	IMAGE_GATHER4_C,
-	IMAGE_GATHER4_C_L,
-	IMAGE_GATHER4_C_LZ,
-	IMAGE_GATHER4_LZ_O,
-	IMAGE_GATHER4_C_O,
-	IMAGE_GATHER4_C_LZ_O,
-	IMAGE_GATHER4H,
+	IMAGE_GATHER4,
 	V_INTERP_P1_F32,
 	V_INTERP_P2_F32,
 	V_INTERP_MOV_F32,
@@ -35,7 +35,9 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
    {0x05u, Opcode::V_SUBREV_F32},
    {0x08u, Opcode::V_MUL_F32, Vop2SdwaProfile::Float32},
    {0x09u, Opcode::V_MUL_I32_I24, Vop2SdwaProfile::IntegerPartialDestination},
+    {0x0au, Opcode::V_MUL_HI_I32_I24, Vop2SdwaProfile::IntegerPartialDestination},
    {0x0bu, Opcode::V_MUL_U32_U24, Vop2SdwaProfile::IntegerPartialDestination},
+    {0x0cu, Opcode::V_MUL_HI_U32_U24, Vop2SdwaProfile::IntegerPartialDestination},
    {0x0fu, Opcode::V_MIN_F32},
    {0x10u, Opcode::V_MAX_F32},
    {0x11u, Opcode::V_MIN_I32, Vop2SdwaProfile::IntegerFullDestination},
@@ -157,6 +159,10 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
    {0x04u, Opcode::V_CVT_F64_I32},
    {0x0fu, Opcode::V_CVT_F32_F64},
    {0x10u, Opcode::V_CVT_F64_F32},
+    {0x11u, Opcode::V_CVT_F32_UBYTE0},
+    {0x12u, Opcode::V_CVT_F32_UBYTE1},
+    {0x13u, Opcode::V_CVT_F32_UBYTE2},
+    {0x14u, Opcode::V_CVT_F32_UBYTE3},
    {0x16u, Opcode::V_CVT_F64_U32},
    {0x17u, Opcode::V_TRUNC_F64},
    {0x18u, Opcode::V_CEIL_F64},
@@ -167,6 +173,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
    {0x07u, Opcode::V_CVT_U32_F32},
    {0x08u, Opcode::V_CVT_I32_F32},
    {0x0au, Opcode::V_CVT_F16_F32},
+    {0x0bu, Opcode::V_CVT_F32_F16},
    {0x0cu, Opcode::V_CVT_RPI_I32_F32},
    {0x0du, Opcode::V_CVT_FLR_I32_F32},
    {0x0eu, Opcode::V_CVT_OFF_F32_I4},
@@ -334,16 +341,20 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
    {0x152u, Opcode::V_MIN3_I32},
    {0x153u, Opcode::V_MIN3_U32},
    {0x351u, Opcode::V_MIN3_F16},
+    {0x352u, Opcode::V_MIN3_I16},
    {0x353u, Opcode::V_MIN3_U16},
    {0x154u, Opcode::V_MAX3_F32},
    {0x155u, Opcode::V_MAX3_I32},
    {0x156u, Opcode::V_MAX3_U32},
    {0x354u, Opcode::V_MAX3_F16},
+    {0x355u, Opcode::V_MAX3_I16},
+    {0x356u, Opcode::V_MAX3_U16},
    {0x157u, Opcode::V_MED3_F32},
    {0x158u, Opcode::V_MED3_I32},
    {0x159u, Opcode::V_MED3_U32},
    {0x357u, Opcode::V_MED3_F16},
    {0x358u, Opcode::V_MED3_I16},
+    {0x359u, Opcode::V_MED3_U16},
    {0x15du, Opcode::V_SAD_U32},
    {0x15eu, Opcode::V_CVT_PK_U8_F32},
    {0x178u, Opcode::V_XOR3_B32},
@@ -354,6 +365,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
    {0x16cu, Opcode::V_MUL_HI_I32},
    {0x2ffu, Opcode::V_LSHLREV_B64},
    {0x300u, Opcode::V_LSHRREV_B64},
+    {0x301u, Opcode::V_ASHRREV_I64},
    {0x303u, Opcode::V_ADD_NC_U16},
    {0x304u, Opcode::V_SUB_NC_U16},
    {0x305u, Opcode::V_MUL_LO_U16},
@@ -370,6 +382,7 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
    {0x311u, Opcode::V_PACK_B32_F16},
    {0x314u, Opcode::V_LSHLREV_B16},
    {0x319u, Opcode::V_SUBREV_I32},
+    {0x340u, Opcode::V_MAD_U16},
    {0x345u, Opcode::V_XAD_U32},
    {0x346u, Opcode::V_LSHL_ADD_U32},
    {0x347u, Opcode::V_ADD_LSHL_U32},
@@ -488,8 +501,10 @@ bool IsNativeVop3F16TernaryOpcode(Opcode opcode) {
 }

 bool IsNativeVop3B16TernaryOpcode(Opcode opcode) {
-	return opcode == Opcode::V_MIN3_U16 || opcode == Opcode::V_MED3_I16 ||
-	       opcode == Opcode::V_MAD_I16;
+	return opcode == Opcode::V_MIN3_I16 || opcode == Opcode::V_MIN3_U16 ||
+	       opcode == Opcode::V_MAX3_I16 || opcode == Opcode::V_MAX3_U16 ||
+	       opcode == Opcode::V_MED3_I16 || opcode == Opcode::V_MED3_U16 ||
+	       opcode == Opcode::V_MAD_I16 || opcode == Opcode::V_MAD_U16;
 }

 bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
@@ -1335,6 +1350,7 @@ uint32_t NativeVop3SourceCount(Opcode opcode) {
 		case Opcode::V_SUB_NC_I16:
 		case Opcode::V_LSHLREV_B64:
 		case Opcode::V_LSHRREV_B64:
+		case Opcode::V_ASHRREV_I64:
 		case Opcode::V_LSHLREV_B16:
 		case Opcode::V_LSHRREV_B16:
 		case Opcode::V_ASHRREV_I16:
@@ -1523,7 +1539,10 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {

 bool SupportsNativeVop3Clamp(Opcode opcode) {
 	return SupportsNativeVop3ResultModifiers(opcode) || UsesInexactClampControl(opcode) ||
-	       opcode == Opcode::V_MAD_I16;
+	       opcode == Opcode::V_MAD_I16 || opcode == Opcode::V_MAD_U16 ||
+	       opcode == Opcode::V_MIN3_I16 || opcode == Opcode::V_MAX3_I16 ||
+	       opcode == Opcode::V_MAX3_U16 || opcode == Opcode::V_MED3_U16 ||
+	       opcode == Opcode::V_MUL_HI_I32_I24 || opcode == Opcode::V_MUL_HI_U32_U24;
 }

 bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mix,
@@ -1554,7 +1573,17 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
 		return clamp != 0u || omod != 0u || neg != 0u;
 	}
 	switch (opcode) {
+		case Opcode::V_MUL_HI_I32_I24:
+		case Opcode::V_MUL_HI_U32_U24:
+			// The high product fits in 32 bits, so clamp has no effect.
+			return abs != 0u || op_sel != 0u || omod != 0u || neg != 0u;
 		case Opcode::V_LDEXP_F32: return (abs & ~1u) != 0u || op_sel != 0u || (neg & ~1u) != 0u;
+		case Opcode::V_CVT_F32_UBYTE0:
+		case Opcode::V_CVT_F32_UBYTE1:
+		case Opcode::V_CVT_F32_UBYTE2:
+		case Opcode::V_CVT_F32_UBYTE3:
+			// Byte conversions ignore input modifiers and support all output modifiers.
+			return op_sel != 0u;
 		case Opcode::V_INTERP_P1LL_F16:
 		case Opcode::V_INTERP_P2_F16:
 			return abs != 0u || (op_sel & 0x7u) != 0u || clamp != 0u || omod != 0u || neg != 0u;
@@ -20,12 +20,12 @@ void Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpco
 	Write16Bits(DestinationOperand(inst), result);
 }

-void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
-	const auto lhs    = ReadU16AsU32(inst.src0, true);
-	const auto rhs    = ReadU16AsU32(inst.src1, true);
-	auto       result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, true));
+void Translator::Integer16Mad(const Decoder::Instruction& inst, bool sign) {
+	const auto lhs    = ReadU16AsU32(inst.src0, sign);
+	const auto rhs    = ReadU16AsU32(inst.src1, sign);
+	auto       result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, sign));
 	if (inst.dst.clamp) {
-		result = Saturate16(result, true);
+		result = Saturate16(result, sign);
 	}
 	Write16Bits(DestinationOperand(inst), result);
 }
@@ -147,6 +147,12 @@ void Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode
 		const auto arg_type = IR::ArgTypeOf(opcode, index);
 		const auto& operand = SourceAt(inst, reverse && index < 2u ? 1u - index : index);
 		args[index]         = ReadOperand(operand, arg_type == IR::Type::Void ? type : arg_type);
+		if (opcode == IR::ValueOpcode::ShiftRightArithmetic64 && index == 0u &&
+		    operand.kind == Decoder::OperandKind::LiteralConstant) {
+			args[index] = ir.ConstructU64(
+			    ReadU32(operand),
+			    IR::U32(IR::Value((operand.value & 0x80000000u) ? 0xffffffffu : 0u)));
+		}
 		if (mask_shift_count && index == 1u) {
 			args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u)));
 		}
@@ -263,7 +269,7 @@ void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
 	WriteOperand(DestinationOperand(inst), result);
 }

-void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
+void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend, bool high) {
 	const auto extract24 = [&](IR::U32 value) {
 		return IR::U32(
 		    ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract,
@@ -271,7 +277,9 @@ void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
 	};
 	const auto lhs    = extract24(ReadU32(inst.src0));
 	const auto rhs    = extract24(ReadU32(inst.src1));
-	auto       result = ir.IMul(lhs, rhs);
+	const auto multiply = high ? (sign ? IR::ValueOpcode::SMulHi : IR::ValueOpcode::UMulHi)
+	                           : IR::ValueOpcode::IMul32;
+	auto result = IR::U32(ir.Emit(multiply, {lhs, rhs}));
 	if (addend) {
 		result = ir.IAdd(result, ReadU32(inst.src2));
 	}
@@ -291,7 +299,7 @@ void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
 	    inst.dst2.kind != Decoder::OperandKind::Unknown) {
 		// The 32x32 product fits in 64 bits, so only the addition can carry.
 		const auto carry = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan64, {result, product}));
-		WriteMask(inst.dst2, carry, VectorMask(carry));
+		WriteVectorMask(inst.dst2, carry);
 	}
 }

@@ -369,7 +377,7 @@ void Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
 	WriteOperand(DestinationOperand(inst), ir.ConstructU64(replicate(low), replicate(high)));
 }

-void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
+void Translator::S_QUADMASK(const Decoder::Instruction& inst, bool is64) {
 	const auto compact = [&](IR::U32 value) {
 		auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, IR::U32(IR::Value(1u))));
 		bits      = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(2u))));
@@ -382,12 +390,17 @@ void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
 		    ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(12u)))),
 		    IR::U32(IR::Value(0xffu)));
 	};
-	const auto source = ReadU32Pair(inst.src0);
-	const auto quads  = ir.BitwiseOr(
-	    compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), IR::U32(IR::Value(8u))));
-	const auto result = ir.ConstructU64(quads, IR::U32(IR::Value(0u)));
-	WriteOperand(DestinationOperand(inst), result);
-	ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+	IR::U32 quads;
+	if (is64) {
+		const auto source = ReadU32Pair(inst.src0);
+		quads = ir.BitwiseOr(
+		    compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), IR::U32(IR::Value(8u))));
+		WriteOperand(DestinationOperand(inst), ir.ConstructU64(quads, IR::U32(IR::Value(0u))));
+	} else {
+		quads = compact(ReadU32(inst.src0));
+		WriteOperand(DestinationOperand(inst), quads);
+	}
+	ir.SetScc(ir.INotEqual(quads, IR::U32(IR::Value(0u))));
 }

 void Translator::BFM_B32(const Decoder::Instruction& inst) {
@@ -322,11 +322,12 @@ IR::Value Translator::MakeImageAddress(const Decoder::Instruction& inst,
                                       const Decoder::Operand&     base) {
 	std::array<IR::Value, 13> components {};
 	components.fill(IR::Value(0u));
-	const auto count =
-	    Decoder::ImageAddressDwordCount(inst.image_sample_flags, inst.image_address_components);
+	const auto count = Decoder::ImageAddressDwordCount(
+	    inst.image_sample_flags, inst.image_dimension, inst.image_address_components);
 	EXIT_IF(count > components.size());
 	const auto nsa_components =
-	    std::min(inst.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
+	    inst.opcode == Decoder::Opcode::IMAGE_BVH8_INTERSECT_RAY && inst.image_nsa_dwords != 0u
+	        ? 10u : std::min(inst.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
 	for (uint32_t index = 0; index < count; index++) {
 		if (index != 0u && index - 1u < nsa_components) {
 			components[index] =
@@ -781,6 +782,15 @@ void Translator::WriteSharedU32(uint32_t width, IR::U32 address,
 	}
 }

+void Translator::DS_BVH_STACK(const Decoder::Instruction& inst) {
+	const auto result = ir.Emit(IR::ValueOpcode::BvhStack,
+	    {ReadU32(inst.src0), ReadU32(inst.src1), ConstructU32x4(inst.src2, 4),
+	     ConstructU32x4(OffsetOperand(inst.src2, 4), 4), ir.GetExec()},
+	    AddMemoryInfo(MemoryInfoFromDecoded(inst), inst.pc));
+	WriteOperand(inst.src0, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(0u)}));
+	WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(1u)}));
+}
+
 void Translator::DS_READ(const Decoder::Instruction& inst) {
 	const auto memory  = MemoryInfoFromDecoded(inst);
 	const auto address = ReadU32(MemorySourceAt(inst, 0));
@@ -939,12 +949,40 @@ void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) {
 	                               {ReadU32(inst.src1), address, ir.GetExec()}));
 }

+void Translator::IMAGE_BVH8(const Decoder::Instruction& inst) {
+	std::array<IR::U32, 10> previous;
+	for (uint32_t i = 0; i < previous.size(); ++i)
+		previous[i] = ReadU32(OffsetOperand(inst.dst, i));
+	const auto result = ir.Emit(IR::ValueOpcode::Bvh8Intersect,
+	    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()});
+	const auto get = [&](uint32_t i) {
+		return IR::U32(ir.Emit(IR::ValueOpcode::Bvh8Extract, {result, IR::Value(i)}));
+	};
+	const auto instance = ir.INotEqual(get(16), IR::U32(IR::Value(0u)));
+	for (uint32_t i = 0; i < 10; ++i) {
+		auto value = get(i);
+		if (i == 0 || i == 1 || i == 4 || i == 5 || i == 8 || i == 9)
+			value = ir.Select(instance, previous[i], value);
+		WriteOperand(OffsetOperand(inst.dst, i), value);
+	}
+	for (uint32_t i = 0; i < 6; ++i) {
+		auto reg = inst.image_nsa_dwords != 0 ? inst.image_nsa_addr[3 + i] : inst.src0.reg + 4 + i;
+		auto target = inst.src0;
+		target.reg = reg;
+		// Non-instance results may alias the ray inputs.
+		WriteOperand(target, ir.Select(instance, get(10 + i), ReadU32(target)));
+	}
+}
+
 void Translator::EmitMemory(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
+		// Translated scalar loads have no cache of their own to invalidate.
+		case Decoder::Opcode::S_DCACHE_INV: return;
 		case Decoder::Opcode::BUFFER_GL0_INV:
 		case Decoder::Opcode::BUFFER_GL1_INV:
 			ir.Emit(IR::ValueOpcode::DataCacheInvalidate);
 			return;
+		case Decoder::Opcode::IMAGE_BVH8_INTERSECT_RAY: return IMAGE_BVH8(inst);
 		case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: {
 			const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect,
 			    {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()},
@@ -1029,6 +1067,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSMin32);
 		case Decoder::Opcode::BUFFER_ATOMIC_UMIN:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMin32);
+		case Decoder::Opcode::BUFFER_ATOMIC_UMIN_X2:
+			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicUMin64);
 		case Decoder::Opcode::BUFFER_ATOMIC_SMAX:
 			return BUFFER_ATOMIC(inst, IR::ValueOpcode::BufferAtomicSMax32);
 		case Decoder::Opcode::BUFFER_ATOMIC_UMAX:
@@ -1146,6 +1186,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::FLAT_LOAD_USHORT:
 		case Decoder::Opcode::FLAT_LOAD_SSHORT:
 		case Decoder::Opcode::FLAT_LOAD_SHORT_D16:
+		case Decoder::Opcode::FLAT_LOAD_SHORT_D16_HI:
 		case Decoder::Opcode::FLAT_LOAD_DWORD:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX2:
 		case Decoder::Opcode::FLAT_LOAD_DWORDX3:
@@ -1165,15 +1206,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::IMAGE_STORE:
 		case Decoder::Opcode::IMAGE_STORE_MIP: return IMAGE_STORE(inst);
 		case Decoder::Opcode::IMAGE_SAMPLE: return IMAGE_SAMPLE(inst);
-		case Decoder::Opcode::IMAGE_GATHER4_L:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_C:
-		case Decoder::Opcode::IMAGE_GATHER4_C_L:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ:
-		case Decoder::Opcode::IMAGE_GATHER4_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_O:
-		case Decoder::Opcode::IMAGE_GATHER4_C_LZ_O:
-		case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst);
+		case Decoder::Opcode::IMAGE_GATHER4: return IMAGE_GATHER(inst);

 		case Decoder::Opcode::DS_MIN_F32:
 			return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false);
@@ -1189,6 +1222,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 			return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataAppend);
 		case Decoder::Opcode::DS_WRITE_ADDTID_B32: return DS_ADDTID(inst, true);
 		case Decoder::Opcode::DS_READ_ADDTID_B32: return DS_ADDTID(inst, false);
+		case Decoder::Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return DS_BVH_STACK(inst);
 		case Decoder::Opcode::DS_READ2_B32:
 		case Decoder::Opcode::DS_READ2ST64_B32:
 		case Decoder::Opcode::DS_READ2_B64:
@@ -248,7 +248,8 @@ void Translator::EmitScalar(const Decoder::Instruction& inst) {
 		case O::S_BITSET0_B64: return S_BITSET_B64(inst, false);
 		case O::S_BITSET1_B64: return S_BITSET_B64(inst, true);
 		case O::S_BITREPLICATE_B64_B32: return S_BITREPLICATE_B64_B32(inst);
-		case O::S_QUADMASK_B64: return S_QUADMASK_B64(inst);
+		case O::S_QUADMASK_B32: return S_QUADMASK(inst, false);
+		case O::S_QUADMASK_B64: return S_QUADMASK(inst, true);
 		case O::S_BFM_B32: return BFM_B32(inst);
 		case O::S_BFM_B64: return S_BFM_B64(inst);
 		case O::S_BFE_U32: return S_BFE_U32(inst, false);
@@ -1165,6 +1165,14 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 		                                               : IR::U32(IR::Value(0u));
 		entry_ir.SetExecLo(initial_lo);
 		entry_ir.SetExecHi(initial_hi);
+		const auto active_lane_count = [&] {
+			auto count = IR::U32(entry_ir.Emit(IR::ValueOpcode::BitCount32, {initial_lo}));
+			if (options.wave_size == 64u) {
+				count = entry_ir.IAdd(count, IR::U32(entry_ir.Emit(
+				    IR::ValueOpcode::BitCount32, {initial_hi})));
+			}
+			return count;
+		};
 		if (options.stage == ShaderType::Compute) {
 			const auto* cs = options.input_info.compute;
 			const auto  thread_ids =
@@ -1302,7 +1310,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 				                      builtin(IR::StageInputKind::WorkgroupId, 1));
 			}
 		} else if (options.stage == ShaderType::Local) {
-			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), IR::U32(IR::Value(64u)));
+			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3), active_lane_count());
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(2),
 			                      builtin(IR::StageInputKind::VertexIndex));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(3), IR::U32(IR::Value(0u)));
@@ -1380,19 +1388,32 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
 		} else if (options.stage == ShaderType::Vertex) {
 			// Vulkan owns primitive assembly; each vertex subgroup is one NGG wave.
 			// ESVertCount describes input vertices, independently of lane ordinals.
-			auto vertex_count = IR::U32(entry_ir.Emit(IR::ValueOpcode::BitCount32, {initial_lo}));
-			if (options.wave_size == 64u) {
-				vertex_count = entry_ir.IAdd(vertex_count, IR::U32(entry_ir.Emit(
-				    IR::ValueOpcode::BitCount32, {initial_hi})));
-			}
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(2),
 			                      IR::U32(IR::Value(options.wave_size << 12u)));
 			entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(3),
-			                      entry_ir.BitwiseOr(vertex_count, IR::U32(IR::Value(1u << 28u))));
+			                      entry_ir.BitwiseOr(active_lane_count(), IR::U32(IR::Value(1u << 28u))));
+			// Vulkan adds an indirect record's bases to the indices; the hardware passes them
+			// only in the user SGPRs the packet patches. SH_NOP leaves those SGPRs unchanged;
+			// it does not make the record's bases zero.
+			const auto& offsets = options.input_info.vertex->indirect_offsets;
+			const auto  index   = [&](uint8_t slot, IR::StageInputKind kind,
+			                          IR::StageInputKind base_kind) {
+				if (!options.input_info.vertex->indirect_draw) {
+					return builtin(kind);
+				}
+				const auto base = builtin(base_kind);
+				if (slot < options.user_data_count) {
+					entry_ir.SetScalarReg(static_cast<IR::ScalarReg>(options.user_data_base + slot),
+					                      base);
+				}
+				return entry_ir.ISub(builtin(kind), base);
+			};
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
-			                      builtin(IR::StageInputKind::VertexIndex));
+			                      index(offsets.vertex, IR::StageInputKind::VertexIndex,
+			                            IR::StageInputKind::BaseVertex));
 			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(8),
-			                      builtin(IR::StageInputKind::InstanceIndex));
+			                      index(offsets.instance, IR::StageInputKind::InstanceIndex,
+			                            IR::StageInputKind::BaseInstance));
 		}
 	}
 	const bool flush_f32_inputs = options.stage == ShaderType::Compute &&
@@ -105,6 +105,8 @@ private:
 	void WriteSharedU32(uint32_t width, IR::U32 address, const std::array<IR::Value, 4>& values,
 	                    const IR::MemoryInfo& memory, uint32_t pc);
 	void DS_READ(const Decoder::Instruction& inst);
+	void IMAGE_BVH8(const Decoder::Instruction& inst);
+	void DS_BVH_STACK(const Decoder::Instruction& inst);
 	void DS_READ2(const Decoder::Instruction& inst);
 	void DS_WRITE(const Decoder::Instruction& inst);
 	void DS_WRITE2(const Decoder::Instruction& inst);
@@ -167,7 +169,7 @@ private:
 	void Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
 	void Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
 	void Integer16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
-	void V_MAD_I16(const Decoder::Instruction& inst);
+	void Integer16Mad(const Decoder::Instruction& inst, bool sign);
 	void PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
 	                          bool arithmetic);
 	void PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
@@ -187,7 +189,7 @@ private:
 	void V_XOR3_B32(const Decoder::Instruction& inst);
 	void S_FF_I32_B64(const Decoder::Instruction& inst, bool find_zero, bool from_msb, bool sign);
 	void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
-	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
+	void Integer24(const Decoder::Instruction& inst, bool sign, bool addend, bool high = false);
 	void V_MAD_U64_U32(const Decoder::Instruction& inst);
 	void V_SAD_U32(const Decoder::Instruction& inst);
 	void V_ADD3_U32(const Decoder::Instruction& inst);
@@ -196,7 +198,7 @@ private:
 	void V_BCNT_U32_B32(const Decoder::Instruction& inst);
 	void V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
 	void S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
-	void S_QUADMASK_B64(const Decoder::Instruction& inst);
+	void S_QUADMASK(const Decoder::Instruction& inst, bool is64);
 	void BFM_B32(const Decoder::Instruction& inst);
 	IR::U32 RightMask32(IR::U32 count);
 	void    S_BFM_B64(const Decoder::Instruction& inst);
@@ -375,9 +375,14 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_SUB_NC_U16:
 		case O::V_SUB_NC_I16: return Integer16Binary(inst, IR::ValueOpcode::ISub32, false);
 		case O::V_MUL_LO_U16: return Integer16Binary(inst, IR::ValueOpcode::IMul32, false);
-		case O::V_MAD_I16: return V_MAD_I16(inst);
+		case O::V_MAD_I16: return Integer16Mad(inst, true);
+		case O::V_MAD_U16: return Integer16Mad(inst, false);
 		case O::V_MED3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMedTri32, true);
 		case O::V_MIN3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMinTri32, false);
+		case O::V_MIN3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMinTri32, true);
+		case O::V_MAX3_I16: return Integer16Ternary(inst, IR::ValueOpcode::SMaxTri32, true);
+		case O::V_MAX3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMaxTri32, false);
+		case O::V_MED3_U16: return Integer16Ternary(inst, IR::ValueOpcode::UMedTri32, false);
 		case O::V_MIN_I16: return Integer16Binary(inst, IR::ValueOpcode::SMin32, true);
 		case O::V_MAX_I16: return Integer16Binary(inst, IR::ValueOpcode::SMax32, true);
 		case O::V_MIN_U16: return Integer16Binary(inst, IR::ValueOpcode::UMin32, false);
@@ -565,6 +570,9 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_LSHRREV_B64:
 			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightLogical64, IR::Type::U64, true,
 			                     false, false);
+		case O::V_ASHRREV_I64:
+			return SimpleInteger(inst, IR::ValueOpcode::ShiftRightArithmetic64, IR::Type::U64, true,
+			                     false, false);

 		case O::V_XNOR_B32:
 			return ComposedIntegerBinary(inst, IR::ValueOpcode::BitwiseXor32, false, true, false);
@@ -578,6 +586,8 @@ void Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_MAD_U32_U24: return Integer24(inst, false, true);
 		case O::V_MUL_I32_I24: return Integer24(inst, true, false);
 		case O::V_MUL_U32_U24: return Integer24(inst, false, false);
+		case O::V_MUL_HI_I32_I24: return Integer24(inst, true, false, true);
+		case O::V_MUL_HI_U32_U24: return Integer24(inst, false, false, true);
 		case O::V_MAD_U64_U32: return V_MAD_U64_U32(inst);
 		case O::V_SAD_U32: return V_SAD_U32(inst);
 		case O::V_ADD3_U32: return V_ADD3_U32(inst);
@@ -118,15 +118,14 @@ ResourcePlan& ResourcePlan::operator=(ResourcePlan&& other) noexcept {
 }

 Program::~Program() {
-	// Planning expressions can refer to block values but outlive block storage in the base class.
+	// Planning expressions can refer to block values but outlive block storage in the base class,
+	// and values cross block boundaries: detach everything before any storage is destroyed.
 	for (auto& inst: value_storage) {
-		inst.Invalidate();
+		inst.DropArgs();
 	}
-	// Values may cross block boundaries. Detach all arguments before any block starts destroying
-	// its instruction storage so reverse-use links always point to live definitions.
 	for (auto* block: blocks) {
 		for (auto& inst: *block) {
-			inst.Invalidate();
+			inst.DropArgs();
 		}
 	}
 }
@@ -169,6 +168,8 @@ bool EquivalentSelectorDomain(const ResourcePlan& program,
                              const DescriptorSource::IndirectDescriptor& right) {
 	return left.selector == right.selector && left.signed_count == right.signed_count &&
 	       left.workgroup_axis == right.workgroup_axis &&
+	       left.workgroup_multiplier == right.workgroup_multiplier &&
+	       left.workgroup_shift == right.workgroup_shift &&
 	       EquivalentValue(program, left.key_count, right.key_count) &&
 	       EquivalentValue(program, left.selector_first, right.selector_first) &&
 	       EquivalentValue(program, left.selector_mask, right.selector_mask);
@@ -148,21 +148,24 @@ template <class S, class T> void Io(S& io, T& value) {
 		IoAll(io, __VA_ARGS__);                                                                    \
 	}
 KYTY_IO_FIELDS(MemoryInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y)
-KYTY_IO_FIELDS(BufferResource, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s)
+KYTY_IO_FIELDS(BufferResource, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r)
 KYTY_IO_FIELDS(ImageResource, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w)
 KYTY_IO_FIELDS(SamplerResource, a, b, c, d, e, f, g, h, i, j, k)
 KYTY_IO_FIELDS(SampledResourcePair, a, b, c)
 KYTY_IO_FIELDS(StageInput, a, b, c, d, e)
 KYTY_IO_FIELDS(StageOutput, a, b, c, d)
 KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
-               A, B, C, D, E, F, G, H, I, J, K)
+               A, B, C, D, E, F, G, H, I, J, K, L, M)
 KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b)
-KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks, a, b, c)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j, k,
+               l)
 KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::PointerTable, a, b, c, d)
-KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)
+KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p,
+               q, r)
 KYTY_IO_FIELDS(DescriptorSource, a, b, c)
 KYTY_IO_FIELDS(SrtRead, a, b)
-KYTY_IO_FIELDS(ResourceBlock, a, b, c, d)
+KYTY_IO_FIELDS(ResourceBlock, a, b, c, d, e)
 KYTY_IO_FIELDS(UniformFill, a, b, c, d, e, f)
 KYTY_IO_FIELDS(UniformFillPlan, a, b, c, d)
 KYTY_IO_FIELDS(ResourceSpecialization::Buffer, a, b, c, d, e, f, g, h)
@@ -122,7 +122,6 @@ struct BufferResource {
 	uint32_t               descriptor_index   = UINT32_MAX;
 	uint32_t               first_use_pc       = 0;
 	uint32_t               max_byte_extent    = 0;
-	uint32_t               write_extent       = 0; // UINT32_MAX once a write is runtime-addressed or formatted
 	uint32_t               packed_stride      = 0;
 	Prospero::BufferFormat descriptor_format  = Prospero::BufferFormat::kInvalid;
 	uint32_t               descriptor_swizzle = DstSel(4, 5, 6, 7);
@@ -251,6 +250,8 @@ enum class StageInputKind {
 	LocalInvocationId,
 	LocalInvocationIndex,
 	GlobalInvocationId,
+	BaseVertex,
+	BaseInstance,
 	Parameter,
 };

@@ -541,6 +542,8 @@ struct ShaderInfo {
 	bool                            uses_image_heap               = false;
 	bool                            uses_dispatch_threads         = false;
 	bool                            bvh                           = false;
+	bool                            bvh8                          = false;
+	bool                            bvh_stack                     = false;
 	bool                            subgroup_ballot               = false;
 	bool                            subgroup_vote                 = false;
 	bool                            subgroup_barrier              = false;
@@ -572,6 +575,13 @@ struct DescriptorSource {

 				bool operator==(const CountedRows& other) const = default;
 			};
+			struct MaskRanks {
+				uint32_t source = UINT32_MAX;
+				uint32_t stride = 0; // Byte spacing of the mask words its load can address.
+				uint32_t offset = 0;
+
+				bool operator==(const MaskRanks& other) const = default;
+			};
 			uint32_t source = UINT32_MAX;
 			uint32_t stride = 0;
 			uint32_t offset = 0;
@@ -579,10 +589,12 @@ struct DescriptorSource {
 			uint32_t bit_offset = 0;
 			uint32_t bit_mask = UINT32_MAX;
 			bool global = false;
-			// Four DWORD bitmap words per record; set-bit ranks subtract from the last key.
+			// Four or eight DWORD bitmap words per record; set ranks subtract from the last key.
 			bool bitset = false;
 			std::optional<CountedRows> rows;
 			std::vector<std::pair<uint32_t, uint32_t>> byte_ranges;
+			std::optional<uint32_t> excluded; // A guard keeps this key from the read.
+			std::optional<MaskRanks> ranks;   // Records are the bits some mask word sets.

 			bool operator==(const SelectorRead& other) const = default;
 		};
@@ -606,8 +618,12 @@ struct DescriptorSource {
 		bool     indexed = false;
 		bool     signed_count = false;
 		uint32_t workgroup_axis  = UINT32_MAX;
+		// Nonzero shift: workgroup keys are (selector_first + ID) * multiplier >> shift.
+		uint32_t workgroup_multiplier = 0;
+		uint32_t workgroup_shift      = 0;
 		Value    key_count;
-		// First selector index, or first descriptor key for a direct or bitmap range.
+		// First selector index, workgroup key base, or first descriptor key for a direct or bitmap
+		// range.
 		Value                 selector_first;
 		Value    selector_mask;
 		std::vector<uint32_t> sources;
@@ -634,6 +650,7 @@ struct ResourceBlock {
 	Value                 condition;
 	std::vector<uint32_t> successors;
 	std::vector<uint32_t> sources;
+	std::vector<uint32_t> images;
 	std::vector<uint32_t> srt_reads;
 };

@@ -31,6 +31,7 @@ std::string TypeName(Type type) {
 	    "SamplerResource",
 	    "ImageAddress",
 	    "F64",
+	    "Bvh8Result",
 	};
 	const auto bits = static_cast<uint32_t>(type);
 	if (bits == 0) {
@@ -28,6 +28,7 @@ enum class Type : uint32_t {
 	SamplerResource = 1u << 18u,
 	ImageAddress    = 1u << 19u,
 	F64             = 1u << 20u,
+	Bvh8Result      = 1u << 21u,
 };

 constexpr Type operator|(Type lhs, Type rhs) {
@@ -25,22 +25,6 @@ Value Value::F32(float value) {
 	return Value(Type::F32, std::bit_cast<uint32_t>(value));
 }

-bool Value::IsEmpty() const {
-	return type == Type::Void;
-}
-
-bool Value::IsImmediate() const {
-	return type != Type::Opaque;
-}
-
-bool Value::IsIdentity() const {
-	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Identity;
-}
-
-bool Value::IsPhi() const {
-	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Phi;
-}
-
 Type Value::GetType() const {
 	if (IsPhi()) {
 		return inst->Flags<Type>();
@@ -56,19 +40,6 @@ Inst* Value::Instruction() const {
 	return inst;
 }

-Inst* Value::TryInstruction() const {
-	return type == Type::Opaque ? inst : nullptr;
-}
-
-Inst* Value::ResolveInstruction() const {
-	EXIT_IF(type != Type::Opaque);
-	return IsIdentity() ? inst->Arg(0).ResolveInstruction() : inst;
-}
-
-Value Value::Resolve() const {
-	return IsIdentity() ? inst->Arg(0).Resolve() : *this;
-}
-
 ScalarReg Value::ScalarRegister() const {
 	EXIT_IF(type != Type::ScalarReg);
 	return scalar_reg;
@@ -94,11 +65,6 @@ uint16_t Value::U16() const {
 	return imm_u16;
 }

-uint32_t Value::U32() const {
-	EXIT_IF(type != Type::U32);
-	return imm_u32;
-}
-
 uint64_t Value::U64() const {
 	EXIT_IF(type != Type::U64);
 	return imm_u64;
@@ -152,10 +118,6 @@ Inst::~Inst() {
 	ClearArgs();
 }

-ValueOpcode Inst::GetOpcode() const {
-	return opcode;
-}
-
 Type Inst::GetType() const {
 	if (opcode == ValueOpcode::Phi) {
 		return static_cast<Type>(flags);
@@ -178,51 +140,25 @@ size_t Inst::UseCount() const {
 	return uses.size();
 }

-size_t Inst::NumArgs() const {
-	return num_args == PhiArity ? phi_args.size() : num_args;
-}
-
 size_t Inst::NumPhiBlocks() const {
 	return num_args == PhiArity ? phi_args.size() : 0;
 }

-Value Inst::Arg(size_t index) const {
-	EXIT_IF(index >= NumArgs());
-	if (num_args <= InlineArity) {
-		return fixed_args[index];
-	}
-	return num_args == PhiArity ? phi_args[index].second : large_args[index];
-}
-
 Block* Inst::PhiBlock(size_t index) const {
 	EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_args.size());
 	return phi_args[index].first;
 }

-Block* Inst::Parent() const {
-	return parent;
-}
-
-const std::vector<Use>& Inst::Uses() const {
-	return uses;
-}
-
 void Inst::SetParent(Block* block) {
 	parent = block;
 }

 void Inst::SetArg(size_t index, Value value) {
-	const auto old = Arg(index);
-	if (auto* old_inst = old.TryInstruction(); old_inst != nullptr) {
+	auto& arg = ArgSlot(*this, index);
+	if (auto* old_inst = arg.TryInstruction(); old_inst != nullptr) {
 		RemoveUse(old_inst, index);
 	}
-	if (num_args <= InlineArity) {
-		fixed_args[index] = value;
-	} else if (num_args == PhiArity) {
-		phi_args[index].second = value;
-	} else {
-		large_args[index] = value;
-	}
+	arg = value;
 	if (auto* new_inst = value.TryInstruction(); new_inst != nullptr) {
 		AddUse(new_inst, index);
 	}
@@ -238,11 +174,15 @@ void Inst::AddPhiOperand(Block* predecessor, Value value) {
 }

 void Inst::ReplaceUsesWith(Value replacement, bool preserve) {
-	const auto old_uses = uses;
-	for (const auto& use: old_uses) {
-		use.user->SetArg(use.operand, replacement);
-	}
+	// Detach the arguments first (a phi can use itself), then move every use over in order.
 	Invalidate();
+	const auto old_uses = std::exchange(uses, {});
+	for (const auto& use: old_uses) {
+		ArgSlot(*use.user, use.operand) = replacement;
+	}
+	if (auto* target = replacement.TryInstruction(); target != nullptr) {
+		target->uses.insert(target->uses.end(), old_uses.begin(), old_uses.end());
+	}
 	if (preserve) {
 		opcode = ValueOpcode::Identity;
 		num_args = 1;
@@ -256,9 +196,6 @@ void Inst::Invalidate() {
 }

 void Inst::AddUse(Inst* used, size_t operand) {
-	const auto found = std::ranges::find_if(
-	    used->uses, [&](const Use& use) { return use.user == this && use.operand == operand; });
-	EXIT_IF(found != used->uses.end());
 	used->uses.push_back({this, operand});
 }

@@ -275,6 +212,10 @@ void Inst::ClearArgs() {
 			RemoveUse(value_inst, index);
 		}
 	}
+	DropArgs();
+}
+
+void Inst::DropArgs() {
 	if (num_args == PhiArity) {
 		std::destroy_at(&phi_args);
 		std::construct_at(&fixed_args);
@@ -33,14 +33,14 @@ public:
 	static Value F16(uint16_t bits);
 	static Value F32(float value);

-	[[nodiscard]] bool IsEmpty() const;
-	[[nodiscard]] bool IsImmediate() const;
+	[[nodiscard]] bool IsEmpty() const { return type == Type::Void; }
+	[[nodiscard]] bool IsImmediate() const { return type != Type::Opaque; }
 	[[nodiscard]] bool IsIdentity() const;
 	[[nodiscard]] bool IsPhi() const;
 	[[nodiscard]] Type GetType() const;

 	[[nodiscard]] Inst*     Instruction() const;
-	[[nodiscard]] Inst*     TryInstruction() const;
+	[[nodiscard]] Inst*     TryInstruction() const { return type == Type::Opaque ? inst : nullptr; }
 	[[nodiscard]] Inst*     ResolveInstruction() const;
 	[[nodiscard]] Value     Resolve() const;
 	[[nodiscard]] ScalarReg ScalarRegister() const;
@@ -112,7 +112,7 @@ public:
 	Inst(Inst&&)                 = delete;
 	Inst& operator=(Inst&&)      = delete;

-	[[nodiscard]] ValueOpcode             GetOpcode() const;
+	[[nodiscard]] ValueOpcode             GetOpcode() const { return opcode; }
 	[[nodiscard]] Type                    GetType() const;
 	[[nodiscard]] bool                    MayHaveSideEffects() const;
 	[[nodiscard]] bool                    HasUses() const;
@@ -121,8 +121,8 @@ public:
 	[[nodiscard]] size_t                  NumPhiBlocks() const;
 	[[nodiscard]] Value                   Arg(size_t index) const;
 	[[nodiscard]] Block*                  PhiBlock(size_t index) const;
-	[[nodiscard]] Block*                  Parent() const;
-	[[nodiscard]] const std::vector<Use>& Uses() const;
+	[[nodiscard]] Block*                  Parent() const { return parent; }
+	[[nodiscard]] const std::vector<Use>& Uses() const { return uses; }
 	// SRT walker output index, assigned by the resource plan that owns this instruction.
 	[[nodiscard]] uint32_t WalkOutput() const { return walk_output; }
 	void SetWalkOutput(uint32_t index) const { walk_output = index; }
@@ -138,6 +138,8 @@ public:
 	void AddPhiOperand(Block* predecessor, Value value);
 	void ReplaceUsesWith(Value replacement, bool preserve = true);
 	void Invalidate();
+	// Drops the arguments without updating their use lists, for teardown that destroys both sides.
+	void DropArgs();

 	template <typename T>
 	requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v<T>)
@@ -157,6 +159,8 @@ public:
 private:
 	friend void EliminateDeadCode(const std::vector<Block*>& blocks);

+	template <typename Self>
+	static auto& ArgSlot(Self& self, size_t index);
 	void AddUse(Inst* used, size_t operand);
 	void RemoveUse(Inst* used, size_t operand);
 	void ClearArgs();
@@ -181,4 +185,44 @@ private:

 static_assert(sizeof(Inst) <= 120, "Inst operand and backend storage unintentionally increased");

+// Called per operand by the IR passes and the resource walker; inline, as the build has no LTO.
+inline bool Value::IsIdentity() const {
+	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Identity;
+}
+
+inline bool Value::IsPhi() const {
+	return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Phi;
+}
+
+inline Inst* Value::ResolveInstruction() const {
+	EXIT_IF(type != Type::Opaque);
+	return IsIdentity() ? inst->Arg(0).ResolveInstruction() : inst;
+}
+
+inline Value Value::Resolve() const {
+	return IsIdentity() ? inst->Arg(0).Resolve() : *this;
+}
+
+inline uint32_t Value::U32() const {
+	EXIT_IF(type != Type::U32);
+	return imm_u32;
+}
+
+inline size_t Inst::NumArgs() const {
+	return num_args == PhiArity ? phi_args.size() : num_args;
+}
+
+template <typename Self>
+auto& Inst::ArgSlot(Self& self, size_t index) {
+	EXIT_IF(index >= self.NumArgs());
+	if (self.num_args <= InlineArity) {
+		return self.fixed_args[index];
+	}
+	return self.num_args == PhiArity ? self.phi_args[index].second : self.large_args[index];
+}
+
+inline Value Inst::Arg(size_t index) const {
+	return ArgSlot(*this, index);
+}
+
 } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -44,6 +44,7 @@ constexpr Type AddressResource = Type::AddressResource;
 constexpr Type ImageResource   = Type::ImageResource;
 constexpr Type SamplerResource = Type::SamplerResource;
 constexpr Type ImageAddress    = Type::ImageAddress;
+constexpr Type Bvh8Result      = Type::Bvh8Result;

 constexpr std::array<OpcodeMeta, static_cast<size_t>(ValueOpcode::Count)> MetaTable = {{
 #define VALUE_OPCODE(name, ...) MakeMeta(#name __VA_OPT__(, ) __VA_ARGS__),
@@ -76,7 +77,8 @@ bool HasSideEffects(ValueOpcode opcode) {
 		return true;
 	}
 	const auto shared_access = SharedAccessOf(opcode);
-	if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic ||
+	if (shared_access == SharedAccess::Write || shared_access == SharedAccess::ReadWrite ||
+	    shared_access == SharedAccess::Atomic ||
 	    shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) {
 		return true;
 	}
@@ -123,6 +125,7 @@ BufferAccess BufferAccessOf(ValueOpcode opcode) {
 		case ValueOpcode::BufferAtomicISub32:
 		case ValueOpcode::BufferAtomicSMin32:
 		case ValueOpcode::BufferAtomicUMin32:
+		case ValueOpcode::BufferAtomicUMin64:
 		case ValueOpcode::BufferAtomicSMax32:
 		case ValueOpcode::BufferAtomicUMax32:
 		case ValueOpcode::BufferAtomicAnd32:
@@ -140,6 +143,7 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {
 	switch (opcode) {
 		case ValueOpcode::BufferAtomicSwap64:
 		case ValueOpcode::BufferAtomicIAdd64:
+		case ValueOpcode::BufferAtomicUMin64:
 		case ValueOpcode::BufferAtomicAnd64:
 		case ValueOpcode::BufferAtomicOr64:
 		case ValueOpcode::LoadBufferU32x2:
@@ -154,6 +158,7 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {

 SharedAccess SharedAccessOf(ValueOpcode opcode) {
 	switch (opcode) {
+		case ValueOpcode::BvhStack: return SharedAccess::ReadWrite;
 		case ValueOpcode::LoadSharedU8:
 		case ValueOpcode::LoadSharedU16:
 		case ValueOpcode::LoadSharedU32:
@@ -17,7 +17,7 @@ enum class ValueOpcode : uint16_t {
 };

 enum class BufferAccess { None, Read, Write, Atomic };
-enum class SharedAccess { None, Read, Write, Atomic, Append, Consume };
+enum class SharedAccess { None, Read, Write, ReadWrite, Atomic, Append, Consume };
 enum class AddressAccess { None, Read, Write };
 enum class ImageAccess { None, Read, Write, Atomic };
 enum class ImageResourceClass { None, Sampled, Storage };
@@ -224,6 +224,7 @@ VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
 VALUE_OPCODE(ReadLane, U32, U32, U32)
 VALUE_OPCODE(SubgroupUMinU32, U32, U32)
 VALUE_OPCODE(SubgroupUMaxU32, U32, U32)
+VALUE_OPCODE(SubgroupOrU32, U32, U32)
 VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
 VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1, U32)
 VALUE_OPCODE(PermuteU32, U32, U32, U32, U1)
@@ -269,6 +270,7 @@ VALUE_OPCODE(BufferAtomicIAdd64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
+VALUE_OPCODE(BufferAtomicUMin64, U64, BufferResource, U32, U32, U32, U64, U1)
 VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
 VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
@@ -317,6 +319,9 @@ VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)

 VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
 VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
+VALUE_OPCODE(Bvh8Intersect, Bvh8Result, U32x4, ImageAddress, U1)
+VALUE_OPCODE(Bvh8Extract, U32, Bvh8Result, U32)
+VALUE_OPCODE(BvhStack, U32x2, U32, U32, U32x4, U32x4, U1)
 VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
 VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
 VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
@@ -1,7 +1,7 @@
 #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"

 #include <algorithm>
-
+#include <optional>
 #include <queue>

 #include <unordered_map>
@@ -186,6 +186,16 @@ private:
 	ValueOpcode  operation;
 };

+// The subgroup reduction that replaces a proven whole-wave scan of operation.
+std::optional<ValueOpcode> WaveReduction(ValueOpcode operation) {
+	switch (operation) {
+		case ValueOpcode::UMin32: return ValueOpcode::SubgroupUMinU32;
+		case ValueOpcode::UMax32: return ValueOpcode::SubgroupUMaxU32;
+		case ValueOpcode::BitwiseOr32: return ValueOpcode::SubgroupOrU32;
+		default: return std::nullopt;
+	}
+}
+
 } // namespace

 ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) {
@@ -197,15 +207,12 @@ ReadLaneStats EliminateReadLane(Program& program, uint32_t wave_size) {
 	for (auto* block: program.blocks) {
 		for (auto& inst: *block) {
 			const auto opcode = inst.GetOpcode();
-			if (opcode == ValueOpcode::UMin32 || opcode == ValueOpcode::UMax32) {
+			if (const auto reduce = WaveReduction(opcode)) {
 				const auto seed = WaveReductionMatcher(block, opcode).Match(inst, wave_size);
 				if (!seed.IsEmpty()) {
 					const auto position = std::find_if(block->begin(), block->end(),
 					                                  [&](const Inst& candidate) { return &candidate == &inst; });
-					const auto reduce    = opcode == ValueOpcode::UMin32
-					                           ? ValueOpcode::SubgroupUMinU32
-					                           : ValueOpcode::SubgroupUMaxU32;
-					auto&      reduction = *block->PrependNewInst(position, reduce, {seed});
+					auto&      reduction = *block->PrependNewInst(position, *reduce, {seed});
 					inst.ReplaceUsesWith(Value(&reduction));
 					stats.rewritten_reductions++;
 				}
@@ -331,6 +331,18 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		keys.resize(count);
 		return true;
 	};
+	// Reads the key of each record whose rank is set.
+	const auto read_ranks = [&](uint64_t base, uint64_t size, uint64_t first, uint64_t step,
+	                            uint32_t ranks) {
+		keys.reserve(std::popcount(ranks));
+		for (; ranks != 0u; ranks &= ranks - 1u) {
+			const auto offset = first + std::countr_zero(ranks) * step;
+			uint32_t key = 0;
+			if (offset > UINT32_MAX || !read_selector(base, size, offset, {&key, 1})) return false;
+			keys.push_back(key);
+		}
+		return true;
+	};
 	if (selector->rows) {
 		const auto& rows = *selector->rows;
 		DescriptorValue count_value;
@@ -377,19 +389,21 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		}
 	} else if (selector->bitset) {
 		ShaderBufferResource material;
-		std::array<uint32_t, 4> masks {};
+		std::array<uint32_t, 8> storage {};
+		const auto record_words = std::min<size_t>(selector->stride / 4u, storage.size());
+		const auto masks = std::span(storage).first(record_words);
 		uint32_t first = 0, count = 0;
 		if (!DecodeBufferDescriptor(material_value, material) ||
 		    !WalkedValue(program, indirect.selector_first, first) ||
 		    !EvaluateIndirectCount(program, indirect, runtime, count) || count != masks.size() * 32u ||
-		    selector->stride != sizeof(masks) || selector->offset != 0u ||
-		    material.Type() != 0u || material.Stride() != sizeof(masks) ||
+		    selector->stride != masks.size_bytes() || selector->offset != 0u ||
+		    material.Type() != 0u || material.Stride() != selector->stride ||
 		    material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
 		    (material.Base48() & 3u) != 0u) return false;
 		if (material.Format() != Prospero::BufferFormat::kInvalid) {
 			const auto words = material.GetSize() / sizeof(uint32_t);
-			// Bitmap storage can grow with resolution while its candidate domain stays
-			// at 128 keys. Bound scratch memory, preserving each coherent read's capture.
+			// Bitmap storage can grow with resolution while its candidate domain stays at 32 keys
+			// per record word. Bound scratch memory, preserving each coherent read's capture.
 			for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) {
 				keys.resize(std::min<uint64_t>(words - first_word, MaxIndirectDescriptorProbes));
 				if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u,
@@ -445,12 +459,35 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		     uint64_t {first} + count > material.NumRecords())) return false;
 		const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u;
 		if (last > UINT32_MAX) return false;
+		// Out-of-bounds scalar reads return zero.
 		const auto end = count != 0u ? last * step + selector->offset + 4u : 0u;
-		if (end > uint64_t {UINT32_MAX} + 1u ||
-		    (indirect.workgroup_axis == UINT32_MAX && end > size))
-			return false;
-		if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count))
+		if (end > uint64_t {UINT32_MAX} + 1u) return false;
+		const auto start = uint64_t {first} * step + selector->offset;
+		if (selector->ranks) {
+			// A scan selects the ranks some word of its mask's address class sets.
+			const auto& mask = *selector->ranks;
+			DescriptorValue mask_value;
+			ShaderBufferResource masks;
+			if (count > 32u || !WalkedDescriptor(program, mask.source, mask_value) ||
+			    !DecodeBufferDescriptor(mask_value, masks)) return false;
+			// Each chunk holds whole strides of the power-of-two class, or one of its words.
+			const auto words = mask.stride / 4u;
+			const auto chunk =
+			    words <= MaxIndirectDescriptorProbes ? MaxIndirectDescriptorProbes : 1u;
+			uint32_t ranks = 0;
+			for (auto at = uint64_t {mask.offset}; at + 4u <= masks.GetSize();
+			     at += uint64_t {std::max(chunk, words)} * 4u) {
+				keys.resize(std::min<uint64_t>((masks.GetSize() - at) / 4u, chunk));
+				if (!ReadScalarTable(masks.Base48(), masks.GetSize(), at, runtime, keys))
+					return false;
+				for (size_t word = 0; word < keys.size(); word += words) ranks |= keys[word];
+			}
+			keys.clear();
+			if (count < 32u) ranks &= (1u << count) - 1u;
+			if (!read_ranks(base, size, start, step, ranks)) return false;
+		} else if (!read_keys(base, size, start, step, count)) {
 			return false;
+		}
 	} else if (!indirect.selector_mask.IsEmpty()) {
 		uint32_t mask = 0;
 		uint32_t count = 0;
@@ -462,18 +499,8 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		if (count < 32u) mask &= (1u << count) - 1u;
 		const auto material_base =
 		    (static_cast<uint64_t>(material_value.dwords[1]) << 32u) | material_value.dwords[0];
-		keys.reserve(std::popcount(mask));
-		while (mask != 0u) {
-			const auto index = std::countr_zero(mask);
-			const auto offset = static_cast<uint64_t>(selector->offset) +
-			                    static_cast<uint64_t>(index) * selector->stride;
-			if (offset > UINT32_MAX) return false;
-			uint32_t key = 0;
-			if (!read_selector(material_base, UINT64_MAX, static_cast<uint32_t>(offset),
-			                   {&key, 1})) return false;
-			keys.push_back(key);
-			mask &= mask - 1u;
-		}
+		if (!read_ranks(material_base, UINT64_MAX, selector->offset, selector->stride, mask))
+			return false;
 	} else if (selector->stride == 0u) {
 		ShaderBufferResource material;
 		if (!DecodeBufferDescriptor(material_value, material) ||
@@ -503,6 +530,7 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		if (selector->bit_offset != 0u || selector->bit_mask != UINT32_MAX || selector->maximum != UINT32_MAX)
 			for (auto& key: keys)
 				key = std::min((key >> selector->bit_offset) & selector->bit_mask, selector->maximum);
+		if (selector->excluded) std::erase(keys, *selector->excluded);
 		std::ranges::sort(keys);
 		keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
 	}
@@ -756,6 +784,12 @@ bool MaterializeIndirectDescriptor(const ResourcePlan&                         p
 			std::iota(keys.begin(), keys.end(), first_key);
 			if (!keys.empty() && keys.back() < keys.front())
 				std::rotate(keys.begin(), keys.begin() + (UINT32_MAX - first_key + 1ull), keys.end());
+			if (indirect.workgroup_shift != 0u) {
+				for (auto& key: keys)
+					key = static_cast<uint32_t>(uint64_t {key} * indirect.workgroup_multiplier >>
+					                            indirect.workgroup_shift);
+				keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
+			}
 		} else {
 			if (!indirect.selector_mask.IsEmpty() && table_value.dword_count != 2u) return false;
 			if (!selector_keys->resolved) {
@@ -1041,6 +1075,26 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			return dimension == Decoder::ImageDimension::Dim1D ||
 			       dimension == Decoder::ImageDimension::Dim2D;
 		};
+		// The descriptor's type selects the address layout. A 2D or 3D candidate reads the leading
+		// coordinates of an equal or larger dimension, but an array with more than one slice reads
+		// the next as its slice. Derivative, LOD, clamp and mip operands shift with the dimension.
+		const auto leading = [&](Decoder::ImageDimension dimension, const DescriptorValue& value) {
+			const bool sliced =
+			    static_cast<Prospero::ImageType>((value.dwords[3] >> 28u) & 0xfu) ==
+			        Prospero::ImageType::kColor2DArray &&
+			    (value.dwords[4] & 0x1fffu) != ((value.dwords[4] >> 16u) & 0x1fffu);
+			return !sliced &&
+			       (dimension == Decoder::ImageDimension::Dim2D || dimension == base.dimension);
+		};
+		const bool leading_coordinates = (base.dimension == Decoder::ImageDimension::Dim2D ||
+		                                  base.dimension == Decoder::ImageDimension::Dim3D) &&
+		    leading(image_class.dimension, snapshot.images[exemplar]) &&
+		    std::ranges::none_of(program.memory_info, [&](const MemoryInfo& memory) {
+			    return memory.kind == ResourceKind::Image && memory.resource == root_index &&
+			           (memory.image_has_mip || (memory.image_sample_flags &
+			               (Decoder::ImageSampleFlagLod | Decoder::ImageSampleFlagDerivative |
+			                Decoder::ImageSampleFlagLodClamp)) != 0u);
+		    });
 		for (uint32_t candidate = 0; candidate < specialization.images.size(); candidate++) {
 			auto& image = specialization.images[candidate];
 			if (image.indirect_root != root_index) {
@@ -1060,8 +1114,10 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
 			    base.dimension == Decoder::ImageDimension::Dim2D &&
 			    !image.cube && !image_class.cube && is_1d_or_2d(image.dimension) &&
 			    is_1d_or_2d(image_class.dimension);
+			const bool prefix_coordinates =
+			    leading_coordinates && leading(image.dimension, snapshot.images[candidate]);
 			if ((!same_coordinates && !(is_2d(image.dimension) && is_2d(image_class.dimension)) &&
-			     !unfiltered_coordinates) ||
+			     !unfiltered_coordinates && !prefix_coordinates) ||
 			    image.conversion_format != image_class.conversion_format ||
 			    (!storage_swizzle && image.shader_swizzle != image_class.shader_swizzle)) {
 				return SpecializationFail(
@@ -1111,7 +1167,7 @@ bool BuildSamplerPlan(const ShaderInfo& base, SamplerPlan& plan) {

 template <typename Predicate>
 static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, const Predicate& predicate) {
-	// The renderer checks captured scalar reads against final buffer and image write ranges.
+	// Address writes can change the values control flow reads.
 	if (program.has_address_writes) {
 		return {};
 	}
@@ -1164,14 +1220,16 @@ static std::vector<ResourceBlock> ResourceControlFlow(const Program& program, co
 				block.sources.push_back(program.info.buffers.at(memory.resource).source);
 			} else {
 				block.sources.push_back(program.info.images.at(memory.resource).source);
+				block.images.push_back(memory.resource);
 				if (image.needs_sampler) {
 					block.sources.push_back(program.info.samplers.at(memory.sampler).source);
 				}
 			}
 		}
-		std::ranges::sort(block.sources);
-		block.sources.erase(std::unique(block.sources.begin(), block.sources.end()),
-		                    block.sources.end());
+		for (auto* list: {&block.sources, &block.images}) {
+			std::ranges::sort(*list);
+			list->erase(std::unique(list->begin(), list->end()), list->end());
+		}
 	}
 	for (auto& block: blocks) block.condition = predicate(block.condition);
 	if (std::ranges::none_of(
@@ -1645,7 +1703,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 		}
 		if (source->indirect_descriptor.has_value()) {
 			snapshot.images[i] = {.dword_count = 8u};
-			if (!WalkedActive(program, image.source)) {
+			if (!WalkedActive(program, image.source) || !WalkedImageActive(program, i)) {
 				continue;
 			}
 			const auto& indirect = *source->indirect_descriptor;
@@ -1665,7 +1723,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 			if (!evaluate(image.source, snapshot.images[i])) {
 				return false;
 			}
-			if (!ValidImageDescriptor(snapshot.images[i], image.r128)) {
+			// An image listed only by unreached blocks reads zero, not its source's descriptor.
+			if (!WalkedImageActive(program, i) ||
+			    !ValidImageDescriptor(snapshot.images[i], image.r128)) {
 				snapshot.images[i].dwords.fill(0);
 			}
 		}
@@ -1711,8 +1771,9 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
 }

 uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value) {
-	// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, and
-	// entries that need shader-side conversion, read as null instead of failing the draw.
+	// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, entries of
+	// another dimension and entries that need shader-side conversion read as null instead of
+	// failing the draw.
 	const auto format = static_cast<Prospero::BufferFormat>((value.dwords[1] >> 20u) & 0x1ffu);
 	const auto tile = static_cast<Prospero::TileMode>((value.dwords[3] >> 20u) & 0x1fu);
 	const auto numeric_class =
@@ -7,6 +7,8 @@

 #include <algorithm>
 #include <bit>
+#include <cmath>
+#include <cstdio>
 #include <fmt/format.h>
 #include <map>
 #include <numeric>
@@ -1014,7 +1016,9 @@ private:
 				    (!shared_image_heap && (a.table_offset != b.table_offset ||
 				                           a.table_immediate != b.table_immediate ||
 				                           a.table_stride != b.table_stride)) || a.indexed != b.indexed ||
-				    a.workgroup_axis != b.workgroup_axis || a.sources != b.sources ||
+				    a.workgroup_axis != b.workgroup_axis ||
+				    a.workgroup_multiplier != b.workgroup_multiplier ||
+				    a.workgroup_shift != b.workgroup_shift || a.sources != b.sources ||
 				    a.signed_count != b.signed_count ||
 				    !EquivalentValue(m_program, a.key_count, b.key_count) ||
 				    a.selector_first.IsEmpty() != b.selector_first.IsEmpty() ||
@@ -1136,31 +1140,63 @@ private:
 		LaneQuantifier lanes = LaneQuantifier::All;
 	};

+	// The immediate that edge proves value differs from on every lane, if any.
+	std::optional<uint32_t> ExcludedBy(const EdgePredicate& edge, Value value) const {
+		if (edge.lanes != LaneQuantifier::All) return {};
+		const auto* test = edge.condition.TryInstruction();
+		if (test != nullptr && test->NumArgs() == 2u &&
+		    ((test->GetOpcode() == ValueOpcode::INotEqual32 && edge.positive) ||
+		     (test->GetOpcode() == ValueOpcode::IEqual32 && !edge.positive))) {
+			for (uint32_t arg = 0; arg < 2u; ++arg) {
+				uint32_t immediate;
+				if (ImmediateU32(test->Arg(arg), immediate) &&
+				    EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) return immediate;
+			}
+		}
+		return {};
+	}
+
 	bool NonzeroOnEntry(Value value, const Block* block) const {
 		if (block == nullptr) return false;
 		bool witnessed = false;
 		const bool guarded = GuardedOnEntry(block, [](const Block* current) {
 			return current->ImmPredecessors().size() != 1u;
 		}, [&](const EdgePredicate& edge) {
-			if (edge.lanes != LaneQuantifier::All) return false;
-			const auto* test = edge.condition.TryInstruction();
-			if (test != nullptr && test->NumArgs() == 2u &&
-			    ((test->GetOpcode() == ValueOpcode::INotEqual32 && edge.positive) ||
-			     (test->GetOpcode() == ValueOpcode::IEqual32 && !edge.positive))) {
-				for (uint32_t arg = 0; arg < 2u; ++arg) {
-					uint32_t immediate;
-					if (ImmediateU32(test->Arg(arg), immediate) && immediate == 0u &&
-					    EquivalentValue(m_program, test->Arg(arg ^ 1u), value)) return witnessed = true;
-				}
-			}
-			return false;
+			return ExcludedBy(edge, value) == 0u && (witnessed = true);
 		});
 		return guarded && witnessed;
 	}

-	uint32_t WorkgroupAxis(Value key) const {
+	// The key that every path from its definition to block rules out, if any.
+	std::optional<uint32_t> ExcludedKeyOnEntry(Value key, const Block* block) const {
+		const auto* definition = key.Resolve().TryInstruction();
+		if (definition == nullptr) return {};
+		std::optional<uint32_t> excluded;
+		const auto defined = [&](const Block* current) { return current == definition->Parent(); };
+		const bool guarded = GuardedOnEntry(block, defined, [&](const EdgePredicate& edge) {
+			const auto immediate = ExcludedBy(edge, key);
+			if (!immediate || (excluded && excluded != immediate)) return false;
+			excluded = immediate;
+			return true;
+		});
+		return guarded ? excluded : std::nullopt;
+	}
+
+	// The workgroup ID axis of key; with first, also of a runtime first + workgroup ID sum.
+	uint32_t WorkgroupAxis(Value key, Value* first = nullptr) const {
 		const auto* builtin = key.Resolve().TryInstruction();
 		uint32_t axis = UINT32_MAX;
+		if (first != nullptr && builtin != nullptr && builtin->GetOpcode() == ValueOpcode::IAdd32) {
+			for (uint32_t arm = 0; arm < 2u; ++arm) {
+				axis = WorkgroupAxis(builtin->Arg(arm));
+				const auto base = builtin->Arg(arm ^ 1u);
+				if (axis == UINT32_MAX ||
+				    !ValidateRuntimeValue(m_program, base, RuntimeValueType::Integer)) continue;
+				*first = base;
+				return axis;
+			}
+			return UINT32_MAX;
+		}
 		if (m_program.stage != ShaderType::Compute || builtin == nullptr ||
 		    builtin->GetOpcode() != ValueOpcode::GetBuiltin || builtin->NumArgs() != 2u ||
 		    builtin->Arg(0) != Value(static_cast<uint32_t>(StageInputKind::WorkgroupId)) ||
@@ -1168,6 +1204,27 @@ private:
 		return axis;
 	}

+	// Matches key = (first + workgroup ID) * multiplier >> shift with shift >= 32; first and the
+	// division are optional.
+	bool MatchWorkgroupKey(Value key, DescriptorSource::IndirectDescriptor& indirect) const {
+		uint32_t bits = 0, multiplier = 0;
+		const auto* shift = key.Resolve().TryInstruction();
+		if (shift != nullptr && shift->GetOpcode() == ValueOpcode::ShiftRightLogical32) {
+			const auto* high = shift->Arg(0).Resolve().TryInstruction();
+			if (high != nullptr && high->GetOpcode() == ValueOpcode::UMulHi &&
+			    ImmediateU32(shift->Arg(1), bits) && bits < 32u &&
+			    ImmediateU32(high->Arg(1), multiplier)) {
+				key = high->Arg(0);
+				bits += 32u;
+			}
+		}
+		indirect.workgroup_axis = WorkgroupAxis(key, &indirect.selector_first);
+		if (indirect.workgroup_axis == UINT32_MAX) return false;
+		indirect.workgroup_multiplier = multiplier;
+		indirect.workgroup_shift = bits;
+		return true;
+	}
+
 	bool MatchTableOffset(Value value, Value& key, uint32_t& offset, uint32_t& stride) const {
 		offset = 0;
 		for (;;) {
@@ -1672,6 +1729,8 @@ private:
 		uint32_t initial_arm;
 	};

+	static bool ZeroBased(uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; }
+
 	std::optional<Induction> MatchInduction(Value key, const auto& accepts_step) const {
 		key = key.Resolve();
 		const auto* phi = key.TryInstruction();
@@ -1733,8 +1792,7 @@ private:
 	}

 	uint32_t PostTestLoopMaximum(Value key) const {
-		const auto induction = MatchInduction(key,
-		    [](uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; });
+		const auto induction = MatchInduction(key, ZeroBased);
 		if (!induction) return UINT32_MAX;
 		const auto [phi, step, initial_arm] = *induction;
 		const auto* incoming = phi->PhiBlock(initial_arm ^ 1u);
@@ -1767,8 +1825,7 @@ private:

 	const Inst* BoundedLoop(Value key, const Block* use, const auto& accepts_bound,
 	                       LaneQuantifier lanes = LaneQuantifier::Any) const {
-		return BoundedInduction(key, use,
-		    [](uint32_t initial, uint32_t step) { return initial == 0u && step == 1u; },
+		return BoundedInduction(key, use, ZeroBased,
 		    [&](const Inst& compare, uint32_t initial) {
 			    return (compare.GetOpcode() == ValueOpcode::SLessThan32 ||
 			            compare.GetOpcode() == ValueOpcode::ULessThan32) &&
@@ -1793,6 +1850,29 @@ private:
 		});
 	}

+	// A zero-based key that only per-lane data bounds takes the count all uniform zero-based
+	// loops in the program share, assuming each lane's count stays within it.
+	bool ProgramLoopCount(Value key, DescriptorSource::IndirectDescriptor& indirect) const {
+		if (!MatchInduction(key, ZeroBased) || BoundedLoop(key, nullptr) != nullptr) return false;
+		const Inst* count = nullptr;
+		for (auto* block: m_program.blocks) {
+			for (auto& phi: *block) {
+				if (phi.GetOpcode() != ValueOpcode::Phi) break;
+				const auto* compare = BoundedLoop(Value(&phi), nullptr);
+				if (compare == nullptr) continue;
+				if (count != nullptr && !EquivalentValue(m_program, compare->Arg(1), count->Arg(1)))
+					return false;
+				count = compare;
+			}
+		}
+		if (count == nullptr) return false;
+		std::printf("Warning: shader 0x%016llx bounds a per-lane table loop by its uniform loops' "
+		            "count\n", static_cast<unsigned long long>(m_program.shader_hash));
+		indirect.key_count    = count->Arg(1);
+		indirect.signed_count = count->GetOpcode() == ValueOpcode::SLessThan32;
+		return true;
+	}
+
 	Value ActiveValue(Value value, Value guard) const {
 		value = value.Resolve();
 		while (const auto* select = value.TryInstruction()) {
@@ -1829,6 +1909,10 @@ private:
 	}

 	bool Dominates(const Block* initial, const Block* use, Value witness = {}) const {
+		// The proven local-zero invocation fails every all-lane bypass condition that excludes it,
+		// such as an all-inactive or an all-nonzero bypass.
+		const auto* negated = SimplifyGuard(witness).TryInstruction();
+		const bool nonzero = negated != nullptr && negated->GetOpcode() == ValueOpcode::LogicalNot;
 		std::vector<const Block*> pending {use};
 		for (size_t index = 0; index < pending.size(); ++index) {
 			const auto* current = pending[index];
@@ -1837,9 +1921,9 @@ private:
 			for (const auto* previous: current->ImmPredecessors()) {
 				if (!witness.IsEmpty()) {
 					const auto edge = ConditionalEdge(previous, current);
-					// The proven local-zero invocation makes an all-inactive bypass impossible.
-					if (edge && edge->lanes == LaneQuantifier::All && !edge->positive &&
-					    Implies(witness, edge->condition)) continue;
+					if (edge && edge->lanes == LaneQuantifier::All &&
+					    (edge->positive ? nonzero && Implies(edge->condition, negated->Arg(0))
+					                    : Implies(witness, edge->condition))) continue;
 				}
 				if (std::ranges::find(pending, previous) == pending.end()) pending.push_back(previous);
 			}
@@ -1948,7 +2032,10 @@ private:

 	bool LocalZeroWitness(Value guard) const {
 		const auto* equal = SimplifyGuard(guard).TryInstruction();
-		if (equal == nullptr || equal->GetOpcode() != ValueOpcode::IEqual32) return false;
+		const bool negated = equal != nullptr && equal->GetOpcode() == ValueOpcode::LogicalNot;
+		if (negated) equal = SimplifyGuard(equal->Arg(0)).TryInstruction();
+		const auto expected = negated ? ValueOpcode::INotEqual32 : ValueOpcode::IEqual32;
+		if (equal == nullptr || equal->GetOpcode() != expected) return false;
 		for (uint32_t arg = 0; arg < 2u; ++arg) {
 			if (equal->Arg(arg).Resolve() != Value(0u)) continue;
 			Value induction;
@@ -1982,6 +2069,10 @@ private:
 		    EquivalentValue(m_program, test->Arg(0), value)) return test->Arg(1);
 		if ((positive ? op == ValueOpcode::UGreaterThan32 : op == ValueOpcode::ULessThanEqual32) &&
 		    EquivalentValue(m_program, test->Arg(1), value)) return test->Arg(0);
+		uint32_t limit = 0;
+		if ((positive ? op == ValueOpcode::ULessThanEqual32 : op == ValueOpcode::UGreaterThan32) &&
+		    ImmediateU32(test->Arg(1), limit) && limit != UINT32_MAX &&
+		    EquivalentValue(m_program, test->Arg(0), value)) return Value(limit + 1u);
 		const auto* scan = value.Resolve().TryInstruction();
 		if (finite_bitscan && scan != nullptr && scan->GetOpcode() == ValueOpcode::FindILsb32 &&
 		    (positive ? op == ValueOpcode::INotEqual32 : op == ValueOpcode::IEqual32)) {
@@ -2056,7 +2147,7 @@ private:
 		const auto* read = ActiveValue(value, active).TryInstruction();
 		if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadSharedU32 ||
 		    m_program.stage != ShaderType::Compute ||
-		    read->Arg(1).Resolve() != Value(true)) return {};
+		    !Implies(active, read->Arg(1))) return {};
 		uint64_t group_size = 1u;
 		for (const auto dimension: m_input.compute->threads_num) {
 			if (dimension == 0u || dimension > m_input.compute->wave_size) return {};
@@ -2087,10 +2178,16 @@ private:
 		const auto* limit = BoundedLoop(read_offset.index, read->Parent(),
 		    [](const Inst& compare, uint32_t) { return compare.GetOpcode() == ValueOpcode::ULessThan32; });
 		if (limit == nullptr) return {};
+		auto counter_active = active;
 		const auto* counter_read = ActiveValue(limit->Arg(1), active).TryInstruction();
+		// Every lane reads the same counter word, so a selected lane carries it.
+		if (counter_read != nullptr && counter_read->GetOpcode() == ValueOpcode::ReadFirstLane) {
+			counter_active = counter_read->Arg(1);
+			counter_read = ActiveValue(counter_read->Arg(0), counter_active).TryInstruction();
+		}
 		uint32_t consumed_counter = 0;
 		if (counter_read == nullptr || counter_read->GetOpcode() != ValueOpcode::LoadSharedU32 ||
-		    counter_read->Arg(1).Resolve() != Value(true) ||
+		    !Implies(counter_active, counter_read->Arg(1)) ||
 		    !constant_address(*counter_read, consumed_counter)) return {};

 		struct Initializer { const Inst* inst; uint32_t address; uint32_t words; };
@@ -2304,25 +2401,23 @@ private:
 		    ((offset | stride | memory->offset) & 3u) == 0u &&
 		    uint64_t {offset} + memory->offset <= UINT32_MAX;
 		if (affine) {
-			workgroup_axis = WorkgroupAxis(index);
-			if (workgroup_axis == UINT32_MAX) {
-				const auto* sum = index.Resolve().TryInstruction();
-				if (sum != nullptr && sum->GetOpcode() == ValueOpcode::IAdd32) {
-					for (uint32_t arm = 0; arm < 2u; ++arm) {
-						const auto axis = WorkgroupAxis(sum->Arg(arm));
-						const auto base = sum->Arg(arm ^ 1u);
-						if (axis == UINT32_MAX ||
-						    !ValidateRuntimeValue(m_program, base, RuntimeValueType::Integer)) continue;
-						workgroup_axis = axis;
-						first = base;
-						break;
-					}
-				}
-			}
+			workgroup_axis = WorkgroupAxis(index, &first);
 			const auto* bound = BoundedLoop(index, read->Parent());
 			if (bound != nullptr && bound->GetOpcode() == ValueOpcode::ULessThan32) {
 				count = bound->Arg(1);
+			} else if (workgroup_axis == UINT32_MAX && !require_bound) {
+				// Other bounded indices, such as bit scans, read the records up to their maximum;
+				// a bit scan of a nonzero mask load reads only the ranks its words set.
+				const auto maximum = SelectorMaximum(index);
+				if (maximum == 31u) selector.ranks = BitScanRanks(index);
+				if (maximum < MaxIndirectDescriptorProbes) count = Value(maximum + 1u);
 			}
+			// A record selected from a shared append list is below the list's bound.
+			const auto* lane = index.Resolve().TryInstruction();
+			if (count.IsEmpty() && lane != nullptr &&
+			    lane->GetOpcode() == ValueOpcode::ReadFirstLane &&
+			    HasActiveLane(lane->Arg(1), lane->Parent()))
+				count = SharedLoadBound(lane->Arg(0), lane->Arg(1));
 		}
 		// Raw pointers have no V# bounds for enumerating an unbounded selector domain.
 		if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty() &&
@@ -2498,7 +2593,8 @@ private:
 	// Each equation is an inductive subset obligation. Phis require every incoming
 	// value; AND needs either operand. Grounding rejects cycles without an initial
 	// value, then the greatest fixed point checks every initial and carried edge.
-	bool BitsetMaskSubset(Value mask, Value active, Value word, const Inst& read) const {
+	bool BitsetMaskSubset(Value mask, Value active, Value word,
+	                      std::span<const Inst* const> reads) const {
 		struct State {
 			Value value;
 			Value guard;
@@ -2510,7 +2606,10 @@ private:
 			bool all = true;
 			std::vector<size_t> terms;
 		};
-		const auto load_guard = read.Arg(4).Resolve();
+		// A scalar load reads one word without EXEC; four-word loads take their guard.
+		const bool scalar = reads[0]->GetOpcode() == ValueOpcode::ReadConstBuffer;
+		const auto load_guard = scalar ? Value(true) : reads[0]->Arg(4).Resolve();
+		const auto words = static_cast<uint32_t>(reads.size()) * (scalar ? 1u : 4u);
 		std::vector<State> states(2);
 		std::vector<Equation> equations(2);
 		const auto add = [&](State state) {
@@ -2521,9 +2620,9 @@ private:
 			equations.emplace_back();
 			return states.size() - 1u;
 		};
-		std::array<size_t, 5> roots {add({.value = active})};
-		for (uint32_t component = 0; component < 4u; ++component)
-			roots[component + 1u] = add({mask, active, component, component});
+		std::vector<size_t> roots {add({.value = active})};
+		for (uint32_t component = 0; component < words; ++component)
+			roots.push_back(add({mask, active, component, component}));
 		for (size_t index = 2; index < states.size(); ++index) {
 			const auto state = states[index];
 			const auto* inst = state.value.TryInstruction();
@@ -2573,11 +2672,14 @@ private:
 				switch (inst->GetOpcode()) {
 					case ValueOpcode::CompositeExtractU32x4: {
 						uint32_t component;
-						if (inst->Arg(0).Resolve().TryInstruction() == &read &&
-						    ImmediateU32(inst->Arg(1), component) && component == state.component)
+						if (inst->Arg(0).Resolve().TryInstruction() == reads[state.component / 4u] &&
+						    ImmediateU32(inst->Arg(1), component) && component == state.component % 4u)
 							equation.terms = {1u};
 						break;
 					}
+					case ValueOpcode::ReadConstBuffer:
+						if (scalar && inst == reads[state.component]) equation.terms = {1u};
+						break;
 					case ValueOpcode::Phi:
 						// The predecessor may be an earlier outer iteration. Establish
 						// this register's invariant without the current word or EXEC.
@@ -2598,17 +2700,17 @@ private:
 								const auto* masked = key.TryInstruction();
 								uint32_t bound = 0;
 								if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 &&
-								    DescendingLoop(word, masked->Parent(), bound) && bound == 3u) {
+								    DescendingLoop(word, masked->Parent(), bound) && bound == words - 1u) {
 									for (uint32_t side = 0; side < 2u; ++side) {
 										uint32_t bits;
-										if (ImmediateU32(masked->Arg(side), bits) && (bits & 3u) == 3u)
+										if (ImmediateU32(masked->Arg(side), bits) && (bits & bound) == bound)
 											key = masked->Arg(side ^ 1u).Resolve();
 									}
 								}
 								return EquivalentValue(m_program, key, word);
 							});
 							uint32_t literal;
-							if (ImmediateU32(equality, literal) && literal < 4u) selected_word = literal;
+							if (ImmediateU32(equality, literal) && literal < words) selected_word = literal;
 							equation.terms.push_back(term(inst->Arg(1), inst->Arg(0), selected_word));
 						}
 						if (!selected || !*selected)
@@ -2646,7 +2748,53 @@ private:
 		return std::ranges::all_of(roots, [&](size_t root) { return valid[root]; });
 	}

-	bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds,
+	// The mask words whose set bits are every record a bit scan of index selects: the scanned
+	// value keeps only bits of one scalar load, which reads a word its address can reach.
+	std::optional<DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks>
+	BitScanRanks(Value index) {
+		const auto* scan = index.Resolve().TryInstruction();
+		if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindILsb32) return {};
+		const auto* masked = scan->Arg(0).Resolve().TryInstruction();
+		if (masked == nullptr) return {};
+		std::vector<const Inst*> values {masked};
+		const Inst* load = nullptr;
+		for (size_t i = 0; i < values.size() && load == nullptr; ++i) {
+			const auto* value = values[i];
+			if (value->GetOpcode() == ValueOpcode::ReadConstBuffer) load = value;
+			if (value->GetOpcode() != ValueOpcode::Phi &&
+			    value->GetOpcode() != ValueOpcode::BitwiseAnd32) continue;
+			for (size_t arg = 0; arg < value->NumArgs(); ++arg) {
+				const auto* next = value->Arg(arg).Resolve().TryInstruction();
+				if (next != nullptr && std::ranges::find(values, next) == values.end())
+					values.push_back(next);
+			}
+		}
+		uint32_t memory_index = 0;
+		const auto* memory = load != nullptr ? ScalarReadMemory(*load, memory_index) : nullptr;
+		if (memory == nullptr || memory->offset > INT32_MAX ||
+		    !MemoryIndexBelongsTo(memory_index, *load) ||
+		    !BitsetMaskSubset(scan->Arg(0), Value(true), Value(0u), std::array {load})) return {};
+		auto address = load->Arg(1);
+		auto active = Value(true);
+		const auto* lane = address.Resolve().TryInstruction();
+		if (lane != nullptr && lane->GetOpcode() == ValueOpcode::ReadFirstLane) {
+			if (!HasActiveLane(lane->Arg(1), lane->Parent())) return {};
+			active = lane->Arg(1);
+			address = lane->Arg(0);
+		}
+		// A left shift keeps the address a multiple of its scale; other addresses read any word.
+		const auto* shift = ActiveValue(address, active).TryInstruction();
+		uint32_t bits = 0;
+		if (shift == nullptr || shift->GetOpcode() != ValueOpcode::ShiftLeftLogical32 ||
+		    !ImmediateU32(shift->Arg(1), bits) || bits >= 32u) bits = 0;
+		DescriptorSource source;
+		if (!MakeRuntimeTableSource(*load, source)) return {};
+		return DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks {
+		    .source = InternSource(source), .stride = std::max(1u << bits, 4u),
+		    .offset = memory->offset & ~3u};
+	}
+
+	bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds, uint32_t words,
 	                         DescriptorSource::IndirectDescriptor& indirect) {
 		const auto selected_value = [&](Value value) {
 			for (;;) {
@@ -2670,7 +2818,7 @@ private:
 		}
 		const auto* word_inst = word.TryInstruction();
 		if (word_inst == nullptr || std::ranges::none_of(bounds, [&](const auto& bound) {
-			return bound.first == word_inst && bound.second == 3u;
+			return bound.first == word_inst && bound.second == words - 1u;
 		})) return false;
 		// FFBH's leading-zero count is inverted again to recover the set-bit rank.
 		for (uint32_t inverse = 0; inverse < 2u; ++inverse) {
@@ -2682,21 +2830,34 @@ private:
 		const auto* scan = bit.TryInstruction();
 		if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindUMsb32 ||
 		    !ImpliesNonzero(active, scan->Arg(0))) return false;
+		const auto bitmap_load = [&](const Inst& read, uint32_t offset) {
+			const auto index = read.Flags<MemoryFlags>().index;
+			if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(read.Parent()) ||
+			    index >= m_program.memory_info.size()) return false;
+			const auto& memory = m_program.memory_info[index];
+			return memory.kind == ResourceKind::Buffer && !memory.typed && !memory.formatted &&
+			       memory.idxen && !memory.offen && memory.offset == offset &&
+			       memory.data_bits == 32u && memory.data_dwords == 4u &&
+			       read.Arg(2).Resolve() == Value(0u) && read.Arg(3).Resolve() == Value(0u);
+		};
 		for (const auto* block: m_program.blocks) {
 			for (const auto& read: *block) {
-				if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(block)) continue;
-				const auto index = read.Flags<MemoryFlags>().index;
-				if (index >= m_program.memory_info.size()) continue;
-				const auto& memory = m_program.memory_info[index];
-				if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
-				    !memory.idxen || memory.offen || memory.offset != 0u || memory.data_bits != 32u ||
-				    memory.data_dwords != 4u || read.Arg(2).Resolve() != Value(0u) ||
-				    read.Arg(3).Resolve() != Value(0u) ||
-				    !BitsetMaskSubset(scan->Arg(0), active, word, read)) continue;
+				if (!bitmap_load(read, 0u)) continue;
+				// An eight-word record continues in a second four-word load of the same buffer and EXEC.
+				std::array<const Inst*, 2> reads {&read};
+				if (words == 8u)
+					for (const auto* other: m_program.blocks)
+						for (const auto& load: *other)
+							if (reads[1] == nullptr && bitmap_load(load, 16u) &&
+							    EquivalentValue(m_program, load.Arg(0), read.Arg(0)) &&
+							    EquivalentValue(m_program, load.Arg(4), read.Arg(4))) reads[1] = &load;
+				const auto parts = std::span(reads).first(words / 4u);
 				DescriptorSource source;
-				if (!MakeRuntimeTableSource(read, source)) continue;
+				if (std::ranges::find(parts, nullptr) != parts.end() ||
+				    !BitsetMaskSubset(scan->Arg(0), active, word, parts) ||
+				    !MakeRuntimeTableSource(read, source)) continue;
 				indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead {
-				    .source = InternSource(source), .stride = 16u, .offset = 0u, .bitset = true});
+				    .source = InternSource(source), .stride = words * 4u, .offset = 0u, .bitset = true});
 				return true;
 			}
 		}
@@ -2726,7 +2887,8 @@ private:
 		first.SetArg(1, Value(maximum));
 		indirect.selector_first = Value(&first);
 		indirect.key_count = Value(maximum + 1u);
-		if (maximum == 127u) MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, indirect);
+		if (maximum == 127u || maximum == 255u)
+			MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, (maximum + 1u) / 32u, indirect);
 		return true;
 	}

@@ -2993,6 +3155,47 @@ private:
 		return true;
 	}

+	// The EXEC predicate of the image access through use; empty for any other use.
+	Value AccessPredicate(const Use& use, bool sampler) const {
+		const auto& access = *use.user;
+		if (use.operand != (sampler ? 1u : 0u) ||
+		    ImageOpcodeInfoOf(access.GetOpcode()).access == ImageAccess::None) return {};
+		if (sampler && (access.GetOpcode() != ValueOpcode::ImageSampleRaw ||
+		    (m_program.memory_info[access.Flags<MemoryFlags>().index].image_sample_flags &
+		     Decoder::ImageSampleFlagAdjust) != 0u)) return {};
+		const auto mask = access.Arg(access.NumArgs() - 1u);
+		return mask.GetType() == Type::U1 ? mask : Value {};
+	}
+
+	// A key no bound covers takes its maximum from a signed clamp to a constant, on the key or on
+	// the key plus a constant in an earlier-tracked sibling read of the same table. Negative keys
+	// and keys past that maximum read the first row.
+	uint32_t ClampedTableMaximum(Value key, const DescriptorSource& table,
+	                             const DescriptorSource::IndirectDescriptor& indirect) const {
+		uint32_t row = 0, offset = 0;
+		const auto clamped = [&](Value value) {
+			const auto* min = value.Resolve().TryInstruction();
+			if (min != nullptr && min->GetOpcode() == ValueOpcode::SMin32)
+				for (uint32_t arg = 0; arg < 2u; ++arg)
+					if (ImmediateU32(min->Arg(arg), row)) return min->Arg(arg ^ 1u);
+			return Value {};
+		};
+		if (!clamped(key).IsEmpty()) return row;
+		for (const auto& sibling: m_indirect_descriptors) {
+			const auto& read = *m_sources[sibling.source].indirect_descriptor;
+			const auto* add = clamped(sibling.key).Resolve().TryInstruction();
+			if (add == nullptr || add->GetOpcode() != ValueOpcode::IAdd32 ||
+			    m_sources[read.table_source] != table ||
+			    read.table_offset != indirect.table_offset ||
+			    read.table_stride != indirect.table_stride ||
+			    read.table_immediate != indirect.table_immediate) continue;
+			for (uint32_t arg = 0; arg < 2u; ++arg)
+				if (ImmediateU32(add->Arg(arg), offset) &&
+				    EquivalentValue(m_program, add->Arg(arg ^ 1u), key)) return row;
+		}
+		return UINT32_MAX;
+	}
+
 	bool TryMakeTableDescriptor(Inst& handle, const DescriptorSource& descriptor,
 	                            IndirectDescriptorPlan& plan) {
 		const bool sampler = handle.GetOpcode() == ValueOpcode::GetSamplerResource;
@@ -3019,6 +3222,7 @@ private:
 		} else if (!plan.table_indexed &&
 		           MatchScalarMaterialKey(key, indirect, material_source,
 		                                  !sampler && table_source.dword_count == 4u, sampler)) {
+			indirect.selector->excluded = ExcludedKeyOnEntry(key, handle.Parent());
 			// Keep live descriptor payloads, but project descriptor-only SMEM onto
 			// the GPU key using the same plan as other indirect image reads.
 			plan.retain_reads |= table_source.dword_count == 4u;
@@ -3080,36 +3284,42 @@ private:
 			if (indirect.key_count.IsEmpty() &&
 			    !MatchUniformizedMaterialKey(key, handle, indirect, material_source, false) &&
 			    !MatchUniformizedMaterialKey(key, handle, indirect, material_source, true) &&
-			    !MatchReversedImageRange(key, indirect)) {
-				const auto maximum = SelectorMaximum(key);
-				if (maximum >= MaxIndirectDescriptorProbes) return false;
-				indirect.key_count = Value(maximum + 1u);
+			    !MatchReversedImageRange(key, indirect) && !MatchWorkgroupKey(key, indirect)) {
+				auto maximum = SelectorMaximum(key);
+				if (maximum >= MaxIndirectDescriptorProbes) {
+					// Each image access bounds the key on the lanes it executes.
+					maximum = 0u;
+					for (const auto& use: handle.Uses()) {
+						const auto active = AccessPredicate(use, false);
+						if (active.IsEmpty()) {
+							maximum = UINT32_MAX;
+							break;
+						}
+						maximum = std::max(maximum, SelectorMaximum(key, {}, 0, active));
+					}
+				}
+				if (maximum >= MaxIndirectDescriptorProbes)
+					maximum = ClampedTableMaximum(key, table_source, indirect);
+				if (maximum < MaxIndirectDescriptorProbes)
+					indirect.key_count = Value(maximum + 1u);
+				else if (!ProgramLoopCount(key, indirect))
+					return false;
 			}
 			if ((table_offset & 3u) != 0u) return false;
 		} else {
 			// Keep the GPU selector and descriptor reads, and preserve the proven scalar
 			// offset domain within the bounded image table.
 			if (sampler || plan.split_offsets) return false;
-			const auto active = [&](const Use& use, bool sampler) -> Value {
-				const auto& access = *use.user;
-				if (use.operand != (sampler ? 1u : 0u) ||
-				    ImageOpcodeInfoOf(access.GetOpcode()).access == ImageAccess::None) return {};
-				if (sampler && (access.GetOpcode() != ValueOpcode::ImageSampleRaw ||
-				    (m_program.memory_info[access.Flags<MemoryFlags>().index].image_sample_flags &
-				     Decoder::ImageSampleFlagAdjust) != 0u)) return {};
-				const auto mask = access.Arg(access.NumArgs() - 1u);
-				return mask.GetType() == Type::U1 ? mask : Value {};
-			};
 			bool shared_selector = false;
 			if (!plan.table_indexed) {
 				for (const auto& owner: m_indirect_descriptors) {
 					const auto& selected = *m_sources[owner.source].indirect_descriptor;
 					if (!selected.selector || !EquivalentValue(m_program, key, owner.key) ||
 					    !std::ranges::all_of(handle.Uses(), [&](const Use& target) {
-						    const auto target_active = active(target, false);
+						    const auto target_active = AccessPredicate(target, false);
 						    return !target_active.IsEmpty() &&
 						        std::ranges::any_of(owner.handle->Uses(), [&](const Use& witness) {
-							        const auto owner_active = active(witness,
+							        const auto owner_active = AccessPredicate(witness,
 							            owner.handle->GetOpcode() == ValueOpcode::GetSamplerResource);
 							        return !owner_active.IsEmpty() && AvailableBefore(*witness.user, *target.user) &&
 							            Implies(target_active, owner_active);
@@ -3430,6 +3640,42 @@ private:
 		return result;
 	}

+	// Bounds of an F32 value on the lanes where guard holds; NaN when unknown.
+	std::pair<float, float> FloatRange(Value value, Value guard, uint32_t depth = 0) const {
+		constexpr std::pair unknown {NAN, NAN};
+		value = ActiveValue(value, guard);
+		if (value.IsImmediate() && value.GetType() == Type::F32)
+			return {value.F32Value(), value.F32Value()};
+		const auto* inst = value.TryInstruction();
+		// Each multiply-add evaluates three operands; keep the search shallow.
+		if (depth > 12u || inst == nullptr) return unknown;
+		const auto range = [&](uint32_t arg) {
+			return FloatRange(inst->Arg(arg), guard, depth + 1u);
+		};
+		switch (inst->GetOpcode()) {
+			case ValueOpcode::BitCastF32U32:
+			case ValueOpcode::BitCastU32F32: return range(0);
+			// Clamping maps NaN to zero.
+			case ValueOpcode::FPSaturate32: return {0.0f, 1.0f};
+			case ValueOpcode::FPFloor32: {
+				const auto [low, high] = range(0);
+				return {std::floor(low), std::floor(high)};
+			}
+			case ValueOpcode::FPMad32: {
+				// The rounded multiply and add are monotonic, so the bounds lie at the corners.
+				const auto a = range(0), b = range(1), c = range(2);
+				if (!std::isfinite(a.first) || !std::isfinite(a.second) ||
+				    !std::isfinite(b.first) || !std::isfinite(b.second)) return unknown;
+				const auto [low, high] = std::minmax({a.first * b.first, a.first * b.second,
+				                                      a.second * b.first, a.second * b.second});
+				const std::pair bounds {low + c.first, high + c.second};
+				if (!std::isfinite(bounds.first) || !std::isfinite(bounds.second)) return unknown;
+				return bounds;
+			}
+			default: return unknown;
+		}
+	}
+
 	uint32_t SelectorMaximum(Value value, SelectorBounds bounds = {}, uint32_t depth = 0,
 	                         Value guard = {}) const {
 		if (depth > 32u) return UINT32_MAX;
@@ -3445,9 +3691,13 @@ private:
 		switch (inst->GetOpcode()) {
 			case ValueOpcode::ReadFirstLane:
 				return SelectedSelectorDomain(*inst, guard).maximum;
-			case ValueOpcode::ReadLane:
-				// ReadLane ignores EXEC; only context-free bounds on every source lane apply.
-				return PossibleU32Bits(inst->Arg(0));
+			case ValueOpcode::ReadLane: {
+				// ReadLane ignores EXEC; only context-free bounds on every source lane apply,
+				// unless guard proves a waterfall key equal to the executing lanes' local value.
+				const auto local = EqualLocalKey(guard, value);
+				return local.IsEmpty() ? PossibleU32Bits(inst->Arg(0))
+				                       : SelectorMaximum(local, bounds, depth + 1u, guard);
+			}
 			case ValueOpcode::Phi: {
 				const auto initial = InitialCandidateMask(value);
 				if (!initial.IsEmpty()) return SelectorMaximum(initial, bounds, depth + 1u, guard);
@@ -3474,6 +3724,18 @@ private:
 				               NonzeroOnEntry(inst->Arg(0), inst->Parent()) ? 31u : UINT32_MAX;
 			case ValueOpcode::UMin32:
 			case ValueOpcode::BitwiseAnd32: return std::min(maximum(0), maximum(1));
+			case ValueOpcode::SMin32: {
+				// Signed and unsigned minimums agree on nonnegative operands.
+				const auto left = maximum(0), right = maximum(1);
+				if (std::max(left, right) <= INT32_MAX) return std::min(left, right);
+				break;
+			}
+			case ValueOpcode::ConvertS32F32: {
+				// The conversion truncates toward zero.
+				const auto [low, high] = FloatRange(inst->Arg(0), guard);
+				if (low >= 0.0f && high < 0x1p31f) return static_cast<uint32_t>(high);
+				break;
+			}
 			case ValueOpcode::SelectU32:
 				if (!guard.IsEmpty() && Implies(guard, inst->Arg(0))) return maximum(1);
 				return SelectorPredicateFalse(inst->Arg(0), bounds, depth + 1u, guard)
@@ -3760,10 +4022,10 @@ private:
 		}
 	}

-	uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, const Inst& inst, uint32_t pc) {
+	uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
 		for (uint32_t i = 0; i < m_info.buffers.size(); i++) {
 			if (m_info.buffers[i].source == source) {
-				Merge(m_info.buffers[i], memory, inst, pc);
+				Merge(m_info.buffers[i], memory, op, pc);
 				return i;
 			}
 		}
@@ -3773,14 +4035,13 @@ private:
 		BufferResource resource;
 		resource.source       = source;
 		resource.first_use_pc = pc;
-		Merge(resource, memory, inst, pc);
+		Merge(resource, memory, op, pc);
 		m_info.buffers.push_back(resource);
 		return static_cast<uint32_t>(m_info.buffers.size() - 1);
 	}

-	static void Merge(BufferResource& resource, const MemoryInfo& memory, const Inst& inst,
+	static void Merge(BufferResource& resource, const MemoryInfo& memory, ValueOpcode op,
 	                  uint32_t pc) {
-		const auto op            = inst.GetOpcode();
 		const auto access        = BufferAccessOf(op);
 		const bool atomic        = access == BufferAccess::Atomic;
 		const bool write         = access == BufferAccess::Write || atomic;
@@ -3788,13 +4049,6 @@ private:
 		resource.max_byte_extent = std::max(resource.max_byte_extent, ByteExtent(memory));
 		resource.read            = resource.read || !write || atomic;
 		resource.written         = resource.written || write;
-		if (write) {
-			// Formatted stores write the whole format; otherwise zero index, VGPR and SGPR offsets
-			// leave only the immediate offset.
-			const bool fixed = !memory.formatted && inst.Arg(1).Resolve() == Value(0u) &&
-			                   inst.Arg(2).Resolve() == Value(0u) && inst.Arg(3).Resolve() == Value(0u);
-			resource.write_extent = std::max(resource.write_extent, fixed ? ByteExtent(memory) : UINT32_MAX);
-		}
 		resource.atomic          = resource.atomic || atomic;
 		resource.formatted       = resource.formatted || memory.formatted;
 		resource.scalar          = resource.scalar || op == ValueOpcode::ReadConstBuffer ||
@@ -3920,7 +4174,7 @@ private:

 	void Collect(Inst& inst) {
 		const auto op           = inst.GetOpcode();
-		if (op == ValueOpcode::BvhIntersect) {
+		if (op == ValueOpcode::BvhIntersect || op == ValueOpcode::Bvh8Intersect) {
 			m_info.uses_dma = true;
 			return;
 		}
@@ -3964,7 +4218,7 @@ private:
 				m_info.uses_dma                         = true;
 				return;
 			}
-			resource = AddBuffer(source, memory, inst, flags.pc);
+			resource = AddBuffer(source, memory, op, flags.pc);
 			if (resource == UINT32_MAX) {
 				Fail(flags.pc, "buffer resource limit exceeded");
 			}
@@ -175,6 +175,8 @@ void AddBuiltinInput(ShaderInfo& info, StageInputKind kind) {
 		case StageInputKind::GlobalInvocationId:
 			AddInput(info, kind, 0, 3, "gl_GlobalInvocationID");
 			break;
+		case StageInputKind::BaseVertex: AddInput(info, kind, 0, 1, "gl_BaseVertex"); break;
+		case StageInputKind::BaseInstance: AddInput(info, kind, 0, 1, "gl_BaseInstance"); break;
 		case StageInputKind::PackedAncillary:
 		case StageInputKind::Parameter: break;
 	}
@@ -444,6 +446,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 				case StageInputKind::InstanceIndex:
 				case StageInputKind::FrontFacing:
 				case StageInputKind::LocalInvocationIndex:
+				case StageInputKind::BaseVertex:
+				case StageInputKind::BaseInstance:
 					if (component != 0u) {
 						return Fail("typed scalar builtin component is out of range");
 					}
@@ -511,6 +515,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 		case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break;
 		case ValueOpcode::DataCacheInvalidate: info.coherent_buffers = true; break;
 		case ValueOpcode::BvhIntersect: info.bvh = true; break;
+		case ValueOpcode::Bvh8Intersect: info.bvh8 = true; break;
+		case ValueOpcode::BvhStack: info.bvh_stack = true; break;
 		case ValueOpcode::ConditionRef:
 			info.subgroup_vote |=
 			    inst.Flags<CFG::BranchCondition>() != CFG::BranchCondition::ScalarInstruction;
@@ -518,7 +524,8 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
 		case ValueOpcode::Ballot: info.subgroup_ballot = true; break;
 		case ValueOpcode::LiveMask: info.subgroup_ballot = info.helper_invocation = true; break;
 		case ValueOpcode::SubgroupUMinU32:
-		case ValueOpcode::SubgroupUMaxU32: info.subgroup_arithmetic = true; break;
+		case ValueOpcode::SubgroupUMaxU32:
+		case ValueOpcode::SubgroupOrU32: info.subgroup_arithmetic = true; break;
 		case ValueOpcode::DppMoveU32:
 		case ValueOpcode::ReadFirstLane:
 		case ValueOpcode::ReadLane: {
@@ -82,6 +82,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
 		case ValueOpcode::BitCastU32F32:
 		case ValueOpcode::BitCastF32U32:
 		case ValueOpcode::ConvertU32F32:
+		case ValueOpcode::ConvertS32F32:
 		case ValueOpcode::ConvertF32U32:
 		case ValueOpcode::CompositeConstructU64:
 		case ValueOpcode::CompositeExtractU64:
@@ -147,8 +148,8 @@ bool ImmediateBits(Value value, uint64_t& bits) {
 }

 // Scratch layout in bytes: per descriptor source its eight DWORDs and a known DWORD, one
-// activity byte per source, then one qword per walked value (all ones when unknown). The
-// walker's value slots, captured addresses, reachability and states follow.
+// activity byte per source, then per image, then one qword per walked value (all ones when
+// unknown). The walker's value slots, captured addresses, reachability and states follow.
 constexpr uint32_t DescriptorStride = 9u * sizeof(uint32_t);

 uint32_t ActiveOffset(const ResourcePlan& plan) {
@@ -156,7 +157,8 @@ uint32_t ActiveOffset(const ResourcePlan& plan) {
 }

 uint32_t OutputOffset(const ResourcePlan& plan) {
-	return (ActiveOffset(plan) + static_cast<uint32_t>(plan.descriptor_sources.size()) + 7u) & ~7u;
+	const auto active = plan.descriptor_sources.size() + plan.info.images.size();
+	return (ActiveOffset(plan) + static_cast<uint32_t>(active) + 7u) & ~7u;
 }

 const uint8_t* Scratch(const ResourcePlan& plan) {
@@ -1038,6 +1040,23 @@ void WalkerCompiler::EmitStrict(const Node& node) {
 			c.L(done);
 			break;
 		}
+		case ValueOpcode::ConvertS32F32: {
+			// The conversion saturates and maps NaN to zero. cvttss2si's out-of-range result,
+			// INT32_MIN, is already right for negative overflow.
+			Xbyak::Label done;
+			c.movd(c.xmm0, c.eax);
+			c.xor_(c.eax, c.eax);
+			c.ucomiss(c.xmm0, c.xmm0);
+			c.jp(done, Near);
+			c.mov(c.ecx, 0x4f000000u);
+			c.movd(c.xmm1, c.ecx);
+			c.mov(c.eax, INT32_MAX);
+			c.ucomiss(c.xmm0, c.xmm1);
+			c.jae(done, Near);
+			c.cvttss2si(c.eax, c.xmm0);
+			c.L(done);
+			break;
+		}
 		case ValueOpcode::FPMul32:
 			c.movd(c.xmm0, c.eax);
 			c.movd(c.xmm1, c.ecx);
@@ -1272,11 +1291,14 @@ void WalkerCompiler::Compile(std::span<const Value> values) {
 			for (const auto slot: blocks[block].srt_reads) flat(slot);
 			c.L(skip);
 		}
-		// A source no block lists is always active.
-		for (uint32_t source = 0; source < sources.size(); ++source) {
+		// A source or image no block lists is always active.
+		for (uint32_t index = 0; index < sources.size() + m_plan.info.images.size(); ++index) {
+			const bool     image = index >= sources.size();
+			const uint32_t id    = image ? index - static_cast<uint32_t>(sources.size()) : index;
 			bool listed = false;
 			for (uint32_t block = 0; block < blocks.size(); ++block) {
-				if (std::ranges::find(blocks[block].sources, source) == blocks[block].sources.end()) continue;
+				const auto& list = image ? blocks[block].images : blocks[block].sources;
+				if (std::ranges::find(list, id) == list.end()) continue;
 				if (listed) {
 					c.or_(c.al, c.byte[c.rdx + m_reach + block]);
 				} else {
@@ -1284,7 +1306,7 @@ void WalkerCompiler::Compile(std::span<const Value> values) {
 				}
 				listed = true;
 			}
-			const auto active = c.byte[c.rdx + ActiveOffset(m_plan) + source];
+			const auto active = c.byte[c.rdx + ActiveOffset(m_plan) + index];
 			if (listed) {
 				c.mov(active, c.al);
 			} else {
@@ -1391,6 +1413,10 @@ bool WalkedActive(const ResourcePlan& plan, uint32_t source) {
 	return Scratch(plan)[ActiveOffset(plan) + source] != 0u;
 }

+bool WalkedImageActive(const ResourcePlan& plan, uint32_t image) {
+	return Scratch(plan)[ActiveOffset(plan) + plan.descriptor_sources.size() + image] != 0u;
+}
+
 bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValueType type) {
 	return RuntimeValidator(program, type).Run(value);
 }
@@ -78,6 +78,7 @@ bool WalkSrt(const ResourcePlan& plan, const SrtRuntime& runtime, std::vector<ui
 bool WalkedValue(const ResourcePlan& plan, Value value, uint32_t& result);
 bool WalkedDescriptor(const ResourcePlan& plan, uint32_t source, DescriptorValue& result);
 bool WalkedActive(const ResourcePlan& plan, uint32_t source);
+bool WalkedImageActive(const ResourcePlan& plan, uint32_t image);
 // Length of the guest load at rip if it belongs to the walker running on this thread, else 0.
 // The exception handler completes such a load as a zero read.
 uint32_t SrtWalkerLoadLength(uint64_t rip);
@@ -348,14 +348,20 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
 } // namespace

 void RewriteToSsa(const BlockList& blocks) {
-	Pass pass;
+	Pass                               pass;
+	std::unordered_map<Block*, size_t> filled_predecessors;
 	for (auto* block: blocks) {
+		if (block->ImmPredecessors().empty()) {
+			pass.Seal(block);
+		}
 		for (auto& inst: *block) {
 			VisitInstruction(pass, block, inst);
 		}
-	}
-	for (auto* block: blocks) {
-		pass.Seal(block);
+		for (auto* successor: block->ImmSuccessors()) {
+			if (++filled_predecessors[successor] == successor->ImmPredecessors().size()) {
+				pass.Seal(successor);
+			}
+		}
 	}
 }

@@ -1,5 +1,6 @@
 #include "graphics/shader/shader.h"

+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/emulatorConfig.h"
@@ -19,6 +20,7 @@
 #include <algorithm>
 #include <bit>
 #include <cstdio>
+#include <cstring>
 #include <memory>
 #include <mutex>
 #include <span>
@@ -59,15 +61,33 @@ void ShaderInit() {
 	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMapEntry>>();
 }

+// The program's code and, at align16(code size), its literal buffer, as sized by the 0x30-byte
+// footer that ends a binary; the whole binary when the footer is unknown.
+static uint32_t ShaderProgramSize(const uint32_t* binary, uint32_t size) {
+	constexpr uint32_t FooterSize = 0x30;
+	if (size < FooterSize) return size;
+	// Magic, then version, code size and literal buffer size at +0x10.
+	const auto* footer = binary + (size - FooterSize) / 4;
+	const auto  end    = Common::AlignUp(uint64_t {footer[5]}, 16) + footer[6];
+	if (std::memcmp(footer, "barefoot", 8) != 0 || footer[4] != 1 || footer[5] == 0 ||
+	    end % 4 != 0 || end > size - FooterSize) {
+		return size;
+	}
+	return static_cast<uint32_t>(end);
+}
+
 void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 	EXIT_IF(g_shader_map == nullptr);

 	if (addr == 0 || data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
-		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n",
+		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid shader_size=0x%08" PRIx32 "\n",
 		     addr, data.code_size_bytes);
 	}
-	const auto hash = XXH3_64bits(reinterpret_cast<const void*>(addr), data.code_size_bytes);
+	const auto*   binary = reinterpret_cast<const uint32_t*>(addr);
 	ShaderMapData mapped {data};
+	// Translation reads the bytes the hash covers.
+	mapped.code_size_bytes = ShaderProgramSize(binary, data.code_size_bytes);
+	const auto hash        = XXH3_64bits(binary, mapped.code_size_bytes);
 	mapped.vertex_error =
 	    ShaderReadVertexMetadata(data, HW::UserSgprInfo::SGPRS_MAX, mapped.vertex);
 	std::scoped_lock lock(g_shader_map_mutex);
@@ -291,10 +311,9 @@ static bool ShaderGetStaticVertexInputInfo(uint64_t shader_addr, const HW::UserS
 	const auto& metadata = data.vertex;

 	if (metadata.vertex_attrib_reg >= 0) {
-		info.fetch_external   = false;
-		info.fetch_embedded   = true;
-		info.fetch_attrib_reg = metadata.vertex_attrib_reg;
-		info.fetch_buffer_reg = metadata.vertex_buffer_reg;
+		info.has_dynamic_vertex_fetch = true;
+		info.fetch_attrib_reg         = metadata.vertex_attrib_reg;
+		info.fetch_buffer_reg         = metadata.vertex_buffer_reg;

 		const auto* attrib = reinterpret_cast<const uint32_t*>(
 		    static_cast<uint64_t>(user_sgpr.value[metadata.vertex_attrib_reg]) |
@@ -413,7 +432,7 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
 void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t>& key) {
 	EXIT_IF(info.resources_num < 0 || info.resources_num > ShaderVertexInputInfo::RES_MAX);
 	key.clear();
-	key.push_back(static_cast<uint32_t>(info.fetch_embedded));
+	key.push_back(static_cast<uint32_t>(info.has_dynamic_vertex_fetch));
 	key.push_back(static_cast<uint32_t>(info.fetch_attrib_reg));
 	key.push_back(static_cast<uint32_t>(info.fetch_buffer_reg));
 	key.push_back(info.resources_num);
@@ -422,6 +441,8 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector<uint32_t
 	key.push_back(info.lds_size_dwords);
 	key.push_back(info.pa_cl_vs_out_cntl);
 	key.push_back(info.user_clip_planes);
+	key.push_back(info.indirect_offsets.vertex | (uint32_t {info.indirect_offsets.instance} << 8u) |
+	              (uint32_t {info.indirect_draw} << 16u));
 	key.push_back(static_cast<uint32_t>(info.clip_space.enabled));
 	if (info.clip_space.enabled) {
 		for (const float value: info.clip_space.scale) {
@@ -692,9 +713,7 @@ void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo& info) {

 	LOGF("ShaderDbgDumpInputInfo()\n");

-	LOGF("\t fetch_external = %s\n"
-	     "\t fetch_embedded = %s\n",
-	     info.fetch_external ? "true" : "false", info.fetch_embedded ? "true" : "false");
+	LOGF("\t has_dynamic_vertex_fetch = %s\n", info.has_dynamic_vertex_fetch ? "true" : "false");

 	for (int i = 0; i < info.resources_num; i++) {
 		LOGF("\t input %d\n", i);
@@ -124,6 +124,17 @@ struct ShaderTessellationInputInfo {
 	uint32_t output_topology       = 0;
 };

+// GS user-data slots an indirect draw packet writes its record's base vertex and first instance
+// to. A packet can leave both slots unused even when its record contains nonzero offsets.
+struct ShaderIndirectOffsets {
+	static constexpr uint8_t None = UINT8_MAX;
+
+	uint8_t vertex   = None;
+	uint8_t instance = None;
+
+	bool operator==(const ShaderIndirectOffsets&) const = default;
+};
+
 struct ShaderVertexInputInfo {
 	static constexpr int RES_MAX = 32;

@@ -146,8 +157,11 @@ struct ShaderVertexInputInfo {
 	ShaderClipSpaceTransform clip_space;
 	ShaderMeshInputInfo      mesh;
 	ShaderTessellationInputInfo tess;
-	bool                    fetch_external      = false;
-	bool                    fetch_embedded      = false;
+	ShaderIndirectOffsets   indirect_offsets;
+	// GPU-read records add Vulkan bases even when the packet patches no user SGPRs.
+	bool                    indirect_draw       = false;
+	// Attribute-table metadata enables dynamic fetch detection; loads are matched later.
+	bool                    has_dynamic_vertex_fetch = false;
 };

 struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo {
@@ -26,9 +26,10 @@ struct DebugStats {
 	uint64_t flexible_total      = 0;
 	uint64_t flexible_used       = 0;
 	uint64_t pool_committed      = 0;
-	uint64_t cpu_page_entries    = 0; // 2 MiB page-table entries in use.
-	uint64_t gpu_page_entries    = 0;
-	uint64_t page_entries_total  = 0; // Per side.
+	uint64_t cpu_page_entries       = 0; // 2 MiB page-table entries in use.
+	uint64_t gpu_page_entries       = 0;
+	uint64_t cpu_page_entries_total = 0;
+	uint64_t gpu_page_entries_total = 0;
 };
 [[nodiscard]] DebugStats GetDebugStats();

@@ -125,7 +126,6 @@ static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
              "KernelMemoryPoolBlockStats struct size is incorrect");

 void                   RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
-void                   SetFlexibleMemorySize(uint64_t size);
 int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size, size_t alignment,
                         int memory_type, int64_t* phys_addr_out, bool automatic = false);
 int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
@@ -197,6 +197,9 @@ int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* outp

 uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
                               Common::VirtualMemory::Mode mode, const char* name);
+// Loaded modules' segments use flexible memory without taking its backing.
+void ChargeFlexibleMemory(uint64_t size);
+void ReleaseFlexibleMemory(uint64_t size);
 void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
 uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
                               Common::VirtualMemory::Mode mode, const char* name,
@@ -222,6 +225,11 @@ bool     TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
 bool     TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
 bool     TestGuestBackingOutsideAddressSpace();
 uint64_t TestGuestBackingSize();
+uint64_t TestFlexibleBackingHighWater();
+uint64_t TestGuestFreeRangeVisits();
+uint64_t TestDirectMappingCount();
+uint64_t TestFlexibleMappingCount();
+uint64_t TestBackingBytesZeroed();
 bool     TestGuestFreeRangeBounds();
 #endif

@@ -203,6 +203,9 @@ public:
 		}
 		std::lock_guard lock(m_mutex);
 		std::memset(m_backing_base + backing_offset, 0, size);
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+		g_test_backing_bytes_zeroed += size;
+#endif
 		return true;
 	}

@@ -287,6 +290,18 @@ public:
 		return true;
 	}

+	bool ProtectViews(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) {
+		std::shared_lock lock(m_mutex);
+		bool             ok = true;
+		ForEachSparseRange(
+		    m_maps, vaddr, size,
+		    [&](const auto&, uint64_t start, uint64_t bytes) {
+			    ok = ok && VirtualMemory::Protect(start, bytes, mode);
+		    },
+		    [&](uint64_t, uint64_t) { ok = false; });
+		return ok;
+	}
+
 	bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset,
 	              VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) {
 		if (!IsAvailable() || !RangeOk(backing_offset, size)) {
@@ -324,15 +339,11 @@ public:
 #endif
 	}

-	bool Unmap(uint64_t vaddr, uint64_t size, bool* placeholder_preserved = nullptr) {
+	bool Unmap(uint64_t vaddr, uint64_t size) {
 		if (size == 0 || UINT64_MAX - vaddr < size) {
 			return false;
 		}

-		if (placeholder_preserved != nullptr) {
-			*placeholder_preserved = false;
-		}
-
 		const auto            end = vaddr + size;
 		std::vector<MapEntry> targets;
 		{
@@ -350,42 +361,36 @@ public:
 			}
 		}

-		std::vector<MapEntry> removed;
-		removed.reserve(targets.size());
+		size_t removed  = 0;
+		auto   rollback = [&] {
+			while (removed != 0) {
+				const auto& previous = targets[--removed];
+				EXIT_IF(!MapFixed(previous.vaddr, previous.size, previous.backing_offset,
+				                  previous.mode));
+			}
+		};
 		for (const auto& entry: targets) {
-			bool part_placeholder_preserved = false;
-			if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) {
-				for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
-					EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
-				}
-				if (placeholder_preserved != nullptr) {
-					*placeholder_preserved = false;
-				}
+			if (!UnmapOne(entry.vaddr, entry.size)) {
+				rollback();
 				return false;
 			}
-			EXIT_IF(!part_placeholder_preserved);
-			removed.push_back(entry);
+			++removed;
 		}

 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		// Each unmapped file view becomes a separate Windows placeholder. The address-space owner
 		// records this contiguous guest span as one free range, so normalize fragmented backing
 		// views into one placeholder before publishing it there.
-		if (removed.size() > 1 && !CoalescePlaceholders(vaddr, size)) {
-			for (auto it = removed.rbegin(); it != removed.rend(); ++it) {
-				EXIT_IF(!MapFixed(it->vaddr, it->size, it->backing_offset, it->mode));
-			}
+		if (removed > 1 && !CoalescePlaceholders(vaddr, size)) {
+			rollback();
 			return false;
 		}
 #endif

-		if (placeholder_preserved != nullptr) {
-			*placeholder_preserved = true;
-		}
 		return true;
 	}

-	bool UnmapOne(uint64_t vaddr, uint64_t size, bool* placeholder_preserved = nullptr) {
+	bool UnmapOne(uint64_t vaddr, uint64_t size) {
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
 		if (g_test_backing_store_unmaps_before_failure == 0) {
 			g_test_backing_store_unmaps_before_failure = UINT32_MAX;
@@ -428,9 +433,7 @@ public:
 				m_maps.emplace(old.vaddr, old);
 				return false;
 			}
-			if (placeholder_preserved != nullptr) {
-				*placeholder_preserved = true;
-			}
+
 			return true;
 		}

@@ -451,9 +454,6 @@ public:
 				     ok;
 			}
 			if (ok) {
-				if (placeholder_preserved != nullptr) {
-					*placeholder_preserved = true;
-				}
 				return true;
 			}

@@ -481,9 +481,6 @@ public:
 		std::lock_guard lock(m_mutex);
 		RemoveMappingsUnlocked(vaddr, size);

-		if (placeholder_preserved != nullptr) {
-			*placeholder_preserved = true;
-		}
 		return true;
 #endif
 	}
@@ -634,8 +631,6 @@ private:

 	HANDLE m_handle = nullptr;
 #elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
-	[[nodiscard]] static uint64_t GetMappableTestSize() { return 0x4000; }
-
 	static int GetProtect(VirtualMemory::Mode mode) {
 		switch (mode) {
 			case VirtualMemory::Mode::Read: return PROT_READ;
@@ -760,7 +755,7 @@ public:
 		if (size == 0 || UINT64_MAX - vaddr < size) {
 			return false;
 		}
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		return std::any_of(m_owned.begin(), m_owned.end(), [vaddr, size](const auto& range) {
 			return VirtualRangesOverlap(vaddr, size, range.first, range.second);
 		});
@@ -864,11 +859,7 @@ public:
 		if (!ContainsMappedUnlocked(vaddr, size, MappingType::Backing)) {
 			return false;
 		}
-		bool placeholder_preserved = false;
-		if (!m_backing->Unmap(vaddr, size, &placeholder_preserved)) {
-			return false;
-		}
-		if (!placeholder_preserved) {
+		if (!m_backing->Unmap(vaddr, size)) {
 			return false;
 		}
 		EXIT_IF(!RemoveMappedUnlocked(vaddr, size, MappingType::Backing));
@@ -876,12 +867,13 @@ public:
 		return true;
 	}

+	// Reservations live in VirtualRanges; these free-range checks only read the host placeholders.
 	uint64_t ReserveAligned(uint64_t search_addr, uint64_t size, uint64_t alignment) {
 		if (size == 0) {
 			return 0;
 		}
 		alignment = std::max<uint64_t>(alignment != 0 ? alignment : PageSize(), PageSize());
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		if (search_addr == 0) {
 			auto addr = FindFreeAlignedUnlocked(HOST_SYSTEM_MANAGED_MIN,
 			                                    HOST_SYSTEM_MANAGED_MAX + 1u, size, alignment);
@@ -898,7 +890,7 @@ public:
 			return 0;
 		}
 		alignment = std::max<uint64_t>(alignment, PageSize());
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		return FindFreeAlignedUnlocked(search_start, search_end, size, alignment);
 	}

@@ -906,19 +898,19 @@ public:
 		if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) {
 			return false;
 		}
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		return FindContaining(vaddr, size) != m_free.end();
 	}

 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
 	bool TestContainsFree(uint64_t vaddr, uint64_t size) {
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		return FindContaining(vaddr, size) != m_free.end();
 	}
 #endif

 	bool ReleaseFree(uint64_t vaddr, uint64_t size) {
-		std::lock_guard lock(m_mutex);
+		std::shared_lock lock(m_mutex);
 		return FindContaining(vaddr, size) != m_free.end();
 	}

@@ -1041,10 +1033,6 @@ private:
 		return (value & (PageSize() - 1u)) == 0;
 	}

-	[[nodiscard]] bool IsGranularityAligned(uint64_t value) const {
-		return (value & (m_granularity - 1u)) == 0;
-	}
-
 	bool ProtectMappedUnlocked(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) {
 		const auto end = vaddr + size;
 		auto       it  = m_mapped.upper_bound(vaddr);
@@ -1055,8 +1043,20 @@ private:
 			const auto& region      = it->second;
 			const auto  range_start = std::max(vaddr, region.vaddr);
 			const auto  range_end   = std::min(end, region.vaddr + region.size);
-			if (range_start < range_end &&
-			    !VirtualMemory::Protect(range_start, range_end - range_start, mode)) {
+			if (range_start >= range_end) {
+				continue;
+			}
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+			// Rollback can split a backing view without changing the owner interval.
+			// Windows protection calls must stay within each actual view.
+			if (region.type == MappingType::Backing) {
+				if (!m_backing->ProtectViews(range_start, range_end - range_start, mode)) {
+					return false;
+				}
+				continue;
+			}
+#endif
+			if (!VirtualMemory::Protect(range_start, range_end - range_start, mode)) {
 				return false;
 			}
 		}
@@ -1312,12 +1312,21 @@ private:
 	}

 	uint64_t FindFreeAlignedUnlocked(uint64_t search_start, uint64_t search_end, uint64_t size,
-	                                 uint64_t alignment) {
+	                                 uint64_t alignment) const {
 		if (search_start >= search_end || size > search_end - search_start) {
 			return 0;
 		}
-		for (const auto& [base, range_size]: m_free) {
-			const auto candidate = AlignUp(std::max(base, search_start), alignment);
+		// Include the predecessor because search_start can lie inside a free range.
+		auto it = m_free.upper_bound(search_start);
+		if (it != m_free.begin()) {
+			--it;
+		}
+		for (; it != m_free.end() && it->first < search_end; ++it) {
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+			g_test_free_range_visits.fetch_add(1, std::memory_order_relaxed);
+#endif
+			const auto& [base, range_size] = *it;
+			const auto candidate           = AlignUp(std::max(base, search_start), alignment);
 			if (candidate < search_end && size <= search_end - candidate &&
 			    GuestFreeRangeContains(base, range_size, candidate, size)) {
 				return candidate;
@@ -1331,9 +1340,8 @@ private:
 		if (next == m_free.begin()) {
 			return m_free.end();
 		}
-		auto it  = std::prev(next);
-		auto end = it->first + it->second;
-		return (vaddr >= it->first && vaddr + size <= end ? it : m_free.end());
+		auto it = std::prev(next);
+		return GuestFreeRangeContains(it->first, it->second, vaddr, size) ? it : m_free.end();
 	}

 	bool SplitUnlocked(std::map<uint64_t, uint64_t>::iterator it, uint64_t vaddr, uint64_t size) {
@@ -2277,7 +2277,10 @@ int KYTY_SYSV_ABI PthreadRwlockDestroy(PthreadRwlock* rwlock) {
 		return KERNEL_ERROR_EINVAL;
 	}

-	EXIT_NOT_IMPLEMENTED(*rwlock == nullptr);
+	// SCE_PTHREAD_RWLOCK_INITIALIZER has no allocated resources until first use.
+	if (*rwlock == nullptr) {
+		return OK;
+	}

 	{
 		std::lock_guard lock((*rwlock)->m);
Author	SHA1	Message	Date
Axcxiii gl-ungus Claude Opus 5.5	bc4333e313	ngs2: size 16-bit PCM blocks in Ngs2CalcWaveformBlock (The Binding of Isaac: Repentance) (#1968 ) Ngs2CalcWaveformBlock returned NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT with a zeroed block for 16-bit PCM (0x12). The Binding of Isaac: Repentance (PPSA03311) ignores the error and queues the zeroed blocks, which the sampler skips, so the game had no audio at all. PCM frames have a fixed size, so the block now starts at sample_pos frames and spans num_samples frames. Fixes #667 Co-authored-by: gl-ungus <notasecondemail63@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-30 05:39:49 +01:00
nmzik	b0c8229bd4	docs: update the README status, Cyberpunk 2077 screenshot and notable forks	2026-10-30 05:17:52 +01:00
nmzik	bc1c779972	launcher: put the icon on a blue tile Icon based on designs by Ant_IV (Discord), earlher (Discord) and cosmo (Discord, GitHub), closest reference first.	2026-10-30 05:03:22 +01:00
Artyom Zabroda	0c2546e850	shader: lower whole-wave OR scans to subgroup reductions	2026-10-30 04:31:24 +01:00
nmzik	eb3cb47296	renderer: drop the per-draw scalar-read overlap guard	2026-10-30 04:19:05 +01:00
nmzikandArtyom Zabroda	a5cfdcfd32	texture_cache: apply DCC fast clears of mipmapped color targets Co-authored-by: Artyom Zabroda <artyomzabroda@gmail.com>	2026-10-30 04:05:24 +01:00
nmzik	ae67e2db51	shader: hash only the program code and literal buffer	2026-10-30 03:29:45 +01:00
nmzik	1703836d06	shader: type packed integer vertex inputs by their component type	2026-10-30 02:59:36 +01:00
nmzikandPhr3d13	b65dd18224	ngs2: let CLEAR reopen a closed waveform block list Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
nmzikandPhr3d13	2f6dccb10b	ngs2: mix ports without matrix levels with the default matrix Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
nmzikandPhr3d13	08a7d7fbed	ngs2: render the reverb voice dry signal Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
nmzikandPhr3d13	2aa0d58ffb	ngs2: report voice port routing and volume Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
nmzikandPhr3d13	9bf311c6c7	ngs2: report voice setup channels from the setup query Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
nmzikandPhr3d13	e66a7f52b5	ngs2: apply mastering voice full band and LFE gain Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
nmzikandPhr3d13	67f8276751	ngs2: patch to a rack handle connects to its voice 0 Co-authored-by: Phr3d13 <phr3d13@gmail.com>	2026-10-30 01:57:41 +01:00
KairosMakkiandAjakfial	3ad21e35cc	Rename near identifier to avoid Windows macro clash (#1983 ) Co-authored-by: Ajakfial <ajakafial@tutamail.com>	2026-10-30 00:41:22 +01:00
nmzik	5eab0ec738	shader: add BVH8 intersections and instance transforms	2026-10-29 18:37:29 +01:00
nmzik	bfd4c34b88	shader: add BVH stack operations	2026-10-29 18:37:29 +01:00
nmzik	6ff6e7551d	graphics: use budget-aware preferred-device image allocation	2026-10-29 18:37:29 +01:00
nmzik	25f3ba4eae	tests: cover low-alpha packed tiler conversion in both directions	2026-10-29 18:37:29 +01:00
Chai T	4592e4c178	kernel: destroy untouched static rwlocks without allocation	2026-10-29 18:32:14 +01:00
Chai T	97bef2992f	appcontent: reject duplicate mounts without transferring ownership	2026-10-29 18:32:14 +01:00
nmzikandChai T	15452a5fee	np: separate PSN emulation from host networking Co-authored-by: Chai T <chaitukca@gmail.com>	2026-10-29 08:30:55 +01:00
nmzikandJoshua Jayasingh	651ef73a03	graphics: add experimental Pro mode and linear PSSR Co-authored-by: Joshua Jayasingh <joshuajayasingh08@gmail.com>	2026-10-29 07:55:50 +01:00
nmzik	5e837a9bea	shader: share formatted indirect buffer load emission	2026-10-29 05:52:04 +01:00
nmzikandFrenzied	a0b99a8c21	ngs2: accept larger custom voice state buffers Co-authored-by: Frenzied <frenziedsaint@gmail.com>	2026-10-29 05:52:04 +01:00
nmzikandFrenzied	8cfe1c4d34	ngs2: apply custom channel matrices before mastering output Co-authored-by: Frenzied <frenziedsaint@gmail.com>	2026-10-29 05:52:04 +01:00
Çağatay Güçlü	f71da54b9b	shader: support fractional-even tessellation spacing	2026-10-29 05:52:04 +01:00
Çağatay Güçlü	63207739bc	shader: initialize local vertex count from active lanes	2026-10-29 05:52:04 +01:00
Çağatay Güçlü	143f640706	shader: preserve counterclockwise tessellation output	2026-10-29 05:52:04 +01:00
nmzik	54b937c85f	shader: support high products of 24-bit integers	2026-10-28 22:49:21 +01:00
nmzik	22610cb93e	shader: support unsigned 16-bit multiply-add	2026-10-28 22:41:29 +01:00
nmzik	0386f33bf3	shader: support four 16-bit integer ternary operations	2026-10-28 22:37:24 +01:00
nmzik	138d5b86c4	shader: support S_QUADMASK_B32	2026-10-28 22:31:16 +01:00
nmzik	963aaeb3b6	gpu: preserve low-alpha 2/10/10/10 render-target storage	2026-10-28 22:29:50 +01:00
Çağatay Güçlü	77458260f1	shader: reflect masked hull control-point addresses	2026-10-28 22:19:09 +01:00
nmzik	41de2fd64e	shader: support five VOP3 float conversion forms	2026-10-28 22:19:02 +01:00
Çağatay Güçlü	128c938514	shader: support quad tessellation domains and factor routing	2026-10-28 22:09:39 +01:00
Chai T	a8e3c29b2e	shader: implement V_ASHRREV_I64 (VOP3 0x301)	2026-10-28 21:51:28 +01:00
Devran Cosmo Uenal	c528324d27	audio: let the AJM Opus decoder write part of a packet (#1914 ) ajm: buffer and resample Opus output	2026-10-28 21:31:16 +01:00
Mr. Big BritchesandClaude Opus 5.5	c67352d5d6	ci: build and run the NGS2 sampler tests (#1957 ) ngs2_sampler_tests is registered with ctest but no job built it or listed it in the test filter, so the NGS2 unit tests never ran in CI. Each platform now builds the target and includes ngs2_sampler in its ctest filter. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-28 19:58:06 +01:00
Nawid3333	6c01401862	graphics: accept video-out buffers with 128-byte independent DCC blocks (Gran Turismo 7) (#1945 ) graphics: accept video-out buffers with 128-byte independent DCC blocks	2026-10-28 04:44:09 +01:00
nmzik	a833e1b735	graphics: fix vertex and instance IDs for unpatched indirect draws	2026-10-28 04:33:03 +01:00
nmzik	1be1c6c4f9	graphics: revert `a358b847c` command-processor display waits	2026-10-27 23:15:01 +01:00
M4andnmzik	69fdb03dc2	Launcher display options (#1938 ) * launcher: add minimize and maximize buttons to the main window The main window is a QDialog, which on Windows only gets a close button. * graphics: keep the game's aspect ratio when presenting to the window The guest frame was blitted across the whole swapchain image, so a window or fullscreen display whose shape differs from the frame stretched the picture. Fit the frame inside the window instead and clear the remaining bars to black. The video-out overlay layer is drawn into the same rectangle so it stays aligned with the frame underneath. * launcher: accept custom screen resolutions such as 21:9 The resolution was a fixed enum of four 16:9 sizes. Make the combo box editable so any WIDTHxHEIGHT between 320 and 16384 can be entered, and add ultrawide presets (2560x1080, 3440x1440, 3840x1600, 5120x2160). Settings saved by older launchers stored the enum key ("R1920X1080"); NormalizeResolution still accepts that form, so they load unchanged. * launcher: add a Stretch to window option Presentation now keeps the game's aspect ratio with black bars. Add --stretch to fill the whole window as before, exposed in the launcher as a "Stretch to window" checkbox under General, off by default. * launcher: make the settings dialog resizable and its tabs scrollable The dialog had no maximize button and pinned its minimum width to its initial 900 px, and without scroll areas it could not be made shorter than its content. Add a maximize button (not minimize, since the dialog is modal), drop the width pin, and wrap each tab page in a frameless scroll area. With the scroll areas the layout's minimum size no longer forces the dialog open, so size it explicitly: grow it to fit the largest tab, capped at the available screen, when there is no saved geometry and whenever the global settings add the game folders panel. * launcher: simplify display options and preserve saved settings --------- Co-authored-by: nmzik <Nmzik@mail.ru>	2026-10-27 21:59:56 +01:00
nmzik	2efd31443f	audio: support 512 object ports per AudioOut2 context	2026-10-27 21:41:30 +01:00
nmzikandMajed-allababedi	9eef2e1434	graphics: reuse range-set nodes when merging additions Co-authored-by: Majed-allababedi <60163390+Majed-allababedi@users.noreply.github.com>	2026-10-27 21:41:30 +01:00
nmzik	39faa62ebd	controller: merge stick and sensor events into the newest unread pad sample	2026-10-27 21:15:03 +01:00
nmzik	7ff8aff5e6	audio: drain the AudioOut2 queue at an exact 48 kHz grain rate	2026-10-27 21:15:03 +01:00
nmzik	64a554dea7	audio: a pushed AudioOut2 grain consumes its ports' PCM	2026-10-27 21:15:03 +01:00
nmzik	4b0282e343	savedata: sort directory searches by the requested key	2026-10-27 21:15:03 +01:00
nmzik	ffc81377b2	memory: refactor (#1941 ) * memory: refactor * tests: avoid Windows small macro collision * memory: retain allocation nodes on partial release	2026-10-27 21:14:11 +01:00
bigjjmoneymanandnmzik	0685bffb2f	texture cache: convert D16<->D24/D32 depth copies instead of crashing (#1873 ) gpu: convert guest D16 depth between native D16 and stenciled four-byte host depth Co-authored-by: nmzik <Nmzik@mail.ru>	2026-10-27 10:41:22 +01:00
nmzik	a5b3673a39	graphics: draw patched-offset indirect draws from GPU-read records	2026-10-27 10:13:15 +01:00
nmzik	09753bf8ff	memory: change defaults	2026-10-27 10:13:15 +01:00
nmzik	9d8c458535	gpu: keep reused smaller image tables resident and drop them a frame after their last draw	2026-10-27 10:13:15 +01:00
Mr. Big Britches Claude Opus 5.5 nmzik	358333125d	libFont: ship Noto Sans as the fallback font (#1917 ) * cmake: download Noto Sans at configure time Fetch the hinted Regular, Bold, Italic and Bold Italic of Noto Sans v2.015 and their licence from the Noto project's font distribution (notofonts.github.io) with FetchContent, like the other third-party dependencies. The commit and each file's SHA-256 are pinned in variables. Only these four styles are needed, so they are fetched individually instead of through the release archive (117 MB, every style and build). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * cmake: ship Noto Sans beside the emulator Copy the downloaded fonts to assets/fonts next to kyty_emulator on every build, like the touch cursors and the trophy sound, and install them there with the SIL Open Font License under licenses/noto-sans. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * libFont: fall back to the shipped Noto Sans instead of Tracy's Roboto Fonts stb_truetype cannot read, including every system font set opened with sceFontOpenFontSet, used Roboto from Tracy's source folder, found relative to the source tree or the working directory. Installed builds have neither, so their system fonts drew every character as '?'. Load Noto Sans Regular, Bold, Italic or Bold Italic from assets/fonts next to the emulator instead, and log when a file is missing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * libFont: fetch Noto Sans with file(DOWNLOAD) and drop the style substitution --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: nmzik <Nmzik@mail.ru>	2026-10-27 07:51:24 +01:00
nmzik	fd6b28c4b2	build: target x86-64-v3 without floating-point contraction	2026-10-27 07:04:38 +01:00
Senaxx	6cee2dc0bd	shader: inline the hot IR operand accessors	2026-10-27 07:04:38 +01:00
Senaxx	a526a8b9c9	shader: hand IR uses over at once on replacement and skip use lists at program teardown	2026-10-27 07:04:38 +01:00
Senaxx	1ec69f8e7d	gpu: make cull mode and front face dynamic state	2026-10-27 07:04:38 +01:00
nmzik	2463e3aaf0	gpu: write end-of-pipe labels beside GPU-written bytes to the backing store	2026-10-27 07:04:38 +01:00
nmzikandstrxno	556538bed5	gpu: write D16 demotion results with one 16-bit store Co-authored-by: strxno <rokas.pettersen@gmail.com>	2026-10-27 07:04:38 +01:00
Mr. Big BritchesandClaude Opus 5.5	770438cee0	ci: fix the Linux build and the macOS SRT walker test crashes (#1922 ) * graphics: give InlinePageOwnerList's const_iterator iterator traits The Linux build failed in textureCache.cpp: std::any_of over the image owners of a page needs std::iterator_traits, which libstdc++ only fills from the iterator's own member types. MSVC's STL and libc++ don't consult them there, so Windows and macOS built. Declare the five member types. The category is input, not forward, because operator* returns a copy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * tests: assert InlinePageOwnerList's const_iterator has an input category Only libstdc++ needs the iterator traits for std::any_of, so removing them would compile on Windows and macOS and break only the Linux build. The assert fails the build on every platform instead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * tests: serve SRT walker test loads on macOS too FakeGuestMemory only installed its fault handler on Linux and Windows, so on macOS the walker's deliberately faulting loads crashed scalar_provenance, resource_materialization and resource_tracking. Share the Linux sigaction path with macOS, read the registers from the macOS thread state, and catch SIGBUS as well as SIGSEGV there, like the emulator's own handler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-27 06:26:08 +01:00
nmzik	db12f0c5fb	shader: select consecutive GPU-selected sampler table entries without a switch	2026-10-27 04:23:01 +01:00
nmzik	3f9666acbb	shader: seal SSA blocks once their predecessors are filled	2026-10-27 04:23:01 +01:00
nmzik	7eb8abb279	shader: drop the duplicate-use scan from Inst::AddUse	2026-10-27 04:23:01 +01:00
Claxten	91a5ff6cb7	shader: decode s_dcache_inv as a no-op	2026-10-27 04:23:01 +01:00
Claxten	95a507a8ff	shader: support 64-bit buffer atomic minimum	2026-10-27 04:23:01 +01:00
Claxten	b2053c2861	shader: bound per-lane descriptor table loops by the program's uniform loop count	2026-10-27 04:23:01 +01:00
Claxten	9801fd862b	shader: zero image descriptors that only unreached blocks use	2026-10-27 04:23:01 +01:00
nmzikandshadowbeat070	78adb9293d	shader: give A16 1D and late 3D gradients their own address VGPRs Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>	2026-10-27 04:23:01 +01:00
Claxten	0a72641678	shader: evaluate saturating float-to-signed conversions in runtime descriptor values	2026-10-27 04:23:01 +01:00
Claxten	99f4eba223	shader: read bit-scan records only at ranks their mask buffer words set	2026-10-27 04:23:01 +01:00
nmzikandshadowbeat070	04bfd9070c	shader: give an A16 bias its own address VGPR Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>	2026-10-27 04:23:01 +01:00
nmzikandshadowbeat070	ffa3992c1a	shader: decode every PS5 image_gather4 encoding Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>	2026-10-27 04:23:00 +01:00
Claxten	1b70938052	agc: emit async memory semaphores and execute them in the CP	2026-10-27 04:23:00 +01:00
Claxten	914a3bb4b8	shader: enumerate set ranks of eight-word light bitmaps	2026-10-27 04:23:00 +01:00
Claxten	ffb6ad6cab	shader: bound selected-lane record indices by guarded shared append lists	2026-10-27 04:23:00 +01:00
Claxten	ff5d16eefc	tests: expect an inclusive child limit to bound one more record	2026-10-27 04:23:00 +01:00
Claxten	d118afbeae	tests: expect scalar selector records past the V# to read zero	2026-10-27 04:23:00 +01:00
Claxten	a3ac6d0d32	shader: bound table keys by an inclusive guard with a constant limit	2026-10-27 04:23:00 +01:00
Claxten	f297c0bfe1	shader: bound bit-scan record keys and drop the key a guard rules out	2026-10-27 04:23:00 +01:00
Claxten	20bb268a48	shader: bound raw image table keys by a signed clamp on the key or a sibling read	2026-10-27 04:23:00 +01:00
Claxten	d3768c00e9	shader: accept indirect image rows whose type reads the leading sample coordinates	2026-10-27 04:23:00 +01:00
Claxten	dfcf1c0b03	shader: bound waterfall image table keys converted from clamped float arithmetic	2026-10-27 04:23:00 +01:00
Claxten	de3ed905c8	shader: track image table keys divided from a runtime base plus the workgroup ID	2026-10-27 04:23:00 +01:00
Claxten	0a140911ad	kernel: default flexible memory to 448 MiB and charge loaded modules' segments to it	2026-10-27 04:23:00 +01:00
nmzik	2483573651	shader: mask V_MAD_U64_U32 carry-out bits of lanes outside EXEC	2026-10-27 04:23:00 +01:00
MrFr3di	986721f5bc	shader: decode FLAT, GLOBAL and SCRATCH D16 high-half loads (#1924 ) FLAT-family opcode 0x25 (FLAT/GLOBAL/SCRATCH_LOAD_SHORT_D16_HI) loads an unsigned halfword into bits [31:16] of the destination and keeps bits [15:0]. Decode it beside the low-half variant and let it select the high half of the existing partial destination write, so FLAT_LOAD needs no new path. Ghost of Yotei CS 0x79b0066f0cd1f3cd stopped at global_load_short_d16_hi v16, v18, s[0:1] offset:196 (0xdc9480c4 0x10000012).	2026-10-26 22:34:54 +01:00