shader: route inner shared exits first

Co-authored-by: Senaxx <Senaxx@users.noreply.github.com>

src/common/emulatorConfig.cpp

@@ -93,6 +93,10 @@ bool SpirvDebugPrintfEnabled() {
 	return g_config->spirv_debug_printf_enabled;
 }
 
+bool GpuAssistedValidationEnabled() {
+	return g_config->gpu_assisted_validation_enabled && g_config->vulkan_validation_enabled;
+}
+
 bool RenderDocEnabled() {
 	return g_config->renderdoc_enabled;
 }

src/common/emulatorConfig.h

@@ -49,6 +49,7 @@ struct ConfigOptions {
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   spirv_debug_printf_enabled  = false;
+	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
 	bool                   playgo_hack_enabled         = false;
@@ -84,6 +85,8 @@ ProfilerDirection GetProfilerDirection();
 
 bool SpirvDebugPrintfEnabled();
 
+bool GpuAssistedValidationEnabled();
+
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
 bool PlayGoHackEnabled();

src/common/slotVector.h

@@ -0,0 +1,132 @@
+#ifndef EMULATOR_SRC_COMMON_SLOTVECTOR_H_
+#define EMULATOR_SRC_COMMON_SLOTVECTOR_H_
+
+#include "common/assert.h"
+#include "common/abi.h"
+
+#include <compare>
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <limits>
+#include <optional>
+#include <utility>
+#include <vector>
+
+namespace Common {
+
+struct SlotId {
+	static constexpr uint32_t INVALID_INDEX = std::numeric_limits<uint32_t>::max();
+
+	constexpr SlotId() noexcept = default;
+	constexpr SlotId(uint32_t value) noexcept: index(value), generation(1) {}
+	constexpr SlotId(uint32_t value, uint32_t slot_generation) noexcept
+	    : index(value), generation(slot_generation) {}
+
+	[[nodiscard]] constexpr explicit operator bool() const noexcept { return index != INVALID_INDEX; }
+	constexpr auto operator<=>(const SlotId&) const noexcept = default;
+
+	uint32_t index = INVALID_INDEX;
+	uint32_t generation = 0;
+};
+
+// Stable-address slot storage for cache resources that are intentionally non-movable.
+template <typename T>
+class SlotVector {
+public:
+	SlotVector() = default;
+	KYTY_CLASS_NO_COPY(SlotVector);
+
+	[[nodiscard]] T& operator[](SlotId id) noexcept {
+		EXIT_IF(!is_allocated(id));
+		return *m_values[id.index].value;
+	}
+
+	[[nodiscard]] const T& operator[](SlotId id) const noexcept {
+		EXIT_IF(!is_allocated(id));
+		return *m_values[id.index].value;
+	}
+
+	[[nodiscard]] T* try_get(SlotId id) noexcept {
+		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+	}
+
+	[[nodiscard]] const T* try_get(SlotId id) const noexcept {
+		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+	}
+
+	[[nodiscard]] bool is_allocated(SlotId id) const noexcept {
+		return id && id.index < m_values.size() &&
+		       m_values[id.index].generation == id.generation && m_values[id.index].value.has_value();
+	}
+
+	template <typename... Args>
+	[[nodiscard]] SlotId insert(Args&&... args) {
+		uint32_t index = 0;
+		if (m_free_list.empty()) {
+			index = static_cast<uint32_t>(m_values.size());
+			m_values.emplace_back();
+			m_values.back().value.emplace(std::forward<Args>(args)...);
+		} else {
+			index = m_free_list.back();
+			m_free_list.pop_back();
+			EXIT_IF(m_values[index].value.has_value());
+			m_values[index].value.emplace(std::forward<Args>(args)...);
+		}
+		++m_size;
+		return SlotId {index, m_values[index].generation};
+	}
+
+	void erase(SlotId id) noexcept {
+		EXIT_IF(!is_allocated(id));
+		auto& slot = m_values[id.index];
+		slot.value.reset();
+		if (++slot.generation == 0) {
+			slot.generation = 1;
+		}
+		m_free_list.push_back(id.index);
+		--m_size;
+	}
+
+	[[nodiscard]] size_t size() const noexcept { return m_size; }
+	[[nodiscard]] size_t capacity() const noexcept { return m_values.size(); }
+
+	template <typename F>
+	void ForEach(F&& fn) {
+		for (uint32_t index = 0; index < m_values.size(); ++index) {
+			if (m_values[index].value) {
+				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+			}
+		}
+	}
+
+	template <typename F>
+	void ForEach(F&& fn) const {
+		for (uint32_t index = 0; index < m_values.size(); ++index) {
+			if (m_values[index].value) {
+				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+			}
+		}
+	}
+
+private:
+	struct Slot {
+		std::optional<T> value;
+		uint32_t         generation = 1;
+	};
+
+	std::deque<Slot>             m_values;
+	std::vector<uint32_t>        m_free_list;
+	size_t                       m_size = 0;
+};
+
+} // namespace Common
+
+template <>
+struct std::hash<Common::SlotId> {
+	[[nodiscard]] size_t operator()(Common::SlotId id) const noexcept {
+		return std::hash<uint64_t> {}((static_cast<uint64_t>(id.generation) << 32u) | id.index);
+	}
+};
+
+#endif // EMULATOR_SRC_COMMON_SLOTVECTOR_H_

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -103,7 +103,7 @@ public:
 	                    uint32_t mode);
 	void DispatchIndirect(uint32_t data_offset, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
-	void TriggerEvent(uint32_t event_type, uint32_t event_index);
+	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);
 
 	void SetUserDataMarker(HW::UserSgprType type) { m_user_data_marker = type; }
 	[[nodiscard]] HW::UserSgprType GetUserDataMarker() const { return m_user_data_marker; }
@@ -178,8 +178,9 @@ private:
 	uint32_t m_const_ram[0x3000] = {0};
 
 	FlipInfo m_flip;
-	uint64_t m_submit_id      = 0;
-	bool     m_predicate_skip = false;
+	uint64_t m_submit_id                   = 0;
+	uint64_t m_synthetic_occlusion_counter = 0;
+	bool     m_predicate_skip              = false;
 };
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -35,7 +35,6 @@ constexpr auto MakeContextDispatchTable() {
 	}
 	g_hw_ctx_func[Pm4::CB_TARGET_MASK]                 = HwCtxSetRenderTargetMask;
 	g_hw_ctx_func[Pm4::PA_SC_GENERIC_SCISSOR_TL]       = HwCtxSetGenericScissor;
-	g_hw_ctx_func[Pm4::CB_BLEND_RED]                   = HwCtxSetBlendColor;
 	g_hw_ctx_func[Pm4::CB_DCC_CONTROL]                 = HwCtxSetCbDccControl;
 	g_hw_ctx_func[Pm4::DB_STENCIL_CONTROL]             = HwCtxSetStencilControl;
 	g_hw_ctx_func[Pm4::DB_STENCILREFMASK]              = HwCtxSetStencilMask;
@@ -82,7 +81,6 @@ constexpr auto MakeContextDispatchTable() {
 	g_hw_ctx_func[Pm4::VGT_PRIMITIVEID_RESET]                 = HwCtxSetPrimitiveIdReset;
 	g_hw_ctx_func[Pm4::PA_CL_OBJPRIM_ID_CNTL]                 = HwCtxSetObjprimIdControl;
 	g_hw_ctx_func[Pm4::VGT_SHADER_STAGES_EN]                  = HwCtxSetShaderStages;
-	g_hw_ctx_func[Pm4::PA_CL_GB_VERT_CLIP_ADJ]                = HwCtxSetGuardBands;
 
 	for (uint32_t slot = 0; slot < 8; slot++) {
 		g_hw_ctx_func[Pm4::CB_COLOR0_BASE + slot * 15] = HwCtxSetRenderTarget;
@@ -217,6 +215,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_COND_EXEC]                 = CpOpCondExec;
 	g_cp_op_func[Pm4::IT_SET_PREDICATION]           = CpOpSetPredication;
 	g_cp_op_func[Pm4::IT_WRITE_DATA]                = CpOpWriteData;
+	g_cp_op_func[Pm4::IT_WAIT_REG_MEM]              = CpOpWaitRegMem32;
 	g_cp_op_func[Pm4::IT_INDIRECT_BUFFER]           = CpOpIndirectBuffer;
 	g_cp_op_func[Pm4::IT_COPY_DATA]                 = CpOpCopyData;
 	g_cp_op_func[Pm4::IT_EVENT_WRITE]               = CpOpEventWrite;
@@ -239,6 +238,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_WAIT_ON_CE_COUNTER]        = CpOpWaitOnCeCounter;
 	g_cp_op_func[Pm4::IT_WAIT_ON_DE_COUNTER_DIFF]   = CpOpWaitOnDeCounterDiff;
 	g_cp_op_func[Pm4::IT_GET_LOD_STATS]             = CpOpGetLodStats;
+	g_cp_op_func[Pm4::IT_WAIT_REG_MEM_64]           = CpOpWaitRegMem64;
 
 	return g_cp_op_func;
 }
@@ -247,12 +247,10 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	std::array<cp_op_parser_func_t, Pm4::R_NUM> g_cp_op_custom_func {};
 
 	g_cp_op_custom_func[Pm4::R_DISPATCH_RESET] = CpOpDispatchReset;
-	g_cp_op_custom_func[Pm4::R_WAIT_MEM_32]    = CpOpWaitRegMem32;
 	g_cp_op_custom_func[Pm4::R_WAIT_FLIP_DONE] = CpOpWaitFlipDone;
 	g_cp_op_custom_func[Pm4::R_PUSH_MARKER]    = CpOpPushMarker;
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_ACQUIRE_MEM]    = CpOpAcquireMem;
-	g_cp_op_custom_func[Pm4::R_WAIT_MEM_64]    = CpOpWaitRegMem64;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
 	g_cp_op_custom_func[Pm4::R_RELEASE_MEM]    = CpOpReleaseMem;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -56,7 +56,6 @@ uint32_t HwCtxSetWindowScissor(CommandProcessor&, uint32_t, uint32_t, const uint
 uint32_t HwCtxSetClipRect(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetRenderTargetMask(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetGenericScissor(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
-uint32_t HwCtxSetBlendColor(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetCbDccControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetStencilControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetStencilMask(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
@@ -90,7 +89,6 @@ uint32_t HwCtxSetDrawPayloadControl(CommandProcessor&, uint32_t, uint32_t, const
 uint32_t HwCtxSetPrimitiveIdReset(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetObjprimIdControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetShaderStages(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
-uint32_t HwCtxSetGuardBands(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetRenderTarget(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetColorInfo(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetBlendControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -71,19 +71,6 @@ constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
 	return raw_offset & ~RegisterSelectorMask;
 }
 
-// Indirect Cx descriptors retain their selector. Selector 1 offsets 0..31 address the
-// SPI_PS_INPUT_CNTL register bank; ordinary context-register offsets remain unchanged.
-constexpr uint32_t DecodeIndirectCxRegisterOffset(uint32_t raw_offset) {
-	const auto offset = NormalizeRegisterOffset(raw_offset);
-	return (raw_offset & RegisterSelectorMask) == Pm4::CX_PS_SHADER_USAGE_BASE && offset < 32u
-	           ? Pm4::SPI_PS_INPUT_CNTL_0 + offset
-	           : offset;
-}
-
-static_assert(DecodeIndirectCxRegisterOffset(Pm4::CX_PS_SHADER_USAGE_BASE + 2u) ==
-              Pm4::SPI_PS_INPUT_CNTL_0 + 2u);
-static_assert(DecodeIndirectCxRegisterOffset(Pm4::DB_Z_INFO) == Pm4::DB_Z_INFO);
-
 bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) {
 	return eop_event_type != 0x28u ||
 	       (gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate |
@@ -530,22 +517,6 @@ KYTY_HW_CTX_PARSER(HwCtxSetAaMask) {
 	return num_values;
 }
 
-KYTY_HW_CTX_PARSER(HwCtxSetBlendColor) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0046900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::CB_BLEND_RED);
-
-	HW::BlendColor r;
-
-	r.red   = *reinterpret_cast<const float*>(&buffer[0]);
-	r.green = *reinterpret_cast<const float*>(&buffer[1]);
-	r.blue  = *reinterpret_cast<const float*>(&buffer[2]);
-	r.alpha = *reinterpret_cast<const float*>(&buffer[3]);
-
-	cp.GetCtx().SetBlendColor(r);
-
-	return 4;
-}
-
 KYTY_HW_CTX_PARSER(HwCtxSetBlendControl) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900);
 
@@ -859,20 +830,6 @@ KYTY_HW_CTX_PARSER(HwCtxSetClipRect) {
 	return value_count;
 }
 
-KYTY_HW_CTX_PARSER(HwCtxSetGuardBands) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0046900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_CL_GB_VERT_CLIP_ADJ);
-
-	auto vert_clip    = *reinterpret_cast<const float*>(&buffer[0]); // PA_CL_GB_VERT_CLIP_ADJ
-	auto vert_discard = *reinterpret_cast<const float*>(&buffer[1]); // PA_CL_GB_VERT_DISC_ADJ
-	auto horz_clip    = *reinterpret_cast<const float*>(&buffer[2]); // PA_CL_GB_HORZ_CLIP_ADJ
-	auto horz_discard = *reinterpret_cast<const float*>(&buffer[3]); // PA_CL_GB_HORZ_DISC_ADJ
-
-	cp.GetCtx().SetGuardBands(horz_clip, vert_clip, horz_discard, vert_discard);
-
-	return 4;
-}
-
 KYTY_HW_CTX_PARSER(HwCtxSetHardwareScreenOffset) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900);
 	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SU_HARDWARE_SCREEN_OFFSET);
@@ -1248,23 +1205,16 @@ KYTY_HW_CTX_PARSER(HwCtxSetStencilInfo) {
 }
 
 KYTY_HW_CTX_PARSER(HwCtxSetStencilMask) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0026900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCILREFMASK);
+	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
+	EXIT_NOT_IMPLEMENTED(num_values == 0);
+	EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::DB_STENCILREFMASK);
+	EXIT_NOT_IMPLEMENTED(cmd_offset + num_values - 1u > Pm4::DB_STENCILREFMASK_BF);
 
-	HW::StencilMask r;
+	for (uint32_t i = 0; i < num_values; i++) {
+		g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, buffer[i]);
+	}
 
-	r.stencil_testval      = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILTESTVAL);
-	r.stencil_mask         = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILMASK);
-	r.stencil_writemask    = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILWRITEMASK);
-	r.stencil_opval        = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILOPVAL);
-	r.stencil_testval_bf   = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILTESTVAL_BF);
-	r.stencil_mask_bf      = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILMASK_BF);
-	r.stencil_writemask_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILWRITEMASK_BF);
-	r.stencil_opval_bf     = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILOPVAL_BF);
-
-	cp.GetCtx().SetStencilMask(r);
-
-	return 2;
+	return num_values;
 }
 
 KYTY_HW_CTX_PARSER(HwCtxSetViewportScaleOffset) {
@@ -1367,10 +1317,6 @@ static void HwShSetCsRegister(CommandProcessor& cp, uint32_t cmd_offset, uint32_
 			                         Pm4::COMPUTE_PGM_RSRC1_BULKY_MASK) != 0u;
 			cs_regs.threadgroup_configuration = ((value >> Pm4::COMPUTE_PGM_RSRC1_WGP_MODE_SHIFT) &
 			                                     Pm4::COMPUTE_PGM_RSRC1_WGP_MODE_MASK) != 0u;
-			cs_regs.wave_size                 = (((value >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) &
-			                                      Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u)
-			                                        ? 32u
-			                                        : 64u;
 			break;
 		case Pm4::COMPUTE_PGM_RSRC2:
 			cs_regs.scratch_en     = ((value >> Pm4::COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT) &
@@ -2201,12 +2147,19 @@ KYTY_CP_OP_PARSER(CpOpEventWrite) {
 
 	EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_EVENT_WRITE);
 
-	uint32_t event_index = (buffer[0] >> 8u) & 0x7u;
-	uint32_t event_type  = (buffer[0]) & 0x3fu;
+	const auto packet_size_dw = KYTY_PM4_LEN(cmd_id);
+	uint32_t   event_index    = (buffer[0] >> 8u) & 0x7u;
+	uint32_t   event_type     = (buffer[0]) & 0x3fu;
+	uint64_t   event_address  = 0;
 
-	cp.TriggerEvent(event_type, event_index);
+	if (event_type == 0x39u) {
+		EXIT_NOT_IMPLEMENTED(packet_size_dw != 4u);
+		event_address = buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u);
+	}
 
-	return KYTY_PM4_LEN(cmd_id) - 1u;
+	cp.TriggerEvent(event_type, event_index, event_address);
+
+	return packet_size_dw - 1u;
 }
 
 KYTY_CP_OP_PARSER(CpOpEventWriteEop) {
@@ -2411,10 +2364,10 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
 		EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
 	}
 	for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
-		// Keep the encoded offset for packet control values, and use the decoded offset only
-		// for register dispatch.
+		// Keep the encoded offset for packet control values, and use the normalized offset
+		// only for register dispatch.
 		auto raw_cmd_offset = indirect_buffer[0];
-		auto cmd_offset     = DecodeIndirectCxRegisterOffset(raw_cmd_offset);
+		auto cmd_offset     = NormalizeRegisterOffset(raw_cmd_offset);
 		auto value          = indirect_buffer[1];
 
 		if (HwCtxTrySetFakeRegister(cmd_offset, value)) {
@@ -2864,6 +2817,20 @@ KYTY_CP_OP_PARSER(CpOpSetShaderReg) {
 	auto pfunc = g_hw_sh_func[cmd_offset];
 
 	if (pfunc == nullptr) {
+		auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
+		bool handled    = num_values != 0;
+		for (uint32_t i = 0; i < num_values; i++) {
+			if (cmd_offset + i >= Pm4::SH_NUM || g_hw_sh_indirect_func[cmd_offset + i] == nullptr) {
+				handled = false;
+				break;
+			}
+		}
+		if (handled) {
+			for (uint32_t i = 0; i < num_values; i++) {
+				g_hw_sh_indirect_func[cmd_offset + i](cp, cmd_offset + i, buffer[1 + i]);
+			}
+			return num_values + 1u;
+		}
 		EXIT("unknown shader register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32
 		     "\n\tcmd_offset = %08" PRIx32 "\n",
 		     num_dw - dw, cmd_id, cmd_offset);
@@ -2877,6 +2844,10 @@ KYTY_CP_OP_PARSER(CpOpSetShaderReg) {
 KYTY_CP_OP_PARSER(CpOpSetUconfigReg) {
 	KYTY_PROFILER_FUNCTION();
 
+	if (Gen5::AgcIsInternalDataPacket(cmd_id, buffer)) {
+		return KYTY_PM4_LEN(cmd_id) - 1u;
+	}
+
 	auto raw_cmd_offset = buffer[0];
 	auto cmd_offset     = NormalizeRegisterOffset(raw_cmd_offset);
 
@@ -2967,19 +2938,21 @@ template <typename T>
 static uint32_t CpOpWaitRegMemSized(CommandProcessor& cp, uint32_t cmd_id, const uint32_t* buffer) {
 	static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
 
-	constexpr auto value_dw   = static_cast<uint32_t>(sizeof(T) / sizeof(uint32_t));
+	constexpr auto value_dw = static_cast<uint32_t>(sizeof(T) / sizeof(uint32_t));
 	constexpr auto payload_dw = 4u + value_dw * 2u;
 	constexpr auto packet_dw  = payload_dw + 1u;
-	constexpr auto register_id =
-	    (sizeof(T) == sizeof(uint32_t) ? Pm4::R_WAIT_MEM_32 : Pm4::R_WAIT_MEM_64);
+	constexpr auto opcode =
+	    (sizeof(T) == sizeof(uint32_t) ? Pm4::IT_WAIT_REG_MEM : Pm4::IT_WAIT_REG_MEM_64);
 
-	EXIT_NOT_IMPLEMENTED(KYTY_PM4_R(cmd_id) != register_id);
+	EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != opcode || KYTY_PM4_R(cmd_id) != 0u);
 	EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != packet_dw);
 
-	auto* addr = reinterpret_cast<const T*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
-	auto  mask = CpOpWaitRegMemReadValue<T>(buffer + 2u);
-	auto  ref  = CpOpWaitRegMemReadValue<T>(buffer + 2u + value_dw);
-	auto  ctrl = buffer[2u + value_dw * 2u];
+	constexpr auto address_align_mask = (sizeof(T) == sizeof(uint32_t) ? 0x3u : 0x7u);
+	auto  ctrl = buffer[0];
+	auto* addr = reinterpret_cast<const T*>((buffer[1] & ~address_align_mask) |
+	                                        (static_cast<uint64_t>(buffer[2] & 0x3ffffu) << 32u));
+	auto  ref  = CpOpWaitRegMemReadValue<T>(buffer + 3u);
+	auto  mask = CpOpWaitRegMemReadValue<T>(buffer + 3u + value_dw);
 	auto  poll = buffer[3u + value_dw * 2u];
 
 	EXIT_NOT_IMPLEMENTED((ctrl & 0x10u) == 0);

src/graphics/guest_gpu/gpu_format.cpp

@@ -9,79 +9,80 @@ namespace Libs::Graphics::Prospero {
 namespace {
 
 struct FormatInfo {
-	BufferFormat format;
-	uint32_t     bytes_per_element;
-	uint32_t     block_compressed_bytes_per_block;
-	uint32_t     render_target_bytes_per_element;
-	bool         supported_sampled_texture_format;
-	bool         unsigned_integer_texture_format;
+	BufferFormat       format;
+	uint32_t           bytes_per_element;
+	uint32_t           block_compressed_bytes_per_block;
+	uint32_t           render_target_bytes_per_element;
+	bool               sampled_texture;
+	bool               uint_texture;
 };
 
 constexpr FormatInfo kFormatInfo[] = {
-    {BufferFormat::k8UNorm, 1, 0, 1, true, false},
-    {BufferFormat::k8SNorm, 0, 0, 1, false, false},
-    {BufferFormat::k8UInt, 1, 0, 1, true, true},
-    {BufferFormat::k16UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k16SNorm, 2, 0, 2, true, false},
-    {BufferFormat::k16UInt, 2, 0, 2, true, true},
-    {BufferFormat::k16SInt, 2, 0, 2, false, false},
-    {BufferFormat::k16Float, 2, 0, 2, true, false},
-    {BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
-    {BufferFormat::k8_8UInt, 2, 0, 2, true, true},
-    {BufferFormat::k8_8SInt, 2, 0, 2, false, false},
-    {BufferFormat::k32UInt, 4, 0, 4, true, true},
-    {BufferFormat::k32SInt, 4, 0, 4, false, false},
-    {BufferFormat::k32Float, 4, 0, 4, true, false},
-    {BufferFormat::k16_16UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k16_16SNorm, 4, 0, 4, true, false},
-    {BufferFormat::k16_16UInt, 4, 0, 4, true, true},
-    {BufferFormat::k16_16SInt, 4, 0, 4, false, false},
-    {BufferFormat::k16_16Float, 4, 0, 4, true, false},
-    {BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
-    {BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
-    {BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k8_8_8_8SNorm, 4, 0, 4, true, false},
-    {BufferFormat::k8_8_8_8UInt, 4, 0, 4, true, true},
-    {BufferFormat::k8_8_8_8SInt, 4, 0, 4, false, false},
-    {BufferFormat::k32_32UInt, 8, 0, 8, true, true},
-    {BufferFormat::k32_32SInt, 8, 0, 8, false, false},
-    {BufferFormat::k32_32Float, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16UNorm, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16SNorm, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16UInt, 8, 0, 8, true, true},
-    {BufferFormat::k16_16_16_16SInt, 8, 0, 8, false, false},
-    {BufferFormat::k16_16_16_16Float, 8, 0, 8, true, false},
-    {BufferFormat::k32_32_32UInt, 12, 0, 12, true, true},
-    {BufferFormat::k32_32_32SInt, 12, 0, 12, false, false},
-    {BufferFormat::k32_32_32Float, 12, 0, 12, true, false},
-    {BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
-    {BufferFormat::k32_32_32_32SInt, 16, 0, 16, false, false},
-    {BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
-    {BufferFormat::k8Srgb, 1, 0, 0, true, false},
-    {BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
-    {BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
-    {BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
-    {BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
-    {BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
-    {BufferFormat::kFmask8_S4_F4, 1, 0, 1, true, false},
-    {BufferFormat::kBc1UNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc1Srgb, 0, 8, 0, true, false},
-    {BufferFormat::kBc2UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc2Srgb, 0, 16, 0, true, false},
-    {BufferFormat::kBc3UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc3Srgb, 0, 16, 0, true, false},
-    {BufferFormat::kBc4UNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc4SNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc5UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc5SNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc6UFloat, 0, 16, 0, true, false},
-    {BufferFormat::kBc6SFloat, 0, 16, 0, true, false},
-    {BufferFormat::kBc7UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc7Srgb, 0, 16, 0, true, false},
+	{BufferFormat::k8UNorm, 1, 0, 1, true, false},
+	{BufferFormat::k8SNorm, 0, 0, 1, false, false},
+	{BufferFormat::k8UInt, 1, 0, 1, true, true},
+	{BufferFormat::k16UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k16SNorm, 2, 0, 2, true, false},
+	{BufferFormat::k16UInt, 2, 0, 2, true, true},
+	{BufferFormat::k16SInt, 2, 0, 2, false, false},
+	{BufferFormat::k16Float, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UInt, 2, 0, 2, true, true},
+	{BufferFormat::k8_8SInt, 2, 0, 2, false, false},
+	{BufferFormat::k32UInt, 4, 0, 4, true, true},
+	{BufferFormat::k32SInt, 4, 0, 4, false, false},
+	{BufferFormat::k32Float, 4, 0, 4, true, false},
+	{BufferFormat::k16_16UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k16_16SNorm, 4, 0, 4, true, false},
+	{BufferFormat::k16_16UInt, 4, 0, 4, true, true},
+	{BufferFormat::k16_16SInt, 4, 0, 4, false, false},
+	{BufferFormat::k16_16Float, 4, 0, 4, true, false},
+	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
+	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
+	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
+	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k8_8_8_8SNorm, 4, 0, 4, true, false},
+	{BufferFormat::k8_8_8_8UInt, 4, 0, 4, true, true},
+	{BufferFormat::k8_8_8_8SInt, 4, 0, 4, false, false},
+	{BufferFormat::k32_32UInt, 8, 0, 8, true, true},
+	{BufferFormat::k32_32SInt, 8, 0, 8, false, false},
+	{BufferFormat::k32_32Float, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16UNorm, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16SNorm, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16UInt, 8, 0, 8, true, true},
+	{BufferFormat::k16_16_16_16SInt, 8, 0, 8, false, false},
+	{BufferFormat::k16_16_16_16Float, 8, 0, 8, true, false},
+	{BufferFormat::k32_32_32UInt, 12, 0, 12, true, true},
+	{BufferFormat::k32_32_32SInt, 12, 0, 12, false, false},
+	{BufferFormat::k32_32_32Float, 12, 0, 12, true, false},
+	{BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
+	{BufferFormat::k32_32_32_32SInt, 16, 0, 16, false, false},
+	{BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
+	{BufferFormat::k8Srgb, 1, 0, 0, true, false},
+	{BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
+	{BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
+	{BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
+	{BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
+	{BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
+	{BufferFormat::kFmask8_S4_F4, 1, 0, 1, true, false},
+	{BufferFormat::kBc1UNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc1Srgb, 0, 8, 0, true, false},
+	{BufferFormat::kBc2UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc2Srgb, 0, 16, 0, true, false},
+	{BufferFormat::kBc3UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc3Srgb, 0, 16, 0, true, false},
+	{BufferFormat::kBc4UNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc4SNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc5UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc5SNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc6UFloat, 0, 16, 0, true, false},
+	{BufferFormat::kBc6SFloat, 0, 16, 0, true, false},
+	{BufferFormat::kBc7UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc7Srgb, 0, 16, 0, true, false},
 };
 
 constexpr uint8_t RenderTargetTypeBit(ChannelType type) {
@@ -201,18 +202,22 @@ uint32_t RenderTargetBytesPerElement(BufferFormat format) {
 	return info != nullptr ? info->render_target_bytes_per_element : 0;
 }
 
-bool IsSupportedTextureFormat(BufferFormat format) {
-	const auto* info = FindFormatInfo(format);
-	return info != nullptr && info->supported_sampled_texture_format;
-}
-
-bool IsUintTextureFormat(BufferFormat format) {
-	const auto* info = FindFormatInfo(format);
-	return info != nullptr && info->unsigned_integer_texture_format;
-}
-
 bool IsFmaskTextureFormat(BufferFormat format) {
 	return format == BufferFormat::kFmask8_S4_F4;
 }
 
+bool IsSampledTextureFormat(BufferFormat format) {
+	const auto* info = FindFormatInfo(format);
+	return info != nullptr && info->sampled_texture;
+}
+
+bool IsUintTextureFormat(BufferFormat format) {
+	const auto* info = FindFormatInfo(format);
+	return info != nullptr && info->uint_texture;
+}
+
+BufferFormat RemapTextureFormat(BufferFormat format) {
+	return format == BufferFormat::k11_11_10UInt ? BufferFormat::k32UInt : format;
+}
+
 } // namespace Libs::Graphics::Prospero

src/graphics/guest_gpu/gpu_format.h

@@ -41,9 +41,10 @@ RenderTargetFormatEncoding ResolveRenderTargetFormat(ChannelLayout layout, Chann
 uint32_t                   NumBytesPerElement(BufferFormat format);
 uint32_t                   BlockCompressedBytesPerBlock(BufferFormat format);
 uint32_t                   RenderTargetBytesPerElement(BufferFormat format);
-bool                       IsSupportedTextureFormat(BufferFormat format);
-bool                       IsUintTextureFormat(BufferFormat format);
 bool                       IsFmaskTextureFormat(BufferFormat format);
+bool                       IsSampledTextureFormat(BufferFormat format);
+bool                       IsUintTextureFormat(BufferFormat format);
+BufferFormat               RemapTextureFormat(BufferFormat format);
 
 } // namespace Libs::Graphics::Prospero
 

src/graphics/guest_gpu/graphicsRun.cpp

@@ -27,6 +27,7 @@
 #include <cstdio>
 #include <deque>
 #include <memory>
+#include <mutex>
 #include <semaphore>
 #include <thread>
 #include <vector>
@@ -199,12 +200,6 @@ void GuestGpu::Done() {
 	if (!IsGpuThread()) {
 		WaitForIdle();
 	}
-	if (RenderDocCaptureInProgress()) {
-		SendCommandSync([this] {
-			Common::LockGuard render_lock(m_renderer.GetMutex());
-			RenderDocEndCapture();
-		});
-	}
 	m_graphics_done = true;
 	m_done_num++;
 }
@@ -359,10 +354,8 @@ template void CommandProcessor::WaitRegMem<uint64_t>(uint32_t, const uint64_t*,
 
 void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num,
                                  uint32_t write_control) {
-	const uint32_t dst_sel      = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
-	const uint32_t cache_policy = (write_control >> 25u) & 0x3u;
-	const uint32_t increment    = (write_control >> 16u) & 0x1u;
-	const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
+	const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
+	const bool     write_one_address = ((write_control >> 16u) & 0x1u) != 0;
 
 	switch (dst_sel) {
 		case 0:
@@ -372,16 +365,17 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
 		case 6: break;
 		default: EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
 	}
-	EXIT_NOT_IMPLEMENTED(increment != 0);
-
-	if (cache_policy > 3 || write_confirm > 1) {
-		LOGF("\t warning: unexpected write_data control 0x%08" PRIx32 "\n", write_control);
-	}
 	if (dw_num == 0) {
 		return;
 	}
 
-	memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
+	if (write_one_address) {
+		for (uint32_t i = 0; i < dw_num; i++) {
+			dst[0] = src[i];
+		}
+	} else {
+		memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
+	}
 }
 
 void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) {
@@ -1107,6 +1101,8 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y,
                                       uint32_t thread_group_z, uint32_t mode) {
+	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
+
 	uint32_t frame_num = 0;
 	// uint32_t local_x   = 1;
 	// uint32_t local_y   = 1;
@@ -1280,7 +1276,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 				if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
 					auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
 					SynchronizeGpu();
-					Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
+					Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
 					              value >> 16u);
 					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
 					                            value >> 16u);
@@ -1472,12 +1468,14 @@ void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id)
 	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id);
 }
 
-void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
+void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
+                                    uint64_t event_address) {
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::TriggerEvent()\n"
 		     "\t event_type  = 0x%08" PRIx32 "\n"
-		     "\t event_index = 0x%08" PRIx32 "\n",
-		     event_type, event_index);
+		     "\t event_index = 0x%08" PRIx32 "\n"
+		     "\t address     = 0x%016" PRIx64 "\n",
+		     event_type, event_index, event_address);
 	}
 
 	const auto valid_cache_event_index = event_index == 0x00000000 || event_index == 0x00000007;
@@ -1514,12 +1512,36 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
 		case 0x0000001a:
 		case 0x0000001b:
 		case 0x00000038:
-		case 0x00000039:
 		case 0x0000003a:
 			LOGF("\t temporary: ignoring unsupported event_write type 0x%08" PRIx32
 			     ", index 0x%08" PRIx32 "\n",
 			     event_type, event_index);
 			break;
+		case 0x00000039: {
+			if (event_index != 0x00000001 || event_address == 0 || (event_address & 0x7u) != 0) {
+				EXIT("invalid occlusion-counter dump: index=0x%08" PRIx32 ", address=0x%016" PRIx64
+				     "\n",
+				     event_index, event_address);
+			}
+			static std::once_flag warning_once;
+			std::call_once(warning_once, [] {
+				std::printf("Warning: game uses occlusion queries, which are currently treated as "
+				            "always visible; GPU usage may be higher and FPS may be lower.\n");
+			});
+
+			// Until host occlusion queries are implemented, publish an always-visible result. The
+			// PS5 layout contains one interleaved begin/end pair per DB, and bit 63 marks a result
+			// ready.
+			constexpr uint64_t ready_bit    = 1ull << 63u;
+			constexpr uint64_t counter_mask = ready_bit - 1u;
+			auto*              results      = reinterpret_cast<volatile uint64_t*>(event_address);
+			const auto         value        = ready_bit | m_synthetic_occlusion_counter;
+			for (uint32_t db = 0; db < 16u; db++) {
+				results[db * 2u] = value;
+			}
+			m_synthetic_occlusion_counter = (m_synthetic_occlusion_counter + 1u) & counter_mask;
+			break;
+		}
 		default:
 			EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, event_index);
 	}

src/graphics/guest_gpu/hardwareContext.h

@@ -1089,6 +1089,7 @@ public:
 	void SetPsShaderChksum(uint32_t value) { m_ps.ps_regs.chksum = value; }
 
 	void SetCsShader(const CsStageRegisters& cs_regs) { m_cs.cs_regs = cs_regs; }
+	void SetCsWaveSize(uint8_t wave_size) { m_cs.cs_regs.wave_size = wave_size; }
 
 	void SetPsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
 		m_ps.ps_user_sgpr.value[id] = value;

src/graphics/guest_gpu/pm4.cpp

@@ -21,12 +21,10 @@ constexpr auto MakeRegisterNames() {
 	names[R_DRAW_RESET]     = "R_DRAW_RESET";
 	names[R_WAIT_FLIP_DONE] = "R_WAIT_FLIP_DONE";
 	names[R_DISPATCH_RESET] = "R_DISPATCH_RESET";
-	names[R_WAIT_MEM_32]    = "R_WAIT_MEM_32";
 	names[R_PUSH_MARKER]    = "R_PUSH_MARKER";
 	names[R_POP_MARKER]     = "R_POP_MARKER";
 	names[R_ACQUIRE_MEM]    = "R_ACQUIRE_MEM";
 	names[R_WRITE_DATA]     = "R_WRITE_DATA";
-	names[R_WAIT_MEM_64]    = "R_WAIT_MEM_64";
 	names[R_FLIP]           = "R_FLIP";
 	names[R_RELEASE_MEM]    = "R_RELEASE_MEM";
 	names[R_DMA_DATA]       = "R_DMA_DATA";
@@ -62,6 +60,7 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_WRITE_DATA]                = "IT_WRITE_DATA";
 	names[IT_MEM_SEMAPHORE]             = "IT_MEM_SEMAPHORE";
 	names[IT_DRAW_INDEX_INDIRECT_MULTI] = "IT_DRAW_INDEX_INDIRECT_MULTI";
+	names[IT_WAIT_REG_MEM]               = "IT_WAIT_REG_MEM";
 	names[IT_INDIRECT_BUFFER]           = "IT_INDIRECT_BUFFER";
 	names[IT_COPY_DATA]                 = "IT_COPY_DATA";
 	names[IT_CP_DMA]                    = "IT_CP_DMA";
@@ -91,6 +90,7 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_DISPATCH_DRAW_PREAMBLE]    = "IT_DISPATCH_DRAW_PREAMBLE";
 	names[IT_DISPATCH_DRAW]             = "IT_DISPATCH_DRAW";
 	names[IT_GET_LOD_STATS]             = "IT_GET_LOD_STATS";
+	names[IT_WAIT_REG_MEM_64]           = "IT_WAIT_REG_MEM_64";
 	names[IT_SET_CONTEXT_REG_INDIRECT]  = "IT_SET_CONTEXT_REG_INDIRECT";
 	return names;
 }

src/graphics/guest_gpu/pm4.h

@@ -42,6 +42,7 @@ constexpr uint32_t IT_WRITE_DATA                = 0x37;
 constexpr uint32_t IT_MEM_SEMAPHORE             = 0x39;
 constexpr uint32_t IT_DRAW_INDEX_INDIRECT_MULTI = 0x38;
 constexpr uint32_t IT_DISPATCH_DRAW_PREAMBLE    = 0x3A;
+constexpr uint32_t IT_WAIT_REG_MEM              = 0x3C;
 constexpr uint32_t IT_INDIRECT_BUFFER           = 0x3F;
 constexpr uint32_t IT_COPY_DATA                 = 0x40;
 constexpr uint32_t IT_CP_DMA                    = 0x41;
@@ -70,6 +71,7 @@ constexpr uint32_t IT_WAIT_ON_CE_COUNTER        = 0x86;
 constexpr uint32_t IT_WAIT_ON_DE_COUNTER_DIFF   = 0x88;
 constexpr uint32_t IT_DISPATCH_DRAW             = 0x8D;
 constexpr uint32_t IT_GET_LOD_STATS             = 0x8E;
+constexpr uint32_t IT_WAIT_REG_MEM_64           = 0x93;
 constexpr uint32_t IT_SET_CONTEXT_REG_INDIRECT  = 0x9F;
 
 /* Custom codes. Implemented via IT_NOP */
@@ -78,12 +80,10 @@ constexpr uint32_t R_ZERO           = 0x00;
 constexpr uint32_t R_DRAW_RESET     = 0x05;
 constexpr uint32_t R_WAIT_FLIP_DONE = 0x06;
 constexpr uint32_t R_DISPATCH_RESET = 0x09;
-constexpr uint32_t R_WAIT_MEM_32    = 0x0A;
 constexpr uint32_t R_PUSH_MARKER    = 0x0B;
 constexpr uint32_t R_POP_MARKER     = 0x0C;
 constexpr uint32_t R_ACQUIRE_MEM    = 0x14;
 constexpr uint32_t R_WRITE_DATA     = 0x15;
-constexpr uint32_t R_WAIT_MEM_64    = 0x16;
 constexpr uint32_t R_FLIP           = 0x17;
 constexpr uint32_t R_RELEASE_MEM    = 0x18;
 constexpr uint32_t R_DMA_DATA       = 0x19;
@@ -317,8 +317,8 @@ constexpr uint32_t FSR_RECURSIONS1              = 0xF2;
 constexpr uint32_t VGT_MULTI_PRIM_IB_RESET_INDX = 0x103;
 constexpr uint32_t CB_RMI_GL2_CACHE_CONTROL     = 0x104;
 constexpr uint32_t CB_BLEND_RED                 = 0x105;
-constexpr uint32_t CB_BLEND_GREEN               = 0x106;
-constexpr uint32_t CB_BLEND_BLUE                = 0x107;
+constexpr uint32_t CB_BLEND_BLUE                = 0x106;
+constexpr uint32_t CB_BLEND_GREEN               = 0x107;
 constexpr uint32_t CB_BLEND_ALPHA               = 0x108;
 constexpr uint32_t CB_DCC_CONTROL               = 0x109;
 
@@ -387,9 +387,6 @@ constexpr uint32_t SPI_SHADER_POS_FORMAT  = 0x1C3;
 constexpr uint32_t SPI_SHADER_Z_FORMAT    = 0x1C4;
 constexpr uint32_t SPI_SHADER_COL_FORMAT  = 0x1C5;
 
-// Indirect Cx descriptor selector for the 32-entry PS input-control register bank.
-constexpr uint32_t CX_PS_SHADER_USAGE_BASE = 0x10000000u;
-
 constexpr uint32_t CB_BLEND0_CONTROL                            = 0x1E0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT       = 0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK        = 0x1F;
@@ -984,8 +981,16 @@ constexpr uint32_t COMPUTE_PGM_RSRC1_FP16_OVFL_SHIFT  = 26;
 constexpr uint32_t COMPUTE_PGM_RSRC1_FP16_OVFL_MASK   = 0x1;
 constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_SHIFT   = 29;
 constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_MASK    = 0x1;
-constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_SHIFT     = 30;
-constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_MASK      = 0x1;
+
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT = 15;
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK  = 0x1;
+
+[[nodiscard]] constexpr uint8_t ComputeWaveSize(uint32_t dispatch_initiator) {
+	return ((dispatch_initiator >> COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT) &
+	        COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK) != 0u
+	           ? 32u
+	           : 64u;
+}
 
 constexpr uint32_t COMPUTE_PGM_RSRC2                      = 0x213;
 constexpr uint32_t COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT     = 0;

src/graphics/host_gpu/graphicContext.h

@@ -24,7 +24,6 @@ struct GraphicContext: public VulkanInstance {
 	[[nodiscard]] uint64_t GetDeviceMemoryUsage() const;
 	[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
 	void                   CreateBuffer(uint64_t size, VulkanBuffer& buffer);
-	void                   DeleteBuffer(VulkanBuffer& buffer);
 	[[nodiscard]] bool     CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
 	void                   DeleteImage(VulkanImage& image);
 	void                   MapMemory(VulkanMemory& memory, void*& data);

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -9,6 +9,7 @@
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
+#include "graphics/host_gpu/vulkanCommon.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -27,60 +28,25 @@ constexpr uint64_t GdsBufferSize = 64 * 1024;
 
 } // namespace
 
-uint64_t BufferCache::AlignDown(uint64_t value) noexcept {
-	return value & ~(CACHING_PAGE_SIZE - 1);
-}
-
-uint64_t BufferCache::AlignUp(uint64_t value) {
-	if (value > UINT64_MAX - (CACHING_PAGE_SIZE - 1)) {
-		EXIT("BufferCache: address alignment overflow\n");
-	}
-	return (value + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
-}
-
-void BufferCache::Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
-                         const void* source, uint64_t size) {
+void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source,
+                                  uint64_t size) {
 	auto* bytes = static_cast<const uint8_t*>(source);
 	while (size != 0) {
-		const auto chunk        = std::min(size, m_staging_buffer.Size());
-		const auto stage_offset = m_staging_buffer.Copy(bytes, chunk, 4);
-		destination.CopyFrom(command, m_staging_buffer, stage_offset, destination_offset, chunk,
-		                     vk::AccessFlagBits::eHostWrite);
+		const auto chunk  = std::min(size, m_staging_buffer.Size());
+		const auto offset = m_staging_buffer.Copy(bytes, chunk, 4);
+		buffer.CopyFrom(m_scheduler.Current(), m_staging_buffer, offset, buffer.Offset(address),
+		                chunk, vk::AccessFlagBits::eHostWrite);
 		bytes += chunk;
-		destination_offset += chunk;
+		address += chunk;
 		size -= chunk;
 	}
 }
 
-bool BufferCache::ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept {
-	if (merged.address == 0 || merged.size == 0 || candidate.address == 0 || candidate.size == 0 ||
-	    merged.size > UINT64_MAX - merged.address ||
-	    candidate.size > UINT64_MAX - candidate.address) {
-		EXIT("BufferCache: invalid overlap-merge range\n");
-	}
-	const auto merged_end    = merged.address + merged.size;
-	const auto candidate_end = candidate.address + candidate.size;
-	if (merged.address >= candidate_end || candidate.address >= merged_end) {
-		return false;
-	}
-	const auto address = std::min(merged.address, candidate.address);
-	const auto end     = std::max(merged_end, candidate_end);
-	merged             = {.address = address, .size = end - address};
-	return true;
-}
-
-struct BufferCache::CachedBuffer {
-	uint64_t                vaddr = 0;
-	uint64_t                size  = 0;
-	std::shared_ptr<Buffer> buffer;
-	uint64_t                tick_accessed_last = 0;
-};
-
 struct BufferCache::DownloadCopy {
-	std::shared_ptr<Buffer> owner;
-	uint64_t                source_offset = 0;
-	uint64_t                address       = 0;
-	uint64_t                size          = 0;
+	Buffer*  buffer        = nullptr;
+	uint64_t source_offset = 0;
+	uint64_t address       = 0;
+	uint64_t size          = 0;
 };
 
 struct BufferCache::DownloadRange {
@@ -89,15 +55,37 @@ struct BufferCache::DownloadRange {
 	uint64_t offset  = 0;
 };
 
-struct BufferCache::RetiredBuffer {
-	uint64_t                address = 0;
-	uint64_t                size    = 0;
-	std::shared_ptr<Buffer> owner;
-};
+void BufferCache::Unregister(BufferId id) {
+	auto& buffer = m_slot_buffers[id];
+	if (buffer.is_deleted) {
+		return;
+	}
+	const auto found = m_buffers.find(buffer.CpuAddress());
+	EXIT_IF(found == m_buffers.end() || found->second != id);
+	m_buffers.erase(found);
+	if (buffer.Size() > m_total_used_memory) {
+		EXIT("BufferCache: allocation accounting underflow\n");
+	}
+	m_total_used_memory -= buffer.Size();
+	buffer.is_deleted = true;
+}
+
+void BufferCache::DeleteBuffer(BufferId id) {
+	auto* buffer = m_slot_buffers.try_get(id);
+	if (buffer == nullptr || buffer->is_deleted) {
+		return;
+	}
+	Unregister(id);
+	if (m_scheduler.Active()) {
+		m_scheduler.DeferOperation([this, id] { m_slot_buffers.erase(id); });
+	} else {
+		m_slot_buffers.erase(id);
+	}
+}
 
 std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
-	if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() ||
-	    copy.size > copy.owner->Size() - copy.source_offset) {
+	if (copy.buffer == nullptr || copy.size == 0 || copy.source_offset > copy.buffer->Size() ||
+	    copy.size > copy.buffer->Size() - copy.source_offset) {
 		EXIT("BufferCache: invalid download copy\n");
 	}
 	const auto begin = copy.source_offset & ~uint64_t {3};
@@ -106,7 +94,7 @@ std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy&
 		EXIT("BufferCache: download copy alignment overflow\n");
 	}
 	const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3};
-	if (end > copy.owner->Size()) {
+	if (end > copy.buffer->Size()) {
 		EXIT("BufferCache: aligned download copy exceeds its owner\n");
 	}
 	return {begin, end - begin};
@@ -143,7 +131,7 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
 	for (const auto& copy: copies) {
 		const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
 		const auto prefix                        = copy.source_offset - source_begin;
-		download.CopyFrom(m_scheduler.Current(), *copy.owner, source_begin, base_offset + cursor,
+		download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
 		                  envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
 		                  vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
 		                  vk::AccessFlagBits::eHostRead);
@@ -162,27 +150,29 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
 	}
 }
 
-void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
-	if (copies.empty()) {
-		return;
-	}
+void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, BufferId id,
+                                       uint64_t address, uint64_t size) {
+	EXIT_IF(copies.empty());
 	auto downloads = RecordDownloads(copies);
-	m_scheduler.DeferOperation(
-	    [this, downloads = std::move(downloads), retire = std::move(retire)]() mutable {
-		    PublishDownloads(downloads);
-		    for (const auto& range: downloads) {
-			    m_gpu_modified_ranges.Subtract(range.address, range.size);
-		    }
-		    // ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
-		    // interval on those pages was downloaded and removed. Clearing the original
-		    // query therefore cannot orphan a dirty sibling on an edge page.
-		    m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size);
-		    if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
-		        !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
-			    EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
-		    }
-		    m_memory_tracker.UntrackMemory(retire.address, retire.size);
-	    });
+	auto retired   = downloads;
+	m_scheduler.DeferPriorityOperation(
+	    [this, downloads = std::move(downloads)]() mutable { PublishDownloads(downloads); });
+	m_scheduler.DeferOperation([this, retired = std::move(retired), id, address, size]() mutable {
+		for (const auto& range: retired) {
+			m_gpu_modified_ranges.Subtract(range.address, range.size);
+		}
+		// ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
+		// interval on those pages was downloaded and removed. Clearing the original
+		// query therefore cannot orphan a dirty sibling on an edge page.
+		m_memory_tracker.UnmarkRegionAsGpuModified(address, size);
+		if (m_memory_tracker.IsRegionGpuModified(address, size) ||
+		    !m_gpu_modified_ranges.Intersections(address, size).empty()) {
+			EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
+		}
+		m_memory_tracker.UntrackMemory(address, size);
+		EXIT_IF(!m_slot_buffers[id].is_deleted);
+		m_slot_buffers.erase(id);
+	});
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -197,6 +187,10 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
       m_texture_cache(texture_cache) {
 	std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
 	m_gds_buffer.Flush(0, m_gds_buffer.Size());
+	const auto null_id =
+	    m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16);
+	EXIT_IF(null_id != NULL_BUFFER_ID);
+	SetVulkanObjectNameF(m_graphics.device, GetBuffer(null_id).Handle(), "Kyty.NullBuffer");
 	if (!m_graphics.CanReportMemoryUsage()) {
 		return;
 	}
@@ -215,38 +209,16 @@ BufferCache::~BufferCache() {
 	if (!m_gpu_modified_ranges.Empty()) {
 		EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
 	}
-	for (const auto& [vaddr, cached]: m_buffers) {
+	for (const auto& [vaddr, id]: m_buffers) {
 		(void)vaddr;
-		if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) {
+		const auto& buffer = m_slot_buffers[id];
+		if (m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size())) {
 			EXIT("BufferCache: destroyed with GPU-modified buffer\n");
 		}
 	}
 	m_buffers.clear();
 }
 
-StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept {
-	switch (usage) {
-		case MemoryUsage::Upload: return m_staging_buffer;
-		case MemoryUsage::Stream: return m_stream_buffer;
-		case MemoryUsage::Download: return m_download_buffer;
-		case MemoryUsage::DeviceLocal: return m_device_buffer;
-	}
-	EXIT("BufferCache: invalid utility-buffer usage\n");
-}
-
-BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint64_t alignment) {
-	EXIT_IF(data == nullptr || size == 0);
-	if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false); mapped != nullptr) {
-		std::memcpy(mapped, data, static_cast<size_t>(size));
-		m_stream_buffer.Commit();
-		return {{}, m_stream_buffer.Handle(), offset};
-	}
-	auto owner =
-	    std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0, AllFlags, size);
-	owner->Write(0, data, size);
-	return {owner, owner->Handle(), 0};
-}
-
 void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
 	    size > TRACKER_ADDRESS_SIZE - vaddr) {
@@ -287,14 +259,14 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
 				    if (owner == m_buffers.begin()) {
 					    EXIT("BufferCache: invalidation readback has no buffer owner\n");
 				    }
-				    auto& cached = *std::prev(owner)->second;
-				    if (!cached.buffer->IsInBounds(copy_address, 1)) {
+				    auto& buffer = m_slot_buffers[std::prev(owner)->second];
+				    if (!buffer.IsInBounds(copy_address, 1)) {
 					    EXIT("BufferCache: invalidation readback is outside its buffer owner\n");
 				    }
-				    const auto copy_size =
-				        std::min(range.size - copied, cached.vaddr + cached.size - copy_address);
-				    copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
-				                      copy_address, copy_size});
+				    const auto copy_size = std::min(
+				        range.size - copied, buffer.CpuAddress() + buffer.Size() - copy_address);
+				    copies.push_back(
+				        {&buffer, buffer.Offset(copy_address), copy_address, copy_size});
 				    copied += copy_size;
 			    }
 		    }
@@ -329,38 +301,39 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	std::vector<DownloadCopy>                  copies;
 	std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
 	std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
-	for (const auto& [begin, cached]: m_buffers) {
-		if (vaddr < begin + cached->size && begin < vaddr + size) {
-			retired_buffers.emplace_back(begin, cached->size);
+	for (const auto& [begin, id]: m_buffers) {
+		const auto& buffer = m_slot_buffers[id];
+		if (vaddr < begin + buffer.Size() && begin < vaddr + size) {
+			retired_buffers.emplace_back(begin, buffer.Size());
 		}
 	}
-	for (const auto& [begin, cached]: m_buffers) {
-		if (vaddr >= begin + cached->size || begin >= vaddr + size ||
-		    !m_memory_tracker.IsRegionGpuModified(begin, cached->size)) {
+	for (const auto& [begin, id]: m_buffers) {
+		const auto& buffer = m_slot_buffers[id];
+		if (vaddr >= begin + buffer.Size() || begin >= vaddr + size ||
+		    !m_memory_tracker.IsRegionGpuModified(begin, buffer.Size())) {
 			continue;
 		}
-		const auto dirty = m_gpu_modified_ranges.Intersections(begin, cached->size);
+		const auto dirty = m_gpu_modified_ranges.Intersections(begin, buffer.Size());
 		if (dirty.empty()) {
 			EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n");
 		}
-		modified_buffers.emplace_back(begin, cached->size);
+		modified_buffers.emplace_back(begin, buffer.Size());
 	}
 	for (const auto& [begin, bytes]: modified_buffers) {
 		auto owner = m_buffers.find(begin);
-		if (owner == m_buffers.end() || owner->second->size != bytes) {
+		if (owner == m_buffers.end() || m_slot_buffers[owner->second].Size() != bytes) {
 			EXIT("BufferCache: unmap owner changed during collection\n");
 		}
-		auto& cached = *owner->second;
+		auto& buffer = m_slot_buffers[owner->second];
 		m_memory_tracker.ForEachDownloadRange<false>(
-		    begin, cached.size,
+		    begin, buffer.Size(),
 		    [&](uint64_t address, uint64_t bytes) noexcept {
 			    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
 			                                           "unmap");
 		    },
 		    [&](uint64_t address, uint64_t bytes) noexcept {
 			    for (const auto& range: m_gpu_modified_ranges.Intersections(address, bytes)) {
-				    copies.push_back(
-				        {cached.buffer, range.address - begin, range.address, range.size});
+				    copies.push_back({&buffer, range.address - begin, range.address, range.size});
 			    }
 		    });
 	}
@@ -384,83 +357,92 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 		EXIT("BufferCache: unmap retained dirty byte ranges\n");
 	}
 	m_memory_tracker.UntrackMemory(vaddr, size);
-	for (auto it = m_buffers.begin(); it != m_buffers.end();) {
-		if (vaddr < it->first + it->second->size && it->first < vaddr + size) {
-			if (it->second->size > m_total_used_memory) {
-				EXIT("BufferCache: allocation accounting underflow\n");
-			}
-			m_total_used_memory -= it->second->size;
-			it = m_buffers.erase(it);
-		} else {
-			++it;
+	for (const auto& [begin, bytes]: retired_buffers) {
+		(void)bytes;
+		const auto found = m_buffers.find(begin);
+		if (found != m_buffers.end()) {
+			DeleteBuffer(found->second);
 		}
 	}
 }
 
-BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
-                                                          uint64_t size) {
-	const auto begin = AlignDown(vaddr);
-	const auto end   = AlignUp(vaddr + size);
+BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
+	if (vaddr == 0) {
+		return NULL_BUFFER_ID;
+	}
+	if (size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) {
+		EXIT("BufferCache: invalid buffer discovery request\n");
+	}
+	auto& command = m_scheduler.Current();
+	EXIT_IF(command.IsInvalid() || command.IsExecute());
+	auto       begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
+	auto       end   = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
 	const auto next  = m_buffers.upper_bound(vaddr);
 	if (next != m_buffers.begin()) {
-		auto owner = std::prev(next);
-		if (owner->second->buffer->IsInBounds(vaddr, size)) {
-			owner->second->tick_accessed_last = m_gc_tick;
-			return *owner->second;
+		const auto owner  = std::prev(next);
+		auto&      buffer = m_slot_buffers[owner->second];
+		if (buffer.IsInBounds(vaddr, size)) {
+			buffer.tick_accessed_last = m_gc_tick;
+			return owner->second;
 		}
 	}
 
-	CacheRange merged {.address = begin, .size = end - begin};
-	using Iterator = decltype(m_buffers.begin());
-	std::vector<Iterator> overlaps;
+	auto merge = [&](const Buffer& buffer) {
+		const auto buffer_begin = buffer.CpuAddress();
+		const auto buffer_end   = buffer_begin + buffer.Size();
+		if (begin >= buffer_end || buffer_begin >= end) {
+			return false;
+		}
+		begin = std::min(begin, buffer_begin);
+		end   = std::max(end, buffer_end);
+		return true;
+	};
+	std::vector<BufferId> overlaps;
 	auto                  first = m_buffers.lower_bound(begin);
 	if (first != m_buffers.begin()) {
-		auto previous = std::prev(first);
-		if (ResolveOverlap(merged, {previous->second->vaddr, previous->second->size})) {
+		const auto previous = std::prev(first);
+		if (merge(m_slot_buffers[previous->second])) {
 			first = previous;
 		}
 	}
 	for (auto candidate = first; candidate != m_buffers.end(); ++candidate) {
-		if (candidate->first >= merged.address + merged.size) {
+		if (candidate->first >= end) {
 			break;
 		}
-		if (ResolveOverlap(merged, {candidate->second->vaddr, candidate->second->size})) {
-			overlaps.push_back(candidate);
+		if (merge(m_slot_buffers[candidate->second])) {
+			overlaps.push_back(candidate->second);
 		}
 	}
 
-	auto cached                = std::make_unique<CachedBuffer>();
-	cached->vaddr              = merged.address;
-	cached->size               = merged.size;
-	cached->tick_accessed_last = m_gc_tick;
-	cached->buffer = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal,
-	                                          merged.address, AllFlags, merged.size);
+	const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
+	                                      AllFlags, end - begin);
+	auto&      buffer         = m_slot_buffers[id];
+	buffer.tick_accessed_last = m_gc_tick;
+	SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
+	                     "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
 	for (const auto overlap: overlaps) {
-		const auto& old = *overlap->second;
-		cached->buffer->CopyFrom(command, *old.buffer, 0, old.vaddr - cached->vaddr, old.size);
-		command.RetainResourceUntilFence(old.buffer);
+		const auto& old = m_slot_buffers[overlap];
+		buffer.CopyFrom(command, old, 0, old.CpuAddress() - begin, old.Size());
 	}
 	for (const auto overlap: overlaps) {
-		if (overlap->second->size > m_total_used_memory) {
-			EXIT("BufferCache: allocation accounting underflow\n");
-		}
-		m_total_used_memory -= overlap->second->size;
-		m_buffers.erase(overlap);
+		DeleteBuffer(overlap);
 	}
-	m_total_used_memory += cached->size;
-	return *m_buffers.emplace(cached->vaddr, std::move(cached)).first->second;
+	m_total_used_memory += buffer.Size();
+	const auto [it, inserted] = m_buffers.emplace(begin, id);
+	(void)it;
+	EXIT_IF(!inserted);
+	return id;
 }
 
-bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
-                                    uint64_t size, bool is_written, bool is_texel_buffer) {
+bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, bool is_written,
+                                    bool is_texel_buffer) {
 	std::vector<std::pair<uint64_t, uint64_t>> uploads;
 	m_memory_tracker.ForEachUploadRange(
 	    vaddr, size, is_written,
 	    [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); },
 	    [&]() noexcept {
 		    for (const auto& [address, bytes]: uploads) {
-			    Upload(command, buffer, buffer.Offset(address),
-			           reinterpret_cast<const void*>(address), bytes);
+			    WriteDataBuffer(buffer, address, reinterpret_cast<const void*>(address), bytes);
 		    }
 	    });
 	if (is_texel_buffer && !is_written) {
@@ -469,71 +451,52 @@ bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint
 	return false;
 }
 
-BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
-                                        bool is_written, bool is_read, bool is_formatted) {
-	if (command.IsInvalid() || command.IsExecute()) {
+std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t size,
+                                                       bool is_written, bool is_texel_buffer,
+                                                       BufferId id) {
+	auto& command = m_scheduler.Current();
+	if (command.IsInvalid() || command.IsExecute() || vaddr == 0 || size == 0 ||
+	    vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
 
-	if (is_read && !is_written && size <= CACHING_PAGE_SIZE &&
+	if (!is_written && size <= CACHING_PAGE_SIZE &&
 	    !m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
 	    m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
 		const auto alignment = std::max<uint64_t>(
 		    m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1);
-		if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false);
-		    mapped != nullptr) {
-			if (Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) {
-				m_stream_buffer.Commit();
-				return {{}, m_stream_buffer.Handle(), offset};
-			}
-		} else {
-			auto owner = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0,
-			                                      AllFlags, size);
-			if (Libs::LibKernel::Memory::TryReadBacking(vaddr, owner->Mapped().data(), size)) {
-				owner->Flush(0, size);
-				return {owner, owner->Handle(), 0};
-			}
+		auto [mapped, offset] = m_stream_buffer.Map(size, alignment);
+		if (mapped != nullptr && Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) {
+			m_stream_buffer.Commit();
+			return {&m_stream_buffer, offset};
 		}
 	}
 
-	if (is_formatted && is_written) {
-		(void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true);
+	auto* buffer = m_slot_buffers.try_get(id);
+	if (buffer == nullptr || buffer->is_deleted || !buffer->IsInBounds(vaddr, size)) {
+		id     = FindBuffer(vaddr, size);
+		buffer = &m_slot_buffers[id];
 	}
-
-	auto& cached = GetOrCreateBuffer(command, vaddr, size);
-	(void)SynchronizeBuffer(command, *cached.buffer, vaddr, size, is_written,
-	                        is_formatted && is_read);
+	(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
-	auto     owner  = cached.buffer;
-	uint64_t offset = cached.buffer->Offset(vaddr);
-	return {owner, owner->Handle(), offset};
+	return {buffer, buffer->Offset(vaddr)};
 }
 
-std::shared_ptr<Buffer> BufferCache::ObtainNullBuffer() {
-	if (m_null_buffer != nullptr) {
-		return m_null_buffer;
-	}
-	m_null_buffer = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0,
-	                                         AllFlags, 16);
-	const std::array<uint8_t, 16> zeros {};
-	Upload(m_scheduler.Current(), *m_null_buffer, 0, zeros.data(), zeros.size());
-	return m_null_buffer;
-}
-
-ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
+std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
 	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
 	    size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: invalid image source\n");
 	}
-	auto find_owner = [&]() -> CachedBuffer* {
+	auto find_owner = [&]() -> Buffer* {
 		auto owner = m_buffers.upper_bound(vaddr);
 		if (owner == m_buffers.begin()) {
 			return nullptr;
 		}
 		--owner;
-		return owner->second->buffer->IsInBounds(vaddr, size) ? owner->second.get() : nullptr;
+		auto* buffer = m_slot_buffers.try_get(owner->second);
+		return buffer != nullptr && buffer->IsInBounds(vaddr, size) ? buffer : nullptr;
 	};
 
 	{
@@ -549,17 +512,18 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 			if (!IsRegionRegistered(vaddr, size)) {
 				EXIT("BufferCache: GPU-dirty image source has no native buffer\n");
 			}
-			auto& cached        = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size);
-			owner               = &cached;
-			const auto resolved = m_buffers.find(cached.vaddr);
-			if (resolved == m_buffers.end() || resolved->second.get() != &cached ||
-			    !cached.buffer->IsInBounds(vaddr, size)) {
+			const auto id       = FindBuffer(vaddr, size);
+			auto&      buffer   = m_slot_buffers[id];
+			owner               = &buffer;
+			const auto resolved = m_buffers.find(buffer.CpuAddress());
+			if (resolved == m_buffers.end() || resolved->second != id ||
+			    !buffer.IsInBounds(vaddr, size)) {
 				EXIT("BufferCache: merged image source does not contain the requested range\n");
 			}
 		}
 		if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) {
 			owner->tick_accessed_last = m_gc_tick;
-			return {owner->buffer.get(), owner->buffer->Offset(vaddr)};
+			return {owner, owner->Offset(vaddr)};
 		}
 		if (has_dirty_buffer_source && owner == nullptr) {
 			EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
@@ -584,8 +548,7 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 		return {&m_staging_buffer, stage_offset};
 	}
 
-	auto& cached              = *owner;
-	cached.tick_accessed_last = m_gc_tick;
+	owner->tick_accessed_last = m_gc_tick;
 	std::vector<std::pair<uint64_t, uint64_t>> uploads;
 	m_memory_tracker.ForEachUploadRange(
 	    vaddr, size, false,
@@ -594,12 +557,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    },
 	    [&]() noexcept {
 		    for (const auto& [address, upload_size]: uploads) {
-			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr,
-			        cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
+			    owner->CopyFrom(m_scheduler.Current(), m_staging_buffer,
+			                    stage_offset + address - vaddr, owner->Offset(address), upload_size,
+			                    vk::AccessFlagBits::eHostWrite);
 		    }
 	    });
-	return {cached.buffer.get(), cached.buffer->Offset(vaddr)};
+	return {owner, owner->Offset(vaddr)};
 }
 
 void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data) {
@@ -609,16 +572,16 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data)
 	Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size());
 
 	const auto end = vaddr + data.size();
-	for (auto& [address, cached]: m_buffers) {
-		const auto cached_end = address + cached->size;
+	for (const auto& [address, id]: m_buffers) {
+		auto&      buffer     = m_slot_buffers[id];
+		const auto buffer_end = address + buffer.Size();
 		const auto begin      = std::max(vaddr, address);
-		const auto range_end  = std::min(end, cached_end);
+		const auto range_end  = std::min(end, buffer_end);
 		if (begin >= range_end) {
 			continue;
 		}
-		Upload(m_scheduler.Current(), *cached->buffer, cached->buffer->Offset(begin),
-		       data.data() + begin - vaddr, range_end - begin);
-		cached->tick_accessed_last = m_gc_tick;
+		WriteDataBuffer(buffer, begin, data.data() + begin - vaddr, range_end - begin);
+		buffer.tick_accessed_last = m_gc_tick;
 	}
 }
 
@@ -656,12 +619,11 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 		}
 	}
 
-	auto& command = m_scheduler.Current();
-	auto  dst     = ObtainBuffer(command, vaddr, size, true, false, true);
-	EXIT_IF(dst.buffer == nullptr || dst.owner == nullptr);
-	command.RetainResourceUntilFence(dst.owner);
-	auto owner = std::static_pointer_cast<Buffer>(dst.owner);
-	owner->Fill(dst.offset, size, value);
+	m_texture_cache.InvalidateMemoryFromGPU(vaddr, size);
+	const auto id          = FindBuffer(vaddr, size);
+	auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true, id);
+	EXIT_IF(dst == nullptr);
+	dst->Fill(dst_offset, size, value);
 }
 
 void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
@@ -703,30 +665,20 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	}
 
 	auto& command = m_scheduler.Current();
-	auto  src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true)
-	                       : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr};
-	auto  dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true)
-	                       : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr};
-	EXIT_IF(src.buffer == nullptr || dst.buffer == nullptr || (dst_memory && dst.owner == nullptr));
-	if (src.owner != nullptr) {
-		command.RetainResourceUntilFence(src.owner);
+	if (dst_memory) {
+		m_texture_cache.InvalidateMemoryFromGPU(dst_vaddr, size);
 	}
-	if (dst.owner != nullptr) {
-		command.RetainResourceUntilFence(dst.owner);
-	}
-	if (src.buffer == dst.buffer && src.offset < dst.offset + size &&
-	    dst.offset < src.offset + size) {
+	const auto src_id      = src_memory ? FindBuffer(src_vaddr, size) : BufferId {};
+	const auto dst_id      = dst_memory ? FindBuffer(dst_vaddr, size) : BufferId {};
+	auto [src, src_offset] = src_memory ? ObtainBuffer(src_vaddr, size, false, true, src_id)
+	                                    : std::pair {&m_gds_buffer, src_vaddr};
+	auto [dst, dst_offset] = dst_memory ? ObtainBuffer(dst_vaddr, size, true, true, dst_id)
+	                                    : std::pair {&m_gds_buffer, dst_vaddr};
+	EXIT_IF(src == nullptr || dst == nullptr);
+	if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) {
 		EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
 	}
-	auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
-	               : src_gds            ? m_gds_buffer
-	                                    : m_stream_buffer;
-	auto& destination =
-	    dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
-	if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
-		EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
-	}
-	destination.CopyFrom(command, source, src.offset, dst.offset, size);
+	dst->CopyFrom(command, *src, src_offset, dst_offset, size);
 }
 
 bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) {
@@ -740,8 +692,8 @@ bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) {
 	if (candidate == m_buffers.begin()) {
 		return false;
 	}
-	const auto& [address, cached] = *std::prev(candidate);
-	return address + cached->size > vaddr;
+	const auto& [address, id] = *std::prev(candidate);
+	return address + m_slot_buffers[id].Size() > vaddr;
 }
 
 bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
@@ -769,70 +721,56 @@ void BufferCache::RunGarbageCollector() {
 	const uint64_t age        = std::min<uint64_t>(aggressive ? 80 : 160, tick);
 	const size_t   limit      = aggressive ? 64 : 32;
 
-	std::vector<RetiredBuffer>                                       retires;
-	std::vector<std::pair<RetiredBuffer, std::vector<DownloadCopy>>> dirty_retires;
-	{
-		std::vector<uint64_t> candidates;
-		for (const auto& [address, owner]: m_buffers) {
-			const auto& cached = *owner;
-			if (tick - std::min(tick, cached.tick_accessed_last) < age) {
-				continue;
-			}
+	std::vector<uint64_t> candidates;
+	for (const auto& [address, id]: m_buffers) {
+		const auto& buffer = m_slot_buffers[id];
+		if (tick - std::min(tick, buffer.tick_accessed_last) >= age) {
 			candidates.push_back(address);
 		}
-		std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) {
-			return m_buffers.at(left)->tick_accessed_last < m_buffers.at(right)->tick_accessed_last;
-		});
-		if (candidates.size() > limit) {
-			candidates.resize(limit);
-		}
-		for (const auto address: candidates) {
-			auto& cached = *m_buffers.at(address);
-			m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
-			                                           cached.size, "garbage collection");
-			retires.push_back({address, cached.size, cached.buffer});
-			// GC runs immediately before submission. Preserve every source referenced by commands
-			// already recorded in the active batch.
-			m_scheduler.Current().RetainResourceUntilFence(cached.buffer);
-		}
-		for (const auto& retire: retires) {
-			if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
-				continue;
-			}
-			auto& copies = dirty_retires.emplace_back(retire, std::vector<DownloadCopy> {}).second;
+	}
+	std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) {
+		return m_slot_buffers[m_buffers.at(left)].tick_accessed_last <
+		       m_slot_buffers[m_buffers.at(right)].tick_accessed_last;
+	});
+	if (candidates.size() > limit) {
+		candidates.resize(limit);
+	}
+
+	for (const auto address: candidates) {
+		auto found = m_buffers.find(address);
+		EXIT_IF(found == m_buffers.end());
+		const auto id     = found->second;
+		auto&      buffer = m_slot_buffers[id];
+		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, buffer.CpuAddress(),
+		                                           buffer.Size(), "garbage collection");
+		const bool dirty = m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size());
+		std::vector<DownloadCopy> copies;
+		if (dirty) {
 			m_memory_tracker.ForEachDownloadRange<false>(
-			    retire.address, retire.size,
-			    [&](uint64_t address, uint64_t size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
-				                                           "garbage collection");
+			    buffer.CpuAddress(), buffer.Size(),
+			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
+				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address,
+				                                           dirty_size, "garbage collection");
 			    },
-			    [&](uint64_t address, uint64_t size) noexcept {
-				    for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
-					    copies.push_back({retire.owner, range.address - retire.address,
+			    [&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
+				    for (const auto range:
+				         m_gpu_modified_ranges.Intersections(dirty_address, dirty_size)) {
+					    copies.push_back({&buffer, range.address - buffer.CpuAddress(),
 					                      range.address, range.size});
 				    }
 			    });
 		}
-	}
 
-	for (auto& [retire, copies]: dirty_retires) {
-		QueueGarbageDownload(copies, std::move(retire));
-	}
-
-	for (const auto& retire: retires) {
-		auto found = m_buffers.find(retire.address);
-		if (found == m_buffers.end() || found->second->size != retire.size ||
-		    found->second->buffer != retire.owner) {
-			EXIT("BufferCache: garbage-collection owner changed during download\n");
+		const auto retire_address = buffer.CpuAddress();
+		const auto retire_size    = buffer.Size();
+		if (dirty) {
+			EXIT_IF(copies.empty());
+			Unregister(id);
+			QueueGarbageDownload(copies, id, retire_address, retire_size);
+		} else {
+			m_memory_tracker.UntrackMemory(retire_address, retire_size);
+			DeleteBuffer(id);
 		}
-		if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
-			m_memory_tracker.UntrackMemory(retire.address, retire.size);
-		}
-		if (retire.size > m_total_used_memory) {
-			EXIT("BufferCache: allocation accounting underflow\n");
-		}
-		m_total_used_memory -= retire.size;
-		m_buffers.erase(found);
 	}
 }
 

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -3,12 +3,12 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
 #include <map>
-#include <memory>
 #include <span>
 #include <utility>
 #include <vector>
@@ -16,49 +16,44 @@
 namespace Libs::Graphics {
 
 struct GraphicContext;
-class CommandBuffer;
 class CommandScheduler;
 class TextureCache;
 
-struct BufferBinding {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	uint64_t              offset = 0;
-};
-
-struct ImageBufferSource {
-	Buffer*  buffer = nullptr;
-	uint64_t offset = 0;
-};
+using BufferId = Common::SlotId;
+inline constexpr BufferId NULL_BUFFER_ID {0};
 
 class BufferCache {
 public:
 	static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
-	static constexpr uint64_t GetBufferOffset(uint64_t vaddr) {
-		return vaddr & (CACHING_PAGE_SIZE - 1);
-	}
 
 	BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
 	            TextureCache& texture_cache);
 	~BufferCache();
 	KYTY_CLASS_NO_COPY(BufferCache);
 
-	void                        InvalidateMemory(uint64_t vaddr, uint64_t size);
-	void                        ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
-	void                        UnmapMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
-	                                         bool is_written = false, bool is_read = true,
-	                                         bool is_formatted = false);
-	[[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept;
-	[[nodiscard]] Buffer&       GetGdsBuffer() noexcept { return m_gds_buffer; }
-	[[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; }
-	[[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size,
-	                                            uint64_t alignment);
-	[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
-	[[nodiscard]] ImageBufferSource       ObtainBufferForImage(uint64_t vaddr, uint64_t size);
-	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
-	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
-	                bool src_gds = false);
+	void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
+	void                   ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
+	void                   UnmapMemory(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] Buffer&  GetBuffer(BufferId id) { return m_slot_buffers[id]; }
+	[[nodiscard]] BufferId FindBuffer(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBuffer(uint64_t vaddr, uint64_t size,
+	                                                        bool     is_written,
+	                                                        bool     is_texel_buffer = false,
+	                                                        BufferId id              = {});
+	[[nodiscard]] StreamBuffer&                GetUtilityBuffer(MemoryUsage usage) noexcept {
+		switch (usage) {
+			case MemoryUsage::Upload: return m_staging_buffer;
+			case MemoryUsage::Stream: return m_stream_buffer;
+			case MemoryUsage::Download: return m_download_buffer;
+			case MemoryUsage::DeviceLocal: return m_device_buffer;
+		}
+		EXIT("BufferCache: invalid utility-buffer usage\n");
+	}
+	[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
+	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
+	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
+	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
+	                bool src_gds);
 	// Cache-index and exact dirty-range queries require GPU-thread serialization.
 	[[nodiscard]] bool IsRegionRegistered(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
@@ -69,51 +64,42 @@ public:
 private:
 	friend struct BufferCacheTestAccess;
 
-	struct CacheRange {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-	struct CachedBuffer;
 	struct DownloadCopy;
 	struct DownloadRange;
-	struct RetiredBuffer;
 	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static uint64_t           AlignDown(uint64_t value) noexcept;
-	[[nodiscard]] static uint64_t           AlignUp(uint64_t value);
 	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
 		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
 	}
 	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
-	[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
-	void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
-	            const void* source, uint64_t size);
-	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
-	                                              uint64_t size);
-	[[nodiscard]] bool SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
-	                                     uint64_t size, bool is_written, bool is_texel_buffer);
+	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
+	void Unregister(BufferId id);
+	void DeleteBuffer(BufferId id);
+	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
+	                                     bool is_written, bool is_texel_buffer);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
 	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
 	void PublishDownloads(std::span<const DownloadRange> downloads);
-	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
+	void QueueGarbageDownload(std::span<const DownloadCopy> copies, BufferId id, uint64_t address,
+	                          uint64_t size);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
 
-	GraphicContext&                                   m_graphics;
-	CommandScheduler&                                 m_scheduler;
-	Buffer                                            m_gds_buffer;
-	std::shared_ptr<Buffer>                           m_null_buffer;
-	std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
-	RangeSet                                          m_gpu_modified_ranges;
-	MemoryTracker                                     m_memory_tracker;
-	StreamBuffer                                      m_staging_buffer;
-	StreamBuffer                                      m_stream_buffer;
-	StreamBuffer                                      m_download_buffer;
-	StreamBuffer                                      m_device_buffer;
-	TextureCache&                                     m_texture_cache;
-	uint64_t                                          m_total_used_memory = 0;
-	uint64_t m_trigger_gc_memory  = 1ull * 1024 * 1024 * 1024;
-	uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
-	uint64_t m_gc_tick            = 0;
+	GraphicContext&              m_graphics;
+	CommandScheduler&            m_scheduler;
+	Buffer                       m_gds_buffer;
+	Common::SlotVector<Buffer>   m_slot_buffers;
+	std::map<uint64_t, BufferId> m_buffers;
+	RangeSet                     m_gpu_modified_ranges;
+	MemoryTracker                m_memory_tracker;
+	StreamBuffer                 m_staging_buffer;
+	StreamBuffer                 m_stream_buffer;
+	StreamBuffer                 m_download_buffer;
+	StreamBuffer                 m_device_buffer;
+	TextureCache&                m_texture_cache;
+	uint64_t                     m_total_used_memory  = 0;
+	uint64_t                     m_trigger_gc_memory  = 1ull * 1024 * 1024 * 1024;
+	uint64_t                     m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
+	uint64_t                     m_gc_tick            = 0;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -65,6 +65,10 @@ public:
 	                                                   vk::AccessFlagBits::eMemoryWrite);
 	void Fill(uint64_t offset, uint64_t size, uint32_t value);
 
+	// BufferCache state lives directly on the resource.
+	bool     is_deleted         = false;
+	uint64_t tick_accessed_last = 0;
+
 protected:
 	[[nodiscard]] GraphicContext&   Graphics() const noexcept { return *m_graphics; }
 	[[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; }

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -4,17 +4,17 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "common/lruCache.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/pageManager.h"
 #include "graphics/host_gpu/regionManager.h"
 #include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/image/blitHelper.h"
 #include "graphics/host_gpu/renderer/image/image.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 
-#include <compare>
 #include <map>
-#include <memory>
-#include <set>
-#include <utility>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -25,8 +25,6 @@ class BufferCache;
 class CommandBuffer;
 class CommandScheduler;
 class RenderExecutor;
-class StreamBuffer;
-class TileManager;
 struct TextureCacheTestAccess;
 
 class TextureCache {
@@ -57,20 +55,26 @@ public:
 	[[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc);
 	[[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc);
 	[[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc);
-	[[nodiscard]] Image&        GetImage(ImageId id);
-	[[nodiscard]] const Image&  GetImage(ImageId id) const;
-	void                        MarkGpuWritten(ImageId id);
+	[[nodiscard]] Image&        GetImage(ImageId id) {
+		auto& image = m_slot_images[id];
+		TouchImage(image);
+		return image;
+	}
+	void MarkGpuWritten(ImageId id);
 
 	[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
-	[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
-	                                           bool formatted_buffer_write = false);
+	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
 	[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
-	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);
+	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice,
+	                                 uint32_t* fill_value = nullptr);
 	[[nodiscard]] bool ClearMeta(uint64_t address);
+	// Returns true when registered DCC absorbed the fill and the caller may skip the dispatch.
+	// False may still record PendingDcc state, but the guest dispatch must execute.
+	[[nodiscard]] bool TryConsumeDccFill(uint64_t address, uint64_t size, uint32_t fill_value);
 	[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
 
 	void UnmapMemory(uint64_t address, uint64_t size);
@@ -82,13 +86,18 @@ private:
 	struct ColorTransferPlan;
 	struct DownloadPlan;
 
-	struct Slot {
-		std::shared_ptr<Image> image;
-		uint32_t               generation = 1;
-	};
-
 	struct MetaDataInfo {
+		// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
+		// PendingDcc retains that exact fill until FindRenderTarget classifies the address,
+		// without exposing an unconfirmed buffer address to the normal metadata heuristics.
+		// Keep all surface metadata in one entry so CMask/FMask can be
+		// registered beside HTile and DCC without introducing parallel tracking paths.
+		enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc };
+
+		Type     type       = Type::PendingDcc;
 		uint32_t clear_mask = 0;
+		uint32_t fill_value = 0xffffffffu;
+		uint64_t fill_size  = 0;
 	};
 
 	struct OverlapResult {
@@ -100,26 +109,21 @@ private:
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
 	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
-	[[nodiscard]] Image&                 ResolveImage(ImageId id);
-	[[nodiscard]] const Image&           ResolveImage(ImageId id) const;
-	[[nodiscard]] std::shared_ptr<Image> ResolveOwner(ImageId id) const;
-	[[nodiscard]] ImageId                InsertImage(const ImageInfo& info);
-	[[nodiscard]] ImageId                GetNullImage(const ImageDesc& desc);
-	void                                 RegisterImage(ImageId id);
-	void                                 UnregisterImage(ImageId id);
-	void                                 DeleteImage(ImageId id);
-	void                                 FreeImage(ImageId id);
-	void                                 RetainImage(CommandBuffer& command, ImageId id);
-	void                                 TouchImage(Image& image);
-	void                                 TrackImage(ImageId id);
-	void                                 TrackImageHead(ImageId id);
-	void                                 TrackImageTail(ImageId id);
-	void                                 UntrackImage(ImageId id);
-	void                                 UntrackImageHead(ImageId id);
-	void                                 UntrackImageTail(ImageId id);
-	void                                 MarkAsMaybeDirty(ImageId id, Image& image);
-	void                                 TrackImageDownload(ImageId id);
-	void                                 TrackImageDownloadLocked(ImageId id, Image& image);
+	[[nodiscard]] ImageId     InsertImage(const ImageInfo& info);
+	[[nodiscard]] ImageId     GetNullImage(const ImageDesc& desc);
+	void                      RegisterImage(ImageId id);
+	void                      UnregisterImage(ImageId id);
+	void                      DeleteImage(ImageId id);
+	void                      FreeImage(ImageId id);
+	void                      TouchImage(Image& image);
+	void                      TrackImage(ImageId id);
+	void                      TrackImageHead(ImageId id);
+	void                      TrackImageTail(ImageId id);
+	void                      UntrackImage(ImageId id);
+	void                      UntrackImageHead(ImageId id);
+	void                      UntrackImageTail(ImageId id);
+	void                      MarkAsMaybeDirty(ImageId id, Image& image);
+	void                      TrackImageDownload(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
 	                                      bool exact_format);
 	[[nodiscard]] static BindingType UploadBinding(const Image& image);
@@ -148,31 +152,24 @@ private:
 	[[nodiscard]] bool CopyD16(Image& destination, Image& source);
 	void               CopyImage(ImageId destination, ImageId source);
 	void               AssociateStencil(ImageId depth, GuestRange stencil);
-	void               AssociateStencilLocked(ImageId depth, GuestRange stencil);
 	void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer);
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
-	void InvalidateCpuAliases(uint64_t address, uint64_t size);
-	void ClearGpuModified(ImageId id);
-
-	void                                        DownloadImage(ImageId id);
-	[[nodiscard]] bool                          TryDownloadImage(ImageId id);
-	[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
-	void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset);
+	void               InvalidateCpuAliases(uint64_t address, uint64_t size);
+	[[nodiscard]] bool TryDownloadImage(ImageId id);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
 	TrackingSpinLock                                  m_lock;
 	PageManager&                                      m_page_manager;
 	BlitHelper                                        m_blit_helper;
-	std::unique_ptr<TileManager>                      m_tiler;
+	TileManager                                       m_tiler;
 	BufferCache&                                      m_buffer_cache;
-	std::vector<Slot>                                 m_slots;
-	std::vector<uint32_t>                             m_free_slots;
+	Common::SlotVector<Image>                         m_slot_images;
 	ImagePageTable                                    m_image_page_table;
-	std::map<vk::Format, ImageId>                     m_null_images;
+	std::unordered_map<vk::Format, ImageId>           m_null_images;
 	Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
-	std::set<ImageId>                                 m_download_images;
+	std::unordered_set<ImageId>                       m_download_images;
 	std::map<uint64_t, MetaDataInfo>                  m_surface_metas;
 	uint64_t                                          m_total_used_memory  = 0;
 	uint64_t                                          m_trigger_gc_memory  = 0;

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -9,7 +9,6 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -21,6 +20,28 @@ namespace Libs::Graphics {
 
 static std::atomic<uint32_t> g_render_color_log_count = 0;
 
+static void ResolveDccClearInfo(RenderColorInfo& info, vk::Format format, bool has_dcc,
+                                uint32_t packed_clear) {
+	// Register-backed DCC clears use the target's packed clear value. Decode one-word guest
+	// formats here; unsupported encodings remain tracked without unsafe materialization.
+	info.metadata_clear_supported =
+	    has_dcc && DecodePackedColorClear(format, packed_clear, info.color_clear_value);
+	switch (format) {
+		case vk::Format::eR8G8B8A8Unorm:
+		case vk::Format::eR8G8B8A8Srgb:
+		case vk::Format::eB8G8R8A8Unorm:
+		case vk::Format::eB8G8R8A8Srgb:
+		case vk::Format::eA2B10G10R10UnormPack32:
+		case vk::Format::eA2R10G10B10UnormPack32:
+			info.metadata_fixed_clear_supported = has_dcc;
+			break;
+		default: info.metadata_fixed_clear_supported = false; break;
+	}
+	if (!info.metadata_clear_supported) {
+		info.color_clear_value = {};
+	}
+}
+
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
                                               RenderColorInfo& r,
@@ -55,20 +76,22 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		}
 
 		// No color output
-		r.type               = RenderColorType::NoColorOutput;
-		r.desc               = {};
-		r.base_addr          = 0;
-		r.image_id           = {};
-		r.image_view         = nullptr;
-		r.format             = vk::Format::eUndefined;
-		r.extent             = {};
-		r.base_mip_level     = 0;
-		r.base_array_layer   = 0;
-		r.buffer_size        = 0;
-		r.samples            = 1;
-		r.export_mapping     = {};
-		r.color_clear_enable = false;
-		r.color_clear_value  = {};
+		r.type                           = RenderColorType::NoColorOutput;
+		r.desc                           = {};
+		r.base_addr                      = 0;
+		r.image_id                       = {};
+		r.image_view                     = nullptr;
+		r.format                         = vk::Format::eUndefined;
+		r.extent                         = {};
+		r.base_mip_level                 = 0;
+		r.base_array_layer               = 0;
+		r.buffer_size                    = 0;
+		r.samples                        = 1;
+		r.export_mapping                 = {};
+		r.color_clear_enable             = false;
+		r.metadata_clear_supported       = false;
+		r.metadata_fixed_clear_supported = false;
+		r.color_clear_value              = {};
 		return;
 	}
 	const auto samples = render_sample_count(rt.attrib.num_fragments);
@@ -107,9 +130,9 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		}
 	}
 
-	// CB_COLOR_CONTROL describes the color-buffer operation / ROP3 logic op.
-	// ROP3 Copy is the normal color write path, not a render-target clear.
-	// SRGB clear words are still encoded as normalized component values.
+	// Color-control state selects the color-buffer operation and logical blend operation.
+	// The normal copy operation is a regular color write, not an attachment clear.
+	// Nonlinear clear values are still stored as normalized components.
 	// Fast color clears are metadata driven and must be handled explicitly when
 	// that metadata path is implemented; render-pass load must preserve contents.
 	r.color_clear_enable = false;
@@ -309,6 +332,13 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	desc.info.bytes_per_block = bytes_per_element;
 	desc.info.samples         = samples;
 	desc.info.tile_mode       = rt.attrib3.tile_mode;
+	const bool has_dcc        = rt.info.dcc_compression_enable && rt.dcc_addr.addr != 0;
+	if (has_dcc) {
+		// DCC uses a separate metadata allocation. Carry its address through ImageInfo so
+		// TextureCache can associate metadata fills observed before target registration.
+		desc.info.metadata.kind          = ImageMetadataKind::Dcc;
+		desc.info.metadata.range.address = rt.dcc_addr.addr;
+	}
 	for (uint32_t level = 0; level < levels; level++) {
 		if (volume) {
 			const auto& mip             = volume_layout.mips[level];
@@ -355,7 +385,7 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	r.samples                  = samples;
 	r.export_mapping           = target_format.export_mapping;
 	r.color_clear_enable       = false;
-	r.color_clear_value        = {};
+	ResolveDccClearInfo(r, target_format.format, has_dcc, rt.clear_word0.word0);
 	BindRenderTarget(r.image_id);
 }
 

src/graphics/host_gpu/renderer/colorRenderTarget.h

@@ -31,7 +31,9 @@ struct RenderColorInfo {
 	uint32_t                        target_slot      = 0;
 	uint32_t                        samples          = 1;
 	Prospero::ColorComponentMapping export_mapping;
-	bool                            color_clear_enable = false;
+	bool                            color_clear_enable             = false;
+	bool                            metadata_clear_supported       = false;
+	bool                            metadata_fixed_clear_supported = false;
 	vk::ClearColorValue             color_clear_value {};
 };
 

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -226,12 +226,17 @@ void CommandScheduler::Wait(uint64_t tick) {
 	EXIT_IF(tick > CurrentTick());
 	if (tick >= CurrentTick()) {
 		// A stream-buffer wrap can wait while a draw is being prepared through a reference to
-		// Current(). Recycle the same command object so that reference remains valid.
-		FinishCurrent();
+		// Current(). Recycle the same command object so that reference remains valid. Deferred
+		// resources are released only at the next GPU operation boundary.
+		SubmitInfo submit;
+		auto&      submitted = SubmitCurrent(submit);
+		submitted.WaitForFenceAndReset();
+		m_master.Refresh();
+		submitted.Begin();
+		m_recording = true;
 		return;
 	}
 	m_master.Wait(tick);
-	PopPendingOperations();
 }
 
 void CommandScheduler::PopPendingOperations() {
@@ -243,17 +248,18 @@ void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) {
 		m_master.Refresh();
 	}
 	for (;;) {
-		Common::UniqueFunction<void> callback;
+		PendingOperation operation;
 		{
 			std::lock_guard lock(m_operation_mutex);
 			if (m_pending_operations.empty() ||
 			    !m_master.IsFree(m_pending_operations.front().tick)) {
 				return;
 			}
-			callback = std::move(m_pending_operations.front().callback);
+			operation = std::move(m_pending_operations.front());
 			m_pending_operations.pop();
 		}
-		RunOperation(std::move(callback));
+		WaitPriorityOperations(operation.tick);
+		RunOperation(std::move(operation.callback));
 	}
 }
 
@@ -423,10 +429,8 @@ void CommandScheduler::BeginNext() {
 	EXIT_IF(m_recording);
 
 	auto candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	bool refreshed = false;
 	if (candidate < 0) {
 		m_master.Refresh();
-		refreshed = true;
 		candidate = FindReusableBuffer(m_master.KnownGpuTick());
 	}
 	if (candidate < 0) {
@@ -435,7 +439,6 @@ void CommandScheduler::BeginNext() {
 
 	m_current = candidate;
 	Current().WaitForFenceAndReset();
-	PopPendingOperations(!refreshed);
 	BindCurrent();
 	Current().Begin();
 	m_recording = true;

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -55,13 +55,14 @@ public:
 	void                      DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
 	[[nodiscard]] static bool InDeferredOperation() noexcept;
 
-	[[nodiscard]] bool            Active() const noexcept { return m_current >= 0; }
-	void                          CheckActive() const;
-	RenderCommandBuffer&          Current() const;
-	[[nodiscard]] uint64_t        CurrentTick() const noexcept { return m_master.CurrentTick(); }
-	[[nodiscard]] bool            IsFree(uint64_t tick);
-	[[nodiscard]] RenderContext&  Context() const noexcept { return m_context; }
-	[[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; }
+	[[nodiscard]] bool             Active() const noexcept { return m_current >= 0; }
+	void                           CheckActive() const;
+	RenderCommandBuffer&           Current() const;
+	[[nodiscard]] uint64_t         CurrentTick() const noexcept { return m_master.CurrentTick(); }
+	[[nodiscard]] bool             IsFree(uint64_t tick);
+	[[nodiscard]] MasterSemaphore& GetMasterSemaphore() noexcept { return m_master; }
+	[[nodiscard]] RenderContext&   Context() const noexcept { return m_context; }
+	[[nodiscard]] GraphicContext&  Graphics() const noexcept { return m_graphics; }
 
 private:
 	static constexpr size_t CommandBufferGrowStep = 4;

src/graphics/host_gpu/renderer/context.cpp

@@ -9,7 +9,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vma.h"
@@ -17,30 +16,27 @@
 
 #include <algorithm>
 #include <bit>
+#include <cstdio>
 #include <cstring>
-#include <memory>
 namespace Libs::Graphics {
 
-FenceResourceRetainer::~FenceResourceRetainer() {
-	if (!m_resources.empty()) {
-		EXIT("fence resource retainer destroyed before release\n");
-	}
+namespace {
+
+void ReportVulkanFatal(const char* what, vk::Result result, uint32_t slot, uint64_t submit_seq,
+                       uint32_t debug_op, uint64_t debug_submit, uint32_t arg0, uint32_t arg1,
+                       uint32_t arg2, uint32_t arg3, uint64_t arg4) {
+	LOGF("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	     what, VulkanToString(result).c_str(), static_cast<int>(result), slot, submit_seq, debug_op,
+	     debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::printf("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64
+	            " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	            what, VulkanToString(result).c_str(), static_cast<int>(result), slot, submit_seq,
+	            debug_op, debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::fflush(stdout);
 }
 
-void FenceResourceRetainer::Retain(std::shared_ptr<void> resource) {
-	if (resource == nullptr) {
-		EXIT("cannot retain a null fence resource\n");
-	}
-	if (std::ranges::none_of(m_resources, [&resource](const auto& retained) {
-		    return retained.get() == resource.get();
-	    })) {
-		m_resources.push_back(std::move(resource));
-	}
-}
-
-void FenceResourceRetainer::ReleaseAfterFence() noexcept {
-	m_resources.clear();
-}
+} // namespace
 
 CommandBuffer::CommandBuffer(CommandScheduler& scheduler)
     : m_context(scheduler.Context()), m_scheduler(scheduler), m_graphics(scheduler.Graphics()),
@@ -71,37 +67,11 @@ void CommandBuffer::Release() {
 
 	m_slot->busy = false;
 	m_slot->Reset();
-	ReleaseResourcesAfterFence();
 	m_slot = nullptr;
 
 	EXIT_NOT_IMPLEMENTED(!IsInvalid());
 }
 
-void CommandBuffer::RetireBufferAfterFence(std::unique_ptr<VulkanBuffer> buffer) {
-	if (IsInvalid() || m_execute || buffer == nullptr || buffer->buffer == nullptr) {
-		EXIT("cannot retire a buffer on an invalid or submitted command buffer\n");
-	}
-	m_retired_buffers.push_back(std::move(buffer));
-}
-
-void CommandBuffer::RetainResourceUntilFence(std::shared_ptr<void> resource) {
-	if (IsInvalid() || m_execute) {
-		EXIT("cannot retain a resource on an invalid or submitted command buffer\n");
-	}
-	m_fence_resources.Retain(std::move(resource));
-}
-
-void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet& set) {
-	m_descriptor_sets_after_fence.push_back(&set);
-}
-
-void CommandBuffer::RecycleDescriptorsAfterFence() {
-	for (auto* set: m_descriptor_sets_after_fence) {
-		m_context.GetDescriptorCache().Recycle(*set);
-	}
-	m_descriptor_sets_after_fence.clear();
-}
-
 void CommandBuffer::Begin() const {
 	EXIT_IF(m_rendering);
 	auto buffer = Handle();
@@ -169,8 +139,9 @@ void CommandBuffer::Execute(const SubmitInfo& submit) {
 
 	auto result = graphics.device.resetFences(1, &fence);
 	if (result != vk::Result::eSuccess) {
-		LOGF("vkResetFences failed before submit: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
+		ReportVulkanFatal("vkResetFences (before submit)", result, m_slot->id, m_submit_seq,
+		                  m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2,
+		                  m_debug_arg3, m_debug_arg4);
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
@@ -192,11 +163,9 @@ void CommandBuffer::Execute(const SubmitInfo& submit) {
 	m_fence_waited = false;
 
 	if (result != vk::Result::eSuccess) {
-		LOGF("vkQueueSubmit failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
+		ReportVulkanFatal("vkQueueSubmit", result, m_slot->id, m_submit_seq, m_debug_op,
+		                  m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
+		                  m_debug_arg4);
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 }
@@ -213,11 +182,9 @@ void CommandBuffer::WaitForFenceOnly() {
 	auto device = m_graphics.device;
 	auto result = device.waitForFences(1, &m_slot->fence, VK_TRUE, UINT64_MAX);
 	if (result != vk::Result::eSuccess) {
-		LOGF("vkWaitForFences failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
+		ReportVulkanFatal("vkWaitForFences", result, m_slot->id, m_submit_seq, m_debug_op,
+		                  m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
+		                  m_debug_arg4);
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	m_fence_waited = true;
@@ -238,22 +205,6 @@ void CommandBuffer::FinalizeFence(bool reset_recording) {
 			m_slot->Reset();
 		}
 	}
-	if (was_executed) {
-		ReleaseResourcesAfterFence();
-	}
-	DeleteBuffersAfterFence();
-}
-
-void CommandBuffer::ReleaseResourcesAfterFence() {
-	RecycleDescriptorsAfterFence();
-	m_fence_resources.ReleaseAfterFence();
-}
-
-void CommandBuffer::DeleteBuffersAfterFence() {
-	for (const auto& buffer: m_retired_buffers) {
-		m_graphics.DeleteBuffer(*buffer);
-	}
-	m_retired_buffers.clear();
 }
 
 void CommandBuffer::BeginRendering(const RenderState& state) const {

src/graphics/host_gpu/renderer/debug.cpp

@@ -266,20 +266,24 @@ static void RtCheck(const HW::RenderTarget& rt) {
 				logged = true;
 			}
 		}
+		const bool native_2x_one_frag = rt.info.fmask_compression_enable &&
+		                                !rt.info.fmask_data_compression_disable &&
+		                                rt.info.cmask_fast_clear_enable && rt.fmask.addr != 0 &&
+		                                rt.cmask.addr != 0 && rt.attrib.num_samples == 1 &&
+		                                rt.attrib.num_fragments == 1;
+		EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode && !native_2x_one_frag);
 		if (rt.info.fmask_compression_enable) {
 			EXIT_NOT_IMPLEMENTED(rt.attrib.num_samples == 0 && rt.attrib.num_fragments == 0);
 			static bool logged = false;
 			if (!logged) {
-				LOGF("RenderTarget: temporary: ignoring PS5 FMASK metadata for single-sample MSAA "
-				     "fallback, fmask=0x%016" PRIx64 "\n",
+				LOGF("RenderTarget: using native Vulkan MSAA without guest FMASK metadata, "
+				     "fmask=0x%016" PRIx64 "\n",
 				     rt.fmask.addr);
 				logged = true;
 			}
 		}
 
-		// EXIT_NOT_IMPLEMENTED(rt.info.fmask_compression_mode != 0x00000000);
 		EXIT_NOT_IMPLEMENTED(rt.info.fmask_data_compression_disable != false);
-		EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode != false);
 
 		if (rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable) {
 			static bool logged = false;
@@ -561,14 +565,11 @@ static void ZCheck(const HW::DepthRenderTarget& z, const HW::DepthControl& dc,
 				     z.depth_view.stencil_write_disable ? "true" : "false");
 			}
 		}
-		EXIT_NOT_IMPLEMENTED(z.z_read_base_addr != z.z_write_base_addr);
-		EXIT_NOT_IMPLEMENTED(z.stencil_read_base_addr != z.stencil_write_base_addr);
-		EXIT_NOT_IMPLEMENTED(z.z_write_base_addr == 0);
-		if (z.stencil_info.format != Prospero::StencilFormat::kInvalid &&
-		    z.stencil_write_base_addr == 0) {
-			LOGF("\t warning: stencil format is set without a stencil base address, continuing "
-			     "without stencil attachment\n");
-		}
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.depth_write_disable &&
+		                     z.z_read_base_addr != z.z_write_base_addr);
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.stencil_write_disable &&
+		                     z.stencil_read_base_addr != z.stencil_write_base_addr);
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.depth_write_disable && z.z_write_base_addr == 0);
 		// EXIT_NOT_IMPLEMENTED(z.pitch_div8_minus1 != 0x000000ff);
 		// EXIT_NOT_IMPLEMENTED(z.height_div8_minus1 != 0x0000008f);
 		// EXIT_NOT_IMPLEMENTED(z.slice_div64_minus1 != 0x00008fff);

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -13,7 +13,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -177,15 +176,17 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	    z.htile_surface.htile_uses_preload_win != 0 || z.htile_surface.preload != 0 ||
 	    z.htile_surface.prefetch_width != 0 || z.htile_surface.prefetch_height != 0 ||
 	    z.htile_surface.dst_outside_zero_to_one != 0 || z.z_read_base_addr == 0 ||
-	    z.z_write_base_addr != z.z_read_base_addr || (z.z_read_base_addr & 0xffffu) != 0 ||
+	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
+	    (z.z_read_base_addr & 0xffffu) != 0 ||
 	    dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways)) {
 		DepthFatal("unsupported depth register state");
 	}
 	if (has_stencil) {
 		if (z.stencil_info.format != Prospero::StencilFormat::k8UInt || !htile_stencil_compat ||
 		    z.stencil_read_base_addr == 0 ||
-		    z.stencil_write_base_addr != z.stencil_read_base_addr ||
-		    (z.stencil_read_base_addr & 0xffffu) != 0 || z.depth_view.stencil_write_disable) {
+		    (!z.depth_view.stencil_write_disable &&
+		     z.stencil_write_base_addr != z.stencil_read_base_addr) ||
+		    (z.stencil_read_base_addr & 0xffffu) != 0) {
 			DepthFatal("unsupported stencil attachment state");
 		}
 	} else if (z.stencil_read_base_addr != 0 || z.stencil_write_base_addr != 0 ||
@@ -286,18 +287,22 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	r.depth_min_bounds         = hw.GetDepthBoundsMin();
 	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
-	r.stencil_clear_enable = has_stencil && rc.stencil_clear_enable;
-	r.stencil_clear_value  = hw.GetStencilClearValue();
-	r.stencil_test_enable  = has_stencil && dc.stencil_enable;
+	r.stencil_clear_enable =
+	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;
+	r.stencil_clear_value = hw.GetStencilClearValue();
+	r.stencil_test_enable = has_stencil && dc.stencil_enable;
 	if (r.stencil_test_enable) {
-		const uint8_t front_write_mask = rc.stencil_clear_enable ? 0 : sm.stencil_writemask;
-		const uint8_t back_write_mask  = rc.stencil_clear_enable ? 0 : sm.stencil_writemask_bf;
+		const bool stencil_ops_disabled =
+		    rc.stencil_clear_enable || z.depth_view.stencil_write_disable;
+		const uint8_t front_write_mask = stencil_ops_disabled ? 0 : sm.stencil_writemask;
+		const uint8_t back_write_mask  = stencil_ops_disabled ? 0 : sm.stencil_writemask_bf;
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
 		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||
-		    (UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
+		    (front_write_mask != 0 &&
+		     UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
 		     sm.stencil_opval != sm.stencil_testval) ||
-		    (dc.backface_enable &&
+		    (dc.backface_enable && back_write_mask != 0 &&
 		     UsesStencilOpValue(sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf) &&
 		     sm.stencil_opval_bf != sm.stencil_testval_bf)) {
 			DepthFatal("unsupported stencil compare or replacement state");

src/graphics/host_gpu/renderer/image/image.h

@@ -2,6 +2,7 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 
 #include "common/assert.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
 
@@ -19,15 +20,7 @@ class Buffer;
 class CommandScheduler;
 struct ImageTestAccess;
 
-struct ImageId {
-	uint32_t index      = std::numeric_limits<uint32_t>::max();
-	uint32_t generation = 0;
-
-	[[nodiscard]] explicit operator bool() const noexcept {
-		return index != std::numeric_limits<uint32_t>::max();
-	}
-	auto operator<=>(const ImageId&) const = default;
-};
+using ImageId = Common::SlotId;
 
 struct CachedImageView {
 	ImageViewInfo info;

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -119,13 +119,16 @@ vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) {
 	return components;
 }
 
-vk::Format TextureGetFormat(Prospero::BufferFormat format) {
-	const auto vk_format = VulkanFormat(format);
+SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
+	const auto backing_format    = Prospero::RemapTextureFormat(format);
+	const auto vk_format         = VulkanFormat(backing_format);
+	const auto conversion_format =
+	    backing_format != format ? format : Prospero::BufferFormat::kInvalid;
 	if (vk_format != vk::Format::eUndefined) {
-		return vk_format;
+		return SurfaceFormatInfo(vk_format, conversion_format);
 	}
 	EXIT("unknown format: fmt = %u\n", static_cast<uint32_t>(format));
-	return vk::Format::eUndefined;
+	return SurfaceFormatInfo(vk::Format::eUndefined, Prospero::BufferFormat::kInvalid);
 }
 
 namespace {

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -3,7 +3,7 @@
 
 #include "common/abi.h"
 #include "common/common.h"
-#include "graphics/guest_gpu/gpu_defs.h"
+#include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
@@ -19,6 +19,15 @@ struct RenderTargetFormatInfo {
 	Prospero::ColorComponentMapping export_mapping;
 };
 
+struct SurfaceFormatInfo {
+	explicit constexpr SurfaceFormatInfo(vk::Format format,
+	                                     Prospero::BufferFormat conversion_format)
+	    : vk_format(format), conversion_format(conversion_format) {}
+
+	vk::Format             vk_format;
+	Prospero::BufferFormat conversion_format;
+};
+
 struct TextureUploadMipLayout {
 	uint64_t offset       = 0;
 	uint64_t size         = 0;
@@ -35,7 +44,7 @@ struct TextureUploadLayout {
 };
 
 vk::ComponentMapping   TextureGetComponentMapping(uint32_t swizzle);
-vk::Format             TextureGetFormat(Prospero::BufferFormat format);
+SurfaceFormatInfo      TextureGetSurfaceFormatInfo(Prospero::BufferFormat format);
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
                                                     Prospero::ChannelType   type,
                                                     Prospero::ChannelOrder  order);

src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp

@@ -1,313 +0,0 @@
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
-
-#include "common/assert.h"
-#include "common/profiler.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-#include <array>
-
-namespace Libs::Graphics {
-namespace {
-
-using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind;
-
-vk::ShaderStageFlags StageFlags(DescriptorCache::Stage stage) {
-	switch (stage) {
-		case DescriptorCache::Stage::Vertex: return vk::ShaderStageFlagBits::eVertex;
-		case DescriptorCache::Stage::Pixel: return vk::ShaderStageFlagBits::eFragment;
-		case DescriptorCache::Stage::Compute: return vk::ShaderStageFlagBits::eCompute;
-		default: EXIT("unknown descriptor stage\n");
-	}
-}
-
-bool IsSampledImage(BindingKind kind) {
-	switch (kind) {
-		case BindingKind::Sampled1D:
-		case BindingKind::Sampled1DArray:
-		case BindingKind::Sampled2D:
-		case BindingKind::Sampled2DArray:
-		case BindingKind::Sampled2DMsaa:
-		case BindingKind::Sampled2DMsaaArray:
-		case BindingKind::Sampled3D:
-		case BindingKind::SampledUint1D:
-		case BindingKind::SampledUint1DArray:
-		case BindingKind::SampledUint2D:
-		case BindingKind::SampledUint2DArray:
-		case BindingKind::SampledUint2DMsaa:
-		case BindingKind::SampledUint2DMsaaArray:
-		case BindingKind::SampledUint3D: return true;
-		default: return false;
-	}
-}
-
-bool IsStorageImage(BindingKind kind) {
-	switch (kind) {
-		case BindingKind::Storage1D:
-		case BindingKind::Storage1DArray:
-		case BindingKind::Storage2D:
-		case BindingKind::Storage2DArray:
-		case BindingKind::Storage3D:
-		case BindingKind::StorageUint1D:
-		case BindingKind::StorageUint1DArray:
-		case BindingKind::StorageUint2D:
-		case BindingKind::StorageUint2DArray:
-		case BindingKind::StorageUint3D: return true;
-		default: return false;
-	}
-}
-
-vk::DescriptorType DescriptorType(BindingKind kind) {
-	if (kind == BindingKind::Samplers) {
-		return vk::DescriptorType::eSampler;
-	}
-	if (IsSampledImage(kind)) {
-		return vk::DescriptorType::eSampledImage;
-	}
-	if (IsStorageImage(kind)) {
-		return vk::DescriptorType::eStorageImage;
-	}
-	return vk::DescriptorType::eStorageBuffer;
-}
-
-uint32_t DescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
-	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
-}
-
-std::vector<uint32_t> LayoutKey(DescriptorCache::Stage               stage,
-                                const ShaderRecompiler::IR::Program& program) {
-	std::vector<uint32_t> key;
-	key.reserve(1u + program.bindings.descriptors.size() * 4u);
-	key.push_back(static_cast<uint32_t>(stage));
-	for (const auto& binding: program.bindings.descriptors) {
-		key.push_back(static_cast<uint32_t>(binding.kind));
-		key.push_back(binding.binding);
-		key.push_back(DescriptorCount(binding));
-		key.push_back(static_cast<uint32_t>(DescriptorType(binding.kind)));
-	}
-	return key;
-}
-
-vk::DescriptorBufferInfo BufferInfo(const BufferView& view) {
-	EXIT_IF(view.buffer == nullptr);
-	return {view.buffer, view.offset, view.range};
-}
-
-} // namespace
-
-vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture,
-                                                       uint32_t              element) {
-	const auto view =
-	    texture.mip_views.empty()
-	        ? (element == 0u ? texture.image_view : vk::ImageView {})
-	        : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {});
-	EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
-	return {nullptr, view, texture.layout};
-}
-
-DescriptorCache::~DescriptorCache() {
-	for (auto& [layout, sets]: m_free_sets_by_layout) {
-		(void)layout;
-		for (auto* set: sets) {
-			delete set;
-		}
-	}
-	for (const auto& [key, layout]: m_descriptor_set_layouts) {
-		(void)key;
-		m_graphics.device.destroyDescriptorSetLayout(layout, nullptr);
-	}
-	for (const auto& pool: m_pools) {
-		m_graphics.device.destroyDescriptorPool(pool.pool, nullptr);
-	}
-}
-
-vk::DescriptorSetLayout
-DescriptorCache::GetDescriptorSetLayoutInternal(Stage                                stage,
-                                                const ShaderRecompiler::IR::Program& program) {
-	const auto key = LayoutKey(stage, program);
-	if (const auto found = m_descriptor_set_layouts.find(key);
-	    found != m_descriptor_set_layouts.end()) {
-		return found->second;
-	}
-
-	std::vector<vk::DescriptorSetLayoutBinding> bindings;
-	bindings.reserve(program.bindings.descriptors.size());
-	for (const auto& descriptor: program.bindings.descriptors) {
-		bindings.push_back({descriptor.binding, DescriptorType(descriptor.kind),
-		                    DescriptorCount(descriptor), StageFlags(stage), nullptr});
-	}
-	if (bindings.empty()) {
-		return nullptr;
-	}
-
-	vk::DescriptorSetLayoutCreateInfo info {};
-	info.sType                     = vk::StructureType::eDescriptorSetLayoutCreateInfo;
-	info.bindingCount              = static_cast<uint32_t>(bindings.size());
-	info.pBindings                 = bindings.data();
-	vk::DescriptorSetLayout layout = nullptr;
-	const auto result = m_graphics.device.createDescriptorSetLayout(&info, nullptr, &layout);
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || layout == nullptr);
-	m_descriptor_set_layouts.emplace(key, layout);
-	return layout;
-}
-
-void DescriptorCache::CreatePool() {
-	KYTY_PROFILER_FUNCTION();
-	constexpr uint32_t           MaxSets = 512;
-	const vk::DescriptorPoolSize sizes[] = {
-	    {vk::DescriptorType::eStorageBuffer,
-	     MaxSets * (ShaderRecompiler::IR::ShaderInfo::MaxBuffers +
-	                ShaderRecompiler::IR::ShaderInfo::MaxAddresses + 3u)},
-	    {vk::DescriptorType::eSampledImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
-	    {vk::DescriptorType::eStorageImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
-	    {vk::DescriptorType::eSampler, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxSamplers},
-	};
-	vk::DescriptorPoolCreateInfo info {};
-	info.sType         = vk::StructureType::eDescriptorPoolCreateInfo;
-	info.poolSizeCount = static_cast<uint32_t>(std::size(sizes));
-	info.pPoolSizes    = sizes;
-	info.maxSets       = MaxSets;
-	const auto pool_id = static_cast<int>(m_pools.size());
-	auto&      pool    = m_pools.emplace_back();
-	const auto result  = m_graphics.device.createDescriptorPool(&info, nullptr, &pool.pool);
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || pool.pool == nullptr);
-	pool.next_free_pool = m_first_free_pool;
-	m_first_free_pool   = pool_id;
-}
-
-VulkanDescriptorSet* DescriptorCache::Allocate(Stage                                stage,
-                                               const ShaderRecompiler::IR::Program& program) {
-	KYTY_PROFILER_FUNCTION();
-	Common::LockGuard lock(m_mutex);
-	const auto        layout = GetDescriptorSetLayoutInternal(stage, program);
-	EXIT_IF(layout == nullptr);
-	auto& free_sets = m_free_sets_by_layout[layout];
-	if (!free_sets.empty()) {
-		auto* result = free_sets.back();
-		free_sets.pop_back();
-		return result;
-	}
-
-	constexpr uint32_t Batch = 32;
-	for (int attempt = 0; attempt < 2; attempt++) {
-		for (int pool_id = m_first_free_pool; pool_id != -1;
-		     pool_id     = m_pools[pool_id].next_free_pool) {
-			auto&                                      pool = m_pools[pool_id];
-			std::array<vk::DescriptorSetLayout, Batch> layouts;
-			layouts.fill(layout);
-			std::array<vk::DescriptorSet, Batch> sets {};
-			vk::DescriptorSetAllocateInfo        info {};
-			info.sType              = vk::StructureType::eDescriptorSetAllocateInfo;
-			info.descriptorPool     = pool.pool;
-			info.descriptorSetCount = Batch;
-			info.pSetLayouts        = layouts.data();
-			if (m_graphics.device.allocateDescriptorSets(&info, sets.data()) ==
-			    vk::Result::eSuccess) {
-				free_sets.reserve(free_sets.size() + Batch - 1u);
-				for (uint32_t i = 0; i + 1u < Batch; i++) {
-					free_sets.push_back(new VulkanDescriptorSet {sets[i], layout, pool_id});
-				}
-				return new VulkanDescriptorSet {sets.back(), layout, pool_id};
-			}
-			m_first_free_pool = pool.next_free_pool;
-			break;
-		}
-		CreatePool();
-	}
-	return nullptr;
-}
-
-void DescriptorCache::Recycle(VulkanDescriptorSet& set) {
-	EXIT_IF(set.set == nullptr || set.layout == nullptr);
-	Common::LockGuard lock(m_mutex);
-	m_free_sets_by_layout[set.layout].push_back(&set);
-}
-
-VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage                                stage,
-                                                    const ShaderRecompiler::IR::Program& program,
-                                                    const NativeDescriptors&             data) {
-	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(data.buffers.size() != program.info.buffers.size() ||
-	        data.images.size() != program.info.images.size() ||
-	        data.samplers.size() != program.info.samplers.size() ||
-	        data.addresses.size() != program.info.addresses.size());
-	auto* set = Allocate(stage, program);
-	EXIT_NOT_IMPLEMENTED(set == nullptr);
-
-	size_t descriptor_count = 0;
-	for (const auto& binding: program.bindings.descriptors) {
-		descriptor_count += DescriptorCount(binding);
-	}
-	std::vector<vk::DescriptorBufferInfo> buffer_infos;
-	std::vector<vk::DescriptorImageInfo>  image_infos;
-	std::vector<vk::WriteDescriptorSet>   writes;
-	buffer_infos.reserve(descriptor_count);
-	image_infos.reserve(descriptor_count);
-	writes.reserve(program.bindings.descriptors.size());
-	std::vector<uint32_t> image_occurrences(data.images.size());
-
-	for (const auto& binding: program.bindings.descriptors) {
-		vk::WriteDescriptorSet write {};
-		write.sType             = vk::StructureType::eWriteDescriptorSet;
-		write.dstSet            = set->set;
-		write.dstBinding        = binding.binding;
-		write.descriptorType    = DescriptorType(binding.kind);
-		write.descriptorCount   = DescriptorCount(binding);
-		const auto buffer_start = buffer_infos.size();
-		const auto image_start  = image_infos.size();
-		switch (binding.kind) {
-			case BindingKind::Buffers:
-				for (const auto resource: binding.resources) {
-					buffer_infos.push_back(BufferInfo(data.buffers.at(resource)));
-				}
-				break;
-			case BindingKind::AddressMemory:
-				for (const auto resource: binding.resources) {
-					buffer_infos.push_back(BufferInfo(data.addresses.at(resource)));
-				}
-				break;
-			case BindingKind::FlattenedSrt:
-				buffer_infos.push_back(BufferInfo(data.flattened_srt));
-				break;
-			case BindingKind::UserData: buffer_infos.push_back(BufferInfo(data.user_data)); break;
-			case BindingKind::Gds: buffer_infos.push_back(BufferInfo(data.gds)); break;
-			case BindingKind::Samplers:
-				for (const auto resource: binding.resources) {
-					const auto sampler = data.samplers.at(resource);
-					EXIT_IF(sampler == nullptr);
-					image_infos.push_back({sampler, nullptr, vk::ImageLayout::eUndefined});
-				}
-				break;
-			default: {
-				for (const auto resource: binding.resources) {
-					const auto& texture = data.images.at(resource);
-					image_infos.push_back(MakeImageInfo(texture, image_occurrences.at(resource)++));
-				}
-				break;
-			}
-		}
-		if (buffer_infos.size() != buffer_start) {
-			write.pBufferInfo = buffer_infos.data() + buffer_start;
-		}
-		if (image_infos.size() != image_start) {
-			write.pImageInfo = image_infos.data() + image_start;
-		}
-		writes.push_back(write);
-	}
-	for (uint32_t i = 0; i < data.images.size(); i++) {
-		const auto expected = data.images[i].mip_views.empty()
-		                          ? 1u
-		                          : static_cast<uint32_t>(data.images[i].mip_views.size());
-		EXIT_IF(image_occurrences[i] != expected);
-	}
-	m_graphics.device.updateDescriptorSets(static_cast<uint32_t>(writes.size()), writes.data(), 0,
-	                                       nullptr);
-	return *set;
-}
-
-vk::DescriptorSetLayout
-DescriptorCache::GetDescriptorSetLayout(Stage stage, const ShaderRecompiler::IR::Program& program) {
-	Common::LockGuard lock(m_mutex);
-	return GetDescriptorSetLayoutInternal(stage, program);
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptorCache.h

@@ -1,116 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_
-
-#include "common/abi.h"
-#include "common/assert.h"
-#include "common/common.h"
-#include "common/threads.h"
-#include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/cache/textureCache.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/shaderBindings.h"
-
-#include <map>
-#include <memory>
-#include <span>
-#include <unordered_map>
-#include <vector>
-
-namespace Libs::Graphics {
-
-namespace ShaderRecompiler::IR {
-struct DescriptorValue;
-struct ImageResource;
-struct Program;
-struct ResourceSnapshot;
-} // namespace ShaderRecompiler::IR
-
-class CommandBuffer;
-struct DescriptorCacheTestAccess;
-struct ShaderStageRuntime;
-
-struct VulkanDescriptorSet {
-	vk::DescriptorSet       set     = nullptr;
-	vk::DescriptorSetLayout layout  = nullptr;
-	int                     pool_id = -1;
-};
-
-struct BufferView {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	vk::DeviceSize        offset = 0;
-	vk::DeviceSize        range  = VK_WHOLE_SIZE;
-};
-
-class DescriptorCache {
-public:
-	enum class Stage { Unknown, Vertex, Pixel, Compute };
-
-	struct TextureBinding {
-		ImageId                    image_id;
-		vk::ImageView              image_view = nullptr;
-		TextureCache::ImageDesc    desc;
-		vk::ImageLayout            layout = vk::ImageLayout::eUndefined;
-		std::vector<vk::ImageView> mip_views;
-	};
-
-	struct NativeDescriptors {
-		std::vector<BufferView>     buffers;
-		std::vector<TextureBinding> images;
-		std::vector<vk::Sampler>    samplers;
-		std::vector<BufferView>     addresses;
-		BufferView                  gds;
-		BufferView                  flattened_srt;
-		BufferView                  user_data;
-	};
-
-	struct PreparedBindings {
-		std::shared_ptr<const ShaderRecompiler::IR::Program>          program;
-		std::shared_ptr<const ShaderRecompiler::IR::ResourceSnapshot> snapshot;
-		NativeDescriptors                                             resources;
-		std::vector<uint32_t>                                         flattened_srt;
-		std::vector<uint32_t>                                         user_data;
-		vk::ShaderStageFlags                                          shader_stage;
-		Stage                                                         stage     = Stage::Unknown;
-		bool                                                          committed = false;
-	};
-
-	explicit DescriptorCache(GraphicContext& graphics): m_graphics(graphics) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	}
-	~DescriptorCache();
-	KYTY_CLASS_NO_COPY(DescriptorCache);
-
-	vk::DescriptorSetLayout GetDescriptorSetLayout(Stage                                stage,
-	                                               const ShaderRecompiler::IR::Program& program);
-	void                    Recycle(VulkanDescriptorSet& set);
-	VulkanDescriptorSet&    GetDescriptor(Stage stage, const ShaderRecompiler::IR::Program& program,
-	                                      const NativeDescriptors& descriptors);
-
-private:
-	friend struct DescriptorCacheTestAccess;
-
-	struct Pool {
-		vk::DescriptorPool pool           = nullptr;
-		int                next_free_pool = -1;
-	};
-
-	static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
-	                                             uint32_t              element = 0);
-	void                           CreatePool();
-	VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program);
-	vk::DescriptorSetLayout
-	GetDescriptorSetLayoutInternal(Stage stage, const ShaderRecompiler::IR::Program& program);
-
-	GraphicContext&   m_graphics;
-	Common::Mutex     m_mutex;
-	std::vector<Pool> m_pools;
-	int               m_first_free_pool = -1;
-	std::unordered_map<vk::DescriptorSetLayout, std::vector<VulkanDescriptorSet*>>
-	                                                         m_free_sets_by_layout;
-	std::map<std::vector<uint32_t>, vk::DescriptorSetLayout> m_descriptor_set_layouts;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp

@@ -0,0 +1,97 @@
+#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
+
+#include "common/assert.h"
+#include "common/profiler.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/masterSemaphore.h"
+
+namespace Libs::Graphics {
+namespace {
+
+constexpr uint32_t   DescriptorHeapCount = 1024;
+constexpr std::array DescriptorPoolSizes = {
+    vk::DescriptorPoolSize {vk::DescriptorType::eStorageBuffer, 8192},
+    vk::DescriptorPoolSize {vk::DescriptorType::eSampledImage, 8192},
+    vk::DescriptorPoolSize {vk::DescriptorType::eStorageImage, 1024},
+    vk::DescriptorPoolSize {vk::DescriptorType::eSampler, 1024},
+};
+
+} // namespace
+
+DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore)
+    : m_graphics(graphics), m_master_semaphore(master_semaphore) {
+	CreatePool();
+}
+
+DescriptorHeap::~DescriptorHeap() {
+	m_graphics.device.destroyDescriptorPool(m_current_pool, nullptr);
+	for (const auto& [pool, tick]: m_pending_pools) {
+		m_master_semaphore.Wait(tick);
+		m_graphics.device.destroyDescriptorPool(pool, nullptr);
+	}
+}
+
+vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) {
+	KYTY_PROFILER_FUNCTION();
+	EXIT_IF(layout == nullptr);
+
+	auto& batch = m_sets[layout];
+	if (batch.size != 0) {
+		return batch.sets[--batch.size];
+	}
+	if (Allocate(layout, batch)) {
+		return batch.sets[--batch.size];
+	}
+
+	m_pending_pools.emplace_back(m_current_pool, m_master_semaphore.CurrentTick());
+	if (const auto& [pool, tick] = m_pending_pools.front(); m_master_semaphore.IsFree(tick)) {
+		m_current_pool = pool;
+		m_pending_pools.pop_front();
+		EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess);
+	} else {
+		CreatePool();
+	}
+
+	m_sets.clear();
+	auto& fresh_batch = m_sets[layout];
+	EXIT_IF(!Allocate(layout, fresh_batch));
+	return fresh_batch.sets[--fresh_batch.size];
+}
+
+bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) {
+	std::array<vk::DescriptorSetLayout, DescriptorSetBatch> layouts;
+	layouts.fill(layout);
+
+	vk::DescriptorSetAllocateInfo allocate {};
+	allocate.sType          = vk::StructureType::eDescriptorSetAllocateInfo;
+	allocate.descriptorPool = m_current_pool;
+	allocate.pSetLayouts    = layouts.data();
+
+	for (;;) {
+		allocate.descriptorSetCount = batch.allocation;
+		const auto result = m_graphics.device.allocateDescriptorSets(&allocate, batch.sets.data());
+		if (result == vk::Result::eSuccess) {
+			batch.size = batch.allocation;
+			return true;
+		}
+		EXIT_IF(result != vk::Result::eErrorOutOfPoolMemory &&
+		        result != vk::Result::eErrorFragmentedPool);
+		if (batch.allocation == 1) {
+			return false;
+		}
+		batch.allocation /= 2;
+	}
+}
+
+void DescriptorHeap::CreatePool() {
+	vk::DescriptorPoolCreateInfo create {};
+	create.sType         = vk::StructureType::eDescriptorPoolCreateInfo;
+	create.flags         = {};
+	create.maxSets       = DescriptorHeapCount;
+	create.poolSizeCount = static_cast<uint32_t>(DescriptorPoolSizes.size());
+	create.pPoolSizes    = DescriptorPoolSizes.data();
+	EXIT_IF(m_graphics.device.createDescriptorPool(&create, nullptr, &m_current_pool) !=
+	        vk::Result::eSuccess);
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h

@@ -0,0 +1,45 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_
+
+#include "common/common.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+
+#include <array>
+#include <deque>
+#include <unordered_map>
+
+namespace Libs::Graphics {
+
+struct GraphicContext;
+class MasterSemaphore;
+
+class DescriptorHeap {
+public:
+	DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore);
+	~DescriptorHeap();
+	KYTY_CLASS_NO_COPY(DescriptorHeap);
+
+	[[nodiscard]] vk::DescriptorSet Commit(vk::DescriptorSetLayout layout);
+
+private:
+	static constexpr uint32_t DescriptorSetBatch = 32;
+
+	struct Batch {
+		std::array<vk::DescriptorSet, DescriptorSetBatch> sets {};
+		uint32_t                                          size       = 0;
+		uint32_t                                          allocation = DescriptorSetBatch;
+	};
+
+	[[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch);
+	void               CreatePool();
+
+	GraphicContext&                                     m_graphics;
+	MasterSemaphore&                                    m_master_semaphore;
+	vk::DescriptorPool                                  m_current_pool = nullptr;
+	std::deque<std::pair<vk::DescriptorPool, uint64_t>> m_pending_pools;
+	std::unordered_map<vk::DescriptorSetLayout, Batch>  m_sets;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -17,7 +17,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
@@ -45,10 +44,99 @@
 
 namespace Libs::Graphics {
 
+namespace {
+
+using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind;
+
+bool IsSampledImage(BindingKind kind) {
+	switch (kind) {
+		case BindingKind::Sampled1D:
+		case BindingKind::Sampled1DArray:
+		case BindingKind::Sampled2D:
+		case BindingKind::Sampled2DArray:
+		case BindingKind::Sampled2DMsaa:
+		case BindingKind::Sampled2DMsaaArray:
+		case BindingKind::Sampled3D:
+		case BindingKind::SampledUint1D:
+		case BindingKind::SampledUint1DArray:
+		case BindingKind::SampledUint2D:
+		case BindingKind::SampledUint2DArray:
+		case BindingKind::SampledUint2DMsaa:
+		case BindingKind::SampledUint2DMsaaArray:
+		case BindingKind::SampledUint3D: return true;
+		default: return false;
+	}
+}
+
+bool IsStorageImage(BindingKind kind) {
+	switch (kind) {
+		case BindingKind::Storage1D:
+		case BindingKind::Storage1DArray:
+		case BindingKind::Storage2D:
+		case BindingKind::Storage2DArray:
+		case BindingKind::Storage3D:
+		case BindingKind::StorageUint1D:
+		case BindingKind::StorageUint1DArray:
+		case BindingKind::StorageUint2D:
+		case BindingKind::StorageUint2DArray:
+		case BindingKind::StorageUint3D:
+		case BindingKind::StorageAtomic1D:
+		case BindingKind::StorageAtomic1DArray:
+		case BindingKind::StorageAtomic2D:
+		case BindingKind::StorageAtomic2DArray:
+		case BindingKind::StorageAtomic3D: return true;
+		default: return false;
+	}
+}
+
+} // namespace
+
+vk::DescriptorType NativeDescriptorType(BindingKind kind) {
+	if (kind == BindingKind::Samplers) {
+		return vk::DescriptorType::eSampler;
+	}
+	if (IsSampledImage(kind)) {
+		return vk::DescriptorType::eSampledImage;
+	}
+	if (IsStorageImage(kind)) {
+		return vk::DescriptorType::eStorageImage;
+	}
+	return vk::DescriptorType::eStorageBuffer;
+}
+
+uint32_t NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
+	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
+}
+
+vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element) {
+	const auto view =
+	    texture.mip_views.empty()
+	        ? (element == 0u ? texture.image_view : vk::ImageView {})
+	        : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {});
+	EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
+	return {nullptr, view, texture.layout};
+}
+
+static const char* ShaderStageResourceName(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return "Vertex";
+		case ShaderType::Pixel: return "Pixel";
+		case ShaderType::Compute: return "Compute";
+		default: return "Unknown";
+	}
+}
+
+static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
+		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
+		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
+		default: EXIT("unknown native shader stage\n");
+	}
+}
+
 static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) {
-	auto owner = context.GetBufferCache().ObtainNullBuffer();
-	dst.buffer = owner->Handle();
-	dst.owner  = std::move(owner);
+	dst.buffer = context.GetBufferCache().GetBuffer(NULL_BUFFER_ID).Handle();
 	dst.offset = 0;
 	dst.range  = 16;
 }
@@ -79,10 +167,11 @@ static bool IsMultisampledTexture(Prospero::ImageType type) {
 	       type == Prospero::ImageType::kColor2DMsaaArray;
 }
 
-static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& command_buffer,
+static BufferView NativeStorageBuffer(RenderContext&                              context,
                                       const ShaderBufferResource&                 descriptor,
                                       const ShaderRecompiler::IR::BufferResource& resource,
-                                      uint32_t&                                   buffer_offset) {
+                                      ShaderType stage, uint32_t slot, uint32_t& buffer_offset,
+                                      BufferId id) {
 	BufferView result;
 	buffer_offset = 0;
 
@@ -104,27 +193,35 @@ static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& com
 	    size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange) {
 		EXIT("storage buffer range or device alignment is unsupported\n");
 	}
-	auto binding = context.GetBufferCache().ObtainBuffer(
-	    command_buffer, address, size, resource.written, resource.read, resource.formatted);
-	const auto aligned_offset = binding.offset - binding.offset % alignment;
-	const auto adjustment     = binding.offset - aligned_offset;
+	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
+	                                                              resource.formatted, id);
+	const auto aligned_offset = offset - offset % alignment;
+	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
 	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
 		EXIT("storage buffer offset adjustment is unsupported\n");
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
-	result.owner  = std::move(binding.owner);
-	result.buffer = binding.buffer;
+	result.buffer = buffer->Handle();
 	result.offset = aligned_offset;
 	result.range  = static_cast<vk::DeviceSize>(size + adjustment);
+	if (resource.formatted && resource.written) {
+		context.GetTextureCache().InvalidateMemoryFromGPU(address, size);
+	}
+	SetVulkanObjectNameF(
+	    graphics.device, result.buffer,
+	    "Kyty.{}.StorageBuffer[slot={} guest=0x{:016x} size=0x{:x} access={} formatted={}]",
+	    ShaderStageResourceName(stage), slot, address, size,
+	    resource.written ? (resource.read ? "ReadWrite" : "Write") : "Read", resource.formatted);
 	return result;
 }
 
 static BufferView
-NativeAddressBuffer(RenderContext& context, CommandBuffer& command_buffer,
-                    const ShaderRecompiler::IR::AddressResource&           resource,
-                    const ShaderRecompiler::IR::ResourceSnapshot::Address& address) {
+NativeAddressBuffer(RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource,
+                    const ShaderRecompiler::IR::ResourceSnapshot::Address& address,
+                    ShaderType stage, uint32_t slot, uint32_t& address_offset, BufferId id) {
 	BufferView result;
+	address_offset = 0;
 	if (address.binding_base == 0) {
 		BindNullStorageBuffer(context, result);
 		return result;
@@ -145,17 +242,19 @@ NativeAddressBuffer(RenderContext& context, CommandBuffer& command_buffer,
 	}
 	const auto& graphics  = context.GetGraphics();
 	const auto  alignment = graphics.StorageMinAlignment();
-	if (alignment == 0 ||
-	    size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange ||
-	    BufferCache::GetBufferOffset(address.binding_base) % alignment != 0) {
-		EXIT("address resource range or alignment is unsupported\n");
-	}
-	auto binding =
-	    context.GetBufferCache().ObtainBuffer(command_buffer, address.binding_base, size);
-	result.owner  = std::move(binding.owner);
-	result.buffer = binding.buffer;
-	result.offset = binding.offset;
-	result.range  = static_cast<vk::DeviceSize>(size);
+	auto [buffer, offset] =
+	    context.GetBufferCache().ObtainBuffer(address.binding_base, size, false, false, id);
+	const auto aligned_offset = offset - offset % alignment;
+	const auto adjustment     = offset - aligned_offset;
+	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
+	const auto visible_size   = std::min(size, static_cast<uint64_t>(max_range - adjustment));
+	address_offset            = static_cast<uint32_t>(adjustment);
+	result.buffer             = buffer->Handle();
+	result.offset             = aligned_offset;
+	result.range              = static_cast<vk::DeviceSize>(visible_size + adjustment);
+	SetVulkanObjectNameF(graphics.device, result.buffer,
+	                     "Kyty.{}.AddressBuffer[slot={} guest=0x{:016x} size=0x{:x}]",
+	                     ShaderStageResourceName(stage), slot, address.binding_base, visible_size);
 	return result;
 }
 
@@ -233,7 +332,8 @@ bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& r
 	       descriptor.BaseArray5() == 0;
 }
 
-bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image) {
+bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image,
+                                      bool r128) {
 	const auto width        = static_cast<uint32_t>(descriptor.Width5()) + 1u;
 	const auto height       = static_cast<uint32_t>(descriptor.Height5()) + 1u;
 	const auto type         = descriptor.Type();
@@ -260,12 +360,12 @@ bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, c
 	                                  descriptor.BaseArray5() <= descriptor.Depth() &&
 	                                  descriptor.Depth() < image.info.resources.layers;
 	const bool levels_ok =
-	    multisampled
-	        ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
-	              descriptor.LastLevel() <= 3 && descriptor.MaxMip() == descriptor.LastLevel() &&
-	              image.info.resources.levels == 1 && image.info.samples == samples
-	        : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
-	              descriptor.MaxMip() == 0 && image.info.samples == 1;
+	    multisampled ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
+	                       descriptor.LastLevel() <= 3 &&
+	                       (r128 || descriptor.MaxMip() == descriptor.LastLevel()) &&
+	                       image.info.resources.levels == 1 && image.info.samples == samples
+	                 : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
+	                       (r128 || descriptor.MaxMip() == 0) && image.info.samples == 1;
 	return image.info.IsDepth() && width == image.info.extent.width &&
 	       height == image.info.extent.height &&
 	       (supported_2d || supported_array || supported_cube || supported_msaa_2d ||
@@ -275,7 +375,8 @@ bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, c
 	       (!descriptor.MsaaDepth() || multisampled) && pitch >= width && pitch == image.info.pitch;
 }
 
-bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image) {
+bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image,
+                                     bool r128) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
 	const uint32_t     field3_expected = descriptor.DstSelXYZW() |
@@ -288,10 +389,14 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 	                                     (static_cast<uint32_t>(descriptor.MaxMip()) << 4u);
 	const bool         common          = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
 	                                     (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	                                     descriptor.fields[3] == field3_expected &&
-	                                     descriptor.fields[4] == field4_expected &&
-	                                     descriptor.fields[5] == field5_expected;
-	if (!common || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
+	                                     descriptor.fields[3] == field3_expected;
+	if (r128) {
+		return common && descriptor.fields[4] == 0 && descriptor.fields[5] == 0 &&
+		       descriptor.fields[6] == 0 && descriptor.fields[7] == 0;
+	}
+	const bool full = common && descriptor.fields[4] == field4_expected &&
+	                  descriptor.fields[5] == field5_expected;
+	if (!full || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
 		return false;
 	}
 	if (descriptor.fields[6] == 0) {
@@ -312,9 +417,9 @@ static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource
                                        vk::Format view_format, uint64_t size) {
 	const bool resource_ok = IsSupportedSampledDepthResource(resource);
 	const bool descriptor_ok =
-	    image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image);
+	    image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image, resource.r128);
 	const bool encoding_ok =
-	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image);
+	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image, resource.r128);
 	const bool format_ok =
 	    image != nullptr && IsSupportedSampledDepthFormat(image->info.pixel_format, view_format);
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
@@ -404,17 +509,19 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_swizzle =
 	    IsValidImageSwizzle(swizzle) &&
 	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
+	const auto max_mip = resource.r128 ? descriptor.LastLevel() : descriptor.MaxMip();
 	const auto view_last_level =
 	    resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage
 	        ? descriptor.LastLevel()
-	        : std::min(descriptor.LastLevel(), descriptor.MaxMip());
+	        : std::min(descriptor.LastLevel(), max_mip);
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       descriptor.BaseLevel() <= view_last_level && view_last_level <= descriptor.MaxMip() &&
+	       descriptor.BaseLevel() <= view_last_level && view_last_level <= max_mip &&
 	       descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
 	       !descriptor.MsaaDepth();
 }
 
-static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descriptor) {
+static bool IsSupportedStorageTextureEncoding(const ShaderRecompiler::IR::ImageResource& resource,
+                                              const ShaderTextureResource& descriptor) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
 	constexpr uint32_t field5_expected      = 0x00700000u;
@@ -426,9 +533,14 @@ static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descr
 	                                     (static_cast<uint32_t>(descriptor.Type()) << 28u);
 	const uint32_t     expected_field4 =
 	    descriptor.Depth() | (static_cast<uint32_t>(descriptor.BaseArray5()) << 16u);
-	return (descriptor.fields[1] & field1_reserved_mask) == 0 &&
-	       (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	       descriptor.fields[3] == expected_field3 && descriptor.fields[4] == expected_field4 &&
+	const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
+	                    (descriptor.fields[2] & field2_reserved_mask) == 0 &&
+	                    descriptor.fields[3] == expected_field3;
+	if (resource.r128) {
+		return common && descriptor.fields[4] == 0 && descriptor.fields[5] == 0 &&
+		       descriptor.fields[6] == 0 && descriptor.fields[7] == 0;
+	}
+	return common && descriptor.fields[4] == expected_field4 &&
 	       (descriptor.fields[5] & ~field5_max_mip_mask) == field5_expected;
 }
 
@@ -437,13 +549,13 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	const auto format        = descriptor.Format();
 	const bool resource_ok   = IsSupportedStorageImageResource(resource);
 	const bool descriptor_ok = IsSupportedStorageTextureDescriptor(resource, descriptor);
-	const bool encoding_ok   = IsSupportedStorageTextureEncoding(descriptor);
+	const bool encoding_ok   = IsSupportedStorageTextureEncoding(resource, descriptor);
 	const bool uint_resource =
 	    resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
 	const bool raw_sint_storage = format == Prospero::BufferFormat::k32SInt && uint_resource &&
 	                              resource.written && !resource.read && !resource.atomic;
 	const bool format_ok =
-	    raw_sint_storage || (Prospero::IsSupportedTextureFormat(format) &&
+	    raw_sint_storage || (Prospero::IsSampledTextureFormat(format) &&
 	                         uint_resource == Prospero::IsUintTextureFormat(format) &&
 	                         (!resource.atomic || format == Prospero::BufferFormat::k32UInt));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
@@ -552,15 +664,17 @@ static void PopulateTextureMipLayout(ImageInfo& info) {
 
 static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource,
                                      const ShaderTextureResource& descriptor, vk::Format format,
-                                     bool storage, uint32_t view_levels, uint32_t image_layers) {
+                                     bool shader_conversion, bool storage, uint32_t view_levels,
+                                     uint32_t image_layers) {
 	ImageViewInfo view {};
 	view.format      = format;
 	view.aspect      = vk::ImageAspectFlagBits::eColor;
 	view.base_level  = descriptor.BaseLevel();
 	view.level_count = view_levels;
-	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
-	view.mapping =
-	    storage ? vk::ComponentMapping {} : TextureGetComponentMapping(descriptor.DstSelXYZW());
+	view.usage   = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
+	view.mapping = storage || shader_conversion
+	                   ? vk::ComponentMapping {}
+	                   : TextureGetComponentMapping(descriptor.DstSelXYZW());
 	switch (resource.dimension) {
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
 			view.type       = vk::ImageViewType::e1D;
@@ -606,9 +720,8 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
 	return view;
 }
 
-DescriptorCache::TextureBinding
-RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
-                               const ShaderRecompiler::IR::DescriptorValue& value) {
+TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
+                                              const ShaderRecompiler::IR::DescriptorValue& value) {
 	ShaderTextureResource descriptor;
 	CopyNativeDescriptor(value, descriptor.fields);
 	const bool storage = resource.written;
@@ -631,20 +744,20 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto last_level   = descriptor.LastLevel();
 	const auto type         = TextureType(descriptor);
 	const bool multisampled = IsMultisampledTexture(type);
-	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
+	const auto max_mip      = resource.r128 ? last_level : descriptor.MaxMip();
+	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(max_mip) + 1u;
 	const bool dynamic_storage =
 	    storage && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
 	const auto view_last_level =
-	    !multisampled && !dynamic_storage ? std::min(last_level, descriptor.MaxMip()) : last_level;
+	    !multisampled && !dynamic_storage ? std::min(last_level, max_mip) : last_level;
 	const auto tile       = descriptor.TileMode();
 	const bool depth_tile = tile == Prospero::TileMode::kDepth;
 	const bool msaa_tile  = depth_tile || tile == Prospero::TileMode::kRenderTarget;
 	const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
 	if ((!multisampled && (base_level > view_last_level || view_last_level >= levels)) ||
 	    (multisampled &&
-	     (base_level != 0 || last_level == 0 || last_level > 3 ||
-	      descriptor.MaxMip() != last_level || !msaa_tile ||
-	      (descriptor.MsaaDepth() && !depth_tile) ||
+	     (base_level != 0 || last_level == 0 || last_level > 3 || max_mip != last_level ||
+	      !msaa_tile || (descriptor.MsaaDepth() && !depth_tile) ||
 	      (!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
 		EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
 		     "kind=%u dimension=%u mip_mode=%u read=%d written=%d "
@@ -660,11 +773,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto view_levels =
 	    multisampled ? 1u : static_cast<uint32_t>(view_last_level - base_level) + 1u;
-	const auto depth  = static_cast<uint32_t>(descriptor.Depth()) + 1u;
-	const auto format = descriptor.Format();
+	const auto depth          = static_cast<uint32_t>(descriptor.Depth()) + 1u;
+	const auto format         = descriptor.Format();
+	const auto surface_format = TextureGetSurfaceFormatInfo(format);
+	const bool shader_conversion =
+	    surface_format.conversion_format != Prospero::BufferFormat::kInvalid;
 	const bool sampled_numeric_class =
-	    storage || ((resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) ==
-	                Prospero::IsUintTextureFormat(format));
+	    storage || (Prospero::IsSampledTextureFormat(format) &&
+	                (resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) ==
+	                    Prospero::IsUintTextureFormat(format));
 	if (!storage &&
 	    (resource.kind == ShaderRecompiler::IR::ResourceKind::Image ||
 	     resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) &&
@@ -700,7 +817,7 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 		ValidateStorageTexture(resource, descriptor, size.size);
 	}
 
-	const auto pixel_format        = TextureGetFormat(format);
+	const auto pixel_format        = surface_format.vk_format;
 	const auto storage_view_format = storage && format == Prospero::BufferFormat::k32SInt
 	                                     ? vk::Format::eR32Uint
 	                                     : SrgbStorageViewFormat(pixel_format);
@@ -725,11 +842,11 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	} else {
 		PopulateTextureMipLayout(desc.info);
 	}
-	desc.view_info = TextureViewInfo(resource, descriptor, view_format, storage, view_levels,
-	                                 desc.info.resources.layers);
+	desc.view_info = TextureViewInfo(resource, descriptor, view_format, shader_conversion, storage,
+	                                 view_levels, desc.info.resources.layers);
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
 
-	auto       id                  = texture_cache.FindImage(desc);
+	auto       id                  = texture_cache.FindImage(desc, shader_conversion);
 	auto*      image               = &texture_cache.GetImage(id);
 	const bool stencil_association = static_cast<bool>(image->depth_id);
 	if (stencil_association) {
@@ -765,25 +882,25 @@ static vk::Sampler NativeSampler(RenderContext&                       context,
 	if (!depth_compare) {
 		descriptor.fields[0] &= ~(0x7u << 12u);
 	}
+	if (program.info.samplers[index].force_point_filtering) {
+		descriptor.SetPointFiltering();
+	}
 	return context.GetSamplerCache().GetSampler(descriptor);
 }
 
-static BufferView NativeUpload(RenderContext& context, CommandBuffer& command_buffer,
-                               std::span<const uint32_t> data) {
+static BufferView NativeUpload(RenderContext& context, std::span<const uint32_t> data) {
 	EXIT_IF(data.empty());
+	auto& command_buffer = context.GetCommandScheduler().Current();
 	EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute());
-	auto binding = context.GetBufferCache().UploadTransient(data.data(), data.size_bytes(), 256);
-	return {.owner  = std::move(binding.owner),
-	        .buffer = binding.buffer,
-	        .offset = binding.offset,
-	        .range  = data.size_bytes()};
+	auto&      buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+	const auto offset = buffer.Copy(data.data(), data.size_bytes(), 256);
+	return {.buffer = buffer.Handle(), .offset = offset, .range = data.size_bytes()};
 }
 
 void RenderExecutor::TrackImageBinding(ImageId id) {
-	auto image = m_context.GetTextureCache().ResolveOwner(id);
-	EXIT_IF(image == nullptr);
-	if (std::ranges::find(m_bound_images, image) == m_bound_images.end()) {
-		m_bound_images.push_back(std::move(image));
+	EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr);
+	if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) {
+		m_bound_images.push_back(id);
 	}
 }
 
@@ -807,16 +924,15 @@ void RenderExecutor::BindRenderTarget(ImageId id) {
 }
 
 void RenderExecutor::ResetBindings() {
-	for (const auto& image: m_bound_images) {
-		image->binding = {};
+	for (const auto id: m_bound_images) {
+		if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(id); image != nullptr) {
+			image->binding = {};
+		}
 	}
 	m_bound_images.clear();
 }
 
-DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&            buffer,
-                                                                  const ShaderStageRuntime& runtime,
-                                                                  vk::ShaderStageFlags shader_stage,
-                                                                  DescriptorCache::Stage stage) {
+PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(!runtime);
 	const auto& program  = *runtime.program;
@@ -826,12 +942,10 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&
 		EXIT("invalid native shader runtime snapshot: %s\n", error.c_str());
 	}
 
-	DescriptorCache::PreparedBindings prepared;
-	prepared.program      = runtime.program;
-	prepared.snapshot     = runtime.resources;
-	prepared.shader_stage = shader_stage;
-	prepared.stage        = stage;
-	auto& descriptors     = prepared.resources;
+	PreparedBindings prepared;
+	prepared.program  = runtime.program.get();
+	prepared.snapshot = runtime.resources.get();
+	auto& descriptors = prepared.resources;
 	descriptors.buffers.reserve(program.info.buffers.size());
 	descriptors.images.reserve(program.info.images.size());
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
@@ -856,44 +970,104 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&
 	prepared.user_data.resize(program.bindings.ShaderDataDwords());
 	if (ShaderRecompiler::IR::FindBinding(
 	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) {
-		descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer().Handle();
+		descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle();
 	}
 	return prepared;
 }
 
-void RenderExecutor::RebindBuffers(CommandBuffer&                     buffer,
-                                   DescriptorCache::PreparedBindings& prepared) {
+void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
+	KYTY_PROFILER_FUNCTION();
+	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
+	const auto& program  = *prepared.program;
+	const auto& snapshot = *prepared.snapshot;
+	auto&       cache    = m_context.GetBufferCache();
+
+	prepared.buffer_ids.clear();
+	prepared.buffer_ids.reserve(program.info.buffers.size());
+	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
+		ShaderBufferResource descriptor;
+		CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields);
+		const auto address = descriptor.Base48();
+		const auto stride  = descriptor.Stride();
+		const auto records = descriptor.NumRecords();
+		EXIT_IF(stride != 0 && records > UINT64_MAX / stride);
+		const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+		if (address == 0 || requested_size == 0) {
+			prepared.buffer_ids.emplace_back();
+			continue;
+		}
+		const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
+		prepared.buffer_ids.push_back(cache.FindBuffer(address, size));
+	}
+
+	prepared.address_ids.clear();
+	prepared.address_ids.reserve(program.info.addresses.size());
+	for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
+		const auto& resource = program.info.addresses[i];
+		const auto& address  = snapshot.addresses[i];
+		if (address.binding_base == 0) {
+			prepared.address_ids.emplace_back();
+			continue;
+		}
+		EXIT_IF(resource.written);
+		const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
+		                       ? ShaderRecompiler::IR::FlatAddressWindowSize
+		                       : static_cast<uint64_t>(m_context.GetGraphics()
+		                                                   .GetPhysicalDeviceProperties()
+		                                                   .limits.maxStorageBufferRange);
+		uint64_t   size  = 0;
+		EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size));
+		prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size));
+	}
+}
+
+void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
 	const auto& program   = *prepared.program;
 	const auto& snapshot  = *prepared.snapshot;
 	auto&       resources = prepared.resources;
 	const auto& layout    = program.bindings;
+	EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size() ||
+	        prepared.address_ids.size() != program.info.addresses.size());
 
 	resources.buffers.clear();
 	resources.buffers.reserve(program.info.buffers.size());
 	EXIT_IF(prepared.user_data.size() != layout.ShaderDataDwords());
-	std::fill(prepared.user_data.begin() + layout.buffer_offset_dword, prepared.user_data.end(), 0);
+	std::fill(prepared.user_data.begin() + layout.memory_offset_dword, prepared.user_data.end(), 0);
+	auto pack_memory_offset = [&](uint32_t index, uint32_t offset) {
+		const auto dword = layout.memory_offset_dword + index / 4u;
+		const auto shift = (index % 4u) * 8u;
+		prepared.user_data[dword] |= offset << shift;
+	};
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		ShaderBufferResource descriptor;
 		CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields);
 		uint32_t buffer_offset = 0;
-		resources.buffers.push_back(NativeStorageBuffer(m_context, buffer, descriptor,
-		                                                program.info.buffers[i], buffer_offset));
-		const auto dword = layout.buffer_offset_dword + i / 4u;
-		const auto shift = (i % 4u) * 8u;
-		prepared.user_data[dword] |= buffer_offset << shift;
+		resources.buffers.push_back(NativeStorageBuffer(m_context, descriptor,
+		                                                program.info.buffers[i], program.stage, i,
+		                                                buffer_offset, prepared.buffer_ids[i]));
+		pack_memory_offset(i, buffer_offset);
 	}
 	resources.addresses.clear();
 	resources.addresses.reserve(program.info.addresses.size());
 	for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
-		resources.addresses.push_back(NativeAddressBuffer(
-		    m_context, buffer, program.info.addresses[i], snapshot.addresses[i]));
+		uint32_t address_offset = 0;
+		resources.addresses.push_back(NativeAddressBuffer(m_context, program.info.addresses[i],
+		                                                  snapshot.addresses[i], program.stage, i,
+		                                                  address_offset, prepared.address_ids[i]));
+		pack_memory_offset(static_cast<uint32_t>(program.info.buffers.size()) + i, address_offset);
+	}
+	if (!prepared.flattened_srt.empty()) {
+		resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt);
+	}
+	if (ShaderRecompiler::IR::FindBinding(
+	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) {
+		resources.user_data = NativeUpload(m_context, prepared.user_data);
 	}
 }
 
-void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
-                                  DescriptorCache::PreparedBindings& prepared) {
+void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
 	const auto& program  = *prepared.program;
@@ -902,7 +1076,7 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		const auto old_image = texture_cache.ResolveOwner(images[i].image_id);
+		const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
@@ -912,6 +1086,8 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
 		}
+	}
+	for (uint32_t i = 0; i < program.info.images.size(); i++) {
 		auto& binding = images[i];
 		binding.mip_views.clear();
 		const auto& resource = program.info.images[i];
@@ -941,121 +1117,215 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 }
 
 RenderExecutor::GraphicsBindings
-RenderExecutor::PrepareGraphicsBindings(CommandBuffer& buffer, const ShaderStageRuntime& vertex,
+RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
                                         const ShaderStageRuntime& pixel, bool pixel_active) {
 	GraphicsBindings bindings {
-	    .vertex = PrepareBindings(buffer, vertex, vk::ShaderStageFlagBits::eVertex,
-	                              DescriptorCache::Stage::Vertex),
+	    .vertex = PrepareBindings(vertex),
 	};
-	RebindBuffers(buffer, bindings.vertex);
-	RebindImages(buffer, bindings.vertex);
 	if (pixel_active) {
-		bindings.pixel.emplace(PrepareBindings(buffer, pixel, vk::ShaderStageFlagBits::eFragment,
-		                                       DescriptorCache::Stage::Pixel));
-		RebindBuffers(buffer, *bindings.pixel);
-		RebindImages(buffer, *bindings.pixel);
+		bindings.pixel.emplace(PrepareBindings(pixel));
+	}
+	FindBuffers(bindings.vertex);
+	if (bindings.pixel) {
+		FindBuffers(*bindings.pixel);
+	}
+	RebindBuffers(bindings.vertex);
+	if (bindings.pixel) {
+		RebindBuffers(*bindings.pixel);
+	}
+	RebindImages(bindings.vertex);
+	if (bindings.pixel) {
+		RebindImages(*bindings.pixel);
 	}
 	return bindings;
 }
 
-static void RetainBindings(CommandBuffer& buffer, DescriptorCache::NativeDescriptors& resources) {
-	if (resources.flattened_srt.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(resources.flattened_srt.owner));
-	}
-	if (resources.user_data.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(resources.user_data.owner));
-	}
-	for (auto& view: resources.buffers) {
-		if (view.owner != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(view.owner));
-		}
-	}
-	for (auto& view: resources.addresses) {
-		if (view.owner != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(view.owner));
-		}
-	}
-}
-
 void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
                                     vk::PipelineBindPoint              pipeline_bind_point,
-                                    vk::PipelineLayout                 layout,
-                                    DescriptorCache::PreparedBindings& prepared) {
+                                    const PipelineCache::Pipeline&     pipeline,
+                                    std::span<PreparedBindings* const> prepared_bindings) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.program == nullptr || prepared.stage == DescriptorCache::Stage::Unknown ||
-	        prepared.committed);
-	const auto& program     = *prepared.program;
-	auto&       descriptors = prepared.resources;
-	if (!prepared.flattened_srt.empty()) {
-		descriptors.flattened_srt = NativeUpload(m_context, buffer, prepared.flattened_srt);
-	}
-	if (ShaderRecompiler::IR::FindBinding(
-	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) {
-		descriptors.user_data = NativeUpload(m_context, buffer, prepared.user_data);
-	}
-	RetainBindings(buffer, descriptors);
-
-	auto       vk_buffer     = buffer.Handle();
-	const auto shader_stages = ShaderPipelineStages(prepared.shader_stage);
-	if (descriptors.gds.buffer != nullptr) {
-		buffer.EndRendering();
-		const auto barrier = MakeGdsDependency(descriptors.gds.buffer);
-		vk_buffer.pipelineBarrier(
-		    vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer |
-		        vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader,
-		    shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr);
-	}
-
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		auto&       image   = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
-		auto&       binding = descriptors.images[i];
-		const auto& view    = binding.desc.view_info;
-		const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
-		                                   view.layer_count};
-		const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
-		if (image.info.data.Empty()) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              storage
-			                  ? vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite
-			                  : vk::AccessFlagBits2::eShaderRead,
-			              range, vk_buffer);
-		} else if ((image.binding.force_general || image.binding.is_target) &&
-		           !image.info.IsDepth()) {
-			const vk::AccessFlags2 storage_access = image.binding.shader_write
-			                                            ? vk::AccessFlagBits2::eShaderWrite
-			                                            : vk::AccessFlags2 {};
-			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead | storage_access |
-			                  vk::AccessFlagBits2::eColorAttachmentRead |
-			                  vk::AccessFlagBits2::eColorAttachmentWrite,
-			              {}, vk_buffer);
-		} else if (storage) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
-			              range, vk_buffer);
-		} else {
-			image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
-			                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
-			              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+	auto   vk_buffer        = buffer.Handle();
+	size_t descriptor_count = 0;
+	size_t write_count      = 0;
+	constexpr auto GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment;
+	const auto push_constant_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics
+	                                      ? vk::ShaderStageFlags {GraphicsStages}
+	                                      : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute};
+	for (const auto* prepared: prepared_bindings) {
+		EXIT_IF(prepared == nullptr || prepared->program == nullptr ||
+		        prepared->snapshot == nullptr || prepared->committed);
+		write_count += prepared->program->bindings.descriptors.size();
+		for (const auto& binding: prepared->program->bindings.descriptors) {
+			descriptor_count += NativeDescriptorCount(binding);
 		}
-		binding.layout = image.backing.state.layout;
+		const auto shader_stage = NativeShaderStage(prepared->program->stage);
+		EXIT_IF((pipeline_bind_point == vk::PipelineBindPoint::eGraphics &&
+		         (shader_stage & GraphicsStages) == vk::ShaderStageFlags {}) ||
+		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
+		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
+	m_descriptor_buffers.clear();
+	m_descriptor_images.clear();
+	m_descriptor_writes.clear();
+	m_push_constants.fill(0);
+	m_descriptor_buffers.reserve(descriptor_count);
+	m_descriptor_images.reserve(descriptor_count);
+	m_descriptor_writes.reserve(write_count);
 
-	if (!program.bindings.descriptors.empty()) {
-		auto& set =
-		    m_context.GetDescriptorCache().GetDescriptor(prepared.stage, program, descriptors);
-		vk_buffer.bindDescriptorSets(pipeline_bind_point, layout, program.bindings.descriptor_set,
-		                             1, &set.set, 0, nullptr);
-		buffer.RecycleDescriptorAfterFence(set);
+	for (auto* prepared: prepared_bindings) {
+		const auto& program       = *prepared->program;
+		auto&       descriptors   = prepared->resources;
+		const auto  shader_stage  = NativeShaderStage(program.stage);
+		const auto  shader_stages = ShaderPipelineStages(shader_stage);
+		if (descriptors.gds.buffer != nullptr) {
+			buffer.EndRendering();
+			const auto barrier = MakeGdsDependency(descriptors.gds.buffer);
+			vk_buffer.pipelineBarrier(
+			    vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer |
+			        vk::PipelineStageFlagBits::eAllGraphics |
+			        vk::PipelineStageFlagBits::eComputeShader,
+			    shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr);
+		}
+
+		for (uint32_t i = 0; i < program.info.images.size(); i++) {
+			auto& image   = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
+			auto& binding = descriptors.images[i];
+			const auto&                 view = binding.desc.view_info;
+			const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
+			                                   view.layer_count};
+			const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
+			if (image.info.data.Empty()) {
+				image.Transit(vk::ImageLayout::eGeneral,
+				              storage ? vk::AccessFlagBits2::eShaderRead |
+				                            vk::AccessFlagBits2::eShaderWrite
+				                      : vk::AccessFlagBits2::eShaderRead,
+				              range, vk_buffer);
+			} else if ((image.binding.force_general || image.binding.is_target) &&
+			           !image.info.IsDepth()) {
+				const vk::AccessFlags2 storage_access = image.binding.shader_write
+				                                            ? vk::AccessFlagBits2::eShaderWrite
+				                                            : vk::AccessFlags2 {};
+				image.Transit(vk::ImageLayout::eGeneral,
+				              vk::AccessFlagBits2::eShaderRead | storage_access |
+				                  vk::AccessFlagBits2::eColorAttachmentRead |
+				                  vk::AccessFlagBits2::eColorAttachmentWrite,
+				              {}, vk_buffer);
+			} else if (storage) {
+				image.Transit(vk::ImageLayout::eGeneral,
+				              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
+				              range, vk_buffer);
+			} else {
+				image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
+				                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
+				              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+			}
+			binding.layout = image.backing.state.layout;
+		}
+
+		m_image_occurrences.assign(descriptors.images.size(), 0);
+		for (const auto& binding: program.bindings.descriptors) {
+			vk::WriteDescriptorSet write {};
+			write.sType          = vk::StructureType::eWriteDescriptorSet;
+			write.dstBinding     = ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind);
+			write.descriptorType = NativeDescriptorType(binding.kind);
+			write.descriptorCount   = NativeDescriptorCount(binding);
+			const auto buffer_start = m_descriptor_buffers.size();
+			const auto image_start  = m_descriptor_images.size();
+			switch (binding.kind) {
+				case BindingKind::Buffers:
+					for (const auto resource: binding.resources) {
+						const auto& view = descriptors.buffers.at(resource);
+						EXIT_IF(view.buffer == nullptr);
+						m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					}
+					break;
+				case BindingKind::AddressMemory:
+					for (const auto resource: binding.resources) {
+						const auto& view = descriptors.addresses.at(resource);
+						EXIT_IF(view.buffer == nullptr);
+						m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					}
+					break;
+				case BindingKind::FlattenedSrt:
+				case BindingKind::UserData:
+				case BindingKind::Gds: {
+					const auto& view =
+					    binding.kind == BindingKind::FlattenedSrt ? descriptors.flattened_srt
+					    : binding.kind == BindingKind::UserData   ? descriptors.user_data
+					                                              : descriptors.gds;
+					EXIT_IF(view.buffer == nullptr);
+					m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					break;
+				}
+				case BindingKind::Samplers:
+					for (const auto resource: binding.resources) {
+						const auto sampler = descriptors.samplers.at(resource);
+						EXIT_IF(sampler == nullptr);
+						m_descriptor_images.emplace_back(sampler, nullptr,
+						                                 vk::ImageLayout::eUndefined);
+					}
+					break;
+				default:
+					for (const auto resource: binding.resources) {
+						m_descriptor_images.push_back(MakeImageInfo(
+						    descriptors.images.at(resource), m_image_occurrences.at(resource)++));
+					}
+					break;
+			}
+			if (m_descriptor_buffers.size() != buffer_start) {
+				write.pBufferInfo = m_descriptor_buffers.data() + buffer_start;
+			}
+			if (m_descriptor_images.size() != image_start) {
+				write.pImageInfo = m_descriptor_images.data() + image_start;
+			}
+			m_descriptor_writes.push_back(write);
+		}
+		for (uint32_t i = 0; i < descriptors.images.size(); i++) {
+			const auto expected =
+			    descriptors.images[i].mip_views.empty()
+			        ? 1u
+			        : static_cast<uint32_t>(descriptors.images[i].mip_views.size());
+			EXIT_IF(m_image_occurrences[i] != expected);
+		}
+
+		if (program.bindings.push_constant_size != 0) {
+			const auto offset = program.bindings.push_constant_offset;
+			EXIT_IF(offset % sizeof(uint32_t) != 0 ||
+			        program.bindings.push_constant_size !=
+			            prepared->user_data.size() * sizeof(uint32_t) ||
+			        offset + program.bindings.push_constant_size >
+			            ShaderRecompiler::IR::NativePushConstantSize);
+			std::copy(prepared->user_data.begin(), prepared->user_data.end(),
+			          m_push_constants.begin() + offset / sizeof(uint32_t));
+		}
 	}
-	if (program.bindings.push_constant_size != 0) {
-		EXIT_IF(program.bindings.push_constant_size !=
-		        prepared.user_data.size() * sizeof(uint32_t));
-		vk_buffer.pushConstants(layout, prepared.shader_stage,
-		                        program.bindings.push_constant_offset,
-		                        program.bindings.push_constant_size, prepared.user_data.data());
+	vk_buffer.pushConstants(pipeline.pipeline_layout, push_constant_stages, 0,
+	                        ShaderRecompiler::IR::NativePushConstantSize,
+	                        m_push_constants.data());
+
+	if (!m_descriptor_writes.empty()) {
+		EXIT_IF(pipeline.descriptor_set_layout == nullptr);
+		if (pipeline.uses_push_descriptors) {
+			vk_buffer.pushDescriptorSetKHR(pipeline_bind_point, pipeline.pipeline_layout, 0,
+			                               static_cast<uint32_t>(m_descriptor_writes.size()),
+			                               m_descriptor_writes.data());
+		} else {
+			const auto set = m_context.GetDescriptorHeap().Commit(pipeline.descriptor_set_layout);
+			for (auto& write: m_descriptor_writes) {
+				write.dstSet = set;
+			}
+			m_context.GetGraphics().device.updateDescriptorSets(
+			    static_cast<uint32_t>(m_descriptor_writes.size()), m_descriptor_writes.data(), 0,
+			    nullptr);
+			vk_buffer.bindDescriptorSets(pipeline_bind_point, pipeline.pipeline_layout, 0, 1, &set,
+			                             0, nullptr);
+		}
+	}
+	for (auto* prepared: prepared_bindings) {
+		prepared->committed = true;
 	}
-	prepared.committed = true;
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -2,6 +2,8 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_
 
 #include "common/assert.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/image/image.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
@@ -10,9 +12,56 @@
 #include <cstdint>
 #include <cstring>
 #include <type_traits>
+#include <vector>
 
 namespace Libs::Graphics {
 
+namespace ShaderRecompiler::IR {
+struct ResourceSnapshot;
+}
+
+struct BufferView {
+	vk::Buffer     buffer = nullptr;
+	vk::DeviceSize offset = 0;
+	vk::DeviceSize range  = VK_WHOLE_SIZE;
+};
+
+struct TextureBinding {
+	ImageId                    image_id;
+	vk::ImageView              image_view = nullptr;
+	TextureCache::ImageDesc    desc;
+	vk::ImageLayout            layout = vk::ImageLayout::eUndefined;
+	std::vector<vk::ImageView> mip_views;
+};
+
+struct NativeDescriptors {
+	std::vector<BufferView>     buffers;
+	std::vector<TextureBinding> images;
+	std::vector<vk::Sampler>    samplers;
+	std::vector<BufferView>     addresses;
+	BufferView                  gds;
+	BufferView                  flattened_srt;
+	BufferView                  user_data;
+};
+
+struct PreparedBindings {
+	const ShaderRecompiler::IR::Program*          program  = nullptr;
+	const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
+	NativeDescriptors                             resources;
+	std::vector<BufferId>                         buffer_ids;
+	std::vector<BufferId>                         address_ids;
+	std::vector<uint32_t>                         flattened_srt;
+	std::vector<uint32_t>                         user_data;
+	bool                                          committed = false;
+};
+
+[[nodiscard]] vk::DescriptorType
+NativeDescriptorType(ShaderRecompiler::IR::DescriptorBindingKind kind);
+[[nodiscard]] uint32_t
+NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding);
+[[nodiscard]] vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
+                                                    uint32_t              element = 0);
+
 template <typename T>
 [[nodiscard]] T DecodeNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& value) {
 	static_assert(std::is_trivially_copyable_v<T>);
@@ -34,9 +83,9 @@ ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource,
                          Prospero::ImageType type, uint32_t base_layer, uint32_t image_layers);
 
 [[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor,
-                                                    const Image&                 image);
+                                                    const Image& image, bool r128 = false);
 [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
-                                                   const Image&                 image);
+                                                   const Image& image, bool r128 = false);
 [[nodiscard]] bool
 IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
                                const ShaderTextureResource& descriptor, const Image& image);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -42,6 +42,7 @@ PipelineCache::~PipelineCache() {
 			(void)key;
 			m_graphics.device.destroyPipeline(pipeline->pipeline, nullptr);
 			m_graphics.device.destroyPipelineLayout(pipeline->pipeline_layout, nullptr);
+			m_graphics.device.destroyDescriptorSetLayout(pipeline->descriptor_set_layout, nullptr);
 		}
 	};
 	destroy(m_graphics_pipelines);
@@ -121,6 +122,12 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 			     depth.samples);
 		}
 	}
+	if (color_count == 0 && !with_depth) {
+		attachment_samples = render_sample_count(ctx.GetAaConfig().msaa_num_samples);
+		EXIT_IF(!static_cast<bool>(
+		    m_graphics.GetPhysicalDeviceProperties().limits.framebufferNoAttachmentsSampleCounts &
+		    vulkan_sample_count(attachment_samples)));
+	}
 	EXIT_IF(attachment_samples == 0 ||
 	        vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {});
 
@@ -204,9 +211,9 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 	auto cached = std::make_unique<GraphicsPipeline>(p);
 	LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0,
 	                 ps_id.crc32);
-	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info,
-	                       vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0,
-	                       vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active);
+	CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vs_spirv, ps_input_info,
+	                       ps_spirv, static_params, vs_id.hash0, vs_id.crc32, ps_id.hash0,
+	                       ps_id.crc32, ps_active);
 	LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0,
 	                 ps_id.crc32);
 
@@ -246,7 +253,7 @@ PipelineCache::CreateComputePipeline(ShaderComputeInputInfo&      input_info,
 	}
 
 	auto cached = std::make_unique<ComputePipeline>(p);
-	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, input_info, cs_spirv);
+	CreatePipelineInternal(m_graphics, *cached, input_info, cs_spirv);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
 	EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -21,7 +21,6 @@ struct GraphicContext;
 struct RenderColorInfo;
 struct RenderDepthInfo;
 class RenderCommandBuffer;
-class DescriptorCache;
 
 namespace HW {
 class Context;
@@ -98,16 +97,17 @@ struct PipelineRenderingState {
 
 class PipelineCache {
 public:
-	PipelineCache(GraphicContext& graphics, DescriptorCache& descriptor_cache)
-	    : m_graphics(graphics), m_descriptor_cache(descriptor_cache) {
+	explicit PipelineCache(GraphicContext& graphics): m_graphics(graphics) {
 		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 	}
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 
 	struct Pipeline {
-		vk::PipelineLayout pipeline_layout = nullptr;
-		vk::Pipeline       pipeline        = nullptr;
+		vk::PipelineLayout      pipeline_layout       = nullptr;
+		vk::Pipeline            pipeline              = nullptr;
+		vk::DescriptorSetLayout descriptor_set_layout = nullptr;
+		bool                    uses_push_descriptors = false;
 	};
 
 	struct GraphicsPipeline: Pipeline {
@@ -201,8 +201,7 @@ private:
 		}
 	};
 
-	GraphicContext&  m_graphics;
-	DescriptorCache& m_descriptor_cache;
+	GraphicContext& m_graphics;
 	std::unordered_map<GraphicsPipelineKey, std::unique_ptr<GraphicsPipeline>,
 	                   GraphicsPipelineKeyHash>
 	    m_graphics_pipelines;
@@ -214,16 +213,14 @@ private:
 void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
                       uint32_t ps_crc32);
 void CreatePipelineInternal(
-    GraphicContext& graphics, DescriptorCache& descriptor_cache,
-    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
-    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
-    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
-    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
-    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active);
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::ComputePipeline& pipeline,
-                            const ShaderComputeInputInfo&   input_info,
-                            std::span<const uint32_t>       cs_shader);
+    GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline,
+    const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info,
+    std::span<const uint32_t> vs_shader, const ShaderPixelInputInfo* ps_input_info,
+    std::span<const uint32_t> ps_shader, const PipelineStaticParameters& static_params,
+    uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline,
+                            const ShaderComputeInputInfo& input_info,
+                            std::span<const uint32_t>     cs_shader);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp

@@ -1,60 +0,0 @@
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
-
-#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
-
-namespace Libs::Graphics {
-
-ShaderSubgroupCapabilities::ShaderSubgroupCapabilities(const GraphicContext& graphics)
-    : subgroup_size(graphics.subgroup_size), min_subgroup_size(graphics.min_subgroup_size),
-      max_subgroup_size(graphics.max_subgroup_size),
-      required_subgroup_size_stages(graphics.required_subgroup_size_stages),
-      subgroup_size_control_enabled(graphics.subgroup_size_control_enabled) {}
-
-ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size) {
-	// A 64-wide guest wave can't assume the host GPU's lane layout matches
-	// GCN's. AMD RDNA also reports a 64-wide subgroup but uses a different
-	// lane/pixel layout, which corrupted rendering under "native wave" mode.
-	// Per-invocation emulation works regardless of the host's lane layout.
-	if (guest_wave_size == 64u) {
-		return ShaderLaneMaskMode::PerInvocation;
-	}
-	return ShaderLaneMaskMode::NativeWave;
-}
-
-ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
-                                                    vk::ShaderStageFlagBits           stage,
-                                                    const ShaderRecompiler::IR::Program& program) {
-	const auto guest_wave_size = program.wave_size;
-	if (guest_wave_size != 32u && guest_wave_size != 64u) {
-		return {};
-	}
-	if (stage != vk::ShaderStageFlagBits::eCompute) {
-		const auto expected = SelectGraphicsLaneMaskMode(guest_wave_size);
-		if (program.lane_mask_mode != expected || (capabilities.subgroup_size != guest_wave_size &&
-		                                           expected != ShaderLaneMaskMode::PerInvocation)) {
-			return {};
-		}
-		return {expected == ShaderLaneMaskMode::NativeWave
-		            ? ShaderSubgroupMode::Natural
-		            : ShaderSubgroupMode::PerInvocationGraphics,
-		        0};
-	}
-	if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) {
-		return {};
-	}
-	if (capabilities.subgroup_size == guest_wave_size) {
-		return {ShaderSubgroupMode::Natural, 0};
-	}
-	if (capabilities.subgroup_size_control_enabled &&
-	    (capabilities.required_subgroup_size_stages & stage) &&
-	    guest_wave_size >= capabilities.min_subgroup_size &&
-	    guest_wave_size <= capabilities.max_subgroup_size) {
-		return {ShaderSubgroupMode::Controlled, guest_wave_size};
-	}
-	if (!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(program)) {
-		return {ShaderSubgroupMode::FlattenedMasks, 0};
-	}
-	return {};
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h

@@ -1,40 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
-
-#include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-namespace Libs::Graphics {
-
-enum class ShaderSubgroupMode {
-	Natural,
-	Controlled,
-	PerInvocationGraphics,
-	FlattenedMasks,
-	Unsupported
-};
-
-struct ShaderSubgroupConfiguration {
-	ShaderSubgroupMode mode          = ShaderSubgroupMode::Unsupported;
-	uint32_t           required_size = 0;
-};
-
-struct ShaderSubgroupCapabilities {
-	explicit ShaderSubgroupCapabilities(const GraphicContext& graphics);
-
-	uint32_t             subgroup_size     = 0;
-	uint32_t             min_subgroup_size = 0;
-	uint32_t             max_subgroup_size = 0;
-	vk::ShaderStageFlags required_subgroup_size_stages;
-	bool                 subgroup_size_control_enabled = false;
-};
-
-ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size);
-
-ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
-                                                    vk::ShaderStageFlagBits           stage,
-                                                    const ShaderRecompiler::IR::Program& program);
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -6,9 +6,8 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
@@ -20,6 +19,7 @@
 #include <algorithm>
 #include <limits>
 #include <span>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -389,82 +389,42 @@ static vk::BlendOp GetBlendOp(uint32_t op) {
 	return vk::BlendOp::eAdd;
 }
 
-static void CreateLayout(DescriptorCache&                   descriptor_cache,
-                         std::span<vk::DescriptorSetLayout> set_layouts, uint32_t& set_layouts_num,
-                         std::span<vk::PushConstantRange>     push_constant_info,
-                         uint32_t&                            push_constant_info_num,
-                         const ShaderRecompiler::IR::Program& program,
-                         vk::ShaderStageFlags vk_stage, DescriptorCache::Stage stage) {
-	const auto& bindings        = program.bindings;
-	const bool  need_descriptor = !bindings.descriptors.empty();
-
-	if (bindings.push_constant_size != 0) {
-		auto index = push_constant_info_num;
-
-		push_constant_info[index].stageFlags = vk_stage;
-		push_constant_info[index].offset     = bindings.push_constant_offset;
-		push_constant_info[index].size       = bindings.push_constant_size;
-		push_constant_info_num++;
-	}
-
-	if (need_descriptor) {
-		EXIT_IF(bindings.descriptor_set != set_layouts_num);
-
-		set_layouts[set_layouts_num] = descriptor_cache.GetDescriptorSetLayout(stage, program);
-		set_layouts_num++;
+static void AddLayoutBindings(std::vector<vk::DescriptorSetLayoutBinding>& descriptor_bindings,
+                              const ShaderRecompiler::IR::Program& program,
+                              vk::ShaderStageFlagBits              stage) {
+	for (const auto& binding: program.bindings.descriptors) {
+		descriptor_bindings.push_back(
+		    {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind),
+		     NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), stage, nullptr});
 	}
 }
 
-static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
-                                  const ShaderRecompiler::IR::Program&                   program,
-                                  vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
-                                  vk::PipelineShaderStageCreateInfo&                     stage) {
-	const auto config =
-	    ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program);
-	switch (config.mode) {
-		case ShaderSubgroupMode::Natural: return;
-		case ShaderSubgroupMode::PerInvocationGraphics: {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation "
-				     "EXEC/VCC on the host-native subgroup\n",
-				     program.wave_size, static_cast<uint32_t>(vk_stage));
-			}
-			return;
-		}
-		case ShaderSubgroupMode::FlattenedMasks: {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation "
-				     "EXEC/VCC\n",
-				     program.wave_size, static_cast<uint32_t>(vk_stage));
-			}
-			return;
-		}
-		case ShaderSubgroupMode::Controlled: break;
-		case ShaderSubgroupMode::Unsupported:
-		default:
-			EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u "
-			     "controlled_stages=0x%08x\n",
-			     program.wave_size, static_cast<uint32_t>(vk_stage), graphics.subgroup_size,
-			     graphics.min_subgroup_size, graphics.max_subgroup_size,
-			     static_cast<vk::ShaderStageFlags::MaskType>(
-			         graphics.required_subgroup_size_stages));
+static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                                   std::span<const vk::DescriptorSetLayoutBinding> bindings) {
+	uint32_t descriptor_count = 0;
+	for (const auto& binding: bindings) {
+		descriptor_count += binding.descriptorCount;
 	}
+	pipeline.uses_push_descriptors = descriptor_count <= graphics.max_push_descriptors;
 
-	required.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
-	required.requiredSubgroupSize = config.required_size;
-	stage.pNext                   = &required;
+	vk::DescriptorSetLayoutCreateInfo create {};
+	create.sType        = vk::StructureType::eDescriptorSetLayoutCreateInfo;
+	create.flags        = pipeline.uses_push_descriptors
+	                          ? vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR
+	                          : vk::DescriptorSetLayoutCreateFlags {};
+	create.bindingCount = static_cast<uint32_t>(bindings.size());
+	create.pBindings    = bindings.data();
+	EXIT_IF(graphics.device.createDescriptorSetLayout(
+	            &create, nullptr, &pipeline.descriptor_set_layout) != vk::Result::eSuccess);
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void CreatePipelineInternal(
-    GraphicContext& graphics, DescriptorCache& descriptor_cache,
-    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
-    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
-    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
-    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
-    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
+    GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline,
+    const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info,
+    std::span<const uint32_t> vs_shader, const ShaderPixelInputInfo* ps_input_info,
+    std::span<const uint32_t> ps_shader, const PipelineStaticParameters& static_params,
+    uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
 	EXIT_IF(ps_active && ps_input_info == nullptr);
 
 	const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList;
@@ -528,8 +488,7 @@ void CreatePipelineInternal(
 	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
 	EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
 
-	vk::PipelineShaderStageCreateInfo                     vert_shader_stage_info {};
-	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {};
+	vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
 
 	vert_shader_stage_info.sType               = vk::StructureType::ePipelineShaderStageCreateInfo;
 	vert_shader_stage_info.pNext               = nullptr;
@@ -538,12 +497,8 @@ void CreatePipelineInternal(
 	vert_shader_stage_info.module              = vert_shader_module;
 	vert_shader_stage_info.pName               = "main";
 	vert_shader_stage_info.pSpecializationInfo = nullptr;
-	EXIT_IF(!vs_input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
-	                      vert_subgroup_size, vert_shader_stage_info);
 
-	vk::PipelineShaderStageCreateInfo                     frag_shader_stage_info {};
-	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {};
+	vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
 	frag_shader_stage_info.sType               = vk::StructureType::ePipelineShaderStageCreateInfo;
 	frag_shader_stage_info.pNext               = nullptr;
 	frag_shader_stage_info.flags               = {};
@@ -551,12 +506,6 @@ void CreatePipelineInternal(
 	frag_shader_stage_info.module              = frag_shader_module;
 	frag_shader_stage_info.pName               = "main";
 	frag_shader_stage_info.pSpecializationInfo = nullptr;
-	if (ps_active) {
-		EXIT_IF(!ps_input_info->stage);
-		ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
-		                      *ps_input_info->stage.program, frag_subgroup_size,
-		                      frag_shader_stage_info);
-	}
 
 	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
 	tess_control_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
@@ -877,40 +826,36 @@ void CreatePipelineInternal(
 	color_blending.blendConstants[2] = static_params.blend_color_blue;
 	color_blending.blendConstants[3] = static_params.blend_color_alpha;
 
-	vk::DescriptorSetLayout set_layouts[2]  = {};
-	uint32_t                set_layouts_num = 0;
-
-	vk::PushConstantRange push_constant_info[2];
-	uint32_t              push_constant_info_num = 0;
-
 	EXIT_IF(!vs_input_info.stage);
-	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num, *vs_input_info.stage.program,
-	             vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex);
+	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
+	AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program,
+	                  vk::ShaderStageFlagBits::eVertex);
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
-		CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-		             push_constant_info_num, *ps_input_info->stage.program,
-		             vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel);
+		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
+		                  vk::ShaderStageFlagBits::eFragment);
 	}
+	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
+	constexpr auto GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment;
+	const vk::PushConstantRange push_constants {
+	    GraphicsStages, 0, ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;
 	pipeline_layout_info.pNext                  = nullptr;
 	pipeline_layout_info.flags                  = {};
-	pipeline_layout_info.setLayoutCount         = set_layouts_num;
-	pipeline_layout_info.pSetLayouts            = (set_layouts_num > 0 ? set_layouts : nullptr);
-	pipeline_layout_info.pushConstantRangeCount = push_constant_info_num;
-	pipeline_layout_info.pPushConstantRanges =
-	    push_constant_info_num > 0 ? push_constant_info : nullptr;
+	pipeline_layout_info.setLayoutCount         = 1;
+	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.pushConstantRangeCount = 1;
+	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
 	if (graphics_debug_dump_enabled()) {
 		LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=0x%08" PRIx32 "/0x%08" PRIx32
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=%" PRIu32 " push_constants=%" PRIu32
-		     "\n",
-		     vs_hash0, vs_crc32, ps_hash0, ps_crc32, set_layouts_num, push_constant_info_num);
+		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=1 push_constants=%" PRIu32 "\n",
+		     vs_hash0, vs_crc32, ps_hash0, ps_crc32, 1u);
 	}
 	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
 	                                              &pipeline.pipeline_layout);
@@ -1042,10 +987,9 @@ void CreatePipelineInternal(
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::ComputePipeline& pipeline,
-                            const ShaderComputeInputInfo&   input_info,
-                            std::span<const uint32_t>       cs_shader) {
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline,
+                            const ShaderComputeInputInfo& input_info,
+                            std::span<const uint32_t>     cs_shader) {
 	vk::ShaderModule comp_shader_module = nullptr;
 
 	vk::ShaderModuleCreateInfo create_info {};
@@ -1074,34 +1018,36 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	comp_shader_stage_info.pName               = "main";
 	comp_shader_stage_info.pSpecializationInfo = nullptr;
 	EXIT_IF(!input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
-	                      comp_subgroup_size, comp_shader_stage_info);
+	const auto wave_size = input_info.stage.program->wave_size;
+	if (graphics.subgroup_size_control_enabled &&
+	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
+	    wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) {
+		comp_subgroup_size.sType =
+		    vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
+		comp_subgroup_size.requiredSubgroupSize = wave_size;
+		comp_shader_stage_info.pNext            = &comp_subgroup_size;
+	}
 
-	vk::DescriptorSetLayout set_layouts[1]  = {};
-	uint32_t                set_layouts_num = 0;
-
-	vk::PushConstantRange push_constant_info[1];
-	uint32_t              push_constant_info_num = 0;
-
-	EXIT_IF(!input_info.stage);
-	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num, *input_info.stage.program,
-	             vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);
+	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
+	AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
+	                  vk::ShaderStageFlagBits::eCompute);
+	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
+	const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
+	                                            ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;
 	pipeline_layout_info.pNext                  = nullptr;
 	pipeline_layout_info.flags                  = {};
-	pipeline_layout_info.setLayoutCount         = set_layouts_num;
-	pipeline_layout_info.pSetLayouts            = (set_layouts_num > 0 ? set_layouts : nullptr);
-	pipeline_layout_info.pushConstantRangeCount = push_constant_info_num;
-	pipeline_layout_info.pPushConstantRanges =
-	    push_constant_info_num > 0 ? push_constant_info : nullptr;
+	pipeline_layout_info.setLayoutCount         = 1;
+	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.pushConstantRangeCount = 1;
+	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
-	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=%u push_constants=%u\n",
-	     set_layouts_num, push_constant_info_num);
+	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n",
+	     1u);
 	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
 	                                              &pipeline.pipeline_layout);
 	LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n",

src/graphics/host_gpu/renderer/render.h

@@ -4,12 +4,14 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
-#include <memory>
+#include <optional>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -24,8 +26,6 @@ struct GraphicContext;
 struct ShaderBufferResource;
 struct ShaderComputeInputInfo;
 struct CommandSlot;
-struct VulkanBuffer;
-struct VulkanDescriptorSet;
 struct RenderDepthInfo;
 struct RenderColorInfo;
 struct DrawCallInfo;
@@ -49,20 +49,6 @@ enum class CommandBufferDebugOp : uint32_t {
 	Unknown,
 };
 
-class FenceResourceRetainer {
-public:
-	FenceResourceRetainer() = default;
-	~FenceResourceRetainer();
-	KYTY_CLASS_NO_COPY(FenceResourceRetainer);
-
-	void               Retain(std::shared_ptr<void> resource);
-	void               ReleaseAfterFence() noexcept;
-	[[nodiscard]] bool Empty() const noexcept { return m_resources.empty(); }
-
-private:
-	std::vector<std::shared_ptr<void>> m_resources;
-};
-
 struct SubmitInfo {
 	static constexpr uint32_t MaxSemaphores = 3;
 
@@ -108,9 +94,6 @@ public:
 	void WaitForFenceOnly();
 	void WaitForFence();
 	void WaitForFenceAndReset();
-	void RetireBufferAfterFence(std::unique_ptr<VulkanBuffer> buffer);
-	void RetainResourceUntilFence(std::shared_ptr<void> resource);
-	void RecycleDescriptorAfterFence(VulkanDescriptorSet& set);
 
 	[[nodiscard]] vk::CommandBuffer Handle() const;
 	[[nodiscard]] GraphicContext&   GetGraphics() const noexcept { return m_graphics; }
@@ -120,29 +103,23 @@ public:
 private:
 	void Release();
 	void FinalizeFence(bool reset_recording);
-	void ReleaseResourcesAfterFence();
-	void DeleteBuffersAfterFence();
-	void RecycleDescriptorsAfterFence();
 
-	RenderContext&                             m_context;
-	CommandScheduler&                          m_scheduler;
-	GraphicContext&                            m_graphics;
-	CommandSlot*                               m_slot            = nullptr;
-	bool                                       m_execute         = false;
-	bool                                       m_fence_waited    = false;
-	uint64_t                                   m_submit_seq      = 0;
-	uint32_t                                   m_debug_op        = 0;
-	uint64_t                                   m_debug_submit_id = 0;
-	uint32_t                                   m_debug_arg0      = 0;
-	uint32_t                                   m_debug_arg1      = 0;
-	uint32_t                                   m_debug_arg2      = 0;
-	uint32_t                                   m_debug_arg3      = 0;
-	uint64_t                                   m_debug_arg4      = 0;
-	std::vector<std::unique_ptr<VulkanBuffer>> m_retired_buffers;
-	FenceResourceRetainer                      m_fence_resources;
-	std::vector<VulkanDescriptorSet*>          m_descriptor_sets_after_fence;
-	mutable RenderState                        m_render_state;
-	mutable bool                               m_rendering = false;
+	RenderContext&      m_context;
+	CommandScheduler&   m_scheduler;
+	GraphicContext&     m_graphics;
+	CommandSlot*        m_slot            = nullptr;
+	bool                m_execute         = false;
+	bool                m_fence_waited    = false;
+	uint64_t            m_submit_seq      = 0;
+	uint32_t            m_debug_op        = 0;
+	uint64_t            m_debug_submit_id = 0;
+	uint32_t            m_debug_arg0      = 0;
+	uint32_t            m_debug_arg1      = 0;
+	uint32_t            m_debug_arg2      = 0;
+	uint32_t            m_debug_arg3      = 0;
+	uint64_t            m_debug_arg4      = 0;
+	mutable RenderState m_render_state;
+	mutable bool        m_rendering = false;
 };
 
 class RenderCommandBuffer final: public CommandBuffer {
@@ -181,25 +158,23 @@ public:
 	void DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x,
 	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode);
 
-	[[nodiscard]] DescriptorCache::PreparedBindings
-	     PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime,
-	                     vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage);
-	void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings);
-	void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings);
+	[[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime);
+	void                           FindBuffers(PreparedBindings& bindings);
+	void                           RebindBuffers(PreparedBindings& bindings);
+	void                           RebindImages(PreparedBindings& bindings);
 	void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
-	                    vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings);
+	                    const PipelineCache::Pipeline&     pipeline,
+	                    std::span<PreparedBindings* const> bindings);
 
 private:
 	struct GraphicsBindings {
-		DescriptorCache::PreparedBindings                vertex;
-		std::optional<DescriptorCache::PreparedBindings> pixel;
+		PreparedBindings                vertex;
+		std::optional<PreparedBindings> pixel;
 	};
 
-	[[nodiscard]] DescriptorCache::TextureBinding
-	ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
-	               const ShaderRecompiler::IR::DescriptorValue& value);
-	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(CommandBuffer&            buffer,
-	                                                       const ShaderStageRuntime& vertex,
+	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
+	                                            const ShaderRecompiler::IR::DescriptorValue& value);
+	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
 	                                                       const ShaderStageRuntime& pixel,
 	                                                       bool                      pixel_active);
 	void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
@@ -229,8 +204,14 @@ private:
 	[[nodiscard]] bool        TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
 	                                                     const RenderCommandBuffer&    buffer);
 
-	RenderContext&                      m_context;
-	std::vector<std::shared_ptr<Image>> m_bound_images;
+	RenderContext&                        m_context;
+	std::vector<ImageId>                  m_bound_images;
+	std::vector<vk::DescriptorBufferInfo> m_descriptor_buffers;
+	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;
+	std::vector<vk::WriteDescriptorSet>   m_descriptor_writes;
+	std::vector<uint32_t>                 m_image_occurrences;
+	std::array<uint32_t, ShaderRecompiler::IR::NativePushConstantSize / sizeof(uint32_t)>
+	    m_push_constants {};
 
 	friend struct RenderExecutorTestAccess;
 };

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -11,11 +11,9 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -115,7 +113,7 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
 	    group_z == 1 && input.dispatch_threads_num[0] == group_x &&
 	    input.dispatch_threads_num[1] == 1 && input.dispatch_threads_num[2] == 1 &&
 	    input.group_id[0] && !input.group_id[1] && !input.group_id[2] &&
-	    input.thread_ids_num == 1 && input.wave_size == 32 && !input.tg_size_en && mode == 0x61u &&
+	    input.thread_ids_num == 1 && input.wave_size == 64 && !input.tg_size_en && mode == 0x61u &&
 	    group_x % input.threads_num[0] == 0 && descriptor.NumRecords() == group_x;
 	const auto size = BufferDescriptorSize(descriptor);
 	if (!full_dispatch || size == 0) {
@@ -139,6 +137,21 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com
 	}
 	auto& cache = command.GetContext().GetTextureCache();
 	if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) {
+		// Recognized metadata-fill shaders access DCC as an ordinary storage buffer and may run
+		// before the render target is bound. TryConsumeDccFill either consumes registered state
+		// or retains a PendingDcc fill while allowing the dispatch to run.
+		const bool registered_metadata =
+		    cache.TryConsumeDccFill(descriptor.Base48(), size, packed_clear);
+		static std::atomic<uint32_t> logged_metadata_clears {0};
+		if (logged_metadata_clears.fetch_add(1, std::memory_order_relaxed) < 32) {
+			LOGF("GraphicsRenderDispatchDirect: %s metadata clear shader=0x%016" PRIx64
+			     " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n",
+			     registered_metadata ? "tracked" : "deferred", input.stage.program->shader_hash,
+			     descriptor.Base48(), size, packed_clear);
+		}
+		if (registered_metadata) {
+			return true;
+		}
 		return false;
 	}
 	static std::atomic<uint32_t> logged_clears {0};
@@ -154,6 +167,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
                                     uint32_t thread_group_x, uint32_t thread_group_y,
                                     uint32_t thread_group_z, uint32_t mode) {
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -264,7 +278,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 			     r.Type() == Prospero::ImageType::kColor2DMsaa ||
 			             r.Type() == Prospero::ImageType::kColor2DMsaaArray
 			         ? 1u
-			         : static_cast<uint32_t>(r.MaxMip()) + 1u,
+			         : static_cast<uint32_t>(image.r128 ? r.LastLevel() : r.MaxMip()) + 1u,
 			     static_cast<uint32_t>(r.TileMode()));
 		}
 		for (uint32_t i = 0; i < program.info.samplers.size(); i++) {
@@ -319,13 +333,15 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 	buffer.EndRendering();
 	auto& pipeline =
 	    m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader);
-	auto bindings = PrepareBindings(buffer, input_info.stage, vk::ShaderStageFlagBits::eCompute,
-	                                DescriptorCache::Stage::Compute);
-	RebindBuffers(buffer, bindings);
-	RebindImages(buffer, bindings);
+	auto bindings = PrepareBindings(input_info.stage);
+	FindBuffers(bindings);
+	RebindBuffers(bindings);
+	RebindImages(bindings);
 
-	auto vk_buffer = buffer.Handle();
-	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline.pipeline_layout, bindings);
+	auto              vk_buffer        = buffer.Handle();
+	PreparedBindings* descriptor_stage = &bindings;
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
 	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
 	has_storage_writes =
 	    std::any_of(program.info.images.begin(), program.info.images.end(),

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -13,8 +13,9 @@ namespace Libs::Graphics {
 
 RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
-      m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache),
-      m_sampler_cache(graphics), m_gpu_resources(graphics, m_command_scheduler) {
+      m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
+      m_pipeline_cache(graphics), m_sampler_cache(graphics),
+      m_gpu_resources(graphics, m_command_scheduler) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 

src/graphics/host_gpu/renderer/renderContext.h

@@ -10,7 +10,7 @@
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "kernel/eventQueue.h"
 
@@ -41,7 +41,7 @@ public:
 	Common::Mutex&      GetMutex() { return m_mutex; }
 	CommandScheduler&   GetCommandScheduler() { return m_command_scheduler; }
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
-	DescriptorCache&    GetDescriptorCache() { return m_descriptor_cache; }
+	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
 	GpuResourceManager& GetGpuResources() { return m_gpu_resources; }
 	BufferCache&        GetBufferCache() { return m_gpu_resources.GetBufferCache(); }
@@ -63,7 +63,7 @@ private:
 	Common::Mutex             m_mutex;
 	RenderExecutor            m_render_executor;
 	CommandScheduler          m_command_scheduler;
-	DescriptorCache           m_descriptor_cache;
+	DescriptorHeap            m_descriptor_heap;
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
 	GpuResourceManager        m_gpu_resources;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -15,10 +15,8 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -142,18 +140,6 @@ static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer
 	                   (bc0.separate_alpha_blend && (IsDualSourceBlendFactor(bc0.alpha_srcblend) ||
 	                                                 IsDualSourceBlendFactor(bc0.alpha_destblend)));
 
-	for (uint32_t i = 1; i < 8; i++) {
-		const auto& rt = ctx.GetRenderTarget(i);
-		if (rt.base.addr != 0 || ps_input_info.target_output_mode[i] != 0 ||
-		    ((rt_mask >> (i * 4u)) & 0x0fu) != 0 || ((cb_shader_mask >> (i * 4u)) & 0x0fu) != 0) {
-			interesting = true;
-		}
-	}
-
-	if (!interesting) {
-		return;
-	}
-
 	auto log_id = g_mrt_state_log_count.fetch_add(1);
 	if (log_id >= 32) {
 		return;
@@ -340,6 +326,9 @@ static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandB
 		framebuffer_extent = colors[0].extent;
 	} else if (depth.image_id) {
 		framebuffer_extent = {depth.width, depth.height};
+	} else {
+		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
+		framebuffer_extent = {limits.maxFramebufferWidth, limits.maxFramebufferHeight};
 	}
 
 	const auto final_scissor = calc_final_scissor(vp, ctx.GetScanModeControl(), framebuffer_extent);
@@ -374,8 +363,8 @@ static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandB
 		    (use_front ? poly_offset.front_scale : poly_offset.back_scale) / 16.0f;
 	}
 
-	// CB_COLOR_CONTROL.operation controls special CB operations, not the normal color component
-	// write mask. Use CB_TARGET_MASK here so scanout passes with mode=Disable do not go black.
+	// Color-control operation selects special color-buffer paths, not the normal component write
+	// mask. Attachment availability therefore follows the target write mask.
 	for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) {
 		ret.color_write_enable[i] =
 		    (i < color_count &&
@@ -549,6 +538,70 @@ struct DrawCallInfo {
 	uint32_t             first_instance = 0;
 };
 
+static bool ResolveDccAttachmentClear(TextureCache& cache, const RenderColorInfo& target,
+                                      const ImageViewInfo& view, vk::ClearColorValue& clear_value) {
+	if (target.desc.info.metadata.kind != ImageMetadataKind::Dcc) {
+		return false;
+	}
+
+	// A DCC fast clear may update metadata only and leave the color allocation stale. Vulkan has
+	// no guest DCC state, so materialize the deferred value when the surface is next bound.
+	const auto metadata_address = target.desc.info.metadata.range.address;
+	uint32_t   metadata_value   = 0xffffffffu;
+	if (!cache.IsMetaCleared(metadata_address, view.base_layer, &metadata_value)) {
+		return false;
+	}
+
+	// Translate recognized constant and register-backed DCC states into a host clear value.
+	clear_value = {};
+	switch (static_cast<uint8_t>(metadata_value)) {
+		case 0x00: break;
+		case 0x20:
+			if (!target.metadata_clear_supported) {
+				return false;
+			}
+			clear_value = target.color_clear_value;
+			break;
+		case 0x40:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[3] = 1.0f;
+			break;
+		case 0x80:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[0] = 1.0f;
+			clear_value.float32[1] = 1.0f;
+			clear_value.float32[2] = 1.0f;
+			break;
+		case 0xc0:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[0] = 1.0f;
+			clear_value.float32[1] = 1.0f;
+			clear_value.float32[2] = 1.0f;
+			clear_value.float32[3] = 1.0f;
+			break;
+		default: return false;
+	}
+
+	for (uint32_t layer = 1; layer < view.layer_count; layer++) {
+		if (!cache.IsMetaCleared(metadata_address, view.base_layer + layer)) {
+			return false;
+		}
+	}
+	// Consume only after every layer in this Vulkan view can be materialized together.
+	for (uint32_t layer = 0; layer < view.layer_count; layer++) {
+		if (!cache.TouchMeta(metadata_address, view.base_layer + layer, false)) {
+			EXIT("failed to consume DCC clear state\n");
+		}
+	}
+	return true;
+}
+
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
                                                  uint32_t color_count, RenderDepthInfo& depth) {
 	EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
@@ -562,7 +615,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
-		const auto old_image = cache.ResolveOwner(target.image_id);
+		const auto old_image = cache.m_slot_images.try_get(target.image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
@@ -573,6 +626,15 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		}
 		target.image_view = cache.FindRenderTarget(target.image_id, target.desc);
 		auto& image       = cache.GetImage(target.image_id);
+		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
+		                     "Kyty.MRT{}.Image[guest=0x{:016x} size=0x{:x} format={}]",
+		                     target.target_slot, image.info.data.address, image.info.data.size,
+		                     static_cast<uint32_t>(image.info.pixel_format));
+		SetVulkanObjectNameF(m_context.GetGraphics().device, target.image_view,
+		                     "Kyty.MRT{}.View[guest=0x{:016x} mip={} layer={}+{}]",
+		                     target.target_slot, image.info.data.address,
+		                     target.desc.view_info.base_level, target.desc.view_info.base_layer,
+		                     target.desc.view_info.layer_count);
 		EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr);
 		if (attachment_samples == 0) {
 			attachment_samples = target.samples;
@@ -596,10 +658,16 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.image_view   = target.image_view;
 		attachment.image_layout = layout;
 		attachment.clear_value  = target.color_clear_value.uint32;
-		attachment.is_clear     = target.color_clear_enable;
+		vk::ClearColorValue metadata_clear_value {};
+		const bool          metadata_clear =
+		    ResolveDccAttachmentClear(cache, target, view, metadata_clear_value);
+		if (metadata_clear) {
+			attachment.clear_value = metadata_clear_value.uint32;
+		}
+		attachment.is_clear = target.color_clear_enable || metadata_clear;
 	}
 	if (depth.image_id) {
-		const auto owner = cache.ResolveOwner(depth.image_id);
+		const auto owner = cache.m_slot_images.try_get(depth.image_id);
 		if (owner == nullptr || !owner->registered || owner->binding.needs_rebind) {
 			EXIT("depth target changed after render-state discovery\n");
 		}
@@ -616,6 +684,14 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			EXIT("failed to consume HTile clear state\n");
 		}
 		auto& image = cache.GetImage(depth.image_id);
+		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
+		                     "Kyty.DepthTarget.Image[guest=0x{:016x} size=0x{:x} format={}]",
+		                     image.info.data.address, image.info.data.size,
+		                     static_cast<uint32_t>(image.info.pixel_format));
+		SetVulkanObjectNameF(m_context.GetGraphics().device, depth.image_view,
+		                     "Kyty.DepthTarget.View[guest=0x{:016x} layer={}+{}]",
+		                     image.info.data.address, depth.desc.view_info.base_layer,
+		                     depth.desc.view_info.layer_count);
 		EXIT_IF(depth.image_view == nullptr || image.backing.samples != depth.samples);
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
@@ -648,8 +724,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.has_stencil    = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
 		attachment.stencil_clear  = depth.stencil_clear_enable;
 	}
-	if (attachment_samples == 0 ||
-	    vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
+	if (color_count == 0 && !depth.image_id) {
+		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
+		state.width        = limits.maxFramebufferWidth;
+		state.height       = limits.maxFramebufferHeight;
+	} else if (attachment_samples == 0 ||
+	           vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
 		EXIT("render state has no valid attachments\n");
 	}
 	if (state.num_layers == std::numeric_limits<uint32_t>::max()) {
@@ -680,10 +760,11 @@ enum class CbColorMode : uint8_t {
 static bool ConsumeMetadataColorOperation(const RenderCommandBuffer& buffer) {
 	const auto& ctx  = buffer.GetRegisters();
 	const auto  mode = ctx.GetColorControl().mode;
-	// These AGC CB modes run color-buffer metadata/decompression operations. The shader is a
-	// dummy vehicle for the CB, and its exported color must not be applied as a normal draw.
-	// Kyty currently stores host images as expanded Vulkan images and does not track CMASK/DCC
-	// metadata state, so the matching host operation is a no-op.
+	// These special modes run color-buffer metadata or decompression operations. The shader is a
+	// vehicle for that operation, and its exported color must not be applied as a normal draw.
+	// Kyty stores expanded Vulkan images rather than compressed guest surfaces, so no equivalent
+	// hardware pass is emitted. Tracked DCC clear state is materialized on attachment bind;
+	// future CMask/FMask support can consume its state through the same TextureCache path.
 	return mode == static_cast<uint8_t>(CbColorMode::EliminateFastClear) ||
 	       mode == static_cast<uint8_t>(CbColorMode::FmaskDecompress) ||
 	       mode == static_cast<uint8_t>(CbColorMode::DccDecompress);
@@ -704,11 +785,10 @@ struct DrawIndexBufferSource {
 };
 
 struct PreparedIndexBuffer {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	uint64_t              size   = 0;
-	vk::DeviceSize        offset = 0;
-	vk::IndexType         type   = vk::IndexType::eUint16;
+	vk::Buffer     buffer = nullptr;
+	uint64_t       size   = 0;
+	vk::DeviceSize offset = 0;
+	vk::IndexType  type   = vk::IndexType::eUint16;
 };
 
 static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) {
@@ -717,10 +797,10 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
 }
 
 struct VertexBufferRange {
-	uint64_t      base_address  = 0;
-	uint64_t      requested_end = 0;
-	uint64_t      acquired_end  = 0;
-	BufferBinding binding;
+	uint64_t                     base_address  = 0;
+	uint64_t                     requested_end = 0;
+	uint64_t                     acquired_end  = 0;
+	std::pair<Buffer*, uint64_t> binding;
 
 	[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
 };
@@ -728,11 +808,9 @@ struct VertexBufferRange {
 struct PreparedVertexBuffers {
 	static constexpr uint32_t MaxBuffers = ShaderVertexInputInfo::RES_MAX;
 
-	std::array<vk::Buffer, MaxBuffers>            buffers {};
-	std::array<vk::DeviceSize, MaxBuffers>        offsets {};
-	std::array<std::shared_ptr<void>, MaxBuffers> owners {};
-	uint32_t                                      count       = 0;
-	uint32_t                                      owner_count = 0;
+	std::array<vk::Buffer, MaxBuffers>     buffers {};
+	std::array<vk::DeviceSize, MaxBuffers> offsets {};
+	uint32_t                               count = 0;
 };
 
 static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         buffer,
@@ -782,21 +860,22 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         b
 		const auto size =
 		    Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
 		range.acquired_end = range.base_address + size;
-		range.binding      = cache.ObtainBuffer(buffer, range.base_address, size);
+		range.binding      = cache.ObtainBuffer(range.base_address, size, false);
+		SetVulkanObjectNameF(
+		    buffer.GetContext().GetGraphics().device, range.binding.first->Handle(),
+		    "Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", range.base_address, size);
 	}
 
 	// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
 	PreparedVertexBuffers prepared;
-	prepared.count = static_cast<uint32_t>(vs_input_info.buffers_num);
-	std::shared_ptr<Buffer> null_owner;
-	vk::Buffer              null_buffer = nullptr;
+	prepared.count         = static_cast<uint32_t>(vs_input_info.buffers_num);
+	vk::Buffer null_buffer = nullptr;
 	for (int i = 0; i < vs_input_info.buffers_num; i++) {
 		const auto& vertex = vs_input_info.buffers[i];
 		const auto  size   = VertexBufferDescriptorSize(vertex);
 		if (size == 0) {
-			if (null_owner == nullptr) {
-				null_owner  = cache.ObtainNullBuffer();
-				null_buffer = null_owner->Handle();
+			if (null_buffer == nullptr) {
+				null_buffer = cache.GetBuffer(NULL_BUFFER_ID).Handle();
 			}
 			prepared.buffers[i] = null_buffer;
 			prepared.offsets[i] = 0;
@@ -813,19 +892,14 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         b
 			     vertex.addr);
 		}
 
-		prepared.buffers[i] = range->binding.buffer;
-		prepared.offsets[i] = range->binding.offset + vertex.addr - range->base_address;
+		prepared.buffers[i] = range->binding.first->Handle();
+		prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address;
+		SetVulkanObjectNameF(
+		    buffer.GetContext().GetGraphics().device, prepared.buffers[i],
+		    "Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i,
+		    vertex.addr, size, vertex.stride, vertex.num_records);
 	}
 
-	if (null_owner != nullptr) {
-		prepared.owners[prepared.owner_count++] = std::move(null_owner);
-	}
-	for (uint32_t i = 0; i < merged_count; i++) {
-		if (merged_ranges[i].binding.owner != nullptr) {
-			EXIT_IF(prepared.owner_count >= PreparedVertexBuffers::MaxBuffers);
-			prepared.owners[prepared.owner_count++] = std::move(merged_ranges[i].binding.owner);
-		}
-	}
 	return prepared;
 }
 
@@ -941,14 +1015,14 @@ bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuf
 	}
 	ResolveRenderDepthTarget(submit_id, buffer, state.depth_info);
 
+	state.ps_active       = DrawHasActivePixelShader(buffer);
 	const bool with_depth = (state.depth_info.format != vk::Format::eUndefined &&
 	                         static_cast<bool>(state.depth_info.image_id));
-	if (state.color_count == 0 && !with_depth) {
+	if (state.color_count == 0 && !with_depth && !state.ps_active) {
 		LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer,
 		                   draw.index_count, draw.flags);
 		return false;
 	}
-	state.ps_active = DrawHasActivePixelShader(buffer);
 
 	return true;
 }
@@ -971,12 +1045,10 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
 	for (uint32_t i = 0; i < state.color_count; i++) {
 		target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
 	}
-	const auto lane_mask_mode = SelectGraphicsLaneMaskMode(64u);
-
 	if (log_phases) {
 		LogDrawPhase(draw.name, "ShaderCompileInfoVS");
 	}
-	if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
+	if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, state.vs_input_info,
 	                         state.vs_shader)) {
 		EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name);
 	}
@@ -987,8 +1059,9 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
 	if (log_phases) {
 		LogDrawPhase(draw.name, "ShaderCompileInfoPS");
 	}
-	if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
-	                         target_export_mapping, state.ps_input_info, state.ps_shader)) {
+	if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, state.vs_input_info,
+	                         target_export_mapping,
+	                         state.ps_input_info, state.ps_shader)) {
 		EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name);
 	}
 }
@@ -1013,28 +1086,29 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffe
 	prepared.size = source.size;
 	prepared.type = source.type;
 	if (source.host_data != nullptr) {
-		auto binding =
-		    buffer.GetContext().GetBufferCache().UploadTransient(source.host_data, source.size, 16);
-		prepared.owner  = std::move(binding.owner);
-		prepared.buffer = binding.buffer;
-		prepared.offset = binding.offset;
+		auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+		prepared.offset = stream.Copy(source.host_data, source.size, 16);
+		prepared.buffer = stream.Handle();
 	} else {
-		auto binding =
-		    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, source.address, source.size);
-		prepared.owner  = std::move(binding.owner);
-		prepared.buffer = binding.buffer;
-		prepared.offset = binding.offset;
+		auto [buffer_ptr, offset] =
+		    buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
+		prepared.buffer = buffer_ptr->Handle();
+		prepared.offset = offset;
+	}
+	if (source.host_data != nullptr) {
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
+		                     static_cast<uint32_t>(source.type));
+	} else {
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
+		                     source.address, source.size, static_cast<uint32_t>(source.type));
 	}
 	return prepared;
 }
 
-static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                                PreparedVertexBuffers& prepared) {
-	for (uint32_t i = 0; i < prepared.owner_count; i++) {
-		if (prepared.owners[i] != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(prepared.owners[i]));
-		}
-	}
+static void CommitVertexBuffers(vk::CommandBuffer            vk_buffer,
+                                const PreparedVertexBuffers& prepared) {
 	for (uint32_t i = 0; i < prepared.count; i++) {
 		EXIT_IF(prepared.buffers[i] == nullptr);
 	}
@@ -1044,14 +1118,10 @@ static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer v
 	}
 }
 
-static void CommitIndexBuffer(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                              PreparedIndexBuffer& prepared) {
+static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBuffer& prepared) {
 	if (prepared.size == 0) {
 		return;
 	}
-	if (prepared.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(prepared.owner));
-	}
 	EXIT_IF(prepared.buffer == nullptr);
 	vk_buffer.bindIndexBuffer(prepared.buffer, prepared.offset, prepared.type);
 }
@@ -1134,8 +1204,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	auto& ucfg = buffer.GetUserConfig();
 
 	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings        = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
-	                                               state.ps_input_info.stage, state.ps_active);
+	auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage,
+	                                        state.ps_active);
 	auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info);
 	auto index_binding   = PrepareIndexBuffer(buffer, index_source);
 	state.rendering =
@@ -1159,17 +1229,20 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	if (set_auto_debug) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
 	}
-	CommitVertexBuffers(buffer, vk_buffer, vertex_bindings);
-	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout,
-	               bindings.vertex);
+	CommitVertexBuffers(vk_buffer, vertex_bindings);
 	if (bindings.pixel.has_value()) {
 		if (set_auto_debug) {
 			SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 		}
-		CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout,
-		               *bindings.pixel);
 	}
-	CommitIndexBuffer(buffer, vk_buffer, index_binding);
+	std::array<PreparedBindings*, 2> descriptor_stages {&bindings.vertex, nullptr};
+	const size_t                     descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u;
+	if (bindings.pixel) {
+		descriptor_stages[1] = &*bindings.pixel;
+	}
+	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
+	               std::span {descriptor_stages.data(), descriptor_stage_count});
+	CommitIndexBuffer(vk_buffer, index_binding);
 
 	const auto dynamic_params =
 	    BuildGraphicsDynamicParams(buffer, state.color_info, state.color_count, state.depth_info);
@@ -1214,6 +1287,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ucfg   = buffer.GetUserConfig();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -1345,6 +1419,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ucfg   = buffer.GetUserConfig();
 	auto& sh_ctx = buffer.GetShaders();
 

src/graphics/host_gpu/renderer/sync.cpp

@@ -390,7 +390,7 @@ int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) {
 	return result;
 }
 
-void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
+void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
 	const auto offset = uint64_t {dw_offset} * sizeof(uint32_t);
 	const auto size   = uint64_t {dw_size} * sizeof(uint32_t);
 	EXIT_IF(dst == nullptr || offset > gds.Size() || size > gds.Size() - offset ||

src/graphics/host_gpu/renderer/sync.h

@@ -62,7 +62,7 @@ void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuf
 int  AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                 void* udata);
 int  DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id);
-void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
+void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
 
 } // namespace Sync
 } // namespace Libs::Graphics

src/graphics/host_gpu/vma.cpp

@@ -199,20 +199,6 @@ void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) {
 	VulkanTrackAllocation(buffer.memory);
 }
 
-void GraphicContext::DeleteBuffer(VulkanBuffer& buffer) {
-	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(allocator == nullptr || buffer.buffer == nullptr ||
-	        buffer.memory.allocation == nullptr);
-
-	VulkanUntrackAllocation(buffer.memory);
-	vmaDestroyBuffer(allocator, buffer.buffer, buffer.memory.allocation);
-	buffer.buffer                 = nullptr;
-	buffer.memory.memory          = nullptr;
-	buffer.memory.allocation      = nullptr;
-	buffer.memory.allocation_info = {};
-	buffer.memory.offset          = 0;
-}
-
 bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanImage& image) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(allocator == nullptr || image.image != nullptr || image.memory.allocation != nullptr);

src/graphics/host_gpu/vulkanCommon.h

@@ -10,10 +10,13 @@
 #define VMA_STATIC_VULKAN_FUNCTIONS  0
 #define VMA_DYNAMIC_VULKAN_FUNCTIONS 1
 
+#include "common/emulatorConfig.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 
 #include <cstdint>
+#include <fmt/format.h>
 #include <string>
+#include <utility>
 #include <vector>
 #include <vulkan/vulkan.hpp>
 #include <vulkan/vulkan_hash.hpp>
@@ -29,6 +32,24 @@ std::string VulkanToString(vk::QueueFlags value);
 vk::Format  VulkanFormat(Prospero::BufferFormat guest_format);
 void        RequireVulkanSuccess(vk::Result result, const char* operation);
 
+template <typename Handle, typename... Args>
+void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<Args...> format,
+                          Args&&... args) {
+	if (!Config::GraphicsDebugDumpEnabled() || device == nullptr || handle == nullptr ||
+	    VULKAN_HPP_DEFAULT_DISPATCHER.vkSetDebugUtilsObjectNameEXT == nullptr) {
+		return;
+	}
+
+	const auto                      name = fmt::format(format, std::forward<Args>(args)...);
+	vk::DebugUtilsObjectNameInfoEXT info {};
+	info.sType        = vk::StructureType::eDebugUtilsObjectNameInfoEXT;
+	info.objectType   = Handle::objectType;
+	info.objectHandle = static_cast<uint64_t>(
+	    reinterpret_cast<uintptr_t>(static_cast<typename Handle::CType>(handle)));
+	info.pObjectName = name.c_str();
+	(void)device.setDebugUtilsObjectNameEXT(&info);
+}
+
 template <typename T, typename Enumerator>
 [[nodiscard]] std::vector<T> EnumerateVulkan(const char* operation, Enumerator&& enumerate) {
 	for (;;) {

src/graphics/host_gpu/vulkanInstance.h

@@ -30,6 +30,7 @@ struct VulkanInstance {
 	uint32_t                           subgroup_size                     = 0;
 	uint32_t                           min_subgroup_size                 = 0;
 	uint32_t                           max_subgroup_size                 = 0;
+	uint32_t                           max_push_descriptors              = 0;
 	vk::ShaderStageFlags               required_subgroup_size_stages     = {};
 	Common::Mutex                      queue_mutex;
 	uint32_t                           queue_family = static_cast<uint32_t>(-1);

src/graphics/presentation/renderDoc.cpp

@@ -25,11 +25,13 @@ namespace Libs::Graphics {
 enum class RenderDocState : uint32_t {
 	Idle,
 	Requested,
+	Starting,
 	Capturing,
 };
 
 static RENDERDOC_API_1_6_0*        g_api             = nullptr;
 static std::atomic<RenderDocState> g_state           = RenderDocState::Idle;
+static std::atomic_uint32_t        g_captured_flips  = 0;
 static std::atomic_bool            g_unavailable_log = false;
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -140,7 +142,7 @@ bool RenderDocCaptureInProgress() {
 
 void RenderDocStartCapture() {
 	RenderDocState expected = RenderDocState::Requested;
-	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Capturing,
+	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Starting,
 	                                                         std::memory_order_acq_rel)) {
 		return;
 	}
@@ -162,6 +164,8 @@ void RenderDocStartCapture() {
 		LOGF("RenderDoc: capture failed to start\n");
 		return;
 	}
+	g_captured_flips.store(0, std::memory_order_release);
+	g_state.store(RenderDocState::Capturing, std::memory_order_release);
 	LOGF("RenderDoc: capture started\n");
 }
 
@@ -175,4 +179,15 @@ void RenderDocEndCapture() {
 	LOGF(ok != 0 ? "RenderDoc: capture finished\n" : "RenderDoc: capture failed\n");
 }
 
+void RenderDocOnGuestFlip() {
+	if (!RenderDocCaptureInProgress()) {
+		return;
+	}
+	const auto flip = g_captured_flips.fetch_add(1, std::memory_order_acq_rel) + 1;
+	LOGF("RenderDoc: captured guest flip %u/2\n", flip);
+	if (flip >= 2) {
+		RenderDocEndCapture();
+	}
+}
+
 } // namespace Libs::Graphics

src/graphics/presentation/renderDoc.h

@@ -7,6 +7,7 @@ void RenderDocInit();
 void RenderDocRequestCapture();
 void RenderDocStartCapture();
 void RenderDocEndCapture();
+void RenderDocOnGuestFlip();
 
 [[nodiscard]] bool RenderDocCaptureRequested();
 [[nodiscard]] bool RenderDocCaptureInProgress();

src/graphics/presentation/videoOut.cpp

@@ -16,6 +16,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/presentation/presenter.h"
+#include "graphics/presentation/renderDoc.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "libs/libs.h"
@@ -42,6 +43,9 @@ constexpr int      VIDEO_OUT_EVENT_PRE_VBLANK_START                     = 2;
 constexpr int      VIDEO_OUT_EVENT_SET_MODE                             = 8;
 constexpr int      VIDEO_OUT_TRUE                                       = 1;
 constexpr int      VIDEO_OUT_FALSE                                      = 0;
+constexpr int      VIDEO_OUT_BUS_TYPE_MAIN                              = 0;
+constexpr int      VIDEO_OUT_BUS_TYPE_OVERLAY                           = 1;
+constexpr int      VIDEO_OUT_BUS_TYPE_SUB                               = 2;
 constexpr int      VIDEO_OUT_FLIP_MODE_VSYNC                            = 1;
 constexpr int      VIDEO_OUT_FLIP_MODE_VSYNC_MULTI                      = 4;
 constexpr int      VIDEO_OUT_BUFFER_INDEX_BLACK                         = -2;
@@ -233,7 +237,7 @@ private:
 
 struct VideoOutDriver::Impl {
 public:
-	static constexpr int VIDEO_OUT_NUM_MAX = 2;
+	static constexpr int VIDEO_OUT_NUM_MAX = 4;
 
 	Impl(uint32_t width, uint32_t height, Graphics::Presenter& presenter)
 	    : m_renderer(presenter.Renderer()), m_presenter(presenter), m_flip_queue(presenter) {
@@ -244,7 +248,7 @@ public:
 	~Impl();
 	KYTY_CLASS_NO_COPY(Impl);
 
-	int             Open();
+	int             Open(int bus_type);
 	bool            Close(int handle);
 	VideoOutConfig* Get(int handle);
 	VideoOutConfig* Get(int handle, uint64_t& generation);
@@ -606,19 +610,11 @@ void VideoOutDriver::Impl::Init(uint32_t width, uint32_t height) {
 	}
 }
 
-int VideoOutDriver::Impl::Open() {
+int VideoOutDriver::Impl::Open(int bus_type) {
 	Common::LockGuard lock(m_mutex);
 
-	int handle = -1;
-
-	for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) {
-		if (!m_video_out_ctx[i].opened) {
-			handle = i;
-			break;
-		}
-	}
-
-	if (handle < 0) {
+	const int handle = bus_type + 1;
+	if (m_video_out_ctx[handle].opened) {
 		return -1;
 	}
 	auto&             config = m_video_out_ctx[handle];
@@ -1127,6 +1123,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 	m_mutex.Unlock();
 
 	m_presenter.Present(*r.frame);
+	Graphics::RenderDocOnGuestFlip();
 
 	m_mutex.Lock();
 	if (m_requests.empty() || m_requests.front().id != r.id ||
@@ -1171,12 +1168,15 @@ KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void*
 	PRINT_NAME();
 
 	EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0);
-	EXIT_NOT_IMPLEMENTED(bus_type != 0);
+	if (bus_type != VIDEO_OUT_BUS_TYPE_MAIN && bus_type != VIDEO_OUT_BUS_TYPE_OVERLAY &&
+	    bus_type != VIDEO_OUT_BUS_TYPE_SUB) {
+		return VIDEO_OUT_ERROR_INVALID_VALUE;
+	}
 	EXIT_NOT_IMPLEMENTED(index != 0);
 
 	LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast<uint64_t>(param));
 
-	int handle = DriverState().Open();
+	int handle = DriverState().Open(bus_type);
 
 	if (handle < 0) {
 		return VIDEO_OUT_ERROR_RESOURCE_BUSY;

src/graphics/presentation/window/vulkanWindow.cpp

@@ -199,7 +199,15 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
 		features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
+#if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
+#else
+		vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
+		fragment_barycentric.sType =
+		    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+		fragment_barycentric.pNext = &depth_clip_control;
+		features12.pNext           = &fragment_barycentric;
+#endif
 		features13.pNext = &features12;
 
 		device_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
@@ -231,6 +239,12 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			skip_device = true;
 #endif
 		}
+#if !defined(__APPLE__)
+		if (fragment_barycentric.fragmentShaderBarycentric != VK_TRUE) {
+			LOGF("fragmentShaderBarycentric is not supported\n");
+			skip_device = true;
+		}
+#endif
 
 		if (features12.samplerMirrorClampToEdge != VK_TRUE) {
 			LOGF("samplerMirrorClampToEdge is not supported\n");
@@ -262,6 +276,22 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			LOGF("depthBiasClamp is not supported\n");
 			skip_device = true;
 		}
+		if (device_features2.features.shaderClipDistance != VK_TRUE) {
+			LOGF("shaderClipDistance is not supported\n");
+			skip_device = true;
+		}
+		if (device_features2.features.shaderCullDistance != VK_TRUE) {
+			LOGF("shaderCullDistance is not supported\n");
+			skip_device = true;
+		}
+		if (device_features2.features.largePoints != VK_TRUE) {
+			LOGF("largePoints is not supported\n");
+			skip_device = true;
+		}
+		if (features12.shaderOutputLayer != VK_TRUE) {
+			LOGF("shaderOutputLayer is not supported\n");
+			skip_device = true;
+		}
 
 		if (device_features2.features.fragmentStoresAndAtomics != VK_TRUE) {
 			LOGF("fragmentStoresAndAtomics is not supported\n");
@@ -286,10 +316,6 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			LOGF("shaderStorageImageWriteWithoutFormat is not supported\n");
 			skip_device = true;
 		}
-		if (device_features2.features.shaderStorageImageReadWithoutFormat != VK_TRUE) {
-			LOGF("shaderStorageImageReadWithoutFormat is not supported\n");
-			skip_device = true;
-		}
 
 		if (device_features2.features.shaderImageGatherExtended != VK_TRUE) {
 			LOGF("shaderImageGatherExtended is not supported\n");
@@ -518,10 +544,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	features12.pNext                    = &depth_clip_control;
 	features12.samplerMirrorClampToEdge = VK_TRUE;
 
-	vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {};
-	subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures;
-	subgroup_size_control.pNext = &features12;
-
 	vk::PhysicalDeviceVulkan13Features supported_features13 {};
 	supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
 
@@ -530,6 +552,14 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	supported_features12.pNext = nullptr;
 	supported_features13.pNext = &supported_features12;
 
+#if !defined(__APPLE__)
+	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_fragment_barycentric {};
+	supported_fragment_barycentric.sType =
+	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+	supported_fragment_barycentric.pNext = nullptr;
+	supported_features12.pNext           = &supported_fragment_barycentric;
+#endif
+
 	const auto robustness2_ext_enabled =
 	    HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
 
@@ -537,7 +567,11 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
 	supported_robustness2.pNext = nullptr;
 	if (robustness2_ext_enabled) {
+#if defined(__APPLE__)
 		supported_features12.pNext = &supported_robustness2;
+#else
+		supported_fragment_barycentric.pNext = &supported_robustness2;
+#endif
 	}
 
 	vk::PhysicalDeviceFeatures2 supported_features2 {};
@@ -552,7 +586,23 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	                     supported_features13.synchronization2 != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE);
+#if !defined(__APPLE__)
+	EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
+#endif
 	features12.timelineSemaphore = VK_TRUE;
+	features12.shaderOutputLayer = VK_TRUE;
+
+	if (Config::GpuAssistedValidationEnabled()) {
+		if (supported_features12.bufferDeviceAddress != VK_TRUE) {
+			EXIT("--gpu-assisted-validation requires bufferDeviceAddress, unsupported by this "
+			     "device\n");
+		}
+		features12.bufferDeviceAddress = VK_TRUE;
+	}
 
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
@@ -562,38 +612,52 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	device_features.depthBounds = VK_TRUE; // unsupported by MoltenVK
 #endif
 	device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
-	device_features.shaderStorageImageReadWithoutFormat  = VK_TRUE;
 	device_features.shaderImageGatherExtended            = VK_TRUE;
 	device_features.independentBlend                     = VK_TRUE;
 	device_features.tessellationShader                   = VK_TRUE;
 	device_features.sampleRateShading                    = VK_TRUE;
 	device_features.depthBiasClamp                       = VK_TRUE;
+	device_features.shaderClipDistance                   = VK_TRUE;
+	device_features.shaderCullDistance                   = VK_TRUE;
+	device_features.largePoints                          = VK_TRUE;
 	graphics.sample_rate_shading_enabled                 = true;
 	device_features.vertexPipelineStoresAndAtomics =
 	    supported_features2.features.vertexPipelineStoresAndAtomics;
-
-	if (graphics.subgroup_size_control_enabled &&
-	    supported_features13.subgroupSizeControl == VK_TRUE) {
-		subgroup_size_control.subgroupSizeControl = VK_TRUE;
+	if (Config::GpuAssistedValidationEnabled()) {
+		device_features.shaderInt64 = supported_features2.features.shaderInt64;
 	}
 
-	const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE
-	                                      ? static_cast<const void*>(&subgroup_size_control)
-	                                      : static_cast<const void*>(&features12));
-
 	vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
 	robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
-	robustness2.pNext = const_cast<void*>(base_feature_chain);
+#if defined(__APPLE__)
+	robustness2.pNext = &features12;
+#else
+	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
+	fragment_barycentric.sType =
+	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+	fragment_barycentric.pNext                     = &features12;
+	fragment_barycentric.fragmentShaderBarycentric = VK_TRUE;
+	robustness2.pNext                              = &fragment_barycentric;
+#endif
 	if (robustness2_ext_enabled) {
 		robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2;
 		robustness2.robustImageAccess2  = supported_robustness2.robustImageAccess2;
 		robustness2.nullDescriptor      = supported_robustness2.nullDescriptor;
 	}
 
+	const bool subgroup_size_control_enabled = graphics.subgroup_size_control_enabled &&
+	                                           supported_features13.subgroupSizeControl == VK_TRUE;
+
 	auto features13 = required_features13;
-	features13.pNext =
-	    robustness2_ext_enabled ? &robustness2 : const_cast<void*>(base_feature_chain);
-	features13.robustImageAccess = supported_features13.robustImageAccess;
+#if defined(__APPLE__)
+	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
+	                                           : static_cast<void*>(&features12);
+#else
+	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
+	                                           : static_cast<void*>(&fragment_barycentric);
+#endif
+	features13.robustImageAccess   = supported_features13.robustImageAccess;
+	features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
 
 	LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
 	     features13.robustImageAccess == VK_TRUE ? "true" : "false",
@@ -833,23 +897,26 @@ void WindowContext::CreateVulkan() {
 	app_info.engineVersion      = 1;
 	app_info.apiVersion         = VULKAN_TARGET_API_VERSION; // NOLINT
 
-	vk::ValidationFeatureEnableEXT enabled_features[] = {
+	if (Config::SpirvDebugPrintfEnabled() && Config::GpuAssistedValidationEnabled()) {
+		EXIT("--spirv-debug-printf and --gpu-assisted-validation are mutually exclusive\n");
+	}
+
+	vk::ValidationFeatureEnableEXT enabled_features[3]    = {};
+	uint32_t                       enabled_features_count = 0;
 #ifdef KYTY_ENABLE_BEST_PRACTICES
-	    vk::ValidationFeatureEnableEXT::eBestPractices,
+	enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eBestPractices;
 #endif
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
-	    vk::ValidationFeatureEnableEXT::eDebugPrintf,
-#endif
-	};
-
-	uint32_t enabled_features_count =
-	    sizeof(enabled_features) / sizeof(vk::ValidationFeatureEnableEXT);
-
-#ifdef KYTY_ENABLE_DEBUG_PRINTF
-	if (!Config::SpirvDebugPrintfEnabled()) {
-		enabled_features_count--;
+	if (Config::SpirvDebugPrintfEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eDebugPrintf;
 	}
 #endif
+	if (Config::GpuAssistedValidationEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eGpuAssisted;
+		LOGF("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::printf("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::fflush(stdout);
+	}
 
 	vk::ValidationFeaturesEXT validation_features {};
 	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
@@ -922,6 +989,7 @@ void WindowContext::CreateVulkan() {
 #else
 	device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
 	device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
+	device_extensions.push_back(VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME);
 #endif
 
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
@@ -939,7 +1007,15 @@ void WindowContext::CreateVulkan() {
 		EXIT("Could not find suitable device");
 	}
 
-	graphic_ctx.physical_device.getProperties(&graphic_ctx.physical_device_properties);
+	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
+	vk::PhysicalDeviceProperties2              physical_device_properties {};
+	push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties;
+	push_descriptor_properties.pNext = nullptr;
+	physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2;
+	physical_device_properties.pNext = &push_descriptor_properties;
+	graphic_ctx.physical_device.getProperties2(&physical_device_properties);
+	graphic_ctx.physical_device_properties = physical_device_properties.properties;
+	graphic_ctx.max_push_descriptors       = push_descriptor_properties.maxPushDescriptors;
 	graphic_ctx.physical_device.getMemoryProperties(&graphic_ctx.physical_device_memory_properties);
 	const auto& device_properties = graphic_ctx.GetPhysicalDeviceProperties();
 

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -46,6 +46,8 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
 	std::string dump = "CFG:\n";
 	dump += CFG::GraphToString(cfg);
 	dump += "\nIR:\n";
+	dump += fmt::format("mode={} scratch_dwords={}\n",
+	                    ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords);
 	dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir);
 	return dump;
 }
@@ -77,85 +79,6 @@ void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg,
 	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
 }
 
-bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
-	switch (inst.op) {
-		case IR::Opcode::SLoadDword:
-		case IR::Opcode::SBufferLoadDword:
-		case IR::Opcode::BufferLoadUbyte:
-		case IR::Opcode::BufferLoadSbyte:
-		case IR::Opcode::BufferLoadUshort:
-		case IR::Opcode::BufferLoadSshort:
-		case IR::Opcode::BufferLoadDword:
-		case IR::Opcode::BufferStoreByte:
-		case IR::Opcode::BufferStoreShort:
-		case IR::Opcode::BufferStoreDword:
-		case IR::Opcode::AtomicSwapU32:
-		case IR::Opcode::AtomicAddU32:
-		case IR::Opcode::AtomicSubU32:
-		case IR::Opcode::AtomicSMinI32:
-		case IR::Opcode::AtomicUMinU32:
-		case IR::Opcode::AtomicSMaxI32:
-		case IR::Opcode::AtomicUMaxU32:
-		case IR::Opcode::AtomicAndU32:
-		case IR::Opcode::AtomicOrU32:
-		case IR::Opcode::AtomicXorU32:
-		case IR::Opcode::AtomicFMinF32:
-		case IR::Opcode::AtomicFMaxF32:
-		case IR::Opcode::FlatLoadUbyte:
-		case IR::Opcode::FlatLoadSbyte:
-		case IR::Opcode::FlatLoadUshort:
-		case IR::Opcode::FlatLoadSshort:
-		case IR::Opcode::FlatLoadDword:
-		case IR::Opcode::FlatStoreByte:
-		case IR::Opcode::FlatStoreShort:
-		case IR::Opcode::FlatStoreDword:
-		case IR::Opcode::DsReadUbyte:
-		case IR::Opcode::DsReadSbyte:
-		case IR::Opcode::DsReadUshort:
-		case IR::Opcode::DsReadSshort:
-		case IR::Opcode::DsReadB32:
-		case IR::Opcode::DsMinF32:
-		case IR::Opcode::DsMaxF32:
-		case IR::Opcode::DsWriteByte:
-		case IR::Opcode::DsWriteShort:
-		case IR::Opcode::DsWriteB32:
-		case IR::Opcode::DsWriteAddtidB32:
-		case IR::Opcode::DsReadAddtidB32:
-		case IR::Opcode::DsAppend:
-		case IR::Opcode::DsConsume:
-		case IR::Opcode::ImageLoad:
-		case IR::Opcode::ImageStore:
-		case IR::Opcode::ImageSample:
-		case IR::Opcode::ImageGather4:
-		case IR::Opcode::Export: return true;
-		default: return false;
-	}
-}
-
-bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, uint32_t* failure_block,
-                                            std::string* reason) {
-	for (const auto& block: ir.blocks) {
-		if (!block.terminator.loop_header) {
-			continue;
-		}
-		for (const auto& inst: block.instructions) {
-			if (!InstructionMaySplitSpirvBlock(inst)) {
-				continue;
-			}
-			if (reason != nullptr) {
-				*reason = fmt::format("loop header block {} contains an instruction whose SPIR-V "
-				                      "lowering emits internal control flow: {}",
-				                      block.id, IR::InstructionToString(inst).c_str());
-			}
-			if (failure_block != nullptr) {
-				*failure_block = block.id;
-			}
-			return true;
-		}
-	}
-	return false;
-}
-
 enum class EmbeddedFetchValueType {
 	Unknown,
 	Constant,
@@ -632,7 +555,9 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputIn
 				continue;
 			}
 			const auto* load = FindEmbeddedFetchLoad(loads, inst.pc);
-			if (load == nullptr || inst.memory.component_index >= load->components) {
+			if (load == nullptr || inst.memory.data_dwords != load->components ||
+			    inst.dst.kind != IR::OperandKind::Register ||
+			    inst.dst.reg.file != IR::RegisterFile::Vector) {
 				continue;
 			}
 
@@ -644,16 +569,16 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputIn
 				continue;
 			}
 
-			inst.op              = IR::Opcode::LoadInputF32;
-			inst.input_info.attr = static_cast<uint32_t>(resource_id);
-			inst.input_info.chan = inst.memory.component_index;
-			inst.memory          = {};
-			inst.src_count       = 0;
-
+			inst.op                         = IR::Opcode::LoadInputF32;
+			inst.input_info.attr            = static_cast<uint32_t>(resource_id);
+			inst.input_info.chan            = 0;
+			inst.input_info.component_count = load->components;
+			inst.memory                     = {};
+			inst.src_count                  = 0;
 			mutable_input_info->resource_fetch_components[resource_id] =
 			    std::max(mutable_input_info->resource_fetch_components[resource_id],
-			             static_cast<int>(inst.input_info.chan + 1u));
-			rewritten++;
+			             static_cast<int>(load->components));
+			rewritten += load->components;
 		}
 	}
 
@@ -758,79 +683,16 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
 			return false;
 		}
-		output.lane_mask_mode  = options.lane_mask_mode;
 		output.shader_hash     = options.shader_hash;
 		output.user_data_base  = options.user_data_base;
 		output.user_data_count = options.user_data_count;
+		output.scratch_dwords  = options.scratch_dwords;
 		return true;
 	};
 	if (!lower_program(ir)) {
 		return false;
 	}
 
-	if (!dispatcher_fallback) {
-		const auto select_dispatcher = [&](uint32_t failure_block, const std::string& reason,
-		                                   const CFG::Graph& diagnostic_cfg) {
-			dispatcher_fallback    = true;
-			dispatcher_reason      = reason;
-			cfg.failure_kind       = CFG::FailureKind::StructuredControlFlow;
-			cfg.failure_block      = failure_block;
-			cfg.unsupported_reason = reason;
-			ir.cfg_failure_kind    = CFG::FailureKind::StructuredControlFlow;
-			ir.fallback_reason     = reason;
-			LogDispatcherFallback(options, diagnostic_cfg, "emit", reason);
-		};
-		const auto retry_budget = cfg.natural_loops.size();
-		for (size_t retry = 0;; retry++) {
-			std::string emitter_reason;
-			uint32_t    failure_block = UINT32_MAX;
-			if (!NeedsDispatcherForStructuredLoopHeader(ir, &failure_block, &emitter_reason)) {
-				break;
-			}
-			if (retry >= retry_budget) {
-				select_dispatcher(failure_block,
-				                  emitter_reason + "; no-clone loop-header isolation exhausted",
-				                  cfg);
-				break;
-			}
-
-			const auto* source_block = cfg.FindBlock(failure_block);
-			const auto  source_pc = source_block != nullptr ? source_block->start_pc : UINT32_MAX;
-			const auto  previous_cfg  = cfg;
-			const auto  blocks_before = cfg.blocks.size();
-			std::string isolation_error;
-			if (!CFG::IsolateLoopHeader(cfg, failure_block, &isolation_error)) {
-				const auto failed_cfg = cfg;
-				cfg                   = previous_cfg;
-				select_dispatcher(failure_block,
-				                  fmt::format("{}; no-clone loop-header isolation failed: {}",
-				                              emitter_reason, isolation_error),
-				                  failed_cfg);
-				break;
-			}
-
-			IR::Program isolated_ir;
-			if (!lower_program(isolated_ir)) {
-				const auto lower_error = error != nullptr ? *error : "IR lowering failed";
-				if (error != nullptr) {
-					error->clear();
-				}
-				cfg = previous_cfg;
-				select_dispatcher(failure_block,
-				                  fmt::format("{}; isolated loop-header lowering failed: {}",
-				                              emitter_reason, lower_error),
-				                  cfg);
-				break;
-			}
-			ir = std::move(isolated_ir);
-			LOGF("%s CFG isolated semantic loop header: stage=%s hash=0x%016" PRIx64
-			     " source_block=%" PRIu32 " pc=0x%08" PRIx32 " blocks=%" PRIu64 "->%" PRIu64
-			     " reason=%s\n",
-			     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
-			     failure_block, source_pc, static_cast<uint64_t>(blocks_before),
-			     static_cast<uint64_t>(cfg.blocks.size()), emitter_reason.c_str());
-		}
-	}
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -867,10 +729,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	ir.blocks.clear();
 	IR::RewriteToSsa(ir.values->blocks);
 	IR::ConstantPropagationPass(ir.values->blocks);
-	for (auto& info: ir.values->block_info) {
-		info.condition       = info.condition.Resolve();
-		info.indirect_target = info.indirect_target.Resolve();
-	}
+	IR::ResolveControlFlowIdentities(*ir.values);
 	IR::RemoveIdentities(ir.values->blocks);
 	IR::EliminateDeadCode(ir.values->blocks);
 	const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size);
@@ -878,6 +737,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
 		     read_lane_stats.rewritten_reads);
 		IR::ConstantPropagationPass(ir.values->blocks);
+		IR::ResolveControlFlowIdentities(*ir.values);
 		IR::RemoveIdentities(ir.values->blocks);
 		IR::EliminateDeadCode(ir.values->blocks);
 	}
@@ -936,10 +796,10 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	if (!IR::CollectShaderInfo(ir, info_options, error)) {
 		return false;
 	}
-	IR::BindingLayoutOptions layout_options;
-	layout_options.descriptor_set       = options.descriptor_set;
-	layout_options.push_constant_offset = options.push_constant_offset;
-	if (!IR::AllocateBindings(ir, layout_options, error)) {
+	if (!IR::AllocateBindings(ir, options.push_constant_offset, error)) {
+		return false;
+	}
+	if (!Spirv::AnalyzeProgramRequirements(ir, error)) {
 		return false;
 	}
 	std::string ir_dump;

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -14,15 +14,14 @@ namespace Libs::Graphics::ShaderRecompiler {
 
 struct CompileOptions {
 	ShaderType                  stage           = ShaderType::Compute;
-	ShaderLaneMaskMode          lane_mask_mode  = ShaderLaneMaskMode::NativeWave;
 	uint32_t                    wave_size       = 64;
 	uint32_t                    user_data_base  = 0;
 	uint32_t                    user_data_count = 64;
+	uint32_t                    scratch_dwords  = 0;
 	uint64_t                    shader_hash     = 0;
 	uint64_t                    shader_base     = 0;
 	std::optional<uint64_t>     flat_memory_base;
-	uint32_t                    descriptor_set             = 0;
-	uint32_t                    push_constant_offset       = 0;
+	uint32_t                    push_constant_offset      = 0;
 	bool                        dump_ir                    = true;
 	bool                        early_dump                 = false;
 	const char*                 dump_label                 = nullptr;

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -1,12 +1,13 @@
 #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 
-#include <cstdio>
+#include "common/debug.h"
+
 #include <cstring>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
 static constexpr size_t InitialSpirvSectionReserve         = 4096;
-static constexpr size_t InitialSpirvFunctionSectionReserve = 450000;
+static constexpr size_t InitialSpirvFunctionSectionReserve = 32768;
 
 static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_t* words,
                                    size_t words_num) {
@@ -22,7 +23,7 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
 Builder::Builder(uint32_t version): m_version(version) {
 	m_debug.reserve(InitialSpirvSectionReserve);
 	m_annotations.reserve(InitialSpirvSectionReserve);
-	m_types.reserve(InitialSpirvSectionReserve);
+	m_declarations.reserve(InitialSpirvSectionReserve);
 	m_functions.reserve(InitialSpirvFunctionSectionReserve);
 }
 
@@ -30,6 +31,110 @@ uint32_t Builder::AllocateId() {
 	return m_next_id++;
 }
 
+void Builder::RequireCapability(uint32_t capability) {
+	if (m_required_capabilities.insert(capability).second) {
+		AddCapability({capability});
+	}
+}
+
+void Builder::RequireExtension(const char* name) {
+	if (m_required_extensions.emplace(name).second) {
+		AddExtension(name);
+	}
+}
+
+uint32_t Builder::Import(const char* name) {
+	if (const auto it = m_import_ids.find(name); it != m_import_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_import_ids.emplace(name, id);
+	AddExtInstImport(id, name);
+	return id;
+}
+
+uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) {
+	return Type(opcode, std::vector<uint32_t>(operands));
+}
+
+uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
+	std::vector<uint32_t> key;
+	key.reserve(operands.size() + 2u);
+	key.push_back(opcode);
+	key.push_back(static_cast<uint32_t>(operands.size()));
+	key.insert(key.end(), operands.begin(), operands.end());
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	return id;
+}
+
+uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+                                std::initializer_list<TypeAnnotation> annotations) {
+	if (annotations.size() == 0) {
+		return Type(opcode, operands);
+	}
+	std::vector<uint32_t> key {opcode, static_cast<uint32_t>(operands.size())};
+	key.insert(key.end(), operands.begin(), operands.end());
+	key.push_back(static_cast<uint32_t>(annotations.size()));
+	for (const auto& annotation: annotations) {
+		key.push_back(annotation.opcode);
+		key.push_back(static_cast<uint32_t>(annotation.operands.size()));
+		key.insert(key.end(), annotation.operands.begin(), annotation.operands.end());
+	}
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	for (const auto& annotation: annotations) {
+		words = {annotation.opcode, id};
+		words.insert(words.end(), annotation.operands.begin(), annotation.operands.end());
+		AppendInstructionWords(m_annotations, words.data(), words.size());
+	}
+	return id;
+}
+
+uint32_t Builder::Constant(uint32_t opcode, uint32_t type,
+                           std::initializer_list<uint32_t> operands) {
+	return Constant(opcode, type, std::vector<uint32_t>(operands));
+}
+
+uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) {
+	std::vector<uint32_t> key;
+	key.reserve(operands.size() + 2u);
+	key.push_back(opcode);
+	key.push_back(type);
+	key.insert(key.end(), operands.begin(), operands.end());
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, type, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	return id;
+}
+
+uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) {
+	const auto id = AllocateId();
+	DefineGlobalVariable(id, pointer_type, storage_class);
+	return id;
+}
+
+void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) {
+	AddType({59u, pointer_type, id, storage_class});
+}
+
 void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
 	const auto len        = text != nullptr ? std::strlen(text) : 0;
 	const auto word_count = (len + 1u + 3u) / 4u;
@@ -103,7 +208,7 @@ void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
 }
 
 void Builder::AddType(std::initializer_list<uint32_t> words) {
-	AppendInstructionWords(m_types, words.begin(), words.size());
+	AppendInstructionWords(m_declarations, words.begin(), words.size());
 }
 
 void Builder::AddFunction(std::initializer_list<uint32_t> words) {
@@ -114,28 +219,33 @@ void Builder::AddFunction(const std::vector<uint32_t>& words) {
 	AppendInstructionWords(m_functions, words.data(), words.size());
 }
 
+DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
+	std::vector<uint32_t> words {245u, type, result};
+	words.resize(words.size() + incoming_count * 2u);
+	const DeferredPhi phi {m_functions.size(), incoming_count};
+	AddFunction(words);
+	m_unpatched_phi_incomings += incoming_count;
+	return phi;
+}
+
+void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
+	EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0);
+	const auto value_word  = phi.word_offset + 3u + incoming * 2u;
+	const auto parent_word = value_word + 1u;
+	EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
+	m_functions[value_word]  = value;
+	m_functions[parent_word] = parent;
+	m_unpatched_phi_incomings--;
+}
+
 std::vector<uint32_t> Builder::Build() const {
-	if (m_debug.size() > InitialSpirvSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=debug size=%zu reserve=%zu\n",
-		            m_debug.size(), InitialSpirvSectionReserve);
-	}
-	if (m_annotations.size() > InitialSpirvSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=annotations size=%zu reserve=%zu\n",
-		            m_annotations.size(), InitialSpirvSectionReserve);
-	}
-	if (m_types.size() > InitialSpirvSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=types size=%zu reserve=%zu\n",
-		            m_types.size(), InitialSpirvSectionReserve);
-	}
-	if (m_functions.size() > InitialSpirvFunctionSectionReserve) {
-		std::printf("SPIR-V builder reserve exceeded: section=functions size=%zu reserve=%zu\n",
-		            m_functions.size(), InitialSpirvFunctionSectionReserve);
-	}
+	EXIT_IF(m_unpatched_phi_incomings != 0);
 
 	std::vector<uint32_t> module;
 	module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() +
 	               m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() +
-	               m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size());
+	               m_debug.size() + m_annotations.size() + m_declarations.size() +
+	               m_functions.size());
 
 	module.push_back(0x07230203u);
 	module.push_back(m_version);
@@ -151,7 +261,7 @@ std::vector<uint32_t> Builder::Build() const {
 	module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end());
 	module.insert(module.end(), m_debug.begin(), m_debug.end());
 	module.insert(module.end(), m_annotations.begin(), m_annotations.end());
-	module.insert(module.end(), m_types.begin(), m_types.end());
+	module.insert(module.end(), m_declarations.begin(), m_declarations.end());
 	module.insert(module.end(), m_functions.begin(), m_functions.end());
 
 	return module;

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -4,10 +4,23 @@
 #include "common/common.h"
 
 #include <initializer_list>
+#include <map>
+#include <set>
+#include <string>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
+struct TypeAnnotation {
+	uint32_t              opcode = 0;
+	std::vector<uint32_t> operands;
+};
+
+struct DeferredPhi {
+	size_t word_offset    = 0;
+	size_t incoming_count = 0;
+};
+
 class Builder {
 public:
 	explicit Builder(uint32_t version = 0x00010300u);
@@ -15,19 +28,29 @@ public:
 	KYTY_CLASS_DEFAULT_COPY(Builder);
 
 	uint32_t AllocateId();
+	void     RequireCapability(uint32_t capability);
+	void     RequireExtension(const char* name);
+	uint32_t Import(const char* name);
+	uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {});
+	uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands);
+	uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+	                       std::initializer_list<TypeAnnotation> annotations);
+	uint32_t Constant(uint32_t opcode, uint32_t type,
+	                  std::initializer_list<uint32_t> operands = {});
+	uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands);
+	uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class);
+	void     DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
 
-	void AddCapability(std::initializer_list<uint32_t> operands);
-	void AddExtension(const char* name);
-	void AddExtInstImport(uint32_t id, const char* name);
-	void AddMemoryModel(std::initializer_list<uint32_t> operands);
-	void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
-	                   const std::vector<uint32_t>& interfaces);
-	void AddExecutionMode(std::initializer_list<uint32_t> operands);
-	void AddName(uint32_t target, const char* name);
-	void AddAnnotation(std::initializer_list<uint32_t> words);
-	void AddType(std::initializer_list<uint32_t> words);
-	void AddFunction(std::initializer_list<uint32_t> words);
-	void AddFunction(const std::vector<uint32_t>& words);
+	void        AddMemoryModel(std::initializer_list<uint32_t> operands);
+	void        AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
+	                          const std::vector<uint32_t>& interfaces);
+	void        AddExecutionMode(std::initializer_list<uint32_t> operands);
+	void        AddName(uint32_t target, const char* name);
+	void        AddAnnotation(std::initializer_list<uint32_t> words);
+	void        AddFunction(std::initializer_list<uint32_t> words);
+	void        AddFunction(const std::vector<uint32_t>& words);
+	DeferredPhi AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count);
+	void        PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent);
 
 	[[nodiscard]] std::vector<uint32_t> Build() const;
 
@@ -37,19 +60,28 @@ private:
 	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
 	                              std::initializer_list<uint32_t> operands);
 	static void AppendString(std::vector<uint32_t>& words, const char* text);
+	void        AddCapability(std::initializer_list<uint32_t> operands);
+	void        AddExtension(const char* name);
+	void        AddExtInstImport(uint32_t id, const char* name);
+	void        AddType(std::initializer_list<uint32_t> words);
 
-	uint32_t              m_next_id = 1;
-	uint32_t              m_version = 0;
-	std::vector<uint32_t> m_capabilities;
-	std::vector<uint32_t> m_extensions;
-	std::vector<uint32_t> m_ext_inst_imports;
-	std::vector<uint32_t> m_memory_model;
-	std::vector<uint32_t> m_entry_points;
-	std::vector<uint32_t> m_execution_modes;
-	std::vector<uint32_t> m_debug;
-	std::vector<uint32_t> m_annotations;
-	std::vector<uint32_t> m_types;
-	std::vector<uint32_t> m_functions;
+	uint32_t                                  m_next_id = 1;
+	uint32_t                                  m_version = 0;
+	std::vector<uint32_t>                     m_capabilities;
+	std::vector<uint32_t>                     m_extensions;
+	std::vector<uint32_t>                     m_ext_inst_imports;
+	std::vector<uint32_t>                     m_memory_model;
+	std::vector<uint32_t>                     m_entry_points;
+	std::vector<uint32_t>                     m_execution_modes;
+	std::vector<uint32_t>                     m_debug;
+	std::vector<uint32_t>                     m_annotations;
+	std::vector<uint32_t>                     m_declarations;
+	std::vector<uint32_t>                     m_functions;
+	std::set<uint32_t>                        m_required_capabilities;
+	std::set<std::string>                     m_required_extensions;
+	std::map<std::string, uint32_t>           m_import_ids;
+	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
+	size_t                                    m_unpatched_phi_incomings = 0;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -6,8 +6,6 @@
 
 #include <algorithm>
 #include <array>
-#include <cinttypes>
-#include <cstdio>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
@@ -15,51 +13,6 @@ using ShaderError::Fail;
 
 namespace {
 
-bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
-	using Kind = IR::DescriptorBindingKind;
-	using Dim  = Decoder::ImageDimension;
-	if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
-		const bool integer = image.kind == IR::ResourceKind::ImageUint;
-		switch (image.dimension) {
-			case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
-			case Dim::Dim1DArray:
-				kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
-				return true;
-			case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
-			case Dim::Dim2DMsaa:
-				kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
-				return true;
-			case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
-			case Dim::Dim2DArray:
-				kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
-				return true;
-			case Dim::Dim2DMsaaArray:
-				kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
-				return true;
-			case Dim::Unknown: return false;
-		}
-	}
-	const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
-	if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
-		return false;
-	}
-	switch (image.dimension) {
-		case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
-		case Dim::Dim1DArray:
-			kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
-			return true;
-		case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
-		case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
-		case Dim::Dim2DArray:
-			kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
-			return true;
-		case Dim::Dim2DMsaa:
-		case Dim::Dim2DMsaaArray: return false;
-		case Dim::Unknown: return false;
-	}
-	return false;
-}
-
 bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 	using Kind                                             = IR::DescriptorBindingKind;
 	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
@@ -81,28 +34,41 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		Expect(Kind::Buffers, Dense(program.info.buffers.size()));
 	}
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		Kind kind;
-		if (!ImageBinding(program.info.images[i], kind)) {
+		const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
+		if (!kind.has_value()) {
 			return Fail(error, "native shader plan has an invalid image class");
 		}
-		present[static_cast<size_t>(kind)] = true;
+		present[static_cast<size_t>(*kind)] = true;
 		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage;
 		const auto count   = dynamic ? program.info.images[i].mip_count : 1u;
 		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
 			return Fail(error, "native shader plan has an invalid image mip descriptor count");
 		}
-		expected[static_cast<size_t>(kind)].insert(expected[static_cast<size_t>(kind)].end(), count,
-		                                           i);
+		expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(),
+		                                            count, i);
 	}
 	if (!program.info.samplers.empty()) {
 		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
 	}
-	const auto uses_gds = program.values != nullptr &&
-	                      std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
-		                      return std::ranges::any_of(*block, [](const IR::Inst& inst) {
-			                      return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
-		                      });
-	                      });
+	bool uses_gds = false;
+	if (program.values != nullptr) {
+		for (const auto* block: program.values->blocks) {
+			for (const auto& inst: *block) {
+				if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
+					continue;
+				}
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.values->memory_info.size()) {
+					return Fail(error, "shared operation has invalid memory metadata");
+				}
+				const auto kind = program.values->memory_info[index].kind;
+				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+					return Fail(error, "shared operation has invalid resource kind");
+				}
+				uses_gds |= kind == IR::ResourceKind::Gds;
+			}
+		}
+	}
 	if (uses_gds) {
 		Expect(Kind::Gds);
 	}
@@ -123,18 +89,12 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 	}
 
 	std::array<bool, KindCount> seen {};
-	for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
-		const auto& binding = program.bindings.descriptors[i];
-		const auto  kind    = static_cast<size_t>(binding.kind);
+	for (const auto& binding: program.bindings.descriptors) {
+		const auto kind = static_cast<size_t>(binding.kind);
 		if (kind >= KindCount || seen[kind] || !present[kind] ||
 		    binding.resources != expected[kind]) {
 			return Fail(error, "native descriptor groups do not match shader topology");
 		}
-		for (uint32_t j = 0; j < i; j++) {
-			if (program.bindings.descriptors[j].binding == binding.binding) {
-				return Fail(error, "native descriptor binding numbers are not unique");
-			}
-		}
 		seen[kind] = true;
 	}
 	for (size_t i = 0; i < KindCount; i++) {
@@ -143,8 +103,12 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 		}
 	}
 	const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
-	if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
-	    program.bindings.buffer_offset_count != program.info.buffers.size() ||
+	if (program.bindings.push_constant_offset % sizeof(uint32_t) != 0 ||
+	    program.bindings.push_constant_offset + program.bindings.push_constant_size >
+	        IR::NativePushConstantSize ||
+	    program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
+	    program.bindings.memory_offset_count !=
+	        program.info.buffers.size() + program.info.addresses.size() ||
 	    (!has_user_storage &&
 	     program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
 	    (has_user_storage && program.bindings.push_constant_size != 0) ||
@@ -192,6 +156,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 						return Fail(error, "typed address handle has an invalid dense resource");
 					}
 					break;
+				case IR::ValueOpcode::GetScratchResource:
+					if (inst.NumArgs() != 0 || program.scratch_dwords == 0) {
+						return Fail(error, "typed scratch handle has invalid shader metadata");
+					}
+					break;
 				case IR::ValueOpcode::GetImageResource:
 					if (dense >= program.info.images.size()) {
 						return Fail(error, "typed image handle has an invalid dense resource");
@@ -219,96 +188,121 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
 
 } // namespace
 
-static constexpr size_t InitialEmitterVectorReserve = 4096;
-
-static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
-                                  size_t size) {
-	if (size <= InitialEmitterVectorReserve) {
-		return;
-	}
-	std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
-	            " stage=%u vector=%s size=%zu reserve=%zu\n",
-	            program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
-	            InitialEmitterVectorReserve);
-}
-
-void CollectValueRegisterMirrors(const IR::ValueProgram&                program,
-                                 std::vector<Emitter::RegisterBinding>& registers) {
-	for (const auto* block: program.blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::SetScalarRegister:
-				case IR::ValueOpcode::SetThreadBitScalarRegister:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(
-						    registers,
-						    {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())});
-					}
-					break;
-				case IR::ValueOpcode::SetVectorRegister:
-					Emitter::CollectRegister(
-					    registers,
-					    {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())});
-					break;
-				case IR::ValueOpcode::SetM0:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(registers, {IR::RegisterFile::M0, 0});
-					}
-					break;
-				case IR::ValueOpcode::SetExecLo:
-				case IR::ValueOpcode::SetExecHi:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(
-						    registers, {IR::RegisterFile::Exec,
-						                inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u});
-					}
-					break;
-				case IR::ValueOpcode::SetVccLo:
-				case IR::ValueOpcode::SetVccHi:
-					if (block != program.blocks.front()) {
-						Emitter::CollectRegister(
-						    registers, {IR::RegisterFile::Vcc,
-						                inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u});
-					}
-					break;
-				default: break;
-			}
-		}
-	}
-}
-
-bool ValueProgramHasOpcode(const IR::ValueProgram& program, IR::ValueOpcode opcode) {
-	return std::ranges::any_of(program.blocks, [opcode](const IR::Block* block) {
-		return std::ranges::any_of(
-		    *block, [opcode](const IR::Inst& inst) { return inst.GetOpcode() == opcode; });
-	});
-}
-
-bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
+bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
+	program.spirv_requirements.reset();
+	IR::SpirvRequirements requirements {};
 	if (program.values == nullptr) {
-		return false;
+		program.spirv_requirements.emplace(requirements);
+		return true;
 	}
+	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
 	for (const auto* block: program.values->blocks) {
 		for (const auto& inst: *block) {
+			if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
+				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
+				if (memory_index >= program.values->memory_info.size()) {
+					return Fail(error, "address operation has invalid memory metadata");
+				}
+				if (program.values->memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+					if (program.scratch_dwords == 0) {
+						return Fail(error, "scratch operation has no per-thread storage");
+					}
+					requirements.function_scratch = true;
+				}
+			}
+			if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
+				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
+				if (memory_index >= program.values->memory_info.size()) {
+					return Fail(error, "buffer operation has invalid memory metadata");
+				}
+				const auto& memory = program.values->memory_info[memory_index];
+				if (memory.kind == IR::ResourceKind::Buffer) {
+					if (memory.resource >= program.info.buffers.size()) {
+						return Fail(error, "buffer operation has invalid resource metadata");
+					}
+					if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
+						if (program.stage != ShaderType::Compute) {
+							return Fail(error, "buffer ADD_TID is only valid for compute shaders");
+						}
+						requirements.subgroup_local_invocation_id = true;
+					}
+				}
+			}
+			const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
+			if (shared_access != IR::SharedAccess::None) {
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.values->memory_info.size()) {
+					return Fail(error, "shared operation has invalid memory metadata");
+				}
+				const auto kind = program.values->memory_info[index].kind;
+				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+					return Fail(error, "shared operation has invalid resource kind");
+				}
+				if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
+					requirements.function_lds = true;
+				}
+				if (shared_access == IR::SharedAccess::Append ||
+				    shared_access == IR::SharedAccess::Consume) {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+				}
+			}
 			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::Ballot:
+				case IR::ValueOpcode::Ballot: MarkBallot(); break;
 				case IR::ValueOpcode::DppMoveU32:
+				case IR::ValueOpcode::ReadFirstLane:
+				case IR::ValueOpcode::ReadLane: {
+					MarkBallot();
+					requirements.subgroup_shuffle = true;
+					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
+						requirements.subgroup_local_invocation_id = true;
+					}
+					break;
+				}
 				case IR::ValueOpcode::DppUpdateU32:
 				case IR::ValueOpcode::WqmMask:
-				case IR::ValueOpcode::ReadFirstLane:
-				case IR::ValueOpcode::ReadLane:
-				case IR::ValueOpcode::WriteLane:
-				case IR::ValueOpcode::Permlane16U32:
-				case IR::ValueOpcode::SwizzleU32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume:
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume: return true;
+				case IR::ValueOpcode::WriteLane: {
+					MarkBallot();
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::Permlane16U32: {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::SwizzleU32: {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::LaneId:
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
+				case IR::ValueOpcode::ImageGatherRaw:
+					requirements.image_gather_extended = true;
+					break;
+				case IR::ValueOpcode::SetAttribute: {
+					const auto index = inst.Flags<IR::ExportFlags>().index;
+					if (index >= program.values->export_info.size()) {
+						return Fail(error, "attribute export has invalid metadata");
+					}
+					if (program.stage == ShaderType::Pixel &&
+					    program.values->export_info[index].vm) {
+						requirements.pixel_valid_mask = true;
+					}
+					break;
+				}
 				default: break;
 			}
 		}
 	}
-	return false;
+	program.spirv_requirements.emplace(requirements);
+	return true;
 }
 
 bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
@@ -322,7 +316,8 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
 		return false;
 	}
 	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete) {
+	    !program.shader_info_complete || !program.binding_layout_complete ||
+	    !program.spirv_requirements.has_value()) {
 		SetError(error, "SPIR-V emitter requires a fully planned native shader program");
 		return false;
 	}
@@ -344,42 +339,19 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
 		return false;
 	}
 	EmitterState state(program, resources, input_info);
-	state.stage                = program.stage;
-	state.wave_size            = program.wave_size;
-	state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
-	state.exact_subgroup_operations =
-	    !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
-	state.registers.reserve(InitialEmitterVectorReserve);
-	state.inputs.reserve(InitialEmitterVectorReserve);
-	state.outputs.reserve(InitialEmitterVectorReserve);
-	state.interface_variables.reserve(InitialEmitterVectorReserve);
-	state.reachable_blocks.reserve(InitialEmitterVectorReserve);
-	CollectValueRegisterMirrors(value_program, state.registers);
+	state.stage     = program.stage;
+	state.wave_size = program.wave_size;
+	state.inputs.reserve(program.info.inputs.size());
+	state.outputs.reserve(program.info.outputs.size());
+	state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
 	CopyProgramInputsAndOutputs(state, program);
-	state.needs_subgroup_ballot  = ProgramNeedsSubgroupBallot(program) ||
-	                               (!state.per_invocation_masks &&
-	                                ValueProgramHasOpcode(value_program, IR::ValueOpcode::Ballot));
-	state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
-	state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
-	state.needs_compute_derivatives          = ProgramNeedsComputeDerivatives(program);
-	state.needs_image_gather_extended        = ProgramNeedsImageGatherExtended(program);
-	state.needs_function_lds                 = ProgramNeedsFunctionLds(program);
-	state.needs_pixel_valid_mask             = ProgramNeedsPixelValidMask(program);
 	AllocateInputVariables(state);
 	AllocateOutputVariables(state);
-	AllocateDescriptorVariables(state);
-	EmitHeaderAndTypes(state);
-	AllocateRegisterVariables(state);
+	DefineModule(state);
 	if (!EmitValueProgram(state, value_program, error)) {
 		return false;
 	}
 
-	ReportReserveExceeded(program, "registers", state.registers.size());
-	ReportReserveExceeded(program, "inputs", state.inputs.size());
-	ReportReserveExceeded(program, "outputs", state.outputs.size());
-	ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
-	ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
-
 	auto binary = state.builder.Build();
 	if (binary.empty()) {
 		SetError(error, "SPIR-V builder returned an empty module");

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -9,7 +9,7 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
+bool AnalyzeProgramRequirements(IR::Program& program, std::string* error);
 
 bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
                  ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -32,92 +32,88 @@ uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint3
 
 Pair ExtractPair(EmitterState& state, uint32_t value) {
 	Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.low, value, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.high, value, 1});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.low, value, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.high, value, 1});
 	return result;
 }
 
 uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, state.uint_pair_type, result, low, high});
+	state.builder.AddFunction({OpCompositeConstruct, TypeU64(state), result, low, high});
 	return result;
 }
 
-uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
-                   uint32_t high_compare, uint32_t low_compare, bool equal_compare = false,
-                   bool not_equal = false) {
-	const auto lhs        = ExtractPair(state, lhs_value);
-	const auto rhs        = ExtractPair(state, rhs_value);
-	const auto high_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.high, rhs.high);
-	const auto low_equal  = NewBinary(state, OpIEqual, state.bool_type, lhs.low, rhs.low);
-	if (equal_compare) {
-		return NewBinary(
-		    state, not_equal ? OpLogicalOr : OpLogicalAnd, state.bool_type,
-		    not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, high_equal) : high_equal,
-		    not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, low_equal) : low_equal);
-	}
-	const auto high_result = NewBinary(state, high_compare, state.bool_type, lhs.high, rhs.high);
-	const auto low_result  = NewBinary(state, low_compare, state.bool_type, lhs.low, rhs.low);
-	const auto low_path = NewBinary(state, OpLogicalAnd, state.bool_type, high_equal, low_result);
-	return NewBinary(state, OpLogicalOr, state.bool_type, high_result, low_path);
+uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                        bool not_equal) {
+	const auto compare = NewBinary(state, not_equal ? OpINotEqual : OpIEqual,
+	                               TypeBoolVector(state, 2), lhs_value, rhs_value);
+	return NewUnary(state, not_equal ? OpAny : OpAll, TypeBool(state), compare);
+}
+
+uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                          uint32_t high_compare, uint32_t low_compare) {
+	const auto lhs         = ExtractPair(state, lhs_value);
+	const auto rhs         = ExtractPair(state, rhs_value);
+	const auto high_equal  = NewBinary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high);
+	const auto high_result = NewBinary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
+	const auto low_result  = NewBinary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
+	const auto low_path = NewBinary(state, OpLogicalAnd, TypeBool(state), high_equal, low_result);
+	return NewBinary(state, OpLogicalOr, TypeBool(state), high_result, low_path);
 }
 
 uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
-	const auto lhs   = ExtractPair(state, lhs_value);
-	const auto rhs   = ExtractPair(state, rhs_value);
-	const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0));
-	const auto high0 = NewBinary(state, OpIAdd, state.uint_type, lhs.high, rhs.high);
-	const auto high  = NewBinary(state, OpIAdd, state.uint_type, high0, carry.carry);
-	return MakePair(state, carry.sum, high);
+	const auto lhs      = ExtractPair(state, lhs_value);
+	const auto rhs      = ExtractPair(state, rhs_value);
+	const auto low_pair = state.builder.AllocateId();
+	const auto low      = state.builder.AllocateId();
+	const auto carry    = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, low_pair, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
+	const auto high0 = NewBinary(state, OpIAdd, TypeU32(state), lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpIAdd, TypeU32(state), high0, carry);
+	return MakePair(state, low, high);
 }
 
 uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
 	const auto lhs    = ExtractPair(state, lhs_value);
 	const auto rhs    = ExtractPair(state, rhs_value);
-	const auto low    = NewBinary(state, OpISub, state.uint_type, lhs.low, rhs.low);
-	const auto borrow = NewBinary(state, OpULessThan, state.bool_type, lhs.low, rhs.low);
+	const auto low    = NewBinary(state, OpISub, TypeU32(state), lhs.low, rhs.low);
+	const auto borrow = NewBinary(state, OpULessThan, TypeBool(state), lhs.low, rhs.low);
 	const auto borrow_u32 =
-	    NewSelect(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0));
-	const auto high0 = NewBinary(state, OpISub, state.uint_type, lhs.high, rhs.high);
-	const auto high  = NewBinary(state, OpISub, state.uint_type, high0, borrow_u32);
+	    NewSelect(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
+	const auto high0 = NewBinary(state, OpISub, TypeU32(state), lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpISub, TypeU32(state), high0, borrow_u32);
 	return MakePair(state, low, high);
 }
 
 uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
-	const auto operand_type = signed_value ? state.int_type : state.uint_type;
-	const auto pair_type    = signed_value ? state.int_pair_type : state.uint_pair_type;
+	const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
+	const auto pair_type    = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
 	uint32_t   lhs_operand  = lhs;
 	uint32_t   rhs_operand  = rhs;
 	if (signed_value) {
-		lhs_operand = NewUnary(state, OpBitcast, state.int_type, lhs);
-		rhs_operand = NewUnary(state, OpBitcast, state.int_type, rhs);
+		lhs_operand = NewUnary(state, OpBitcast, TypeI32(state), lhs);
+		rhs_operand = NewUnary(state, OpBitcast, TypeI32(state), rhs);
 	}
 	const auto extended = state.builder.AllocateId();
 	state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended,
 	                           lhs_operand, rhs_operand});
 	const auto high = state.builder.AllocateId();
 	state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1});
-	return signed_value ? NewUnary(state, OpBitcast, state.uint_type, high) : high;
+	return signed_value ? NewUnary(state, OpBitcast, TypeU32(state), high) : high;
 }
 
 uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
 	const auto lhs   = ExtractPair(state, lhs_value);
 	const auto rhs   = ExtractPair(state, rhs_value);
-	const auto low   = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.low);
+	const auto low   = NewBinary(state, OpIMul, TypeU32(state), lhs.low, rhs.low);
 	const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
-	const auto high1 = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.high);
-	const auto high2 = NewBinary(state, OpIMul, state.uint_type, lhs.high, rhs.low);
+	const auto high1 = NewBinary(state, OpIMul, TypeU32(state), lhs.low, rhs.high);
+	const auto high2 = NewBinary(state, OpIMul, TypeU32(state), lhs.high, rhs.low);
 	return MakePair(state, low,
-	                NewBinary(state, OpIAdd, state.uint_type,
-	                          NewBinary(state, OpIAdd, state.uint_type, high0, high1), high2));
-}
-
-uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value,
-                       uint32_t rhs_value) {
-	const auto lhs = ExtractPair(state, lhs_value);
-	const auto rhs = ExtractPair(state, rhs_value);
-	return MakePair(state, NewBinary(state, opcode, state.uint_type, lhs.low, rhs.low),
-	                NewBinary(state, opcode, state.uint_type, lhs.high, rhs.high));
+	                NewBinary(state, OpIAdd, TypeU32(state),
+	                          NewBinary(state, OpIAdd, TypeU32(state), high0, high1), high2));
 }
 
 uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
@@ -129,52 +125,94 @@ uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint3
 	} else if (opcode == OpShiftRightLogical) {
 		EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
 	} else {
-		const auto logical      = EmitShift64(state, OpShiftRightLogical, value, shift);
-		const auto logical_pair = ExtractPair(state, logical);
-		const auto sign =
-		    NewBinary(state, OpSLessThan, state.bool_type, pair.high, ConstantU32(state, 0));
-		const auto inverse =
-		    NewBinary(state, OpISub, state.uint_type, ConstantU32(state, 64), shift);
-		const auto fill = NewBinary(state, OpShiftLeftLogical, state.uint_type,
-		                            ConstantU32(state, 0xffffffffu), inverse);
-		const auto high_shift =
-		    NewBinary(state, OpISub, state.uint_type, shift, ConstantU32(state, 32));
-		const auto high_fill = NewSelect(
-		    state, state.uint_type,
-		    NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, ConstantU32(state, 32)),
-		    ConstantU32(state, 0xffffffffu), fill);
-		const auto fill_low =
-		    NewSelect(state, state.uint_type, sign,
-		              NewSelect(state, state.uint_type,
-		                        NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift,
-		                                  ConstantU32(state, 32)),
-		                        NewBinary(state, OpShiftLeftLogical, state.uint_type,
-		                                  ConstantU32(state, 0xffffffffu), high_shift),
-		                        ConstantU32(state, 0)),
+		const auto amount =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
+		const auto word_shift =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
+		const auto at_least_32 =
+		    NewBinary(state, OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
+		const auto nonzero =
+		    NewBinary(state, OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
+		const auto high_shifted =
+		    NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
+		const auto low_shifted =
+		    NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
+		const auto carry_count =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state),
+		              NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
+		              ConstantU32(state, 31));
+		const auto carry =
+		    NewSelect(state, TypeU32(state), nonzero,
+		              NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
 		              ConstantU32(state, 0));
-		const auto fill_high =
-		    NewSelect(state, state.uint_type, sign, high_fill, ConstantU32(state, 0));
-		return MakePair(
-		    state, NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.low, fill_low),
-		    NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.high, fill_high));
+		const auto low_below_32 = NewBinary(state, OpBitwiseOr, TypeU32(state), low_shifted, carry);
+		const auto negative =
+		    NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
+		const auto sign_fill = NewSelect(state, TypeU32(state), negative,
+		                                 ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
+		return MakePair(state,
+		                NewSelect(state, TypeU32(state), at_least_32, high_shifted, low_below_32),
+		                NewSelect(state, TypeU32(state), at_least_32, sign_fill, high_shifted));
 	}
 	return MakePair(state, low, high);
 }
 
+uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
+	shift &= 63u;
+	if (shift == 0u) {
+		return value;
+	}
+	const auto pair = ExtractPair(state, value);
+	if (opcode == OpShiftLeftLogical) {
+		if (shift < 32u) {
+			const auto low  = NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.low,
+			                            ConstantU32(state, shift));
+			const auto high = NewBinary(state, OpBitwiseOr, TypeU32(state),
+			                            NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+			                                      pair.high, ConstantU32(state, shift)),
+			                            NewBinary(state, OpShiftRightLogical, TypeU32(state),
+			                                      pair.low, ConstantU32(state, 32u - shift)));
+			return MakePair(state, low, high);
+		}
+		return MakePair(state, ConstantU32(state, 0),
+		                shift == 32u ? pair.low
+		                             : NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+		                                         pair.low, ConstantU32(state, shift - 32u)));
+	}
+	if (shift < 32u) {
+		const auto low = NewBinary(state, OpBitwiseOr, TypeU32(state),
+		                           NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low,
+		                                     ConstantU32(state, shift)),
+		                           NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high,
+		                                     ConstantU32(state, 32u - shift)));
+		const auto high =
+		    NewBinary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
+		return MakePair(state, low, high);
+	}
+	const auto high = opcode == OpShiftRightArithmetic
+	                      ? NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high,
+	                                  ConstantU32(state, 31u))
+	                      : ConstantU32(state, 0);
+	const auto low  = shift == 32u ? pair.high
+	                               : NewBinary(state, opcode, TypeU32(state), pair.high,
+	                                           ConstantU32(state, shift - 32u));
+	return MakePair(state, low, high);
+}
+
 uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
 	const auto pair   = ExtractPair(state, value);
 	const auto high_i = state.builder.AllocateId();
 	const auto low_i  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.int_type, high_i, state.glsl_std450, GlslFindUMsb, pair.high});
+	    {OpExtInst, TypeI32(state), high_i, GlslStd450(state), GlslFindUMsb, pair.high});
 	state.builder.AddFunction(
-	    {OpExtInst, state.int_type, low_i, state.glsl_std450, GlslFindUMsb, pair.low});
-	const auto high = NewUnary(state, OpBitcast, state.uint_type, high_i);
-	const auto low  = NewUnary(state, OpBitcast, state.uint_type, low_i);
+	    {OpExtInst, TypeI32(state), low_i, GlslStd450(state), GlslFindUMsb, pair.low});
+	const auto high = NewUnary(state, OpBitcast, TypeU32(state), high_i);
+	const auto low  = NewUnary(state, OpBitcast, TypeU32(state), low_i);
 	const auto high_nonzero =
-	    NewBinary(state, OpINotEqual, state.bool_type, pair.high, ConstantU32(state, 0));
-	return NewSelect(state, state.uint_type, high_nonzero,
-	                 NewBinary(state, OpIAdd, state.uint_type, high, ConstantU32(state, 32)), low);
+	    NewBinary(state, OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
+	return NewSelect(state, TypeU32(state), high_nonzero,
+	                 NewBinary(state, OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
 }
 
 uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
@@ -188,10 +226,10 @@ uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bo
 uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
 	const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
 	const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
-	const auto ab      = NewBinary(state, OpIAdd, state.uint_type, a, b);
-	const auto abc     = NewBinary(state, OpIAdd, state.uint_type, ab, c);
-	return NewBinary(state, OpISub, state.uint_type,
-	                 NewBinary(state, OpISub, state.uint_type, abc, minimum), maximum);
+	const auto ab      = NewBinary(state, OpIAdd, TypeU32(state), a, b);
+	const auto abc     = NewBinary(state, OpIAdd, TypeU32(state), ab, c);
+	return NewBinary(state, OpISub, TypeU32(state),
+	                 NewBinary(state, OpISub, TypeU32(state), abc, minimum), maximum);
 }
 
 uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
@@ -204,17 +242,17 @@ uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
 	const auto max_ab   = EmitMinMaxF32Value(state, a, b, true);
 	const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
 	const auto median   = EmitMinMaxF32Value(state, min_ab, high_min, true);
-	const auto nan_ab   = NewBinary(state, OpLogicalOr, state.bool_type,
+	const auto nan_ab   = NewBinary(state, OpLogicalOr, TypeBool(state),
 	                                EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
 	const auto any_nan =
-	    NewBinary(state, OpLogicalOr, state.bool_type, nan_ab, EmitClassifyF32(state, c).nan);
-	return NewSelect(state, state.float_type, any_nan, min3, median);
+	    NewBinary(state, OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
+	return NewSelect(state, TypeF32(state), any_nan, min3, median);
 }
 
 uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
                  std::initializer_list<uint32_t> args) {
 	const auto            result = state.builder.AllocateId();
-	std::vector<uint32_t> words {OpExtInst, type, result, state.glsl_std450, opcode};
+	std::vector<uint32_t> words {OpExtInst, type, result, GlslStd450(state), opcode};
 	words.insert(words.end(), args.begin(), args.end());
 	state.builder.AddFunction(words);
 	return result;
@@ -225,41 +263,41 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
 	const auto converted_raw = state.builder.AllocateId();
 	if (signed_value) {
 		const auto converted_i = state.builder.AllocateId();
-		state.builder.AddFunction({OpConvertFToS, state.int_type, converted_i, trunc});
-		state.builder.AddFunction({OpBitcast, state.uint_type, converted_raw, converted_i});
+		state.builder.AddFunction({OpConvertFToS, TypeI32(state), converted_i, trunc});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), converted_raw, converted_i});
 	} else {
-		state.builder.AddFunction({OpConvertFToU, state.uint_type, converted_raw, trunc});
+		state.builder.AddFunction({OpConvertFToU, TypeU32(state), converted_raw, trunc});
 	}
 	const auto nan = EmitClassifyF32(state, src).nan;
 	if (signed_value) {
-		const auto below = NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src,
+		const auto below = NewBinary(state, OpFOrdLessThanEqual, TypeBool(state), src,
 		                             ConstantF32(state, 0xcf000000u));
-		const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
+		const auto above = NewBinary(state, OpFOrdGreaterThanEqual, TypeBool(state), src,
 		                             ConstantF32(state, 0x4f000000u));
-		const auto high  = NewSelect(state, state.uint_type, above, ConstantU32(state, 0x7fffffffu),
-		                             converted_raw);
+		const auto high =
+		    NewSelect(state, TypeU32(state), above, ConstantU32(state, 0x7fffffffu), converted_raw);
 		const auto low =
-		    NewSelect(state, state.uint_type, below, ConstantU32(state, 0x80000000u), high);
-		return NewSelect(state, state.uint_type, nan, ConstantU32(state, 0), low);
+		    NewSelect(state, TypeU32(state), below, ConstantU32(state, 0x80000000u), high);
+		return NewSelect(state, TypeU32(state), nan, ConstantU32(state, 0), low);
 	}
 	const auto below =
-	    NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, ConstantF32(state, 0));
-	const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
+	    NewBinary(state, OpFOrdLessThanEqual, TypeBool(state), src, ConstantF32(state, 0));
+	const auto above = NewBinary(state, OpFOrdGreaterThanEqual, TypeBool(state), src,
 	                             ConstantF32(state, 0x4f800000u));
-	const auto zero  = NewBinary(state, OpLogicalOr, state.bool_type, nan, below);
+	const auto zero  = NewBinary(state, OpLogicalOr, TypeBool(state), nan, below);
 	const auto high =
-	    NewSelect(state, state.uint_type, above, ConstantU32(state, 0xffffffffu), converted_raw);
-	return NewSelect(state, state.uint_type, zero, ConstantU32(state, 0), high);
+	    NewSelect(state, TypeU32(state), above, ConstantU32(state, 0xffffffffu), converted_raw);
+	return NewSelect(state, TypeU32(state), zero, ConstantU32(state, 0), high);
 }
 
 uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
-	return EmitExt(state, state.uint_type, GlslPackHalf2x16, {src});
+	return EmitExt(state, TypeU32(state), GlslPackHalf2x16, {src});
 }
 
 uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) {
-	const auto pair   = EmitExt(state, state.vec2_float_type, GlslUnpackHalf2x16, {bits});
+	const auto pair   = EmitExt(state, TypeF32Vector(state, 2), GlslUnpackHalf2x16, {bits});
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, result, pair, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), result, pair, 0});
 	return result;
 }
 
@@ -277,7 +315,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		return true;
 	};
 	const auto ext_unary = [&](uint32_t ext) {
-		ctx.Define(inst, EmitExt(state, state.float_type, ext, {ctx.Arg(inst, 0)}));
+		ctx.Define(inst, EmitExt(state, TypeF32(state), ext, {ctx.Arg(inst, 0)}));
 		return true;
 	};
 	switch (op) {
@@ -285,8 +323,6 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::BitCastF16U16:
 		case IR::ValueOpcode::ConvertU32U16:
 		case IR::ValueOpcode::ConvertU32U8:
-		case IR::ValueOpcode::PackUint2x32:
-		case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
 		case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
 		case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
 		case IR::ValueOpcode::ConvertU16U32:
@@ -299,7 +335,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::ConvertF16F32: {
 			const auto pair = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair,
+			state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 2), pair,
 			                           ctx.Arg(inst, 0), ConstantF32(state, 0)});
 			ctx.Define(inst, EmitPackHalf(state, pair));
 			return true;
@@ -314,15 +350,23 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
 			return true;
 		case IR::ValueOpcode::ConvertF32S32: {
-			const auto signed_value = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0));
+			const auto signed_value = NewUnary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 0));
 			ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value});
 			return true;
 		}
 		case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
+		case IR::ValueOpcode::CompositeConstructU64:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U64,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
 		case IR::ValueOpcode::CompositeConstructU32x2:
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
 			return true;
+		case IR::ValueOpcode::CompositeConstructU32x3:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x3,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
 		case IR::ValueOpcode::CompositeConstructF32x2:
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
@@ -331,28 +375,28 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
 			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
 			return true;
+		case IR::ValueOpcode::CompositeExtractU64:
 		case IR::ValueOpcode::CompositeExtractU32x2:
+		case IR::ValueOpcode::CompositeExtractU32x3:
 		case IR::ValueOpcode::CompositeExtractU32x4:
 			ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
 			         {ctx.Arg(inst, 0), inst.Arg(1).U32()});
 			return true;
 		case IR::ValueOpcode::PackHalf2x16:
-			ctx.Define(inst, EmitExt(state, state.uint_type, GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
 			return true;
 		case IR::ValueOpcode::PackSnorm2x16:
-			ctx.Define(inst,
-			           EmitExt(state, state.uint_type, GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
 			return true;
 		case IR::ValueOpcode::PackUnorm2x16:
-			ctx.Define(inst,
-			           EmitExt(state, state.uint_type, GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
 			return true;
 		case IR::ValueOpcode::PackFloat2x16Rtz: {
 			const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
 			const auto high =
-			    NewBinary(state, OpShiftLeftLogical, state.uint_type,
+			    NewBinary(state, OpShiftLeftLogical, TypeU32(state),
 			              EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
-			ctx.Define(inst, NewBinary(state, OpBitwiseOr, state.uint_type, low, high));
+			ctx.Define(inst, NewBinary(state, OpBitwiseOr, TypeU32(state), low, high));
 			return true;
 		}
 		case IR::ValueOpcode::FPAbs32:
@@ -362,7 +406,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
 			return true;
 		case IR::ValueOpcode::FPSaturate32:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslFClamp,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFClamp,
 			                         {ctx.Arg(inst, 0), ConstantF32(state, 0),
 			                          ConstantF32(state, 0x3f800000u)}));
 			return true;
@@ -414,10 +458,10 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::IAbs32: {
 			const auto value = ctx.Arg(inst, 0);
-			const auto neg   = NewUnary(state, OpSNegate, state.uint_type, value);
+			const auto neg   = NewUnary(state, OpSNegate, TypeU32(state), value);
 			const auto negative =
-			    NewBinary(state, OpSLessThan, state.bool_type, value, ConstantU32(state, 0));
-			ctx.Define(inst, NewSelect(state, state.uint_type, negative, neg, value));
+			    NewBinary(state, OpSLessThan, TypeBool(state), value, ConstantU32(state, 0));
+			ctx.Define(inst, NewSelect(state, TypeU32(state), negative, neg, value));
 			return true;
 		}
 		case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32);
@@ -426,15 +470,25 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::ShiftRightArithmetic32:
 			return binary(OpShiftRightArithmetic, IR::Type::U32);
 		case IR::ValueOpcode::ShiftLeftLogical64:
-			ctx.Define(inst,
-			           EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
+			                     ? EmitConstantShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::ShiftRightLogical64:
-			ctx.Define(inst,
-			           EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
+			                     ? EmitConstantShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::ShiftRightArithmetic64:
-			ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			ctx.Define(inst,
+			           inst.Arg(1).Resolve().IsImmediate()
+			               ? EmitConstantShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			                                     inst.Arg(1).Resolve().U32())
+			               : EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
 			                             ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
@@ -442,41 +496,30 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
 		case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
 		case IR::ValueOpcode::BitwiseAnd64:
-			ctx.Define(inst,
-			           EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::BitwiseOr64:
-			ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			ctx.Define(inst, NewBinary(state, OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0),
+			                           ctx.Arg(inst, 1)));
 			return true;
 		case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
 		case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
 		case IR::ValueOpcode::BitCount64: {
-			const auto pair = ExtractPair(state, ctx.Arg(inst, 0));
-			ctx.Define(inst, NewBinary(state, OpIAdd, state.uint_type,
-			                           NewUnary(state, OpBitCount, state.uint_type, pair.low),
-			                           NewUnary(state, OpBitCount, state.uint_type, pair.high)));
+			const auto pair =
+			    ExtractPair(state, NewUnary(state, OpBitCount, TypeU64(state), ctx.Arg(inst, 0)));
+			ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state), pair.low, pair.high));
 			return true;
 		}
 		case IR::ValueOpcode::FindILsb32: {
-			const auto value = EmitExt(state, state.int_type, GlslFindILsb, {ctx.Arg(inst, 0)});
-			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			const auto value = EmitExt(state, TypeI32(state), GlslFindILsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, TypeU32(state), value));
 			return true;
 		}
 		case IR::ValueOpcode::FindUMsb32: {
-			const auto value = EmitExt(state, state.int_type, GlslFindUMsb, {ctx.Arg(inst, 0)});
-			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			const auto value = EmitExt(state, TypeI32(state), GlslFindUMsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, TypeU32(state), value));
 			return true;
 		}
 		case IR::ValueOpcode::FindUMsb64:
 			ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0)));
 			return true;
-		case IR::ValueOpcode::FindSMsb32: {
-			const auto value =
-			    EmitExt(state, state.int_type, 74u,
-			            {NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0))});
-			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
-			return true;
-		}
 		case IR::ValueOpcode::SMin32:
 			ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
 			return true;
@@ -524,53 +567,23 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
 		case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
 		case IR::ValueOpcode::IEqual64:
-			ctx.Define(inst,
-			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false));
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
 			return true;
 		case IR::ValueOpcode::INotEqual64:
-			ctx.Define(inst,
-			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true));
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
 			return true;
 		case IR::ValueOpcode::ULessThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan,
-			                           OpULessThan));
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpULessThan, OpULessThan));
 			return true;
 		case IR::ValueOpcode::SLessThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan,
-			                           OpULessThan));
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpSLessThan, OpULessThan));
 			return true;
 		case IR::ValueOpcode::UGreaterThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan,
-			                           OpUGreaterThan));
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpUGreaterThan, OpUGreaterThan));
 			return true;
-		case IR::ValueOpcode::SGreaterThan64:
-			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan,
-			                           OpUGreaterThan));
-			return true;
-		case IR::ValueOpcode::ULessThanEqual64: {
-			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                               OpUGreaterThan, OpUGreaterThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
-			return true;
-		}
-		case IR::ValueOpcode::SLessThanEqual64: {
-			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
-			                               OpSGreaterThan, OpUGreaterThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
-			return true;
-		}
-		case IR::ValueOpcode::UGreaterThanEqual64: {
-			const auto less =
-			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
-			return true;
-		}
-		case IR::ValueOpcode::SGreaterThanEqual64: {
-			const auto less =
-			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan);
-			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
-			return true;
-		}
 		case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
 		case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
 		case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);
@@ -602,7 +615,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32);
 		case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32);
 		case IR::ValueOpcode::FPFma32:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslFma,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFma,
 			                         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
 			return true;
 		case IR::ValueOpcode::FPMin32:
@@ -625,44 +638,44 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::FPRecip32: {
 			const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
-			ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
+			ctx.Define(inst, NewBinary(state, OpFDiv, TypeF32(state),
 			                           ConstantF32(state, 0x3f800000u), source));
 			return true;
 		}
 		case IR::ValueOpcode::FPRecipIFlag32:
 			// Integer-to-float inputs used by IFLAG cannot be denormal.
-			ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
+			ctx.Define(inst, NewBinary(state, OpFDiv, TypeF32(state),
 			                           ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
 			return true;
 		case IR::ValueOpcode::FPRecipSqrt32:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslInverseSqrt,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslInverseSqrt,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPSqrt:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslSqrt,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslSqrt,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPSin:
 		case IR::ValueOpcode::FPCos: {
 			auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
 			source =
-			    NewBinary(state, OpFMul, state.float_type, source, ConstantF32(state, 0x40c90fdbu));
-			ctx.Define(inst, EmitExt(state, state.float_type,
+			    NewBinary(state, OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
+			ctx.Define(inst, EmitExt(state, TypeF32(state),
 			                         op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source}));
 			return true;
 		}
 		case IR::ValueOpcode::FPExp2:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslExp2,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslExp2,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPLog2:
-			ctx.Define(inst, EmitExt(state, state.float_type, GlslLog2,
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslLog2,
 			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
 			return true;
 		case IR::ValueOpcode::FPLdexp: {
-			const auto exponent = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 1));
+			const auto exponent = NewUnary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 1));
 			ctx.Define(inst,
-			           EmitExt(state, state.float_type, GlslLdexp, {ctx.Arg(inst, 0), exponent}));
+			           EmitExt(state, TypeF32(state), GlslLdexp, {ctx.Arg(inst, 0), exponent}));
 			return true;
 		}
 		case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven);

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -5,7 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
                             uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, true_value, false_value});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value});
 	return ret;
 }
 
@@ -26,20 +26,20 @@ void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t h
 	const auto high_less        = state.builder.AllocateId();
 	const auto high_ge          = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
+	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
+	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, carry_part, low, carry_count});
+	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
 	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, high_less, high_part, carry});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_ge, low, dword_shift});
+	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
 	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
 	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
 }
@@ -59,79 +59,59 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t
 	const auto low_less         = state.builder.AllocateId();
 	const auto low_ge           = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
+	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
+	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_part, low, dword_shift});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, high_part, high, dword_shift});
-	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, carry_part, high, carry_count});
+	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
 	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, low_less, low_part, carry});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_ge, high, dword_shift});
+	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
 	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
 	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
 }
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, mask)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
 	return ret;
 }
 
-AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
-                                  uint32_t carry_in) {
-	const auto pair0  = state.builder.AllocateId();
-	const auto sum0   = state.builder.AllocateId();
-	const auto carry0 = state.builder.AllocateId();
-	const auto pair1  = state.builder.AllocateId();
-	const auto sum1   = state.builder.AllocateId();
-	const auto carry1 = state.builder.AllocateId();
-	const auto carry  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair0, lhs, rhs});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum0, pair0, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry0, pair0, 1});
-	state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair1, sum0, carry_in});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum1, pair1, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry1, pair1, 1});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry0, carry1});
-	return {sum1, carry};
-}
-
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, ret, value, ConstantU32(state, shift)});
+	    {OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
 	return ret;
 }
 
 uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                 uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
 uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, state.uint_type, ret, ConstantU32(state, constant), value});
+	state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
 	return ret;
 }
 
 uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, f32});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32});
 
 	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
 	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
@@ -142,12 +122,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
 	const auto normal_payload = state.builder.AllocateId();
 	const auto normal         = state.builder.AllocateId();
-	state.builder.AddFunction({OpISub, state.uint_type, half_exp, exp, ConstantU32(state, 112)});
+	state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, normal_exp, half_exp, ConstantU32(state, 10)});
+	    {OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, state.uint_type, normal_payload, normal_exp, normal_mant});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, normal, sign, normal_payload});
+	    {OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
 
 	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
 	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
@@ -160,8 +140,8 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
 	const auto sub_mant  = state.builder.AllocateId();
 	const auto subnormal = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, sub_mant, mant_with_hidden, sub_shift});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, subnormal, sign, sub_mant});
+	    {OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
 
 	const auto nan_payload =
 	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
@@ -185,8 +165,8 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {max_value ? OpUGreaterThan : OpULessThan, state.bool_type, cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
+	    {max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
 	return ret;
 }
 
@@ -194,44 +174,44 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto cond = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {max_value ? OpSGreaterThan : OpSLessThan, state.bool_type, cond, lhs, rhs});
-	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
+	    {max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalOr, state.bool_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value});
+	state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value});
 	return ret;
 }
 
@@ -243,12 +223,8 @@ F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
 	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
 	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
 	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
-	const auto quiet_bit_clear =
-	    EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0);
-	cls.nan        = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
-	cls.snan       = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear);
-	cls.zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
-	cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u));
+	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
+	cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
 	return cls;
 }
 
@@ -316,7 +292,7 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 	const auto rhs_class = EmitClassifyF32(state, rhs);
 
 	const auto numeric_cond = state.builder.AllocateId();
-	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type,
+	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state),
 	                           numeric_cond, lhs, rhs});
 	const auto ordered_bits =
 	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
@@ -328,19 +304,14 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
 
 	const auto rhs_nan_bits =
 	    EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits);
-	const auto non_snan_bits =
-	    EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
-	const auto rhs_snan_bits =
-	    EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits);
-	const auto result_bits =
-	    EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits);
+	const auto result_bits = EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
 	return EmitBitcastU32ToF32(state, result_bits);
 }
 
 uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.float_type, ret, state.glsl_std450, GlslTrunc, value});
+	    {OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value});
 	return ret;
 }
 
@@ -353,18 +324,18 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
 	const auto flush     = state.builder.AllocateId();
 	const auto selected  = state.builder.AllocateId();
 	const auto ret       = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)});
+	    {OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, non_zero, abs_bits, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, flush, non_zero, subnormal});
-	state.builder.AddFunction({OpSelect, state.uint_type, selected, flush, sign_bits, bits});
-	state.builder.AddFunction({OpBitcast, state.float_type, ret, selected});
+	    {OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
+	state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected});
 	return ret;
 }
 
@@ -377,46 +348,45 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
 	const auto large_finite = state.builder.AllocateId();
 	const auto reduced      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.float_type, fract, state.glsl_std450, GlslFract, src});
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, src});
+	    {OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)});
+	    {OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)});
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, large_finite, large, finite});
+	    {OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
 	state.builder.AddFunction(
-	    {OpSelect, state.float_type, reduced, large_finite, ConstantF32(state, 0), fract});
+	    {OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
 	if (!preserve_signed_zero) {
 		return reduced;
 	}
 	const auto zero = state.builder.AllocateId();
 	const auto ret  = state.builder.AllocateId();
-	state.builder.AddFunction({OpIEqual, state.bool_type, zero, abs_bits, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpSelect, state.float_type, ret, zero, src, reduced});
+	state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced});
 	return ret;
 }
 
 uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpFNegate, state.float_type, ret, value});
+	state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, state.float_type, ret, state.glsl_std450, GlslFAbs, value});
+	state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value});
 	return ret;
 }
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, bits});
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, ret, unpacked, 0});
+	state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
+	                           GlslUnpackHalf2x16, bits});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -32,24 +32,17 @@ bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
 	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
 }
 
+bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
+	return state.stage == ShaderType::Pixel &&
+	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
+}
+
 void SetError(std::string* error, const char* message) {
 	if (error != nullptr) {
 		*error = message;
 	}
 }
 
-void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg) {
-	if (std::any_of(registers.begin(), registers.end(),
-	                [reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
-		return;
-	}
-	registers.push_back({reg, 0});
-}
-
-bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
-	return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
-}
-
 bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
                uint32_t index) {
 	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
@@ -59,8 +52,8 @@ bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind ki
 
 void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
 	for (const auto& input: program.info.inputs) {
-		state.inputs.push_back(
-		    {input.kind, input.location, input.component_count, 0, input.debug_name});
+		state.inputs.push_back({input.kind, input.location, input.component_count, 0,
+		                        input.debug_name, input.per_vertex});
 	}
 	for (const auto& output: program.info.outputs) {
 		if (HasOutput(state.outputs, output.kind, output.index)) {
@@ -71,7 +64,7 @@ void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program
 }
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
-	if (exp.kind == IR::ExportTargetKind::Position) {
+	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
 		return state.per_vertex_variable;
 	}
 	if (exp.kind == IR::ExportTargetKind::MrtZ) {
@@ -88,170 +81,6 @@ uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo
 	return 0;
 }
 
-bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() == IR::ValueOpcode::ImageQueryLod) {
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() == IR::ValueOpcode::ImageGatherRaw) {
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsFunctionLds(const IR::Program& program) {
-	if (program.stage == ShaderType::Compute || program.values == nullptr) {
-		return false;
-	}
-
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::SwizzleU32: return true;
-				case IR::ValueOpcode::LoadSharedU8:
-				case IR::ValueOpcode::LoadSharedU16:
-				case IR::ValueOpcode::LoadSharedU32:
-				case IR::ValueOpcode::WriteSharedU8:
-				case IR::ValueOpcode::WriteSharedU16:
-				case IR::ValueOpcode::WriteSharedU32:
-				case IR::ValueOpcode::SharedAtomicFMin32:
-				case IR::ValueOpcode::SharedAtomicFMax32:
-				case IR::ValueOpcode::SharedAtomicSwap32:
-				case IR::ValueOpcode::SharedAtomicIAdd32:
-				case IR::ValueOpcode::SharedAtomicISub32:
-				case IR::ValueOpcode::SharedAtomicSMin32:
-				case IR::ValueOpcode::SharedAtomicUMin32:
-				case IR::ValueOpcode::SharedAtomicSMax32:
-				case IR::ValueOpcode::SharedAtomicUMax32:
-				case IR::ValueOpcode::SharedAtomicAnd32:
-				case IR::ValueOpcode::SharedAtomicOr32:
-				case IR::ValueOpcode::SharedAtomicXor32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume: {
-					const auto index = inst.Flags<IR::MemoryFlags>().index;
-					if (index < program.values->memory_info.size() &&
-					    program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
-						return true;
-					}
-					break;
-				}
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsPixelValidMask(const IR::Program& program) {
-	if (program.stage != ShaderType::Pixel || program.values == nullptr) {
-		return false;
-	}
-
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != IR::ValueOpcode::SetAttribute) {
-				continue;
-			}
-			const auto index = inst.Flags<IR::ExportFlags>().index;
-			if (index < program.values->export_info.size() &&
-			    program.values->export_info[index].vm) {
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
-		for (const auto* block: program.values->blocks) {
-			for (const auto& inst: *block) {
-				if (inst.GetOpcode() == IR::ValueOpcode::WqmMask) {
-					return true;
-				}
-			}
-		}
-	}
-	return ProgramRequiresExactSubgroupSize(program);
-}
-
-bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::DppMoveU32:
-				case IR::ValueOpcode::ReadFirstLane:
-				case IR::ValueOpcode::ReadLane:
-				case IR::ValueOpcode::Permlane16U32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume:
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume:
-				case IR::ValueOpcode::SwizzleU32: return true;
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
-	if (program.values == nullptr) {
-		return false;
-	}
-	for (const auto* block: program.values->blocks) {
-		for (const auto& inst: *block) {
-			switch (inst.GetOpcode()) {
-				case IR::ValueOpcode::DppMoveU32:
-				case IR::ValueOpcode::DppUpdateU32:
-				case IR::ValueOpcode::WqmMask:
-				case IR::ValueOpcode::LaneId:
-				case IR::ValueOpcode::WriteLane:
-				case IR::ValueOpcode::Permlane16U32:
-				case IR::ValueOpcode::DataAppend:
-				case IR::ValueOpcode::DataConsume:
-				case IR::ValueOpcode::GdsDataAppend:
-				case IR::ValueOpcode::GdsDataConsume:
-				case IR::ValueOpcode::SwizzleU32: return true;
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
-	for (const auto& binding: state.registers) {
-		if (binding.reg == reg) {
-			return binding.pointer_id;
-		}
-	}
-	return 0;
-}
-
 uint32_t          ConstantU32(EmitterState& state, uint32_t value);
 [[noreturn]] void ExitDescriptorBindingFailure(const EmitterState&       state,
                                                IR::DescriptorBindingKind kind, uint32_t resource,
@@ -310,11 +139,7 @@ ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryIn
 	return ImageViewKindFromDimension(mem.image_dimension);
 }
 
-ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   bool uint_image, uint32_t use_pc) {
-	(void)state;
-	(void)uint_image;
-	(void)use_pc;
+ImageViewKind StorageImageViewKind(const IR::MemoryInfo& mem) {
 	return ImageViewKindFromDimension(mem.image_dimension);
 }
 
@@ -344,45 +169,70 @@ uint32_t ImageViewSpatialComponents(ImageViewKind view) {
 	}
 }
 
-uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) {
-	return state.sampled_images[SampledImageIndex(integer, view)].image_type;
+uint32_t ImageViewImageType(EmitterState& state, ImageViewKind view, bool integer) {
+	const auto component = integer ? TypeU32(state) : TypeF32(state);
+	return state.builder.Type(OpTypeImage,
+	                          {component, ImageSpirvDimension(view), 0, ImageSpirvArrayed(view),
+	                           ImageSpirvMultisampled(view), 1, ImageFormatUnknown});
 }
 
-uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) {
-	return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type;
+uint32_t ImageViewSampledImageType(EmitterState& state, ImageViewKind view, bool integer) {
+	return state.builder.Type(OpTypeSampledImage, {ImageViewImageType(state, view, integer)});
 }
 
-uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) {
+uint32_t ImageViewSizeType(EmitterState& state, ImageViewKind view) {
 	switch (ImageViewCoordinateComponents(view)) {
-		case 1u: return state.uint_type;
-		case 2u: return state.vec2_uint_type;
-		case 3u: return state.vec3_uint_type;
+		case 1u: return TypeU32(state);
+		case 2u: return TypeU32Vector(state, 2);
+		case 3u: return TypeU32Vector(state, 3);
 		default: return 0;
 	}
 }
 
-uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) {
-	return state.storage_images[StorageImageIndex(uint_image, view)].image_type;
+uint32_t StorageImageType(EmitterState& state, StorageImageClass image_class, ImageViewKind view) {
+	uint32_t component = TypeF32(state);
+	uint32_t format    = ImageFormatUnknown;
+	if (image_class == StorageImageClass::FormatlessUint) {
+		component = TypeU32(state);
+	} else if (image_class == StorageImageClass::AtomicUint) {
+		component = TypeU32(state);
+		format    = ImageFormatR32ui;
+	}
+	return state.builder.Type(OpTypeImage, {component, ImageSpirvDimension(view), 0,
+	                                        ImageSpirvArrayed(view), 0, 2, format});
 }
 
-uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) {
-	return state.storage_images[StorageImageIndex(uint_image, view)].pointer_type;
+uint32_t StorageImagePointerType(EmitterState& state, StorageImageClass image_class,
+                                 ImageViewKind view) {
+	return state.builder.Type(
+	    OpTypePointer, {StorageClassUniformConstant, StorageImageType(state, image_class, view)});
 }
 
-uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) {
-	return state.storage_images[StorageImageIndex(uint_image, view)].variable;
+StorageImageClass StorageImageClassForResource(const EmitterState& state, uint32_t resource) {
+	const auto& image      = state.program.info.images.at(resource);
+	const bool  uint_image = image.kind == IR::ResourceKind::StorageImageUint;
+	EXIT_IF(image.kind != IR::ResourceKind::StorageImage && !uint_image);
+	EXIT_IF(image.atomic && !uint_image);
+	return StorageImageClassFor(uint_image, image.atomic);
+}
+
+uint32_t StorageImageVariable(const EmitterState& state, StorageImageClass image_class,
+                              ImageViewKind view) {
+	return state.storage_image_variables[StorageImageIndex(image_class, view)];
 }
 
 uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
                                     ImageViewKind view) {
 	(void)use_pc;
-	const bool  integer     = mem.kind == IR::ResourceKind::ImageUint;
-	const auto  kind        = SampledBindingKind(integer, view);
-	const auto  binding     = ResourceForDescriptor(state, kind, mem.resource);
-	const auto& descriptors = state.sampled_images[SampledImageIndex(integer, view)];
-	const auto  pointer     = DescriptorElementPointer(
-	    state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
-	    mem.resource, "sampled image descriptor array was not emitted");
+	const bool integer      = mem.kind == IR::ResourceKind::ImageUint;
+	const auto kind         = SampledBindingKind(integer, view);
+	const auto binding      = ResourceForDescriptor(state, kind, mem.resource);
+	const auto variable     = state.sampled_image_variables[SampledImageIndex(integer, view)];
+	const auto pointer_type = state.builder.Type(
+	    OpTypePointer, {StorageClassUniformConstant, ImageViewImageType(state, view, integer)});
+	const auto pointer =
+	    DescriptorElementPointer(state, pointer_type, variable, binding.array_index, kind,
+	                             mem.resource, "sampled image descriptor array was not emitted");
 	const auto image = state.builder.AllocateId();
 	state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
 	return image;
@@ -391,11 +241,14 @@ uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& m
 uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
 	(void)use_pc;
 	const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
+	const auto sampler_type = state.builder.Type(OpTypeSampler);
+	const auto pointer_type =
+	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type});
 	const auto pointer = DescriptorElementPointer(
-	    state, state.ptr_uniform_sampler, state.sampler_variable, binding.array_index,
+	    state, pointer_type, state.sampler_variable, binding.array_index,
 	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
 	const auto sampler_id = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.sampler_type, sampler_id, pointer});
+	state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer});
 	return sampler_id;
 }
 
@@ -423,43 +276,33 @@ uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32
 	return MakeSampledImage(state, selected, use_pc, view);
 }
 
-uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
-                                       uint32_t use_pc, ImageViewKind view) {
-	(void)use_pc;
-	const auto kind     = StorageBindingKind(uint_image, view);
+uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, ImageViewKind view) {
+	const auto image_class = StorageImageClassForResource(state, resource);
+	const auto kind        = StorageBindingKind(image_class, view);
 	const auto binding  = ResourceForDescriptor(state, kind, resource);
-	const auto ptr_type = StorageImagePointerType(state, uint_image, view);
-	const auto variable = StorageImageVariable(state, uint_image, view);
+	const auto ptr_type    = StorageImagePointerType(state, image_class, view);
+	const auto variable    = StorageImageVariable(state, image_class, view);
 	return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
 	                                "storage image descriptor array was not emitted");
 }
 
-uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
-                                    uint32_t use_pc, ImageViewKind view) {
-	const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
-	if (pointer == 0) {
-		ExitDescriptorBindingFailure(state, StorageBindingKind(uint_image, view), resource,
-		                             "storage image descriptor pointer creation failed");
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, ImageViewKind view,
+                           uint32_t mip_lod, uint32_t coord, uint32_t texel) {
+	const auto image_class = StorageImageClassForResource(state, resource);
+	if (image_class != StorageImageClass::AtomicUint) {
+		state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat);
 	}
-	const auto type  = StorageImageType(state, uint_image, view);
-	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, type, image, pointer});
-	return image;
-}
-
-void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_image,
-                           ImageViewKind view, uint32_t mip_lod, uint32_t coord, uint32_t texel) {
-	const auto  kind    = StorageBindingKind(uint_image, view);
+	const auto  kind    = StorageBindingKind(image_class, view);
 	const auto  binding = ResourceForDescriptor(state, kind, resource);
 	const auto& image   = state.program.info.images.at(resource);
 	const auto  LoadAt  = [&](uint32_t array_index) {
 		const auto pointer = DescriptorElementPointer(
-		    state, StorageImagePointerType(state, uint_image, view),
-		    StorageImageVariable(state, uint_image, view), array_index, kind, resource,
+		    state, StorageImagePointerType(state, image_class, view),
+		    StorageImageVariable(state, image_class, view), array_index, kind, resource,
 		    "storage image descriptor array was not emitted");
 		const auto descriptor = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpLoad, StorageImageType(state, uint_image, view), descriptor, pointer});
+		    {OpLoad, StorageImageType(state, image_class, view), descriptor, pointer});
 		return descriptor;
 	};
 	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
@@ -482,11 +325,11 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_ima
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction(words);
 	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
-		state.builder.AddFunction({OpLabel, labels[mip]});
+		EmitLabel(state, labels[mip]);
 		state.builder.AddFunction({OpImageWrite, LoadAt(binding.array_index + mip), coord, texel});
 		state.builder.AddFunction({OpBranch, merge_label});
 	}
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 }
 
 uint32_t ExecutionModelForStage(ShaderType stage) {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp

@@ -1,52 +0,0 @@
-#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
-
-#include <algorithm>
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-
-bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
-	if (reg.file != IR::RegisterFile::Scalar) {
-		return false;
-	}
-	const auto& registers = state.program.bindings.user_data_registers;
-	const auto  found     = std::lower_bound(registers.begin(), registers.end(), reg.index);
-	if (found == registers.end() || *found != reg.index) {
-		return false;
-	}
-	dword_index = static_cast<uint32_t>(found - registers.begin());
-	return true;
-}
-
-uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
-	if (state.push_constant_variable != 0) {
-		const auto pointer = state.builder.AllocateId();
-		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
-		                           state.push_constant_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-		return value;
-	}
-	if (state.vsharp_storage_variable != 0) {
-		const auto pointer = state.builder.AllocateId();
-		const auto value   = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-		                           state.vsharp_storage_variable, ConstantU32(state, 0),
-		                           ConstantU32(state, dword_index)});
-		state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-		return value;
-	}
-	return ConstantU32(state, 0);
-}
-
-uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
-	if (reg.file != IR::RegisterFile::Exec) {
-		return 0;
-	}
-	if (state.per_invocation_masks) {
-		return reg.index == 0 ? 1u : 0u;
-	}
-	return reg.index == 0 || state.wave_size == 64u ? 0xffffffffu : 0u;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -5,44 +5,57 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 namespace {
 
-uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
-	uint32_t ret = ConstantU32(state, 0);
-	for (uint32_t i = 0; i < 8u; i++) {
-		const auto mask     = 0x0fu << (i * 4u);
-		const auto masked   = state.builder.AllocateId();
-		const auto non_zero = state.builder.AllocateId();
-		const auto expanded = state.builder.AllocateId();
-		const auto combined = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
-		state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
-		                           ConstantU32(state, mask), ConstantU32(state, 0)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
-		ret = combined;
+bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
+	if (reg.file != IR::RegisterFile::Scalar) {
+		return false;
 	}
-	return ret;
+	const auto& registers = state.program.bindings.user_data_registers;
+	const auto  found     = std::lower_bound(registers.begin(), registers.end(), reg.index);
+	if (found == registers.end() || *found != reg.index) {
+		return false;
+	}
+	dword_index = static_cast<uint32_t>(found - registers.begin());
+	return true;
 }
 
-uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
-	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value,
-	                               ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
+uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
 	return result;
 }
 
-void StoreRegisterMirror(ValueEmitContext& ctx, IR::Register reg, uint32_t value) {
-	if (const auto pointer = PointerForRegister(ctx.state, reg); pointer != 0) {
-		if (IsInactiveWave32ExecHigh(ctx.state, reg)) {
-			value = ConstantU32(ctx.state, 0);
-		}
-		ctx.state.builder.AddFunction({OpStore, pointer, value});
-	}
-}
-
-bool IsPrologue(const ValueEmitContext& ctx) {
-	return !ctx.program.blocks.empty() && ctx.current_block == ctx.program.blocks.front();
+uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
+	                           ConstantU64(state, 0x0000000100000001ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                           ConstantU64(state, 0x0000000200000002ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                           ConstantU64(state, 0x1111111111111111ull)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
+	return result;
 }
 
 uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
@@ -57,26 +70,39 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 	if (kind == IR::StageInputKind::FrontFacing) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, state.bool_type, value, variable});
+		state.builder.AddFunction({OpLoad, TypeBool(state), value, variable});
 		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		    {OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) {
 		const auto value = state.builder.AllocateId();
 		const auto bits  = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, state.int_type, value, variable});
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+		state.builder.AddFunction({OpLoad, TypeI32(state), value, variable});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	if (kind == IR::StageInputKind::FragCoord) {
 		const auto pointer = state.builder.AllocateId();
 		const auto value   = state.builder.AllocateId();
 		const auto bits    = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_input_float, pointer, variable,
-		                           ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+		state.builder.AddFunction({OpAccessChain,
+		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		                           variable, ConstantU32(state, component)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::BaryCoordSmooth ||
+	    kind == IR::StageInputKind::BaryCoordNoPerspective) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		const auto bits    = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain,
+		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		                           variable, ConstantU32(state, component + 1u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
 		return bits;
 	}
 	return EmitInputComponentU32(state, kind, component);
@@ -85,30 +111,34 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
 uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
 	auto&      state  = ctx.state;
 	const auto ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 	                           ConstantU32(state, ScopeSubgroup), active});
-	const auto low  = state.builder.AllocateId();
-	const auto high = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
-	const auto wqm_low  = EmitWqmLaneU32(state, low);
-	const auto wqm_high = EmitWqmLaneU32(state, high);
+	const auto low = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
+	const auto wqm_low  = EmitWqmWordU32(state, low);
 	const auto lane     = EmitSubgroupLocalInvocationId(state);
-	const auto upper    = state.builder.AllocateId();
-	const auto mask     = state.builder.AllocateId();
-	const auto bit_lane = state.builder.AllocateId();
-	const auto bit      = state.builder.AllocateId();
-	const auto hit      = state.builder.AllocateId();
-	const auto result   = state.builder.AllocateId();
+	uint32_t   mask     = wqm_low;
+	uint32_t   bit_lane = lane;
+	if (state.wave_size == 64u) {
+		const auto high = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
+		const auto wqm_high = EmitWqmWordU32(state, high);
+		const auto upper    = state.builder.AllocateId();
+		mask                = state.builder.AllocateId();
+		bit_lane            = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, TypeU32(state), mask, upper, wqm_high, wqm_low});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)});
+	}
+	const auto bit    = state.builder.AllocateId();
+	const auto hit    = state.builder.AllocateId();
+	const auto result = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 32)});
-	state.builder.AddFunction({OpSelect, state.uint_type, mask, upper, wqm_high, wqm_low});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bit_lane, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_lane});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, result, hit, ConstantU32(state, 0)});
+	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, TypeBool(state), result, hit, ConstantU32(state, 0)});
 	return result;
 }
 
@@ -128,35 +158,35 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
 	const auto row_ok     = state.builder.AllocateId();
 	const auto masks_ok   = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, bank_shift, lane, ConstantU32(state, 2)});
+	    {OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, row_shift, lane, ConstantU32(state, 4)});
+	    {OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)});
+	    {OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)});
+	    {OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
+	    {OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
+	    {OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
+	    {OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, flags.row_mask), row_bit});
+	    {OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, bank_ok, bank_hit, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, row_ok, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, masks_ok, bank_ok, row_ok});
+	    {OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
 	uint32_t writable = masks_ok;
 	if (!flags.bound_control) {
 		const auto target  = EmitDppTargetLane(state, flags.control);
 		const auto bounded = state.builder.AllocateId();
-		state.builder.AddFunction({OpLogicalAnd, state.bool_type, bounded, writable, target.valid});
+		state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
 		writable = bounded;
 	}
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, result, exec, writable});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable});
 	return result;
 }
 
@@ -169,25 +199,93 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
 	if (state.stage == ShaderType::Vertex) {
 		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
 	}
+	const auto load_per_vertex = [&](uint32_t vertex) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		return value;
+	};
+	if (input->per_vertex) {
+		const auto barycentric_kind = state.input_info.pixel->ps_no_perspective
+		                                  ? IR::StageInputKind::BaryCoordNoPerspective
+		                                  : IR::StageInputKind::BaryCoordSmooth;
+		const auto barycentric      = InputVariableForKind(state, barycentric_kind);
+		uint32_t   sum              = 0;
+		for (uint32_t vertex = 0; vertex < 3u; vertex++) {
+			const auto pointer = state.builder.AllocateId();
+			const auto weight  = state.builder.AllocateId();
+			const auto product = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain,
+			                           TypePointer(state, StorageClassInput, TypeF32(state)),
+			                           pointer, barycentric, ConstantU32(state, vertex)});
+			state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer});
+			state.builder.AddFunction(
+			    {OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
+			if (vertex == 0u) {
+				sum = product;
+			} else {
+				const auto next = state.builder.AllocateId();
+				state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product});
+				sum = next;
+			}
+		}
+		const auto bits = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum});
+		return bits;
+	}
 	const auto vector    = state.builder.AllocateId();
 	const auto component = state.builder.AllocateId();
 	const auto bits      = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.vec4_float_type, vector, input->variable_id});
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, component, vector, chan & 3u});
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
+	state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component});
 	return bits;
 }
 
-bool MrtUsesUint(const EmitterState& state, const IR::ExportInfo& exp) {
-	return state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
-	       exp.index < std::size(state.input_info.pixel->target_output_mode) &&
-	       state.input_info.pixel->target_output_mode[exp.index] == 7u;
+uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32_t chan,
+                                    uint32_t mode) {
+	auto&       state = ctx.state;
+	const auto* input = InputBindingForParameter(state, attr);
+	if (!input->per_vertex) {
+		return EmitAttribute(ctx, attr, chan);
+	}
+	const auto load_vertex = [&](uint32_t vertex) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		return value;
+	};
+
+	const auto selected_vertex = (mode + 1u) % 3u;
+	uint32_t   value           = load_vertex(selected_vertex);
+	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
+		const auto delta = state.builder.AllocateId();
+		state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)});
+		value = delta;
+	}
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	return bits;
+}
+
+uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
+	if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
+	    exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
+		return 0;
+	}
+	return state.input_info.pixel->target_output_mode[exp.index];
 }
 
 uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
 	const auto value = ctx.state.builder.AllocateId();
 	ctx.state.builder.AddFunction(
-	    {OpCompositeExtract, ctx.state.uint_type, value, vector, component});
+	    {OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
 	return value;
 }
 
@@ -195,6 +293,8 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
                       bool uint_output) {
 	auto& state = ctx.state;
 	if (exp.compr && !uint_output) {
+		const auto unpack =
+		    MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
 		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
 		                   ConstantF32(state, 0x3f800000u)};
 		for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -203,21 +303,21 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 			}
 			const auto packed   = state.builder.AllocateId();
 			const auto unpacked = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, state.uint_type, packed, data, pair});
-			state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
-			                           state.glsl_std450, GlslUnpackHalf2x16, packed});
+			state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
+			state.builder.AddFunction(
+			    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed});
 			for (uint32_t lane = 0; lane < 2u; lane++) {
 				const auto component = pair * 2u + lane;
 				if (((exp.en >> component) & 1u) != 0u) {
 					f32[component] = state.builder.AllocateId();
 					state.builder.AddFunction(
-					    {OpCompositeExtract, state.float_type, f32[component], unpacked, lane});
+					    {OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
 				}
 			}
 		}
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
+		state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
+		                           f32[1], f32[2], f32[3]});
 		return vector;
 	}
 	uint32_t raw[4] = {
@@ -238,7 +338,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 					continue;
 				}
 				raw[component] = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, raw[component],
+				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component],
 				                           packed, ConstantU32(state, lane * 16u),
 				                           ConstantU32(state, 16)});
 			}
@@ -252,30 +352,73 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
 	}
 	if (uint_output) {
 		const auto vector = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeConstruct, state.vec4_uint_type, vector, raw[0], raw[1], raw[2], raw[3]});
+		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0],
+		                           raw[1], raw[2], raw[3]});
 		return vector;
 	}
 	uint32_t f32[4] {};
 	for (uint32_t component = 0; component < 4u; component++) {
 		f32[component] = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, state.float_type, f32[component], raw[component]});
+		state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]});
 	}
 	const auto vector = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
+	    {OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]});
 	return vector;
 }
 
+void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp) {
+	auto& state = ctx.state;
+	for (uint32_t component = 0; component < 4; component++) {
+		if ((exp.en & (1u << component)) == 0) {
+			continue;
+		}
+		const auto output = IR::DecodePositionExportComponent(
+		    state.input_info.vertex->pa_cl_vs_out_cntl, exp.index, component);
+		if (output.layer) {
+			const auto raw   = ExportRawComponent(ctx, data, component);
+			const auto layer = state.builder.AllocateId();
+			state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
+			                           ConstantU32(state, 0x7ffu)});
+			state.builder.AddFunction({OpStore, state.layer_variable, layer});
+			continue;
+		}
+		if (!output.point_size && output.clip_distance == UINT32_MAX &&
+		    output.cull_distance == UINT32_MAX) {
+			continue;
+		}
+
+		const auto raw = ExportRawComponent(ctx, data, component);
+		const auto f32 = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+		if (output.point_size) {
+			state.builder.AddFunction({OpStore, state.point_size_variable, f32});
+			continue;
+		}
+		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
+			if (index == UINT32_MAX) {
+				return;
+			}
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer,
+			     variable, ConstantU32(state, index)});
+			state.builder.AddFunction({OpStore, pointer, f32});
+		};
+		StoreDistance(state.clip_distance_variable, output.clip_distance);
+		StoreDistance(state.cull_distance_variable, output.cull_distance);
+	}
+}
+
 void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto&       state = ctx.state;
 	const auto& exp   = ctx.Export(inst);
 	const auto  exec  = ctx.Arg(inst, 1);
-	if (state.stage == ShaderType::Pixel && exp.vm && state.needs_pixel_valid_mask &&
+	if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
 	    state.pixel_valid_mask_variable != 0) {
 		const auto value = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
+		    {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
 		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
 	}
 	if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
@@ -284,20 +427,25 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 	}
 	EmitIfCondition(state, exec, [&]() {
 		const auto data = ctx.Arg(inst, 0);
+		if (exp.kind == IR::ExportTargetKind::Position && exp.index != 0) {
+			EmitAuxPositionExport(ctx, data, exp);
+			return;
+		}
 		if (exp.kind == IR::ExportTargetKind::MrtZ) {
 			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
 				const auto raw = ExportRawComponent(ctx, data, 0);
 				const auto f32 = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, state.float_type, f32, raw});
+				state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
 				state.builder.AddFunction({OpStore, state.depth_variable, f32});
 			}
 			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
 				const auto raw     = ExportRawComponent(ctx, data, 2);
 				const auto value   = state.builder.AllocateId();
 				const auto pointer = state.builder.AllocateId();
-				state.builder.AddFunction({OpBitcast, state.int_type, value, raw});
-				state.builder.AddFunction({OpAccessChain, state.ptr_output_int, pointer,
-				                           state.sample_mask_variable, ConstantU32(state, 0)});
+				state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw});
+				state.builder.AddFunction(
+				    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer,
+				     state.sample_mask_variable, ConstantU32(state, 0)});
 				state.builder.AddFunction({OpStore, pointer, value});
 			}
 			return;
@@ -306,8 +454,8 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		if (variable == 0) {
 			return;
 		}
-		const bool uint_output = MrtUsesUint(state, exp);
-		const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
+		const bool uint_output = MrtOutputMode(state, exp) == 7u;
+		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
 		auto       value       = ExportVector(ctx, data, exp, uint_output);
 		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
 		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
@@ -322,8 +470,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 		}
 		if (exp.kind == IR::ExportTargetKind::Position) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer,
-			                           variable, ConstantU32(state, 0)});
+			state.builder.AddFunction(
+			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
+			     pointer, variable, ConstantU32(state, 0)});
 			state.builder.AddFunction({OpStore, pointer, value});
 		} else {
 			state.builder.AddFunction({OpStore, variable, value});
@@ -336,63 +485,10 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
 bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 	auto& state = ctx.state;
 	switch (inst.GetOpcode()) {
-		case IR::ValueOpcode::Identity:
-		case IR::ValueOpcode::ConditionRef: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
-		case IR::ValueOpcode::SetScalarRegister:
-			StoreRegisterMirror(
-			    ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
-			    ctx.Arg(inst, 1));
-			return true;
-		case IR::ValueOpcode::SetThreadBitScalarRegister:
-			StoreRegisterMirror(
-			    ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
-			    BoolAsU32(ctx, ctx.Arg(inst, 1)));
-			return true;
-		case IR::ValueOpcode::SetVectorRegister:
-			StoreRegisterMirror(
-			    ctx, {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())},
-			    ctx.Arg(inst, 1));
-			return true;
-		case IR::ValueOpcode::SetScc: ctx.Arg(inst, 0); return true;
-		case IR::ValueOpcode::SetExec: ctx.Arg(inst, 0); return true;
-		case IR::ValueOpcode::SetExecLo:
-		case IR::ValueOpcode::SetExecHi:
-			if (!IsPrologue(ctx)) {
-				StoreRegisterMirror(ctx,
-				                    {IR::RegisterFile::Exec,
-				                     inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u},
-				                    ctx.Arg(inst, 0));
-			}
-			return true;
-		case IR::ValueOpcode::SetVcc: ctx.Arg(inst, 0); return true;
-		case IR::ValueOpcode::SetVccLo:
-		case IR::ValueOpcode::SetVccHi:
-			if (!IsPrologue(ctx)) {
-				StoreRegisterMirror(ctx,
-				                    {IR::RegisterFile::Vcc,
-				                     inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u},
-				                    ctx.Arg(inst, 0));
-			}
-			return true;
-		case IR::ValueOpcode::SetM0:
-			StoreRegisterMirror(ctx, {IR::RegisterFile::M0, 0}, ctx.Arg(inst, 0));
-			return true;
-		case IR::ValueOpcode::GetScalarRegister:
-		case IR::ValueOpcode::GetThreadBitScalarRegister:
-		case IR::ValueOpcode::GetVectorRegister:
-		case IR::ValueOpcode::GetScc:
-		case IR::ValueOpcode::GetExec:
-		case IR::ValueOpcode::GetExecLo:
-		case IR::ValueOpcode::GetExecHi:
-		case IR::ValueOpcode::GetVcc:
-		case IR::ValueOpcode::GetVccLo:
-		case IR::ValueOpcode::GetVccHi:
-		case IR::ValueOpcode::GetM0: return true;
+		case IR::ValueOpcode::Identity: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
 		case IR::ValueOpcode::Void:
 		case IR::ValueOpcode::Reference:
 		case IR::ValueOpcode::ReferenceU32:
-		case IR::ValueOpcode::Prologue:
-		case IR::ValueOpcode::Epilogue:
 		case IR::ValueOpcode::ControlNop:
 		case IR::ValueOpcode::Waitcnt:
 		case IR::ValueOpcode::Sendmsg:
@@ -430,7 +526,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto flags    = inst.Flags<IR::DppMoveFlags>();
 			const auto target   = EmitDppTargetLane(state, flags.control);
 			const auto shuffled = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+			state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
 			                           target.lane});
 			if (flags.fetch_inactive) {
@@ -438,12 +534,12 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 				return true;
 			}
 			const auto ballot = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+			state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
 			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
 			const auto can_fetch     = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
+			    {OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
 			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
 			return true;
 		}
@@ -453,6 +549,9 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
 			return true;
 		}
+		case IR::ValueOpcode::WqmU64:
+			ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
+			return true;
 		case IR::ValueOpcode::WqmMask:
 			ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
 			return true;
@@ -460,23 +559,15 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
 			return true;
 		case IR::ValueOpcode::Ballot:
-			if (state.per_invocation_masks) {
-				const auto result = state.builder.AllocateId();
-				state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result,
-				                           BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
-				                           ConstantU32(state, 0), ConstantU32(state, 0)});
-				ctx.Define(inst, result);
-			} else {
-				ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
-				         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
-			}
+			ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
 			return true;
 		case IR::ValueOpcode::ReadFirstLane: {
 			const auto ballot = state.builder.AllocateId();
 			const auto lane   = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+			state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
-			state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, lane,
+			state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), lane,
 			                           ConstantU32(state, ScopeSubgroup), ballot});
 			ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
 			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
@@ -488,7 +579,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			return true;
 		case IR::ValueOpcode::WriteLane: {
 			const auto hit = state.builder.AllocateId();
-			state.builder.AddFunction({OpIEqual, state.bool_type, hit,
+			state.builder.AddFunction({OpIEqual, TypeBool(state), hit,
 			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
 			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
 			return true;
@@ -507,40 +598,40 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto index     = state.builder.AllocateId();
 			const auto target    = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
+			    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
 			if (flags.x16) {
 				state.builder.AddFunction(
-				    {OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
+				    {OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
 			} else {
-				state.builder.AddFunction({OpCopyObject, state.uint_type, row_value, row});
+				state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row});
 			}
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+			    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
+			    {OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
 			state.builder.AddFunction(
-			    {OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
+			    {OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
 			state.builder.AddFunction(
-			    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
+			    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
+			    {OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
 			state.builder.AddFunction(
-			    {OpShiftRightLogical, state.uint_type, shifted, selected, shift});
+			    {OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
 			state.builder.AddFunction(
-			    {OpBitwiseAnd, state.uint_type, index, shifted, ConstantU32(state, 15)});
-			state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index});
+			    {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
+			state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
 			const auto shuffled = state.builder.AllocateId();
-			state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+			state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
 			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
 			                           target});
 			uint32_t result = shuffled;
 			if (!flags.fetch_inactive) {
 				const auto source_exec = state.builder.AllocateId();
-				state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
+				state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec,
 				                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
 				                           target});
 				result = state.builder.AllocateId();
-				state.builder.AddFunction({OpSelect, state.uint_type, result, source_exec, shuffled,
+				state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
 				                           ConstantU32(state, 0)});
 			}
 			ctx.Define(inst, result);
@@ -549,6 +640,10 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetAttribute:
 			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
 			return true;
+		case IR::ValueOpcode::GetInterpolationParameter:
+			ctx.Define(inst, EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(),
+			                                            inst.Arg(2).U32()));
+			return true;
 		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
 		case IR::ValueOpcode::GetShaderBase:
 			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
@@ -558,10 +653,9 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
 		case IR::ValueOpcode::GetSrtResource:
 		case IR::ValueOpcode::GetBufferResource:
 		case IR::ValueOpcode::GetAddressResource:
+		case IR::ValueOpcode::GetScratchResource:
 		case IR::ValueOpcode::GetImageResource:
 		case IR::ValueOpcode::GetSamplerResource:
-		case IR::ValueOpcode::GetLdsResource:
-		case IR::ValueOpcode::GetGdsResource:
 		case IR::ValueOpcode::MakeImageAddress: return true;
 		default: return false;
 	}

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -3,10 +3,7 @@
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t EmitTrueBool(EmitterState& state) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
-	return ret;
+	return ConstantBool(state, true);
 }
 
 DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
@@ -17,15 +14,15 @@ DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto selected  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
+	    {OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
+	    {OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
 	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
+	    {OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
+	    {OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected});
 	return {target, EmitTrueBool(state)};
 }
 
@@ -37,20 +34,20 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
 	const auto target       = state.builder.AllocateId();
 	const auto valid        = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 	if (left) {
 		state.builder.AddFunction(
-		    {OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
+		    {OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
 		state.builder.AddFunction(
-		    {OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
+		    {OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
 	} else {
 		state.builder.AddFunction(
-		    {OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
+		    {OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
 		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
+		    {OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
 	}
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
 	return {target, valid};
 }
 
@@ -64,15 +61,15 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
 	const auto selected = state.builder.AllocateId();
 	const auto target   = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
 	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
+	    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
 	state.builder.AddFunction(
-	    {OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
-	state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
+	    {OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
+	state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected});
 	return {target, EmitTrueBool(state)};
 }
 
@@ -84,12 +81,12 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
 	const auto mirrored  = state.builder.AllocateId();
 	const auto target    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
+	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
+	    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
 	state.builder.AddFunction(
-	    {OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
+	    {OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored});
 	return {target, EmitTrueBool(state)};
 }
 
@@ -118,11 +115,11 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
 
 uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
 	if (state.subgroup_local_invocation_id_variable == 0) {
-		return ConstantU32(state, 0);
+		EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n");
 	}
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
+	    {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
 	return value;
 }
 
@@ -151,9 +148,9 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
 	}
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
-	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+	state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)),
+	                           pointer, variable, ConstantU32(state, component)});
+	state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
 	return value;
 }
 
@@ -163,7 +160,7 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
 		return ConstantU32(state, 0);
 	}
 	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
+	state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
 	return value;
 }
 
@@ -200,47 +197,47 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
 	}
 
 	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw});
 	return bits;
 }
 
 uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
 	const auto active_ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
+	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
 	                           ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
 	return EmitBallotLaneActiveBool(state, active_ballot, lane);
 }
 uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
 	const auto low = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
 	uint32_t mask = low;
 	if (state.wave_size == 64u) {
 		const auto high     = state.builder.AllocateId();
 		const auto in_high  = state.builder.AllocateId();
 		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
+		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
 		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
+		    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low});
 		mask = selected;
 	}
 
 	const auto lane_low = state.builder.AllocateId();
 	const auto bit      = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
+	    {OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
+	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
 
 	const auto hit      = state.builder.AllocateId();
 	const auto active   = state.builder.AllocateId();
 	const auto in_range = state.builder.AllocateId();
 	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
+	    {OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.wave_size)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range});
 	return ret;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -1,3 +1,4 @@
+#include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
 
 #include <algorithm>
@@ -14,6 +15,13 @@ uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
 	return index;
 }
 
+uint32_t DmaskComponent(uint32_t dmask, uint32_t index) {
+	for (uint32_t component = 0; component < 4u; component++) {
+		if (((dmask >> component) & 1u) != 0u && index-- == 0u) return component;
+	}
+	return 0u;
+}
+
 uint32_t ImageGatherComponent(uint32_t dmask) {
 	switch (dmask) {
 		case 0x2u: return 1;
@@ -78,10 +86,10 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	auto value = ctx.Def(address.Arg(packed));
 	if (ComponentUsesA16(mem, component)) {
 		if ((half & 1u) != 0u) {
-			value = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, value,
+			value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value,
 			               ConstantU32(ctx.state, 16));
 		}
-		value = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, value,
+		value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value,
 		               ConstantU32(ctx.state, 0xffffu));
 	}
 	return value;
@@ -92,7 +100,7 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
 	const auto value = AddressU32(ctx, mem, address, component);
 	return ComponentUsesA16(mem, component)
 	           ? EmitF16BitsToF32(ctx.state, value)
-	           : Unary(ctx.state, OpBitcast, ctx.state.float_type, value);
+	           : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value);
 }
 
 ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) {
@@ -117,28 +125,20 @@ uint32_t ZeroF32(EmitterState& state) {
 	return ConstantF32(state, 0);
 }
 
-uint32_t ZeroU32x4(EmitterState& state) {
-	const auto zero  = ConstantU32(state, 0);
-	const auto value = state.builder.AllocateId();
-	state.builder.AddType(
-	    {OpConstantComposite, state.vec4_uint_type, value, zero, zero, zero, zero});
-	return value;
-}
-
 uint32_t CubeAxis(EmitterState& state, uint32_t value) {
-	return Binary(state, OpFSub, state.float_type, value, ConstantF32(state, 0x3f800000u));
+	return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
 }
 
 uint32_t CubeLayer(EmitterState& state, uint32_t value) {
 	const auto guest = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertFToU, state.uint_type, guest, value});
+	state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value});
 	const auto padding =
-	    Binary(state, OpShiftLeftLogical, state.uint_type,
-	           Binary(state, OpShiftRightLogical, state.uint_type, guest, ConstantU32(state, 3)),
+	    Binary(state, OpShiftLeftLogical, TypeU32(state),
+	           Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
 	           ConstantU32(state, 1));
-	const auto host   = Binary(state, OpISub, state.uint_type, guest, padding);
+	const auto host   = Binary(state, OpISub, TypeU32(state), guest, padding);
 	const auto result = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, state.float_type, result, host});
+	state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host});
 	return result;
 }
 
@@ -159,10 +159,10 @@ uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		             : ZeroF32(ctx.state);
 		if (mem.image_cube) z = CubeLayer(ctx.state, z);
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec3_float_type, result, x, y, z});
+		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec2_float_type, result, x, y});
+		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
 	}
 	return result;
 }
@@ -179,10 +179,10 @@ uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
 		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
 		                                                 : ConstantU32(ctx.state, 0);
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec3_uint_type, result, x, y, z});
+		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec2_uint_type, result, x, y});
+		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
 	}
 	return result;
 }
@@ -196,10 +196,52 @@ uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst
 }
 
 uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
-	return Unary(ctx.state, OpBitcast, ctx.state.uint_type, value);
+	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
 }
 
-uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) {
+uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref,
+                      const IR::MemoryInfo& mem, bool gather = false) {
+	if (mem.data_bits == 16u) {
+		uint32_t   packed[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+		                        ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+		const auto scalar    = [&](uint32_t index) {
+			if (dref) return value;
+			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
+			const auto result    = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction({OpCompositeExtract,
+			                               integer ? TypeU32(ctx.state) : TypeF32(ctx.state),
+			                               result, value, component});
+			return result;
+		};
+		for (uint32_t word = 0; word < mem.data_dwords; word++) {
+			const auto low_index  = word * 2u;
+			const auto high_index = low_index + 1u;
+			const auto low        = scalar(low_index);
+			const auto high = high_index < mem.component_count
+			                      ? scalar(high_index)
+			                      : (integer ? ConstantU32(ctx.state, 0) : ZeroF32(ctx.state));
+			if (integer) {
+				const auto mask = ConstantU32(ctx.state, 0xffffu);
+				packed[word] =
+				    Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
+				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low, mask),
+				           Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
+				                  Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high, mask),
+				                  ConstantU32(ctx.state, 16u)));
+			} else {
+				const auto pair = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction(
+				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				packed[word] = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
+				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+			}
+		}
+		const auto result = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+		                               packed[0], packed[1], packed[2], packed[3]});
+		return result;
+	}
 	uint32_t component[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		if (dref) {
@@ -208,12 +250,12 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool
 		}
 		const auto scalar = ctx.state.builder.AllocateId();
 		ctx.state.builder.AddFunction({OpCompositeExtract,
-		                               integer ? ctx.state.uint_type : ctx.state.float_type, scalar,
+		                               integer ? TypeU32(ctx.state) : TypeF32(ctx.state), scalar,
 		                               value, index});
 		component[index] = integer ? scalar : FloatBits(ctx, scalar);
 	}
 	const auto result = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, result,
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
 	                               component[0], component[1], component[2], component[3]});
 	return result;
 }
@@ -240,15 +282,15 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::
 		for (uint32_t index = 0; index < components; index++) {
 			result[index] = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, ctx.state.uint_type, result[index], size, index});
+			    {OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
 		}
 	}
 	if (ImageSpirvMultisampled(view) == 0u) {
 		result[3] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpImageQueryLevels, ctx.state.uint_type, result[3], image});
+		ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
 	}
 	const auto vector = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, vector,
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
 	                               result[0], result[1], result[2], result[3]});
 	return vector;
 }
@@ -262,55 +304,51 @@ uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR
 		const auto result = ctx.state.builder.AllocateId();
 		if (components == 3u) {
 			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, ctx.state.vec3_int_type, result, zero, zero, zero});
+			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
 		} else {
 			ctx.state.builder.AddFunction(
-			    {OpCompositeConstruct, ctx.state.vec2_int_type, result, zero, zero});
+			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
 		}
 		return result;
 	}
-	const auto packed    = Unary(ctx.state, OpBitcast, ctx.state.int_type,
+	const auto packed    = Unary(ctx.state, OpBitcast, TypeI32(ctx.state),
 	                             AddressU32(ctx, mem, address, layout.offset));
 	uint32_t   values[3] = {zero, zero, zero};
 	for (uint32_t index = 0; index < components; index++) {
 		values[index] = ctx.state.builder.AllocateId();
-		ctx.state.builder.AddFunction({OpBitFieldSExtract, ctx.state.int_type, values[index],
+		ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index],
 		                               packed, ConstantU32(ctx.state, index * 8u),
 		                               ConstantU32(ctx.state, 6)});
 	}
 	if (components == 1u) return values[0];
 	const auto result = ctx.state.builder.AllocateId();
 	if (components == 3u) {
-		ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec3_int_type, result,
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result,
 		                               values[0], values[1], values[2]});
 	} else {
 		ctx.state.builder.AddFunction(
-		    {OpCompositeConstruct, ctx.state.vec2_int_type, result, values[0], values[1]});
+		    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
 	}
 	return result;
 }
 
 uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) {
 	const auto components   = ImageViewSpatialComponents(view);
-	const auto array_type   = state.builder.AllocateId();
 	const auto count        = ConstantU32(state, 4);
-	const auto element_type = components == 1u ? state.int_type : state.vec2_int_type;
-	state.builder.AddType({OpTypeArray, array_type, element_type, count});
-	uint32_t offsets[4] {};
+	const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
+	const auto array_type   = state.builder.Type(OpTypeArray, {element_type, count});
+	uint32_t   offsets[4] {};
 	for (uint32_t index = 0; index < 4u; index++) {
 		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
 		if (components == 1u) {
 			offsets[index] = x;
 		} else {
-			offsets[index] = state.builder.AllocateId();
-			state.builder.AddType({OpConstantComposite, state.vec2_int_type, offsets[index], x,
-			                       ConstantI32(state, 0)});
+			offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2),
+			                                        {x, ConstantI32(state, 0)});
 		}
 	}
-	const auto value = state.builder.AllocateId();
-	state.builder.AddType(
-	    {OpConstantComposite, array_type, value, offsets[0], offsets[1], offsets[2], offsets[3]});
-	return value;
+	return state.builder.Constant(OpConstantComposite, array_type,
+	                              {offsets[0], offsets[1], offsets[2], offsets[3]});
 }
 
 uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
@@ -320,8 +358,120 @@ uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
 	return UINT32_MAX;
 }
 
+Format::BufferFormatInfo ImageConversionFormat(const EmitterState&   state,
+                                               const IR::MemoryInfo& mem) {
+	const auto format = state.program.info.images[mem.resource].conversion_format;
+	if (format == Prospero::BufferFormat::kInvalid) return {};
+	const auto info = Format::GetFormatInfo(format);
+	EXIT_IF(!Prospero::IsSampledTextureFormat(format) ||
+	        !Prospero::IsUintTextureFormat(format) ||
+	        Prospero::RemapTextureFormat(format) == format ||
+	        info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
+	        info.byte_size != sizeof(uint32_t) || info.component_count == 0u ||
+	        info.component_count > 4u);
+	return info;
+}
+
+uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+
+	const auto packed = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
+	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	for (uint32_t component = 0; component < info.component_count; component++) {
+		components[component] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state),
+		                               components[component], packed,
+		                               ConstantU32(ctx.state, info.component_bit_offset[component]),
+		                               ConstantU32(ctx.state, info.component_bits[component])});
+	}
+	for (uint32_t component = info.component_count; component < 4u; component++) {
+		components[component] = components[component % info.component_count];
+	}
+
+	const auto swizzle = ctx.state.program.info.images[mem.resource].shader_swizzle;
+	uint32_t   selected[4] {};
+	for (uint32_t component = 0; component < 4u; component++) {
+		const auto selector = (swizzle >> (component * 3u)) & 7u;
+		if (selector == 1u) {
+			selected[component] = ConstantU32(ctx.state, 1u);
+		} else if (selector >= 4u) {
+			selected[component] = components[selector - 4u];
+		} else {
+			selected[component] = ConstantU32(ctx.state, 0u);
+		}
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               selected[0], selected[1], selected[2], selected[3]});
+	return result;
+}
+
+uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t gathered) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return gathered;
+
+	const auto component = ImageGatherComponent(mem.dmask);
+	const auto selector =
+	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	if (selector < 4u) {
+		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
+		return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4),
+		                                  {value, value, value, value});
+	}
+
+	uint32_t values[4] {};
+	for (uint32_t lane = 0; lane < 4u; lane++) {
+		const auto packed = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
+		values[lane]        = ctx.state.builder.AllocateId();
+		const auto physical = (selector - 4u) % info.component_count;
+		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed,
+		                               ConstantU32(ctx.state, info.component_bit_offset[physical]),
+		                               ConstantU32(ctx.state, info.component_bits[physical])});
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               values[0], values[1], values[2], values[3]});
+	return result;
+}
+
+uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+
+	auto packed = ConstantU32(ctx.state, 0u);
+	for (uint32_t component = 0; component < info.component_count; component++) {
+		const auto value = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
+		const auto maximum =
+		    ConstantU32(ctx.state, info.component_bits[component] == 32u
+		                               ? UINT32_MAX
+		                               : (1u << info.component_bits[component]) - 1u);
+		const auto within  = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum);
+		const auto clamped = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
+		const auto shifted =
+		    info.component_bit_offset[component] == 0u
+		        ? clamped
+		        : Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped,
+		                 ConstantU32(ctx.state, info.component_bit_offset[component]));
+		packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
+	}
+	const auto zero   = ConstantU32(ctx.state, 0u);
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(
+	    {OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
+	return result;
+}
+
 uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) {
-	const auto swizzle = ctx.state.program.info.images[mem.resource].storage_swizzle;
+	const auto swizzle = ctx.state.program.info.images[mem.resource].shader_swizzle;
 	uint32_t   values[4] {};
 	const auto dmask = mem.dmask != 0u ? mem.dmask : 1u;
 	for (uint32_t component = 0; component < 4u; component++) {
@@ -331,15 +481,27 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
 			const auto packed_index = DmaskComponentIndex(dmask, source);
 			raw                     = ctx.state.builder.AllocateId();
 			ctx.state.builder.AddFunction(
-			    {OpCompositeExtract, ctx.state.uint_type, raw, data, packed_index});
+			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
+			if (mem.data_bits == 16u) {
+				if ((packed_index & 1u) != 0u) {
+					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					             ConstantU32(ctx.state, 16u));
+				}
+				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				             ConstantU32(ctx.state, 0xffffu));
+			}
 		}
-		values[component] = integer ? raw : Unary(ctx.state, OpBitcast, ctx.state.float_type, raw);
+		values[component] = integer ? raw
+		                    : mem.data_bits == 16u
+		                        ? EmitF16BitsToF32(ctx.state, raw)
+		                        : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
 	}
 	const auto texel = ctx.state.builder.AllocateId();
-	ctx.state.builder.AddFunction({OpCompositeConstruct,
-	                               integer ? ctx.state.vec4_uint_type : ctx.state.vec4_float_type,
-	                               texel, values[0], values[1], values[2], values[3]});
-	return texel;
+	ctx.state.builder.AddFunction(
+	    {OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
+	     texel, values[0], values[1], values[2], values[3]});
+	return PackImageTexel(ctx, mem, texel);
 }
 
 uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
@@ -357,8 +519,9 @@ uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
 } // namespace
 
 bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
-	const auto op = inst.GetOpcode();
-	if (op < IR::ValueOpcode::ImageQueryDimensions || op > IR::ValueOpcode::ImageAtomicXor32) {
+	const auto op         = inst.GetOpcode();
+	const auto image_info = IR::ImageOpcodeInfoOf(op);
+	if (image_info.access == IR::ImageAccess::None) {
 		return false;
 	}
 	auto&       state     = ctx.state;
@@ -366,32 +529,30 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 	const auto  pc        = inst.Flags<IR::MemoryFlags>().pc;
 	const auto  image_arg = inst.Arg(0);
 	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
-	const bool  sampled = op == IR::ValueOpcode::ImageQueryLod ||
-	                      op == IR::ValueOpcode::ImageSampleRaw ||
-	                      op == IR::ValueOpcode::ImageGatherRaw;
-	const auto* address = ctx.ImageAddress(inst.Arg(sampled ? 2 : 1));
+	const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
 	if (address == nullptr) return true;
 	if (op == IR::ValueOpcode::ImageQueryDimensions) {
+		state.builder.RequireCapability(CapabilityImageQuery);
 		ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address));
 		return true;
 	}
 	if (op == IR::ValueOpcode::ImageQueryLod) {
+		state.builder.RequireCapability(CapabilityImageQuery);
 		const auto view    = SampledImageViewKind(state, mem, pc);
 		const auto sampled = MakeSampledImage(state, mem, pc, view);
 		const auto lod     = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpImageQueryLod, state.vec2_float_type, lod, sampled,
+		    {OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
 		     CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))});
 		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
 		                      ConstantU32(state, 0)};
 		for (uint32_t index = 0; index < 2u; index++) {
 			const auto component = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpCompositeExtract, state.float_type, component, lod, index});
+			state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index});
 			values[index] = FloatBits(ctx, component);
 		}
 		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, values[0],
+		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0],
 		                           values[1], values[2], values[3]});
 		ctx.Define(inst, result);
 		return true;
@@ -403,28 +564,31 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		ctx.Define(
 		    inst,
 		    EmitValueOrDefaultIfCondition(
-		        state, condition, state.vec4_uint_type, ZeroU32x4(state), [&]() {
+		        state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4),
+		        [&]() {
 			        const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
 			        const auto color = state.builder.AllocateId();
 			        const auto coord = CoordU32(ctx, mem, *address, view);
 			        if (ImageSpirvMultisampled(view) != 0u) {
 				        state.builder.AddFunction(
-				            {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
-				             color, image, coord, ImageOperandsSampleMask,
+				            {OpImageFetch,
+				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
+				             image, coord, ImageOperandsSampleMask,
 				             AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))});
 			        } else {
 				        state.builder.AddFunction(
-				            {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
-				             color, image, coord, ImageOperandsLodMask,
-				             LodU32(ctx, mem, *address, view)});
+				            {OpImageFetch,
+				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
+				             image, coord, ImageOperandsLodMask, LodU32(ctx, mem, *address, view)});
 			        }
-			        return ResultVector(ctx, color, integer, false);
+			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), integer, false,
+			                            mem);
 		        }));
 		return true;
 	}
 	if (op == IR::ValueOpcode::ImageWrite) {
 		const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint;
-		const auto view       = StorageImageViewKind(state, mem, uint_image, pc);
+		const auto view       = StorageImageViewKind(mem);
 		EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
 			const auto mip_lod =
 			    state.program.info.images[mem.resource].mip_mode == IR::ImageMipMode::DynamicStorage
@@ -432,7 +596,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			        : 0u;
 			const auto coord = CoordU32(ctx, mem, *address, view);
 			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
-			EmitStorageImageWrite(state, mem.resource, uint_image, view, mip_lod, coord, texel);
+			EmitStorageImageWrite(state, mem.resource, view, mip_lod, coord, texel);
 		});
 		return true;
 	}
@@ -441,6 +605,11 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		const auto layout  = Layout(mem, view);
 		const bool integer = mem.kind == IR::ResourceKind::ImageUint;
 		const bool dref    = HasFlag(mem, Decoder::ImageSampleFlagCompare);
+		if (dref && state.program.info.images[mem.resource].conversion_format !=
+		                Prospero::BufferFormat::kInvalid) {
+			ctx.Fail(inst, "uses depth comparison with a packed integer image");
+			return true;
+		}
 		const auto coord =
 		    CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view));
 		if (op == IR::ValueOpcode::ImageGatherRaw) {
@@ -448,7 +617,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			const auto            sample  = state.builder.AllocateId();
 			std::vector<uint32_t> words =
 			    dref ? std::vector<uint32_t> {OpImageDrefGather,
-			                                  state.vec4_float_type,
+			                                  TypeF32Vector(state, 4),
 			                                  sample,
 			                                  sampled,
 			                                  coord,
@@ -456,9 +625,15 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 			                                      ? AddressF32(ctx, mem, *address, layout.dref)
 			                                      : ZeroF32(state)}
 			         : std::vector<uint32_t> {
-			               OpImageGather, integer ? state.vec4_uint_type : state.vec4_float_type,
-			               sample,        sampled,
-			               coord,         ConstantU32(state, ImageGatherComponent(mem.dmask))};
+			               OpImageGather,
+			               integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4),
+			               sample,
+			               sampled,
+			               coord,
+			               ConstantU32(state, ImageConversionFormat(state, mem).format ==
+			                                          Prospero::BufferFormat::kInvalid
+			                                      ? ImageGatherComponent(mem.dmask)
+			                                      : 0u)};
 			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
 				words.push_back(ImageOperandsConstOffsetsMask);
 				words.push_back(HorizontalOffsets(state, view));
@@ -467,7 +642,8 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 				words.push_back(PackedOffset(ctx, mem, *address, layout, view));
 			}
 			state.builder.AddFunction(words);
-			ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false));
+			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
+			                              integer && !dref, false, mem, true));
 			return true;
 		}
 		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
@@ -478,7 +654,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		                        ? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod)
 		                        : (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod);
 		const auto result_type =
-		    dref ? state.float_type : (integer ? state.vec4_uint_type : state.vec4_float_type);
+		    dref ? TypeF32(state) : (integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4));
 		const auto dref_value =
 		    dref ? (layout.dref != NoImageComponent ? AddressF32(ctx, mem, *address, layout.dref)
 		                                            : ZeroF32(state))
@@ -517,7 +693,9 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		};
 		const auto& image = state.program.info.images[mem.resource];
 		if (image.indirect_root != mem.resource) {
-			ctx.Define(inst, ResultVector(ctx, EmitSample(mem.resource), integer, dref));
+			const auto sample = EmitSample(mem.resource);
+			ctx.Define(inst, ResultVector(ctx, dref ? sample : UnpackImageTexel(ctx, mem, sample),
+			                              integer, dref, mem));
 			return true;
 		}
 		const auto* handle = image_arg.ResolveInstruction();
@@ -537,10 +715,11 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		}
 		const auto LoadMapping = [&](uint32_t index) {
 			const auto pointer = state.builder.AllocateId();
-			state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-			                           state.flattened_srt_variable, ConstantU32(state, 0), index});
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state),
+			                           pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+			                           index});
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+			state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
 			return value;
 		};
 		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
@@ -549,39 +728,38 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		auto       selected = ConstantU32(state, 0u);
 		for (uint32_t iteration = 0; iteration < std::bit_width(image.indirect_mapping_capacity);
 		     iteration++) {
-			const auto active = Binary(state, OpULessThan, state.bool_type, low, high);
+			const auto active = Binary(state, OpULessThan, TypeBool(state), low, high);
 			const auto mid =
-			    Binary(state, OpShiftRightLogical, state.uint_type,
-			           Binary(state, OpIAdd, state.uint_type, low, high), ConstantU32(state, 1u));
+			    Binary(state, OpShiftRightLogical, TypeU32(state),
+			           Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u));
 			const auto probe = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, probe, active, mid, ConstantU32(state, 0u)});
-			const auto entry      = Binary(state, OpIAdd, state.uint_type, mapping,
-			                               Binary(state, OpIAdd, state.uint_type,
-			                                      Binary(state, OpShiftLeftLogical, state.uint_type,
+			    {OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
+			const auto entry      = Binary(state, OpIAdd, TypeU32(state), mapping,
+			                               Binary(state, OpIAdd, TypeU32(state),
+			                                      Binary(state, OpShiftLeftLogical, TypeU32(state),
 			                                             probe, ConstantU32(state, 1u)),
 			                                      ConstantU32(state, 1u)));
 			const auto mapped_key = LoadMapping(entry);
 			const auto candidate =
-			    LoadMapping(Binary(state, OpIAdd, state.uint_type, entry, ConstantU32(state, 1u)));
-			const auto equal         = Binary(state, OpIEqual, state.bool_type, mapped_key, key);
-			const auto match         = Binary(state, OpLogicalAnd, state.bool_type, active, equal);
+			    LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
+			const auto equal         = Binary(state, OpIEqual, TypeBool(state), mapped_key, key);
+			const auto match         = Binary(state, OpLogicalAnd, TypeBool(state), active, equal);
 			const auto next_selected = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, next_selected, match, candidate, selected});
+			    {OpSelect, TypeU32(state), next_selected, match, candidate, selected});
 			selected              = next_selected;
-			const auto less       = Binary(state, OpULessThan, state.bool_type, mapped_key, key);
-			const auto take_upper = Binary(state, OpLogicalAnd, state.bool_type, active, less);
-			const auto take_lower = Binary(state, OpLogicalAnd, state.bool_type, active,
-			                               Unary(state, OpLogicalNot, state.bool_type, less));
+			const auto less       = Binary(state, OpULessThan, TypeBool(state), mapped_key, key);
+			const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less);
+			const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active,
+			                               Unary(state, OpLogicalNot, TypeBool(state), less));
 			const auto next_low   = state.builder.AllocateId();
 			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, next_low, take_upper,
-			     Binary(state, OpIAdd, state.uint_type, mid, ConstantU32(state, 1u)), low});
+			    {OpSelect, TypeU32(state), next_low, take_upper,
+			     Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
 			low                  = next_low;
 			const auto next_high = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpSelect, state.uint_type, next_high, take_lower, mid, high});
+			state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high});
 			high = next_high;
 		}
 		const auto            default_label = state.builder.AllocateId();
@@ -596,32 +774,36 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
 		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 		state.builder.AddFunction(switch_words);
 		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
-		state.builder.AddFunction({OpLabel, default_label});
+		EmitLabel(state, default_label);
 		phi_words.push_back(EmitSample(image.indirect_resources[0]));
 		phi_words.push_back(default_label);
 		state.builder.AddFunction({OpBranch, merge_label});
 		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
-			state.builder.AddFunction({OpLabel, labels[candidate - 1u]});
+			EmitLabel(state, labels[candidate - 1u]);
 			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
 			phi_words.push_back(labels[candidate - 1u]);
 			state.builder.AddFunction({OpBranch, merge_label});
 		}
-		state.builder.AddFunction({OpLabel, merge_label});
+		EmitLabel(state, merge_label);
 		state.builder.AddFunction(phi_words);
-		ctx.Define(inst, ResultVector(ctx, phi_words[2], integer, dref));
+		ctx.Define(inst,
+		           ResultVector(ctx, dref ? phi_words[2] : UnpackImageTexel(ctx, mem, phi_words[2]),
+		                        integer, dref, mem));
 		return true;
 	}
 	const auto atomic_opcode = ImageAtomicOpcode(op);
 	if (atomic_opcode != 0u) {
-		const auto view = StorageImageViewKind(state, mem, true, pc);
+		const auto view = StorageImageViewKind(mem);
 		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
 			           const auto pointer = state.builder.AllocateId();
+			           const auto pointer_type =
+			               state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)});
 			           state.builder.AddFunction(
-			               {OpImageTexelPointer, state.ptr_image_uint, pointer,
-			                StorageImageDescriptorPointer(state, mem.resource, true, pc, view),
+			               {OpImageTexelPointer, pointer_type, pointer,
+			                StorageImageDescriptorPointer(state, mem.resource, view),
 			                CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)});
 			           const auto old = state.builder.AllocateId();
-			           state.builder.AddFunction({atomic_opcode, state.uint_type, old, pointer,
+			           state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
 			                                      ConstantU32(state, ScopeDevice),
 			                                      ConstantU32(state, MemorySemanticsNone),
 			                                      ctx.Arg(inst, 2)});

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -39,15 +39,18 @@ enum : uint32_t {
 	MemoryModelGLSL450                       = 1,
 	CapabilityShader                         = 1,
 	CapabilityImageGatherExtended            = 25,
+	CapabilityClipDistance                   = 32,
+	CapabilityCullDistance                   = 33,
 	CapabilitySampled1D                      = 43,
 	CapabilityImage1D                        = 44,
 	CapabilityImageQuery                     = 50,
-	CapabilityStorageImageReadWithoutFormat  = 55,
 	CapabilityStorageImageWriteWithoutFormat = 56,
 	CapabilityGroupNonUniform                = 61,
 	CapabilityGroupNonUniformBallot          = 64,
 	CapabilityGroupNonUniformShuffle         = 65,
 	CapabilitySignedZeroInfNanPreserve       = 4466,
+	CapabilityShaderViewportIndexLayerEXT    = 5254,
+	CapabilityFragmentBarycentricKHR         = 5284,
 	CapabilityComputeDerivativeGroupQuadsKHR = 5288,
 	StorageClassUniformConstant              = 0,
 	StorageClassInput                        = 1,
@@ -72,10 +75,15 @@ enum : uint32_t {
 	DecorationBinding       = 33,
 	DecorationDescriptorSet = 34,
 	DecorationOffset        = 35,
+	DecorationPerVertexKHR  = 5285,
 };
 
 enum : uint32_t {
 	BuiltInPosition                  = 0,
+	BuiltInPointSize                 = 1,
+	BuiltInClipDistance              = 3,
+	BuiltInCullDistance              = 4,
+	BuiltInLayer                     = 9,
 	BuiltInFragCoord                 = 15,
 	BuiltInFrontFacing               = 17,
 	BuiltInSampleMask                = 20,
@@ -87,6 +95,8 @@ enum : uint32_t {
 	BuiltInSubgroupLocalInvocationId = 41,
 	BuiltInVertexIndex               = 42,
 	BuiltInInstanceIndex             = 43,
+	BuiltInBaryCoordKHR              = 5286,
+	BuiltInBaryCoordNoPerspKHR       = 5287,
 };
 
 enum : uint32_t {
@@ -132,6 +142,8 @@ enum : uint32_t {
 	OpTypeStruct                   = 30,
 	OpTypePointer                  = 32,
 	OpTypeFunction                 = 33,
+	OpConstantTrue                 = 41,
+	OpConstantFalse                = 42,
 	OpConstant                     = 43,
 	OpConstantComposite            = 44,
 	OpUndef                        = 1,
@@ -180,6 +192,8 @@ enum : uint32_t {
 	OpIAddCarry                    = 149,
 	OpUMulExtended                 = 151,
 	OpSMulExtended                 = 152,
+	OpAny                          = 154,
+	OpAll                          = 155,
 	OpLogicalNotEqual              = 165,
 	OpLogicalOr                    = 166,
 	OpLogicalAnd                   = 167,
@@ -248,34 +262,30 @@ enum : uint32_t {
 };
 
 enum : uint32_t {
-	GlslRoundEven      = 2,
-	GlslTrunc          = 3,
-	GlslFAbs           = 4,
-	GlslFloor          = 8,
-	GlslCeil           = 9,
-	GlslFract          = 10,
-	GlslSin            = 13,
-	GlslCos            = 14,
-	GlslExp2           = 29,
-	GlslLog2           = 30,
-	GlslSqrt           = 31,
-	GlslInverseSqrt    = 32,
-	GlslFMin           = 37,
-	GlslFMax           = 40,
-	GlslFClamp         = 43,
-	GlslLdexp          = 53,
-	GlslFma            = 50,
-	GlslPackSnorm2x16  = 56,
-	GlslPackUnorm2x16  = 57,
-	GlslPackHalf2x16   = 58,
-	GlslUnpackHalf2x16 = 62,
-	GlslFindILsb       = 73,
-	GlslFindUMsb       = 75,
-};
-
-struct RegisterBinding {
-	IR::Register reg;
-	uint32_t     pointer_id = 0;
+	GlslRoundEven       = 2,
+	GlslTrunc           = 3,
+	GlslFAbs            = 4,
+	GlslFloor           = 8,
+	GlslCeil            = 9,
+	GlslFract           = 10,
+	GlslSin             = 13,
+	GlslCos             = 14,
+	GlslExp2            = 29,
+	GlslLog2            = 30,
+	GlslSqrt            = 31,
+	GlslInverseSqrt     = 32,
+	GlslFMin            = 37,
+	GlslFMax            = 40,
+	GlslFClamp          = 43,
+	GlslLdexp           = 53,
+	GlslFma             = 50,
+	GlslPackSnorm2x16   = 56,
+	GlslPackUnorm2x16   = 57,
+	GlslPackHalf2x16    = 58,
+	GlslUnpackUnorm2x16 = 61,
+	GlslUnpackHalf2x16  = 62,
+	GlslFindILsb        = 73,
+	GlslFindUMsb        = 75,
 };
 
 struct InputBinding {
@@ -284,6 +294,7 @@ struct InputBinding {
 	uint32_t           component_count = 1;
 	uint32_t           variable_id     = 0;
 	std::string        debug_name;
+	bool               per_vertex = false;
 };
 
 struct OutputBinding {
@@ -299,147 +310,102 @@ struct DescriptorResourceBinding {
 	uint32_t                     array_index = 0;
 };
 
-struct SampledImageDescriptors {
-	uint32_t image_type         = 0;
-	uint32_t sampled_image_type = 0;
-	uint32_t pointer_type       = 0;
-	uint32_t array_type         = 0;
-	uint32_t array_pointer_type = 0;
-	uint32_t variable           = 0;
+enum class ImageViewKind {
+	Dim1D,
+	Dim1DArray,
+	Dim2D,
+	Dim2DArray,
+	Dim3D,
+	Dim2DMsaa,
+	Dim2DMsaaArray,
+	Count,
 };
 
-struct StorageImageDescriptors {
-	uint32_t image_type         = 0;
-	uint32_t pointer_type       = 0;
-	uint32_t array_type         = 0;
-	uint32_t array_pointer_type = 0;
-	uint32_t variable           = 0;
+constexpr uint32_t SampledImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Count);
+constexpr uint32_t StorageImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Dim2DMsaa);
+
+enum class StorageImageClass : uint32_t {
+	FormatlessFloat,
+	FormatlessUint,
+	AtomicUint,
+	Count,
 };
 
 struct EmitterState {
 	EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_,
 	             ShaderStageInputInfo input_info_)
-	    : program(program_), resources(resources_), input_info(input_info_) {}
+	    : program(program_), resources(resources_), input_info(input_info_),
+	      requirements(*program_.spirv_requirements) {}
 
-	Builder                                          builder;
-	const IR::Program&                               program;
-	const IR::ResourceSnapshot&                      resources;
-	ShaderStageInputInfo                             input_info;
-	ShaderType                                       stage     = ShaderType::Unknown;
-	uint32_t                                         wave_size = 64;
-	bool                                             exact_subgroup_operations    = false;
-	bool                                             per_invocation_masks         = false;
-	uint32_t                                         void_type                    = 0;
-	uint32_t                                         bool_type                    = 0;
-	uint32_t                                         uint_type                    = 0;
-	uint32_t                                         uint_pair_type               = 0;
-	uint32_t                                         int_pair_type                = 0;
-	uint32_t                                         int_type                     = 0;
-	uint32_t                                         float_type                   = 0;
-	uint32_t                                         vec2_uint_type               = 0;
-	uint32_t                                         vec3_uint_type               = 0;
-	uint32_t                                         vec4_uint_type               = 0;
-	uint32_t                                         vec2_int_type                = 0;
-	uint32_t                                         vec3_int_type                = 0;
-	uint32_t                                         vec4_int_type                = 0;
-	uint32_t                                         vec2_float_type              = 0;
-	uint32_t                                         vec3_float_type              = 0;
-	uint32_t                                         vec4_float_type              = 0;
-	uint32_t                                         ptr_func_uint                = 0;
-	uint32_t                                         ptr_func_bool                = 0;
-	uint32_t                                         ptr_func_float               = 0;
-	uint32_t                                         ptr_func_uint_pair           = 0;
-	uint32_t                                         ptr_func_vec4_uint           = 0;
-	uint32_t                                         ptr_func_vec2_float          = 0;
-	uint32_t                                         ptr_input_float              = 0;
-	uint32_t                                         ptr_input_bool               = 0;
-	uint32_t                                         ptr_input_int                = 0;
-	uint32_t                                         ptr_input_uint               = 0;
-	uint32_t                                         ptr_input_vec2_float         = 0;
-	uint32_t                                         ptr_input_vec3_float         = 0;
-	uint32_t                                         ptr_input_vec2_int           = 0;
-	uint32_t                                         ptr_input_vec3_int           = 0;
-	uint32_t                                         ptr_input_vec4_int           = 0;
-	uint32_t                                         ptr_input_vec2_uint          = 0;
-	uint32_t                                         ptr_input_vec3_uint          = 0;
-	uint32_t                                         ptr_input_vec4_uint          = 0;
-	uint32_t                                         ptr_input_vec4_float         = 0;
-	uint32_t                                         sample_mask_array_type       = 0;
-	uint32_t                                         ptr_output_int               = 0;
-	uint32_t                                         ptr_output_sample_mask_array = 0;
-	uint32_t                                         ptr_output_float             = 0;
-	uint32_t                                         ptr_output_vec4_float        = 0;
-	uint32_t                                         per_vertex_type              = 0;
-	uint32_t                                         ptr_output_per_vertex        = 0;
-	uint32_t                                         storage_runtime_array_type   = 0;
-	uint32_t                                         storage_buffer_type          = 0;
-	uint32_t                                         ptr_storage_buffer           = 0;
-	uint32_t                                         ptr_storage_buffer_uint      = 0;
-	uint32_t                                         storage_buffer_array_type    = 0;
-	uint32_t                                         ptr_storage_buffer_array     = 0;
-	uint32_t                                         storage_buffer_variable      = 0;
-	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
-	uint32_t                                         address_memory_array_type = 0;
-	uint32_t                                         ptr_address_memory_array  = 0;
-	uint32_t                                         address_memory_variable   = 0;
-	uint32_t                                         gds_variable              = 0;
-	uint32_t                                         push_constant_array_type  = 0;
-	uint32_t                                         push_constant_block_type  = 0;
-	uint32_t                                         ptr_push_constant_block   = 0;
-	uint32_t                                         ptr_push_constant_uint    = 0;
-	uint32_t                                         push_constant_variable    = 0;
-	uint32_t                                         vsharp_storage_variable   = 0;
-	uint32_t                                         flattened_srt_variable    = 0;
-	uint32_t                                         lds_array_type            = 0;
-	uint32_t                                         ptr_workgroup_array       = 0;
-	uint32_t                                         ptr_workgroup_uint        = 0;
-	uint32_t                                         lds_variable              = 0;
-	std::array<SampledImageDescriptors, 14>          sampled_images;
-	std::array<StorageImageDescriptors, 10>          storage_images;
-	uint32_t                                         sampler_type                          = 0;
-	uint32_t                                         sampler_array_type                    = 0;
-	uint32_t                                         ptr_uniform_sampler                   = 0;
-	uint32_t                                         ptr_uniform_sampler_array             = 0;
-	uint32_t                                         sampler_variable                      = 0;
-	uint32_t                                         ptr_image_uint                        = 0;
-	uint32_t                                         func_type                             = 0;
-	uint32_t                                         main_func                             = 0;
-	uint32_t                                         entry_label                           = 0;
-	uint32_t                                         pixel_valid_mask_variable             = 0;
-	bool                                             dispatcher_fallback                   = false;
-	uint32_t                                         dispatch_pc_variable                  = 0;
-	uint32_t                                         dispatch_header_label                 = 0;
-	uint32_t                                         dispatch_select_label                 = 0;
-	uint32_t                                         dispatch_default_label                = 0;
-	uint32_t                                         dispatch_after_switch_label           = 0;
-	uint32_t                                         dispatch_continue_label               = 0;
-	uint32_t                                         dispatch_merge_label                  = 0;
-	uint32_t                                         glsl_std450                           = 0;
-	uint32_t                                         subgroup_local_invocation_id_variable = 0;
-	uint32_t                                         per_vertex_variable                   = 0;
-	uint32_t                                         depth_variable                        = 0;
-	uint32_t                                         sample_mask_variable                  = 0;
-	bool                                             needs_subgroup_ballot                 = false;
-	bool                                             needs_subgroup_shuffle                = false;
-	bool                                             needs_subgroup_local_invocation_id    = false;
-	bool                                             needs_compute_derivatives             = false;
-	bool                                             needs_image_gather_extended           = false;
-	bool                                             needs_function_lds                    = false;
-	bool                                             needs_pixel_valid_mask                = false;
-	bool                                             unsupported_ir_instruction            = false;
-	IR::Opcode                                       unsupported_ir_opcode = IR::Opcode::Count;
-	uint32_t                                         unsupported_ir_pc     = 0;
-	std::vector<RegisterBinding>                     registers;
-	std::vector<InputBinding>                        inputs;
-	std::vector<OutputBinding>                       outputs;
-	std::vector<uint32_t>                            interface_variables;
-	std::vector<bool>                                reachable_blocks;
-	std::map<uint32_t, uint32_t>                     block_labels;
-	std::map<uint32_t, uint32_t>                     constants;
-	std::map<uint32_t, uint32_t>                     signed_constants;
-	std::map<uint32_t, uint32_t>                     float_constants;
+	Builder                      builder;
+	const IR::Program&           program;
+	const IR::ResourceSnapshot&  resources;
+	ShaderStageInputInfo         input_info;
+	const IR::SpirvRequirements& requirements;
+	ShaderType                   stage                   = ShaderType::Unknown;
+	uint32_t                     wave_size               = 64;
+	uint32_t                     storage_buffer_variable = 0;
+	std::array<uint32_t, IR::ShaderInfo::MaxBuffers + IR::ShaderInfo::MaxAddresses>
+	                                                     memory_byte_offsets {};
+	uint32_t                                             address_memory_variable = 0;
+	uint32_t                                             gds_variable            = 0;
+	uint32_t                                             gds_length              = 0;
+	uint32_t                                             push_constant_variable  = 0;
+	uint32_t                                             vsharp_storage_variable = 0;
+	uint32_t                                             flattened_srt_variable  = 0;
+	uint32_t                                             lds_variable            = 0;
+	uint32_t                                             scratch_variable        = 0;
+	std::array<uint32_t, 2u * SampledImageViewKindCount> sampled_image_variables {};
+	std::array<uint32_t,
+	           static_cast<uint32_t>(StorageImageClass::Count) * StorageImageViewKindCount>
+	                           storage_image_variables {};
+	uint32_t                   sampler_variable                      = 0;
+	uint32_t                   main_func                             = 0;
+	uint32_t                   entry_label                           = 0;
+	uint32_t                   current_label                         = 0;
+	uint32_t                   pixel_valid_mask_variable             = 0;
+	uint32_t                   subgroup_local_invocation_id_variable = 0;
+	uint32_t                   per_vertex_variable                   = 0;
+	uint32_t                   point_size_variable                   = 0;
+	uint32_t                   clip_distance_variable                = 0;
+	uint32_t                   cull_distance_variable                = 0;
+	uint32_t                   layer_variable                        = 0;
+	uint32_t                   clip_distance_count                   = 0;
+	uint32_t                   cull_distance_count                   = 0;
+	uint32_t                   depth_variable                        = 0;
+	uint32_t                   sample_mask_variable                  = 0;
+	std::vector<InputBinding>  inputs;
+	std::vector<OutputBinding> outputs;
+	std::vector<uint32_t>      interface_variables;
 };
 
+uint32_t TypeVoid(EmitterState& state);
+uint32_t TypeBool(EmitterState& state);
+uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
+uint32_t TypeU32(EmitterState& state);
+uint32_t TypeU64(EmitterState& state);
+uint32_t TypeU32Pair(EmitterState& state);
+uint32_t TypeI32(EmitterState& state);
+uint32_t TypeI32Pair(EmitterState& state);
+uint32_t TypeF32(EmitterState& state);
+uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
+
+uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
+uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
+uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
+uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee);
+uint32_t TypeFunction(EmitterState& state);
+uint32_t TypeStorageBufferPointer(EmitterState& state);
+uint32_t TypeStorageBufferElementPointer(EmitterState& state);
+uint32_t TypePushConstantElementPointer(EmitterState& state);
+uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
+uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
+
+inline void EmitLabel(EmitterState& state, uint32_t label) {
+	state.current_label = label;
+	state.builder.AddFunction({OpLabel, label});
+}
+
 struct ValueEmitContext {
 	ValueEmitContext(EmitterState& state_, const IR::ValueProgram& program_)
 	    : state(state_), program(program_) {}
@@ -456,19 +422,18 @@ struct ValueEmitContext {
 	const IR::MemoryInfo& Memory(const IR::Inst& inst) const;
 	const IR::ExportInfo& Export(const IR::Inst& inst) const;
 	uint32_t              Label(const IR::Block* block) const;
-	size_t                BlockIndex(const IR::Block* block) const;
 	void                  Fail(const IR::Inst& inst, const char* reason);
 
-	EmitterState&                                  state;
-	const IR::ValueProgram&                        program;
-	std::unordered_map<const IR::Inst*, uint32_t>  definitions;
-	std::unordered_map<const IR::Block*, uint32_t> labels;
-	std::unordered_map<const IR::Inst*, uint32_t>  phi_variables;
-	std::unordered_map<const IR::Inst*, uint32_t>  dispatcher_variables;
-	const IR::Block*                               current_block        = nullptr;
-	uint32_t                                       scratch_u32_variable = 0;
-	bool                                           failed               = false;
-	std::string                                    error;
+	EmitterState&                                                      state;
+	const IR::ValueProgram&                                            program;
+	std::unordered_map<const IR::Inst*, uint32_t>                      definitions;
+	std::unordered_map<const IR::Block*, uint32_t>                     labels;
+	const std::unordered_map<const IR::Inst*, uint32_t>*               dispatcher_spills = nullptr;
+	std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
+	const IR::Block*                                                   current_block = nullptr;
+	uint32_t                                                           scratch_u32_variable = 0;
+	bool                                                               failed               = false;
+	std::string                                                        error;
 };
 
 enum class VertexInputScalarKind { Float, Sint, Uint };
@@ -490,26 +455,23 @@ struct ImageSampleLayout {
 	uint32_t grad_y = NoImageComponent;
 };
 
-enum class ImageViewKind {
-	Dim1D,
-	Dim1DArray,
-	Dim2D,
-	Dim2DArray,
-	Dim3D,
-	Dim2DMsaa,
-	Dim2DMsaaArray,
-	Count,
-};
-
-constexpr uint32_t SampledImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Count);
-constexpr uint32_t StorageImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Dim2DMsaa);
-
 constexpr uint32_t SampledImageIndex(bool integer, ImageViewKind view) {
 	return static_cast<uint32_t>(view) + (integer ? SampledImageViewKindCount : 0u);
 }
 
-constexpr uint32_t StorageImageIndex(bool integer, ImageViewKind view) {
-	return static_cast<uint32_t>(view) + (integer ? StorageImageViewKindCount : 0u);
+constexpr StorageImageClass StorageImageClassFor(bool uint_image, bool atomic) {
+	if (atomic) {
+		return StorageImageClass::AtomicUint;
+	}
+	if (uint_image) {
+		return StorageImageClass::FormatlessUint;
+	}
+	return StorageImageClass::FormatlessFloat;
+}
+
+constexpr uint32_t StorageImageIndex(StorageImageClass image_class, ImageViewKind view) {
+	return static_cast<uint32_t>(view) +
+	       static_cast<uint32_t>(image_class) * StorageImageViewKindCount;
 }
 
 constexpr IR::DescriptorBindingKind SampledBindingKind(bool integer, ImageViewKind view) {
@@ -539,24 +501,44 @@ constexpr IR::DescriptorBindingKind SampledBindingKind(bool integer, ImageViewKi
 	return IR::DescriptorBindingKind::Count;
 }
 
-constexpr IR::DescriptorBindingKind StorageBindingKind(bool integer, ImageViewKind view) {
-	if (integer) {
-		switch (view) {
-			case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageUint1D;
-			case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::StorageUint1DArray;
-			case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageUint2D;
-			case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::StorageUint2DArray;
-			case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
-			default: break;
-		}
-	}
-	switch (view) {
-		case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::Storage1D;
-		case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::Storage1DArray;
-		case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::Storage2D;
-		case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
-		case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
-		default: break;
+constexpr IR::DescriptorBindingKind StorageBindingKind(StorageImageClass image_class,
+                                                       ImageViewKind     view) {
+	switch (image_class) {
+		case StorageImageClass::FormatlessFloat:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::Storage1D;
+				case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::Storage1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::Storage2D;
+				case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::FormatlessUint:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageUint1D;
+				case ImageViewKind::Dim1DArray:
+					return IR::DescriptorBindingKind::StorageUint1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageUint2D;
+				case ImageViewKind::Dim2DArray:
+					return IR::DescriptorBindingKind::StorageUint2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::AtomicUint:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageAtomic1D;
+				case ImageViewKind::Dim1DArray:
+					return IR::DescriptorBindingKind::StorageAtomic1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageAtomic2D;
+				case ImageViewKind::Dim2DArray:
+					return IR::DescriptorBindingKind::StorageAtomic2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageAtomic3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::Count: break;
 	}
 	return IR::DescriptorBindingKind::Count;
 }
@@ -586,17 +568,10 @@ constexpr uint32_t ImageSpirvMultisampled(ImageViewKind view) {
 	return view == ImageViewKind::Dim2DMsaa || view == ImageViewKind::Dim2DMsaaArray ? 1u : 0u;
 }
 
-struct AddCarryResult {
-	uint32_t sum   = 0;
-	uint32_t carry = 0;
-};
-
 struct F32Class {
-	uint32_t bits       = 0;
-	uint32_t nan        = 0;
-	uint32_t snan       = 0;
-	uint32_t zero       = 0;
-	uint32_t quiet_bits = 0;
+	uint32_t bits = 0;
+	uint32_t nan  = 0;
+	uint32_t zero = 0;
 };
 
 uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr);
@@ -605,48 +580,30 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr);
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr);
 
+bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr);
+
 VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location);
 
 uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input);
 
-uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind);
+uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind);
 
-uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind);
+uint32_t VertexParameterScalarPointerType(EmitterState& state, VertexInputScalarKind kind);
 
-uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind,
+uint32_t VertexParameterVectorOrScalarType(EmitterState& state, VertexInputScalarKind kind,
                                            uint32_t components);
 
-uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind,
+uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
                                          uint32_t components);
 
 void SetError(std::string* error, const char* message);
 
-void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg);
-
-bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg);
-
 bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
 
 void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
 
 uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
 
-bool ProgramNeedsComputeDerivatives(const IR::Program& program);
-
-bool ProgramNeedsImageGatherExtended(const IR::Program& program);
-
-bool ProgramNeedsFunctionLds(const IR::Program& program);
-
-bool ProgramNeedsPixelValidMask(const IR::Program& program);
-
-bool ProgramNeedsSubgroupBallot(const IR::Program& program);
-
-bool ProgramNeedsSubgroupShuffle(const IR::Program& program);
-
-bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program);
-
-uint32_t PointerForRegister(const EmitterState& state, IR::Register reg);
-
 uint32_t ConstantU32(EmitterState& state, uint32_t value);
 
 uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
@@ -670,11 +627,16 @@ uint32_t ImageViewCoordinateComponents(ImageViewKind view);
 
 uint32_t ImageViewSpatialComponents(ImageViewKind view);
 
-uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer);
+uint32_t ImageViewImageType(EmitterState& state, ImageViewKind view, bool integer);
 
-uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer);
+uint32_t ImageViewSampledImageType(EmitterState& state, ImageViewKind view, bool integer);
 
-uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view);
+uint32_t ImageViewSizeType(EmitterState& state, ImageViewKind view);
+
+uint32_t StorageImageType(EmitterState& state, StorageImageClass image_class, ImageViewKind view);
+
+uint32_t StorageImagePointerType(EmitterState& state, StorageImageClass image_class,
+                                 ImageViewKind view);
 
 uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
                                     ImageViewKind view);
@@ -686,18 +648,12 @@ uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32
 uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
                           ImageViewKind view, uint32_t image_resource);
 
-ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   bool uint_image, uint32_t use_pc);
+ImageViewKind StorageImageViewKind(const IR::MemoryInfo& mem);
 
-uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
-                                       uint32_t      use_pc = UINT32_MAX,
-                                       ImageViewKind view   = ImageViewKind::Dim2D);
+uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, ImageViewKind view);
 
-uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
-                                    uint32_t use_pc, ImageViewKind view = ImageViewKind::Dim2D);
-
-void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_image,
-                           ImageViewKind view, uint32_t mip_lod, uint32_t coord, uint32_t texel);
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, ImageViewKind view,
+                           uint32_t mip_lod, uint32_t coord, uint32_t texel);
 
 uint32_t ExecutionModelForStage(ShaderType stage);
 
@@ -711,6 +667,14 @@ uint32_t FloatBits(float value);
 
 uint32_t ConstantF32Value(EmitterState& state, float value);
 
+uint32_t ConstantBool(EmitterState& state, bool value);
+
+uint32_t ConstantU64(EmitterState& state, uint64_t value);
+
+uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
+
+uint32_t GlslStd450(EmitterState& state);
+
 void AllocateInputVariables(EmitterState& state);
 
 void AllocateOutputVariables(EmitterState& state);
@@ -721,16 +685,12 @@ void AddInputAnnotationsAndNames(EmitterState& state);
 
 void AddOutputAnnotationsAndNames(EmitterState& state);
 
-void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, uint32_t set,
-                        uint32_t binding);
+void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name,
+                        IR::DescriptorBindingKind kind);
 
 void AddDescriptorAnnotationsAndNames(EmitterState& state);
 
-void EmitHeaderAndTypes(EmitterState& state);
-
-void AllocateRegisterVariables(EmitterState& state);
-
-void AllocateDescriptorVariables(EmitterState& state);
+void DefineModule(EmitterState& state);
 
 uint32_t EmitTrueBool(EmitterState& state);
 
@@ -768,40 +728,32 @@ uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint3
 
 uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
 
-bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index);
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
 
-uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   uint32_t use_pc);
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
 
-Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t use_pc);
-
-void EmitStorageBufferOffsets(EmitterState& state);
-
-DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState&         state,
-                                                        const IR::MemoryInfo& mem, uint32_t use_pc);
-
-uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                        uint32_t use_pc);
-
-uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc);
-
-uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                         uint32_t index, uint32_t use_pc);
-
-uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index);
+void EmitMemoryOffsets(EmitterState& state);
 
 uint32_t LdsDwordCount(const EmitterState& state);
 
-uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index);
+struct MemoryResourceAccess {
+	IR::ResourceKind kind             = IR::ResourceKind::None;
+	uint32_t         object_pointer   = 0;
+	uint32_t         length           = 0;
+	uint32_t         index_offset     = 0;
+	bool             add_index_offset = false;
+};
 
-uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index);
+MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
 
-uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index);
+uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
+                                uint32_t raw_index);
 
-uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
-                                  uint32_t use_pc);
+uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
+                                   uint32_t index);
+
+uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                  uint32_t index);
 
 uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value);
 
@@ -831,8 +783,6 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
                             uint32_t false_value);
 
-uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
-
 void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
                                    uint32_t& out_low, uint32_t& out_high);
 
@@ -841,9 +791,6 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t
 
 uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
 
-AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
-                                  uint32_t carry_in);
-
 uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
 
 uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
@@ -892,8 +839,6 @@ uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
 
 uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
 
-uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg);
-
 bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
 
 bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -907,42 +852,33 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
 // These templates accept local lambdas from several emitter translation units.
 template <typename Fn>
 void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
-	if (condition == 0) {
-		fn();
-		return;
-	}
-
 	const auto then_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label});
-	state.builder.AddFunction({OpLabel, then_label});
+	EmitLabel(state, then_label);
 	fn();
 	state.builder.AddFunction({OpBranch, merge_label});
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 }
 
 template <typename Fn>
 uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
                                        uint32_t default_value, Fn&& fn) {
-	if (condition == 0) {
-		return fn();
-	}
-
 	const auto then_label  = state.builder.AllocateId();
 	const auto then_exit   = state.builder.AllocateId();
 	const auto else_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
-	state.builder.AddFunction({OpLabel, then_label});
+	EmitLabel(state, then_label);
 	const auto then_value = fn();
 	state.builder.AddFunction({OpBranch, then_exit});
-	state.builder.AddFunction({OpLabel, then_exit});
+	EmitLabel(state, then_exit);
 	state.builder.AddFunction({OpBranch, merge_label});
-	state.builder.AddFunction({OpLabel, else_label});
+	EmitLabel(state, else_label);
 	state.builder.AddFunction({OpBranch, merge_label});
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 	const auto value = state.builder.AllocateId();
 	state.builder.AddFunction(
 	    {OpPhi, type, value, then_value, then_exit, default_value, else_label});
@@ -951,7 +887,7 @@ uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition,
 
 template <typename Fn>
 uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
-	return EmitValueOrDefaultIfCondition(state, condition, state.uint_type, ConstantU32(state, 0),
+	return EmitValueOrDefaultIfCondition(state, condition, TypeU32(state), ConstantU32(state, 0),
 	                                     std::forward<Fn>(fn));
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -2,21 +2,41 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
+uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
+	if (state.push_constant_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer,
+		                           state.push_constant_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		return value;
+	}
+	if (state.vsharp_storage_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+		                           state.vsharp_storage_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		return value;
+	}
+	return ConstantU32(state, 0);
+}
+
 uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({OpIAdd, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
 uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs});
+	state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
 	return ret;
 }
 
-uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
-                                   uint32_t use_pc) {
-	(void)use_pc;
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
 		                             "buffer specialization is missing");
@@ -24,9 +44,7 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn
 	return state.program.info.buffers[mem.resource].packed_stride;
 }
 
-Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t use_pc) {
-	(void)use_pc;
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem) {
 	if (mem.resource >= state.program.info.buffers.size()) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
 		                             "buffer specialization is missing");
@@ -34,160 +52,160 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
 	return state.program.info.buffers[mem.resource].descriptor_format;
 }
 
-void EmitStorageBufferOffsets(EmitterState& state) {
-	for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
+void EmitMemoryOffsets(EmitterState& state) {
+	for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
 		const auto word =
-		    EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
-		const auto shift                = ConstantU32(state, (i % 4u) * 8u + 2u);
-		state.storage_buffer_offsets[i] = EmitBinaryU32(
+		    EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
+		const auto shift             = ConstantU32(state, (i % 4u) * 8u);
+		state.memory_byte_offsets[i] = EmitBinaryU32(
 		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
-		    ConstantU32(state, 0x3fu));
+		    ConstantU32(state, 0xffu));
 	}
 }
 
-bool IsAddressMemoryKind(IR::ResourceKind kind) {
-	return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat ||
-	       kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch;
-}
-
-DescriptorResourceBinding
-StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) {
-	(void)use_pc;
-	const auto binding =
-	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
-	if (state.storage_buffer_variable == 0) {
-		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
-		                             "storage buffer descriptor array was not emitted");
-	}
-	return binding;
-}
-
-uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
-                                uint32_t use_pc) {
-	if (IsAddressMemoryKind(mem.kind)) {
-		return index;
-	}
-	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
-	return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]);
-}
-
-uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                        uint32_t use_pc) {
-	if (IsAddressMemoryKind(mem.kind)) {
-		if (state.address_memory_variable == 0) {
-			ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
-			                             mem.resource, "address memory binding was not emitted");
-		}
-		const auto binding =
-		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
-		const auto pointer = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-		                           state.address_memory_variable,
-		                           ConstantU32(state, binding.array_index)});
-		return pointer;
-	}
-	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-	                           state.storage_buffer_variable,
-	                           ConstantU32(state, binding.array_index)});
-	return pointer;
-}
-
-uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc) {
-	index                = EmitStorageBufferIndex(state, mem, index, use_pc);
-	const auto object    = EmitStorageBufferObjectPointer(state, mem, use_pc);
-	const auto length    = state.builder.AllocateId();
-	const auto in_bounds = state.builder.AllocateId();
-	state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
-	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
-	return in_bounds;
-}
-
-uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                         uint32_t index, uint32_t use_pc) {
-	index = EmitStorageBufferIndex(state, mem, index, use_pc);
-	if (IsAddressMemoryKind(mem.kind)) {
-		if (state.address_memory_variable == 0) {
-			ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
-			                             mem.resource, "address memory binding was not emitted");
-		}
-		const auto binding =
-		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
-		const auto pointer = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable,
-		     ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
-		return pointer;
-	}
-	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable,
-	     ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
-	return pointer;
-}
-
-uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) {
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index});
-	return pointer;
-}
-
 uint32_t LdsDwordCount(const EmitterState& state) {
 	return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
 }
 
-uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) {
-	const auto in_bounds = state.builder.AllocateId();
+static void EnsureLdsStorage(EmitterState& state) {
+	if (state.lds_variable != 0) {
+		return;
+	}
+	if (state.stage != ShaderType::Compute) {
+		EXIT("function LDS was not prepared before SPIR-V function emission\n");
+	}
+	state.lds_variable = state.builder.DefineGlobalVariable(
+	    TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)),
+	    StorageClassWorkgroup);
+	state.builder.AddName(state.lds_variable, "lds_dwords");
+}
+
+MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) {
+	MemoryResourceAccess access {.kind = mem.kind};
+	switch (mem.kind) {
+		case IR::ResourceKind::Lds:
+			EnsureLdsStorage(state);
+			access.object_pointer = state.lds_variable;
+			access.length         = ConstantU32(state, LdsDwordCount(state));
+			return access;
+		case IR::ResourceKind::Gds:
+			if (state.gds_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Gds, mem.resource,
+				                             "GDS binding was not emitted");
+			}
+			access.object_pointer = state.gds_variable;
+			if (state.gds_length == 0) {
+				EXIT("GDS length was not prepared at function entry\n");
+			}
+			access.length = state.gds_length;
+			return access;
+		case IR::ResourceKind::Scratch:
+			if (state.scratch_variable == 0) {
+				EXIT("scratch storage was not prepared before SPIR-V function emission\n");
+			}
+			access.object_pointer = state.scratch_variable;
+			access.length         = ConstantU32(state, state.program.scratch_dwords);
+			return access;
+		case IR::ResourceKind::ScalarAddress:
+		case IR::ResourceKind::Flat:
+		case IR::ResourceKind::Global: {
+			if (state.address_memory_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
+				                             mem.resource,
+				                             "address memory binding was not emitted");
+			}
+			const auto binding = ResourceForDescriptor(
+			    state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
+			access.object_pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
+			                           access.object_pointer, state.address_memory_variable,
+			                           ConstantU32(state, binding.array_index)});
+			break;
+		}
+		case IR::ResourceKind::ScalarBuffer:
+		case IR::ResourceKind::Buffer: {
+			if (state.storage_buffer_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers,
+				                             mem.resource,
+				                             "storage buffer descriptor array was not emitted");
+			}
+			const auto binding =
+			    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
+			access.object_pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
+			                           access.object_pointer, state.storage_buffer_variable,
+			                           ConstantU32(state, binding.array_index)});
+			access.index_offset     = EmitBinaryU32(state, OpShiftRightLogical,
+			                                        state.memory_byte_offsets[binding.array_index],
+			                                        ConstantU32(state, 2u));
+			access.add_index_offset = true;
+			break;
+		}
+		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
+	}
+	access.length = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))});
+	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	return access;
+}
+
+uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
+                                uint32_t raw_index) {
+	return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
+}
+
+uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
+                                   uint32_t index) {
+	const auto in_bounds = state.builder.AllocateId();
+	state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length});
 	return in_bounds;
 }
 
-uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
-                                  uint32_t use_pc) {
-	if (mem.kind == IR::ResourceKind::Lds) {
-		return EmitLdsElementPointer(state, index);
+uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                  uint32_t index) {
+	const auto pointer = state.builder.AllocateId();
+	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
+		const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
+		                           : state.stage == ShaderType::Compute     ? StorageClassWorkgroup
+		                                                                    : StorageClassFunction;
+		state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
+		                           pointer, access.object_pointer, index});
+	} else {
+		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+		                           access.object_pointer, ConstantU32(state, 0), index});
 	}
-	if (mem.kind == IR::ResourceKind::Gds) {
-		return EmitGdsElementPointer(state, index);
-	}
-	return EmitStorageBufferElementPointer(state, mem, index, use_pc);
+	return pointer;
 }
 
 uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.int_type, ret, value});
+	state.builder.AddFunction({OpBitcast, TypeI32(state), ret, value});
 	return ret;
 }
 
 uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                        uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
 uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
                               uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpSelect, state.float_type, ret, condition, true_value, false_value});
+	state.builder.AddFunction({OpSelect, TypeF32(state), ret, condition, true_value, false_value});
 	return ret;
 }
 
@@ -200,8 +218,8 @@ uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) {
 	const auto unpacked = state.builder.AllocateId();
 	const auto value    = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw});
-	state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0});
+	    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), GlslUnpackHalf2x16, raw});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), value, unpacked, 0});
 	return EmitTBufferBitcastF32ToU32(state, value);
 }
 
@@ -229,9 +247,9 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
 
 	const auto mantissa_f32 = state.builder.AllocateId();
 	const auto subnormal    = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa});
+	state.builder.AddFunction({OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
 	state.builder.AddFunction(
-	    {OpFMul, state.float_type, subnormal, mantissa_f32,
+	    {OpFMul, TypeF32(state), subnormal, mantissa_f32,
 	     ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
 
 	const auto zero_exp    = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0);
@@ -249,22 +267,22 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 		case Format::ComponentType::Sint: return raw;
 		case Format::ComponentType::Uscaled: {
 			const auto value = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
+			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
 			return EmitTBufferBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Sscaled: {
 			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
 			const auto value      = state.builder.AllocateId();
-			state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
+			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
 			return EmitTBufferBitcastF32ToU32(state, value);
 		}
 		case Format::ComponentType::Unorm: {
 			const auto value      = state.builder.AllocateId();
 			const auto normalized = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << bits) - 1u);
-			state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
+			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
 			state.builder.AddFunction(
-			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
+			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
 			return EmitTBufferBitcastF32ToU32(state, normalized);
 		}
 		case Format::ComponentType::Snorm: {
@@ -273,10 +291,10 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			const auto normalized = state.builder.AllocateId();
 			const auto clamped    = state.builder.AllocateId();
 			const auto max_value  = static_cast<float>((1u << (bits - 1u)) - 1u);
-			state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
+			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
 			state.builder.AddFunction(
-			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
-			state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
+			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
+			state.builder.AddFunction({OpExtInst, TypeF32(state), clamped, GlslStd450(state),
 			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
 			return EmitTBufferBitcastF32ToU32(state, clamped);
 		}
@@ -298,21 +316,6 @@ void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
 	    {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
 }
 
-uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
-	const auto length    = state.builder.AllocateId();
-	const auto in_bounds = state.builder.AllocateId();
-	state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0});
-	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
-	return in_bounds;
-}
-
-uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
-	const auto pointer = state.builder.AllocateId();
-	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                           state.gds_variable, ConstantU32(state, 0), index});
-	return pointer;
-}
-
 uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
 	if ((control & 0xc000u) == 0xc000u) {
 		const uint32_t mask         = control & 0x1fu;
@@ -336,16 +339,16 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 		const auto base   = state.builder.AllocateId();
 		const auto target = state.builder.AllocateId();
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)});
+		    {OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
 		state.builder.AddFunction(
-		    {OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)});
+		    {OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
 		state.builder.AddFunction(
-		    {OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift});
+		    {OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)});
+		    {OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
 		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm});
+		    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
+		state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, perm});
 		return target;
 	}
 
@@ -355,16 +358,16 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
+	    {OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
+	    {OpBitwiseOr, TypeU32(state), ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
+	    {OpBitwiseXor, TypeU32(state), xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
 	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
+	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, xored});
 	return target;
 }
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -4,6 +4,7 @@
 #include <bit>
 #include <fmt/format.h>
 #include <functional>
+#include <optional>
 #include <unordered_set>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -13,12 +14,12 @@ void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
 	const auto kill_label  = state.builder.AllocateId();
 	const auto merge_label = state.builder.AllocateId();
 	const auto inactive    = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
+	state.builder.AddFunction({OpLogicalNot, TypeBool(state), inactive, active});
 	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
 	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
-	state.builder.AddFunction({OpLabel, kill_label});
+	EmitLabel(state, kill_label);
 	state.builder.AddFunction({OpKill});
-	state.builder.AddFunction({OpLabel, merge_label});
+	EmitLabel(state, merge_label);
 }
 
 void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
@@ -29,54 +30,49 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
 	const auto mask_value = state.builder.AllocateId();
 	const auto active     = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
+	    {OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
 	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
+	    {OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
 	EmitKillIfBoolFalse(state, active);
 }
 
-uint32_t BoolConstant(EmitterState& state, bool value) {
-	const auto id = state.builder.AllocateId();
-	state.builder.AddType({value ? 41u : 42u, state.bool_type, id});
-	return id;
+uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
+	const auto value_type = ctx.TypeId(type);
+	return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type);
 }
 
-uint32_t ConstantPair(EmitterState& state, uint64_t value) {
-	const auto id = state.builder.AllocateId();
-	state.builder.AddType({OpConstantComposite, state.uint_pair_type, id,
-	                       ConstantU32(state, static_cast<uint32_t>(value)),
-	                       ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
-	return id;
-}
+struct DeferredPhiPatch {
+	DeferredPhi     phi;
+	const IR::Inst* instruction = nullptr;
+};
 
-uint32_t PhiPointerType(const EmitterState& state, IR::Type type) {
-	switch (type) {
-		case IR::Type::U1: return state.ptr_func_bool;
-		case IR::Type::U8:
-		case IR::Type::U16:
-		case IR::Type::U32:
-		case IR::Type::F16: return state.ptr_func_uint;
-		case IR::Type::F32: return state.ptr_func_float;
-		case IR::Type::U64:
-		case IR::Type::U32x2: return state.ptr_func_uint_pair;
-		case IR::Type::U32x4: return state.ptr_func_vec4_uint;
-		case IR::Type::F32x2: return state.ptr_func_vec2_float;
-		default: return 0;
-	}
-}
+struct StructuredFunctionState {
+	std::unordered_map<const IR::Block*, uint32_t> block_exit_labels;
+	std::vector<DeferredPhiPatch>                  deferred_phis;
+};
 
-void StorePhiEdge(ValueEmitContext& ctx, const IR::Block* from, const IR::Block* to) {
+struct DispatcherFunctionState {
+	std::unordered_map<const IR::Inst*, uint32_t> spills;
+	uint32_t                                      header_label       = 0;
+	uint32_t                                      select_label       = 0;
+	uint32_t                                      after_switch_label = 0;
+	uint32_t                                      continue_label     = 0;
+	uint32_t                                      merge_label        = 0;
+};
+
+void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                            const IR::Block* from, const IR::Block* to) {
 	if (to == nullptr) {
 		return;
 	}
 	for (const auto& phi: *to) {
 		if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
-			continue;
+			break;
 		}
 		for (size_t index = 0; index < phi.NumArgs(); index++) {
 			if (phi.PhiBlock(index) == from) {
 				ctx.state.builder.AddFunction(
-				    {OpStore, ctx.phi_variables.at(&phi), ctx.Def(phi.Arg(index))});
+				    {OpStore, dispatcher.spills.at(&phi), ctx.Def(phi.Arg(index))});
 				break;
 			}
 		}
@@ -124,7 +120,6 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				EmitReturn(ctx);
 				return;
 			}
-			StorePhiEdge(ctx, block, target);
 			emit_merge();
 			ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
 			return;
@@ -136,8 +131,6 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				EmitReturn(ctx);
 				return;
 			}
-			StorePhiEdge(ctx, block, true_block);
-			StorePhiEdge(ctx, block, false_block);
 			const auto condition = ctx.Def(info.condition);
 			emit_merge();
 			ctx.state.builder.AddFunction(
@@ -148,39 +141,34 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
 	}
 }
 
-void EmitDispatcherTarget(ValueEmitContext& ctx, const IR::Block* from, uint32_t target) {
+void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                          const IR::Block* from, uint32_t target) {
 	const auto* block = TargetBlock(ctx.program, target);
 	if (block != nullptr) {
-		StorePhiEdge(ctx, from, block);
+		StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
 	}
 }
 
-void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
-                              const IR::ValueBlockInfo& info, uint32_t exit_label) {
+uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                              const IR::Block* block, const IR::ValueBlockInfo& info) {
 	const auto& term = info.terminator;
 	switch (term.kind) {
 		case CFG::TerminatorKind::Branch:
-			EmitDispatcherTarget(ctx, block, term.true_block);
-			ctx.state.builder.AddFunction(
-			    {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, term.true_block)});
-			break;
+			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
+			return ConstantU32(ctx.state, term.true_block);
 		case CFG::TerminatorKind::ConditionalBranch: {
-			EmitDispatcherTarget(ctx, block, term.true_block);
-			EmitDispatcherTarget(ctx, block, term.false_block);
+			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
+			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
 			const auto selected = ctx.state.builder.AllocateId();
-			ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, selected,
+			ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
 			                               ctx.Def(info.condition),
 			                               ConstantU32(ctx.state, term.true_block),
 			                               ConstantU32(ctx.state, term.false_block)});
-			ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
-			break;
+			return selected;
 		}
 		case CFG::TerminatorKind::IndirectBranch: {
 			for (const auto target: term.indirect_targets) {
-				EmitDispatcherTarget(ctx, block, target);
-			}
-			for (const auto target: term.indirect_selector_targets) {
-				EmitDispatcherTarget(ctx, block, target);
+				EmitDispatcherTarget(ctx, dispatcher, block, target);
 			}
 			uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
 			if (!info.indirect_target.IsEmpty()) {
@@ -194,108 +182,193 @@ void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
 				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
 					const auto match = ctx.state.builder.AllocateId();
 					const auto next  = ctx.state.builder.AllocateId();
-					ctx.state.builder.AddFunction({OpIEqual, ctx.state.bool_type, match, selector,
+					ctx.state.builder.AddFunction({OpIEqual, TypeBool(ctx.state), match, selector,
 					                               ConstantU32(ctx.state, values[index])});
-					ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, next, match,
+					ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), next, match,
 					                               ConstantU32(ctx.state, targets[index]),
 					                               selected});
 					selected = next;
 				}
 			}
-			ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
-			break;
+			return selected;
 		}
-		default:
-			ctx.state.builder.AddFunction(
-			    {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
-			break;
+		default: return ConstantU32(ctx.state, UINT32_MAX);
 	}
-	ctx.state.builder.AddFunction({OpBranch, exit_label});
 }
 
-bool EmitInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
-	const auto finish = [&]() {
-		if (const auto found = ctx.dispatcher_variables.find(&inst);
-		    found != ctx.dispatcher_variables.end()) {
-			ctx.state.builder.AddFunction(
-			    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
-		}
-		return true;
-	};
-	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
-		ctx.state.builder.AddFunction(
-		    {OpLoad, ctx.TypeId(inst.GetType()), ctx.Result(inst), ctx.phi_variables.at(&inst)});
-		return finish();
-	}
+bool EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
 	if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
 	    EmitValueImage(ctx, inst)) {
-		return finish();
+		return true;
 	}
 	ctx.Fail(inst, "has no direct SPIR-V emitter");
 	return false;
 }
 
-void EmitBlock(ValueEmitContext& ctx, const IR::Block* block) {
+bool EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
+                               const IR::Inst& inst) {
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		const auto type = ctx.TypeId(inst.GetType());
+		if (type == 0 || inst.NumArgs() == 0) {
+			ctx.Fail(inst, "has no native SPIR-V representation");
+			return false;
+		}
+		for (size_t index = 0; index < inst.NumArgs(); index++) {
+			const auto* predecessor = inst.PhiBlock(index);
+			if (predecessor == nullptr || !ctx.labels.contains(predecessor)) {
+				ctx.Fail(inst, "has a predecessor outside the structured function");
+				return false;
+			}
+		}
+		structured.deferred_phis.push_back(
+		    {ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst});
+		return true;
+	}
+	return EmitDirectInstruction(ctx, inst);
+}
+
+bool EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                               const IR::Inst& inst) {
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		const auto type = ctx.TypeId(inst.GetType());
+		if (type == 0) {
+			ctx.Fail(inst, "cannot be loaded by the dispatcher");
+			return false;
+		}
+		ctx.state.builder.AddFunction(
+		    {OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)});
+		return true;
+	}
+	if (!EmitDirectInstruction(ctx, inst)) {
+		return false;
+	}
+	if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) {
+		ctx.state.builder.AddFunction(
+		    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
+	}
+	return true;
+}
+
+template <typename EmitInstruction>
+void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
 	ctx.current_block = block;
-	ctx.state.builder.AddFunction({OpLabel, ctx.Label(block)});
+	EmitLabel(ctx.state, ctx.Label(block));
+	bool emitted_non_phi = false;
 	for (const auto& inst: *block) {
-		if (!EmitInstruction(ctx, inst)) {
+		if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+			if (emitted_non_phi) {
+				ctx.Fail(inst, "appears after a non-Phi instruction");
+				return;
+			}
+		} else {
+			emitted_non_phi = true;
+		}
+		if (!emit_instruction(inst)) {
 			return;
 		}
 	}
 }
 
-void EmitStructuredFunction(ValueEmitContext& ctx) {
-	ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
-	for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
-		EmitBlock(ctx, ctx.program.blocks[index]);
-		EmitStructuredTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index]);
+void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) {
+	for (const auto& deferred: structured.deferred_phis) {
+		for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) {
+			const auto* predecessor = deferred.instruction->PhiBlock(index);
+			const auto  found       = structured.block_exit_labels.find(predecessor);
+			if (found == structured.block_exit_labels.end()) {
+				ctx.failed = true;
+				ctx.error  = "typed Phi predecessor was not emitted";
+				return;
+			}
+			ctx.state.builder.PatchDeferredPhi(
+			    deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second);
+			if (ctx.failed) {
+				return;
+			}
+		}
 	}
 }
 
-void EmitDispatcherFunction(ValueEmitContext& ctx) {
+void EmitStructuredFunction(ValueEmitContext& ctx) {
+	StructuredFunctionState structured;
+	ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
+	for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
+		const auto* block = ctx.program.blocks[index];
+		EmitBlock(ctx, block, [&](const IR::Inst& inst) {
+			return EmitStructuredInstruction(ctx, structured, inst);
+		});
+		if (ctx.failed) {
+			break;
+		}
+		structured.block_exit_labels.emplace(block, ctx.state.current_label);
+		EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]);
+	}
+	if (!ctx.failed) {
+		PatchStructuredPhis(ctx, structured);
+	}
+}
+
+void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
 	auto&       state = ctx.state;
 	const auto* entry = ctx.program.blocks.front();
 	state.builder.AddFunction({OpBranch, ctx.Label(entry)});
-	EmitBlock(ctx, entry);
-	EmitDispatcherTerminator(ctx, entry, ctx.program.block_info.front(),
-	                         state.dispatch_header_label);
+	EmitBlock(ctx, entry, [&](const IR::Inst& inst) {
+		return EmitDispatcherInstruction(ctx, dispatcher, inst);
+	});
+	if (ctx.failed) {
+		return;
+	}
+	const auto initial_pc =
+	    EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front());
+	const auto initial_parent = state.current_label;
+	state.builder.AddFunction({OpBranch, dispatcher.header_label});
 
-	state.builder.AddFunction({OpLabel, state.dispatch_header_label});
-	const auto pc = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.uint_type, pc, state.dispatch_pc_variable});
+	EmitLabel(state, dispatcher.header_label);
+	const auto pc      = state.builder.AllocateId();
+	const auto next_pc = state.builder.AllocateId();
+	state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
+	                           dispatcher.continue_label});
 	const auto done = state.builder.AllocateId();
 	state.builder.AddFunction(
-	    {OpIEqual, state.bool_type, done, pc, ConstantU32(ctx.state, UINT32_MAX)});
+	    {OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
 	state.builder.AddFunction(
-	    {OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone});
+	    {OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label, LoopControlNone});
 	state.builder.AddFunction(
-	    {OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
+	    {OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
 
-	state.builder.AddFunction({OpLabel, state.dispatch_select_label});
+	EmitLabel(state, dispatcher.select_label);
 	state.builder.AddFunction(
-	    {OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
-	std::vector<uint32_t> words {OpSwitch, pc, state.dispatch_default_label};
+	    {OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone});
+	std::vector<uint32_t> words {OpSwitch, pc, dispatcher.after_switch_label};
 	for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
 		words.push_back(ctx.program.block_info[index].id);
 		words.push_back(ctx.Label(ctx.program.blocks[index]));
 	}
 	state.builder.AddFunction(words);
-	state.builder.AddFunction({OpLabel, state.dispatch_default_label});
-	state.builder.AddFunction(
-	    {OpStore, state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
-	state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
+	std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc,
+	                                     ConstantU32(state, UINT32_MAX), dispatcher.select_label};
 
 	for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) {
-		EmitBlock(ctx, ctx.program.blocks[index]);
-		EmitDispatcherTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index],
-		                         state.dispatch_after_switch_label);
+		EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) {
+			return EmitDispatcherInstruction(ctx, dispatcher, inst);
+		});
+		if (ctx.failed) {
+			break;
+		}
+		const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index],
+		                                           ctx.program.block_info[index]);
+		next_pc_words.push_back(selected);
+		next_pc_words.push_back(state.current_label);
+		state.builder.AddFunction({OpBranch, dispatcher.after_switch_label});
 	}
-	state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
-	state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
-	state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
-	state.builder.AddFunction({OpBranch, state.dispatch_header_label});
-	state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
+	if (ctx.failed) {
+		return;
+	}
+	EmitLabel(state, dispatcher.after_switch_label);
+	state.builder.AddFunction(next_pc_words);
+	state.builder.AddFunction({OpBranch, dispatcher.continue_label});
+	EmitLabel(state, dispatcher.continue_label);
+	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	EmitLabel(state, dispatcher.merge_label);
 	EmitReturn(ctx);
 }
 
@@ -303,16 +376,17 @@ void EmitDispatcherFunction(ValueEmitContext& ctx) {
 
 uint32_t ValueEmitContext::TypeId(IR::Type type) const {
 	switch (type) {
-		case IR::Type::U1: return state.bool_type;
+		case IR::Type::U1: return TypeBool(state);
 		case IR::Type::U8:
 		case IR::Type::U16:
 		case IR::Type::U32:
-		case IR::Type::F16: return state.uint_type;
-		case IR::Type::U64:
-		case IR::Type::U32x2: return state.uint_pair_type;
-		case IR::Type::F32: return state.float_type;
-		case IR::Type::U32x4: return state.vec4_uint_type;
-		case IR::Type::F32x2: return state.vec2_float_type;
+		case IR::Type::F16: return TypeU32(state);
+		case IR::Type::U64: return TypeU64(state);
+		case IR::Type::U32x2: return TypeU32Pair(state);
+		case IR::Type::F32: return TypeF32(state);
+		case IR::Type::U32x3: return TypeU32Vector(state, 3);
+		case IR::Type::U32x4: return TypeU32Vector(state, 4);
+		case IR::Type::F32x2: return TypeF32Vector(state, 2);
 		default: return 0;
 	}
 }
@@ -321,11 +395,11 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
 	value = value.Resolve();
 	if (value.IsImmediate()) {
 		switch (value.GetType()) {
-			case IR::Type::U1: return BoolConstant(state, value.U1());
+			case IR::Type::U1: return ConstantBool(state, value.U1());
 			case IR::Type::U8: return ConstantU32(state, value.U8());
 			case IR::Type::U16: return ConstantU32(state, value.U16());
 			case IR::Type::U32: return ConstantU32(state, value.U32());
-			case IR::Type::U64: return ConstantPair(state, value.U64());
+			case IR::Type::U64: return ConstantU64(state, value.U64());
 			case IR::Type::F16: return ConstantU32(state, value.F16Bits());
 			case IR::Type::F32:
 				return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
@@ -338,12 +412,17 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
 		error  = "direct SPIR-V emitter received a non-value argument";
 		return ConstantU32(state, 0);
 	}
-	if (program.dispatcher_fallback && current_block != nullptr &&
+	if (dispatcher_spills != nullptr && current_block != nullptr &&
 	    inst->Parent() != current_block) {
-		if (const auto found = dispatcher_variables.find(inst);
-		    found != dispatcher_variables.end()) {
+		if (const auto found = dispatcher_spills->find(inst); found != dispatcher_spills->end()) {
+			if (const auto loaded = dispatcher_block_loads.find(inst);
+			    loaded != dispatcher_block_loads.end() &&
+			    loaded->second.first == state.current_label) {
+				return loaded->second.second;
+			}
 			const auto id = state.builder.AllocateId();
 			state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
+			dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
 			return id;
 		}
 	}
@@ -419,48 +498,48 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const {
 	return labels.at(block);
 }
 
-size_t ValueEmitContext::BlockIndex(const IR::Block* block) const {
-	return static_cast<size_t>(std::ranges::find(program.blocks, block) - program.blocks.begin());
-}
-
 void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
 	failed = true;
 	error  = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
 }
 
 bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) {
-	ValueEmitContext ctx(state, program);
+	ValueEmitContext                       ctx(state, program);
+	std::optional<DispatcherFunctionState> dispatcher;
+	if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
+		state.pixel_valid_mask_variable = state.builder.AllocateId();
+		state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
+	}
 	for (const auto* block: program.blocks) {
 		ctx.labels.emplace(block, state.builder.AllocateId());
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
-				continue;
+	}
+	if (state.program.dispatcher_fallback) {
+		auto& dispatch = dispatcher.emplace();
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
+					continue;
+				}
+				if (SpillPointerType(ctx, inst.GetType()) == 0) {
+					ctx.Fail(inst, "cannot be stored by the dispatcher");
+					break;
+				}
+				dispatch.spills.emplace(&inst, state.builder.AllocateId());
 			}
-			const auto ptr_type = PhiPointerType(state, inst.GetType());
-			if (ptr_type == 0) {
-				ctx.failed = true;
-				ctx.error  = fmt::format("typed opcode Phi of type {} cannot be spilled across a "
-				                         "control-flow edge",
-				                         IR::TypeName(inst.GetType()));
-				break;
-			}
-			ctx.phi_variables.emplace(&inst, state.builder.AllocateId());
 		}
-	}
-	if (ctx.failed) {
-		SetError(error, ctx.error.c_str());
-		return false;
-	}
-	if (program.dispatcher_fallback) {
 		const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
 			value                  = value.Resolve();
 			const auto* definition = value.TryInstruction();
 			if (definition == nullptr || definition->Parent() == consumer ||
-			    PhiPointerType(state, definition->GetType()) == 0) {
+			    definition->Parent() == program.blocks.front()) {
 				return;
 			}
-			if (!ctx.dispatcher_variables.contains(definition)) {
-				ctx.dispatcher_variables.emplace(definition, state.builder.AllocateId());
+			if (SpillPointerType(ctx, definition->GetType()) == 0) {
+				ctx.Fail(*definition, "cannot be stored by the dispatcher");
+				return;
+			}
+			if (!dispatch.spills.contains(definition)) {
+				dispatch.spills.emplace(definition, state.builder.AllocateId());
 			}
 		};
 		for (const auto* block: program.blocks) {
@@ -476,83 +555,76 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
 			mark_cross_block(program.block_info[index].condition, program.blocks[index]);
 			mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
 		}
-		state.dispatcher_fallback         = true;
-		state.dispatch_pc_variable        = state.builder.AllocateId();
-		state.dispatch_header_label       = state.builder.AllocateId();
-		state.dispatch_select_label       = state.builder.AllocateId();
-		state.dispatch_default_label      = state.builder.AllocateId();
-		state.dispatch_after_switch_label = state.builder.AllocateId();
-		state.dispatch_continue_label     = state.builder.AllocateId();
-		state.dispatch_merge_label        = state.builder.AllocateId();
+		dispatch.header_label       = state.builder.AllocateId();
+		dispatch.select_label       = state.builder.AllocateId();
+		dispatch.after_switch_label = state.builder.AllocateId();
+		dispatch.continue_label     = state.builder.AllocateId();
+		dispatch.merge_label        = state.builder.AllocateId();
+		ctx.dispatcher_spills       = &dispatch.spills;
+	}
+	if (ctx.failed) {
+		SetError(error, ctx.error.c_str());
+		return false;
 	}
 	for (const auto* block: program.blocks) {
 		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
 			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
 			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
-			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
 		    })) {
 			ctx.scratch_u32_variable = state.builder.AllocateId();
 			break;
 		}
 	}
 	state.builder.AddFunction(
-	    {OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
-	state.builder.AddFunction({OpLabel, state.entry_label});
-	if (state.needs_function_lds) {
+	    {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
+	EmitLabel(state, state.entry_label);
+	if (state.requirements.function_lds) {
 		state.builder.AddFunction(
-		    {OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
+		    {OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)),
+		     state.lds_variable, StorageClassFunction});
+	}
+	if (state.requirements.function_scratch) {
+		state.builder.AddFunction(
+		    {OpVariable,
+		     TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
+		     state.scratch_variable, StorageClassFunction});
 	}
 	if (state.pixel_valid_mask_variable != 0) {
-		state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable,
-		                           StorageClassFunction});
+		state.builder.AddFunction({OpVariable,
+		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
+		                           state.pixel_valid_mask_variable, StorageClassFunction});
 	}
-	if (program.dispatcher_fallback) {
-		state.builder.AddFunction(
-		    {OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
-	}
-	for (const auto& binding: state.registers) {
-		state.builder.AddFunction(
-		    {OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
-	}
-	for (const auto* block: program.blocks) {
-		for (const auto& inst: *block) {
-			if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
-				continue;
-			}
-			state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
-			                           ctx.phi_variables.at(&inst), StorageClassFunction});
-		}
-	}
-	if (program.dispatcher_fallback) {
+	if (state.program.dispatcher_fallback) {
 		for (const auto* block: program.blocks) {
 			for (const auto& inst: *block) {
-				if (const auto found = ctx.dispatcher_variables.find(&inst);
-				    found != ctx.dispatcher_variables.end()) {
-					state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
+				if (const auto found = dispatcher->spills.find(&inst);
+				    found != dispatcher->spills.end()) {
+					state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()),
 					                           found->second, StorageClassFunction});
 				}
 			}
 		}
 	}
 	if (ctx.scratch_u32_variable != 0) {
-		state.builder.AddFunction(
-		    {OpVariable, state.ptr_func_uint, ctx.scratch_u32_variable, StorageClassFunction});
+		state.builder.AddFunction({OpVariable,
+		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
+		                           ctx.scratch_u32_variable, StorageClassFunction});
 	}
-	for (const auto& binding: state.registers) {
-		state.builder.AddFunction({OpStore, binding.pointer_id,
-		                           ConstantU32(state, InitialRegisterValue(state, binding.reg))});
+	if (state.gds_variable != 0) {
+		state.gds_length = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
 	}
 	if (state.pixel_valid_mask_variable != 0) {
 		state.builder.AddFunction(
 		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
 	}
-	EmitStorageBufferOffsets(state);
+	EmitMemoryOffsets(state);
 	if (program.blocks.empty()) {
 		EmitReturn(ctx);
-	} else if (program.dispatcher_fallback) {
-		EmitDispatcherFunction(ctx);
+	} else if (state.program.dispatcher_fallback) {
+		EmitDispatcherFunction(ctx, *dispatcher);
 	} else {
 		EmitStructuredFunction(ctx);
 	}

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp

@@ -1,7 +1,6 @@
 #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 
 #include <algorithm>
-#include <array>
 #include <fmt/format.h>
 #include <iterator>
 #include <map>
@@ -683,6 +682,66 @@ void RebuildPredecessors(Graph& graph) {
 	}
 }
 
+void PruneUnreachableBlocks(Graph& graph) {
+	if (graph.entry_block >= graph.blocks.size()) {
+		return;
+	}
+
+	std::vector<bool>     reachable(graph.blocks.size(), false);
+	std::vector<uint32_t> pending {graph.entry_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id >= graph.blocks.size() || reachable[block_id]) {
+			continue;
+		}
+		reachable[block_id] = true;
+		for (const auto successor: graph.blocks[block_id].successors) {
+			pending.push_back(successor);
+		}
+	}
+	if (std::ranges::all_of(reachable, [](bool value) { return value; })) {
+		return;
+	}
+
+	std::vector<uint32_t>   id_map(graph.blocks.size(), UINT32_MAX);
+	std::vector<BasicBlock> blocks;
+	blocks.reserve(std::ranges::count(reachable, true));
+	for (uint32_t old_id = 0; old_id < graph.blocks.size(); old_id++) {
+		if (!reachable[old_id]) {
+			continue;
+		}
+		id_map[old_id] = static_cast<uint32_t>(blocks.size());
+		blocks.push_back(std::move(graph.blocks[old_id]));
+	}
+
+	graph.entry_block   = RemapId(graph.entry_block, id_map);
+	graph.failure_block = RemapId(graph.failure_block, id_map);
+	graph.blocks        = std::move(blocks);
+	for (auto& block: graph.blocks) {
+		block.id = RemapId(block.id, id_map);
+		RemapIds(block.successors, id_map);
+		block.predecessors.clear();
+		block.dominators.clear();
+		block.post_dominators.clear();
+		auto& terminator          = block.terminator;
+		terminator.true_block     = RemapId(terminator.true_block, id_map);
+		terminator.false_block    = RemapId(terminator.false_block, id_map);
+		terminator.merge_block    = RemapId(terminator.merge_block, id_map);
+		terminator.continue_block = RemapId(terminator.continue_block, id_map);
+		for (auto& target: terminator.indirect_targets) {
+			target = RemapId(target, id_map);
+		}
+		for (auto& target: terminator.indirect_selector_targets) {
+			target = RemapId(target, id_map);
+		}
+	}
+	graph.back_edges.clear();
+	graph.natural_loops.clear();
+	graph.components.clear();
+	RebuildPredecessors(graph);
+}
+
 void ComputeDominators(Graph& graph) {
 	const auto count = static_cast<uint32_t>(graph.blocks.size());
 	const auto all   = AllBlockIds(count);
@@ -1079,11 +1138,75 @@ bool IsLoopControlGateway(const Graph& graph, const NaturalLoop& loop, uint32_t
 	       is_control_target(block->terminator.false_block);
 }
 
+bool HasLinearPathToTerminal(const Graph& graph, uint32_t start) {
+	std::vector<bool> visited(graph.blocks.size(), false);
+	for (auto block_id = start;;) {
+		const auto* block = graph.FindBlock(block_id);
+		if (block == nullptr || block_id >= visited.size() || visited[block_id]) {
+			return false;
+		}
+		if (block->successors.empty()) {
+			return true;
+		}
+		if (block->successors.size() != 1) {
+			return false;
+		}
+		visited[block_id] = true;
+		block_id          = block->successors.front();
+	}
+}
+
+bool IsEnclosingLinearExit(const Graph& graph, uint32_t header, uint32_t block_id) {
+	// AGC commonly lowers nested early returns through a terminal epilogue shared with an
+	// enclosing conditional. Such a path is an exit boundary, not part of the inner selection.
+	const auto* block = graph.FindBlock(block_id);
+	if (block == nullptr || graph.Dominates(header, block_id) ||
+	    !HasLinearPathToTerminal(graph, block_id) || block->predecessors.empty()) {
+		return false;
+	}
+	return std::ranges::all_of(block->predecessors, [&](uint32_t predecessor) {
+		return graph.Dominates(header, predecessor) || graph.Dominates(predecessor, header);
+	});
+}
+
+bool CanReachBefore(const Graph& graph, uint32_t start, uint32_t target, uint32_t stop) {
+	std::vector<uint32_t> pending = {start};
+	std::vector<bool>     visited(graph.blocks.size(), false);
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id == target) {
+			return true;
+		}
+		if (block_id == stop || block_id >= visited.size() || visited[block_id]) {
+			continue;
+		}
+		visited[block_id] = true;
+		const auto* block = graph.FindBlock(block_id);
+		if (block != nullptr) {
+			pending.insert(pending.end(), block->successors.begin(), block->successors.end());
+		}
+	}
+	return false;
+}
+
 uint32_t FindSelectionMerge(const Graph& graph, const BasicBlock& block) {
 	const auto  global_merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
 	                                                                block.terminator.false_block);
 	const auto* loop         = FindInnermostContainingLoop(graph, block.id);
 	if (loop == nullptr) {
+		const auto* global_block = graph.FindBlock(global_merge);
+		const auto  true_target  = block.terminator.true_block;
+		const auto  false_target = block.terminator.false_block;
+		if (global_block != nullptr && global_block->successors.empty()) {
+			const bool false_reaches_true =
+			    CanReachBefore(graph, false_target, true_target, global_merge);
+			const bool true_reaches_false =
+			    CanReachBefore(graph, true_target, false_target, global_merge);
+			if (false_reaches_true != true_reaches_false) {
+				return false_reaches_true ? true_target : false_target;
+			}
+		}
 		return global_merge;
 	}
 
@@ -1120,6 +1243,11 @@ bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& blo
 		};
 		return is_repeat_target(true_target) && is_repeat_target(false_target);
 	}
+	const bool true_in_body  = Contains(loop->body_blocks, true_target);
+	const bool false_in_body = Contains(loop->body_blocks, false_target);
+	if (true_in_body != false_in_body) {
+		return true;
+	}
 	const auto is_control_target = [&](uint32_t target) {
 		return target == loop->merge || target == loop->continue_block;
 	};
@@ -1198,6 +1326,37 @@ bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
 	return false;
 }
 
+bool IsolateSemanticLoopHeaders(Graph& graph, std::string* error) {
+	const auto isolation_budget = graph.natural_loops.size() + 1u;
+	for (size_t isolation = 0; isolation < isolation_budget; isolation++) {
+		const auto loop = std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(),
+		                               [&](const auto& value) {
+			                               const auto* header = graph.FindBlock(value.header);
+			                               return header != nullptr &&
+			                                      header->inst_begin != header->inst_end;
+		                               });
+		if (loop == graph.natural_loops.end()) {
+			return true;
+		}
+
+		// SPIR-V requires OpLoopMerge and its branch to remain in the loop header's
+		// physical block. Keep that header as a dedicated control node, exactly like
+		// the separate Loop node in shadPS4's structured AST. Guest instructions live
+		// in the body so later lowering may introduce bounds/EXEC control flow without
+		// displacing OpLoopMerge into a helper-created block.
+		if (!IsolateSemanticLoopHeader(graph, loop->header)) {
+			SetFailure(graph, FailureKind::StructuredControlFlow, loop->header,
+			           fmt::format("failed to isolate semantic loop header {}", loop->header),
+			           error);
+			return false;
+		}
+	}
+
+	SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
+	           "CFG semantic loop-header isolation exceeded rewrite budget", error);
+	return false;
+}
+
 bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
                            const std::vector<uint32_t>& construct_blocks,
                            bool                         force_split = false) {
@@ -1269,17 +1428,24 @@ std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& head
 	std::vector<uint32_t> region;
 	std::vector<uint32_t> pending = {header.terminator.true_block, header.terminator.false_block};
 	const auto*           loop    = FindInnermostContainingLoop(graph, header.id);
+	const auto global_merge = graph.FindNearestCommonPostDominator(header.terminator.true_block,
+	                                                               header.terminator.false_block);
 	while (!pending.empty()) {
 		const auto block_id = pending.back();
 		pending.pop_back();
 		if (block_id == merge || Contains(region, block_id) ||
-		    (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block))) {
+		    (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block)) ||
+		    IsEnclosingLinearExit(graph, header.id, block_id) ||
+		    (block_id == global_merge && HasLinearPathToTerminal(graph, block_id))) {
 			continue;
 		}
 		const auto* block = graph.FindBlock(block_id);
 		if (block == nullptr) {
 			continue;
 		}
+		if (block->successors.empty()) {
+			continue;
+		}
 		AddUnique(region, block_id);
 		pending.insert(pending.end(), block->successors.begin(), block->successors.end());
 	}
@@ -1324,7 +1490,11 @@ bool SplitOneSelectionMerge(Graph& graph, std::string* error) {
 		}
 		const auto region   = SelectionRegion(graph, *block, merge);
 		const auto external = std::find_if(region.begin(), region.end(), [&](uint32_t member) {
-			return !graph.Dominates(block_id, member);
+			const auto* member_block = graph.FindBlock(member);
+			return member_block != nullptr &&
+			       std::ranges::any_of(member_block->predecessors, [&](uint32_t predecessor) {
+				       return predecessor != block_id && !Contains(region, predecessor);
+			       });
 		});
 		if (external != region.end()) {
 			SetFailure(
@@ -1449,8 +1619,8 @@ uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_
 namespace {
 
 struct GotoRouteBlocks {
-	uint32_t                before = UINT32_MAX;
-	std::array<uint32_t, 5> blocks {};
+	uint32_t              before = UINT32_MAX;
+	std::vector<uint32_t> blocks;
 };
 
 void PlaceGotoRouteBlocks(Graph& graph, uint32_t original_block_count,
@@ -1514,6 +1684,48 @@ uint32_t AppendGotoSetBlock(Graph& graph, uint32_t route_variable, bool value, u
 	return block_id;
 }
 
+void AppendEnclosingRouteBlocks(Graph& graph, uint32_t original_block_count,
+                                uint32_t route_variable, uint32_t shared, uint32_t other,
+                                uint32_t route_select, uint32_t route_root,
+                                std::vector<uint32_t>& route_blocks) {
+	for (uint32_t depth = 0; depth < original_block_count; depth++) {
+		uint32_t predecessor = UINT32_MAX;
+		uint32_t leaf        = UINT32_MAX;
+		for (uint32_t candidate = 0; candidate < original_block_count; candidate++) {
+			const auto* candidate_block = graph.FindBlock(candidate);
+			if (candidate_block == nullptr ||
+			    candidate_block->terminator.kind != TerminatorKind::ConditionalBranch) {
+				continue;
+			}
+			const auto candidate_true  = candidate_block->terminator.true_block;
+			const auto candidate_false = candidate_block->terminator.false_block;
+			if (candidate_true == route_root &&
+			    (candidate_false == shared || candidate_false == other)) {
+				predecessor = candidate;
+				leaf        = candidate_false;
+				break;
+			}
+			if (candidate_false == route_root &&
+			    (candidate_true == shared || candidate_true == other)) {
+				predecessor = candidate;
+				leaf        = candidate_true;
+				break;
+			}
+		}
+		if (predecessor == UINT32_MAX) {
+			break;
+		}
+
+		const auto routed_leaf =
+		    AppendGotoSetBlock(graph, route_variable, leaf == other, route_select);
+		auto* predecessor_block = graph.FindBlock(predecessor);
+		ReplaceValue(predecessor_block->successors, leaf, routed_leaf);
+		ReplaceTerminatorTarget(predecessor_block->terminator, leaf, routed_leaf);
+		route_blocks.push_back(routed_leaf);
+		route_root = predecessor;
+	}
+}
+
 // Turn H0 -> shared/H1, H1 -> shared/other into nested selections whose empty
 // forwarding blocks set one typed SSA route value. Guest semantic blocks remain unique.
 bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t outer_id,
@@ -1538,66 +1750,139 @@ bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t out
 
 		const bool true_is_shared  = inner->terminator.true_block == shared;
 		const bool false_is_shared = inner->terminator.false_block == shared;
-		if (true_is_shared == false_is_shared) {
-			continue;
-		}
-		const auto other =
-		    true_is_shared ? inner->terminator.false_block : inner->terminator.true_block;
-		if (other == outer_id || other == inner_id || other == shared) {
-			continue;
-		}
-		const auto first_arm = std::min(shared, other);
-		if (first_arm >= original_block_count || outer_id >= inner_id || inner_id >= first_arm) {
-			continue;
-		}
-		if (graph.FindNearestCommonPostDominator(shared, other) == UINT32_MAX) {
-			continue;
+		if (true_is_shared != false_is_shared) {
+			const auto other =
+			    true_is_shared ? inner->terminator.false_block : inner->terminator.true_block;
+			if (other == outer_id || other == inner_id || other == shared) {
+				continue;
+			}
+			const auto first_arm = std::min(shared, other);
+			if (first_arm >= original_block_count || outer_id >= inner_id ||
+			    inner_id >= first_arm ||
+			    graph.FindNearestCommonPostDominator(shared, other) == UINT32_MAX) {
+				continue;
+			}
+
+			const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared);
+			const auto inner_merge  = AppendSyntheticBranchBlock(graph, route_select);
+			const auto outer_shared =
+			    AppendGotoSetBlock(graph, route_variable, false, route_select);
+			const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge);
+			const auto inner_other  = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
+
+			outer                         = graph.FindBlock(outer_id);
+			auto* mutable_inner           = graph.FindBlock(inner_id);
+			outer->terminator.true_block  = inner_is_true ? inner_id : outer_shared;
+			outer->terminator.false_block = inner_is_true ? outer_shared : inner_id;
+			outer->successors = {outer->terminator.true_block, outer->terminator.false_block};
+			mutable_inner->terminator.true_block  = true_is_shared ? inner_shared : inner_other;
+			mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared;
+			mutable_inner->successors             = {mutable_inner->terminator.true_block,
+			                                         mutable_inner->terminator.false_block};
+
+			std::vector<uint32_t> route_blocks = {inner_shared, inner_other, inner_merge,
+			                                      outer_shared};
+			AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared, other,
+			                           route_select, outer_id, route_blocks);
+			route_blocks.push_back(route_select);
+			route = {first_arm, std::move(route_blocks)};
+			return true;
 		}
 
-		const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared);
-		const auto inner_merge  = AppendSyntheticBranchBlock(graph, route_select);
-		const auto outer_shared = AppendGotoSetBlock(graph, route_variable, false, route_select);
-		const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge);
-		const auto inner_other  = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
+		// H0 -> arm/H1, H1 -> exit/body, and both arms reach a common continuation.
+		// Route exit/continuation after joining H0 and H1 so neither has a nonlocal edge.
+		for (const bool exit_is_true: {true, false}) {
+			const auto other =
+			    exit_is_true ? inner->terminator.true_block : inner->terminator.false_block;
+			const auto body =
+			    exit_is_true ? inner->terminator.false_block : inner->terminator.true_block;
+			if (other == outer_id || other == inner_id || other == shared || body == outer_id ||
+			    body == inner_id || body == shared || !graph.Dominates(inner_id, body)) {
+				continue;
+			}
 
-		outer                         = graph.FindBlock(outer_id);
-		auto* mutable_inner           = graph.FindBlock(inner_id);
-		outer->terminator.true_block  = inner_is_true ? inner_id : outer_shared;
-		outer->terminator.false_block = inner_is_true ? outer_shared : inner_id;
-		outer->successors = {outer->terminator.true_block, outer->terminator.false_block};
-		mutable_inner->terminator.true_block  = true_is_shared ? inner_shared : inner_other;
-		mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared;
-		mutable_inner->successors             = {mutable_inner->terminator.true_block,
-		                                         mutable_inner->terminator.false_block};
-		route = {first_arm, {inner_shared, inner_other, inner_merge, outer_shared, route_select}};
-		return true;
+			const auto continuation = graph.FindNearestCommonPostDominator(shared, body);
+			const auto* continuation_block = graph.FindBlock(continuation);
+			if (continuation_block == nullptr || continuation == other ||
+			    CanReachBefore(graph, other, continuation, UINT32_MAX)) {
+				continue;
+			}
+			std::vector<uint32_t> outer_predecessors;
+			std::vector<uint32_t> inner_predecessors;
+			bool                  external_predecessor = false;
+			for (const auto predecessor: continuation_block->predecessors) {
+				if (predecessor == outer_id || graph.Dominates(shared, predecessor)) {
+					outer_predecessors.push_back(predecessor);
+				} else if (predecessor == inner_id || graph.Dominates(body, predecessor)) {
+					inner_predecessors.push_back(predecessor);
+				} else {
+					external_predecessor = true;
+				}
+			}
+			const auto first_arm = std::min(continuation, other);
+			if (outer_predecessors.empty() || inner_predecessors.empty() ||
+			    external_predecessor || first_arm >= original_block_count ||
+			    outer_id >= inner_id || inner_id >= first_arm ||
+			    graph.FindNearestCommonPostDominator(continuation, other) == UINT32_MAX) {
+				continue;
+			}
+
+			const auto route_select =
+			    AppendGotoSelectBlock(graph, route_variable, other, continuation);
+			const auto inner_merge  = AppendSyntheticBranchBlock(graph, route_select);
+			const auto outer_continue =
+			    AppendGotoSetBlock(graph, route_variable, false, route_select);
+			const auto inner_continue =
+			    AppendGotoSetBlock(graph, route_variable, false, inner_merge);
+			const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
+
+			auto* mutable_inner = graph.FindBlock(inner_id);
+			if (exit_is_true) {
+				mutable_inner->terminator.true_block = inner_other;
+			} else {
+				mutable_inner->terminator.false_block = inner_other;
+			}
+			ReplaceValue(mutable_inner->successors, other, inner_other);
+			for (const auto predecessor: outer_predecessors) {
+				auto* predecessor_block = graph.FindBlock(predecessor);
+				ReplaceValue(predecessor_block->successors, continuation, outer_continue);
+				ReplaceTerminatorTarget(predecessor_block->terminator, continuation,
+				                        outer_continue);
+			}
+			for (const auto predecessor: inner_predecessors) {
+				auto* predecessor_block = graph.FindBlock(predecessor);
+				ReplaceValue(predecessor_block->successors, continuation, inner_continue);
+				ReplaceTerminatorTarget(predecessor_block->terminator, continuation,
+				                        inner_continue);
+			}
+
+			std::vector<uint32_t> route_blocks = {inner_continue, inner_other, inner_merge,
+			                                      outer_continue};
+			if (continuation == shared) {
+				AppendEnclosingRouteBlocks(graph, original_block_count, route_variable, shared,
+				                           other, route_select, outer_id, route_blocks);
+			}
+			route_blocks.push_back(route_select);
+			route = {first_arm, std::move(route_blocks)};
+			return true;
+		}
 	}
 	return false;
 }
 
-bool RouteSharedSelectionArms(Graph& graph) {
+bool RouteSharedSelectionArm(Graph& graph, uint32_t route_variable) {
 	if (graph.irreducible) {
 		return false;
 	}
-	const auto route_budget = static_cast<uint32_t>(graph.blocks.size());
-	bool       found        = false;
-	for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) {
-		const auto block_count = static_cast<uint32_t>(graph.blocks.size());
-		bool       routed      = false;
-		for (uint32_t block_id = 0; block_id < block_count; block_id++) {
-			GotoRouteBlocks route;
-			if (!RouteOneSharedArm(graph, block_count, block_id, route_variable, route)) {
-				continue;
-			}
+	const auto block_count = static_cast<uint32_t>(graph.blocks.size());
+	// Route later/deeper candidates first so earlier shared arms remain structured.
+	for (uint32_t block_id = block_count; block_id-- > 0;) {
+		GotoRouteBlocks route;
+		if (RouteOneSharedArm(graph, block_count, block_id, route_variable, route)) {
 			PlaceGotoRouteBlocks(graph, block_count, route);
 			RebuildPredecessors(graph);
 			RecomputeAnalyses(graph);
-			found  = true;
-			routed = true;
-			break;
-		}
-		if (!routed) {
-			return found;
+			return true;
 		}
 	}
 	return false;
@@ -1633,7 +1918,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 	labels.insert(end_pc);
 
 	std::map<uint32_t, SetpcTargetInfo> setpc_targets;
-	bool                                indirect_setpc = false;
 	for (uint32_t i = 0; i < program.instructions.size(); i++) {
 		const auto& inst    = program.instructions[i];
 		const auto  next_pc = InstructionEndPc(inst);
@@ -1670,7 +1954,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 				}
 				labels.insert(target);
 			}
-			indirect_setpc = indirect_setpc || target_info.indirect;
 			setpc_targets.emplace(inst.pc, std::move(target_info));
 			if (next_pc <= end_pc) {
 				labels.insert(next_pc);
@@ -1748,9 +2031,6 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 					block.terminator.indirect_selector_targets.push_back(
 					    pc_to_block.at(target_info.selector_target_pcs[i]));
 				}
-				if (target_info.table_load_pc != UINT32_MAX) {
-					AddUnique(graph.code_table_load_pcs, target_info.table_load_pc);
-				}
 			} else {
 				block.terminator.kind       = TerminatorKind::Branch;
 				block.terminator.condition  = BranchCondition::Always;
@@ -1811,6 +2091,22 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
 	for (auto& block: graph.blocks) {
 		SortUnique(block.predecessors);
 	}
+	PruneUnreachableBlocks(graph);
+	graph.code_table_load_pcs.clear();
+	bool indirect_setpc = false;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.kind != TerminatorKind::IndirectBranch) {
+			continue;
+		}
+		indirect_setpc = true;
+		if (block.inst_begin >= block.inst_end || block.inst_end > program.instructions.size()) {
+			continue;
+		}
+		const auto found = setpc_targets.find(program.instructions[block.inst_end - 1u].pc);
+		if (found != setpc_targets.end() && found->second.table_load_pc != UINT32_MAX) {
+			AddUnique(graph.code_table_load_pcs, found->second.table_load_pc);
+		}
+	}
 	SortUnique(graph.code_table_load_pcs);
 
 	ComputeDominators(graph);
@@ -1855,6 +2151,9 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
 	if (!SplitSharedMergeBlocks(graph, error)) {
 		return false;
 	}
+	if (!IsolateSemanticLoopHeaders(graph, error)) {
+		return false;
+	}
 	ClearStructuredTerminators(graph);
 
 	std::map<uint32_t, uint32_t> merge_headers;
@@ -1881,6 +2180,14 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
 			    error);
 			return false;
 		}
+		if (header->inst_begin != header->inst_end ||
+		    header->terminator.kind != TerminatorKind::Branch) {
+			SetFailure(graph, FailureKind::StructuredControlFlow, loop.header,
+			           fmt::format("loop header {} is not a dedicated empty control block",
+			                       loop.header),
+			           error);
+			return false;
+		}
 		if (!reserve_merge_block(loop.header, loop.merge)) {
 			return false;
 		}
@@ -1922,17 +2229,25 @@ bool Structurize(Graph& graph, std::string* error) {
 	if (StructurizeImpl(graph, error)) {
 		return true;
 	}
-	if (!RouteSharedSelectionArms(original)) {
-		return false;
-	}
+
 	Graph       failed_graph = std::move(graph);
 	std::string failed_error = error != nullptr ? *error : std::string {};
 	graph                    = std::move(original);
-	if (error != nullptr) {
-		error->clear();
-	}
-	if (StructurizeImpl(graph, error)) {
-		return true;
+	const auto route_budget  = static_cast<uint32_t>(graph.blocks.size());
+	// Apply one route at a time and retry. Eagerly routing every matching diamond can
+	// rewrite unrelated selections that were already structurally valid.
+	for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) {
+		if (!RouteSharedSelectionArm(graph, route_variable)) {
+			break;
+		}
+		Graph routed = graph;
+		if (error != nullptr) {
+			error->clear();
+		}
+		if (StructurizeImpl(routed, error)) {
+			graph = std::move(routed);
+			return true;
+		}
 	}
 	graph = std::move(failed_graph);
 	if (error != nullptr) {
@@ -1941,31 +2256,6 @@ bool Structurize(Graph& graph, std::string* error) {
 	return false;
 }
 
-bool IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error) {
-	const auto loop =
-	    std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(),
-	                 [header_id](const auto& value) { return value.header == header_id; });
-	if (loop == graph.natural_loops.end()) {
-		SetFailure(graph, FailureKind::StructuredControlFlow, header_id,
-		           fmt::format("block {} is not a natural loop header", header_id), error);
-		return false;
-	}
-
-	ClearStructuredTerminators(graph);
-	if (!IsolateSemanticLoopHeader(graph, header_id)) {
-		SetFailure(graph, FailureKind::StructuredControlFlow, header_id,
-		           fmt::format("loop header block {} has no semantic instructions", header_id),
-		           error);
-		return false;
-	}
-
-	graph.unsupported   = false;
-	graph.failure_kind  = FailureKind::None;
-	graph.failure_block = UINT32_MAX;
-	graph.unsupported_reason.clear();
-	return Structurize(graph, error);
-}
-
 std::string BranchConditionToString(BranchCondition condition) {
 	switch (condition) {
 		case BranchCondition::Always: return "always";

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -110,7 +110,6 @@ struct Graph {
 
 bool        BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error);
 bool        Structurize(Graph& graph, std::string* error);
-bool        IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error);
 std::string BranchConditionToString(BranchCondition condition);
 std::string FailureKindToString(FailureKind kind);
 std::string GraphToString(const Graph& graph);

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -153,28 +153,38 @@ constexpr MimgSampleInfo MIMG_SAMPLE_OPCODE_LIST[] = {
     SampleInfo(0x6fu, "image_sample_c_cd_cl_o",
                ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd |
                    ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xa0u, "image_sample_a", 0),
-    SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp),
-    SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias),
-    SampleInfo(0xa6u, "image_sample_b_cl_a", ImageSampleFlagBias | ImageSampleFlagLodClamp),
-    SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare),
-    SampleInfo(0xa9u, "image_sample_c_cl_a", ImageSampleFlagCompare | ImageSampleFlagLodClamp),
-    SampleInfo(0xadu, "image_sample_c_b_a", ImageSampleFlagCompare | ImageSampleFlagBias),
+    SampleInfo(0xa0u, "image_sample_a", ImageSampleFlagAdjust),
+    SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias | ImageSampleFlagAdjust),
+    SampleInfo(0xa6u, "image_sample_b_cl_a",
+               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare | ImageSampleFlagAdjust),
+    SampleInfo(0xa9u, "image_sample_c_cl_a",
+               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagAdjust),
+    SampleInfo(0xadu, "image_sample_c_b_a",
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagAdjust),
     SampleInfo(0xaeu, "image_sample_c_b_cl_a",
-               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp),
-    SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset),
-    SampleInfo(0xb1u, "image_sample_cl_o_a", ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xb5u, "image_sample_b_o_a", ImageSampleFlagBias | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
+                   ImageSampleFlagAdjust),
+    SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset | ImageSampleFlagAdjust),
+    SampleInfo(0xb1u, "image_sample_cl_o_a",
+               ImageSampleFlagLodClamp | ImageSampleFlagOffset | ImageSampleFlagAdjust),
+    SampleInfo(0xb5u, "image_sample_b_o_a",
+               ImageSampleFlagBias | ImageSampleFlagOffset | ImageSampleFlagAdjust),
     SampleInfo(0xb6u, "image_sample_b_cl_o_a",
-               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
-    SampleInfo(0xb8u, "image_sample_c_o_a", ImageSampleFlagCompare | ImageSampleFlagOffset),
+               ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
+    SampleInfo(0xb8u, "image_sample_c_o_a",
+               ImageSampleFlagCompare | ImageSampleFlagOffset | ImageSampleFlagAdjust),
     SampleInfo(0xb9u, "image_sample_c_cl_o_a",
-               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
     SampleInfo(0xbdu, "image_sample_c_b_o_a",
-               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset),
+               ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset |
+                   ImageSampleFlagAdjust),
     SampleInfo(0xbeu, "image_sample_c_b_cl_o_a",
                ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
-                   ImageSampleFlagOffset),
+                   ImageSampleFlagOffset | ImageSampleFlagAdjust),
 };
 
 constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
@@ -299,7 +309,9 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	const uint32_t vaddr  = word1 & 0xffu;
 	const uint32_t srsrc  = (word1 >> 16u) & 0x1fu;
 	const uint32_t ssamp  = (word1 >> 21u) & 0x1fu;
+	const bool     r128   = ((word0 >> 15u) & 0x1u) != 0u;
 	const bool     a16    = ((word1 >> 30u) & 0x1u) != 0u;
+	const bool     d16    = ((word1 >> 31u) & 0x1u) != 0u;
 	const auto*    sample = LookupSample(opcode);
 	const auto*    gather = LookupGather(opcode);
 	const auto*    atomic = LookupAtomic(opcode);
@@ -311,7 +323,9 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	inst.opcode_id          = opcode;
 	inst.opcode             = DecodeMimgOpcode(opcode, sample, gather, atomic);
 	inst.dmask              = (word0 >> 8u) & 0xfu;
-	inst.data_dwords        = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
+	inst.data_components    = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
+	inst.data_bits          = d16 ? 16u : 32u;
+	inst.data_dwords        = d16 ? (inst.data_components + 1u) / 2u : inst.data_components;
 	inst.glc                = ((word0 >> 13u) & 1u) != 0;
 	inst.slc                = ((word0 >> 25u) & 1u) != 0;
 	inst.image_sample_flags = DecodeMimgSampleFlags(sample, gather);
@@ -319,6 +333,7 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		inst.image_sample_flags |= ImageSampleFlagA16;
 	}
 	inst.image_dimension  = dimension;
+	inst.image_r128       = r128;
 	inst.image_nsa_dwords = nsa_dwords;
 	for (uint32_t i = 0; i < nsa_dwords * 4u; i++) {
 		inst.image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
@@ -334,6 +349,11 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		SetUnsupported(inst, Family::MIMG, opcode,
 		               "MIMG image gather requires exactly one dmask bit");
 	}
+	const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u ||
+	                          opcode == 0x01u || opcode == 0x08u || opcode == 0x09u;
+	if (d16 && !supports_d16) {
+		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data");
+	}
 
 	DecodeVectorGpr(vdata, inst.dst);
 	DecodeVectorGpr(vaddr, inst.src0);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -96,7 +96,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
     {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32},      {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
     {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32},      {0x36u, Opcode::DS_READ_B32, 1, 32},
-    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2_B32, 2, 32},
+    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2ST64_B32, 2, 32},
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
@@ -403,10 +403,11 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 		SetUnsupported(inst, Family::DS, opcode,
 		               "DS read addtid data operands are not implemented");
 	}
-	if (inst.opcode == Opcode::DS_WRITE2_B32 || opcode == 0x37u) {
+	if (inst.opcode == Opcode::DS_WRITE2_B32 || inst.opcode == Opcode::DS_READ2_B32) {
 		inst.offset           = offset0 * 4u;
 		inst.secondary_offset = offset1 * 4u;
-	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || opcode == 0x38u) {
+	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 ||
+	           inst.opcode == Opcode::DS_READ2ST64_B32) {
 		inst.offset           = offset0 * 256u;
 		inst.secondary_offset = offset1 * 256u;
 	} else if (inst.opcode == Opcode::DS_WRITE2_B64 || inst.opcode == Opcode::DS_READ2_B64) {

src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h

@@ -1,6 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
 
+#include "common/assert.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <array>
@@ -27,7 +28,7 @@ consteval auto MakeOpcodeTable(const Entry (&entries)[EntryCount]) {
 	for (size_t i = 0; i < EntryCount; i++) {
 		const auto encoding = entries[i].encoding;
 		if (encoding >= EncodingCount || table.indices[encoding] != 0) {
-			throw "invalid opcode table";
+			EXIT("invalid opcode table\n");
 		}
 		table.entries[i]        = entries[i];
 		table.indices[encoding] = static_cast<uint16_t>(i + 1);

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -52,6 +52,7 @@ constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
     {0x20u, Opcode::S_SETPC_B64},
     {0x24u, Opcode::S_AND_SAVEEXEC_B64},
     {0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
+    {0x2du, Opcode::S_QUADMASK_B64},
     {0x34u, Opcode::S_ABS_I32},
     {0x37u, Opcode::S_ANDN1_SAVEEXEC_B64},
     {0x3bu, Opcode::S_BITREPLICATE_B64_B32},

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -140,6 +140,7 @@ std::string ImageSampleFlagsToString(uint32_t flags) {
 	AppendFlag(&text, &first, flags, ImageSampleFlagA16, "a16");
 	AppendFlag(&text, &first, flags, ImageSampleFlagCd, "cd");
 	AppendFlag(&text, &first, flags, ImageSampleFlagGatherHorizontal, "gather_horizontal");
+	AppendFlag(&text, &first, flags, ImageSampleFlagAdjust, "adjust");
 	return text;
 }
 
@@ -153,6 +154,12 @@ std::string FormatMimg(const Instruction& inst) {
 	                OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(),
 	                OperandToString(inst.src1).c_str(), OperandToString(inst.src2).c_str(),
 	                inst.dmask, ImageDimensionToString(inst.image_dimension));
+	if (inst.data_bits == 16u) {
+		text += " d16=1";
+	}
+	if (inst.image_r128) {
+		text += " r128=1";
+	}
 	switch (inst.opcode) {
 		case Opcode::IMAGE_SAMPLE:
 		case Opcode::IMAGE_GATHER4_LZ:
@@ -498,6 +505,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::S_FF1_I32_B64:
 		case Opcode::S_NOT_B64:
 		case Opcode::S_WQM_B64:
+		case Opcode::S_QUADMASK_B64:
 		case Opcode::S_AND_SAVEEXEC_B32:
 		case Opcode::S_ANDN1_SAVEEXEC_B32:
 		case Opcode::S_AND_SAVEEXEC_B64:
@@ -658,6 +666,7 @@ std::string InstructionToString(const Instruction& inst) {
 		case Opcode::DS_READ_I16:
 		case Opcode::DS_READ_U16:
 		case Opcode::DS_READ2_B32:
+		case Opcode::DS_READ2ST64_B32:
 		case Opcode::DS_READ_B32:
 		case Opcode::DS_READ_B64:
 		case Opcode::DS_READ2_B64:

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -47,6 +47,7 @@ enum class Opcode {
 	S_FF1_I32_B64,
 	S_FLBIT_I32_B64,
 	S_BITREPLICATE_B64_B32,
+	S_QUADMASK_B64,
 	S_GETPC_B64,
 	S_SETPC_B64,
 	S_AND_SAVEEXEC_B32,
@@ -490,6 +491,7 @@ enum class Opcode {
 	DS_READ_I16,
 	DS_READ_U16,
 	DS_READ2_B32,
+	DS_READ2ST64_B32,
 	DS_READ_B32,
 	DS_READ_B64,
 	DS_READ2_B64,
@@ -584,6 +586,7 @@ enum ImageSampleFlag : uint32_t {
 	ImageSampleFlagA16              = 1u << 7u,
 	ImageSampleFlagCd               = 1u << 8u,
 	ImageSampleFlagGatherHorizontal = 1u << 9u,
+	ImageSampleFlagAdjust           = 1u << 10u,
 };
 
 enum class ImageDimension : uint32_t {
@@ -645,6 +648,7 @@ struct Instruction {
 	uint32_t       offset                                       = 0;
 	uint32_t       secondary_offset                             = 0;
 	uint32_t       dmask                                        = 0;
+	uint32_t       data_components                              = 0;
 	uint32_t       data_dwords                                  = 1;
 	uint32_t       data_bits                                    = 32;
 	uint32_t       data_format                                  = 0;
@@ -663,6 +667,7 @@ struct Instruction {
 	bool           slc                                          = false;
 	bool           idxen                                        = false;
 	bool           offen                                        = false;
+	bool           image_r128                                   = false;
 	int32_t        branch_offset                                = 0;
 	uint32_t       branch_target                                = 0;
 	struct {

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -11,7 +11,6 @@ using Detail::OpcodeMap;
 // architectural SDWA legality classes.
 enum class Vop2SdwaProfile {
 	None,
-	Cndmask,
 	Float32,
 	Float16,
 	IntegerFullDestination,
@@ -29,7 +28,7 @@ struct Vop2OpcodeInfo {
 };
 
 constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
-    {0x01u, Opcode::V_CNDMASK_B32, Vop2SdwaProfile::Cndmask},
+    {0x01u, Opcode::V_CNDMASK_B32, Vop2SdwaProfile::IntegerPartialDestination},
     {0x02u, Opcode::V_DOT2C_F32_F16},
     {0x03u, Opcode::V_ADD_F32, Vop2SdwaProfile::Float32},
     {0x04u, Opcode::V_SUB_F32, Vop2SdwaProfile::Float32},
@@ -929,8 +928,6 @@ constexpr uint32_t SdwaSelAll() {
 
 constexpr Vop2SdwaRule VOP2_SDWA_RULES[] = {
     {},
-    {SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(),
-     true, false},
     {SdwaSelFull(), SdwaSelFull(), SdwaSelFull(), false, true},
     {SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(), SdwaSelWords() | SdwaSelFull(),
      true, true},

src/graphics/shader/recompiler/frontend/translate/Attribute.cpp

@@ -13,9 +13,16 @@ IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) {
 
 bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) {
 	if (inst.op == IR::Opcode::LoadInputF32) {
-		WriteOperand(inst.dst,
-		             ir.Emit(IR::ValueOpcode::GetAttribute,
-		                     {IR::Value(inst.input_info.attr), IR::Value(inst.input_info.chan)}));
+		for (uint32_t component = 0; component < inst.input_info.component_count; component++) {
+			const auto chan  = inst.input_info.chan + component;
+			const auto value = inst.input_info.interpolation_mode < 3u
+			                       ? ir.Emit(IR::ValueOpcode::GetInterpolationParameter,
+			                                 {IR::Value(inst.input_info.attr), IR::Value(chan),
+			                                  IR::Value(inst.input_info.interpolation_mode)})
+			                       : ir.Emit(IR::ValueOpcode::GetAttribute,
+			                                 {IR::Value(inst.input_info.attr), IR::Value(chan)});
+			WriteOperand(OffsetOperand(inst.dst, component), value);
+		}
 		return true;
 	}
 	if (inst.op == IR::Opcode::Export) {

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -7,37 +7,26 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
 	switch (inst.op) {
 		case IR::Opcode::SaveexecB32:
 		case IR::Opcode::SaveexecB64: {
-			const auto old_low  = ir.GetExecLo();
-			const auto old_high = ir.GetExecHi();
-			const auto src_low  = ReadRawU32(inst.src[0]);
-			const auto src_high = inst.op == IR::Opcode::SaveexecB64
-			                          ? ReadRawU32(OffsetOperand(inst.src[0], 1))
-			                          : IR::U32(IR::Value(0u));
-			const auto apply    = [&](IR::U32 old_value, IR::U32 src_value) {
-				switch (inst.saveexec_mode) {
-					case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value);
-					case IR::SaveexecMode::Andn1:
-						return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value));
-					case IR::SaveexecMode::Orn2:
-						return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value);
-				}
-				return old_value;
-			};
-			const auto result_low = apply(old_low, src_low);
-			const auto result_high =
-			    inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
-			if (inst.op == IR::Opcode::SaveexecB64) {
-				WriteOperand(inst.dst, ir.PackUint2x32(old_low, old_high));
-			} else {
-				WriteOperand(inst.dst, old_low);
+			const auto old = ir.GetExec();
+			const auto src = ReadMask(inst.src[0]);
+			IR::U1     result;
+			switch (inst.saveexec_mode) {
+				case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break;
+				case IR::SaveexecMode::Andn1:
+					result = ir.LogicalAnd(old, ir.LogicalNot(src));
+					break;
+				case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break;
 			}
-			ir.SetExecLo(result_low);
-			ir.SetExecHi(result_high);
-			ir.SetExec(ThreadBit(result_low, result_high));
-			const auto nonzero = inst.op == IR::Opcode::SaveexecB64
-			                         ? ir.BitwiseOr(result_low, result_high)
-			                         : result_low;
-			ir.SetScc(ir.INotEqual(nonzero, zero));
+			if (inst.op == IR::Opcode::SaveexecB64) {
+				WriteMask64(inst.dst, old);
+			} else {
+				WriteMask(inst.dst, old);
+			}
+			const auto mask = BallotMask(result);
+			ir.SetExec(result);
+			ir.SetExecLo(mask[0]);
+			ir.SetExecHi(mask[1]);
+			ir.SetScc(result);
 			return true;
 		}
 		case IR::Opcode::ScalarAddCarryU32:
@@ -157,26 +146,61 @@ bool Translator::TranslateMove(const IR::Instruction& inst) {
 			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
 			return true;
 		case IR::Opcode::MoveU64: {
-			const bool scalar_pair = inst.dst.kind == IR::OperandKind::Register &&
+			const auto is_exec_or_vcc = [](const IR::Operand& operand) {
+				return operand.kind == IR::OperandKind::Register &&
+				       (operand.reg.file == IR::RegisterFile::Exec ||
+				        operand.reg.file == IR::RegisterFile::Vcc);
+			};
+			if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) {
+				WriteMask64(inst.dst, ReadMask(inst.src[0]));
+				return true;
+			}
+			const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register &&
 			                         inst.src[0].kind == IR::OperandKind::Register &&
 			                         inst.dst.reg.file == IR::RegisterFile::Scalar &&
 			                         inst.src[0].reg.file == IR::RegisterFile::Scalar;
-			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U64));
-			if (scalar_pair) {
-				ir.SetThreadBitScalarReg(
-				    static_cast<IR::ScalarReg>(inst.dst.reg.index),
-				    ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
+			IR::U1     source_mask;
+			IR::U1     source_mask_valid;
+			if (scalar_copy) {
+				const auto src    = static_cast<IR::ScalarReg>(inst.src[0].reg.index);
+				source_mask       = ir.GetThreadBitScalarReg(src);
+				source_mask_valid = ir.GetScalarMaskTag(src);
+			}
+			WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0]));
+			if (scalar_copy) {
+				ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index),
+				                         source_mask);
+				ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg.index),
+				                    source_mask_valid);
 			}
 			return true;
 		}
 		case IR::Opcode::WqmB64: {
-			if (!current_per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
-			    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-				const auto result =
-				    ExpandWholeQuadMask(IR::U64(ReadOperand(inst.src[0], IR::Type::U64)));
+			const auto is_exec_or_vcc = [](const IR::Operand& operand) {
+				return operand.kind == IR::OperandKind::Register &&
+				       (operand.reg.file == IR::RegisterFile::Exec ||
+				        operand.reg.file == IR::RegisterFile::Vcc);
+			};
+			if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) {
+				const auto mask_valid = ReadMaskValid(inst.src[0]);
+				const auto invocation_result =
+				    IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
+				const auto result = IR::U64(
+				    ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
 				WriteOperand(inst.dst, result);
-				ir.SetScc(IR::U1(
-				    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+				if (inst.dst.kind == IR::OperandKind::Register &&
+				    inst.dst.reg.file == IR::RegisterFile::Scalar) {
+					const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
+					ir.SetThreadBitScalarReg(dst, invocation_result);
+					ir.SetScalarMaskTag(dst, mask_valid);
+					const auto raw_nonzero = IR::U1(
+					    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
+					ir.SetScc(IR::U1(ir.Emit(
+					    IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
+				} else {
+					ir.SetScc(IR::U1(
+					    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+				}
 				return true;
 			}
 			const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -3,24 +3,85 @@
 namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
 
 bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
-	if (current_per_invocation_masks) {
-		return TranslatePerInvocationU64Mask(inst);
+	const auto invocation_result = EvaluateU64Mask(inst);
+	const auto is_exec_or_vcc    = [](const IR::Operand& operand) {
+		return operand.kind == IR::OperandKind::Register &&
+		       (operand.reg.file == IR::RegisterFile::Exec ||
+		        operand.reg.file == IR::RegisterFile::Vcc);
+	};
+	if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) ||
+	    (inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) {
+		WriteMask64(inst.dst, invocation_result);
+		ir.SetScc(invocation_result);
+		return true;
 	}
-	return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64
-	           ? TranslateSimpleInteger(inst)
-	           : TranslateComposedInteger(inst);
+
+	const auto lhs        = ReadU32Pair(inst.src[0]);
+	auto       mask_valid = ReadMaskValid(inst.src[0]);
+	if (inst.src_count > 1u) {
+		mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1]));
+	}
+	std::array<IR::U32, 2> result;
+	if (inst.op == IR::Opcode::BitwiseNotU64) {
+		result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
+	} else {
+		const auto rhs = ReadU32Pair(inst.src[1]);
+		for (uint32_t component = 0; component < 2u; component++) {
+			switch (inst.op) {
+				case IR::Opcode::BitwiseAndU64:
+					result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
+					break;
+				case IR::Opcode::BitwiseAndNotU64:
+					result[component] =
+					    ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
+					break;
+				case IR::Opcode::BitwiseOrU64:
+					result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
+					break;
+				case IR::Opcode::BitwiseOrNotU64:
+					result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
+					break;
+				case IR::Opcode::BitwiseXorU64:
+					result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
+					break;
+				case IR::Opcode::BitwiseNandU64:
+					result[component] =
+					    ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
+					break;
+				case IR::Opcode::BitwiseNorU64:
+					result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
+					break;
+				case IR::Opcode::BitwiseXnorU64:
+					result[component] =
+					    ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
+					break;
+				default: return false;
+			}
+		}
+	}
+	WriteU32Pair(inst.dst, result);
+	if (inst.dst.kind == IR::OperandKind::Register &&
+	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
+		const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
+		ir.SetThreadBitScalarReg(dst, invocation_result);
+		ir.SetScalarMaskTag(dst, mask_valid);
+		const auto raw_nonzero =
+		    ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
+		ir.SetScc(IR::U1(
+		    ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
+	} else {
+		ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
+	}
+	return true;
 }
 
-bool Translator::TranslateInstruction(const IR::Instruction&          inst,
-                                      const BufferAddressValues*      address_snapshot,
-                                      const ScalarMemorySourceValues* scalar_source_snapshot) {
+bool Translator::TranslateInstruction(const IR::Instruction& inst) {
 	switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
 		case IR::LoweringClass::Control: return TranslateControlOperation(inst);
 		case IR::LoweringClass::Move: return TranslateMove(inst);
 		case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
 		case IR::LoweringClass::State: return TranslateStateOperation(inst);
-		case IR::LoweringClass::Memory:
-			return TranslateMemoryOperation(inst, address_snapshot, scalar_source_snapshot);
+		case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst);
 		case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
 		case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
 		case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -16,11 +16,12 @@ bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) {
 		default: return false;
 	}
 	const auto translate_lane = [&](bool high) {
-		const auto lhs = ReadF16LaneAsF32(inst.src[0], high);
-		const auto rhs = ReadF16LaneAsF32(inst.src[1], high);
+		const auto lhs = ReadF16LaneAsF32(inst.src[0], high, true);
+		const auto rhs = ReadF16LaneAsF32(inst.src[1], high, true);
 		IR::F32    result;
 		if (inst.src_count == 3u) {
-			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high)}));
+			result = IR::F32(
+			    ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high, true)}));
 		} else {
 			result = IR::F32(ir.Emit(opcode, {lhs, rhs}));
 		}
@@ -88,6 +89,16 @@ bool Translator::TranslateFloat16Operation(const IR::Instruction& inst) {
 		case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break;
 		default: EXIT("invalid half-float source count: %u", inst.src_count);
 	}
+	if (inst.op == IR::Opcode::SqrtF16 || inst.op == IR::Opcode::InverseSqrtF16 ||
+	    inst.op == IR::Opcode::Log2F16) {
+		const auto negative = IR::U1(
+		    ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {args[0], IR::Value::F32(0.0f)}));
+		result = ApplyF32ResultModifiers(inst.dst, result);
+		const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
+		const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
+		WriteU16(inst.dst, ir.Select(negative, invalid, bits));
+		return true;
+	}
 	WriteF16(inst.dst, result);
 	return true;
 }

src/graphics/shader/recompiler/frontend/translate/Integer.cpp

@@ -126,34 +126,23 @@ bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) {
 	}
 }
 
-bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) {
-	if (!current_per_invocation_masks) {
-		return false;
-	}
-
+IR::U1 Translator::EvaluateU64Mask(const IR::Instruction& inst) {
 	const auto lhs = [&] { return ReadMask(inst.src[0]); };
 	const auto rhs = [&] { return ReadMask(inst.src[1]); };
-	IR::U1     result;
 	switch (inst.op) {
-		case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break;
-		case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break;
+		case IR::Opcode::BitwiseAndU64: return ir.LogicalAnd(lhs(), rhs());
+		case IR::Opcode::BitwiseOrU64: return ir.LogicalOr(lhs(), rhs());
 		case IR::Opcode::BitwiseXorU64:
-			result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
-			break;
-		case IR::Opcode::BitwiseAndNotU64:
-			result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
-			break;
-		case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break;
-		case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break;
-		case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break;
+			return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
+		case IR::Opcode::BitwiseAndNotU64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
+		case IR::Opcode::BitwiseOrNotU64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs()));
+		case IR::Opcode::BitwiseNandU64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs()));
+		case IR::Opcode::BitwiseNorU64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs()));
 		case IR::Opcode::BitwiseXnorU64:
-			result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
-			break;
-		case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break;
-		default: return false;
+			return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
+		case IR::Opcode::BitwiseNotU64: return ir.LogicalNot(lhs());
+		default: EXIT("invalid 64-bit mask opcode");
 	}
-	WriteMask64(inst.dst, result);
-	return true;
 }
 
 bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
@@ -187,14 +176,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
 			type   = IR::Type::U64;
 			break;
 		case IR::Opcode::FindLsbU32: opcode = IR::ValueOpcode::FindILsb32; break;
-		case IR::Opcode::BitwiseAndU64:
-			opcode = IR::ValueOpcode::BitwiseAnd64;
-			type   = IR::Type::U64;
-			break;
-		case IR::Opcode::BitwiseOrU64:
-			opcode = IR::ValueOpcode::BitwiseOr64;
-			type   = IR::Type::U64;
-			break;
 		case IR::Opcode::ShiftLeftLogicalU32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break;
 		case IR::Opcode::ShiftRightLogicalU32: opcode = IR::ValueOpcode::ShiftRightLogical32; break;
 		case IR::Opcode::ShiftRightArithmeticI32:
@@ -219,10 +200,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
 		                    opcode == IR::ValueOpcode::ShiftRightLogical32 ||
 		                    opcode == IR::ValueOpcode::ShiftRightArithmetic32)) {
 			args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u)));
-		} else if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical64 ||
-		                           opcode == IR::ValueOpcode::ShiftRightLogical64 ||
-		                           opcode == IR::ValueOpcode::ShiftRightArithmetic64)) {
-			args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(63u)));
 		}
 	}
 	IR::Value result;
@@ -233,18 +210,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
 		default: EXIT("invalid simple integer source count: %u", inst.src_count);
 	}
 	WriteOperand(inst.dst, result);
-	if (type == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register &&
-	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-		IR::U1 mask;
-		if (inst.op == IR::Opcode::BitwiseAndU64) {
-			mask = ir.LogicalAnd(ReadMask(inst.src[0]), ReadMask(inst.src[1]));
-		} else if (inst.op == IR::Opcode::BitwiseOrU64) {
-			mask = ir.LogicalOr(ReadMask(inst.src[0]), ReadMask(inst.src[1]));
-		}
-		if (!mask.IsEmpty()) {
-			ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index), mask);
-		}
-	}
 	return true;
 }
 
@@ -254,11 +219,6 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 		const auto rhs = ReadU32(inst.src[1]);
 		return operation(lhs, rhs);
 	};
-	const auto binary_u64 = [&](auto operation) {
-		const auto lhs = UnpackU64(ReadU64(inst.src[0]));
-		const auto rhs = UnpackU64(ReadU64(inst.src[1]));
-		return ir.PackUint2x32(operation(lhs[0], rhs[0]), operation(lhs[1], rhs[1]));
-	};
 
 	IR::Value result;
 	switch (inst.op) {
@@ -282,34 +242,6 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 			result = binary_u32(
 			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); });
 			break;
-		case IR::Opcode::BitwiseXorU64:
-			result = binary_u64([&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseXor(lhs, rhs); });
-			break;
-		case IR::Opcode::BitwiseAndNotU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); });
-			break;
-		case IR::Opcode::BitwiseOrNotU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); });
-			break;
-		case IR::Opcode::BitwiseNandU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); });
-			break;
-		case IR::Opcode::BitwiseNorU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); });
-			break;
-		case IR::Opcode::BitwiseXnorU64:
-			result = binary_u64(
-			    [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); });
-			break;
-		case IR::Opcode::BitwiseNotU64: {
-			const auto source = UnpackU64(ReadU64(inst.src[0]));
-			result            = ir.PackUint2x32(ir.BitwiseNot(source[0]), ir.BitwiseNot(source[1]));
-			break;
-		}
 		case IR::Opcode::BitwiseAndOrU32: {
 			const auto lhs = ReadU32(inst.src[0]);
 			const auto rhs = ReadU32(inst.src[1]);
@@ -326,7 +258,7 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 			                       ReadU32(inst.src[2]));
 			break;
 		case IR::Opcode::FindLsbU64: {
-			const auto source        = UnpackU64(ReadU64(inst.src[0]));
+			const auto source        = ExtractU64(ReadU64(inst.src[0]));
 			const auto low_lsb       = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]}));
 			const auto high_lsb      = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]}));
 			const auto high_position = ir.IAdd(high_lsb, IR::U32(IR::Value(32u)));
@@ -349,44 +281,13 @@ bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
 			const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb);
 			const auto nonzero  = IR::U1(
 			    ir.Emit(IR::ValueOpcode::INotEqual64,
-			            {source, ir.PackUint2x32(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
+			            {source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
 			result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu)));
 			break;
 		}
 		default: return false;
 	}
 	WriteOperand(inst.dst, result);
-	if (result.GetType() == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register &&
-	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-		const auto lhs = ReadMask(inst.src[0]);
-		IR::U1     mask;
-		switch (inst.op) {
-			case IR::Opcode::BitwiseXorU64:
-				mask = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])}));
-				break;
-			case IR::Opcode::BitwiseAndNotU64:
-				mask = ir.LogicalAnd(lhs, ir.LogicalNot(ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseOrNotU64:
-				mask = ir.LogicalOr(lhs, ir.LogicalNot(ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseNandU64:
-				mask = ir.LogicalNot(ir.LogicalAnd(lhs, ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseNorU64:
-				mask = ir.LogicalNot(ir.LogicalOr(lhs, ReadMask(inst.src[1])));
-				break;
-			case IR::Opcode::BitwiseXnorU64:
-				mask = ir.LogicalNot(
-				    IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])})));
-				break;
-			case IR::Opcode::BitwiseNotU64: mask = ir.LogicalNot(lhs); break;
-			default: break;
-		}
-		if (!mask.IsEmpty()) {
-			ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index), mask);
-		}
-	}
 	return true;
 }
 
@@ -402,18 +303,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 		return IR::U32(
 		    ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), imm(0), count}));
 	};
-	const auto select_u64 = [&](IR::U1 condition, IR::U64 true_value, IR::U64 false_value) {
-		const auto lhs = UnpackU64(true_value);
-		const auto rhs = UnpackU64(false_value);
-		return ir.PackUint2x32(ir.Select(condition, lhs[0], rhs[0]),
-		                       ir.Select(condition, lhs[1], rhs[1]));
-	};
 	const auto right_mask64 = [&](IR::U32 count) {
 		const auto below32    = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, imm(32)}));
 		const auto above32    = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, imm(32)}));
 		const auto low_count  = ir.Select(below32, count, imm(32));
 		const auto high_count = ir.Select(above32, ir.ISub(count, imm(32)), imm(0));
-		return ir.PackUint2x32(right_mask32(low_count), right_mask32(high_count));
+		return ir.ConstructU64(right_mask32(low_count), right_mask32(high_count));
 	};
 
 	IR::Value result;
@@ -436,7 +331,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 		case IR::Opcode::UMadU64U32: {
 			const auto lhs       = ReadU32(inst.src[0]);
 			const auto rhs       = ReadU32(inst.src[1]);
-			const auto add       = UnpackU64(ReadU64(inst.src[2]));
+			const auto add       = ExtractU64(ReadU64(inst.src[2]));
 			const auto mul_low   = ir.IMul(lhs, rhs);
 			const auto mul_high  = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
 			const auto low       = ir.IAdd(mul_low, add[0]);
@@ -445,7 +340,7 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 			const auto carry0    = ir.ULessThan(high0, mul_high);
 			const auto high      = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0)));
 			const auto carry1    = ir.ULessThan(high, high0);
-			WriteOperand(inst.dst, ir.PackUint2x32(low, high));
+			WriteOperand(inst.dst, ir.ConstructU64(low, high));
 			if (inst.dst2.kind != IR::OperandKind::Null) {
 				WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
 			}
@@ -503,10 +398,25 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 				return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1)));
 			};
 			const auto source = ReadU32(inst.src[0]);
-			result            = ir.PackUint2x32(replicate(mask(source, 0xffffu)),
+			result            = ir.ConstructU64(replicate(mask(source, 0xffffu)),
 			                                    replicate(ir.ShiftRightLogical(source, imm(16))));
 			break;
 		}
+		case IR::Opcode::QuadmaskB64: {
+			const auto compact = [&](IR::U32 value) {
+				auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, imm(1)));
+				bits      = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(2)));
+				bits      = mask(bits, 0x11111111u);
+				bits = mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(3))), 0x03030303u);
+				bits = mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(6))), 0x000f000fu);
+				return mask(ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, imm(12))), 0xffu);
+			};
+			const auto source = ReadU32Pair(inst.src[0]);
+			const auto quads =
+			    ir.BitwiseOr(compact(source[0]), ir.ShiftLeftLogical(compact(source[1]), imm(8)));
+			result = ir.ConstructU64(quads, imm(0));
+			break;
+		}
 		case IR::Opcode::BitFieldMaskU32: {
 			const auto count  = mask(ReadU32(inst.src[0]), 31u);
 			const auto offset = mask(ReadU32(inst.src[1]), 31u);
@@ -590,12 +500,12 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 			const auto lo           = ReadU32(inst.src[1]);
 			const auto byte_offset  = mask(ReadU32(inst.src[2]), 31u);
 			const auto bit_offset   = ir.ShiftLeftLogical(byte_offset, imm(3));
-			const auto concatenated = ir.PackUint2x32(lo, hi);
+			const auto concatenated = ir.ConstructU64(lo, hi);
 			const auto shifted      = IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64,
 			                                          {concatenated, mask(bit_offset, 63u)}));
 			const auto in_range =
 			    IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, imm(8)}));
-			result = ir.Select(in_range, UnpackU64(shifted)[0], imm(0));
+			result = ir.Select(in_range, ExtractU64(shifted)[0], imm(0));
 			break;
 		}
 		case IR::Opcode::ShiftLeftAddU32:
@@ -633,10 +543,32 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
 			                                ReadOperand(inst.src[2], IR::Type::F32)});
 			break;
 		}
-		case IR::Opcode::SelectU64:
-			result =
-			    select_u64(ReadCondition(inst.src[0]), ReadU64(inst.src[1]), ReadU64(inst.src[2]));
-			break;
+		case IR::Opcode::SelectU64: {
+			const auto condition = ReadCondition(inst.src[0]);
+			const auto selected_mask =
+			    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
+			                   {condition, ReadMask(inst.src[1]), ReadMask(inst.src[2])}));
+			const auto selected_mask_valid =
+			    IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src[1]),
+			                                               ReadMaskValid(inst.src[2])}));
+			if (inst.dst.kind == IR::OperandKind::Register &&
+			    (inst.dst.reg.file == IR::RegisterFile::Exec ||
+			     inst.dst.reg.file == IR::RegisterFile::Vcc)) {
+				WriteMask64(inst.dst, selected_mask);
+				return true;
+			}
+			const auto lhs = ReadU32Pair(inst.src[1]);
+			const auto rhs = ReadU32Pair(inst.src[2]);
+			WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]),
+			                        ir.Select(condition, lhs[1], rhs[1])});
+			if (inst.dst.kind == IR::OperandKind::Register &&
+			    inst.dst.reg.file == IR::RegisterFile::Scalar) {
+				const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
+				ir.SetThreadBitScalarReg(dst, selected_mask);
+				ir.SetScalarMaskTag(dst, selected_mask_valid);
+			}
+			return true;
+		}
 		case IR::Opcode::PackLowLowU16:
 		case IR::Opcode::PackLowHighU16:
 		case IR::Opcode::PackHighHighU16:

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -34,7 +34,7 @@ Translator::AddressOperands Translator::ReadAddressOperands(const IR::Instructio
 		                            high_or_base.reg.file != IR::RegisterFile::Vector
 		                        ? ReadU32(high_or_base)
 		                        : low;
-		return {GetAddressResource(offset, IR::Value(0u)), offset, IR::Value(0u)};
+		return {ir.Emit(IR::ValueOpcode::GetScratchResource), offset, IR::Value(0u)};
 	}
 	if (inst.memory.kind == IR::ResourceKind::Global &&
 	    high_or_base.kind == IR::OperandKind::Register &&
@@ -52,11 +52,13 @@ IR::Value Translator::GetScalarAddressResource(uint32_t base) {
 }
 
 IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) {
+	const auto dword = [&](uint32_t index) {
+		return memory.image_r128 && index >= 4u ? IR::U32(IR::Value(0u))
+		                                         : GetResourceDword(memory.resource, index);
+	};
 	return ir.Emit(IR::ValueOpcode::GetImageResource,
-	               {GetResourceDword(memory.resource, 0), GetResourceDword(memory.resource, 1),
-	                GetResourceDword(memory.resource, 2), GetResourceDword(memory.resource, 3),
-	                GetResourceDword(memory.resource, 4), GetResourceDword(memory.resource, 5),
-	                GetResourceDword(memory.resource, 6), GetResourceDword(memory.resource, 7)});
+	               {dword(0), dword(1), dword(2), dword(3), dword(4), dword(5), dword(6),
+	                dword(7)});
 }
 
 IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) {
@@ -65,11 +67,6 @@ IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) {
 	                GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)});
 }
 
-IR::Value Translator::GetSharedResource(IR::ResourceKind kind) {
-	return ir.Emit(kind == IR::ResourceKind::Gds ? IR::ValueOpcode::GetGdsResource
-	                                             : IR::ValueOpcode::GetLdsResource);
-}
-
 IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) {
 	std::array<IR::Value, 13> components {};
 	components[0] = ReadRawU32(PlainOperand(base));
@@ -99,9 +96,17 @@ IR::Value Translator::ConstructU32x4(const IR::Operand& base, uint32_t count) {
 	               {components[0], components[1], components[2], components[3]});
 }
 
-void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
-                                      uint32_t component_limit) {
-	const auto mask      = dmask != 0u ? dmask : 1u;
+void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value,
+                                      const IR::MemoryInfo& memory, uint32_t component_limit) {
+	if (memory.data_bits == 16u) {
+		for (uint32_t index = 0; index < memory.data_dwords; index++) {
+			WriteOperand(OffsetOperand(dst, index),
+			             ir.Emit(IR::ValueOpcode::CompositeExtractU32x4,
+			                     {value, IR::Value(index)}));
+		}
+		return;
+	}
+	const auto mask      = memory.dmask != 0u ? memory.dmask : 1u;
 	uint32_t   dst_index = 0;
 	for (uint32_t component = 0; component < component_limit; component++) {
 		if (((mask >> component) & 1u) == 0u) {
@@ -125,8 +130,8 @@ IR::ValueOpcode Translator::ImageAtomicOpcode(IR::Opcode opcode) {
 	}
 }
 
-BufferAddressValues Translator::ReadBufferAddress(const IR::Instruction& inst,
-                                                  uint32_t               first_source) {
+Translator::BufferAddress Translator::ReadBufferAddress(const IR::Instruction& inst,
+                                                        uint32_t               first_source) {
 	uint32_t   cursor = first_source;
 	const auto next   = [&]() {
 		return cursor < inst.src_count ? ReadU32(inst.src[cursor++]) : IR::U32(IR::Value(0u));
@@ -170,22 +175,7 @@ IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) {
 	}
 }
 
-IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
-	if (gds) {
-		switch (opcode) {
-			case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::GdsAtomicSwap32;
-			case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::GdsAtomicIAdd32;
-			case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::GdsAtomicISub32;
-			case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::GdsAtomicSMin32;
-			case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::GdsAtomicUMin32;
-			case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::GdsAtomicSMax32;
-			case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::GdsAtomicUMax32;
-			case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::GdsAtomicAnd32;
-			case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::GdsAtomicOr32;
-			case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::GdsAtomicXor32;
-			default: EXIT("invalid GDS atomic opcode");
-		}
-	}
+IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode) {
 	switch (opcode) {
 		case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32;
 		case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32;
@@ -197,29 +187,11 @@ IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
 		case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32;
 		case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32;
 		case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32;
-		default: EXIT("invalid LDS atomic opcode");
+		default: EXIT("invalid shared atomic opcode");
 	}
 }
 
-bool Translator::IsBufferLoadOperation(IR::Opcode opcode) {
-	return opcode == IR::Opcode::BufferLoadUbyte || opcode == IR::Opcode::BufferLoadSbyte ||
-	       opcode == IR::Opcode::BufferLoadUshort || opcode == IR::Opcode::BufferLoadSshort ||
-	       opcode == IR::Opcode::BufferLoadDword;
-}
-
-bool Translator::IsScalarMemoryLoadOperation(IR::Opcode opcode) {
-	return opcode == IR::Opcode::SLoadDword || opcode == IR::Opcode::SBufferLoadDword;
-}
-
-ScalarMemorySourceValues Translator::ReadScalarMemorySource(const IR::Instruction& inst) {
-	const auto resource = inst.op == IR::Opcode::SLoadDword
-	                          ? GetScalarAddressResource(inst.memory.resource)
-	                          : GetBufferResource(inst.memory);
-	return {resource, ReadU32(inst.src[0])};
-}
-
-bool Translator::TranslateScalarMemory(const IR::Instruction&          inst,
-                                       const ScalarMemorySourceValues* source_snapshot) {
+bool Translator::TranslateScalarMemory(const IR::Instruction& inst) {
 	switch (inst.op) {
 		case IR::Opcode::LoadSrtDword: {
 			const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource);
@@ -227,28 +199,37 @@ bool Translator::TranslateScalarMemory(const IR::Instruction&          inst,
 			             ir.Emit(IR::ValueOpcode::ReadConst, {resource, ReadU32(inst.src[0])}));
 			return true;
 		}
-		case IR::Opcode::SBufferLoadDword: {
-			const auto source =
-			    source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst);
-			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ReadConstBuffer,
-			                               {source.resource, source.offset}, AddMemoryInfo(inst)));
-			return true;
-		}
+		case IR::Opcode::SBufferLoadDword:
 		case IR::Opcode::SLoadDword: {
-			const auto source =
-			    source_snapshot != nullptr ? *source_snapshot : ReadScalarMemorySource(inst);
-			WriteOperand(inst.dst,
-			             ir.Emit(IR::ValueOpcode::LoadAddressU32,
-			                     {source.resource, source.offset, IR::Value(0u), IR::Value(true)},
-			                     AddMemoryInfo(inst)));
+			const auto resource = inst.op == IR::Opcode::SLoadDword
+			                          ? GetScalarAddressResource(inst.memory.resource)
+			                          : GetBufferResource(inst.memory);
+			const auto offset = ReadU32(inst.src[0]);
+			std::array<IR::Value, 16> loaded {};
+			for (uint32_t component = 0; component < inst.memory.data_dwords; component++) {
+				auto scalar             = inst;
+				scalar.memory.offset += component * sizeof(uint32_t);
+				scalar.memory.data_dwords     = 1u;
+				scalar.memory.component_index = component;
+				if (inst.op == IR::Opcode::SBufferLoadDword) {
+					loaded[component] = ir.Emit(IR::ValueOpcode::ReadConstBuffer,
+					                            {resource, offset}, AddMemoryInfo(scalar));
+				} else {
+					loaded[component] = ir.Emit(
+					    IR::ValueOpcode::LoadAddressU32,
+					    {resource, offset, IR::Value(0u), IR::Value(true)}, AddMemoryInfo(scalar));
+				}
+			}
+			for (uint32_t component = 0; component < inst.memory.data_dwords; component++) {
+				WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]);
+			}
 			return true;
 		}
 		default: return false;
 	}
 }
 
-bool Translator::TranslateBufferLoad(const IR::Instruction&     inst,
-                                     const BufferAddressValues* address_snapshot) {
+bool Translator::TranslateBufferLoad(const IR::Instruction& inst) {
 	IR::ValueOpcode opcode;
 	uint32_t        bits;
 	bool            sign;
@@ -266,19 +247,33 @@ bool Translator::TranslateBufferLoad(const IR::Instruction&     inst,
 			sign   = inst.op == IR::Opcode::BufferLoadSshort;
 			break;
 		case IR::Opcode::BufferLoadDword:
-			opcode = IR::ValueOpcode::LoadBufferU32;
-			bits   = 32u;
-			sign   = false;
+			switch (inst.memory.data_dwords) {
+				case 1u: opcode = IR::ValueOpcode::LoadBufferU32; break;
+				case 2u: opcode = IR::ValueOpcode::LoadBufferU32x2; break;
+				case 3u: opcode = IR::ValueOpcode::LoadBufferU32x3; break;
+				case 4u: opcode = IR::ValueOpcode::LoadBufferU32x4; break;
+				default: return false;
+			}
+			bits = 32u;
+			sign = false;
 			break;
 		default: return false;
 	}
 	const auto resource = GetBufferResource(inst.memory);
-	const auto address =
-	    address_snapshot != nullptr ? *address_snapshot : ReadBufferAddress(inst, 0);
+	const auto address  = ReadBufferAddress(inst, 0);
 	const auto loaded =
 	    ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, ir.GetExec()},
 	            AddMemoryInfo(inst));
-	WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
+	if (bits != 32u) {
+		WriteOperand(inst.dst, WidenSubdword(loaded, bits, sign));
+	} else if (inst.memory.data_dwords == 1u) {
+		WriteOperand(inst.dst, loaded);
+	} else {
+		for (uint32_t component = 0; component < inst.memory.data_dwords; component++) {
+			WriteOperand(OffsetOperand(inst.dst, component),
+			             ir.CompositeExtract(loaded, component));
+		}
+	}
 	return true;
 }
 
@@ -298,8 +293,31 @@ bool Translator::TranslateBufferStore(const IR::Instruction& inst) {
 			value  = NarrowSubdword(data, 16u);
 			break;
 		case IR::Opcode::BufferStoreDword:
-			opcode = IR::ValueOpcode::StoreBufferU32;
-			value  = data;
+			switch (inst.memory.data_dwords) {
+				case 1u:
+					opcode = IR::ValueOpcode::StoreBufferU32;
+					value  = data;
+					break;
+				case 2u:
+					opcode = IR::ValueOpcode::StoreBufferU32x2;
+					value  = ir.Emit(IR::ValueOpcode::CompositeConstructU32x2,
+					                 {data, ReadU32(OffsetOperand(inst.src[0], 1u))});
+					break;
+				case 3u:
+					opcode = IR::ValueOpcode::StoreBufferU32x3;
+					value  = ir.Emit(IR::ValueOpcode::CompositeConstructU32x3,
+					                 {data, ReadU32(OffsetOperand(inst.src[0], 1u)),
+					                  ReadU32(OffsetOperand(inst.src[0], 2u))});
+					break;
+				case 4u:
+					opcode = IR::ValueOpcode::StoreBufferU32x4;
+					value  = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
+					                 {data, ReadU32(OffsetOperand(inst.src[0], 1u)),
+					                  ReadU32(OffsetOperand(inst.src[0], 2u)),
+					                  ReadU32(OffsetOperand(inst.src[0], 3u))});
+					break;
+				default: return false;
+			}
 			break;
 		default: return false;
 	}
@@ -333,13 +351,9 @@ bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) {
 		}
 		case IR::ResourceKind::Lds:
 		case IR::ResourceKind::Gds: {
-			const bool gds      = inst.memory.kind == IR::ResourceKind::Gds;
-			const auto resource = GetSharedResource(inst.memory.kind);
-			const auto address =
-			    ir.IAdd(ReadU32(inst.src[1]), IR::U32(IR::Value(inst.memory.offset)));
-			result = ir.Emit(SharedAtomicOpcode(inst.op, gds),
-			                 {resource, address, ReadU32(inst.src[0]), ir.GetExec()},
-			                 AddMemoryInfo(inst));
+			const auto address = ReadU32(inst.src[1]);
+			result = ir.Emit(SharedAtomicOpcode(inst.op),
+			                 {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst));
 			break;
 		}
 		default: return false;
@@ -431,20 +445,20 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
 		case IR::Opcode::ImageGetResinfo: {
 			const auto result =
 			    ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, flags);
-			WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
+			WriteImageComponents(inst.dst, result, inst.memory, 4u);
 			return true;
 		}
 		case IR::Opcode::ImageGetLod: {
 			const auto sampler = GetSamplerResource(inst.memory);
 			const auto result =
 			    ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, flags);
-			WriteImageComponents(inst.dst, result, inst.memory.dmask, 2u);
+			WriteImageComponents(inst.dst, result, inst.memory, 2u);
 			return true;
 		}
 		case IR::Opcode::ImageLoad: {
 			const auto result =
 			    ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, flags);
-			WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
+			WriteImageComponents(inst.dst, result, inst.memory, 4u);
 			return true;
 		}
 		case IR::Opcode::ImageStore: {
@@ -461,7 +475,7 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
 			const auto result  = ir.Emit(opcode, {resource, sampler, address}, flags);
 			const bool dref =
 			    (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u;
-			if (inst.op == IR::Opcode::ImageSample && dref) {
+			if (inst.op == IR::Opcode::ImageSample && dref && inst.memory.data_bits != 16u) {
 				const auto component =
 				    ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)});
 				for (uint32_t index = 0; index < inst.memory.data_dwords; index++) {
@@ -474,7 +488,7 @@ bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
 					                     {result, IR::Value(index)}));
 				}
 			} else {
-				WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
+				WriteImageComponents(inst.dst, result, inst.memory, 4u);
 			}
 			return true;
 		}
@@ -488,10 +502,51 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 	if (!shared && inst.op != IR::Opcode::DsSwizzleB32) {
 		return false;
 	}
-	const bool gds            = inst.memory.kind == IR::ResourceKind::Gds;
-	const auto resource       = shared ? GetSharedResource(inst.memory.kind) : IR::Value {};
-	const auto shared_address = [&](uint32_t source) {
-		return ir.IAdd(ReadU32(inst.src[source]), IR::U32(IR::Value(inst.memory.offset)));
+	const auto shared_address = [&](uint32_t source) { return ReadU32(inst.src[source]); };
+	const auto load_u32 = [&](uint32_t width, IR::U32 address, const IR::Instruction& source) {
+		IR::ValueOpcode opcode;
+		switch (width) {
+			case 1u: opcode = IR::ValueOpcode::LoadSharedU32; break;
+			case 2u: opcode = IR::ValueOpcode::LoadSharedU32x2; break;
+			case 3u: opcode = IR::ValueOpcode::LoadSharedU32x3; break;
+			case 4u: opcode = IR::ValueOpcode::LoadSharedU32x4; break;
+			default: EXIT("invalid shared load width");
+		}
+		return ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(source));
+	};
+	const auto extract_u32 = [&](IR::Value value, uint32_t width, uint32_t index) {
+		if (width == 1u) {
+			return value;
+		}
+		const auto opcode = width == 2u   ? IR::ValueOpcode::CompositeExtractU32x2
+		                    : width == 3u ? IR::ValueOpcode::CompositeExtractU32x3
+		                                  : IR::ValueOpcode::CompositeExtractU32x4;
+		return ir.Emit(opcode, {value, IR::Value(index)});
+	};
+	const auto write_u32 = [&](uint32_t width, IR::U32 address,
+	                           const std::array<IR::Value, 4>& values,
+	                           const IR::Instruction&          source) {
+		switch (width) {
+			case 1u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32, {address, values[0], ir.GetExec()},
+				        AddMemoryInfo(source));
+				break;
+			case 2u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32x2,
+				        {address, values[0], values[1], ir.GetExec()}, AddMemoryInfo(source));
+				break;
+			case 3u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32x3,
+				        {address, values[0], values[1], values[2], ir.GetExec()},
+				        AddMemoryInfo(source));
+				break;
+			case 4u:
+				ir.Emit(IR::ValueOpcode::WriteSharedU32x4,
+				        {address, values[0], values[1], values[2], values[3], ir.GetExec()},
+				        AddMemoryInfo(source));
+				break;
+			default: EXIT("invalid shared store width");
+		}
 	};
 	switch (inst.op) {
 		case IR::Opcode::DsReadUbyte:
@@ -503,65 +558,109 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 			uint32_t        bits;
 			bool            sign;
 			if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) {
-				opcode = gds ? IR::ValueOpcode::LoadGdsU8 : IR::ValueOpcode::LoadSharedU8;
+				opcode = IR::ValueOpcode::LoadSharedU8;
 				bits   = 8u;
 				sign   = inst.op == IR::Opcode::DsReadSbyte;
 			} else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) {
-				opcode = gds ? IR::ValueOpcode::LoadGdsU16 : IR::ValueOpcode::LoadSharedU16;
+				opcode = IR::ValueOpcode::LoadSharedU16;
 				bits   = 16u;
 				sign   = inst.op == IR::Opcode::DsReadSshort;
 			} else {
-				opcode = gds ? IR::ValueOpcode::LoadGdsU32 : IR::ValueOpcode::LoadSharedU32;
-				bits   = 32u;
-				sign   = false;
+				const auto width  = inst.memory.data_dwords;
+				const auto loaded = load_u32(width, shared_address(0), inst);
+				for (uint32_t index = 0; index < width; index++) {
+					WriteOperand(OffsetOperand(inst.dst, index), extract_u32(loaded, width, index));
+				}
+				return true;
 			}
 			const auto loaded =
-			    ir.Emit(opcode, {resource, shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
+			    ir.Emit(opcode, {shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
 			WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
 			return true;
 		}
+		case IR::Opcode::DsRead2B32: {
+			const auto width             = inst.memory.data_dwords / 2u;
+			const auto address           = shared_address(0);
+			auto       first             = inst;
+			first.memory.data_dwords     = width;
+			first.memory.component_count = width;
+			const auto first_value       = load_u32(width, address, first);
+			IR::Value  second_value      = first_value;
+			if (inst.memory.secondary_offset != inst.memory.offset) {
+				auto second          = first;
+				second.memory.offset = inst.memory.secondary_offset;
+				second_value         = load_u32(width, address, second);
+			}
+			for (uint32_t index = 0; index < width; index++) {
+				WriteOperand(OffsetOperand(inst.dst, index),
+				             extract_u32(first_value, width, index));
+				WriteOperand(OffsetOperand(inst.dst, width + index),
+				             extract_u32(second_value, width, index));
+			}
+			return true;
+		}
 		case IR::Opcode::DsWriteByte:
 		case IR::Opcode::DsWriteShort:
 		case IR::Opcode::DsWriteB32: {
-			const auto      data    = ReadU32(inst.src[0]);
+			const auto               width = inst.memory.data_dwords;
+			std::array<IR::Value, 4> values {};
+			for (uint32_t index = 0; index < width; index++) {
+				values[index] = ReadU32(OffsetOperand(inst.src[0], index));
+			}
+			const auto      data    = IR::U32(values[0]);
 			const auto      address = shared_address(1);
 			IR::ValueOpcode opcode;
 			IR::Value       value;
 			if (inst.op == IR::Opcode::DsWriteByte) {
-				opcode = gds ? IR::ValueOpcode::WriteGdsU8 : IR::ValueOpcode::WriteSharedU8;
+				opcode = IR::ValueOpcode::WriteSharedU8;
 				value  = NarrowSubdword(data, 8u);
 			} else if (inst.op == IR::Opcode::DsWriteShort) {
-				opcode = gds ? IR::ValueOpcode::WriteGdsU16 : IR::ValueOpcode::WriteSharedU16;
+				opcode = IR::ValueOpcode::WriteSharedU16;
 				value  = NarrowSubdword(data, 16u);
 			} else {
-				opcode = gds ? IR::ValueOpcode::WriteGdsU32 : IR::ValueOpcode::WriteSharedU32;
-				value  = data;
+				write_u32(width, address, values, inst);
+				return true;
+			}
+			ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(inst));
+			return true;
+		}
+		case IR::Opcode::DsWrite2B32: {
+			const auto               width   = inst.memory.data_dwords / 2u;
+			const auto               address = shared_address(1);
+			std::array<IR::Value, 4> first_values {};
+			std::array<IR::Value, 4> second_values {};
+			for (uint32_t index = 0; index < width; index++) {
+				first_values[index]  = ReadU32(OffsetOperand(inst.src[0], index));
+				second_values[index] = ReadU32(OffsetOperand(inst.src[2], index));
+			}
+			auto first                   = inst;
+			first.memory.data_dwords     = width;
+			first.memory.component_count = width;
+			write_u32(width, address, first_values, first);
+			if (inst.memory.secondary_offset != inst.memory.offset) {
+				auto second          = first;
+				second.memory.offset = inst.memory.secondary_offset;
+				write_u32(width, address, second_values, second);
 			}
-			ir.Emit(opcode, {resource, address, value, ir.GetExec()}, AddMemoryInfo(inst));
 			return true;
 		}
 		case IR::Opcode::DsMinF32:
 		case IR::Opcode::DsMaxF32: {
-			const auto opcode =
-			    gds ? (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::GdsAtomicFMin32
-			                                           : IR::ValueOpcode::GdsAtomicFMax32)
-			        : (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::SharedAtomicFMin32
-			                                           : IR::ValueOpcode::SharedAtomicFMax32);
+			const auto opcode = inst.op == IR::Opcode::DsMinF32
+			                        ? IR::ValueOpcode::SharedAtomicFMin32
+			                        : IR::ValueOpcode::SharedAtomicFMax32;
 			ir.Emit(opcode,
-			        {resource, shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]),
-			         ir.GetExec()},
+			        {shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), ir.GetExec()},
 			        AddMemoryInfo(inst));
 			return true;
 		}
 		case IR::Opcode::DsAppend:
 		case IR::Opcode::DsConsume: {
 			const bool append = inst.op == IR::Opcode::DsAppend;
-			const auto opcode =
-			    gds ? (append ? IR::ValueOpcode::GdsDataAppend : IR::ValueOpcode::GdsDataConsume)
-			        : (append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume);
+			const auto opcode = append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume;
 			WriteOperand(inst.dst, ir.Emit(opcode,
-			                               {resource, ReadU32(inst.src[0]), ir.GetExec(),
-			                                ir.GetExecLo(), ir.GetExecHi()},
+			                               {ReadU32(inst.src[0]), ir.GetExec(), ir.GetExecLo(),
+			                                ir.GetExecHi()},
 			                               AddMemoryInfo(inst)));
 			return true;
 		}
@@ -571,16 +670,13 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 			const auto base =
 			    ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu)));
 			const auto lane    = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
-			const auto address = ir.IAdd(ir.IAdd(base, IR::U32(IR::Value(inst.memory.offset))),
-			                             ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
+			const auto address = ir.IAdd(base, ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
 			if (inst.op == IR::Opcode::DsWriteAddtidB32) {
 				ir.Emit(IR::ValueOpcode::WriteSharedU32,
-				        {resource, address, ReadU32(inst.src[0]), ir.GetExec()},
-				        AddMemoryInfo(inst));
+				        {address, ReadU32(inst.src[0]), ir.GetExec()}, AddMemoryInfo(inst));
 			} else {
-				WriteOperand(inst.dst,
-				             ir.Emit(IR::ValueOpcode::LoadSharedU32,
-				                     {resource, address, ir.GetExec()}, AddMemoryInfo(inst)));
+				WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32,
+				                               {address, ir.GetExec()}, AddMemoryInfo(inst)));
 			}
 			return true;
 		}
@@ -593,20 +689,18 @@ bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
 	}
 }
 
-bool Translator::TranslateMemoryOperation(const IR::Instruction&          inst,
-                                          const BufferAddressValues*      address_snapshot,
-                                          const ScalarMemorySourceValues* scalar_source_snapshot) {
+bool Translator::TranslateMemoryOperation(const IR::Instruction& inst) {
 	switch (inst.op) {
 		case IR::Opcode::LoadSrtDword:
 		case IR::Opcode::SLoadDword:
 		case IR::Opcode::SBufferLoadDword:
-			return TranslateScalarMemory(inst, scalar_source_snapshot);
+			return TranslateScalarMemory(inst);
 
 		case IR::Opcode::BufferLoadUbyte:
 		case IR::Opcode::BufferLoadSbyte:
 		case IR::Opcode::BufferLoadUshort:
 		case IR::Opcode::BufferLoadSshort:
-		case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst, address_snapshot);
+		case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst);
 
 		case IR::Opcode::BufferStoreByte:
 		case IR::Opcode::BufferStoreShort:
@@ -647,9 +741,11 @@ bool Translator::TranslateMemoryOperation(const IR::Instruction&          inst,
 		case IR::Opcode::DsReadUshort:
 		case IR::Opcode::DsReadSshort:
 		case IR::Opcode::DsReadB32:
+		case IR::Opcode::DsRead2B32:
 		case IR::Opcode::DsWriteByte:
 		case IR::Opcode::DsWriteShort:
 		case IR::Opcode::DsWriteB32:
+		case IR::Opcode::DsWrite2B32:
 		case IR::Opcode::DsMinF32:
 		case IR::Opcode::DsMaxF32:
 		case IR::Opcode::DsSwizzleB32:

src/graphics/shader/recompiler/frontend/translate/Translate.cpp

@@ -37,6 +37,29 @@ IR::Operand Translator::OffsetOperand(const IR::Operand& operand, uint32_t offse
 	return result;
 }
 
+IR::Operand Translator::ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset) {
+	EXIT_IF(operand.kind != IR::OperandKind::Register);
+	uint32_t code = 0;
+	switch (operand.reg.file) {
+		case IR::RegisterFile::Scalar: code = operand.reg.index; break;
+		case IR::RegisterFile::Vcc: code = 106u + operand.reg.index; break;
+		default: EXIT("invalid scalar-memory destination");
+	}
+	code += offset;
+	IR::Operand result;
+	result.kind = IR::OperandKind::Register;
+	if (code < 106u) {
+		result.reg = {IR::RegisterFile::Scalar, code};
+	} else {
+		switch (code) {
+			case 106u: result.reg = {IR::RegisterFile::Vcc, 0u}; break;
+			case 107u: result.reg = {IR::RegisterFile::Vcc, 1u}; break;
+			default: EXIT("scalar-memory destination crosses an invalid register");
+		}
+	}
+	return result;
+}
+
 IR::Operand Translator::PlainOperand(const IR::Operand& operand) {
 	auto result               = operand;
 	result.sdwa_sel           = 6;
@@ -64,26 +87,12 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) {
 	const auto address =
 	    ir.Emit(IR::ValueOpcode::IAdd64, {ir.Emit(IR::ValueOpcode::GetShaderBase),
 	                                      IR::Value(static_cast<uint64_t>(operand.imm))});
-	const auto pair = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {address});
-	return IR::U32(
-	    ir.Emit(IR::ValueOpcode::CompositeExtractU32x2,
-	            {pair, IR::Value(operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u)}));
+	return ir.CompositeExtract(address, operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u);
 }
 
 std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
-	if (current_per_invocation_masks) {
-		return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
-		        IR::U32(IR::Value(0u))};
-	}
-	const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value});
-	const auto low =
-	    IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)}));
-	// RDNA2 wave32 masks have one architectural dword; the upper half is unused.
-	const auto high =
-	    current_wave_size == 32u
-	        ? IR::U32(IR::Value(0u))
-	        : IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)}));
-	return {low, high};
+	return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
+	        IR::U32(IR::Value(0u))};
 }
 
 IR::U32 Translator::ReadRawU32(const IR::Operand& operand) {
@@ -182,22 +191,8 @@ IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) {
 		return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u)));
 	}
 	if (type == IR::Type::U64) {
-		if (operand.kind == IR::OperandKind::Register &&
-		    operand.reg.file == IR::RegisterFile::Exec) {
-			return ir.PackUint2x32(ir.GetExecLo(), ir.GetExecHi());
-		}
-		if (operand.kind == IR::OperandKind::Register &&
-		    operand.reg.file == IR::RegisterFile::Vcc) {
-			return ir.PackUint2x32(ir.GetVccLo(), ir.GetVccHi());
-		}
-		const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
-		IR::U32    high(IR::Value(0u));
-		if (operand.kind == IR::OperandKind::Register) {
-			high = ReadRawU32(OffsetOperand(operand, 1));
-		} else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) {
-			high = IR::U32(IR::Value(0xffffffffu));
-		}
-		return ir.PackUint2x32(low, high);
+		const auto pair = ReadU32Pair(operand);
+		return ir.ConstructU64(pair[0], pair[1]);
 	}
 	auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
 	if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) {
@@ -265,9 +260,17 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
 		}
 	}
 	switch (operand.reg.file) {
-		case IR::RegisterFile::Scalar:
-			ir.SetScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), value);
+		case IR::RegisterFile::Scalar: {
+			const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
+			ir.SetScalarReg(reg, value);
+			ir.SetThreadBitScalarReg(reg, ThreadBit(value));
+			ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
+			if (IR::RegIndex(reg) > 0u) {
+				ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
+				                    IR::U1(IR::Value(false)));
+			}
 			break;
+		}
 		case IR::RegisterFile::Vector: {
 			const auto reg = static_cast<IR::VectorReg>(operand.reg.index);
 			const auto old = ir.GetVectorReg(reg);
@@ -290,41 +293,26 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
 		case IR::RegisterFile::Vcc:
 			if (operand.reg.index == 0) {
 				ir.SetVccLo(IR::U32(value));
-				ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi()));
+				ir.SetVcc(ThreadBit(IR::U32(value)));
 			} else {
 				ir.SetVccHi(IR::U32(value));
-				ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value)));
+				ir.SetVcc(ThreadBit(ir.GetVccLo()));
 			}
 			break;
 		case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break;
 		case IR::RegisterFile::Exec:
 			if (operand.reg.index == 0) {
 				ir.SetExecLo(IR::U32(value));
-				ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi()));
+				ir.SetExec(ThreadBit(IR::U32(value)));
 			} else {
 				ir.SetExecHi(IR::U32(value));
-				ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value)));
+				ir.SetExec(ThreadBit(ir.GetExecLo()));
 			}
 			break;
 		case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break;
 	}
 }
 
-IR::U64 Translator::ExpandWholeQuadMask(IR::U64 value) {
-	const auto shifted1 =
-	    IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(1u)}));
-	const auto shifted2 =
-	    IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(2u)}));
-	const auto shifted3 =
-	    IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(3u)}));
-	const auto merged1 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {value, shifted1});
-	const auto merged2 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {merged1, shifted2});
-	const auto merged3 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {merged2, shifted3});
-	const auto low_bits =
-	    ir.Emit(IR::ValueOpcode::BitwiseAnd64, {merged3, IR::Value(0x1111111111111111ull)});
-	return IR::U64(ir.Emit(IR::ValueOpcode::IMul64, {low_bits, IR::Value(uint64_t {0xfull})}));
-}
-
 IR::F32 Translator::ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value) {
 	if (operand.omod != 0u) {
 		float multiplier = 0.5f;
@@ -388,31 +376,7 @@ void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) {
 		return;
 	}
 	if (type == IR::Type::U64) {
-		const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value});
-		const auto low      = ir.CompositeExtract(unpacked, 0);
-		const auto high     = ir.CompositeExtract(unpacked, 1);
-		if (operand.kind == IR::OperandKind::Register) {
-			const auto thread_bit = ThreadBit(low, high);
-			switch (operand.reg.file) {
-				case IR::RegisterFile::Exec:
-					ir.SetExec(thread_bit);
-					ir.SetExecLo(low);
-					ir.SetExecHi(high);
-					return;
-				case IR::RegisterFile::Vcc:
-					ir.SetVcc(thread_bit);
-					ir.SetVccLo(low);
-					ir.SetVccHi(high);
-					return;
-				case IR::RegisterFile::Scalar:
-					ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index),
-					                         thread_bit);
-					break;
-				default: break;
-			}
-		}
-		WriteRawU32(operand, low);
-		WriteRawU32(OffsetOperand(operand, 1), high);
+		WriteU32Pair(operand, {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)});
 		return;
 	}
 	if (type == IR::Type::F32) {
@@ -466,11 +430,44 @@ IR::U32 Translator::ReadU32(const IR::Operand& operand) {
 	return IR::U32(ReadOperand(operand, IR::Type::U32));
 }
 
+std::array<IR::U32, 2> Translator::ReadU32Pair(const IR::Operand& operand) {
+	if (operand.kind == IR::OperandKind::Register) {
+		if (operand.reg.file == IR::RegisterFile::Exec) {
+			return {ir.GetExecLo(), ir.GetExecHi()};
+		}
+		if (operand.reg.file == IR::RegisterFile::Vcc) {
+			return {ir.GetVccLo(), ir.GetVccHi()};
+		}
+	}
+	const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
+	IR::U32    high(IR::Value(0u));
+	if (operand.kind == IR::OperandKind::Register) {
+		high = ReadRawU32(OffsetOperand(operand, 1));
+	} else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) {
+		high = IR::U32(IR::Value(0xffffffffu));
+	}
+	return {low, high};
+}
+
 IR::U64 Translator::ReadU64(const IR::Operand& operand) {
 	return IR::U64(ReadOperand(operand, IR::Type::U64));
 }
 
-IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane) {
+IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed) {
+	if (operand.float_inline) {
+		const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel);
+		auto       value    = use_zero ? IR::F32(IR::Value::F32(0.0f))
+		                               : ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
+		const auto half     = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
+		value               = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
+		if (operand.absolute) {
+			value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
+		}
+		if (high_lane ? operand.negate_hi : operand.negate) {
+			value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
+		}
+		return value;
+	}
 	auto raw_operand      = operand;
 	raw_operand.sdwa_sel  = 6;
 	raw_operand.sdwa_sext = false;
@@ -552,26 +549,37 @@ IR::U32 Translator::ReadF16LaneBits(const IR::Operand& operand, bool high_lane)
 	return value;
 }
 
-std::array<IR::U32, 2> Translator::UnpackU64(IR::U64 value) {
-	const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value});
-	return {ir.CompositeExtract(unpacked, 0), ir.CompositeExtract(unpacked, 1)};
+std::array<IR::U32, 2> Translator::ExtractU64(IR::U64 value) {
+	return {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)};
 }
 
-IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) {
-	if (current_per_invocation_masks) {
-		return ir.INotEqual(low, IR::U32(IR::Value(0u)));
+void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U32, 2>& value) {
+	if (operand.kind == IR::OperandKind::Null) {
+		return;
 	}
-	const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
-	auto       word = low;
-	if (current_wave_size == 64u) {
-		const auto high_lane =
-		    IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)}));
-		word = ir.Select(high_lane, high, low);
+	if (operand.kind == IR::OperandKind::Register) {
+		const auto thread_bit = ThreadBit(value[0]);
+		switch (operand.reg.file) {
+			case IR::RegisterFile::Exec:
+				ir.SetExec(thread_bit);
+				ir.SetExecLo(value[0]);
+				ir.SetExecHi(value[1]);
+				return;
+			case IR::RegisterFile::Vcc:
+				ir.SetVcc(thread_bit);
+				ir.SetVccLo(value[0]);
+				ir.SetVccHi(value[1]);
+				return;
+			case IR::RegisterFile::Scalar: break;
+			default: break;
+		}
 	}
-	const auto bit =
-	    ir.BitwiseAnd(ir.ShiftRightLogical(word, ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)))),
-	                  IR::U32(IR::Value(1u)));
-	return ir.INotEqual(bit, IR::U32(IR::Value(0u)));
+	WriteRawU32(operand, value[0]);
+	WriteRawU32(OffsetOperand(operand, 1), value[1]);
+}
+
+IR::U1 Translator::ThreadBit(IR::U32 low) {
+	return ir.INotEqual(low, IR::U32(IR::Value(0u)));
 }
 
 IR::U1 Translator::ReadCondition(const IR::Operand& operand) {
@@ -593,11 +601,7 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
 	switch (operand.reg.file) {
 		case IR::RegisterFile::Scalar: {
 			const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
-			const auto high =
-			    current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs
-			        ? ir.GetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u))
-			        : IR::U32(IR::Value(0u));
-			return ThreadBit(ir.GetScalarReg(reg), high);
+			return ir.GetThreadBitScalarReg(reg);
 		}
 		case IR::RegisterFile::Exec: return ir.GetExec();
 		case IR::RegisterFile::Vcc: return ir.GetVcc();
@@ -606,6 +610,25 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
 	}
 }
 
+IR::U1 Translator::ReadMaskValid(const IR::Operand& operand) {
+	if (operand.kind == IR::OperandKind::ImmediateU32) {
+		const bool zero = operand.imm == 0u && !operand.sext_64;
+		const bool ones = operand.imm == 0xffffffffu && operand.sext_64;
+		return IR::U1(IR::Value(zero || ones));
+	}
+	if (operand.kind != IR::OperandKind::Register) {
+		return IR::U1(IR::Value(false));
+	}
+	switch (operand.reg.file) {
+		case IR::RegisterFile::Scalar:
+			return ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(operand.reg.index));
+		case IR::RegisterFile::Exec:
+		case IR::RegisterFile::Vcc:
+		case IR::RegisterFile::Scc: return IR::U1(IR::Value(true));
+		default: return IR::U1(IR::Value(false));
+	}
+}
+
 void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
 	EXIT_IF(operand.kind != IR::OperandKind::Register);
 	switch (operand.reg.file) {
@@ -613,10 +636,18 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
 			const auto reg  = static_cast<IR::ScalarReg>(operand.reg.index);
 			const auto mask = BallotMask(value);
 			ir.SetThreadBitScalarReg(reg, value);
+			ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
+			if (IR::RegIndex(reg) > 0u) {
+				ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
+				                    IR::U1(IR::Value(false)));
+			}
 			ir.SetScalarReg(reg, mask[0]);
 			// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
 			if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
-				ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
+				const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
+				ir.SetScalarReg(high, mask[1]);
+				ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
+				ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
 			}
 			return;
 		}
@@ -650,9 +681,17 @@ void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) {
 	const auto reg  = static_cast<IR::ScalarReg>(operand.reg.index);
 	const auto mask = BallotMask(value);
 	ir.SetThreadBitScalarReg(reg, value);
+	ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
+	if (IR::RegIndex(reg) > 0u) {
+		ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
+		                    IR::U1(IR::Value(false)));
+	}
 	ir.SetScalarReg(reg, mask[0]);
 	if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
-		ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
+		const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
+		ir.SetScalarReg(high, mask[1]);
+		ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
+		ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
 	}
 }
 
@@ -668,43 +707,13 @@ void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) {
 }
 
 bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) {
-	BufferAddressValues      buffer_address_snapshot {};
-	ScalarMemorySourceValues scalar_source_snapshot {};
-	uint32_t                 buffer_address_pc           = 0;
-	uint32_t                 scalar_source_pc            = 0;
-	bool                     has_buffer_address_snapshot = false;
-	bool                     has_scalar_source_snapshot  = false;
 	for (const auto& source_inst: source.instructions) {
 		current_opcode = source_inst.op;
 		current_pc     = source_inst.pc;
-		const bool grouped_buffer_load =
-		    IsBufferLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
-		if (grouped_buffer_load &&
-		    (!has_buffer_address_snapshot || buffer_address_pc != source_inst.pc ||
-		     source_inst.memory.component_index == 0u)) {
-			buffer_address_snapshot     = ReadBufferAddress(source_inst, 0);
-			buffer_address_pc           = source_inst.pc;
-			has_buffer_address_snapshot = true;
-		} else if (!grouped_buffer_load) {
-			has_buffer_address_snapshot = false;
-		}
-		const bool grouped_scalar_load =
-		    IsScalarMemoryLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
-		if (grouped_scalar_load &&
-		    (!has_scalar_source_snapshot || scalar_source_pc != source_inst.pc ||
-		     source_inst.memory.component_index == 0u)) {
-			scalar_source_snapshot     = ReadScalarMemorySource(source_inst);
-			scalar_source_pc           = source_inst.pc;
-			has_scalar_source_snapshot = true;
-		} else if (!grouped_scalar_load) {
-			has_scalar_source_snapshot = false;
-		}
 		if (static_cast<size_t>(source_inst.op) >= static_cast<size_t>(IR::Opcode::Count)) {
 			return Fail(error, "value IR input opcode is out of range");
 		}
-		if (TranslateInstruction(source_inst,
-		                         grouped_buffer_load ? &buffer_address_snapshot : nullptr,
-		                         grouped_scalar_load ? &scalar_source_snapshot : nullptr)) {
+		if (TranslateInstruction(source_inst)) {
 			continue;
 		}
 		return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering",
@@ -737,23 +746,17 @@ bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlock
 	if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
 		return true;
 	}
-	IR::U1     condition;
-	const auto mask_non_zero = [&](IR::U1 value) {
-		const auto mask = BallotMask(value);
-		return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)));
-	};
+	// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
+	// invocations leave the region without reconstructing a host-subgroup mask.
+	IR::U1 condition;
 	switch (source.terminator.condition) {
 		case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
 		case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
 		case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
-		case CFG::BranchCondition::VccZero:
-			condition = ir.LogicalNot(mask_non_zero(ir.GetVcc()));
-			break;
-		case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break;
-		case CFG::BranchCondition::ExecZero:
-			condition = ir.LogicalNot(mask_non_zero(ir.GetExec()));
-			break;
-		case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break;
+		case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
+		case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
+		case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
+		case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
 		case CFG::BranchCondition::GotoVariable:
 			if (source.terminator.goto_variable == UINT32_MAX) {
 				return Fail(error,
@@ -775,16 +778,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
                       const ShaderVertexInputInfo*  vertex_input_info,
                       const ShaderPixelInputInfo*   pixel_input_info,
                       const ShaderComputeInputInfo* compute_input_info, std::string* error) {
-	result                          = {};
-	result.dispatcher_fallback      = source.dispatcher_fallback;
-	const uint32_t wave_size        = source.wave_size;
-	const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
-	uint32_t   vector_limit         = 1u;
-	const auto include_vector       = [&](const IR::Operand& operand) {
+	result                        = {};
+	const uint32_t wave_size      = source.wave_size;
+	uint32_t       vector_limit   = 1u;
+	const auto     include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
 		if (operand.kind == IR::OperandKind::Register &&
 		    operand.reg.file == IR::RegisterFile::Vector) {
 			vector_limit =
-			    std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + 1u));
+			    std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + count));
 		}
 	};
 	for (const auto& block: source.blocks) {
@@ -794,6 +795,21 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
 			for (uint32_t index = 0; index < inst.src_count; index++) {
 				include_vector(inst.src[index]);
 			}
+			if (inst.op == IR::Opcode::BufferLoadDword) {
+				include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::BufferStoreDword) {
+				include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::DsReadB32 || inst.op == IR::Opcode::DsRead2B32) {
+				include_vector(inst.dst, std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::DsWriteB32) {
+				include_vector(inst.src[0], std::max(inst.memory.data_dwords, 1u));
+			} else if (inst.op == IR::Opcode::DsWrite2B32) {
+				const auto width = std::max(inst.memory.data_dwords / 2u, 1u);
+				include_vector(inst.src[0], width);
+				include_vector(inst.src[2], width);
+			} else if (inst.op == IR::Opcode::LoadInputF32) {
+				include_vector(inst.dst, std::max(inst.input_info.component_count, 1u));
+			}
 		}
 	}
 	result.block_storage.reserve(source.blocks.size() + 1u);
@@ -805,7 +821,7 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
 			    return block.id;
 		    })->id;
 		if (max_id == UINT32_MAX) {
-			return Fail(error, "cannot allocate a typed prologue block id");
+			return Fail(error, "cannot allocate a typed entry block id");
 		}
 		CFG::Terminator terminator;
 		terminator.kind       = CFG::TerminatorKind::Branch;
@@ -831,47 +847,37 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
 	}
 	if (!result.blocks.empty()) {
 		result.blocks.front()->AddBranch(result.blocks[1]);
-		IR::IREmitter prologue(result.blocks.front());
+		IR::IREmitter entry_ir(result.blocks.front());
 		const auto    builtin = [&](IR::StageInputKind kind, uint32_t component = 0u) {
 			return IR::U32(
-			    prologue.Emit(IR::ValueOpcode::GetBuiltin,
+			    entry_ir.Emit(IR::ValueOpcode::GetBuiltin,
 			                  {IR::Value(static_cast<uint32_t>(kind)), IR::Value(component)}));
 		};
-		prologue.Emit(IR::ValueOpcode::Prologue);
-		const IR::U32 zero(IR::Value(0u));
-		for (uint32_t index = 0; index < IR::NumScalarRegs; index++) {
-			prologue.SetScalarReg(static_cast<IR::ScalarReg>(index), zero);
-		}
-		for (uint32_t index = 0; index < vector_limit; index++) {
-			prologue.SetVectorReg(static_cast<IR::VectorReg>(index), zero);
-		}
 		for (uint32_t index = 0; index < source.user_data_count; index++) {
 			const auto reg = static_cast<IR::ScalarReg>(source.user_data_base + index);
 			if (IR::RegIndex(reg) >= IR::NumScalarRegs) {
 				break;
 			}
-			prologue.SetScalarReg(reg, prologue.GetUserData(reg));
+			const auto value = entry_ir.GetUserData(reg);
+			entry_ir.SetScalarReg(reg, value);
+			entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u))));
+			entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
 		}
-		prologue.SetScc(IR::U1(IR::Value(false)));
-		prologue.SetExec(IR::U1(IR::Value(true)));
-		prologue.SetExecLo(IR::U32(IR::Value(0xffffffffu)));
-		prologue.SetExecHi(IR::U32(IR::Value(source.wave_size > 32u ? 0xffffffffu : 0u)));
-		prologue.SetVcc(IR::U1(IR::Value(false)));
-		prologue.SetVccLo(IR::U32(IR::Value(0u)));
-		prologue.SetVccHi(IR::U32(IR::Value(0u)));
-		prologue.SetM0(IR::U32(IR::Value(0u)));
+		entry_ir.SetExec(IR::U1(IR::Value(true)));
+		entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
+		entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
 		if (source.stage == ShaderType::Compute) {
 			const auto* cs = compute_input_info;
 			const auto  thread_ids =
 			    cs->thread_ids_num > 0 ? std::min<uint32_t>(cs->thread_ids_num, 3u) : 0u;
 			for (uint32_t index = 0; index < thread_ids; index++) {
-				prologue.SetVectorReg(static_cast<IR::VectorReg>(index),
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(index),
 				                      builtin(IR::StageInputKind::LocalInvocationId, index));
 			}
 			uint32_t reg_offset = 0;
 			for (uint32_t index = 0; index < 3u; index++) {
 				if (cs->group_id[index]) {
-					prologue.SetScalarReg(
+					entry_ir.SetScalarReg(
 					    static_cast<IR::ScalarReg>(cs->workgroup_register + reg_offset++),
 					    builtin(IR::StageInputKind::WorkgroupId, index));
 				}
@@ -884,49 +890,55 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
 				const auto waves = std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu);
 				const auto local_index = builtin(IR::StageInputKind::LocalInvocationIndex);
 				const auto wave_id     = IR::U32(
-				    prologue.Emit(IR::ValueOpcode::UDiv32, {local_index, IR::Value(wave_size)}));
-				const auto wave_bits = prologue.ShiftLeftLogical(wave_id, IR::U32(IR::Value(20u)));
+				    entry_ir.Emit(IR::ValueOpcode::UDiv32, {local_index, IR::Value(wave_size)}));
+				const auto wave_bits = entry_ir.ShiftLeftLogical(wave_id, IR::U32(IR::Value(20u)));
 				const auto first_bit =
-				    prologue.Select(prologue.IEqual(wave_id, IR::U32(IR::Value(0u))),
+				    entry_ir.Select(entry_ir.IEqual(wave_id, IR::U32(IR::Value(0u))),
 				                    IR::U32(IR::Value(0x80000000u)), IR::U32(IR::Value(0u)));
-				prologue.SetScalarReg(
+				entry_ir.SetScalarReg(
 				    static_cast<IR::ScalarReg>(cs->workgroup_register + reg_offset),
-				    prologue.BitwiseOr(prologue.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))),
+				    entry_ir.BitwiseOr(entry_ir.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))),
 				                       first_bit));
 			}
 		} else if (source.stage == ShaderType::Pixel) {
-			const auto* ps  = pixel_input_info;
-			uint32_t    reg = ps->ps_system_input_base;
+			const auto* ps = pixel_input_info;
+			if (ps->ps_perspective_center_vgpr != UINT32_MAX) {
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(ps->ps_perspective_center_vgpr),
+				                      builtin(IR::StageInputKind::BaryCoordSmooth, 0));
+				entry_ir.SetVectorReg(
+				    static_cast<IR::VectorReg>(ps->ps_perspective_center_vgpr + 1u),
+				    builtin(IR::StageInputKind::BaryCoordSmooth, 1));
+			}
+			uint32_t reg = ps->ps_system_input_base;
 			if (ps->ps_pos_x) {
-				prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(reg++),
 				                      builtin(IR::StageInputKind::FragCoord, 0));
 			}
 			if (ps->ps_pos_y) {
-				prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(reg++),
 				                      builtin(IR::StageInputKind::FragCoord, 1));
 			}
 			if (ps->ps_pos_z) {
-				prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(reg++),
 				                      builtin(IR::StageInputKind::FragCoord, 2));
 			}
 			if (ps->ps_pos_w) {
-				prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(reg++),
 				                      builtin(IR::StageInputKind::FragCoord, 3));
 			}
 			if (ps->ps_front_face) {
-				prologue.SetVectorReg(static_cast<IR::VectorReg>(reg),
+				entry_ir.SetVectorReg(static_cast<IR::VectorReg>(reg),
 				                      builtin(IR::StageInputKind::FrontFacing));
 			}
 		} else if (source.stage == ShaderType::Vertex) {
-			prologue.SetVectorReg(static_cast<IR::VectorReg>(5),
+			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(5),
 			                      builtin(IR::StageInputKind::VertexIndex));
-			prologue.SetVectorReg(static_cast<IR::VectorReg>(8),
+			entry_ir.SetVectorReg(static_cast<IR::VectorReg>(8),
 			                      builtin(IR::StageInputKind::InstanceIndex));
 		}
 	}
 	for (size_t index = 0; index < source.blocks.size(); index++) {
-		Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size,
-		                              per_invocation_masks);
+		Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size);
 		if (!translator.TranslateBlock(source.blocks[index], error) ||
 		    !translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u],
 		                                   error)) {

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -7,23 +7,12 @@
 
 namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
 
-struct BufferAddressValues {
-	IR::U32 index;
-	IR::U32 offset;
-	IR::U32 soffset;
-};
-
-struct ScalarMemorySourceValues {
-	IR::Value resource;
-	IR::U32   offset;
-};
-
 class Translator {
 public:
 	Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
-	           uint32_t wave_size, bool per_invocation_masks)
+	           uint32_t wave_size)
 	    : value_program(program), ir(block), current_vector_limit(vector_limit),
-	      current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {}
+	      current_wave_size(wave_size) {}
 
 	bool TranslateBlock(const IR::BasicBlock& source, std::string* error);
 	bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
@@ -31,6 +20,7 @@ public:
 
 private:
 	IR::Operand            OffsetOperand(const IR::Operand& operand, uint32_t offset);
+	IR::Operand            ScalarDestinationOperand(const IR::Operand& operand, uint32_t offset);
 	IR::Operand            PlainOperand(const IR::Operand& operand);
 	IR::U32                ReadPcRelativeU32(const IR::Operand& operand);
 	std::array<IR::U32, 2> BallotMask(IR::U1 value);
@@ -38,30 +28,32 @@ private:
 	IR::U32                ReadScalarCode(uint32_t code);
 	IR::U32                ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value);
 	IR::Value              ReadOperand(const IR::Operand& operand, IR::Type type);
-	IR::U1                 ThreadBit(IR::U32 low, IR::U32 high);
+	IR::U1                 ThreadBit(IR::U32 low);
 	void                   WriteRawU32(const IR::Operand& operand, IR::U32 value);
-	IR::U64                ExpandWholeQuadMask(IR::U64 value);
 	IR::F32                ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value);
 	void                   WriteOperand(const IR::Operand& operand, IR::Value value);
 	IR::U32                PackHalf2x16(IR::F32 low, IR::F32 high);
 	void                   WriteF16(const IR::Operand& operand, IR::F32 value);
 	void                   WriteU16(const IR::Operand& operand, IR::U32 value);
 	IR::U32                ReadU32(const IR::Operand& operand);
+	std::array<IR::U32, 2> ReadU32Pair(const IR::Operand& operand);
 	IR::U64                ReadU64(const IR::Operand& operand);
-	IR::F32                ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane);
-	IR::F32                ReadF16AsF32(const IR::Operand& operand);
-	IR::F32                ReadMixF32(const IR::Operand& operand);
-	IR::U32                ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
+	IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false);
+	IR::F32 ReadF16AsF32(const IR::Operand& operand);
+	IR::F32 ReadMixF32(const IR::Operand& operand);
+	IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
 	IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
 	IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
 	IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
-	std::array<IR::U32, 2> UnpackU64(IR::U64 value);
-	IR::U1                 ReadCondition(const IR::Operand& operand);
-	IR::U32                ConditionBit(const IR::Operand& operand);
-	IR::U1                 ReadMask(const IR::Operand& operand);
-	void                   WriteMask(const IR::Operand& operand, IR::U1 value);
-	void                   WriteMask64(const IR::Operand& operand, IR::U1 value);
-	void                   WriteCompareResult(const IR::Operand& operand, IR::U1 value);
+	std::array<IR::U32, 2> ExtractU64(IR::U64 value);
+	void    WriteU32Pair(const IR::Operand& operand, const std::array<IR::U32, 2>& value);
+	IR::U1  ReadCondition(const IR::Operand& operand);
+	IR::U32 ConditionBit(const IR::Operand& operand);
+	IR::U1  ReadMask(const IR::Operand& operand);
+	IR::U1  ReadMaskValid(const IR::Operand& operand);
+	void    WriteMask(const IR::Operand& operand, IR::U1 value);
+	void    WriteMask64(const IR::Operand& operand, IR::U1 value);
+	void    WriteCompareResult(const IR::Operand& operand, IR::U1 value);
 
 	IR::MemoryFlags AddMemoryInfo(const IR::Instruction& inst);
 	IR::ExportFlags AddExportInfo(const IR::Instruction& inst);
@@ -70,6 +62,11 @@ private:
 		IR::Value low;
 		IR::Value high;
 	};
+	struct BufferAddress {
+		IR::U32 index;
+		IR::U32 offset;
+		IR::U32 soffset;
+	};
 	AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source);
 	IR::U32         GetResourceDword(uint32_t index, uint32_t dword);
 	IR::Value       GetBufferResource(const IR::MemoryInfo& memory);
@@ -77,30 +74,24 @@ private:
 	IR::Value       GetScalarAddressResource(uint32_t base);
 	IR::Value       GetImageResource(const IR::MemoryInfo& memory);
 	IR::Value       GetSamplerResource(const IR::MemoryInfo& memory);
-	IR::Value       GetSharedResource(IR::ResourceKind kind);
 	IR::Value       MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
 	IR::Value       ConstructU32x4(const IR::Operand& base, uint32_t count);
-	void            WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
-	                                     uint32_t component_limit);
+	void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory,
+	                          uint32_t component_limit);
 	IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
-	BufferAddressValues      ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
-	IR::U32                  WidenSubdword(IR::Value value, uint32_t bits, bool sign);
-	IR::Value                NarrowSubdword(IR::U32 value, uint32_t bits);
-	IR::ValueOpcode          BufferAtomicOpcode(IR::Opcode opcode);
-	IR::ValueOpcode          SharedAtomicOpcode(IR::Opcode opcode, bool gds);
-	bool                     IsBufferLoadOperation(IR::Opcode opcode);
-	bool                     IsScalarMemoryLoadOperation(IR::Opcode opcode);
-	ScalarMemorySourceValues ReadScalarMemorySource(const IR::Instruction& inst);
-	bool                     TranslateScalarMemory(const IR::Instruction&          inst,
-	                                               const ScalarMemorySourceValues* source_snapshot);
-	bool                     TranslateBufferLoad(const IR::Instruction&     inst,
-	                                             const BufferAddressValues* address_snapshot);
-	bool                     TranslateBufferStore(const IR::Instruction& inst);
-	bool                     TranslateAtomicMemory(const IR::Instruction& inst);
-	bool                     TranslateFlatLoad(const IR::Instruction& inst);
-	bool                     TranslateFlatStore(const IR::Instruction& inst);
-	bool                     TranslateImageMemory(const IR::Instruction& inst);
-	bool                     TranslateSharedMemory(const IR::Instruction& inst);
+	BufferAddress   ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
+	IR::U32         WidenSubdword(IR::Value value, uint32_t bits, bool sign);
+	IR::Value       NarrowSubdword(IR::U32 value, uint32_t bits);
+	IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
+	IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode);
+	bool            TranslateScalarMemory(const IR::Instruction& inst);
+	bool            TranslateBufferLoad(const IR::Instruction& inst);
+	bool            TranslateBufferStore(const IR::Instruction& inst);
+	bool            TranslateAtomicMemory(const IR::Instruction& inst);
+	bool            TranslateFlatLoad(const IR::Instruction& inst);
+	bool            TranslateFlatStore(const IR::Instruction& inst);
+	bool            TranslateImageMemory(const IR::Instruction& inst);
+	bool            TranslateSharedMemory(const IR::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
@@ -110,39 +101,34 @@ private:
 	                                 uint32_t upper_result);
 	IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
 
-	bool TranslateInstruction(const IR::Instruction&          inst,
-	                          const BufferAddressValues*      address_snapshot,
-	                          const ScalarMemorySourceValues* scalar_source_snapshot);
-	bool TranslateStateOperation(const IR::Instruction& inst);
-	bool TranslateControlOperation(const IR::Instruction& inst);
-	bool TranslateMove(const IR::Instruction& inst);
-	bool TranslateLaneOperation(const IR::Instruction& inst);
-	bool TranslateAttributeOperation(const IR::Instruction& inst);
-	bool TranslateMemoryOperation(const IR::Instruction&          inst,
-	                              const BufferAddressValues*      address_snapshot,
-	                              const ScalarMemorySourceValues* scalar_source_snapshot);
-	bool TranslateIntegerCompare(const IR::Instruction& inst);
-	bool TranslateInteger16Compare(const IR::Instruction& inst);
-	bool TranslateFloatCompare(const IR::Instruction& inst);
-	bool TranslateConversion(const IR::Instruction& inst);
-	bool TranslateInteger16Operation(const IR::Instruction& inst);
-	bool TranslatePackedInteger16(const IR::Instruction& inst);
-	bool TranslatePackedFloat16(const IR::Instruction& inst);
-	bool TranslateFloat16Operation(const IR::Instruction& inst);
-	bool TranslateFloatOperation(const IR::Instruction& inst);
-	bool TranslatePerInvocationU64Mask(const IR::Instruction& inst);
-	bool TranslateU64MaskOperation(const IR::Instruction& inst);
-	bool TranslateSimpleInteger(const IR::Instruction& inst);
-	bool TranslateComposedInteger(const IR::Instruction& inst);
-	bool TranslateExtendedInteger(const IR::Instruction& inst);
+	bool   TranslateInstruction(const IR::Instruction& inst);
+	bool   TranslateStateOperation(const IR::Instruction& inst);
+	bool   TranslateControlOperation(const IR::Instruction& inst);
+	bool   TranslateMove(const IR::Instruction& inst);
+	bool   TranslateLaneOperation(const IR::Instruction& inst);
+	bool   TranslateAttributeOperation(const IR::Instruction& inst);
+	bool   TranslateMemoryOperation(const IR::Instruction& inst);
+	bool   TranslateIntegerCompare(const IR::Instruction& inst);
+	bool   TranslateInteger16Compare(const IR::Instruction& inst);
+	bool   TranslateFloatCompare(const IR::Instruction& inst);
+	bool   TranslateConversion(const IR::Instruction& inst);
+	bool   TranslateInteger16Operation(const IR::Instruction& inst);
+	bool   TranslatePackedInteger16(const IR::Instruction& inst);
+	bool   TranslatePackedFloat16(const IR::Instruction& inst);
+	bool   TranslateFloat16Operation(const IR::Instruction& inst);
+	bool   TranslateFloatOperation(const IR::Instruction& inst);
+	IR::U1 EvaluateU64Mask(const IR::Instruction& inst);
+	bool   TranslateU64MaskOperation(const IR::Instruction& inst);
+	bool   TranslateSimpleInteger(const IR::Instruction& inst);
+	bool   TranslateComposedInteger(const IR::Instruction& inst);
+	bool   TranslateExtendedInteger(const IR::Instruction& inst);
 
 	IR::ValueProgram& value_program;
 	IR::IREmitter     ir;
-	IR::Opcode        current_opcode               = IR::Opcode::ControlNop;
-	uint32_t          current_pc                   = 0;
-	uint32_t          current_vector_limit         = 1;
-	uint32_t          current_wave_size            = 64;
-	bool              current_per_invocation_masks = false;
+	IR::Opcode        current_opcode       = IR::Opcode::ControlNop;
+	uint32_t          current_pc           = 0;
+	uint32_t          current_vector_limit = 1;
+	uint32_t          current_wave_size    = 64;
 };
 
 } // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

src/graphics/shader/recompiler/ir/Block.h

@@ -49,6 +49,7 @@ public:
 
 	std::array<Value, NumScalarRegs> ssa_sreg_values {};
 	std::array<Value, NumScalarRegs> ssa_thread_bit_sreg_values {};
+	std::array<Value, NumScalarRegs> ssa_sreg_mask_tags {};
 	std::array<Value, NumVectorRegs> ssa_vreg_values {};
 
 private:

src/graphics/shader/recompiler/ir/IREmitter.cpp

@@ -36,6 +36,14 @@ void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) {
 	Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value});
 }
 
+U1 IREmitter::GetScalarMaskTag(ScalarReg reg) {
+	return U1(Emit(ValueOpcode::GetScalarMaskTag, {Value(reg)}));
+}
+
+void IREmitter::SetScalarMaskTag(ScalarReg reg, U1 value) {
+	Emit(ValueOpcode::SetScalarMaskTag, {Value(reg), value});
+}
+
 U32 IREmitter::GetVectorReg(VectorReg reg) {
 	return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)}));
 }
@@ -134,13 +142,20 @@ U32 IREmitter::BitCastU32(F16 value) {
 	return U32(Emit(ValueOpcode::ConvertU32U16, {half}));
 }
 
-U64 IREmitter::PackUint2x32(U32 low, U32 high) {
-	const auto pair = Emit(ValueOpcode::CompositeConstructU32x2, {low, high});
-	return U64(Emit(ValueOpcode::PackUint2x32, {pair}));
+U64 IREmitter::ConstructU64(U32 low, U32 high) {
+	return U64(Emit(ValueOpcode::CompositeConstructU64, {low, high}));
 }
 
 U32 IREmitter::CompositeExtract(Value composite, uint32_t index) {
-	return U32(Emit(ValueOpcode::CompositeExtractU32x2, {composite, Value(index)}));
+	ValueOpcode opcode;
+	switch (composite.GetType()) {
+		case Type::U64: opcode = ValueOpcode::CompositeExtractU64; break;
+		case Type::U32x2: opcode = ValueOpcode::CompositeExtractU32x2; break;
+		case Type::U32x3: opcode = ValueOpcode::CompositeExtractU32x3; break;
+		case Type::U32x4: opcode = ValueOpcode::CompositeExtractU32x4; break;
+		default: EXIT("invalid U32 composite type\n");
+	}
+	return U32(Emit(opcode, {composite, Value(index)}));
 }
 
 U32 IREmitter::IAdd(U32 lhs, U32 rhs) {

src/graphics/shader/recompiler/ir/IREmitter.h

@@ -29,6 +29,8 @@ public:
 	void SetScalarReg(ScalarReg reg, U32 value);
 	U1   GetThreadBitScalarReg(ScalarReg reg);
 	void SetThreadBitScalarReg(ScalarReg reg, U1 value);
+	U1   GetScalarMaskTag(ScalarReg reg);
+	void SetScalarMaskTag(ScalarReg reg, U1 value);
 	U32  GetVectorReg(VectorReg reg);
 	void SetVectorReg(VectorReg reg, U32 value);
 	U1   GetGotoVariable(uint32_t id);
@@ -54,7 +56,7 @@ public:
 	U32 BitCastU32(F32 value);
 	F16 BitCastF16(U32 value);
 	U32 BitCastU32(F16 value);
-	U64 PackUint2x32(U32 low, U32 high);
+	U64 ConstructU64(U32 low, U32 high);
 	U32 CompositeExtract(Value composite, uint32_t index);
 
 	U32 IAdd(U32 lhs, U32 rhs);

src/graphics/shader/recompiler/ir/Reg.h

@@ -7,10 +7,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
 enum class ScalarReg : uint16_t {};
 enum class VectorReg : uint16_t {};
 
-// Architectural SGPRs occupy the low range. The current shader-only migration also accepts the
-// scalar shadow registers produced by Kyty's pre-existing read-lane pass; that pass disappears
-// once read-lane elimination operates on the SSA graph.
-constexpr uint32_t NumScalarRegs = 512;
+constexpr uint32_t NumScalarRegs = 106;
 constexpr uint32_t NumVectorRegs = 256;
 
 constexpr uint32_t RegIndex(ScalarReg reg) {

src/graphics/shader/recompiler/ir/ShaderIR.cpp

@@ -44,6 +44,23 @@ bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBloc
 	if (!ScalarResultWritesSccNonZero(decoded.opcode)) {
 		return true;
 	}
+	switch (decoded.opcode) {
+		case Decoder::Opcode::S_AND_B64:
+		case Decoder::Opcode::S_ANDN2_B64:
+		case Decoder::Opcode::S_NOT_B64:
+		case Decoder::Opcode::S_OR_B64:
+		case Decoder::Opcode::S_ORN2_B64:
+		case Decoder::Opcode::S_XOR_B64:
+		case Decoder::Opcode::S_NAND_B64:
+		case Decoder::Opcode::S_NOR_B64:
+		case Decoder::Opcode::S_XNOR_B64:
+		case Decoder::Opcode::S_WQM_B64:
+			// These instructions maintain an invocation-local mask shadow and update SCC
+			// while lowering the operation. A raw dword comparison would disagree after
+			// complementing a mask temporary.
+			return true;
+		default: break;
+	}
 	if (block.instructions.empty()) {
 		SetError(error, "internal IR error: missing scalar result for SCC update");
 		return false;
@@ -157,10 +174,11 @@ bool LowerSourceOperand(const Decoder::Operand& decoded, Operand& operand, std::
 		case Decoder::OperandKind::FloatInlineConstant:
 			operand = {};
 			CopyOperandModifiers(decoded, operand);
-			operand.kind    = OperandKind::ImmediateU32;
-			operand.imm     = decoded.value;
-			operand.sext_64 = decoded.kind == Decoder::OperandKind::IntegerInlineConstant &&
-			                  decoded.signed_val < 0;
+			operand.kind         = OperandKind::ImmediateU32;
+			operand.imm          = decoded.value;
+			operand.float_inline = decoded.kind == Decoder::OperandKind::FloatInlineConstant;
+			operand.sext_64      = decoded.kind == Decoder::OperandKind::IntegerInlineConstant &&
+			                       decoded.signed_val < 0;
 			return true;
 		case Decoder::OperandKind::Null:
 		case Decoder::OperandKind::PopsExitingWaveId:
@@ -205,13 +223,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) {
 
 MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) {
 	MemoryInfo mem;
-	mem.kind                     = kind;
-	mem.offset                   = decoded.offset;
-	mem.secondary_offset         = decoded.secondary_offset;
-	mem.dmask                    = decoded.dmask;
-	mem.data_dwords              = decoded.data_dwords;
-	mem.data_bits                = decoded.data_bits;
-	mem.component_count          = decoded.data_dwords;
+	mem.kind             = kind;
+	mem.offset           = decoded.offset;
+	mem.secondary_offset = decoded.secondary_offset;
+	mem.dmask            = decoded.dmask;
+	mem.data_dwords      = decoded.data_dwords;
+	mem.data_bits        = decoded.data_bits;
+	mem.component_count =
+	    decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords;
 	mem.data_format              = decoded.data_format;
 	mem.number_format            = decoded.number_format;
 	mem.image_sample_flags       = decoded.image_sample_flags;
@@ -223,20 +242,21 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
 	}
 	mem.memory_segment = decoded.memory_segment;
 	mem.address_is_full =
-	    kind == ResourceKind::Flat || kind == ResourceKind::Scratch ||
+	    kind == ResourceKind::Flat ||
 	    (kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
 	mem.data_signed   = decoded.data_signed;
 	mem.typed         = decoded.typed;
 	mem.formatted     = decoded.formatted;
 	mem.image_has_mip = decoded.opcode == Decoder::Opcode::IMAGE_LOAD_MIP ||
 	                    decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP;
+	mem.image_r128    = decoded.image_r128;
 	mem.glc           = decoded.glc;
 	mem.slc           = decoded.slc;
 	mem.idxen         = decoded.idxen;
 	mem.offen         = decoded.offen;
 	mem.resource      = ResourceIndexFromOperand(decoded.src1);
 	mem.sampler       = ResourceIndexFromOperand(decoded.src2);
-	if (kind == ResourceKind::Lds || kind == ResourceKind::Gds) {
+	if (kind == ResourceKind::Lds || kind == ResourceKind::Gds || kind == ResourceKind::Scratch) {
 		mem.resource = 0;
 		mem.sampler  = 0;
 	}
@@ -273,7 +293,7 @@ void ClearRegisterOffsetModifiers(Decoder::Operand& operand) {
 	operand.dpp                = false;
 }
 
-bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code) {
+bool TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code) {
 	switch (operand.kind) {
 		case Decoder::OperandKind::Sgpr: code = operand.reg; return true;
 		case Decoder::OperandKind::VccLo: code = SCALAR_DST_VCC_LO; return true;
@@ -290,7 +310,7 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
 	// aliases such as VCC and EXEC sit beside ordinary SGPRs. Re-decode after adding the
 	// dword index so s_buffer_load_dwordx2 vcc_lo writes vcc_lo, then vcc_hi.
 	uint32_t code = 0;
-	return TryGetScalarDestinationCode(operand, code) &&
+	return TryGetEncodedScalarCode(operand, code) &&
 	       Decoder::DecodeScalarDestination(code + index, 0, decoded, nullptr);
 }
 
@@ -328,54 +348,6 @@ MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind ki
 	return mem;
 }
 
-uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
-                                         uint32_t                    component_index) {
-	if (!decoded.typed || !decoded.formatted) {
-		return component_index * 4u;
-	}
-
-	const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format);
-	if (!Format::IsKnownFormat(format)) {
-		return component_index * 4u;
-	}
-	return Format::GetFormatComponentByteOffset(format, component_index);
-}
-
-uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded) {
-	if (!decoded.typed || !decoded.formatted) {
-		return decoded.data_dwords;
-	}
-
-	const auto format = Format::DecodeTBufferFormat(decoded.data_format, decoded.number_format);
-	if (!Format::IsKnownFormat(format)) {
-		return decoded.data_dwords;
-	}
-	return Format::GetFormatComponentCount(format);
-}
-
-bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
-                           BasicBlock& block, std::string* error) {
-	Instruction inst;
-	inst.pc        = pc;
-	inst.op        = Opcode::MoveU32;
-	inst.src[0]    = MakeImmediateU32(value);
-	inst.src_count = 1;
-	if (!LowerRegisterOperand(dst, inst.dst, error)) {
-		return false;
-	}
-	block.instructions.push_back(inst);
-	return true;
-}
-
-MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index) {
-	auto mem = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
-	mem.offset += TypedBufferComponentOffsetBytes(decoded, component_index);
-	mem.data_dwords     = 1;
-	mem.component_index = component_index;
-	mem.component_count = decoded.data_dwords;
-	return mem;
-}
-
 MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
                                 uint32_t byte_offset) {
 	auto mem             = MemoryInfoFromDecoded(decoded, kind);
@@ -894,9 +866,9 @@ bool LowerVInterpP1F32(const Decoder::Instruction& decoded, BasicBlock& block) {
 
 bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock& block,
                          std::string* error) {
-	if (decoded.opcode == Decoder::Opcode::V_INTERP_MOV_F32 && decoded.src0.value != 2u) {
+	if (decoded.opcode == Decoder::Opcode::V_INTERP_MOV_F32 && decoded.src0.value >= 3u) {
 		if (error != nullptr) {
-			*error = fmt::format("v_interp_mov_f32 mode {} is not implemented at pc 0x{:08x}",
+			*error = fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}",
 			                     decoded.src0.value, decoded.pc);
 		}
 		return false;
@@ -907,6 +879,9 @@ bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock& block,
 	inst.op              = Opcode::LoadInputF32;
 	inst.input_info.attr = decoded.src1.value;
 	inst.input_info.chan = decoded.src2.value;
+	if (decoded.opcode == Decoder::Opcode::V_INTERP_MOV_F32) {
+		inst.input_info.interpolation_mode = decoded.src0.value;
+	}
 	if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) {
 		return false;
 	}
@@ -1229,12 +1204,11 @@ bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, Shader
 		EXIT("invalid shader wave size: %u\n", wave_size);
 	}
 
-	program                     = {};
-	program.stage               = stage;
-	program.wave_size           = wave_size;
-	program.dispatcher_fallback = cfg.irreducible;
-	program.cfg_failure_kind    = cfg.failure_kind;
-	program.fallback_reason     = cfg.unsupported_reason;
+	program                  = {};
+	program.stage            = stage;
+	program.wave_size        = wave_size;
+	program.cfg_failure_kind = cfg.failure_kind;
+	program.fallback_reason  = cfg.unsupported_reason;
 	program.blocks.clear();
 
 	if (cfg.blocks.empty()) {

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -7,10 +7,13 @@
 #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
 #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 #include "graphics/shader/recompiler/ir/opcodes/Opcodes.h"
+#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h"
 #include "graphics/shader/shader.h"
 
 #include <array>
+#include <bit>
 #include <memory>
+#include <optional>
 #include <string_view>
 #include <vector>
 
@@ -35,6 +38,7 @@ struct Operand {
 	OperandKind kind = OperandKind::Null;
 	Register    reg;
 	uint32_t    imm                = 0;
+	bool        float_inline       = false;
 	bool        sext_64            = false;
 	uint32_t    sdwa_sel           = 6;
 	uint32_t    sdwa_dst_unused    = 2;
@@ -73,6 +77,24 @@ enum class ResourceKind {
 	Sampler
 };
 
+[[nodiscard]] constexpr bool IsAddressResourceKind(ResourceKind kind) {
+	return kind == ResourceKind::ScalarAddress || kind == ResourceKind::Flat ||
+	       kind == ResourceKind::Global || kind == ResourceKind::Scratch;
+}
+
+[[nodiscard]] constexpr bool ImageResourceKindMatches(ResourceKind       kind,
+                                                      ImageResourceClass resource_class) {
+	switch (resource_class) {
+		case ImageResourceClass::Sampled:
+			return kind == ResourceKind::Image || kind == ResourceKind::ImageUint;
+		case ImageResourceClass::Storage:
+			return kind == ResourceKind::StorageImage || kind == ResourceKind::StorageImageUint;
+		case ImageResourceClass::StorageUint: return kind == ResourceKind::StorageImageUint;
+		case ImageResourceClass::None: return false;
+	}
+	return false;
+}
+
 struct MemoryInfo {
 	ResourceKind            kind                     = ResourceKind::None;
 	uint32_t                resource                 = 0;
@@ -98,6 +120,7 @@ struct MemoryInfo {
 	bool                    formatted                                             = false;
 	bool                    image_has_mip                                         = false;
 	bool                    image_cube                                            = false;
+	bool                    image_r128                                            = false;
 	bool                    glc                                                   = false;
 	bool                    slc                                                   = false;
 	bool                    idxen                                                 = false;
@@ -122,8 +145,10 @@ struct ExportInfo {
 };
 
 struct InputInfo {
-	uint32_t attr = 0;
-	uint32_t chan = 0;
+	uint32_t attr               = 0;
+	uint32_t chan               = 0;
+	uint32_t component_count    = 1;
+	uint32_t interpolation_mode = 3;
 
 	bool operator==(const InputInfo& other) const = default;
 };
@@ -187,7 +212,7 @@ struct BufferResource {
 
 enum class ImageMipMode { None, DynamicStorage };
 
-constexpr uint32_t StorageImageIdentitySwizzle = 0x00000facu;
+constexpr uint32_t ShaderImageIdentitySwizzle = 0x00000facu;
 
 struct ImageResource {
 	static constexpr uint32_t NoIndirectImage = UINT32_MAX;
@@ -198,12 +223,14 @@ struct ImageResource {
 	Decoder::ImageDimension dimension                 = Decoder::ImageDimension::Unknown;
 	ImageMipMode            mip_mode                  = ImageMipMode::None;
 	uint32_t                mip_count                 = 1;
-	uint32_t                storage_swizzle           = StorageImageIdentitySwizzle;
+	Prospero::BufferFormat  conversion_format         = Prospero::BufferFormat::kInvalid;
+	uint32_t                shader_swizzle            = ShaderImageIdentitySwizzle;
 	bool                    read                      = false;
 	bool                    written                   = false;
 	bool                    atomic                    = false;
 	bool                    depth_compare             = false;
 	bool                    cube                      = false;
+	bool                    r128                      = false;
 	uint32_t                indirect_root             = NoIndirectImage;
 	uint32_t                indirect_mapping_offset   = 0;
 	uint32_t                indirect_mapping_capacity = 0;
@@ -213,8 +240,9 @@ struct ImageResource {
 };
 
 struct SamplerResource {
-	uint32_t source       = 0;
-	uint32_t first_use_pc = 0;
+	uint32_t source                = 0;
+	uint32_t first_use_pc          = 0;
+	bool     force_point_filtering = false;
 
 	bool operator==(const SamplerResource& other) const = default;
 };
@@ -246,6 +274,8 @@ enum class StageInputKind {
 	InstanceIndex,
 	FragCoord,
 	FrontFacing,
+	BaryCoordSmooth,
+	BaryCoordNoPerspective,
 	WorkgroupId,
 	LocalInvocationId,
 	LocalInvocationIndex,
@@ -253,13 +283,77 @@ enum class StageInputKind {
 	Parameter,
 };
 
-enum class StageOutputKind { Position, Parameter, Mrt, Depth, SampleMask };
+enum class StageOutputKind {
+	Position,
+	Parameter,
+	Mrt,
+	Depth,
+	SampleMask,
+	PointSize,
+	ClipDistance,
+	CullDistance,
+	Layer
+};
+
+struct PositionExportComponent {
+	uint32_t clip_distance = UINT32_MAX;
+	uint32_t cull_distance = UINT32_MAX;
+	bool     valid          = false;
+	bool     point_size     = false;
+	bool     layer          = false;
+	bool     viewport       = false;
+};
+
+inline PositionExportComponent DecodePositionExportComponent(uint32_t control,
+	                                                           uint32_t pos_index,
+	                                                           uint32_t component) {
+	PositionExportComponent result;
+	if (pos_index == 0 || component >= 4) {
+		return result;
+	}
+
+	uint32_t slot   = pos_index - 1;
+	uint32_t vector = 3;
+	for (uint32_t i = 0; i < 3; i++) {
+		if ((control & (1u << (21u + i))) != 0) {
+			if (slot == 0) {
+				vector = i;
+				break;
+			}
+			slot--;
+		}
+	}
+	if (vector == 3) {
+		return result;
+	}
+
+	result.valid = true;
+	if (vector == 0) {
+		result.point_size = component == 0 && (control & (1u << 16u)) != 0;
+		result.layer      = component == 2 && (control & (1u << 18u)) != 0;
+		result.viewport   = component == 2 && (control & (1u << 19u)) != 0;
+		return result;
+	}
+
+	const auto scalar = (vector - 1) * 4 + component;
+	const auto lower  = (1u << scalar) - 1u;
+	const auto clip   = control & 0xffu;
+	const auto cull   = (control >> 8u) & 0xffu;
+	if ((clip & (1u << scalar)) != 0) {
+		result.clip_distance = std::popcount(clip & lower);
+	}
+	if ((cull & (1u << scalar)) != 0) {
+		result.cull_distance = std::popcount(cull & lower);
+	}
+	return result;
+}
 
 struct StageInput {
 	StageInputKind kind            = StageInputKind::VertexIndex;
 	uint32_t       location        = 0;
 	uint32_t       component_count = 1;
 	std::string    debug_name;
+	bool           per_vertex = false;
 
 	bool operator==(const StageInput& other) const = default;
 };
@@ -299,6 +393,11 @@ enum class DescriptorBindingKind {
 	StorageUint2D,
 	StorageUint2DArray,
 	StorageUint3D,
+	StorageAtomic1D,
+	StorageAtomic1DArray,
+	StorageAtomic2D,
+	StorageAtomic2DArray,
+	StorageAtomic3D,
 	Samplers,
 	Gds,
 	AddressMemory,
@@ -307,25 +406,103 @@ enum class DescriptorBindingKind {
 	Count,
 };
 
+constexpr uint32_t NativePushConstantSize = 128;
+
+[[nodiscard]] constexpr uint32_t NativeBinding(ShaderType stage, DescriptorBindingKind kind) {
+	return static_cast<uint32_t>(kind) +
+	       (stage == ShaderType::Pixel ? static_cast<uint32_t>(DescriptorBindingKind::Count) : 0u);
+}
+
+[[nodiscard]] inline std::optional<DescriptorBindingKind>
+DescriptorBindingForImage(const ImageResource& image) {
+	using Dimension = Decoder::ImageDimension;
+	using Kind      = DescriptorBindingKind;
+
+	switch (image.kind) {
+		case ResourceKind::Image:
+			switch (image.dimension) {
+				case Dimension::Dim1D: return Kind::Sampled1D;
+				case Dimension::Dim1DArray: return Kind::Sampled1DArray;
+				case Dimension::Dim2D: return Kind::Sampled2D;
+				case Dimension::Dim2DArray: return Kind::Sampled2DArray;
+				case Dimension::Dim2DMsaa: return Kind::Sampled2DMsaa;
+				case Dimension::Dim2DMsaaArray: return Kind::Sampled2DMsaaArray;
+				case Dimension::Dim3D: return Kind::Sampled3D;
+				default: return std::nullopt;
+			}
+		case ResourceKind::ImageUint:
+			switch (image.dimension) {
+				case Dimension::Dim1D: return Kind::SampledUint1D;
+				case Dimension::Dim1DArray: return Kind::SampledUint1DArray;
+				case Dimension::Dim2D: return Kind::SampledUint2D;
+				case Dimension::Dim2DArray: return Kind::SampledUint2DArray;
+				case Dimension::Dim2DMsaa: return Kind::SampledUint2DMsaa;
+				case Dimension::Dim2DMsaaArray: return Kind::SampledUint2DMsaaArray;
+				case Dimension::Dim3D: return Kind::SampledUint3D;
+				default: return std::nullopt;
+			}
+		case ResourceKind::StorageImage:
+			if (image.atomic) {
+				return std::nullopt;
+			}
+			switch (image.dimension) {
+				case Dimension::Dim1D: return Kind::Storage1D;
+				case Dimension::Dim1DArray: return Kind::Storage1DArray;
+				case Dimension::Dim2D: return Kind::Storage2D;
+				case Dimension::Dim2DArray: return Kind::Storage2DArray;
+				case Dimension::Dim3D: return Kind::Storage3D;
+				default: return std::nullopt;
+			}
+		case ResourceKind::StorageImageUint:
+			switch (image.dimension) {
+				case Dimension::Dim1D:
+					if (image.atomic) {
+						return Kind::StorageAtomic1D;
+					}
+					return Kind::StorageUint1D;
+				case Dimension::Dim1DArray:
+					if (image.atomic) {
+						return Kind::StorageAtomic1DArray;
+					}
+					return Kind::StorageUint1DArray;
+				case Dimension::Dim2D:
+					if (image.atomic) {
+						return Kind::StorageAtomic2D;
+					}
+					return Kind::StorageUint2D;
+				case Dimension::Dim2DArray:
+					if (image.atomic) {
+						return Kind::StorageAtomic2DArray;
+					}
+					return Kind::StorageUint2DArray;
+				case Dimension::Dim3D:
+					if (image.atomic) {
+						return Kind::StorageAtomic3D;
+					}
+					return Kind::StorageUint3D;
+				default: return std::nullopt;
+			}
+		default: return std::nullopt;
+	}
+}
+
 struct DescriptorBinding {
-	DescriptorBindingKind kind    = DescriptorBindingKind::Buffers;
-	uint32_t              binding = 0;
+	DescriptorBindingKind kind = DescriptorBindingKind::Buffers;
 	std::vector<uint32_t> resources;
 
 	bool operator==(const DescriptorBinding& other) const = default;
 };
 
 struct BindingLayout {
-	uint32_t                       descriptor_set       = 0;
 	uint32_t                       push_constant_offset = 0;
-	uint32_t                       push_constant_size   = 0;
-	uint32_t                       buffer_offset_dword  = 0;
-	uint32_t                       buffer_offset_count  = 0;
+	uint32_t                       push_constant_size  = 0;
+	uint32_t                       memory_offset_dword = 0;
+	uint32_t                       memory_offset_count = 0;
 	std::vector<uint32_t>          user_data_registers;
 	std::vector<DescriptorBinding> descriptors;
 
 	[[nodiscard]] uint32_t ShaderDataDwords() const {
-		return buffer_offset_dword + (buffer_offset_count + 3u) / 4u;
+		return memory_offset_dword + (memory_offset_count + 3u) / 4u;
 	}
 
 	bool operator==(const BindingLayout& other) const = default;
@@ -351,13 +528,24 @@ struct ShaderInfo {
 	bool operator==(const ShaderInfo& other) const = default;
 };
 
+struct SpirvRequirements {
+	bool subgroup_ballot              = false;
+	bool subgroup_shuffle             = false;
+	bool subgroup_local_invocation_id = false;
+	bool compute_derivatives          = false;
+	bool image_gather_extended        = false;
+	bool function_lds                 = false;
+	bool function_scratch             = false;
+	bool pixel_valid_mask             = false;
+};
+
 struct Program {
 	ShaderType                    stage               = ShaderType::Unknown;
-	ShaderLaneMaskMode            lane_mask_mode      = ShaderLaneMaskMode::NativeWave;
 	uint64_t                      shader_hash         = 0;
 	uint32_t                      wave_size           = 64;
 	uint32_t                      user_data_base      = 0;
 	uint32_t                      user_data_count     = 64;
+	uint32_t                      scratch_dwords      = 0;
 	bool                          dispatcher_fallback = false;
 	CFG::FailureKind              cfg_failure_kind    = CFG::FailureKind::None;
 	std::string                   fallback_reason;
@@ -369,6 +557,8 @@ struct Program {
 	bool                          shader_info_complete       = false;
 	BindingLayout                 bindings;
 	bool                          binding_layout_complete = false;
+
+	std::optional<SpirvRequirements> spirv_requirements;
 };
 
 bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage,

src/graphics/shader/recompiler/ir/ShaderIRInternal.h

@@ -30,21 +30,15 @@ uint32_t         ResourceIndexFromOperand(const Decoder::Operand& operand);
 MemoryInfo       MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind);
 uint32_t         RawScalarLoadBase(const Decoder::Operand& operand);
 void             ClearRegisterOffsetModifiers(Decoder::Operand& operand);
-bool             TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code);
+bool             TryGetEncodedScalarCode(const Decoder::Operand& operand, uint32_t& code);
 bool             TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
                                             Decoder::Operand& result);
 Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
 MemoryInfo       OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
                                   uint32_t dword_index);
-uint32_t         TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
-                                                 uint32_t                    component_index);
-uint32_t         TypedBufferFormatComponentCount(const Decoder::Instruction& decoded);
-bool             LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
-                                       BasicBlock& block, std::string* error);
-MemoryInfo   OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index);
-MemoryInfo   ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
-                                  uint32_t byte_offset);
-ResourceKind FlatSegmentResourceKind(uint32_t segment);
+MemoryInfo       ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
+                                      uint32_t byte_offset);
+ResourceKind     FlatSegmentResourceKind(uint32_t segment);
 const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index);
 Decoder::Operand        M0Operand();
 

src/graphics/shader/recompiler/ir/ShaderIRLog.cpp

@@ -98,14 +98,14 @@ std::string MemoryInfoToString(const MemoryInfo& mem) {
 	    " ; {} resource={} sampler={} offset={} offset2={} dmask=0x{:x} data_dwords={} "
 	    "data_bits={} component={}/{} dfmt={} nfmt={} signed={} typed={} formatted={} segment={} "
 	    "glc={} slc={} idxen={} offen={} image_flags=0x{:x} image_dim={} image_addr={} "
-	    "image_mip={}",
+	    "image_mip={} image_r128={}",
 	    ResourceKindToString(mem.kind).c_str(), mem.resource, mem.sampler, mem.offset,
 	    mem.secondary_offset, mem.dmask, mem.data_dwords, mem.data_bits, mem.component_index,
 	    mem.component_count, mem.data_format, mem.number_format, mem.data_signed ? 1u : 0u,
 	    mem.typed ? 1u : 0u, mem.formatted ? 1u : 0u, mem.memory_segment, mem.glc ? 1u : 0u,
 	    mem.slc ? 1u : 0u, mem.idxen ? 1u : 0u, mem.offen ? 1u : 0u, mem.image_sample_flags,
 	    Decoder::ImageDimensionToString(mem.image_dimension), mem.image_address_components,
-	    mem.image_has_mip ? 1u : 0u);
+	    mem.image_has_mip ? 1u : 0u, mem.image_r128 ? 1u : 0u);
 	if (mem.image_nsa_dwords != 0) {
 		text += fmt::format(" image_nsa_dwords={} image_nsa_addr=", mem.image_nsa_dwords);
 		const auto count =
@@ -130,7 +130,8 @@ std::string InputInfoToString(Opcode op, const InputInfo& input) {
 	if (op != Opcode::LoadInputF32) {
 		return "";
 	}
-	return fmt::format(" ; input_attr={} input_chan={}", input.attr, input.chan);
+	return fmt::format(" ; input_attr={} input_chan={} input_components={}", input.attr, input.chan,
+	                   input.component_count);
 }
 
 std::string TerminatorToString(const CFG::Terminator& term) {
@@ -191,10 +192,8 @@ std::string ExportTargetKindToString(ExportTargetKind kind) {
 
 std::string ProgramToString(const Program& program) {
 	std::string text;
-	text += fmt::format("mode={} masks={} fallback={} reason={}\n",
+	text += fmt::format("mode={} fallback={} reason={}\n",
 	                    program.dispatcher_fallback ? "dispatcher" : "structured",
-	                    program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave"
-	                                                                             : "per-invocation",
 	                    CFG::FailureKindToString(program.cfg_failure_kind).c_str(),
 	                    program.fallback_reason.c_str());
 	for (const auto& block: program.blocks) {

src/graphics/shader/recompiler/ir/ShaderIRMemory.cpp

@@ -77,24 +77,40 @@ Opcode LoweredImageOpcode(Decoder::Opcode opcode) {
 
 bool LowerScalarMemoryLoadDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
                                 std::string* error) {
-	for (uint32_t i = 0; i < decoded.data_dwords; i++) {
-		Instruction inst;
-		inst.pc        = decoded.pc;
-		inst.op        = op;
-		inst.src_count = 1;
-		const auto kind =
-		    op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer;
-		inst.memory = OffsetMemoryInfo(decoded, kind, i);
-		// Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR
-		// descriptor resource index.
-		inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
-		                                                : ResourceIndexFromOperand(decoded.src0);
-		if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
-		    !LowerSourceOperand(decoded.src1, inst.src[0], error)) {
-			return false;
-		}
-		block.instructions.push_back(inst);
+	uint32_t destination_code = 0;
+	const auto alignment = decoded.data_dwords == 1u ? 1u
+	                       : decoded.data_dwords == 2u ? 2u
+	                                                    : 4u;
+	if (!TryGetEncodedScalarCode(decoded.dst, destination_code) ||
+	    destination_code % alignment != 0u ||
+	    destination_code + decoded.data_dwords > 108u) {
+		SetError(error, "scalar-memory destination has an invalid aligned SGPR span");
+		return false;
 	}
+	uint32_t base_code = 0;
+	const auto base_alignment = op == Opcode::SLoadDword ? 2u : 4u;
+	const auto base_width     = op == Opcode::SLoadDword ? 2u : 4u;
+	if (!TryGetEncodedScalarCode(decoded.src0, base_code) ||
+	    base_code % base_alignment != 0u || base_code + base_width > 108u) {
+		SetError(error, "scalar-memory base has an invalid aligned SGPR span");
+		return false;
+	}
+	Instruction inst;
+	inst.pc        = decoded.pc;
+	inst.op        = op;
+	inst.src_count = 1;
+	const auto kind =
+	    op == Opcode::SLoadDword ? ResourceKind::ScalarAddress : ResourceKind::ScalarBuffer;
+	inst.memory = MemoryInfoFromDecoded(decoded, kind);
+	// Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR
+	// descriptor resource index.
+	inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
+	                                                : ResourceIndexFromOperand(decoded.src0);
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
+	    !LowerSourceOperand(decoded.src1, inst.src[0], error)) {
+		return false;
+	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
@@ -121,52 +137,29 @@ bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction&
 }
 
 bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto ir_op                   = BufferLoadOpcode(decoded);
-	const auto count                   = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
-	for (uint32_t i = 0; i < count; i++) {
-		if (i >= typed_format_components) {
-			if (!LowerMoveImmediateU32(decoded.pc, OffsetDecodedRegister(decoded.dst, i), 0, block,
-			                           error)) {
-				return false;
-			}
-			continue;
-		}
-
-		Instruction inst;
-		inst.pc     = decoded.pc;
-		inst.op     = ir_op;
-		inst.memory = OffsetBufferMemoryInfo(decoded, i);
-		if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
-		    !LowerBufferAddressSources(decoded, inst, 0, error)) {
-			return false;
-		}
-		block.instructions.push_back(inst);
+	Instruction inst;
+	inst.pc     = decoded.pc;
+	inst.op     = BufferLoadOpcode(decoded);
+	inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
+	    !LowerBufferAddressSources(decoded, inst, 0, error)) {
+		return false;
 	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
 bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto ir_op = BufferStoreOpcode(decoded.data_bits);
-	auto       count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	if (decoded.data_bits == 32u) {
-		const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
-		if (typed_format_components < count) {
-			count = typed_format_components;
-		}
-	}
-	for (uint32_t i = 0; i < count; i++) {
-		Instruction inst;
-		inst.pc       = decoded.pc;
-		inst.op       = ir_op;
-		inst.memory   = OffsetBufferMemoryInfo(decoded, i);
-		inst.dst.kind = OperandKind::Null;
-		if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), inst.src[0], error) ||
-		    !LowerBufferAddressSources(decoded, inst, 1, error)) {
-			return false;
-		}
-		block.instructions.push_back(inst);
+	Instruction inst;
+	inst.pc       = decoded.pc;
+	inst.op       = BufferStoreOpcode(decoded.data_bits);
+	inst.memory   = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
+	inst.dst.kind = OperandKind::Null;
+	if (!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
+	    !LowerBufferAddressSources(decoded, inst, 1, error)) {
+		return false;
 	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
@@ -191,42 +184,33 @@ ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) {
 }
 
 bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto ir_op = DsReadOpcode(decoded);
-	const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	for (uint32_t i = 0; i < count; i++) {
-		Instruction inst;
-		inst.pc        = decoded.pc;
-		inst.op        = ir_op;
-		inst.src_count = 1;
-		inst.memory    = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
-		if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
-		    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
-			return false;
-		}
-		block.instructions.push_back(inst);
+	const auto  ir_op = DsReadOpcode(decoded);
+	Instruction inst;
+	inst.pc        = decoded.pc;
+	inst.op        = ir_op;
+	inst.src_count = 1;
+	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
+	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
+		return false;
 	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
 bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read,
                   std::string* error) {
-	const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
-	for (uint32_t read = 0; read < 2u; read++) {
-		for (uint32_t dword = 0; dword < dwords_per_read; dword++) {
-			const auto  index = read * dwords_per_read + dword;
-			Instruction inst;
-			inst.pc        = decoded.pc;
-			inst.op        = Opcode::DsReadB32;
-			inst.src_count = 1;
-			inst.memory =
-			    ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u);
-			if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), inst.dst, error) ||
-			    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
-				return false;
-			}
-			block.instructions.push_back(inst);
-		}
+	Instruction inst;
+	inst.pc                 = decoded.pc;
+	inst.op                 = Opcode::DsRead2B32;
+	inst.src_count          = 1;
+	inst.memory             = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
+	inst.memory.data_dwords = dwords_per_read * 2u;
+	if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
+	    !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
+		return false;
 	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
@@ -431,45 +415,36 @@ bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std:
 }
 
 bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
-	const auto ir_op = DsWriteOpcode(decoded.data_bits);
-	const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
-	for (uint32_t i = 0; i < count; i++) {
-		Instruction inst;
-		inst.pc        = decoded.pc;
-		inst.op        = ir_op;
-		inst.src_count = 2;
-		inst.memory    = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
-		inst.dst.kind  = OperandKind::Null;
-		if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), inst.src[0], error) ||
-		    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
-			return false;
-		}
-		block.instructions.push_back(inst);
+	const auto  ir_op = DsWriteOpcode(decoded.data_bits);
+	Instruction inst;
+	inst.pc        = decoded.pc;
+	inst.op        = ir_op;
+	inst.src_count = 2;
+	inst.memory    = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
+	inst.dst.kind  = OperandKind::Null;
+	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
+	    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
+		return false;
 	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
 bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block,
                    uint32_t dwords_per_write, std::string* error) {
-	const Decoder::Operand* data[]    = {&decoded.src1, &decoded.src2};
-	const uint32_t          offsets[] = {decoded.offset, decoded.secondary_offset};
-	for (uint32_t write = 0; write < 2u; write++) {
-		for (uint32_t dword = 0; dword < dwords_per_write; dword++) {
-			Instruction inst;
-			inst.pc        = decoded.pc;
-			inst.op        = Opcode::DsWriteB32;
-			inst.src_count = 2;
-			inst.memory =
-			    ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u);
-			inst.dst.kind = OperandKind::Null;
-			if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), inst.src[0],
-			                        error) ||
-			    !LowerSourceOperand(decoded.src0, inst.src[1], error)) {
-				return false;
-			}
-			block.instructions.push_back(inst);
-		}
+	Instruction inst;
+	inst.pc                 = decoded.pc;
+	inst.op                 = Opcode::DsWrite2B32;
+	inst.src_count          = 3;
+	inst.memory             = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
+	inst.memory.data_dwords = dwords_per_write * 2u;
+	inst.dst.kind           = OperandKind::Null;
+	if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
+	    !LowerSourceOperand(decoded.src0, inst.src[1], error) ||
+	    !LowerSourceOperand(decoded.src2, inst.src[2], error)) {
+		return false;
 	}
+	block.instructions.push_back(inst);
 	return true;
 }
 
@@ -641,7 +616,8 @@ bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& blo
 			return LowerDsWriteAddtidB32(decoded, block, error);
 		case Decoder::Opcode::DS_READ_ADDTID_B32:
 			return LowerDsReadAddtidB32(decoded, block, error);
-		case Decoder::Opcode::DS_READ2_B32: return LowerDsRead2(decoded, block, 1, error);
+		case Decoder::Opcode::DS_READ2_B32:
+		case Decoder::Opcode::DS_READ2ST64_B32: return LowerDsRead2(decoded, block, 1, error);
 		case Decoder::Opcode::DS_READ2_B64:
 		case Decoder::Opcode::DS_READ2ST64_B64: return LowerDsRead2(decoded, block, 2, error);
 		case Decoder::Opcode::DS_READ_I8:
@@ -788,6 +764,7 @@ bool IsMemoryOpcode(Decoder::Opcode opcode) {
 		case Decoder::Opcode::DS_READ_I16:
 		case Decoder::Opcode::DS_READ_U16:
 		case Decoder::Opcode::DS_READ2_B32:
+		case Decoder::Opcode::DS_READ2ST64_B32:
 		case Decoder::Opcode::DS_READ_B32:
 		case Decoder::Opcode::DS_READ_B64:
 		case Decoder::Opcode::DS_READ2_B64:

src/graphics/shader/recompiler/ir/Type.cpp

@@ -24,15 +24,11 @@ std::string TypeName(Type type) {
 	    "U32x3",
 	    "U32x4",
 	    "F32x2",
-	    "F32x3",
-	    "F32x4",
 	    "SrtResource",
 	    "BufferResource",
 	    "AddressResource",
 	    "ImageResource",
 	    "SamplerResource",
-	    "LdsResource",
-	    "GdsResource",
 	    "ImageAddress",
 	};
 	const auto bits = static_cast<uint32_t>(type);

src/graphics/shader/recompiler/ir/Type.h

@@ -21,16 +21,12 @@ enum class Type : uint32_t {
 	U32x3           = 1u << 11u,
 	U32x4           = 1u << 12u,
 	F32x2           = 1u << 13u,
-	F32x3           = 1u << 14u,
-	F32x4           = 1u << 15u,
-	SrtResource     = 1u << 16u,
-	BufferResource  = 1u << 17u,
-	AddressResource = 1u << 18u,
-	ImageResource   = 1u << 19u,
-	SamplerResource = 1u << 20u,
-	LdsResource     = 1u << 21u,
-	GdsResource     = 1u << 22u,
-	ImageAddress    = 1u << 23u,
+	SrtResource     = 1u << 14u,
+	BufferResource  = 1u << 15u,
+	AddressResource = 1u << 16u,
+	ImageResource   = 1u << 17u,
+	SamplerResource = 1u << 18u,
+	ImageAddress    = 1u << 19u,
 };
 
 constexpr Type operator|(Type lhs, Type rhs) {