gpu emulation: remove unnecessary Vulkan synchronization

This reverts commit 90bf3c9236.

src/common/threads.cpp

@@ -28,7 +28,8 @@
 #ifdef KYTY_WIN_CS
 #include <windows.h> // IWYU pragma: keep
 // IWYU pragma: no_include <winbase.h>
-constexpr DWORD KYTY_CS_SPIN_COUNT = 4000;
+constexpr DWORD    KYTY_CS_SPIN_COUNT          = 4000;
+constexpr uint64_t KYTY_SLEEP_SPIN_LIMIT_100NS = 500; // 50 us
 
 #ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION
 #define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002
@@ -39,7 +40,10 @@ static void SleepHighResolution100ns(uint64_t units_100ns) {
 		return;
 	}
 
-	if (units_100ns <= 10000) {
+	// Keep spinning only where a kernel transition is
+	// likely to cost more than the requested delay; ordinary millisecond sleeps use the
+	// per-thread high-resolution waitable timer below.
+	if (units_100ns <= KYTY_SLEEP_SPIN_LIMIT_100NS) {
 		LARGE_INTEGER frequency {};
 		LARGE_INTEGER start {};
 		if (QueryPerformanceFrequency(&frequency) != 0 && QueryPerformanceCounter(&start) != 0 &&

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -2682,14 +2682,21 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 	const bool gl2_writeback = ((gcr_cntl & GcrGl2Writeback) != 0);
 
 	auto trigger_interrupt = [&]() {
+		bool queued = false;
 		switch (interrupt_selector) {
 			case 0x00:
 			case 0x03: break;
 			case 0x01:
 			case 0x02:
-			case 0x04: cp.TriggerEopEventAtEndOfPipe(interrupt_context_id); break;
+			case 0x04:
+				cp.TriggerEopEventAtEndOfPipe(interrupt_context_id);
+				queued = true;
+				break;
 			default: EXIT("unknown release_mem interrupt selector\n");
 		}
+		if (queued) {
+			cp.BufferFlush();
+		}
 	};
 
 	if (data_sel == 0 || interrupt_selector == 4) {
@@ -2697,10 +2704,6 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 			cp.EmitGlobalBarrier();
 		}
 
-		if (gl2_writeback) {
-			cp.SynchronizeGpu();
-		}
-
 		trigger_interrupt();
 
 		return 7;
@@ -2711,10 +2714,6 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 			cp.EmitGlobalBarrier();
 		}
 
-		if (gl2_writeback) {
-			cp.SynchronizeGpu();
-		}
-
 		trigger_interrupt();
 
 		return 7;
@@ -2728,17 +2727,6 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 	cache_policy      = 0;
 
 	if (data_sel == 1) {
-		static std::atomic<uint32_t> log_count {0};
-		const auto                   dst_addr = reinterpret_cast<uint64_t>(dst_gpu_addr);
-		if ((dst_addr & 0xffffffffff000000ull) == 0x0000001021000000ull || log_count.load() < 64) {
-			if (log_count.fetch_add(1) < 256) {
-				LOGF(
-				    "\t runtime release_mem32: dst=0x%016" PRIx64 ", value=0x%08" PRIx32
-				    ", action=0x%02" PRIx32 ", gcr=0x%04" PRIx32 ", cache_action=0x%08" PRIx32 "\n",
-				    dst_addr, static_cast<uint32_t>(value), eop_event_type, gcr_cntl, cache_action);
-			}
-		}
-
 		eop_event_type    = 0x2f;
 		event_index       = 6;
 		auto event_source = 2u;
@@ -2753,7 +2741,6 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 
 	if (data_sel == 5) {
 		if (gl2_writeback) {
-			cp.SynchronizeGpu();
 			cache_action = 0;
 		}
 
@@ -2764,7 +2751,9 @@ KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 		cp.WriteAtEndOfPipe32(cache_policy, event_write_dest, eop_event_type, cache_action,
 		                      event_index, event_source, dst_gpu_addr, static_cast<uint32_t>(value),
 		                      interrupt_selector, interrupt_context_id);
-		cp.BufferFlush();
+		if (interrupt_selector == 0x01) {
+			cp.BufferFlush();
+		}
 
 		return 7;
 	}

src/graphics/guest_gpu/graphicsRun.cpp

@@ -124,7 +124,6 @@ private:
 		uint64_t       flip_request_id               = 0;
 	};
 
-	void              WaitLocked();
 	void              Enqueue(Submission submission);
 	void              WaitForIdle();
 	void              ProcessCommands();
@@ -276,10 +275,8 @@ void GpuState::SubmitFlipPreparation(uint64_t request_id) {
 
 void GpuState::Done() {
 	GpuMutexLock lock(m_submission_mutex);
-	if (IsGpuThread()) {
-		m_gfx_cp->BufferWait();
-	} else {
-		WaitLocked();
+	if (!IsGpuThread()) {
+		WaitForIdle();
 	}
 	m_graphics_done = true;
 	m_done_num++;
@@ -289,11 +286,6 @@ int GpuState::GetFrameNum() {
 	return m_done_num;
 }
 
-void GpuState::WaitLocked() {
-	WaitForIdle();
-	SendCommandSync([this] { m_gfx_cp->BufferWait(); });
-}
-
 CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) {
 	EXIT_IF(queue_id >= QueueCount);
 	if (queue_id == 0) {
@@ -307,8 +299,6 @@ CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) {
 }
 
 void CommandProcessor::Reset() {
-	BufferWait();
-
 	m_sh_ctx.Reset();
 	m_ucfg.Reset();
 	m_ctx.Reset();
@@ -385,7 +375,6 @@ void CommandProcessor::WaitDeDiff(uint32_t diff) {
 }
 
 void CommandProcessor::IncrementDe() {
-	BufferWait();
 	m_de_count++;
 }
 
@@ -1352,6 +1341,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					              value >> 16u);
 					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
 					                            value >> 16u);
+					if (with_interrupt) {
+						m_renderer.TriggerEopEvent(interrupt_context_id);
+					}
 					return;
 				}
 			} else if (eop_event_type == 0x04 && cache_action == 0x00 && event_index == 0x05) {
@@ -1554,7 +1546,6 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
 				     event_index);
 			}
 			EmitGlobalBarrier();
-			SynchronizeGpu();
 			break;
 		// DbDataWritebackInvalidate, DbMetadataWritebackInvalidate, CbMetadataWritebackInvalidate.
 		case 0x0000002a:

src/graphics/host_gpu/memoryTracker.cpp

@@ -79,7 +79,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
-	std::lock_guard access(m_access_mutex);
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -88,7 +87,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 
 bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
-	std::lock_guard access(m_access_mutex);
 	return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -97,7 +95,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 
 void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
-	std::lock_guard access(m_access_mutex);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
@@ -106,7 +103,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
-	std::lock_guard access(m_access_mutex);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		manager->ChangeState<DirtySource::Gpu, true>(manager->GetCpuAddr() + offset, bytes);
@@ -115,14 +111,13 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 
 void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
-	std::lock_guard access(m_access_mutex);
 	Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		manager->ChangeState<DirtySource::Gpu, false>(manager->GetCpuAddr() + offset, bytes);
 	});
 }
 
-void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
+void MemoryTracker::UntrackMemoryImpl(uint64_t vaddr, uint64_t size) {
 	std::vector<RegionManager*> managers;
 	managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
 	                 TRACKER_REGION_SIZE);
@@ -148,8 +143,7 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
 
 void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
-	std::lock_guard access(m_access_mutex);
-	UntrackMemoryLocked(vaddr, size);
+	UntrackMemoryImpl(vaddr, size);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/memoryTracker.h

@@ -68,7 +68,6 @@ public:
 		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
 		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
 		CheckNotInUploadCallback();
-		std::lock_guard             access(m_access_mutex);
 		std::vector<RegionManager*> managers;
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
 			managers.push_back(manager);
@@ -109,7 +108,6 @@ public:
 		static_assert(std::is_nothrow_invocable_v<RangeFunc&, uint64_t, uint64_t>);
 		static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
 		CheckNotInUploadCallback();
-		std::unique_lock access(m_access_mutex);
 		Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
 		const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -173,13 +171,12 @@ private:
 	}
 
 	static void    ValidateRange(uint64_t vaddr, uint64_t size);
-	void           UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
+	void           UntrackMemoryImpl(uint64_t vaddr, uint64_t size);
 	RegionManager* GetOrCreateRegion(uint64_t index);
 
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
 	std::vector<std::unique_ptr<RegionManager>>    m_region_storage;
 	std::mutex                                     m_region_mutex;
-	std::mutex                                     m_access_mutex;
 	PageManager&                                   m_page_manager;
 };
 

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -99,7 +99,10 @@ bool CommandScheduler::InDeferredOperation() noexcept {
 
 CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics)
     : m_master(graphics), m_context(context), m_graphics(graphics),
-      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {}
+      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {
+	m_buffers.reserve(CommandBufferGrowStep);
+	m_buffer_ticks.reserve(CommandBufferGrowStep);
+}
 
 CommandScheduler::~CommandScheduler() {
 	Shutdown();
@@ -154,10 +157,7 @@ void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config
 	m_shaders     = &shaders;
 
 	if (!Active()) {
-		for (auto& buffer: m_buffers) {
-			buffer = std::make_unique<RenderCommandBuffer>(*this);
-		}
-		m_current = 0;
+		m_current = static_cast<int>(GrowCommandBuffers());
 	}
 
 	BindCurrent();
@@ -235,7 +235,13 @@ void CommandScheduler::Wait(uint64_t tick) {
 }
 
 void CommandScheduler::PopPendingOperations() {
-	m_master.Refresh();
+	PopPendingOperations(true);
+}
+
+void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) {
+	if (refresh_gpu_tick) {
+		m_master.Refresh();
+	}
 	for (;;) {
 		Common::UniqueFunction<void> callback;
 		{
@@ -362,7 +368,7 @@ uint64_t CommandScheduler::NextSubmitSequence() noexcept {
 }
 
 void CommandScheduler::CheckActive() const {
-	EXIT_IF(!Active() || m_current >= BufferCount);
+	EXIT_IF(!Active() || static_cast<size_t>(m_current) >= m_buffers.size());
 }
 
 RenderCommandBuffer& CommandScheduler::Current() const {
@@ -384,15 +390,52 @@ CommandBuffer& CommandScheduler::SubmitCurrent(SubmitInfo& submit) {
 	const auto signal_tick = m_master.NextTick();
 	submit.AddSignal(m_master.Handle(), signal_tick);
 	submitted.Execute(submit);
-	m_recording = false;
+	m_buffer_ticks[static_cast<size_t>(m_current)] = signal_tick;
+	m_recording                                    = false;
 	return submitted;
 }
 
+int CommandScheduler::FindReusableBuffer(uint64_t gpu_tick) const {
+	EXIT_IF(m_buffer_ticks.size() != m_buffers.size());
+	const auto buffer_count = m_buffers.size();
+	for (size_t offset = 1; offset <= buffer_count; ++offset) {
+		const auto candidate = (static_cast<size_t>(m_current) + offset) % buffer_count;
+		if (gpu_tick >= m_buffer_ticks[candidate]) {
+			return static_cast<int>(candidate);
+		}
+	}
+	return -1;
+}
+
+size_t CommandScheduler::GrowCommandBuffers() {
+	const auto first = m_buffers.size();
+	const auto end   = first + CommandBufferGrowStep;
+	m_buffers.reserve(end);
+	m_buffer_ticks.reserve(end);
+	for (size_t i = first; i < end; ++i) {
+		m_buffers.emplace_back(std::make_unique<RenderCommandBuffer>(*this));
+		m_buffer_ticks.push_back(0);
+	}
+	return first;
+}
+
 void CommandScheduler::BeginNext() {
 	EXIT_IF(m_recording);
-	m_current = (m_current + 1) % BufferCount;
+
+	auto candidate = FindReusableBuffer(m_master.KnownGpuTick());
+	bool refreshed = false;
+	if (candidate < 0) {
+		m_master.Refresh();
+		refreshed = true;
+		candidate = FindReusableBuffer(m_master.KnownGpuTick());
+	}
+	if (candidate < 0) {
+		candidate = static_cast<int>(GrowCommandBuffers());
+	}
+
+	m_current = candidate;
 	Current().WaitForFenceAndReset();
-	PopPendingOperations();
+	PopPendingOperations(!refreshed);
 	BindCurrent();
 	Current().Begin();
 	m_recording = true;

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -16,6 +16,7 @@
 #include <queue>
 
 #include <thread>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -31,8 +32,6 @@ struct CommandSlot {
 
 class CommandScheduler {
 public:
-	static constexpr int BufferCount = 8;
-
 	CommandScheduler(RenderContext& context, GraphicContext& graphics);
 	~CommandScheduler();
 	KYTY_CLASS_NO_COPY(CommandScheduler);
@@ -65,6 +64,8 @@ public:
 	[[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; }
 
 private:
+	static constexpr size_t CommandBufferGrowStep = 4;
+
 	class CommandPool {
 	public:
 		CommandPool() = default;
@@ -94,30 +95,34 @@ private:
 	void                       BindCurrent() const;
 	CommandBuffer&             SubmitCurrent(SubmitInfo& submit);
 	void                       BeginNext();
+	void                       PopPendingOperations(bool refresh_gpu_tick);
+	[[nodiscard]] int          FindReusableBuffer(uint64_t gpu_tick) const;
+	[[nodiscard]] size_t       GrowCommandBuffers();
 	void                       PriorityOperationsThread(std::stop_token stop);
 	void                       RunOperation(Common::UniqueFunction<void>&& operation);
 	[[nodiscard]] CommandSlot* AllocateCommandBuffer();
 	[[nodiscard]] uint64_t     NextSubmitSequence() noexcept;
 
-	MasterSemaphore                                               m_master;
-	RenderContext&                                                m_context;
-	GraphicContext&                                               m_graphics;
-	CommandPool                                                   m_command_pool;
-	std::array<std::unique_ptr<RenderCommandBuffer>, BufferCount> m_buffers;
-	std::queue<PendingOperation>                                  m_pending_operations;
-	std::queue<PendingOperation>                                  m_priority_operations;
-	std::mutex                                                    m_operation_mutex;
-	std::condition_variable                                       m_operation_available;
-	std::jthread                                                  m_priority_thread;
-	bool                                                          m_priority_active      = false;
-	uint64_t                                                      m_priority_active_tick = 0;
-	OperationState        m_operation_state = OperationState::Open;
-	int                   m_current         = -1;
-	bool                  m_recording       = false;
-	HW::Context*          m_registers       = nullptr;
-	HW::UserConfig*       m_user_config     = nullptr;
-	HW::Shader*           m_shaders         = nullptr;
-	std::atomic<uint64_t> m_submit_sequence = 0;
+	MasterSemaphore                                   m_master;
+	RenderContext&                                    m_context;
+	GraphicContext&                                   m_graphics;
+	CommandPool                                       m_command_pool;
+	std::vector<std::unique_ptr<RenderCommandBuffer>> m_buffers;
+	std::vector<uint64_t>                             m_buffer_ticks;
+	std::queue<PendingOperation>                      m_pending_operations;
+	std::queue<PendingOperation>                      m_priority_operations;
+	std::mutex                                        m_operation_mutex;
+	std::condition_variable                           m_operation_available;
+	std::jthread                                      m_priority_thread;
+	bool                                              m_priority_active      = false;
+	uint64_t                                          m_priority_active_tick = 0;
+	OperationState                                    m_operation_state      = OperationState::Open;
+	int                                               m_current              = -1;
+	bool                                              m_recording            = false;
+	HW::Context*                                      m_registers            = nullptr;
+	HW::UserConfig*                                   m_user_config          = nullptr;
+	HW::Shader*                                       m_shaders              = nullptr;
+	std::atomic<uint64_t>                             m_submit_sequence      = 0;
 
 	friend class CommandBuffer;
 };

src/graphics/host_gpu/renderer/debug.cpp

@@ -10,6 +10,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 
 #include <algorithm>
+#include <array>
 #include <atomic>
 #include <cmath>
 #include <fmt/format.h>
@@ -1059,18 +1060,24 @@ static ScissorRect ScissorRectClamp(ScissorRect r, uint32_t width, uint32_t heig
 	return r;
 }
 
+static constexpr std::array<uint16_t, 16> MakeScissorIntersectionRules() {
+	std::array<uint16_t, 16> rules {};
+	for (uint32_t candidate = 0; candidate < rules.size(); candidate++) {
+		for (uint32_t combination = 0; combination < 16; combination++) {
+			if ((combination & candidate) == candidate) {
+				rules[candidate] |= static_cast<uint16_t>(1u << combination);
+			}
+		}
+	}
+	return rules;
+}
+
 static bool ScissorClipRuleToIntersectionMask(uint16_t rule, uint8_t* mask) {
 	EXIT_IF(mask == nullptr);
 
-	for (uint32_t candidate = 0; candidate < 16; candidate++) {
-		uint16_t expected = 0;
-		for (uint32_t combination = 0; combination < 16; combination++) {
-			if ((combination & candidate) == candidate) {
-				expected |= static_cast<uint16_t>(1u << combination);
-			}
-		}
-
-		if (expected == rule) {
+	static constexpr auto rules = MakeScissorIntersectionRules();
+	for (uint32_t candidate = 0; candidate < rules.size(); candidate++) {
+		if (rules[candidate] == rule) {
 			*mask = static_cast<uint8_t>(candidate);
 			return true;
 		}

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -38,7 +38,10 @@ namespace Libs::Graphics {
 	EXIT("unsupported render state; details were printed above\n");
 }
 
-static vk::StencilOp ConvertStencilOp(uint8_t value) {
+static vk::StencilOp ConvertStencilOp(uint8_t value, uint8_t write_mask, uint8_t op_value) {
+	if (write_mask == 0) {
+		return vk::StencilOp::eKeep;
+	}
 	switch (static_cast<Prospero::StencilOp>(value)) {
 		case Prospero::StencilOp::kKeep: return vk::StencilOp::eKeep;
 		case Prospero::StencilOp::kZero: return vk::StencilOp::eZero;
@@ -49,7 +52,17 @@ static vk::StencilOp ConvertStencilOp(uint8_t value) {
 		case Prospero::StencilOp::kInvert: return vk::StencilOp::eInvert;
 		case Prospero::StencilOp::kAddWrap: return vk::StencilOp::eIncrementAndWrap;
 		case Prospero::StencilOp::kSubWrap: return vk::StencilOp::eDecrementAndWrap;
-		default: DepthFatal("unsupported stencil operation");
+		case Prospero::StencilOp::kXor:
+			if ((write_mask & op_value) == 0) {
+				return vk::StencilOp::eKeep;
+			}
+			if ((write_mask & ~op_value) != 0) {
+				DepthFatal("unsupported stencil XOR operands: write mask=0x%02" PRIx8
+				           ", operation value=0x%02" PRIx8,
+				           write_mask, op_value);
+			}
+			return vk::StencilOp::eInvert;
+		default: DepthFatal("unsupported stencil operation: 0x%02" PRIx8, value);
 	}
 }
 
@@ -283,6 +296,8 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	r.stencil_clear_value  = hw.GetStencilClearValue();
 	r.stencil_test_enable  = has_stencil && dc.stencil_enable;
 	if (r.stencil_test_enable) {
+		const uint8_t front_write_mask = rc.stencil_clear_enable ? 0 : sm.stencil_writemask;
+		const uint8_t back_write_mask  = rc.stencil_clear_enable ? 0 : sm.stencil_writemask_bf;
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
 		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||
@@ -294,19 +309,18 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 			DepthFatal("unsupported stencil compare or replacement state");
 		}
 		r.stencil_static_front = {
-		    ConvertStencilOp(sc.stencil_fail), ConvertStencilOp(sc.stencil_zpass),
-		    ConvertStencilOp(sc.stencil_zfail), static_cast<vk::CompareOp>(dc.stencilfunc)};
-		r.stencil_dynamic_front = {sm.stencil_mask,
-		                           rc.stencil_clear_enable ? 0u : sm.stencil_writemask,
-		                           sm.stencil_testval};
+		    ConvertStencilOp(sc.stencil_fail, front_write_mask, sm.stencil_opval),
+		    ConvertStencilOp(sc.stencil_zpass, front_write_mask, sm.stencil_opval),
+		    ConvertStencilOp(sc.stencil_zfail, front_write_mask, sm.stencil_opval),
+		    static_cast<vk::CompareOp>(dc.stencilfunc)};
+		r.stencil_dynamic_front = {sm.stencil_mask, front_write_mask, sm.stencil_testval};
 		if (dc.backface_enable) {
-			r.stencil_static_back  = {ConvertStencilOp(sc.stencil_fail_bf),
-			                          ConvertStencilOp(sc.stencil_zpass_bf),
-			                          ConvertStencilOp(sc.stencil_zfail_bf),
-			                          static_cast<vk::CompareOp>(dc.stencilfunc_bf)};
-			r.stencil_dynamic_back = {sm.stencil_mask_bf,
-			                          rc.stencil_clear_enable ? 0u : sm.stencil_writemask_bf,
-			                          sm.stencil_testval_bf};
+			r.stencil_static_back = {
+			    ConvertStencilOp(sc.stencil_fail_bf, back_write_mask, sm.stencil_opval_bf),
+			    ConvertStencilOp(sc.stencil_zpass_bf, back_write_mask, sm.stencil_opval_bf),
+			    ConvertStencilOp(sc.stencil_zfail_bf, back_write_mask, sm.stencil_opval_bf),
+			    static_cast<vk::CompareOp>(dc.stencilfunc_bf)};
+			r.stencil_dynamic_back = {sm.stencil_mask_bf, back_write_mask, sm.stencil_testval_bf};
 		} else {
 			r.stencil_static_back  = r.stencil_static_front;
 			r.stencil_dynamic_back = r.stencil_dynamic_front;

src/graphics/presentation/videoOut.cpp

@@ -788,14 +788,10 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
 	while (!token.stop_requested()) {
 		const auto sleep_begin = Common::Timer::QueryPerformanceCounter();
 		if (total_wait > 0) {
-			const auto sleep_end = sleep_begin + static_cast<uint64_t>(total_wait);
-			auto       now       = sleep_begin;
-			while (!token.stop_requested() && now < sleep_end) {
-				const auto remaining_us = (sleep_end - now) * 1000000u / frequency;
-				Common::Thread::SleepMicro(
-				    static_cast<uint32_t>(std::clamp<uint64_t>(remaining_us, 1, 1000)));
-				now = Common::Timer::QueryPerformanceCounter();
-			}
+			const auto remaining_us =
+			    (static_cast<uint64_t>(total_wait) * 1000000u + frequency - 1) / frequency;
+			Common::Thread::SleepMicro(static_cast<uint32_t>(
+			    std::clamp<uint64_t>(remaining_us, 1, std::numeric_limits<uint32_t>::max())));
 		}
 		if (token.stop_requested()) {
 			break;

src/graphics/presentation/window/window.cpp

@@ -752,26 +752,18 @@ void WindowContext::Run() {
 #if defined(__APPLE__)
 		DrainMainThreadTasks();
 #endif
-		if (SDL_PollEvent(&loop.event) != 0) {
-			ProcessEvent(timer.GetTimeS());
-			continue;
-		}
-
 		if (loop.paused.load(std::memory_order_acquire)) {
 			if (!timer.IsPaused()) {
 				timer.Pause();
 			}
-			if (SDL_WaitEvent(&loop.event) == 0) {
-				EXIT("%s\n", SDL_GetError());
-			}
-			ProcessEvent(timer.GetTimeS());
-			continue;
-		}
-
-		if (timer.IsPaused()) {
+		} else if (timer.IsPaused()) {
 			timer.Resume();
 		}
-		Common::Thread::SleepMicro(1000);
+
+		if (SDL_WaitEvent(&loop.event) == 0) {
+			EXIT("%s\n", SDL_GetError());
+		}
+		ProcessEvent(timer.GetTimeS());
 	}
 }
 

src/graphics/shader/recompiler/ir/SrtWalker.cpp

@@ -562,8 +562,7 @@ static bool EvaluateRuntimeSourcesImpl(const Program&
 		const auto  dynamic =
 		    std::find(program.srt.dynamic_sources.begin(), program.srt.dynamic_sources.end(),
 		              request.source) != program.srt.dynamic_sources.end();
-		if (descriptor == nullptr || dynamic ||
-		    !DescriptorSourceResolved(program, request.source)) {
+		if (descriptor == nullptr || dynamic) {
 			if (error != nullptr) {
 				*error = Diagnostic(program, request.use_pc,
 				                    fmt::format("descriptor source {} is {}", request.source,

src/kernel/pthread.cpp

@@ -1849,8 +1849,9 @@ static int PthreadMutexInitNamed(PthreadMutex* mutex, const PthreadMutexattr* at
 	}
 }
 
-int KYTY_SYSV_ABI PthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr) {
-	return PthreadMutexInitNamed(mutex, attr, "");
+int KYTY_SYSV_ABI PthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr,
+                                   const char* name) {
+	return PthreadMutexInitNamed(mutex, attr, name);
 }
 
 int KYTY_SYSV_ABI PthreadMutexDestroy(PthreadMutex* mutex) {
@@ -2443,8 +2444,9 @@ static int PthreadRwlockInitNamed(PthreadRwlock* rwlock, const PthreadRwlockattr
 	return OK;
 }
 
-int KYTY_SYSV_ABI PthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr) {
-	return PthreadRwlockInitNamed(rwlock, attr, "");
+int KYTY_SYSV_ABI PthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr,
+                                    const char* name) {
+	return PthreadRwlockInitNamed(rwlock, attr, name);
 }
 
 static PthreadRwlockPrivate::Reader* RwlockFindReader(PthreadRwlock rwlock, Pthread thread) {
@@ -2938,8 +2940,9 @@ static int PthreadCondInitNamed(PthreadCond* cond, const PthreadCondattr* attr,
 	}
 }
 
-int KYTY_SYSV_ABI PthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr) {
-	return PthreadCondInitNamed(cond, attr, "");
+int KYTY_SYSV_ABI PthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr,
+                                  const char* name) {
+	return PthreadCondInitNamed(cond, attr, name);
 }
 
 int KYTY_SYSV_ABI PthreadCondSignal(PthreadCond* cond) {
@@ -4417,6 +4420,13 @@ int KYTY_SYSV_ABI pthread_cond_signal(LibKernel::PthreadCond* cond) {
 	return POSIX_PTHREAD_CALL(LibKernel::PthreadCondSignal(cond));
 }
 
+int KYTY_SYSV_ABI pthread_cond_init(LibKernel::PthreadCond*           cond,
+                                    const LibKernel::PthreadCondattr* attr) {
+	PRINT_NAME();
+
+	return POSIX_PTHREAD_CALL(LibKernel::PthreadCondInit(cond, attr, nullptr));
+}
+
 int KYTY_SYSV_ABI pthread_condattr_setclock(LibKernel::PthreadCondattr* attr,
                                             LibKernel::KernelClockid    clock_id) {
 	PRINT_NAME();
@@ -4525,6 +4535,13 @@ int KYTY_SYSV_ABI pthread_rwlock_destroy(LibKernel::PthreadRwlock* rwlock) {
 	return POSIX_PTHREAD_CALL(LibKernel::PthreadRwlockDestroy(rwlock));
 }
 
+int KYTY_SYSV_ABI pthread_rwlock_init(LibKernel::PthreadRwlock*           rwlock,
+                                      const LibKernel::PthreadRwlockattr* attr) {
+	PRINT_NAME();
+
+	return POSIX_PTHREAD_CALL(LibKernel::PthreadRwlockInit(rwlock, attr, nullptr));
+}
+
 int KYTY_SYSV_ABI pthread_key_create(LibKernel::PthreadKey*                   key,
                                      LibKernel::pthread_key_destructor_func_t destructor) {
 	PRINT_NAME();
@@ -4560,7 +4577,7 @@ int KYTY_SYSV_ABI pthread_mutex_init(LibKernel::PthreadMutex*           mutex,
                                      const LibKernel::PthreadMutexattr* attr) {
 	PRINT_NAME();
 
-	return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexInit(mutex, attr));
+	return POSIX_PTHREAD_CALL(LibKernel::PthreadMutexInit(mutex, attr, nullptr));
 }
 
 int KYTY_SYSV_ABI pthread_mutexattr_init(LibKernel::PthreadMutexattr* attr) {

src/kernel/pthread.h

@@ -77,7 +77,8 @@ int KYTY_SYSV_ABI PthreadMutexattrInit(PthreadMutexattr* attr);
 int KYTY_SYSV_ABI PthreadMutexattrDestroy(PthreadMutexattr* attr);
 int KYTY_SYSV_ABI PthreadMutexattrSettype(PthreadMutexattr* attr, int type);
 int KYTY_SYSV_ABI PthreadMutexattrSetprotocol(PthreadMutexattr* attr, int protocol);
-int KYTY_SYSV_ABI PthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr);
+int KYTY_SYSV_ABI PthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr,
+                                   const char* name);
 int KYTY_SYSV_ABI PthreadMutexDestroy(PthreadMutex* mutex);
 int KYTY_SYSV_ABI PthreadMutexLock(PthreadMutex* mutex);
 int KYTY_SYSV_ABI PthreadMutexTrylock(PthreadMutex* mutex);
@@ -159,7 +160,8 @@ int KYTY_SYSV_ABI PthreadAttrSetstackaddr(PthreadAttr* attr, void* addr);
 int KYTY_SYSV_ABI PthreadAttrSetstacksize(PthreadAttr* attr, size_t stack_size);
 
 int KYTY_SYSV_ABI PthreadRwlockDestroy(PthreadRwlock* rwlock);
-int KYTY_SYSV_ABI PthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr);
+int KYTY_SYSV_ABI PthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockattr* attr,
+                                    const char* name);
 int KYTY_SYSV_ABI PthreadRwlockRdlock(PthreadRwlock* rwlock);
 int KYTY_SYSV_ABI PthreadRwlockTimedrdlock(PthreadRwlock* rwlock, KernelUseconds usec);
 int KYTY_SYSV_ABI PthreadRwlockTimedwrlock(PthreadRwlock* rwlock, KernelUseconds usec);
@@ -177,7 +179,7 @@ int KYTY_SYSV_ABI PthreadCondattrInit(PthreadCondattr* attr);
 int KYTY_SYSV_ABI PthreadCondattrSetclock(PthreadCondattr* attr, KernelClockid clock_id);
 int KYTY_SYSV_ABI PthreadCondBroadcast(PthreadCond* cond);
 int KYTY_SYSV_ABI PthreadCondDestroy(PthreadCond* cond);
-int KYTY_SYSV_ABI PthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr);
+int KYTY_SYSV_ABI PthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name);
 int KYTY_SYSV_ABI PthreadCondSignal(PthreadCond* cond);
 int KYTY_SYSV_ABI PthreadCondSignalto(PthreadCond* cond, Pthread thread);
 int KYTY_SYSV_ABI PthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, KernelUseconds usec);
@@ -267,6 +269,8 @@ int KYTY_SYSV_ABI pthread_rwlock_rdlock(LibKernel::PthreadRwlock* rwlock);
 int KYTY_SYSV_ABI pthread_rwlock_unlock(LibKernel::PthreadRwlock* rwlock);
 int KYTY_SYSV_ABI pthread_rwlock_wrlock(LibKernel::PthreadRwlock* rwlock);
 int KYTY_SYSV_ABI pthread_rwlock_destroy(LibKernel::PthreadRwlock* rwlock);
+int KYTY_SYSV_ABI pthread_rwlock_init(LibKernel::PthreadRwlock*           rwlock,
+                                      const LibKernel::PthreadRwlockattr* attr);
 int KYTY_SYSV_ABI pthread_key_create(LibKernel::PthreadKey*                   key,
                                      LibKernel::pthread_key_destructor_func_t destructor);
 int KYTY_SYSV_ABI pthread_key_delete(LibKernel::PthreadKey key);

src/libs/libC.cpp

@@ -481,13 +481,13 @@ static KYTY_SYSV_ABI int MtxInit(LibKernel::PthreadMutex* mtx, int type) {
 
 	constexpr int mtx_recursive = 0x100;
 	if ((type & mtx_recursive) == 0) {
-		return mtx_result(LibKernel::PthreadMutexInit(mtx, nullptr));
+		return mtx_result(LibKernel::PthreadMutexInit(mtx, nullptr, nullptr));
 	}
 
 	LibKernel::PthreadMutexattr attr   = nullptr;
 	int                         result = LibKernel::PthreadMutexattrInit(&attr);
 	result = (result == OK ? LibKernel::PthreadMutexattrSettype(&attr, 2) : result);
-	result = (result == OK ? LibKernel::PthreadMutexInit(mtx, &attr) : result);
+	result = (result == OK ? LibKernel::PthreadMutexInit(mtx, &attr, nullptr) : result);
 	(void)LibKernel::PthreadMutexattrDestroy(&attr);
 
 	return mtx_result(result);
@@ -659,8 +659,8 @@ void KYTY_SYSV_ABI LibcHeapGetTraceInfo(Info* info) {
 	info->mstate_table          = g_mstate_table;
 }
 
-int KYTY_SYSV_ABI LibcInternalExtCxaThreadAtexit(thread_atexit_destructor_t destructor, void* object,
-                                                 void* /*module_id*/) {
+int KYTY_SYSV_ABI LibcInternalExtCxaThreadAtexit(thread_atexit_destructor_t destructor,
+                                                 void* object, void* /*module_id*/) {
 	PRINT_NAME();
 
 	g_thread_atexit_destructors.push_back({destructor, object});

src/libs/libKernel.cpp

@@ -2172,9 +2172,11 @@ LIB_DEFINE(InitLibKernel_1_Posix) {
 	LIB_FUNC("ltCfaGr2JGE", Posix::pthread_mutex_destroy);
 	LIB_FUNC("mkx2fVhNMsg", Posix::pthread_cond_broadcast);
 	LIB_FUNC("2MOy+rUfuhQ", Posix::pthread_cond_signal);
+	LIB_FUNC("0TyVk4MSLt0", Posix::pthread_cond_init);
 	LIB_FUNC("Op8TBGY5KHg", Posix::pthread_cond_wait);
 	LIB_FUNC("Z4QosVuAsA0", Posix::pthread_once);
 	LIB_FUNC("1471ajPzxh0", Posix::pthread_rwlock_destroy);
+	LIB_FUNC("ytQULN-nhL4", Posix::pthread_rwlock_init);
 	LIB_FUNC("sIlRvQqsN2Y", Posix::pthread_rwlock_wrlock);
 	LIB_FUNC("SFxTMOfuCkE", LibKernel::PthreadRwlockTryrdlock);
 	LIB_FUNC("XhWHn6P5R7U", LibKernel::PthreadRwlockTrywrlock);
@@ -3211,7 +3213,6 @@ LIB_DEFINE(InitLibKernel_1_Pthread) {
 	LIB_FUNC("FXPWHNk8Of0", LibKernel::PthreadAttrGetschedparam);
 
 	LIB_FUNC("6ULAa0fq4jA", LibKernel::PthreadRwlockInit);
-	LIB_FUNC("ytQULN-nhL4", LibKernel::PthreadRwlockInit);
 	LIB_FUNC("BB+kb08Tl9A", LibKernel::PthreadRwlockDestroy);
 	LIB_FUNC("Ox9i0c7L5w0", LibKernel::PthreadRwlockRdlock);
 	LIB_FUNC("iGjsr1WAtI0", LibKernel::PthreadRwlockRdlock);
@@ -3227,7 +3228,6 @@ LIB_DEFINE(InitLibKernel_1_Pthread) {
 	LIB_FUNC("h-OifiouBd8", LibKernel::PthreadRwlockattrSettype);
 
 	LIB_FUNC("2Tb92quprl0", LibKernel::PthreadCondInit);
-	LIB_FUNC("0TyVk4MSLt0", LibKernel::PthreadCondInit);
 	LIB_FUNC("g+PZd2hiacg", LibKernel::PthreadCondDestroy);
 	LIB_FUNC("RXXqi4CtF8w", LibKernel::PthreadCondDestroy);
 	LIB_FUNC("WKAXJ4XBPQ4", LibKernel::PthreadCondWait);

tests/MemoryTrackerTests.cpp

@@ -2,6 +2,8 @@
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
 
+#include <atomic>
+#include <chrono>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
@@ -211,6 +213,40 @@ void TestQueriesDoNotRequireMappedOwnership() {
         "unowned tracker range did not expose its initial CPU-dirty state");
 }
 
+void TestConcurrentRegionPublication() {
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  const auto page_size = page_manager.GetPageSize();
+  auto *memory = Allocate(page_manager, 1);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+
+  std::binary_semaphore start_first{0};
+  std::binary_semaphore start_second{0};
+  std::atomic_uint32_t cpu_dirty_results{0};
+  std::jthread first([&] {
+    start_first.acquire();
+    if (tracker.IsRegionCpuModified(address, page_size)) {
+      cpu_dirty_results.fetch_add(1, std::memory_order_relaxed);
+    }
+  });
+  std::jthread second([&] {
+    start_second.acquire();
+    if (tracker.IsRegionCpuModified(address, page_size)) {
+      cpu_dirty_results.fetch_add(1, std::memory_order_relaxed);
+    }
+  });
+  start_first.release();
+  start_second.release();
+  first.join();
+  second.join();
+
+  tracker.UntrackMemory(address, page_size);
+  Release(page_manager, memory, page_size);
+  Check(cpu_dirty_results.load(std::memory_order_relaxed) == 2,
+        "concurrent region publication lost initial CPU ownership");
+}
+
 void TestCpuDirtyUpload() {
   TrackerHarness harness;
   auto &tracker = harness.tracker;
@@ -497,6 +533,114 @@ void TestCrossRegionUpload() {
   Release(page_manager, memory, region_size * 2);
 }
 
+void TestUploadDoesNotSerializeDisjointRegion() {
+  constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
+  constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  auto *memory = Allocate(page_manager, region_size * 2 / page_size);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto second_region =
+      (allocation_base & ~(region_size - 1)) + region_size;
+  Check(second_region + page_size <= allocation_base + region_size * 2,
+        "test allocation does not span two tracker regions");
+
+  // Publish both managers before the concurrent section so this test measures
+  // tracker access serialization rather than manager allocation.
+  Check(tracker.IsRegionCpuModified(allocation_base, page_size) &&
+            tracker.IsRegionCpuModified(second_region, page_size),
+        "disjoint upload setup did not initialize both regions");
+
+  std::binary_semaphore upload_entered{0};
+  std::binary_semaphore finish_upload{0};
+  std::binary_semaphore query_finished{0};
+  std::atomic_bool query_result{false};
+  std::jthread uploader([&] {
+    tracker.ForEachUploadRange(
+        allocation_base, page_size, true, [](uint64_t, uint64_t) noexcept {},
+        [&]() noexcept {
+          upload_entered.release();
+          finish_upload.acquire();
+        });
+  });
+  upload_entered.acquire();
+  std::jthread query([&] {
+    query_result.store(tracker.IsRegionCpuModified(second_region, page_size),
+                       std::memory_order_relaxed);
+    query_finished.release();
+  });
+
+  const bool completed_while_upload_blocked =
+      query_finished.try_acquire_for(std::chrono::seconds(5));
+  finish_upload.release();
+  uploader.join();
+  query.join();
+
+  tracker.UnmarkRegionAsGpuModified(allocation_base, page_size);
+  tracker.MarkRegionAsCpuModified(allocation_base, page_size);
+  tracker.UntrackMemory(allocation_base, region_size * 2);
+  Release(page_manager, memory, region_size * 2);
+  Check(completed_while_upload_blocked &&
+            query_result.load(std::memory_order_relaxed),
+        "upload callback serialized an unrelated tracker region");
+}
+
+void TestDownloadDoesNotSerializeDisjointRegion() {
+  constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
+  constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  auto *memory = Allocate(page_manager, region_size * 2 / page_size);
+  const auto allocation_base = reinterpret_cast<uint64_t>(memory);
+  const auto second_region =
+      (allocation_base & ~(region_size - 1)) + region_size;
+  Check(second_region + page_size <= allocation_base + region_size * 2,
+        "test allocation does not span two tracker regions");
+
+  tracker.ForEachUploadRange(
+      allocation_base, page_size, true, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  tracker.ForEachUploadRange(
+      second_region, page_size, false, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+
+  std::binary_semaphore download_entered{0};
+  std::binary_semaphore finish_download{0};
+  std::binary_semaphore mutation_finished{0};
+  std::jthread downloader([&] {
+    tracker.ForEachDownloadRange<false>(allocation_base, page_size,
+                                        [&](uint64_t, uint64_t) noexcept {
+                                          download_entered.release();
+                                          finish_download.acquire();
+                                        });
+  });
+  download_entered.acquire();
+  std::jthread mutation([&] {
+    tracker.MarkRegionAsGpuModified(second_region, page_size);
+    mutation_finished.release();
+  });
+
+  const bool completed_while_download_blocked =
+      mutation_finished.try_acquire_for(std::chrono::seconds(5));
+  finish_download.release();
+  downloader.join();
+  mutation.join();
+
+  const bool both_gpu_owned =
+      tracker.IsRegionGpuModified(allocation_base, page_size) &&
+      tracker.IsRegionGpuModified(second_region, page_size);
+  tracker.UnmarkRegionAsGpuModified(allocation_base, page_size);
+  tracker.UnmarkRegionAsGpuModified(second_region, page_size);
+  tracker.MarkRegionAsCpuModified(allocation_base, page_size);
+  tracker.MarkRegionAsCpuModified(second_region, page_size);
+  tracker.UntrackMemory(allocation_base, region_size * 2);
+  Release(page_manager, memory, region_size * 2);
+  Check(completed_while_download_blocked && both_gpu_owned,
+        "download callback serialized an unrelated tracker region");
+}
+
 void TestGpuUnmarkUsesRegionMask() {
   constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
   constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
@@ -683,6 +827,7 @@ int main(int argc, char **argv) {
   }
   TestRangeSet();
   TestQueriesDoNotRequireMappedOwnership();
+  TestConcurrentRegionPublication();
   TestCpuDirtyUpload();
   TestRangeInvalidation();
   TestGpuReacquisitionAfterInvalidation();
@@ -690,6 +835,8 @@ int main(int argc, char **argv) {
   TestExactDirtyIntervalsSharingTrackerPage();
   TestGpuDownloadProtectionMirrors();
   TestCrossRegionUpload();
+  TestUploadDoesNotSerializeDisjointRegion();
+  TestDownloadDoesNotSerializeDisjointRegion();
   TestGpuUnmarkUsesRegionMask();
   TestFullRegionGpuUnmarkBatching();
   TestFatalPaths();

tests/ShaderRecompilerComputeTests.cpp

@@ -1323,10 +1323,28 @@ public:
 		const auto         external_wait_tick  = scheduler.CurrentTick();
 		SubmitInfo         external_wait;
 		external_wait.AddWait(external_timeline, external_wait_value);
-		scheduler.Flush(external_wait);
-		Require("SchedulerTimeline", "external timeline blocked",
-		        !scheduler.IsFree(external_wait_tick),
-		        "submission ignored the external timeline wait value");
+		constexpr size_t blocked_submission_count = 6;
+		std::array<vk::CommandBuffer, blocked_submission_count> blocked_handles {};
+		for (size_t i = 0; i < blocked_submission_count; ++i) {
+			blocked_handles[i] = scheduler.Current().Handle();
+			if (i == 0) {
+				scheduler.Flush(external_wait);
+			} else {
+				scheduler.Flush();
+			}
+		}
+		bool distinct_handles = true;
+		for (size_t i = 0; i < blocked_handles.size(); ++i) {
+			for (size_t j = i + 1; j < blocked_handles.size(); ++j) {
+				distinct_handles &= blocked_handles[i] != blocked_handles[j];
+			}
+		}
+		const auto last_blocked_tick = scheduler.CurrentTick() - 1;
+		Require("SchedulerTimeline", "external timeline overflow",
+		        distinct_handles && last_blocked_tick ==
+		                                external_wait_tick + blocked_submission_count - 1 &&
+		            !scheduler.IsFree(last_blocked_tick),
+		        "blocked submissions did not grow the command-buffer pool");
 		vk::SemaphoreSignalInfo signal_info {};
 		signal_info.sType     = vk::StructureType::eSemaphoreSignalInfo;
 		signal_info.semaphore = external_timeline;
@@ -1334,8 +1352,14 @@ public:
 		Require("SchedulerTimeline", "external timeline signal",
 		        m_runtime_context.device.signalSemaphore(&signal_info) == vk::Result::eSuccess,
 		        "failed to signal the external timeline semaphore");
-		scheduler.Wait(external_wait_tick);
+		scheduler.Wait(last_blocked_tick);
 		m_runtime_context.device.destroySemaphore(external_timeline, nullptr);
+		scheduler.Flush();
+		scheduler.Flush();
+		Require("SchedulerTimeline", "timeline pool reuse",
+		        std::ranges::find(blocked_handles, scheduler.Current().Handle()) !=
+		            blocked_handles.end(),
+		        "completed command buffers were not reused after timeline progress");
 		scheduler.Shutdown();
 
 		CommandScheduler draining(Renderer(), m_runtime_context);
@@ -1583,6 +1607,145 @@ public:
 		        order == 2 && gpu_thread != caller_thread && nested_thread == gpu_thread,
 		        "host command did not run on the GPU thread or nested sync deadlocked");
 
+		{
+			const auto make_release_mem = [](uint32_t data_sel, uint32_t interrupt_selector,
+			                                 void* destination, uint64_t value) {
+				std::array<uint32_t, 8> packet {};
+				const auto              address = reinterpret_cast<uint64_t>(destination);
+				constexpr uint32_t       eop_none = 0x28u;
+				constexpr uint32_t       gcr_gl2_writeback = 1u << 9u;
+				packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
+				packet[1] = eop_none | (gcr_gl2_writeback << 12u);
+				packet[2] = (interrupt_selector << 24u) | (data_sel << 29u);
+				packet[3] = static_cast<uint32_t>(address);
+				packet[4] = static_cast<uint32_t>(address >> 32u);
+				packet[5] = static_cast<uint32_t>(value);
+				packet[6] = static_cast<uint32_t>(value >> 32u);
+				packet[7] = 0x123u;
+				return packet;
+			};
+
+			vk::SemaphoreTypeCreateInfo timeline_type {};
+			timeline_type.sType         = vk::StructureType::eSemaphoreTypeCreateInfo;
+			timeline_type.semaphoreType = vk::SemaphoreType::eTimeline;
+			vk::SemaphoreCreateInfo timeline_create {};
+			timeline_create.sType           = vk::StructureType::eSemaphoreCreateInfo;
+			timeline_create.pNext           = &timeline_type;
+			vk::Semaphore blocked_timeline = nullptr;
+			Require("GpuCommandLane", "blocked timeline create",
+			        m_runtime_context.device.createSemaphore(
+			            &timeline_create, nullptr, &blocked_timeline) == vk::Result::eSuccess &&
+			            blocked_timeline != nullptr,
+			        "failed to create the blocked GPU timeline");
+
+			constexpr uint64_t release_tick = 1;
+			auto&              gpu_scheduler = context.GetCommandScheduler();
+			auto               processor     = std::make_unique<CommandProcessor>(context);
+			gpu.SendCommandSync([&] {
+				SubmitInfo blocked_submit;
+				blocked_submit.AddWait(blocked_timeline, release_tick);
+				gpu_scheduler.Flush(blocked_submit);
+			});
+
+			std::binary_semaphore parser_complete {0};
+			alignas(uint64_t) uint64_t interrupt_only_gds_label = UINT64_MAX;
+			bool                       parser_kept_nonblocking_boundaries = false;
+			std::jthread               no_gpu_wait([&] {
+				gpu.SendCommandSync([&] {
+					processor->BufferInit();
+					const auto tick_before = gpu_scheduler.CurrentTick();
+					processor->TriggerEvent(0x16u, 0u);
+					auto         packet = make_release_mem(0, 0, nullptr, 0);
+					Pm4Execution execution;
+					const auto   result = processor->Process(execution, packet.data(), packet.size());
+					processor->IncrementDe();
+					const bool cache_and_counter_did_not_submit =
+					    result == Pm4ProcessResult::Complete &&
+					    gpu_scheduler.CurrentTick() == tick_before;
+
+					auto gds_interrupt_only =
+					    make_release_mem(5, 1, &interrupt_only_gds_label, 1ull << 16u);
+					Pm4Execution gds_interrupt_execution;
+					const auto   interrupt_tick = gpu_scheduler.CurrentTick();
+					const auto   interrupt_result = processor->Process(
+					    gds_interrupt_execution, gds_interrupt_only.data(), gds_interrupt_only.size());
+					parser_kept_nonblocking_boundaries =
+					    cache_and_counter_did_not_submit &&
+					    interrupt_result == Pm4ProcessResult::Complete &&
+					    gpu_scheduler.CurrentTick() == interrupt_tick + 1 &&
+					    interrupt_only_gds_label == UINT64_MAX;
+				});
+				parser_complete.release();
+			});
+			const bool parser_did_not_wait_gpu =
+			    parser_complete.try_acquire_for(std::chrono::seconds(2));
+
+			vk::SemaphoreSignalInfo signal_info {};
+			signal_info.sType     = vk::StructureType::eSemaphoreSignalInfo;
+			signal_info.semaphore = blocked_timeline;
+			signal_info.value     = release_tick;
+			Require("GpuCommandLane", "blocked timeline signal",
+			        m_runtime_context.device.signalSemaphore(&signal_info) == vk::Result::eSuccess,
+			        "failed to release the blocked GPU timeline");
+			no_gpu_wait.join();
+			gpu.SendCommandSync([&] { gpu_scheduler.Finish(); });
+			m_runtime_context.device.destroySemaphore(blocked_timeline, nullptr);
+			Require("GpuCommandLane", "nonblocking GPU packets",
+			        parser_did_not_wait_gpu && parser_kept_nonblocking_boundaries,
+			        "a cache event, GL2 writeback, DE counter, or interrupt boundary used the wrong "
+			        "submission behavior");
+
+			alignas(uint64_t) uint64_t release_label = 0;
+			alignas(uint64_t) uint64_t gds_label     = UINT64_MAX;
+			bool                       release_mem_submission_counts = false;
+			gpu.SendCommandSync([&] {
+				processor->BufferInit();
+
+				auto         immediate = make_release_mem(1, 0, &release_label, 0x11223344u);
+				Pm4Execution immediate_execution;
+				const auto   immediate_tick = gpu_scheduler.CurrentTick();
+				const auto   immediate_result =
+				    processor->Process(immediate_execution, immediate.data(), immediate.size());
+				const bool immediate_split_once =
+				    gpu_scheduler.CurrentTick() == immediate_tick + 1;
+
+				auto         gds = make_release_mem(5, 0, &gds_label, 1ull << 16u);
+				Pm4Execution gds_execution;
+				const auto   gds_tick   = gpu_scheduler.CurrentTick();
+				const auto   gds_result = processor->Process(gds_execution, gds.data(), gds.size());
+				const bool   gds_waited_once = gpu_scheduler.CurrentTick() == gds_tick + 1;
+
+				auto         interrupt_only = make_release_mem(0, 4, nullptr, 0);
+				Pm4Execution interrupt_execution;
+				const auto   interrupt_tick = gpu_scheduler.CurrentTick();
+				const auto interrupt_result = processor->Process(
+				    interrupt_execution, interrupt_only.data(), interrupt_only.size());
+				const bool interrupt_split_once =
+				    gpu_scheduler.CurrentTick() == interrupt_tick + 1;
+
+				auto         gds_interrupt = make_release_mem(5, 2, &gds_label, 1ull << 16u);
+				Pm4Execution gds_interrupt_execution;
+				const auto   gds_interrupt_tick = gpu_scheduler.CurrentTick();
+				const auto   gds_interrupt_result = processor->Process(
+				    gds_interrupt_execution, gds_interrupt.data(), gds_interrupt.size());
+				const bool gds_interrupt_waited_once =
+				    gpu_scheduler.CurrentTick() == gds_interrupt_tick + 1;
+
+				release_mem_submission_counts =
+				    immediate_result == Pm4ProcessResult::Complete && immediate_split_once &&
+				    gds_result == Pm4ProcessResult::Complete && gds_waited_once &&
+				    interrupt_result == Pm4ProcessResult::Complete && interrupt_split_once &&
+				    gds_interrupt_result == Pm4ProcessResult::Complete &&
+				    gds_interrupt_waited_once;
+			});
+			gpu.SendCommandSync([&] { gpu_scheduler.Finish(); });
+			Require("GpuCommandLane", "RELEASE_MEM submission counts",
+			        release_mem_submission_counts &&
+			            static_cast<uint32_t>(release_label) == 0x11223344u &&
+			            static_cast<uint32_t>(gds_label) == 0,
+			        "RELEASE_MEM lost its required split/readback or retained a redundant GPU wait");
+		}
+
 		alignas(uint32_t) uint32_t packet_marker_a = 0;
 		alignas(uint32_t) uint32_t packet_marker_b = 0;
 		const auto write_packet = [](uint32_t* packet, uint32_t* destination, uint32_t value) {
@@ -18396,14 +18559,16 @@ void CheckPm4CeCompletion(RenderContext& renderer) {
 	uint32_t         suffix  = 0;
 	const auto       address = reinterpret_cast<uint64_t>(&suffix);
 
-	std::array<uint32_t, 7> commands {};
+	std::array<uint32_t, 9> commands {};
 	commands[0] = 0xc0008600u;
 	commands[1] = 1;
-	commands[2] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
+	commands[2] = 0xc0008500u;
 	commands[3] = 0;
-	commands[4] = static_cast<uint32_t>(address);
-	commands[5] = static_cast<uint32_t>(address >> 32u);
-	commands[6] = 33;
+	commands[4] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
+	commands[5] = 0;
+	commands[6] = static_cast<uint32_t>(address);
+	commands[7] = static_cast<uint32_t>(address >> 32u);
+	commands[8] = 33;
 
 	processor.ResetDeCe();
 	Pm4Execution execution;
@@ -18413,11 +18578,30 @@ void CheckPm4CeCompletion(RenderContext& renderer) {
 	            suffix == 0,
 	        "DE did not wait for an active CE stream");
 
-	processor.SetCeComplete(true);
-	Require("Pm4CeCompletion", "complete",
+	std::array<uint32_t, 2> ce_increment {0xc0008400u, 1};
+	Pm4Execution            ce_execution;
+	Require("Pm4CeCompletion", "CE increment",
+	        processor.Process(ce_execution, ce_increment.data(), ce_increment.size()) ==
+	            Pm4ProcessResult::Complete,
+	        "CE counter increment did not complete");
+	Require("Pm4CeCompletion", "counter gate",
 	        processor.Process(execution, commands.data(), commands.size()) ==
 	                Pm4ProcessResult::Complete &&
 	            suffix == 33,
+	        "DE did not resume after CE advanced or failed to increment its counter");
+
+	suffix = 0;
+	Pm4Execution balanced_execution;
+	Require("Pm4CeCompletion", "balanced wait",
+	        processor.Process(balanced_execution, commands.data(), commands.size()) ==
+	                Pm4ProcessResult::Blocked &&
+	            suffix == 0,
+	        "balanced CE/DE counters did not gate the next DE packet");
+	processor.SetCeComplete(true);
+	Require("Pm4CeCompletion", "stream complete",
+	        processor.Process(balanced_execution, commands.data(), commands.size()) ==
+	                Pm4ProcessResult::Complete &&
+	            suffix == 33,
 	        "DE remained blocked after the CE stream completed");
 	std::printf("[host]    %-32s ok\n", "Pm4CeCompletion");
 }