gpu: write end-of-pipe labels beside GPU-written bytes to the backing store

src/graphics/guest_gpu/graphicsRun.cpp

@@ -976,16 +976,28 @@ void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector
 	// Dumped occlusion counters are written before any end-of-pipe signal reaches the guest.
 	m_renderer.GetOcclusionQueries().Publish();
 
+	// A label beside GPU-written bytes goes to the backing store, so they are not downloaded.
+	// Other labels go through the guest mapping: its fault keeps a clean cached copy current and
+	// downloads GPU-written bytes under the label before the store.
+	const auto write_label = [this, address](const void* value, uint64_t size) {
+		const auto vaddr   = reinterpret_cast<uint64_t>(address);
+		auto&      buffers = m_renderer.GetBufferCache();
+		if (!GuestRange {vaddr, size}.Valid() || !buffers.IsRegionGpuModified(vaddr, size) ||
+		    buffers.HasGpuDirtyBytes(vaddr, size) ||
+		    !LibKernel::Memory::TryWriteBacking(vaddr, value, size)) {
+			std::memcpy(address, value, size);
+		}
+	};
 	// EndOfPipeData: kNone, kImmediate32b, kImmediate64b, kRefClk; 5 reads GDS.
 	switch (data_sel) {
 		case 0: break;
 		case 1: {
 			const auto value = static_cast<uint32_t>(data);
-			std::memcpy(address, &value, sizeof(value));
+			write_label(&value, sizeof(value));
 			break;
 		}
 		case 3: data = Sync::ReadReferenceClock(); [[fallthrough]];
-		case 2: std::memcpy(address, &data, sizeof(data)); break;
+		case 2: write_label(&data, sizeof(data)); break;
 		case 5: {
 			GetScheduler().Finish();
 			const auto gds = static_cast<uint32_t>(data);

tests/ShaderRecompilerComputeTests.cpp

@@ -3942,6 +3942,62 @@ public:
             "write invalidation without a remaining copy did not establish CPU "
             "ownership");
 
+    // A RELEASE_MEM label beside GPU-written bytes lands without downloading
+    // them; one over GPU-written bytes downloads them first, and one on a clean
+    // cached page leaves it CPU-modified for the next upload.
+    constexpr uint64_t label_gpu_address = fault_base + 0xf000;
+    constexpr uint64_t label_beside_address = fault_base + 0xf100;
+    constexpr uint64_t label_clean_address = memory_dst + 0x100;
+    constexpr uint32_t label_gpu_value = 0x7e57ab1eu;
+    constexpr uint64_t label_value = 0x0123456789abcdefull;
+    gpu.SendCommandSync([&] {
+      auto dirty = buffer_cache.ObtainBuffer(
+          label_gpu_address, sizeof(label_gpu_value), true, false);
+      Require("GpuCommandLane", "label page setup",
+              dirty.first != nullptr &&
+                  buffer_cache.IsRegionRegistered(label_clean_address,
+                                                  sizeof(label_value)) &&
+                  !buffer_cache.IsRegionCpuModified(label_clean_address,
+                                                    sizeof(label_value)) &&
+                  !buffer_cache.IsRegionGpuModified(label_clean_address,
+                                                    sizeof(label_value)),
+              "label test has no GPU-written page or clean cached page");
+      buffer_cache.FillBuffer(label_gpu_address, sizeof(label_gpu_value),
+                              label_gpu_value, false);
+    });
+    const auto release_label = [&](uint64_t address) {
+      const auto packet = make_interrupt_packet(
+          2, 0, reinterpret_cast<void *>(address), label_value, 0);
+      gpu.Submit(packet);
+      gpu.WaitForIdle();
+      uint64_t label = 0;
+      return LibKernel::Memory::TryReadBacking(address, &label,
+                                               sizeof(label)) &&
+             label == label_value;
+    };
+    LibKernel::Memory::InstallGpuResources(&resources);
+    const bool label_beside_stored =
+        release_label(label_beside_address) &&
+        buffer_cache.HasGpuDirtyBytes(label_gpu_address,
+                                      sizeof(label_gpu_value));
+    const bool label_over_stored =
+        release_label(label_gpu_address) &&
+        !buffer_cache.HasGpuDirtyBytes(label_gpu_address, sizeof(label_value));
+    const bool label_clean_stored =
+        release_label(label_clean_address) &&
+        buffer_cache.IsRegionCpuModified(label_clean_address,
+                                         sizeof(label_value));
+    LibKernel::Memory::InstallGpuResources(nullptr);
+    Require("GpuCommandLane", "RELEASE_MEM label beside GPU-written bytes",
+            label_beside_stored,
+            "a label downloaded the GPU-written bytes beside it");
+    Require("GpuCommandLane", "RELEASE_MEM label over GPU-written bytes",
+            label_over_stored,
+            "a label over GPU-written bytes left them to a later download");
+    Require("GpuCommandLane", "RELEASE_MEM label on a clean cached page",
+            label_clean_stored,
+            "a label left a clean cached copy of its page stale");
+
     resources.UnmapMemory(fault_base, fault_size);
     Require("GpuCommandLane", "processor fault unmap",
             Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,