shader: hash only the program code and literal buffer

src/graphics/shader/shader.cpp

@@ -1,5 +1,6 @@
 #include "graphics/shader/shader.h"
 
+#include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common.h"
 #include "common/emulatorConfig.h"
@@ -19,6 +20,7 @@
 #include <algorithm>
 #include <bit>
 #include <cstdio>
+#include <cstring>
 #include <memory>
 #include <mutex>
 #include <span>
@@ -59,15 +61,33 @@ void ShaderInit() {
 	g_shader_map = std::make_unique<std::unordered_map<uint64_t, ShaderMapEntry>>();
 }
 
+// The program's code and, at align16(code size), its literal buffer, as sized by the 0x30-byte
+// footer that ends a binary; the whole binary when the footer is unknown.
+static uint32_t ShaderProgramSize(const uint32_t* binary, uint32_t size) {
+	constexpr uint32_t FooterSize = 0x30;
+	if (size < FooterSize) return size;
+	// Magic, then version, code size and literal buffer size at +0x10.
+	const auto* footer = binary + (size - FooterSize) / 4;
+	const auto  end    = Common::AlignUp(uint64_t {footer[5]}, 16) + footer[6];
+	if (std::memcmp(footer, "barefoot", 8) != 0 || footer[4] != 1 || footer[5] == 0 ||
+	    end % 4 != 0 || end > size - FooterSize) {
+		return size;
+	}
+	return static_cast<uint32_t>(end);
+}
+
 void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data) {
 	EXIT_IF(g_shader_map == nullptr);
 
 	if (addr == 0 || data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
-		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n",
+		EXIT("ShaderMapUserData shader=0x%016" PRIx64 " has invalid shader_size=0x%08" PRIx32 "\n",
 		     addr, data.code_size_bytes);
 	}
-	const auto hash = XXH3_64bits(reinterpret_cast<const void*>(addr), data.code_size_bytes);
+	const auto*   binary = reinterpret_cast<const uint32_t*>(addr);
 	ShaderMapData mapped {data};
+	// Translation reads the bytes the hash covers.
+	mapped.code_size_bytes = ShaderProgramSize(binary, data.code_size_bytes);
+	const auto hash        = XXH3_64bits(binary, mapped.code_size_bytes);
 	mapped.vertex_error =
 	    ShaderReadVertexMetadata(data, HW::UserSgprInfo::SGPRS_MAX, mapped.vertex);
 	std::scoped_lock lock(g_shader_map_mutex);

tests/shaderCfgTests.cpp

@@ -1941,6 +1941,42 @@ void TestVertexBufferGrouping() {
   Check(!input.has_dynamic_vertex_fetch, "remapping a shader kept its previous vertex metadata");
 }
 
+void TestShaderProgramHash() {
+  using Binary = std::array<uint32_t, 22>;
+  // One code word, a 16-byte literal buffer at 16, 8 bytes of metadata, then the 0x30-byte
+  // footer, whose key and checksum cover the whole binary.
+  const auto Make = [](uint32_t literal, uint32_t metadata, uint32_t version = 1,
+                       uint32_t code_size = 4) {
+    Binary words{};
+    words[0] = EncodeSopp(0x01);
+    words[4] = literal;
+    words[8] = metadata;
+    std::memcpy(&words[10], "barefoot", 8);
+    words[12] = metadata;
+    words[14] = version;
+    words[15] = code_size;
+    words[16] = 16;
+    words[18] = metadata;
+    return words;
+  };
+  const auto Map = [](const Binary &binary) {
+    ShaderMappedData mapped{};
+    mapped.code_size_bytes = sizeof(binary);
+    HW::ComputeShaderInfo regs{};
+    regs.cs_regs.data_addr = reinterpret_cast<uint64_t>(binary.data());
+    ShaderMapUserData(regs.cs_regs.data_addr, mapped);
+    ShaderComputeInputInfo input{};
+    return PrepareProgram(regs, input);
+  };
+  Check(Map(Make(7, 1)).hash == Map(Make(7, 2)).hash,
+        "binaries differing only in metadata got different programs");
+  Check(Map(Make(7, 1)).hash != Map(Make(8, 1)).hash, "the program hash missed the literal buffer");
+  Check(Map(Make(7, 1)).code.size() == 8, "translation must read only the hashed program bytes");
+  Check(Map(Make(7, 1, 2)).hash != Map(Make(7, 2, 2)).hash &&
+            Map(Make(7, 1, 1, 0)).hash != Map(Make(7, 2, 1, 0)).hash,
+        "a binary with an unknown footer must hash whole");
+}
+
 void TestNggVertexEntryState() {
   using namespace ShaderRecompiler;
   // PPSA03309 shader 5a39eb2021a5d2c1: retain its NGG prologue and replace
@@ -17221,6 +17257,7 @@ int main() {
   TestNewShaderRecompilerSMovB32();
   TestShaderStageBarriers();
   TestVertexBufferGrouping();
+  TestShaderProgramHash();
   TestNggVertexEntryState();
   TestNewShaderRecompilerClipDisabledPosition();
   TestNewShaderRecompilerAuxPositionExports();