shader: enumerate set ranks of eight-word light bitmaps

src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp

@@ -377,19 +377,21 @@ bool ResolveSelectorKeys(const ResourcePlan& program, const DescriptorSource::In
 		}
 	} else if (selector->bitset) {
 		ShaderBufferResource material;
-		std::array<uint32_t, 4> masks {};
+		std::array<uint32_t, 8> storage {};
+		const auto record_words = std::min<size_t>(selector->stride / 4u, storage.size());
+		const auto masks = std::span(storage).first(record_words);
 		uint32_t first = 0, count = 0;
 		if (!DecodeBufferDescriptor(material_value, material) ||
 		    !WalkedValue(program, indirect.selector_first, first) ||
 		    !EvaluateIndirectCount(program, indirect, runtime, count) || count != masks.size() * 32u ||
-		    selector->stride != sizeof(masks) || selector->offset != 0u ||
-		    material.Type() != 0u || material.Stride() != sizeof(masks) ||
+		    selector->stride != masks.size_bytes() || selector->offset != 0u ||
+		    material.Type() != 0u || material.Stride() != selector->stride ||
 		    material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u ||
 		    (material.Base48() & 3u) != 0u) return false;
 		if (material.Format() != Prospero::BufferFormat::kInvalid) {
 			const auto words = material.GetSize() / sizeof(uint32_t);
-			// Bitmap storage can grow with resolution while its candidate domain stays
-			// at 128 keys. Bound scratch memory, preserving each coherent read's capture.
+			// Bitmap storage can grow with resolution while its candidate domain stays at 32 keys
+			// per record word. Bound scratch memory, preserving each coherent read's capture.
 			for (uint64_t first_word = 0; first_word < words; first_word += keys.size()) {
 				keys.resize(std::min<uint64_t>(words - first_word, MaxIndirectDescriptorProbes));
 				if (!ReadScalarTable(material.Base48(), material.GetSize(), first_word * 4u,

src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp

@@ -2567,7 +2567,8 @@ private:
 	// Each equation is an inductive subset obligation. Phis require every incoming
 	// value; AND needs either operand. Grounding rejects cycles without an initial
 	// value, then the greatest fixed point checks every initial and carried edge.
-	bool BitsetMaskSubset(Value mask, Value active, Value word, const Inst& read) const {
+	bool BitsetMaskSubset(Value mask, Value active, Value word,
+	                      std::span<const Inst* const> reads) const {
 		struct State {
 			Value value;
 			Value guard;
@@ -2579,7 +2580,8 @@ private:
 			bool all = true;
 			std::vector<size_t> terms;
 		};
-		const auto load_guard = read.Arg(4).Resolve();
+		const auto load_guard = reads[0]->Arg(4).Resolve();
+		const auto words = static_cast<uint32_t>(reads.size()) * 4u;
 		std::vector<State> states(2);
 		std::vector<Equation> equations(2);
 		const auto add = [&](State state) {
@@ -2590,9 +2592,9 @@ private:
 			equations.emplace_back();
 			return states.size() - 1u;
 		};
-		std::array<size_t, 5> roots {add({.value = active})};
-		for (uint32_t component = 0; component < 4u; ++component)
-			roots[component + 1u] = add({mask, active, component, component});
+		std::vector<size_t> roots {add({.value = active})};
+		for (uint32_t component = 0; component < words; ++component)
+			roots.push_back(add({mask, active, component, component}));
 		for (size_t index = 2; index < states.size(); ++index) {
 			const auto state = states[index];
 			const auto* inst = state.value.TryInstruction();
@@ -2642,8 +2644,8 @@ private:
 				switch (inst->GetOpcode()) {
 					case ValueOpcode::CompositeExtractU32x4: {
 						uint32_t component;
-						if (inst->Arg(0).Resolve().TryInstruction() == &read &&
-						    ImmediateU32(inst->Arg(1), component) && component == state.component)
+						if (inst->Arg(0).Resolve().TryInstruction() == reads[state.component / 4u] &&
+						    ImmediateU32(inst->Arg(1), component) && component == state.component % 4u)
 							equation.terms = {1u};
 						break;
 					}
@@ -2667,17 +2669,17 @@ private:
 								const auto* masked = key.TryInstruction();
 								uint32_t bound = 0;
 								if (masked != nullptr && masked->GetOpcode() == ValueOpcode::BitwiseAnd32 &&
-								    DescendingLoop(word, masked->Parent(), bound) && bound == 3u) {
+								    DescendingLoop(word, masked->Parent(), bound) && bound == words - 1u) {
 									for (uint32_t side = 0; side < 2u; ++side) {
 										uint32_t bits;
-										if (ImmediateU32(masked->Arg(side), bits) && (bits & 3u) == 3u)
+										if (ImmediateU32(masked->Arg(side), bits) && (bits & bound) == bound)
 											key = masked->Arg(side ^ 1u).Resolve();
 									}
 								}
 								return EquivalentValue(m_program, key, word);
 							});
 							uint32_t literal;
-							if (ImmediateU32(equality, literal) && literal < 4u) selected_word = literal;
+							if (ImmediateU32(equality, literal) && literal < words) selected_word = literal;
 							equation.terms.push_back(term(inst->Arg(1), inst->Arg(0), selected_word));
 						}
 						if (!selected || !*selected)
@@ -2715,7 +2717,7 @@ private:
 		return std::ranges::all_of(roots, [&](size_t root) { return valid[root]; });
 	}
 
-	bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds,
+	bool MatchBitsetSelector(Value rank, Value active, SelectorBounds bounds, uint32_t words,
 	                         DescriptorSource::IndirectDescriptor& indirect) {
 		const auto selected_value = [&](Value value) {
 			for (;;) {
@@ -2739,7 +2741,7 @@ private:
 		}
 		const auto* word_inst = word.TryInstruction();
 		if (word_inst == nullptr || std::ranges::none_of(bounds, [&](const auto& bound) {
-			return bound.first == word_inst && bound.second == 3u;
+			return bound.first == word_inst && bound.second == words - 1u;
 		})) return false;
 		// FFBH's leading-zero count is inverted again to recover the set-bit rank.
 		for (uint32_t inverse = 0; inverse < 2u; ++inverse) {
@@ -2751,21 +2753,34 @@ private:
 		const auto* scan = bit.TryInstruction();
 		if (scan == nullptr || scan->GetOpcode() != ValueOpcode::FindUMsb32 ||
 		    !ImpliesNonzero(active, scan->Arg(0))) return false;
+		const auto bitmap_load = [&](const Inst& read, uint32_t offset) {
+			const auto index = read.Flags<MemoryFlags>().index;
+			if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(read.Parent()) ||
+			    index >= m_program.memory_info.size()) return false;
+			const auto& memory = m_program.memory_info[index];
+			return memory.kind == ResourceKind::Buffer && !memory.typed && !memory.formatted &&
+			       memory.idxen && !memory.offen && memory.offset == offset &&
+			       memory.data_bits == 32u && memory.data_dwords == 4u &&
+			       read.Arg(2).Resolve() == Value(0u) && read.Arg(3).Resolve() == Value(0u);
+		};
 		for (const auto* block: m_program.blocks) {
 			for (const auto& read: *block) {
-				if (read.GetOpcode() != ValueOpcode::LoadBufferU32x4 || InCycle(block)) continue;
-				const auto index = read.Flags<MemoryFlags>().index;
-				if (index >= m_program.memory_info.size()) continue;
-				const auto& memory = m_program.memory_info[index];
-				if (memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
-				    !memory.idxen || memory.offen || memory.offset != 0u || memory.data_bits != 32u ||
-				    memory.data_dwords != 4u || read.Arg(2).Resolve() != Value(0u) ||
-				    read.Arg(3).Resolve() != Value(0u) ||
-				    !BitsetMaskSubset(scan->Arg(0), active, word, read)) continue;
+				if (!bitmap_load(read, 0u)) continue;
+				// An eight-word record continues in a second four-word load of the same buffer and EXEC.
+				std::array<const Inst*, 2> reads {&read};
+				if (words == 8u)
+					for (const auto* other: m_program.blocks)
+						for (const auto& load: *other)
+							if (reads[1] == nullptr && bitmap_load(load, 16u) &&
+							    EquivalentValue(m_program, load.Arg(0), read.Arg(0)) &&
+							    EquivalentValue(m_program, load.Arg(4), read.Arg(4))) reads[1] = &load;
+				const auto parts = std::span(reads).first(words / 4u);
 				DescriptorSource source;
-				if (!MakeRuntimeTableSource(read, source)) continue;
+				if (std::ranges::find(parts, nullptr) != parts.end() ||
+				    !BitsetMaskSubset(scan->Arg(0), active, word, parts) ||
+				    !MakeRuntimeTableSource(read, source)) continue;
 				indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead {
-				    .source = InternSource(source), .stride = 16u, .offset = 0u, .bitset = true});
+				    .source = InternSource(source), .stride = words * 4u, .offset = 0u, .bitset = true});
 				return true;
 			}
 		}
@@ -2795,7 +2810,8 @@ private:
 		first.SetArg(1, Value(maximum));
 		indirect.selector_first = Value(&first);
 		indirect.key_count = Value(maximum + 1u);
-		if (maximum == 127u) MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, indirect);
+		if (maximum == 127u || maximum == 255u)
+			MatchBitsetSelector(lane->Arg(0), lane->Arg(1), bounds, (maximum + 1u) / 32u, indirect);
 		return true;
 	}
 

tests/ResourceTrackingTests.cpp

@@ -2505,11 +2505,15 @@ void TestDescendingImageRange() {
   enum class Variant {
     Valid, WrongStep, IncrementBypass, MissingNonzero, NegativeInitial,
     Bitmap, BitmapAddsBit, BitmapWrongWord, BitmapUnknownInitial,
-    BitmapByteOffset, BitmapScalarOffset, BitmapOuterClobber, BitmapWrongLoadMask
+    BitmapByteOffset, BitmapScalarOffset, BitmapOuterClobber, BitmapWrongLoadMask,
+    Bitmap8, Bitmap8MisplacedLoad, Bitmap8OtherBuffer, Bitmap8OuterClobber, Bitmap8WrongUpperWord,
+    Bitmap8NarrowMask, Bitmap8OtherGuard
   };
   const auto make_plan = [](Variant variant) {
     Fixture fixture;
     const bool bitmap = variant >= Variant::Bitmap;
+    // Eight-word records are two four-word loads of one buffer record.
+    const uint32_t bitmap_words = variant >= Variant::Bitmap8 ? 8u : 4u;
     auto *entry = fixture.block;
     auto *header = fixture.AddBlock();
     auto *guard = fixture.AddBlock();
@@ -2532,11 +2536,11 @@ void TestDescendingImageRange() {
     branch(latch, header);
     auto &induction = header->AppendNewInst(ValueOpcode::Phi);
     induction.SetFlags(Type::U32);
-    std::array<Inst *, 4> outer_words {}, remaining_words {};
-    std::array<Value, 4> loaded_words {};
+    std::array<Inst *, 8> outer_words {}, remaining_words {};
+    std::array<Value, 8> loaded_words {};
     if (bitmap)
-      for (auto& word : outer_words)
-        word = &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
+      for (uint32_t word = 0; word < bitmap_words; ++word)
+        outer_words[word] = &header->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32));
     const auto bound = fixture.Emit(ValueOpcode::SGreaterThanEqual32,
         {Value(&induction), Value(0u)}, 0, header);
     choose(header, guard, variant == Variant::IncrementBypass ? latch : exit, bound);
@@ -2553,14 +2557,27 @@ void TestDescendingImageRange() {
       read.idxen = true;
       read.offen = variant == Variant::BitmapByteOffset;
       read.data_dwords = 4u;
-      const auto packed = fixture.Emit(ValueOpcode::LoadBufferU32x4,
-          {buffer, lane,
-           Value(variant == Variant::BitmapByteOffset ? 4u : 0u),
-           Value(variant == Variant::BitmapScalarOffset ? 4u : 0u), load_active},
-          fixture.AddMemory(read, 0x10));
-      for (uint32_t word = 0; word < 4u; ++word) {
+      std::array<Value, 2> packed;
+      for (uint32_t part = 0; part < bitmap_words / 4u; ++part) {
+        read.offset = part * (variant == Variant::Bitmap8MisplacedLoad ? 32u : 16u);
+        const auto source = variant == Variant::Bitmap8OtherBuffer && part != 0u
+            ? fixture.Buffer({fixture.UserData(8), fixture.UserData(9), fixture.UserData(10),
+                              fixture.UserData(11)}) : buffer;
+        const auto guard = variant == Variant::Bitmap8OtherGuard && part != 0u
+            ? fixture.Emit(ValueOpcode::LogicalAnd,
+                  {load_active, fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(16u)})})
+            : load_active;
+        packed[part] = fixture.Emit(ValueOpcode::LoadBufferU32x4,
+            {source, lane,
+             Value(variant == Variant::BitmapByteOffset ? 4u : 0u),
+             Value(variant == Variant::BitmapScalarOffset ? 4u : 0u), guard},
+            fixture.AddMemory(read, 0x10 + part * 8u));
+      }
+      for (uint32_t word = 0; word < bitmap_words; ++word) {
+        const bool wrong_word = variant == Variant::BitmapWrongWord ||
+                                (variant == Variant::Bitmap8WrongUpperWord && word >= 4u);
         auto component = fixture.Emit(ValueOpcode::CompositeExtractU32x4,
-            {packed, Value(variant == Variant::BitmapWrongWord ? (word + 1u) % 4u : word)}, 0, entry);
+            {packed[word / 4u], Value((wrong_word ? word + 1u : word) % 4u)}, 0, entry);
         component = fixture.Emit(ValueOpcode::SelectU32,
             {load_active, component, fixture.Emit(ValueOpcode::UndefU32)});
         loaded_words[word] = component;
@@ -2573,7 +2590,7 @@ void TestDescendingImageRange() {
         remaining_words[word]->AddPhiOperand(header, Value(outer_words[word]));
       }
       auto selected = fixture.Emit(ValueOpcode::UndefU32, {}, 0, guard);
-      for (uint32_t word = 0; word < 4u; ++word) {
+      for (uint32_t word = 0; word < bitmap_words; ++word) {
         selected = fixture.Emit(ValueOpcode::SelectU32,
             {fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)}, 0, guard),
              Value(remaining_words[word]), selected}, 0, guard);
@@ -2636,32 +2653,39 @@ void TestDescendingImageRange() {
               {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(1u), bit})})});
       if (variant == Variant::BitmapAddsBit)
         next = fixture.Emit(ValueOpcode::BitwiseOr32, {next, Value(1u << 21u)});
-      for (uint32_t word = 0; word < 4u; ++word) {
+      for (uint32_t word = 0; word < bitmap_words; ++word) {
+        // Relative register writes index with M0 & 0xff; a narrower mask aliases upper words.
+        const auto index = bitmap_words == 4u ? Value(&induction)
+            : fixture.Emit(ValueOpcode::BitwiseAnd32,
+                  {Value(&induction), Value(variant == Variant::Bitmap8NarrowMask ? 3u : 255u)});
         const auto selected_word = fixture.Emit(ValueOpcode::LogicalAnd,
-            {active, fixture.Emit(ValueOpcode::IEqual32, {Value(&induction), Value(word)})});
+            {active, fixture.Emit(ValueOpcode::IEqual32, {index, Value(word)})});
         remaining_words[word]->AddPhiOperand(body, fixture.Emit(ValueOpcode::SelectU32,
             {selected_word, next, Value(remaining_words[word])}));
         auto carried = Value(remaining_words[word]);
-        if (variant == Variant::BitmapOuterClobber)
+        if (variant == Variant::BitmapOuterClobber || variant == Variant::Bitmap8OuterClobber)
           carried = fixture.Emit(ValueOpcode::SelectU32,
               {fixture.Emit(ValueOpcode::IEqual32,
                    {Value(&induction), Value(word + 1u)}, 0, latch),
-               loaded_words[(word + 1u) % 4u], carried}, 0, latch);
+               loaded_words[(word + 1u) % bitmap_words], carried}, 0, latch);
         outer_words[word]->AddPhiOperand(latch, carried);
       }
     }
     const auto step = fixture.Emit(ValueOpcode::IAdd32,
         {Value(&induction), Value(variant == Variant::WrongStep ? UINT32_MAX - 1u : UINT32_MAX)}, 0, latch);
-    induction.AddPhiOperand(entry, Value(variant == Variant::NegativeInitial ? UINT32_MAX : 3u));
+    induction.AddPhiOperand(entry,
+        Value(variant == Variant::NegativeInitial ? UINT32_MAX : bitmap_words - 1u));
     induction.AddPhiOperand(latch, step);
     fixture.PlanAndTrack();
     Check(ordinal.Instruction()->Arg(1).Resolve() == bit.Resolve(),
           "descending range retained impossible relative-register stores");
     const auto &indirect = fixture.program.descriptor_sources[
         fixture.program.info.images[0].source].indirect_descriptor;
-    Check(indirect && indirect->selector.has_value() == (variant == Variant::Bitmap) &&
-              (!indirect->selector || indirect->selector->bitset) &&
-              indirect->key_count == Value(128u) &&
+    const bool selects = variant == Variant::Bitmap || variant == Variant::Bitmap8;
+    Check(indirect && indirect->selector.has_value() == selects &&
+              (!indirect->selector || (indirect->selector->bitset &&
+                                       indirect->selector->stride == bitmap_words * 4u)) &&
+              indirect->key_count == Value(bitmap_words * 32u) &&
               !indirect->selector_first.IsEmpty(),
           "descending bitscan range lost its bound or accepted an unproved bitmap subset");
     return ExtractResourcePlan(fixture.program);
@@ -2701,10 +2725,14 @@ void TestDescendingImageRange() {
   }
 
   auto bitmap_plan = make_plan(Variant::Bitmap);
+  auto wide_plan = make_plan(Variant::Bitmap8);
   for (const auto variant : {Variant::BitmapAddsBit, Variant::BitmapWrongWord,
                              Variant::BitmapUnknownInitial, Variant::BitmapByteOffset,
                              Variant::BitmapScalarOffset, Variant::BitmapOuterClobber,
-                             Variant::BitmapWrongLoadMask})
+                             Variant::BitmapWrongLoadMask, Variant::Bitmap8MisplacedLoad,
+                             Variant::Bitmap8OtherBuffer, Variant::Bitmap8OuterClobber,
+                             Variant::Bitmap8WrongUpperWord, Variant::Bitmap8NarrowMask,
+                             Variant::Bitmap8OtherGuard})
     make_plan(variant);
   // Every descriptor is distinct, so retained keys are visible in the snapshot as well as
   // its lookup table. An unrelated table slot must never be read, even when it is malformed.
@@ -2719,23 +2747,25 @@ void TestDescendingImageRange() {
   const SrtRuntime bitmap_runtime {.user_data = bitmap_data, .userdata = &memory,
       .read_specialization_memory = ReadLinearTestMemory};
   memory.watched_address = bitmap_address;
+  auto *checked_plan = &bitmap_plan;
   const auto check_bitmap = [&](std::initializer_list<uint32_t> ranks, uint32_t count) {
     bitmap_data[2] = count;
     memory.watched_reads = memory.descriptor_reads = 0u;
     std::vector<uint32_t> keys;
     for (const auto rank : ranks) keys.push_back(count - rank);
     std::ranges::sort(keys);
-    Check(MaterializeResources(bitmap_plan, bitmap_runtime, snapshot, specialization) &&
+    const auto dwords = bitmap_data[6] * (bitmap_data[5] >> 18u);
+    Check(MaterializeResources(*checked_plan, bitmap_runtime, snapshot, specialization) &&
               memory.watched_reads == uint32_t(bitmap_data[6] != 0u) &&
               (bitmap_data[6] == 0u || memory.watched_dwords ==
-                  std::min(bitmap_data[6] * 4u, MaxIndirectDescriptorProbes)) &&
+                  std::min(dwords, MaxIndirectDescriptorProbes)) &&
               memory.descriptor_reads == keys.size() &&
               snapshot.images.size() == std::max<size_t>(keys.size(), 1u),
           "bitmap selector lost bounded reads or retained unreachable descriptors");
-    for (uint32_t first = 0; first < bitmap_data[6] * 4u; first += MaxIndirectDescriptorProbes)
+    for (uint32_t first = 0; first < dwords; first += MaxIndirectDescriptorProbes)
       Check(std::ranges::find(snapshot.specialization_reads,
                 std::pair{uint64_t{bitmap_address} + first * 4u,
-                    uint64_t{std::min(bitmap_data[6] * 4u - first, MaxIndirectDescriptorProbes)} * 4u}) !=
+                    uint64_t{std::min(dwords - first, MaxIndirectDescriptorProbes)} * 4u}) !=
                 snapshot.specialization_reads.end(),
             "bitmap chunk was omitted from same-dispatch write-alias protection");
     if (keys.empty()) {
@@ -2787,8 +2817,8 @@ void TestDescendingImageRange() {
   memory.fail_address = UINT64_MAX;
 
   const auto valid_data = bitmap_data;
-  for (const auto [word, value] : std::array<std::pair<uint32_t, uint32_t>, 6>{{
-           {4u, bitmap_address + 1u}, {5u, 8u << 16u}, {5u, (16u << 16u) | (1u << 31u)},
+  for (const auto [word, value] : std::array<std::pair<uint32_t, uint32_t>, 7>{{
+           {4u, bitmap_address + 1u}, {5u, 8u << 16u}, {5u, 32u << 16u}, {5u, (16u << 16u) | (1u << 31u)},
            {7u, 0x5204u | (1u << 23u)}, {7u, 0x5204u | (1u << 28u)},
            {7u, 0x5204u | (1u << 30u)}}}) {
     bitmap_data[word] = value;
@@ -2828,6 +2858,23 @@ void TestDescendingImageRange() {
               return read.first >= memory.fail_address;
             }),
         "a complete bitmap domain read redundant chunks or lost candidate keys");
+
+  // Eight-word records keep their upper words' ranks, and a key below the table is never read.
+  checked_plan = &wide_plan;
+  bitmap_data = valid_data;
+  bitmap_data[5] = 32u << 16u;
+  bitmap_data[6] = 2u;
+  std::fill_n(memory.words.begin() + bitmap_word, 16u, 0u);
+  memory.words[bitmap_word] = 1u | (1u << 2u);
+  memory.words[bitmap_word + 12u] = 1u << 1u;
+  memory.fail_address = 0x2000u - 32u;
+  check_bitmap({0u, 2u, 129u}, 2u);
+  memory.fail_address = UINT64_MAX;
+  for (const auto stride : {16u, 64u}) {
+    bitmap_data[5] = stride << 16u;
+    Check(!MaterializeResources(wide_plan, bitmap_runtime, snapshot, specialization),
+          "an eight-word bitmap selector accepted a different record stride");
+  }
 }
 
 void TestUniformizedMaterialImageKeys() {