diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index edde1fc62..527dd20e9 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1885,6 +1885,10 @@ private: } bool Dominates(const Block* initial, const Block* use, Value witness = {}) const { + // The proven local-zero invocation fails every all-lane bypass condition that excludes it, + // such as an all-inactive or an all-nonzero bypass. + const auto* negated = SimplifyGuard(witness).TryInstruction(); + const bool nonzero = negated != nullptr && negated->GetOpcode() == ValueOpcode::LogicalNot; std::vector pending {use}; for (size_t index = 0; index < pending.size(); ++index) { const auto* current = pending[index]; @@ -1893,9 +1897,9 @@ private: for (const auto* previous: current->ImmPredecessors()) { if (!witness.IsEmpty()) { const auto edge = ConditionalEdge(previous, current); - // The proven local-zero invocation makes an all-inactive bypass impossible. - if (edge && edge->lanes == LaneQuantifier::All && !edge->positive && - Implies(witness, edge->condition)) continue; + if (edge && edge->lanes == LaneQuantifier::All && + (edge->positive ? nonzero && Implies(edge->condition, negated->Arg(0)) + : Implies(witness, edge->condition))) continue; } if (std::ranges::find(pending, previous) == pending.end()) pending.push_back(previous); } @@ -2004,7 +2008,10 @@ private: bool LocalZeroWitness(Value guard) const { const auto* equal = SimplifyGuard(guard).TryInstruction(); - if (equal == nullptr || equal->GetOpcode() != ValueOpcode::IEqual32) return false; + const bool negated = equal != nullptr && equal->GetOpcode() == ValueOpcode::LogicalNot; + if (negated) equal = SimplifyGuard(equal->Arg(0)).TryInstruction(); + const auto expected = negated ? ValueOpcode::INotEqual32 : ValueOpcode::IEqual32; + if (equal == nullptr || equal->GetOpcode() != expected) return false; for (uint32_t arg = 0; arg < 2u; ++arg) { if (equal->Arg(arg).Resolve() != Value(0u)) continue; Value induction; @@ -2116,7 +2123,7 @@ private: const auto* read = ActiveValue(value, active).TryInstruction(); if (read == nullptr || read->GetOpcode() != ValueOpcode::LoadSharedU32 || m_program.stage != ShaderType::Compute || - read->Arg(1).Resolve() != Value(true)) return {}; + !Implies(active, read->Arg(1))) return {}; uint64_t group_size = 1u; for (const auto dimension: m_input.compute->threads_num) { if (dimension == 0u || dimension > m_input.compute->wave_size) return {}; @@ -2147,10 +2154,16 @@ private: const auto* limit = BoundedLoop(read_offset.index, read->Parent(), [](const Inst& compare, uint32_t) { return compare.GetOpcode() == ValueOpcode::ULessThan32; }); if (limit == nullptr) return {}; + auto counter_active = active; const auto* counter_read = ActiveValue(limit->Arg(1), active).TryInstruction(); + // Every lane reads the same counter word, so a selected lane carries it. + if (counter_read != nullptr && counter_read->GetOpcode() == ValueOpcode::ReadFirstLane) { + counter_active = counter_read->Arg(1); + counter_read = ActiveValue(counter_read->Arg(0), counter_active).TryInstruction(); + } uint32_t consumed_counter = 0; if (counter_read == nullptr || counter_read->GetOpcode() != ValueOpcode::LoadSharedU32 || - counter_read->Arg(1).Resolve() != Value(true) || + !Implies(counter_active, counter_read->Arg(1)) || !constant_address(*counter_read, consumed_counter)) return {}; struct Initializer { const Inst* inst; uint32_t address; uint32_t words; }; @@ -2373,6 +2386,12 @@ private: const auto maximum = SelectorMaximum(index); if (maximum < MaxIndirectDescriptorProbes) count = Value(maximum + 1u); } + // A record selected from a shared append list is below the list's bound. + const auto* lane = index.Resolve().TryInstruction(); + if (count.IsEmpty() && lane != nullptr && + lane->GetOpcode() == ValueOpcode::ReadFirstLane && + HasActiveLane(lane->Arg(1), lane->Parent())) + count = SharedLoadBound(lane->Arg(0), lane->Arg(1)); } // Raw pointers have no V# bounds for enumerating an unbounded selector domain. if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty() && diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 5e98cd620..ad5d7563b 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -4812,7 +4812,8 @@ void TestSharedAppendSelectorBounds() { Valid, PayloadOob, MissingInit, InactiveInit, UnknownInit, DuplicateAtomic, WrongAtomicGuard, UnguardedCandidate, WrongStep, WrongLocalIndex, Multiwave, OuterReentry, InnerAppendCycle, EarlyCounterRead, WrongConsumerBound, ExtraWriter, CounterAlias, - PayloadWrap, CounterOob + PayloadWrap, CounterOob, Lane, LaneUnimpliedRead, LaneUnimpliedCounter, LaneCounterMask, + LaneOtherBypass, LaneInactiveSelect }; const auto make_plan = [](Variant variant, uint32_t first_base = 1032u, uint32_t consumed_base = 8u) { Fixture fixture; @@ -4845,6 +4846,7 @@ void TestSharedAppendSelectorBounds() { auto *entry = fixture.block; auto *initialize = fixture.AddBlock(); auto *initialized = fixture.AddBlock(); + const bool lane = variant >= Variant::Lane; const auto material = fixture.Buffer({fixture.UserData(0), fixture.UserData(1), fixture.UserData(2), fixture.UserData(3)}); const auto table = fixture.Address(fixture.UserData(4), fixture.UserData(5)); @@ -4863,12 +4865,18 @@ void TestSharedAppendSelectorBounds() { {Value(static_cast(StageInputKind::LocalInvocationId)), Value(1u)}); const auto local = fixture.Emit(ValueOpcode::IAdd32, {x, fixture.Emit(ValueOpcode::ShiftLeftLogical32, {y, Value(3u)})}); - auto zero_lane = fixture.Emit(ValueOpcode::IEqual32, {local, Value(0u)}); + // The negated form skips the initializer only when every lane is nonzero. + const auto local_is = [&](uint32_t value) { + return fixture.Emit(ValueOpcode::LogicalNot, + {fixture.Emit(ValueOpcode::INotEqual32, {local, Value(value)})}); + }; + auto zero_lane = lane ? local_is(0u) : fixture.Emit(ValueOpcode::IEqual32, {local, Value(0u)}); if (variant == Variant::InactiveInit) zero_lane = Value(false); if (variant == Variant::UnknownInit) zero_lane = fixture.Emit(ValueOpcode::LogicalAnd, {zero_lane, fixture.Emit(ValueOpcode::IEqual32, {fixture.UserData(8), Value(0u)})}); - choose(entry, initialize, initialized, zero_lane, CFG::BranchCondition::ExecNonZero); + choose(entry, initialize, initialized, variant == Variant::LaneOtherBypass ? local_is(1u) : zero_lane, + CFG::BranchCondition::ExecNonZero); if (variant != Variant::MissingInit) fixture.Emit(ValueOpcode::WriteSharedU32x2, {Value(0u), Value(0u), Value(0u), zero_lane}, shared(0u), initialize); @@ -4937,13 +4945,25 @@ void TestSharedAppendSelectorBounds() { if (variant == Variant::ExtraWriter) fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(UINT32_MAX), Value(true)}, shared(12u), previous); - if (final_counter.IsEmpty()) - final_counter = fixture.Emit(ValueOpcode::LoadSharedU32, - {Value(0u), Value(true)}, shared(0u), previous); + // Lane variants consume under a lane mask, which their counter and record reads use. + Value lanes(true), other; + if (lane) { + lanes = fixture.Emit(ValueOpcode::ULessThan32, {y, fixture.UserData(8)}, 0, previous); + other = fixture.Emit(ValueOpcode::ULessThan32, {x, fixture.UserData(8)}, 0, previous); + } + auto *guarded = lane ? fixture.AddBlock() : previous; auto *header = fixture.AddBlock(); auto *body = fixture.AddBlock(); auto *latch = fixture.AddBlock(); auto *exit = fixture.AddBlock(); + if (lane) choose(previous, guarded, exit, lanes, CFG::BranchCondition::ExecNonZero); + previous = guarded; + if (final_counter.IsEmpty()) + final_counter = fixture.Emit(ValueOpcode::LoadSharedU32, + {Value(0u), variant == Variant::LaneUnimpliedCounter ? other : lanes}, shared(0u), previous); + if (lane) + final_counter = fixture.Emit(ValueOpcode::ReadFirstLane, + {final_counter, variant == Variant::LaneCounterMask ? other : lanes}, 0, previous); branch(previous, header); auto &index = header->AppendNewInst(ValueOpcode::Phi); index.SetFlags(Type::U32); @@ -4952,12 +4972,22 @@ void TestSharedAppendSelectorBounds() { choose(header, body, exit, in_list, CFG::BranchCondition::VccNonZero); fixture.block = body; const auto selected = fixture.Emit(ValueOpcode::LoadSharedU32, - {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(&index), Value(2u)}), Value(true)}, - shared(consumed_base)); - const auto offset32 = fixture.Emit(ValueOpcode::ShiftLeftLogical32, {selected, Value(5u)}); - const auto offset = fixture.Emit(ValueOpcode::ReadFirstLane, - {fixture.Emit(ValueOpcode::IAdd32, - {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {offset32, Value(2u)}), offset32}), Value(true)}); + {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {Value(&index), Value(2u)}), + variant == Variant::LaneUnimpliedRead ? other : lanes}, shared(consumed_base)); + Value offset; + if (lane) { + // The record is selected from one lane and scaled on the scalar side. + const auto mask = variant == Variant::LaneInactiveSelect + ? fixture.Emit(ValueOpcode::LogicalAnd, {lanes, other}) : lanes; + offset = fixture.Emit(ValueOpcode::IMul32, + {fixture.Emit(ValueOpcode::ReadFirstLane, {selected, mask}), Value(160u)}); + } else { + const auto offset32 = fixture.Emit(ValueOpcode::ShiftLeftLogical32, {selected, Value(5u)}); + offset = fixture.Emit(ValueOpcode::ReadFirstLane, + {fixture.Emit(ValueOpcode::IAdd32, + {fixture.Emit(ValueOpcode::ShiftLeftLogical32, {offset32, Value(2u)}), offset32}), + Value(true)}); + } MemoryInfo scalar; scalar.kind = ResourceKind::ScalarBuffer; scalar.offset = 144u; @@ -4986,7 +5016,8 @@ void TestSharedAppendSelectorBounds() { fixture.PlanAndTrack({.compute = &compute}); const auto &indirect = fixture.program.descriptor_sources[ fixture.program.info.images[0].source].indirect_descriptor; - const bool proven = variant == Variant::Valid || variant == Variant::PayloadOob; + const bool proven = variant == Variant::Valid || variant == Variant::PayloadOob || + variant == Variant::Lane; Check(indirect && indirect->selector && indirect->selector->offset == 144u && indirect->selector->stride == (proven ? 160u : 32u) && indirect->key_count.IsEmpty() != proven && @@ -5003,6 +5034,11 @@ void TestSharedAppendSelectorBounds() { Variant::InnerAppendCycle, Variant::EarlyCounterRead, Variant::WrongConsumerBound, Variant::ExtraWriter, Variant::CounterAlias, Variant::PayloadWrap, Variant::CounterOob}) make_plan(variant); + auto lane_plan = make_plan(Variant::Lane); + for (const auto variant : {Variant::LaneUnimpliedRead, Variant::LaneUnimpliedCounter, + Variant::LaneCounterMask, Variant::LaneOtherBypass, Variant::LaneInactiveSelect}) + CheckFatal([&] { make_plan(variant); }, "not a valid runtime value", + "shared append provenance accepted an unimplied lane mask or initializer bypass"); LinearTestMemory memory; for (uint32_t index = 0; index < 8u; ++index) memory.words[(144u + 160u * index) / 4u] = index + 1u; @@ -5015,7 +5051,7 @@ void TestSharedAppendSelectorBounds() { memory.words[word + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) | (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); } - std::array user_data {0x1000u, 160u << 16u, 8u, 0u, 0x2000u, 0u, 0x1800u, 0u, 0u}; + std::array user_data {0x1000u, 160u << 16u, 8u, 0u, 0x2000u, 0u, 0x1800u, 0u, 8u}; SrtRuntime runtime {.user_data = user_data, .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; FakeGuestMemory fake(ReadRuntimeMemory, &runtime); @@ -5030,10 +5066,11 @@ void TestSharedAppendSelectorBounds() { memory.words[0x800u / 4u] = counts[0]; memory.words[0x804u / 4u] = counts[1]; const auto expected = counts[1]; - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.images.size() == std::max(expected, 1u) && - (expected == 0u || snapshot.images.back().dwords[0] == 0x100u + expected), - "disjoint shared append lists or refreshed counts selected unrelated material keys"); + for (const auto *tested : {&plan, &lane_plan}) + Check(MaterializeResources(*tested, runtime, snapshot, specialization) && + snapshot.images.size() == std::max(expected, 1u) && + (expected == 0u || snapshot.images.back().dwords[0] == 0x100u + expected), + "disjoint shared append lists or refreshed counts selected unrelated material keys"); } memory.words[0x800u / 4u] = 24u; memory.words[0x804u / 4u] = 2u;