mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-27 07:52:39 +00:00
shader: fold complementary masks so wave reductions see full EXEC
Co-authored-by: Senaxx <michelbeld@gmail.com> Co-authored-by: Mac <me@itsmemac.com>
This commit is contained in:
committed
1 parent
6357740fa1
commit
ce5e57aa36
3 files changed
+63
-89
No files matched your search
@@ -167,6 +167,16 @@ bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Whether one operand is the other's negation.
|
||||
bool IsComplementPair(const Inst& inst, ValueOpcode negation) {
|
||||
for (uint32_t arg = 0; arg < 2u; ++arg) {
|
||||
const auto* negate = Arg(inst, arg).TryInstruction();
|
||||
if (negate != nullptr && negate->GetOpcode() == negation &&
|
||||
Arg(*negate, 0) == Arg(inst, arg ^ 1u)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FoldSelect(Inst& inst) {
|
||||
const auto condition = Arg(inst, 0);
|
||||
const auto true_value = Arg(inst, 1);
|
||||
@@ -626,8 +636,9 @@ void FoldInstruction(Block& block, Block::iterator instruction,
|
||||
case ValueOpcode::BitwiseXor32:
|
||||
if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
|
||||
return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b;
|
||||
})) {
|
||||
ReplaceBinaryIdentity(inst, Type::U32, 0u);
|
||||
}) && !ReplaceBinaryIdentity(inst, Type::U32, 0u) &&
|
||||
IsComplementPair(inst, ValueOpcode::BitwiseNot32)) {
|
||||
Replace(inst, Value(UINT32_MAX));
|
||||
}
|
||||
return;
|
||||
case ValueOpcode::BitwiseNot32: {
|
||||
@@ -802,6 +813,8 @@ void FoldInstruction(Block& block, Block::iterator instruction,
|
||||
Replace(inst, lhs.U1() ? lhs : rhs);
|
||||
} else if (IsImmediate(rhs, Type::U1)) {
|
||||
Replace(inst, rhs.U1() ? rhs : lhs);
|
||||
} else if (IsComplementPair(inst, ValueOpcode::LogicalNot)) {
|
||||
Replace(inst, Value(true));
|
||||
} else {
|
||||
const auto* left = lhs.TryInstruction();
|
||||
const auto* right = rhs.TryInstruction();
|
||||
|
||||
@@ -104,11 +104,10 @@ Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_s
|
||||
// Fragment subgroups may have no physical invocation at either endpoint.
|
||||
class WaveReductionMatcher {
|
||||
public:
|
||||
WaveReductionMatcher(Block* block, ValueOpcode operation)
|
||||
: block(block), operation(operation),
|
||||
identity(operation == ValueOpcode::UMin32 ? UINT32_MAX : 0u) {}
|
||||
WaveReductionMatcher(const Block* block, ValueOpcode operation)
|
||||
: block(block), operation(operation) {}
|
||||
|
||||
Value Match(const Inst& reduction, uint32_t wave_size) {
|
||||
Value Match(const Inst& reduction, uint32_t wave_size) const {
|
||||
const auto* low = Get(reduction.Arg(0), ValueOpcode::ReadLane);
|
||||
const auto* high = Get(reduction.Arg(1), ValueOpcode::ReadLane);
|
||||
if (low == nullptr || high == nullptr || low->Arg(0).Resolve() != high->Arg(0).Resolve()) {
|
||||
@@ -124,46 +123,33 @@ public:
|
||||
(lane(low->Arg(1), wave_size - 1u) && lane(high->Arg(1), half - 1u)))) {
|
||||
return {};
|
||||
}
|
||||
if (wave_size == 32u) return MatchScan(Unwrap(low->Arg(0)));
|
||||
const auto* joined = Get(Unwrap(low->Arg(0)), operation);
|
||||
if (wave_size == 32u) return MatchScan(low->Arg(0));
|
||||
const auto* joined = Get(low->Arg(0), operation);
|
||||
if (joined == nullptr) return {};
|
||||
for (size_t index = 0; index < 2; index++) {
|
||||
auto attempt = *this;
|
||||
const auto* cross = attempt.Get(attempt.Unwrap(joined->Arg(index)), ValueOpcode::Permlane16U32);
|
||||
const auto* cross = Get(joined->Arg(index), ValueOpcode::Permlane16U32);
|
||||
if (cross == nullptr) continue;
|
||||
const auto flags = cross->Flags<PermlaneFlags>();
|
||||
auto prefix = attempt.Unwrap(joined->Arg(1 - index));
|
||||
const auto prefix = joined->Arg(1 - index).Resolve();
|
||||
if (!flags.x16 || flags.fetch_inactive || !flags.bound_control ||
|
||||
cross->Arg(1).Resolve() != Value(UINT32_MAX) ||
|
||||
cross->Arg(2).Resolve() != Value(UINT32_MAX) ||
|
||||
!attempt.FullExec(cross->Arg(3)) || attempt.Unwrap(cross->Arg(0)) != prefix) {
|
||||
cross->Arg(2).Resolve() != Value(UINT32_MAX) || !FullExec(cross->Arg(3)) ||
|
||||
cross->Arg(0).Resolve() != prefix) {
|
||||
continue;
|
||||
}
|
||||
if (const auto seed = attempt.MatchScan(prefix); !seed.IsEmpty()) return seed;
|
||||
if (const auto seed = MatchScan(prefix); !seed.IsEmpty()) return seed;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
private:
|
||||
// The row scan's input, whose lanes outside the saved EXEC hold the operation's identity.
|
||||
Value MatchScan(Value prefix) {
|
||||
// The row scan's input.
|
||||
Value MatchScan(Value prefix) const {
|
||||
for (const uint32_t shift: {8u, 4u, 2u, 1u}) {
|
||||
prefix = MatchRow(prefix, shift);
|
||||
}
|
||||
prefix = Unwrap(prefix);
|
||||
if (prefix.IsEmpty() || saved_exec.IsEmpty()) return prefix;
|
||||
const auto* seed = Get(prefix, ValueOpcode::SelectU32);
|
||||
if (seed != nullptr && seed->Arg(2).Resolve() == Value(identity)) {
|
||||
const auto predicate = seed->Arg(0).Resolve();
|
||||
const auto* subset = Get(predicate, ValueOpcode::LogicalAnd);
|
||||
if (predicate == saved_exec ||
|
||||
(subset != nullptr && (subset->Arg(0).Resolve() == saved_exec ||
|
||||
subset->Arg(1).Resolve() == saved_exec))) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Inst* Get(Value value, ValueOpcode opcode) const {
|
||||
auto* inst = value.Resolve().TryInstruction();
|
||||
@@ -171,28 +157,8 @@ private:
|
||||
? inst : nullptr;
|
||||
}
|
||||
|
||||
bool FullExec(Value predicate) {
|
||||
predicate = predicate.Resolve();
|
||||
if (predicate == Value(true)) return true;
|
||||
const auto* either = Get(predicate, ValueOpcode::LogicalOr);
|
||||
if (either == nullptr) return false;
|
||||
for (size_t index = 0; index < 2; index++) {
|
||||
const auto* negate = Get(either->Arg(index), ValueOpcode::LogicalNot);
|
||||
if (negate == nullptr || negate->Arg(0).Resolve() != either->Arg(1 - index).Resolve()) continue;
|
||||
const auto original = negate->Arg(0).Resolve();
|
||||
if (saved_exec.IsEmpty()) saved_exec = original;
|
||||
return saved_exec == original;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Value Unwrap(Value value) {
|
||||
while (const auto* select = Get(value, ValueOpcode::SelectU32)) {
|
||||
if (!FullExec(select->Arg(0))) break;
|
||||
value = select->Arg(1);
|
||||
}
|
||||
return value.Resolve();
|
||||
}
|
||||
// Constant propagation folds the x | ~x EXEC of s_orn2_saveexec to true.
|
||||
static bool FullExec(Value predicate) { return predicate.Resolve() == Value(true); }
|
||||
|
||||
static bool IsRowShift(const Inst& inst, uint32_t shift) {
|
||||
const auto flags = inst.Flags<DppMoveFlags>();
|
||||
@@ -200,24 +166,24 @@ private:
|
||||
flags.bank_mask == 0xf && !flags.fetch_inactive && !flags.bound_control && !flags.dpp8;
|
||||
}
|
||||
|
||||
Value MatchRow(Value value, uint32_t shift) {
|
||||
const auto* update = Get(Unwrap(value), ValueOpcode::DppUpdateU32);
|
||||
Value MatchRow(Value value, uint32_t shift) const {
|
||||
const auto* update = Get(value, ValueOpcode::DppUpdateU32);
|
||||
if (update == nullptr || !IsRowShift(*update, shift) || !FullExec(update->Arg(2))) return {};
|
||||
const auto* combined = Get(update->Arg(0), operation);
|
||||
if (combined == nullptr) return {};
|
||||
for (size_t index = 0; index < 2; index++) {
|
||||
const auto* move = Get(combined->Arg(index), ValueOpcode::DppMoveU32);
|
||||
if (move == nullptr || !IsRowShift(*move, shift) || !FullExec(move->Arg(1))) continue;
|
||||
const auto previous = Unwrap(combined->Arg(1 - index));
|
||||
if (Unwrap(move->Arg(0)) == previous && Unwrap(update->Arg(1)) == previous) return previous;
|
||||
const auto previous = combined->Arg(1 - index).Resolve();
|
||||
if (move->Arg(0).Resolve() == previous && update->Arg(1).Resolve() == previous) {
|
||||
return previous;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Block* block;
|
||||
const Block* block;
|
||||
ValueOpcode operation;
|
||||
uint32_t identity;
|
||||
Value saved_exec;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -630,39 +630,39 @@ void TestReadLaneElimination() {
|
||||
|
||||
void TestCompleteWaveReduction() {
|
||||
enum class Case {
|
||||
Masked, Subset, Literal, LiteralMasked, LiteralMove, Wave32, Wave32Rows,
|
||||
Wave32Maximum, UnrelatedSeed, UnmaskedSymbolic, WrongNeutral, RowMask, BankMask,
|
||||
Masked, ScalarExec, Wave32, Wave32Rows, Wave32Maximum, RowMask, BankMask,
|
||||
Shift, FetchInactive, BoundControl, Dpp8, MoveExec, UpdateExec,
|
||||
Previous, DppSource, CrossExec, CrossRow, CrossSelectorLow,
|
||||
CrossSelectorHigh, CrossFetch, CrossBound, CrossSource, FinalExec,
|
||||
Endpoint, EndpointSource, DifferentWitness, CrossBlock, WrongOperation
|
||||
Endpoint, EndpointSource, CrossBlock, WrongOperation
|
||||
};
|
||||
for (const auto test : {Case::Masked, Case::Subset, Case::Literal,
|
||||
Case::LiteralMasked, Case::LiteralMove, Case::Wave32, Case::Wave32Rows,
|
||||
Case::Wave32Maximum, Case::UnrelatedSeed,
|
||||
Case::UnmaskedSymbolic, Case::WrongNeutral, Case::RowMask, Case::BankMask,
|
||||
for (const auto test : {Case::Masked, Case::ScalarExec, Case::Wave32,
|
||||
Case::Wave32Rows, Case::Wave32Maximum, Case::RowMask, Case::BankMask,
|
||||
Case::Shift, Case::FetchInactive, Case::BoundControl, Case::Dpp8,
|
||||
Case::MoveExec, Case::UpdateExec, Case::Previous, Case::DppSource,
|
||||
Case::CrossExec, Case::CrossRow, Case::CrossSelectorLow,
|
||||
Case::CrossSelectorHigh, Case::CrossFetch, Case::CrossBound,
|
||||
Case::CrossSource, Case::FinalExec, Case::Endpoint, Case::EndpointSource,
|
||||
Case::DifferentWitness, Case::CrossBlock, Case::WrongOperation}) {
|
||||
Case::CrossBlock, Case::WrongOperation}) {
|
||||
Fixture fixture(test == Case::CrossBlock ? 2 : 1);
|
||||
const auto source = fixture.Emit(ValueOpcode::LaneId);
|
||||
const auto mask = fixture.Emit(ValueOpcode::ULessThan32, {source, Value(19u)});
|
||||
const auto other_mask = fixture.Emit(ValueOpcode::IEqual32, {source, Value(7u)});
|
||||
const auto negated = fixture.Emit(ValueOpcode::LogicalNot, {mask});
|
||||
const bool literal = test == Case::Literal || test == Case::LiteralMasked;
|
||||
const auto enabled = literal ? Value(true) :
|
||||
fixture.Emit(ValueOpcode::LogicalOr, {negated, mask});
|
||||
auto seed_mask = test == Case::UnrelatedSeed ? other_mask : mask;
|
||||
if (test == Case::Subset)
|
||||
seed_mask = fixture.Emit(ValueOpcode::LogicalAnd, {other_mask, mask});
|
||||
auto enabled = fixture.Emit(ValueOpcode::LogicalOr, {negated, mask});
|
||||
if (test == Case::ScalarExec) {
|
||||
// s_orn2_saveexec of a mask that is not a ballot, read back as a lane bit.
|
||||
const auto exec = fixture.Emit(ValueOpcode::GetUserData, {Value(static_cast<ScalarReg>(2))});
|
||||
const auto full = fixture.Emit(ValueOpcode::BitwiseOr32,
|
||||
{fixture.Emit(ValueOpcode::BitwiseNot32, {exec}), exec});
|
||||
const auto bit = fixture.Emit(ValueOpcode::BitwiseAnd32, {source, Value(31u)});
|
||||
const auto shifted = fixture.Emit(ValueOpcode::ShiftRightLogical32, {full, bit});
|
||||
enabled = fixture.Emit(ValueOpcode::INotEqual32,
|
||||
{fixture.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(1u)}), Value(0u)});
|
||||
}
|
||||
// A UMax scan seeds lanes outside EXEC with zero.
|
||||
const bool maximum = test == Case::Wave32Maximum;
|
||||
const uint32_t neutral = test == Case::WrongNeutral || maximum ? 0u : UINT32_MAX;
|
||||
const auto seed = test == Case::Literal || test == Case::UnmaskedSymbolic ? source :
|
||||
fixture.Emit(ValueOpcode::SelectU32, {seed_mask, source, Value(neutral)});
|
||||
const uint32_t neutral = maximum ? 0u : UINT32_MAX;
|
||||
const auto seed = fixture.Emit(ValueOpcode::SelectU32, {mask, source, Value(neutral)});
|
||||
auto prefix = fixture.Emit(ValueOpcode::SelectU32, {enabled, seed, source});
|
||||
Value preserved;
|
||||
auto operation = maximum ? ValueOpcode::UMax32 : ValueOpcode::UMin32;
|
||||
@@ -678,8 +678,7 @@ void TestCompleteWaveReduction() {
|
||||
if (test == Case::Dpp8) flags.dpp8 = true;
|
||||
}
|
||||
const auto move = fixture.Emit(ValueOpcode::DppMoveU32,
|
||||
{test == Case::DppSource ? source : prefix,
|
||||
test == Case::MoveExec ? mask : test == Case::LiteralMove ? Value(true) : enabled});
|
||||
{test == Case::DppSource ? source : prefix, test == Case::MoveExec ? mask : enabled});
|
||||
move.Instruction()->SetFlags(flags);
|
||||
const auto combined = fixture.Emit(operation, {move, prefix});
|
||||
const auto update = fixture.Emit(ValueOpcode::DppUpdateU32,
|
||||
@@ -689,11 +688,7 @@ void TestCompleteWaveReduction() {
|
||||
prefix = update;
|
||||
if (shift == 2) preserved = fixture.Emit(ValueOpcode::ReferenceU32, {prefix});
|
||||
}
|
||||
auto cross_exec = test == Case::CrossExec ? mask : enabled;
|
||||
if (test == Case::DifferentWitness) {
|
||||
const auto other_not = fixture.Emit(ValueOpcode::LogicalNot, {other_mask});
|
||||
cross_exec = fixture.Emit(ValueOpcode::LogicalOr, {other_mask, other_not});
|
||||
}
|
||||
const auto cross_exec = test == Case::CrossExec ? mask : enabled;
|
||||
const auto cross = fixture.Emit(ValueOpcode::Permlane16U32,
|
||||
{test == Case::CrossSource ? source : prefix,
|
||||
Value(test == Case::CrossSelectorLow ? 0xfffffff0u : UINT32_MAX),
|
||||
@@ -708,7 +703,7 @@ void TestCompleteWaveReduction() {
|
||||
{test == Case::FinalExec ? mask : enabled, combined, prefix});
|
||||
const uint32_t result_block = test == Case::CrossBlock ? 1u : 0u;
|
||||
// Wave32 reads each half's row endpoint from the scan itself.
|
||||
const bool rows = test == Case::Wave32Rows || maximum;
|
||||
const bool rows = test == Case::Wave32Rows || test == Case::ScalarExec || maximum;
|
||||
const auto endpoints = rows ? prefix : final;
|
||||
uint32_t low_lane = test == Case::Endpoint ? 30u : 31u;
|
||||
if (rows) low_lane = 15u;
|
||||
@@ -727,11 +722,11 @@ void TestCompleteWaveReduction() {
|
||||
fixture.BlockAt().terminator.true_block = &fixture.BlockAt(1);
|
||||
}
|
||||
ValidateProgram(fixture.program, true);
|
||||
const auto stats =
|
||||
EliminateReadLane(fixture.program, test == Case::Wave32 || rows ? 32u : 64u);
|
||||
const bool matches = test == Case::Masked || test == Case::Subset ||
|
||||
test == Case::Literal || test == Case::LiteralMasked || test == Case::LiteralMove ||
|
||||
rows;
|
||||
// The pipeline folds full-EXEC predicates to true before eliminating reads.
|
||||
const uint32_t wave_size = test == Case::Wave32 || rows ? 32u : 64u;
|
||||
ConstantPropagationPass(fixture.program.blocks, wave_size);
|
||||
const auto stats = EliminateReadLane(fixture.program, wave_size);
|
||||
const bool matches = test == Case::Masked || rows;
|
||||
Check(stats.rewritten_reductions == (matches ? 1u : 0u) && stats.rewritten_reads == 0,
|
||||
("wave reduction proof failed case " + std::to_string(static_cast<unsigned>(test))).c_str());
|
||||
if (matches) {
|
||||
@@ -754,7 +749,7 @@ void TestCompleteWaveReduction() {
|
||||
ValidateProgram(fixture.program, true);
|
||||
} else {
|
||||
Check(use.Instruction()->Arg(0).Resolve() == result,
|
||||
"rejected UMin proof changed its original result");
|
||||
"rejected wave reduction proof changed its original result");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user