gremling/KytyPS5
1
0 Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-28 08:13:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

shader: reuse native emitters and simplify memory dispatch

Browse Source
This commit is contained in:
nmzik committed 2026-09-12 13:42:15 +04:00
1 parent 59ef88c959
commit 20e57994cf
10 files changed
+721 -993
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp
+156 -298
View File
@@ -41,32 +41,6 @@ uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_
return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path); return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
} }
uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high);
}
uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) { uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state); const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state); const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
@@ -84,58 +58,42 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high; return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
} }
uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
const auto lhs = ExtractPair(state, lhs_value); const auto pair = ExtractPair(state, value);
const auto rhs = ExtractPair(state, rhs_value); const auto amount = EmitAndConstant(state, shift, 63u);
const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low); const auto word_shift = EmitAndConstant(state, amount, 31u);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false); const auto at_least_32 =
const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high); Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low); const auto nonzero =
return MakePair(state, low, Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
Binary(state, spv::OpIAdd, TypeU32(state), const auto carry_count = EmitAndConstant(
Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2)); state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u);
}
uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value);
uint32_t low = 0;
uint32_t high = 0;
if (opcode == spv::OpShiftLeftLogical) { if (opcode == spv::OpShiftLeftLogical) {
EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high); const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift);
} else if (opcode == spv::OpShiftRightLogical) {
EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else {
const auto amount =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
const auto word_shift =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
const auto at_least_32 = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount,
ConstantU32(state, 32));
const auto nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
const auto high_shifted =
Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
const auto low_shifted =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
const auto carry_count =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
ConstantU32(state, 31));
const auto carry = const auto carry =
Select(state, TypeU32(state), nonzero, Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count), Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count),
ConstantU32(state, 0)); ConstantU32(state, 0));
const auto low_below_32 = const auto high =
Binary(state, spv::OpBitwiseOr, TypeU32(state), low_shifted, carry); Binary(state, spv::OpBitwiseOr, TypeU32(state),
const auto negative = Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry);
Binary(state, spv::OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
const auto sign_fill = Select(state, TypeU32(state), negative,
ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
return MakePair(state, return MakePair(state,
Select(state, TypeU32(state), at_least_32, high_shifted, low_below_32), Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low),
Select(state, TypeU32(state), at_least_32, sign_fill, high_shifted)); Select(state, TypeU32(state), at_least_32, low, high));
} }
return MakePair(state, low, high); const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
ConstantU32(state, 0));
const auto low = Binary(
state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry);
const auto fill = opcode == spv::OpShiftRightArithmetic
? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31))
: ConstantU32(state, 0);
return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low),
Select(state, TypeU32(state), at_least_32, fill, high));
} }
uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
@@ -176,26 +134,10 @@ uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t valu
: ConstantU32(state, 0); : ConstantU32(state, 0);
const auto low = shift == 32u ? pair.high const auto low = shift == 32u ? pair.high
: Binary(state, opcode, TypeU32(state), pair.high, : Binary(state, opcode, TypeU32(state), pair.high,
ConstantU32(state, shift - 32u)); ConstantU32(state, shift - 32u));
return MakePair(state, low, high); return MakePair(state, low, high);
} }
uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
const auto pair = ExtractPair(state, value);
const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high});
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low});
const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero,
Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
}
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value, uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
bool max_value) { bool max_value) {
const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value) const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
@@ -217,19 +159,6 @@ uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, b
return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value); return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
} }
uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
const auto min_ab = EmitMinMaxF32Value(state, a, b, false);
const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false);
const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state),
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
const auto any_nan =
Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
return Select(state, TypeF32(state), any_nan, min3, median);
}
uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode, uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
std::initializer_list<uint32_t> args) { std::initializer_list<uint32_t> args) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
@@ -271,121 +200,127 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high); return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high);
} }
uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {src});
}
} // namespace } // namespace
uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
uint32_t EmitConvertU16U32(ValueEmitContext& ctx, uint32_t arg0) { const auto min_ab = EmitMinMaxF32Value(state, a, b, false);
auto& state = ctx.state; const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false);
return EmitNative<spv::OpBitwiseAnd, IR::Type::U16>(ctx, arg0, ConstantU32(state, 0xffffu)); const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state),
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
const auto any_nan =
Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
return Select(state, TypeF32(state), any_nan, min3, median);
} }
uint32_t EmitConvertU8U32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
auto& state = ctx.state; const auto pair = ExtractPair(state, value);
return EmitNative<spv::OpBitwiseAnd, IR::Type::U8>(ctx, arg0, ConstantU32(state, 0xffu)); const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high});
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low});
const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero,
Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
} }
uint32_t EmitConvertF16F32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
auto& state = ctx.state; const auto lhs = ExtractPair(state, lhs_value);
const auto pair = state.builder.AllocateId(); const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
return MakePair(state, low,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
}
uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high);
}
uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpBitwiseAnd, IR::Type::U16>(state, arg0, ConstantU32(state, 0xffffu));
}
uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpBitwiseAnd, IR::Type::U8>(state, arg0, ConstantU32(state, 0xffu));
}
uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0) {
const auto pair = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{spv::OpCompositeConstruct, TypeF32Vector(state, 2), pair, arg0, ConstantF32(state, 0)}); {spv::OpCompositeConstruct, TypeF32Vector(state, 2), pair, arg0, ConstantF32(state, 0)});
return EmitPackHalf(state, pair); return EmitPackHalf2x16(state, pair);
} }
uint32_t EmitConvertF32F16(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitF16BitsToF32(state, arg0);
}
uint32_t EmitConvertS32F32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitF32ToU32(state, arg0, true); return EmitF32ToU32(state, arg0, true);
} }
uint32_t EmitConvertU32F32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitF32ToU32(state, arg0, false); return EmitF32ToU32(state, arg0, false);
} }
uint32_t EmitConvertF32S32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0); const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0);
return EmitNative<spv::OpConvertSToF, IR::Type::F32>(ctx, signed_value); return EmitNative<spv::OpConvertSToF, IR::Type::F32>(state, signed_value);
} }
uint32_t EmitCompositeExtractU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) { uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(ctx, arg0, arg1.U32()); return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
} }
uint32_t EmitPackHalf2x16(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state; const auto low = EmitF32ToF16RtzBits(state, arg0);
return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {arg0}); const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
} EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16));
uint32_t EmitPackSnorm2x16(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeU32(state), GLSLstd450PackSnorm2x16, {arg0});
}
uint32_t EmitPackUnorm2x16(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeU32(state), GLSLstd450PackUnorm2x16, {arg0});
}
uint32_t EmitPackFloat2x16Rtz(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
const auto low = EmitF32ToF16RtzBits(state, arg0);
const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16));
return Binary(state, spv::OpBitwiseOr, TypeU32(state), low, high); return Binary(state, spv::OpBitwiseOr, TypeU32(state), low, high);
} }
uint32_t EmitFPAbs32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitFAbsValue(state, arg0);
}
uint32_t EmitFPNeg32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitFNegateValue(state, arg0);
}
uint32_t EmitFPSaturate32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450FClamp, return EmitExt(state, TypeF32(state), GLSLstd450FClamp,
{arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)}); {arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
} }
uint32_t EmitIAdd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitAdd64(state, arg0, arg1);
}
uint32_t EmitISub64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitSub64(state, arg0, arg1);
}
uint32_t EmitIMul64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMul64(state, arg0, arg1);
}
uint32_t EmitSMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMulHigh(state, arg0, arg1, true); return EmitMulHigh(state, arg0, arg1, true);
} }
uint32_t EmitUMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMulHigh(state, arg0, arg1, false); return EmitMulHigh(state, arg0, arg1, false);
} }
uint32_t EmitIAbs32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
const auto value = arg0; const auto value = arg0;
const auto neg = Unary(state, spv::OpSNegate, TypeU32(state), value); const auto neg = Unary(state, spv::OpSNegate, TypeU32(state), value);
const auto negative = const auto negative =
@@ -414,225 +349,148 @@ uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Va
: EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1)); : EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1));
} }
uint32_t EmitBitwiseAnd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1); return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1);
} }
uint32_t EmitBitCount64(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state; const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high); return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
} }
uint32_t EmitFindILsb32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindILsb, {arg0}); const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindILsb, {arg0});
return Unary(state, spv::OpBitcast, TypeU32(state), value); return Unary(state, spv::OpBitcast, TypeU32(state), value);
} }
uint32_t EmitFindUMsb32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindUMsb, {arg0}); const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindUMsb, {arg0});
return Unary(state, spv::OpBitcast, TypeU32(state), value); return Unary(state, spv::OpBitcast, TypeU32(state), value);
} }
uint32_t EmitFindUMsb64(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitFindMsb64(state, arg0);
}
uint32_t EmitSMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMinMaxI32Value(state, arg0, arg1, false); return EmitMinMaxI32Value(state, arg0, arg1, false);
} }
uint32_t EmitSMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMinMaxI32Value(state, arg0, arg1, true); return EmitMinMaxI32Value(state, arg0, arg1, true);
} }
uint32_t EmitUMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMinMaxU32Value(state, arg0, arg1, false); return EmitMinMaxU32Value(state, arg0, arg1, false);
} }
uint32_t EmitUMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMinMaxU32Value(state, arg0, arg1, true); return EmitMinMaxU32Value(state, arg0, arg1, true);
} }
uint32_t EmitSMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitMinMax3(state, arg0, arg1, arg2, true, false); return EmitMinMax3(state, arg0, arg1, arg2, true, false);
} }
uint32_t EmitSMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitMinMax3(state, arg0, arg1, arg2, true, true); return EmitMinMax3(state, arg0, arg1, arg2, true, true);
} }
uint32_t EmitUMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitMinMax3(state, arg0, arg1, arg2, false, false); return EmitMinMax3(state, arg0, arg1, arg2, false, false);
} }
uint32_t EmitUMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitMinMax3(state, arg0, arg1, arg2, false, true); return EmitMinMax3(state, arg0, arg1, arg2, false, true);
} }
uint32_t EmitSMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitMed3(state, arg0, arg1, arg2, true); return EmitMed3(state, arg0, arg1, arg2, true);
} }
uint32_t EmitUMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitMed3(state, arg0, arg1, arg2, false); return EmitMed3(state, arg0, arg1, arg2, false);
} }
uint32_t EmitIEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return CompareEqual64(state, arg0, arg1, false); return CompareEqual64(state, arg0, arg1, false);
} }
uint32_t EmitINotEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return CompareEqual64(state, arg0, arg1, true); return CompareEqual64(state, arg0, arg1, true);
} }
uint32_t EmitULessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan); return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan);
} }
uint32_t EmitSLessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan); return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan);
} }
uint32_t EmitUGreaterThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan); return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
} }
uint32_t EmitFPIsNan32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(ctx, arg0, arg0); return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
} }
uint32_t EmitFPCmpClass32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitClassMaskF32(state, arg0, arg1);
}
uint32_t EmitFPFma32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Fma, {arg0, arg1, arg2});
}
uint32_t EmitFPMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMinMaxF32Value(state, arg0, arg1, false); return EmitMinMaxF32Value(state, arg0, arg1, false);
} }
uint32_t EmitFPMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMinMaxF32Value(state, arg0, arg1, true); return EmitMinMaxF32Value(state, arg0, arg1, true);
} }
uint32_t EmitFPMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitFMinMax3(state, arg0, arg1, arg2, false); return EmitFMinMax3(state, arg0, arg1, arg2, false);
} }
uint32_t EmitFPMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitFMinMax3(state, arg0, arg1, arg2, true); return EmitFMinMax3(state, arg0, arg1, arg2, true);
} }
uint32_t EmitFPMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitFMed3(state, arg0, arg1, arg2);
}
uint32_t EmitFPRecip32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
const auto source = EmitFlushF32DenormToSignedZero(state, arg0); const auto source = EmitFlushF32DenormToSignedZero(state, arg0);
return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), source); return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), source);
} }
uint32_t EmitFPRecipIFlag32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
// Integer-to-float inputs used by IFLAG cannot be denormal. // Integer-to-float inputs used by IFLAG cannot be denormal.
return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), arg0); return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), arg0);
} }
uint32_t EmitFPRecipSqrt32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450InverseSqrt, return EmitExt(state, TypeF32(state), GLSLstd450InverseSqrt,
{EmitFlushF32DenormToSignedZero(state, arg0)}); {EmitFlushF32DenormToSignedZero(state, arg0)});
} }
uint32_t EmitFPSqrt(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Sqrt, return EmitExt(state, TypeF32(state), GLSLstd450Sqrt,
{EmitFlushF32DenormToSignedZero(state, arg0)}); {EmitFlushF32DenormToSignedZero(state, arg0)});
} }
uint32_t EmitFPExp2(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Exp2, return EmitExt(state, TypeF32(state), GLSLstd450Exp2,
{EmitFlushF32DenormToSignedZero(state, arg0)}); {EmitFlushF32DenormToSignedZero(state, arg0)});
} }
uint32_t EmitFPLog2(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Log2, return EmitExt(state, TypeF32(state), GLSLstd450Log2,
{EmitFlushF32DenormToSignedZero(state, arg0)}); {EmitFlushF32DenormToSignedZero(state, arg0)});
} }
uint32_t EmitFPLdexp(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
const auto exponent = Unary(state, spv::OpBitcast, TypeI32(state), arg1); const auto exponent = Unary(state, spv::OpBitcast, TypeI32(state), arg1);
return EmitExt(state, TypeF32(state), GLSLstd450Ldexp, {arg0, exponent}); return EmitExt(state, TypeF32(state), GLSLstd450Ldexp, {arg0, exponent});
} }
uint32_t EmitFPRoundEven32(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPSin(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state; auto source = EmitTrigCycleF32(state, arg0, true);
return EmitExt(state, TypeF32(state), GLSLstd450RoundEven, {arg0});
}
uint32_t EmitFPFloor32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Floor, {arg0});
}
uint32_t EmitFPCeil32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Ceil, {arg0});
}
uint32_t EmitFPTrunc32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Trunc, {arg0});
}
uint32_t EmitFPFract32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Fract, {arg0});
}
uint32_t EmitFPSin(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
auto source = EmitTrigCycleF32(state, arg0, true);
source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu)); source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
return EmitExt(state, TypeF32(state), GLSLstd450Sin, {source}); return EmitExt(state, TypeF32(state), GLSLstd450Sin, {source});
} }
uint32_t EmitFPCos(ValueEmitContext& ctx, uint32_t arg0) { uint32_t EmitFPCos(EmitterState& state, uint32_t arg0) {
auto& state = ctx.state; auto source = EmitTrigCycleF32(state, arg0, false);
auto source = EmitTrigCycleF32(state, arg0, false);
source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu)); source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
return EmitExt(state, TypeF32(state), GLSLstd450Cos, {source}); return EmitExt(state, TypeF32(state), GLSLstd450Cos, {source});
} }
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp
-144
View File
@@ -2,88 +2,6 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
uint32_t false_value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, true_value, false_value});
return ret;
}
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
uint32_t& out_low, uint32_t& out_high) {
const auto amount = EmitAndConstant(state, shift, 63u);
const auto dword_shift = EmitAndConstant(state, amount, 31u);
const auto less32 = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto carry_count_base = state.builder.AllocateId();
const auto carry_count = state.builder.AllocateId();
const auto low_part = state.builder.AllocateId();
const auto high_part = state.builder.AllocateId();
const auto carry_part = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
const auto high_less = state.builder.AllocateId();
const auto high_ge = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
state.builder.AddFunction(
{spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
state.builder.AddFunction(
{spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
state.builder.AddFunction({spv::OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
}
void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
const auto amount = EmitAndConstant(state, shift, 63u);
const auto dword_shift = EmitAndConstant(state, amount, 31u);
const auto less32 = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto carry_count_base = state.builder.AllocateId();
const auto carry_count = state.builder.AllocateId();
const auto low_part = state.builder.AllocateId();
const auto high_part = state.builder.AllocateId();
const auto carry_part = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
const auto low_less = state.builder.AllocateId();
const auto low_ge = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
state.builder.AddFunction(
{spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
state.builder.AddFunction(
{spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
out_low = EmitSelectValueU32(state, less32, low_less, low_ge);
out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
}
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) { uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
@@ -98,12 +16,6 @@ uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t sh
return ret; return ret;
} }
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
uint32_t constant) { uint32_t constant) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
@@ -189,42 +101,6 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
return ret; return ret;
} }
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitcast, TypeU32(state), ret, value});
return ret;
}
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitcast, TypeF32(state), ret, value});
return ret;
}
uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLogicalOr, TypeBool(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), ret, value});
return ret;
}
F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) { F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
F32Class cls; F32Class cls;
cls.bits = bits; cls.bits = bits;
@@ -324,13 +200,6 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
return EmitBitcastU32ToF32(state, result_bits); return EmitBitcastU32ToF32(state, result_bits);
} }
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450Trunc, value});
return ret;
}
uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
const auto abs_bits = state.builder.AllocateId(); const auto abs_bits = state.builder.AllocateId();
@@ -386,19 +255,6 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
return ret; return ret;
} }
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpFNegate, TypeF32(state), ret, value});
return ret;
}
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450FAbs, value});
return ret;
}
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) { uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
const auto unpacked = state.builder.AllocateId(); const auto unpacked = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+34 -48
View File
@@ -16,29 +16,7 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
return true; return true;
} }
uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
const auto shifted_two = state.builder.AllocateId();
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull)});
state.builder.AddFunction(
{spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull)});
state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full)});
return result;
}
uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
auto& state = ctx.state;
if (kind == IR::StageInputKind::LocalInvocationIndex) { if (kind == IR::StageInputKind::LocalInvocationIndex) {
return EmitLocalInvocationIndex(state); return EmitLocalInvocationIndex(state);
} }
@@ -155,8 +133,7 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
return result; return result;
} }
uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) { uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
auto& state = ctx.state;
const auto* input = InputBindingForParameter(state, attr); const auto* input = InputBindingForParameter(state, attr);
if (input == nullptr || input->variable_id == 0) { if (input == nullptr || input->variable_id == 0) {
return ConstantU32(state, 0); return ConstantU32(state, 0);
@@ -217,7 +194,7 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
auto& state = ctx.state; auto& state = ctx.state;
const auto* input = InputBindingForParameter(state, attr); const auto* input = InputBindingForParameter(state, attr);
if (!input->per_vertex) { if (!input->per_vertex) {
return EmitAttribute(ctx, attr, chan); return EmitAttribute(ctx.state, attr, chan);
} }
const auto load_vertex = [&](uint32_t vertex) { const auto load_vertex = [&](uint32_t vertex) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
@@ -427,6 +404,26 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) {
} }
} // namespace } // namespace
uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
const auto shifted_two = state.builder.AllocateId();
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull)});
state.builder.AddFunction(
{spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull)});
state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full)});
return result;
}
void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state; auto& state = ctx.state;
@@ -523,8 +520,7 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
void EmitVoid(ValueEmitContext&) {} void EmitVoid(ValueEmitContext&) {}
void EmitBarrier(ValueEmitContext& ctx) { void EmitBarrier(EmitterState& state) {
auto& state = ctx.state;
const auto semantics = const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
@@ -547,8 +543,7 @@ uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
return result; return result;
} }
uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg) { uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg) {
auto& state = ctx.state;
uint32_t dword = 0; uint32_t dword = 0;
if (!UserDataDwordIndex(state, reg, dword)) { if (!UserDataDwordIndex(state, reg, dword)) {
@@ -559,13 +554,12 @@ uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg) {
} }
uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) { uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) {
return EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(kind.U32()), index.U32()); return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
} }
uint32_t EmitUndefU1(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
auto& state = ctx.state;
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpUndef, ctx.TypeId(inst.GetType()), result}); state.builder.AddFunction({spv::OpUndef, TypeId(state, inst.GetType()), result});
return result; return result;
} }
@@ -582,23 +576,15 @@ uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto can_fetch = state.builder.AllocateId(); const auto can_fetch = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active}); {spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx, can_fetch, shuffled, return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, can_fetch, shuffled,
ConstantU32(state, 0)); ConstantU32(state, 0));
} }
uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto flags = inst.Flags<IR::DppMoveFlags>(); const auto flags = inst.Flags<IR::DppMoveFlags>();
const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2)); const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx, write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)); return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, write, ctx.Arg(inst, 0),
} ctx.Arg(inst, 1));
uint32_t EmitWqmU64(ValueEmitContext& ctx, uint32_t value) {
return EmitWqmU64(ctx.state, value);
}
uint32_t EmitLaneId(ValueEmitContext& ctx) {
auto& state = ctx.state;
return EmitSubgroupLocalInvocationId(state);
} }
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) { uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
@@ -620,7 +606,8 @@ uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto hit = state.builder.AllocateId(); const auto hit = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIEqual, TypeBool(state), hit, state.builder.AddFunction({spv::OpIEqual, TypeBool(state), hit,
EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)}); EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx, hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)); return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, hit, ctx.Arg(inst, 1),
ctx.Arg(inst, 0));
} }
uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -671,7 +658,7 @@ uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
} }
uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
return EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()); return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32());
} }
uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -684,7 +671,6 @@ uint32_t EmitGetShaderBase(ValueEmitContext& ctx) {
return ctx.Def(IR::Value(uint64_t {0})); return ctx.Def(IR::Value(uint64_t {0}));
} }
void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Fail(inst, "must be lowered before SPIR-V emission"); ctx.Fail(inst, "must be lowered before SPIR-V emission");
} }
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
+146 -141
View File
@@ -3,13 +3,8 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
// Direct SPIR-V operations share emission as well as argument/result marshalling. // Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
template <spv::Op opcode, IR::Type type, typename... Args> // (IR::Value), and instructions whose metadata or operand loads must stay lazy.
uint32_t EmitNative(ValueEmitContext& ctx, Args... args) {
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({opcode, ctx.TypeId(type), result, args...});
return result;
}
#define EMIT_NATIVE(name, opcode, type, ...) \ #define EMIT_NATIVE(name, opcode, type, ...) \
inline constexpr auto Emit##name = EmitNative<spv::opcode, IR::Type::type, __VA_ARGS__>; inline constexpr auto Emit##name = EmitNative<spv::opcode, IR::Type::type, __VA_ARGS__>;
EMIT_NATIVE(BitCastU16F16, OpBitcast, U32, uint32_t) EMIT_NATIVE(BitCastU16F16, OpBitcast, U32, uint32_t)
@@ -18,13 +13,13 @@ inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16;
inline constexpr auto EmitConvertU32U8 = EmitBitCastU16F16; inline constexpr auto EmitConvertU32U8 = EmitBitCastU16F16;
inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16; inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16;
EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t) EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t)
uint32_t EmitConvertU16U32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertU8U32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF16F32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF32F16(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32;
uint32_t EmitConvertS32F32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertU32F32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF32S32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t) EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
@@ -32,17 +27,19 @@ EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint
EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t, EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
uint32_t) uint32_t)
uint32_t EmitCompositeExtractU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1); uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64; inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64; inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64; inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
uint32_t EmitPackHalf2x16(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
uint32_t EmitPackSnorm2x16(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitPackSnorm2x16 =
uint32_t EmitPackUnorm2x16(ValueEmitContext& ctx, uint32_t arg0); EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
uint32_t EmitPackFloat2x16Rtz(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); inline constexpr auto EmitPackUnorm2x16 =
uint32_t EmitFPAbs32(ValueEmitContext& ctx, uint32_t arg0); EmitGlsl<GLSLstd450PackUnorm2x16, IR::Type::U32, uint32_t>;
uint32_t EmitFPNeg32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPSaturate32(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitFPAbs32 = EmitFAbsValue;
inline constexpr auto EmitFPNeg32 = EmitFNegateValue;
uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(BitFieldInsert, OpBitFieldInsert, U32, uint32_t, uint32_t, uint32_t, uint32_t) EMIT_NATIVE(BitFieldInsert, OpBitFieldInsert, U32, uint32_t, uint32_t, uint32_t, uint32_t)
EMIT_NATIVE(BitFieldUExtract, OpBitFieldUExtract, U32, uint32_t, uint32_t, uint32_t) EMIT_NATIVE(BitFieldUExtract, OpBitFieldUExtract, U32, uint32_t, uint32_t, uint32_t)
EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint32_t) EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint32_t)
@@ -53,13 +50,13 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t) EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t) EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t) EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
uint32_t EmitIAdd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitISub64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitIMul64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t) EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
uint32_t EmitSMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitIAbs32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
@@ -70,23 +67,23 @@ EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t) EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
uint32_t EmitBitwiseAnd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t) EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t) EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
uint32_t EmitBitCount64(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
uint32_t EmitFindILsb32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0);
uint32_t EmitFindUMsb32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0);
uint32_t EmitFindUMsb64(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0);
uint32_t EmitSMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitSMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitSMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
EMIT_NATIVE(SLessThan32, OpSLessThan, U1, uint32_t, uint32_t) EMIT_NATIVE(SLessThan32, OpSLessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(ULessThan32, OpULessThan, U1, uint32_t, uint32_t) EMIT_NATIVE(ULessThan32, OpULessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(IEqual32, OpIEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(IEqual32, OpIEqual, U1, uint32_t, uint32_t)
@@ -97,11 +94,11 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
uint32_t EmitIEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitINotEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
@@ -118,31 +115,32 @@ EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
uint32_t EmitFPIsNan32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPCmpClass32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t) EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t) EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t) EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
uint32_t EmitFPFma32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); inline constexpr auto EmitFPFma32 =
uint32_t EmitFPMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
uint32_t EmitFPMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPRecip32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPRecipIFlag32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPRecipSqrt32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPSqrt(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPExp2(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0);
uint32_t EmitFPLog2(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0);
uint32_t EmitFPLdexp(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0);
uint32_t EmitFPRoundEven32(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPFloor32(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitFPRoundEven32 = EmitGlsl<GLSLstd450RoundEven, IR::Type::F32, uint32_t>;
uint32_t EmitFPCeil32(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitFPFloor32 = EmitGlsl<GLSLstd450Floor, IR::Type::F32, uint32_t>;
uint32_t EmitFPTrunc32(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitFPCeil32 = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
uint32_t EmitFPFract32(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitFPTrunc32 = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
uint32_t EmitFPSin(ValueEmitContext& ctx, uint32_t arg0); inline constexpr auto EmitFPFract32 = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
uint32_t EmitFPCos(ValueEmitContext& ctx, uint32_t arg0); uint32_t EmitFPSin(EmitterState& state, uint32_t arg0);
uint32_t EmitFPCos(EmitterState& state, uint32_t arg0);
#undef EMIT_NATIVE #undef EMIT_NATIVE
uint32_t EmitIdentity(ValueEmitContext& ctx, uint32_t value); uint32_t EmitIdentity(ValueEmitContext& ctx, uint32_t value);
@@ -154,20 +152,20 @@ inline constexpr auto EmitWaitcnt = EmitVoid;
inline constexpr auto EmitSendmsg = EmitVoid; inline constexpr auto EmitSendmsg = EmitVoid;
inline constexpr auto EmitTtraceData = EmitVoid; inline constexpr auto EmitTtraceData = EmitVoid;
inline constexpr auto EmitInstPrefetch = EmitVoid; inline constexpr auto EmitInstPrefetch = EmitVoid;
void EmitBarrier(ValueEmitContext& ctx); void EmitBarrier(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg); uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index); uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
uint32_t EmitUndefU1(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst);
inline constexpr auto EmitUndefU8 = EmitUndefU1; inline constexpr auto EmitUndefU8 = EmitUndefU1;
inline constexpr auto EmitUndefU16 = EmitUndefU1; inline constexpr auto EmitUndefU16 = EmitUndefU1;
inline constexpr auto EmitUndefU32 = EmitUndefU1; inline constexpr auto EmitUndefU32 = EmitUndefU1;
inline constexpr auto EmitUndefU64 = EmitUndefU1; inline constexpr auto EmitUndefU64 = EmitUndefU1;
uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitWqmU64(ValueEmitContext& ctx, uint32_t value); uint32_t EmitWqmU64(EmitterState& state, uint32_t value);
uint32_t EmitLaneId(ValueEmitContext& ctx); inline constexpr auto EmitLaneId = EmitSubgroupLocalInvocationId;
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate); uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -184,72 +182,79 @@ inline constexpr auto EmitGetScratchResource = EmitGetSrtResource;
inline constexpr auto EmitGetImageResource = EmitGetSrtResource; inline constexpr auto EmitGetImageResource = EmitGetSrtResource;
inline constexpr auto EmitGetSamplerResource = EmitGetSrtResource; inline constexpr auto EmitGetSamplerResource = EmitGetSrtResource;
inline constexpr auto EmitMakeImageAddress = EmitGetSrtResource; inline constexpr auto EmitMakeImageAddress = EmitGetSrtResource;
void EmitMemory(ValueEmitContext& ctx, const IR::Inst& inst); void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitBpermuteU32 = EmitMemory; void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitReadConst = EmitMemory; uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitReadConstBuffer = EmitMemory; uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitLoadAddressU8 = EmitMemory; uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitLoadAddressU16 = EmitMemory; void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitLoadAddressU32 = EmitMemory; uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitStoreAddressU8 = EmitMemory; uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitStoreAddressU16 = EmitMemory; uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitStoreAddressU32 = EmitMemory; uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitLoadBufferU8 = EmitMemory; void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitLoadBufferU16 = EmitMemory; inline constexpr auto EmitLoadAddressU8 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32 = EmitMemory; inline constexpr auto EmitLoadAddressU16 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32x2 = EmitMemory; inline constexpr auto EmitLoadAddressU32 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32x3 = EmitMemory; inline constexpr auto EmitStoreAddressU8 = EmitStoreMemory;
inline constexpr auto EmitLoadBufferU32x4 = EmitMemory; inline constexpr auto EmitStoreAddressU16 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU8 = EmitMemory; inline constexpr auto EmitStoreAddressU32 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU16 = EmitMemory; inline constexpr auto EmitLoadBufferU8 = EmitLoadMemory;
inline constexpr auto EmitStoreBufferU32 = EmitMemory; inline constexpr auto EmitLoadBufferU16 = EmitLoadMemory;
inline constexpr auto EmitStoreBufferU32x2 = EmitMemory; inline constexpr auto EmitLoadBufferU32 = EmitLoadMemory;
inline constexpr auto EmitStoreBufferU32x3 = EmitMemory; inline constexpr auto EmitLoadBufferU32x2 = EmitLoadMemory;
inline constexpr auto EmitStoreBufferU32x4 = EmitMemory; inline constexpr auto EmitLoadBufferU32x3 = EmitLoadMemory;
inline constexpr auto EmitBufferAtomicSwap32 = EmitMemory; inline constexpr auto EmitLoadBufferU32x4 = EmitLoadMemory;
inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitMemory; inline constexpr auto EmitStoreBufferU8 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicSwap64 = EmitMemory; inline constexpr auto EmitStoreBufferU16 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicIAdd32 = EmitMemory; inline constexpr auto EmitStoreBufferU32 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicISub32 = EmitMemory; inline constexpr auto EmitStoreBufferU32x2 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicSMin32 = EmitMemory; inline constexpr auto EmitStoreBufferU32x3 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicUMin32 = EmitMemory; inline constexpr auto EmitStoreBufferU32x4 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicSMax32 = EmitMemory; inline constexpr auto EmitBufferAtomicSwap32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicUMax32 = EmitMemory; inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicAnd32 = EmitMemory; inline constexpr auto EmitBufferAtomicSwap64 = EmitBufferAtomic64;
inline constexpr auto EmitBufferAtomicOr32 = EmitMemory; inline constexpr auto EmitBufferAtomicIAdd32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicOr64 = EmitMemory; inline constexpr auto EmitBufferAtomicISub32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicXor32 = EmitMemory; inline constexpr auto EmitBufferAtomicSMin32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicFMin32 = EmitMemory; inline constexpr auto EmitBufferAtomicUMin32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicFMax32 = EmitMemory; inline constexpr auto EmitBufferAtomicSMax32 = EmitAtomic32;
inline constexpr auto EmitLoadSharedU8 = EmitMemory; inline constexpr auto EmitBufferAtomicUMax32 = EmitAtomic32;
inline constexpr auto EmitLoadSharedU16 = EmitMemory; inline constexpr auto EmitBufferAtomicAnd32 = EmitAtomic32;
inline constexpr auto EmitLoadSharedU32 = EmitMemory; inline constexpr auto EmitBufferAtomicOr32 = EmitAtomic32;
inline constexpr auto EmitLoadSharedU32x2 = EmitMemory; inline constexpr auto EmitBufferAtomicOr64 = EmitBufferAtomic64;
inline constexpr auto EmitLoadSharedU32x3 = EmitMemory; inline constexpr auto EmitBufferAtomicXor32 = EmitAtomic32;
inline constexpr auto EmitLoadSharedU32x4 = EmitMemory; inline constexpr auto EmitBufferAtomicFMin32 = EmitBufferFloatAtomic;
inline constexpr auto EmitWriteSharedU8 = EmitMemory; inline constexpr auto EmitBufferAtomicFMax32 = EmitBufferFloatAtomic;
inline constexpr auto EmitWriteSharedU16 = EmitMemory; inline constexpr auto EmitLoadSharedU8 = EmitLoadMemory;
inline constexpr auto EmitWriteSharedU32 = EmitMemory; inline constexpr auto EmitLoadSharedU16 = EmitLoadMemory;
inline constexpr auto EmitWriteSharedU32x2 = EmitMemory; inline constexpr auto EmitLoadSharedU32 = EmitLoadMemory;
inline constexpr auto EmitWriteSharedU32x3 = EmitMemory; inline constexpr auto EmitLoadSharedU32x2 = EmitLoadMemory;
inline constexpr auto EmitWriteSharedU32x4 = EmitMemory; inline constexpr auto EmitLoadSharedU32x3 = EmitLoadMemory;
inline constexpr auto EmitSharedAtomicFMin32 = EmitMemory; inline constexpr auto EmitLoadSharedU32x4 = EmitLoadMemory;
inline constexpr auto EmitSharedAtomicFMax32 = EmitMemory; inline constexpr auto EmitWriteSharedU8 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicSwap32 = EmitMemory; inline constexpr auto EmitWriteSharedU16 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicIAdd32 = EmitMemory; inline constexpr auto EmitWriteSharedU32 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicISub32 = EmitMemory; inline constexpr auto EmitWriteSharedU32x2 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicInc32 = EmitMemory; inline constexpr auto EmitWriteSharedU32x3 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicDec32 = EmitMemory; inline constexpr auto EmitWriteSharedU32x4 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicSMin32 = EmitMemory; inline constexpr auto EmitSharedAtomicFMin32 = EmitSharedFloatAtomic;
inline constexpr auto EmitSharedAtomicUMin32 = EmitMemory; inline constexpr auto EmitSharedAtomicFMax32 = EmitSharedFloatAtomic;
inline constexpr auto EmitSharedAtomicSMax32 = EmitMemory; inline constexpr auto EmitSharedAtomicSwap32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicUMax32 = EmitMemory; inline constexpr auto EmitSharedAtomicIAdd32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicAnd32 = EmitMemory; inline constexpr auto EmitSharedAtomicISub32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicOr32 = EmitMemory; inline constexpr auto EmitSharedAtomicInc32 = EmitSharedIncDec;
inline constexpr auto EmitSharedAtomicXor32 = EmitMemory; inline constexpr auto EmitSharedAtomicDec32 = EmitSharedIncDec;
inline constexpr auto EmitDataAppend = EmitMemory; inline constexpr auto EmitSharedAtomicSMin32 = EmitAtomic32;
inline constexpr auto EmitDataConsume = EmitMemory; inline constexpr auto EmitSharedAtomicUMin32 = EmitAtomic32;
inline constexpr auto EmitSwizzleU32 = EmitMemory; inline constexpr auto EmitSharedAtomicSMax32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicUMax32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicAnd32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicOr32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicXor32 = EmitAtomic32;
inline constexpr auto EmitDataAppend = EmitAppendConsume;
inline constexpr auto EmitDataConsume = EmitAppendConsume;
uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst); void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitImageQueryDimensions = EmitImage; inline constexpr auto EmitImageQueryDimensions = EmitImage;
inline constexpr auto EmitImageQueryLod = EmitImage; inline constexpr auto EmitImageQueryLod = EmitImage;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+43 -35
View File
@@ -165,6 +165,23 @@ inline void EmitLabel(EmitterState& state, uint32_t label) {
state.builder.AddFunction({spv::OpLabel, label}); state.builder.AddFunction({spv::OpLabel, label});
} }
uint32_t TypeId(EmitterState& state, IR::Type type);
// Shared instruction construction; typed aliases add no forwarding functions.
template <spv::Op opcode, IR::Type type, typename... Args>
uint32_t EmitNative(EmitterState& state, Args... args) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, TypeId(state, type), result, args...});
return result;
}
uint32_t GlslStd450(EmitterState& state);
template <GLSLstd450 opcode, IR::Type type, typename... Args>
uint32_t EmitGlsl(EmitterState& state, Args... args) {
return EmitNative<spv::OpExtInst, type>(state, GlslStd450(state), opcode, args...);
}
inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, value}); state.builder.AddFunction({opcode, type, result, value});
@@ -172,14 +189,14 @@ inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint3
} }
inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
uint32_t rhs) { uint32_t rhs) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, lhs, rhs}); state.builder.AddFunction({opcode, type, result, lhs, rhs});
return result; return result;
} }
inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
uint32_t true_value, uint32_t false_value) { uint32_t false_value) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpSelect, type, result, condition, true_value, false_value}); state.builder.AddFunction({spv::OpSelect, type, result, condition, true_value, false_value});
return result; return result;
@@ -195,7 +212,6 @@ struct ValueEmitContext {
uint32_t FirstLane(uint32_t ballot); uint32_t FirstLane(uint32_t ballot);
uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane); uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane);
uint32_t Result(const IR::Inst& inst); uint32_t Result(const IR::Inst& inst);
uint32_t TypeId(IR::Type type) const;
uint32_t Define(const IR::Inst& inst, uint32_t value); uint32_t Define(const IR::Inst& inst, uint32_t value);
uint32_t ResourceIndex(IR::Value value, IR::ValueOpcode opcode); uint32_t ResourceIndex(IR::Value value, IR::ValueOpcode opcode);
const IR::Inst* ImageAddress(IR::Value value); const IR::Inst* ImageAddress(IR::Value value);
@@ -313,8 +329,6 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value);
uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components); uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
uint32_t GlslStd450(EmitterState& state);
void AllocateInputVariables(EmitterState& state); void AllocateInputVariables(EmitterState& state);
void AllocateOutputVariables(EmitterState& state); void AllocateOutputVariables(EmitterState& state);
@@ -330,7 +344,7 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name
void AddDescriptorAnnotationsAndNames(EmitterState& state); void AddDescriptorAnnotationsAndNames(EmitterState& state);
void DefineModule(EmitterState& state); void DefineModule(EmitterState& state);
void DefineMeshOutputs(EmitterState& state); void DefineMeshOutputs(EmitterState& state);
void EmitMeshEntryPoint(EmitterState& state); void EmitMeshEntryPoint(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -365,7 +379,7 @@ uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane); uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane);
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs); inline constexpr auto EmitAddU32 = EmitNative<spv::OpIAdd, IR::Type::U32, uint32_t, uint32_t>;
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs); uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
@@ -390,10 +404,9 @@ struct MemoryResourceAccess {
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem, const IR::MemoryInfo& mem,
uint32_t variable, uint32_t variable, uint32_t pointer_type);
uint32_t pointer_type);
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access, uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
uint32_t raw_index); uint32_t raw_index);
@@ -404,14 +417,13 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access, uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index); uint32_t index);
uint32_t EmitStorageBufferElementPointer(EmitterState& state, uint32_t EmitStorageBufferElementPointer(EmitterState& state, const MemoryResourceAccess& access,
const MemoryResourceAccess& access, uint32_t index, uint32_t index, uint32_t pointer_type);
uint32_t pointer_type);
uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value); uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value);
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, inline constexpr auto EmitTBufferSelectF32 =
uint32_t false_value); EmitNative<spv::OpSelect, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
bool IsSignedFormatComponent(Format::ComponentType type); bool IsSignedFormatComponent(Format::ComponentType type);
@@ -427,20 +439,14 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control); uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, inline constexpr auto EmitSelectValueU32 =
uint32_t false_value); EmitNative<spv::OpSelect, IR::Type::U32, uint32_t, uint32_t, uint32_t>;
void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
uint32_t& out_low, uint32_t& out_high);
void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
uint32_t shift, uint32_t& out_low, uint32_t& out_high);
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift); uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs); inline constexpr auto EmitOrU32 = EmitNative<spv::OpBitwiseOr, IR::Type::U32, uint32_t, uint32_t>;
uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
uint32_t constant); uint32_t constant);
@@ -453,17 +459,19 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value); inline constexpr auto EmitBitcastF32ToU32 = EmitNative<spv::OpBitcast, IR::Type::U32, uint32_t>;
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value); inline constexpr auto EmitBitcastU32ToF32 = EmitNative<spv::OpBitcast, IR::Type::F32, uint32_t>;
uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs); inline constexpr auto EmitAndU32 = EmitNative<spv::OpBitwiseAnd, IR::Type::U32, uint32_t, uint32_t>;
uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs); inline constexpr auto EmitLogicalAndBool =
EmitNative<spv::OpLogicalAnd, IR::Type::U1, uint32_t, uint32_t>;
uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs); inline constexpr auto EmitLogicalOrBool =
EmitNative<spv::OpLogicalOr, IR::Type::U1, uint32_t, uint32_t>;
uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value); inline constexpr auto EmitLogicalNotBool = EmitNative<spv::OpLogicalNot, IR::Type::U1, uint32_t>;
F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits); F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits);
@@ -476,15 +484,15 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value); inline constexpr auto EmitTruncF32Value = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value); uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero); uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value); inline constexpr auto EmitFNegateValue = EmitNative<spv::OpFNegate, IR::Type::F32, uint32_t>;
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value); inline constexpr auto EmitFAbsValue = EmitGlsl<GLSLstd450FAbs, IR::Type::F32, uint32_t>;
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits); uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+251 -302
View File
@@ -599,23 +599,6 @@ uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
}); });
} }
uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) {
const auto scope =
mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
}
template <typename Fn> template <typename Fn>
uint32_t EmitAtomicUpdate(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t EmitAtomicUpdate(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem, Fn&& replacement) { const IR::MemoryInfo& mem, Fn&& replacement) {
@@ -643,137 +626,6 @@ uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) {
return Select(state, TypeU32(state), wrap, limit, next); return Select(state, TypeU32(state), wrap, limit, next);
} }
uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem) {
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
const auto resource = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
ByteAddress(ctx, inst, mem), resource.byte_offset);
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
ConstantU32(state, 3u));
return EmitValueOrDefaultIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
ConstantU64(state, 0), [&]() {
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state),
ctx.Arg(inst, inst.NumArgs() - 2));
const auto old = state.builder.AllocateId();
state.builder.AddFunction(
{SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old,
EmitStorageBufferElementPointer(state, resource, index,
TypeStorageBufferU64ElementPointer(state)),
ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, spv::MemorySemanticsMaskNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return Unary(state, spv::OpBitcast, TypeU64(state), old);
});
});
}
uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
return EmitAtomicUpdate(ctx, inst, mem, [max_value](EmitterState& state, uint32_t old,
uint32_t value) {
return EmitFloatAtomicReplacement(state, old, value, max_value);
});
}
uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
mem.kind, [&](uint32_t old) {
const auto old_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data);
const auto compare = Binary(
ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f);
return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
});
return 0;
}
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) {
auto& state = ctx.state;
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, ctx.Memory(inst).offset));
const auto raw_index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count = Binary(state, spv::OpIAdd, TypeU32(state),
Unary(state, spv::OpBitCount, TypeU32(state), low),
Unary(state, spv::OpBitCount, TypeU32(state), high));
const auto first = ctx.FirstLane(ballot);
const auto source_lane =
state.lane_count == 2
? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31))
: first;
const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first,
AndCondition(state,
state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice
: spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsMaskNone), count});
return value;
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane});
return result;
}
struct PreparedFormattedMemory { struct PreparedFormattedMemory {
Format::BufferFormatInfo info; Format::BufferFormatInfo info;
MemoryResourceAccess resource; MemoryResourceAccess resource;
@@ -1044,162 +896,259 @@ void DefineGetBdaPointer(EmitterState& state) {
state.builder.AddFunction({spv::OpFunctionEnd}); state.builder.AddFunction({spv::OpFunctionEnd});
} }
void EmitMemory(ValueEmitContext& ctx, const IR::Inst& inst) { uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state; const auto& mem = ctx.Memory(inst);
const auto op = inst.GetOpcode(); return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) {
const auto scope =
mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
}
uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
const auto resource = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
ByteAddress(ctx, inst, mem), resource.byte_offset);
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
ConstantU32(state, 3u));
return EmitValueOrDefaultIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
ConstantU64(state, 0), [&]() {
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state),
ctx.Arg(inst, inst.NumArgs() - 2));
const auto old = state.builder.AllocateId();
state.builder.AddFunction(
{SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old,
EmitStorageBufferElementPointer(state, resource, index,
TypeStorageBufferU64ElementPointer(state)),
ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, spv::MemorySemanticsMaskNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return Unary(state, spv::OpBitcast, TypeU64(state), old);
});
});
}
uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::BufferAtomicFMax32;
return EmitAtomicUpdate(ctx, inst, mem,
[max_value](EmitterState& state, uint32_t old, uint32_t value) {
return EmitFloatAtomicReplacement(state, old, value, max_value);
});
}
void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
mem.kind, [&](uint32_t old) {
const auto old_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data);
const auto compare = Binary(
ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f);
return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
});
}
uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
const bool append = inst.GetOpcode() == IR::ValueOpcode::DataAppend;
auto& state = ctx.state;
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, ctx.Memory(inst).offset));
const auto raw_index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count = Binary(state, spv::OpIAdd, TypeU32(state),
Unary(state, spv::OpBitCount, TypeU32(state), low),
Unary(state, spv::OpBitCount, TypeU32(state), high));
const auto first = ctx.FirstLane(ballot);
const auto source_lane =
state.lane_count == 2
? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31))
: first;
const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first,
AndCondition(state,
state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice
: spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsMaskNone), count});
return value;
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane});
return result;
}
uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
if (state.flattened_srt_variable == 0) {
ctx.Fail(inst, "requires the flattened SRT descriptor");
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.flattened_srt_variable, ConstantU32(state, 0),
ctx.Arg(inst, 1)});
return EmitNative<spv::OpLoad, IR::Type::U32>(state, pointer);
}
void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
auto mem = ctx.Memory(inst);
if (mem.planning_only) return;
auto& state = ctx.state;
mem.kind = IR::ResourceKind::ScalarBuffer;
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, mem.offset));
const auto index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto element = EmitMemoryElementIndex(state, access, index);
const auto condition = EmitMemoryElementInBounds(state, access, element);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
return EmitNative<spv::OpLoad, IR::Type::U32>(
state, EmitMemoryElementPointer(state, access, element));
}));
}
void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
const auto& mem = ctx.Memory(inst);
if (op == IR::ValueOpcode::LoadAddressU32 && mem.planning_only) return;
const auto buffer_components = IR::BufferComponentCount(op); const auto buffer_components = IR::BufferComponentCount(op);
if (buffer_components > 1u) {
const auto access = IR::BufferAccessOf(op);
if (access == IR::BufferAccess::Read) {
ctx.Define(inst, LoadWideBuffer(ctx, inst, buffer_components));
return;
}
if (access == IR::BufferAccess::Write) {
StoreWideBuffer(ctx, inst, buffer_components);
return;
}
}
const auto shared_components = IR::SharedComponentCount(op); const auto shared_components = IR::SharedComponentCount(op);
if (shared_components > 1u) { const auto address_info = IR::AddressOpcodeInfoOf(op);
if (IR::SharedAccessOf(op) == IR::SharedAccess::Read) { uint32_t value;
ctx.Define(inst, LoadWideShared(ctx, inst, shared_components)); if (buffer_components > 1u)
} else { value = LoadWideBuffer(ctx, inst, buffer_components);
StoreWideShared(ctx, inst, shared_components); else if (shared_components > 1u)
} value = LoadWideShared(ctx, inst, shared_components);
return; else if (address_info.access == IR::AddressAccess::Read &&
} mem.kind != IR::ResourceKind::Scratch)
if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) && value = LoadBda(ctx, inst, mem, address_info.data_bits);
ctx.Memory(inst).planning_only) { else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
return; value = FormattedLoad(ctx, inst, mem);
} else if (inst.GetType() == IR::Type::U8)
if (op == IR::ValueOpcode::ReadConst) { value = LoadSubword(ctx, inst, mem, 8, false);
if (state.flattened_srt_variable == 0) { else if (inst.GetType() == IR::Type::U16)
ctx.Fail(inst, "requires the flattened SRT descriptor"); value = LoadSubword(ctx, inst, mem, 16, false);
return; else
} value = LoadWord(ctx, inst, mem);
const auto pointer = state.builder.AllocateId(); ctx.Define(inst, value);
state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state), }
pointer, state.flattened_srt_variable, ConstantU32(state, 0),
ctx.Arg(inst, 1)}); void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitNative<spv::OpLoad, IR::Type::U32>(ctx, pointer)); const auto op = inst.GetOpcode();
return; const auto& mem = ctx.Memory(inst);
} const auto buffer_components = IR::BufferComponentCount(op);
if (op == IR::ValueOpcode::ReadConstBuffer) { const auto shared_components = IR::SharedComponentCount(op);
auto mem = ctx.Memory(inst); const auto type = inst.Arg(inst.NumArgs() - 2).GetType();
mem.kind = IR::ResourceKind::ScalarBuffer; if (buffer_components > 1u)
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), StoreWideBuffer(ctx, inst, buffer_components);
ConstantU32(state, mem.offset)); else if (shared_components > 1u)
const auto index = StoreWideShared(ctx, inst, shared_components);
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); else if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
const auto access = PrepareMemoryResourceAccess(state, mem); FormattedStore(ctx, inst, mem);
const auto element = EmitMemoryElementIndex(state, access, index); else if (type == IR::Type::U8)
const auto condition = EmitMemoryElementInBounds(state, access, element); StoreSubword(ctx, inst, mem, 8);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { else if (type == IR::Type::U16)
const auto value = state.builder.AllocateId(); StoreSubword(ctx, inst, mem, 16);
state.builder.AddFunction( else
{spv::OpLoad, TypeU32(state), value, StoreWord(ctx, inst, mem);
EmitMemoryElementPointer(state, access, element)}); }
return value;
})); uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst) {
return; const auto replacement =
} inst.GetOpcode() == IR::ValueOpcode::SharedAtomicInc32 ? AtomicIncrement : AtomicDecrement;
const auto address_info = IR::AddressOpcodeInfoOf(op); return EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement);
const bool load_address = address_info.access == IR::AddressAccess::Read; }
if (load_address && ctx.Memory(inst).kind != IR::ResourceKind::Scratch) {
ctx.Define(inst, LoadBda(ctx, inst, ctx.Memory(inst), address_info.data_bits)); uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) {
return; auto& state = ctx.state;
} state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)});
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 || const auto source = EmitNative<spv::OpLoad, IR::Type::U32>(state, ctx.scratch_u32_variable);
op == IR::ValueOpcode::LoadBufferU16 || const auto target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
op == IR::ValueOpcode::LoadBufferU32; inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
const bool load_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Read; return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2));
if (load_address || load_buffer || load_shared) { }
const auto mem = ctx.Memory(inst);
uint32_t value = 0; uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) {
if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) auto& state = ctx.state;
value = FormattedLoad(ctx, inst, mem); const auto source = ctx.Arg(inst, 0);
else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 || const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
op == IR::ValueOpcode::LoadSharedU8) Binary(state, spv::OpShiftRightLogical, TypeU32(state),
value = LoadSubword(ctx, inst, mem, 8, false); ctx.Arg(inst, 1), ConstantU32(state, 2)),
else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 || ConstantU32(state, 31));
op == IR::ValueOpcode::LoadSharedU16) const auto base = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
value = LoadSubword(ctx, inst, mem, 16, false); EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u));
else const auto target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index);
value = LoadWord(ctx, inst, mem); return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2));
ctx.Define(inst, value);
return;
}
const bool store_address = address_info.access == IR::AddressAccess::Write;
const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::StoreBufferU32;
const bool store_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Write;
if (store_address || store_buffer || store_shared) {
const auto mem = ctx.Memory(inst);
if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
FormattedStore(ctx, inst, mem);
else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::WriteSharedU8)
StoreSubword(ctx, inst, mem, 8);
else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::WriteSharedU16)
StoreSubword(ctx, inst, mem, 16);
else
StoreWord(ctx, inst, mem);
return;
}
const auto atomic_opcode = SpirvAtomicOpcode(op);
if (atomic_opcode != 0 && inst.GetType() == IR::Type::U64) {
ctx.Define(inst, EmitBufferAtomic64(ctx, inst, ctx.Memory(inst)));
return;
}
if (atomic_opcode != 0) {
ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst)));
return;
}
if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst),
op == IR::ValueOpcode::BufferAtomicFMax32));
return;
}
if (op == IR::ValueOpcode::SharedAtomicInc32 || op == IR::ValueOpcode::SharedAtomicDec32) {
const auto replacement = op == IR::ValueOpcode::SharedAtomicInc32
? AtomicIncrement : AtomicDecrement;
ctx.Define(inst, EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement));
return;
}
if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) {
SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32);
return;
}
if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume) {
ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend));
return;
}
if (op == IR::ValueOpcode::SwizzleU32 || op == IR::ValueOpcode::BpermuteU32) {
uint32_t source = ctx.Arg(inst, 0);
uint32_t target = 0;
if (op == IR::ValueOpcode::SwizzleU32) {
state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, source});
source = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpLoad, TypeU32(state), source, ctx.scratch_u32_variable});
target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
} else {
const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state),
ctx.Arg(inst, 1), ConstantU32(state, 2)),
ConstantU32(state, 31));
const auto base =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u));
target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index);
}
ctx.Define(inst, EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2)));
return;
}
ctx.Fail(inst, "has no memory SPIR-V emitter");
} }
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
-14
View File
@@ -25,12 +25,6 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
return ConstantU32(state, 0); return ConstantU32(state, 0);
} }
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIAdd, TypeU32(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs}); state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
@@ -194,14 +188,6 @@ uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
return ret; return ret;
} }
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpSelect, TypeF32(state), ret, condition, true_value, false_value});
return ret;
}
bool IsSignedFormatComponent(Format::ComponentType type) { bool IsSignedFormatComponent(Format::ComponentType type) {
return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm || return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm ||
type == Format::ComponentType::Sscaled; type == Format::ComponentType::Sscaled;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+19 -11
View File
@@ -40,7 +40,7 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
} }
uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) { uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
const auto value_type = ctx.TypeId(type); const auto value_type = TypeId(ctx.state, type);
return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type); return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type);
} }
@@ -243,14 +243,22 @@ decltype(auto) Arg(ValueEmitContext& ctx, const IR::Inst& inst, size_t index) {
} }
} }
template <typename Return, typename... Args> template <typename Context, typename Return, typename... Args>
void Invoke(Return (*emit)(ValueEmitContext&, Args...), ValueEmitContext& ctx, void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::Inst& inst) {
const IR::Inst& inst) {
// A full instruction keeps metadata and predicated/lane operand loads lazy. // A full instruction keeps metadata and predicated/lane operand loads lazy.
static_assert(std::is_same_v<Context, ValueEmitContext> ||
std::is_same_v<Context, EmitterState>);
auto& context = [&]() -> Context& {
if constexpr (std::is_same_v<Context, EmitterState>)
return ctx.state;
else
return ctx;
}();
constexpr bool has_inst = (std::is_same_v<Args, const IR::Inst&> || ...); constexpr bool has_inst = (std::is_same_v<Args, const IR::Inst&> || ...);
[&]<size_t... I>(std::index_sequence<I...>) { [&]<size_t... I>(std::index_sequence<I...>) {
static_assert(((!std::is_same_v<Args, const IR::Inst&> || I == 0) && ...));
const auto call = [&] { const auto call = [&] {
return emit(ctx, Arg<Args>(ctx, inst, I - (has_inst && I != 0))...); return emit(context, Arg<Args>(ctx, inst, I - (has_inst && I != 0))...);
}; };
if constexpr (std::is_void_v<Return>) { if constexpr (std::is_void_v<Return>) {
call(); call();
@@ -274,7 +282,7 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured, void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
const IR::Inst& inst) { const IR::Inst& inst) {
if (inst.GetOpcode() == IR::ValueOpcode::Phi) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
const auto type = ctx.TypeId(inst.GetType()); const auto type = TypeId(ctx.state, inst.GetType());
if (type == 0 || inst.NumArgs() == 0) { if (type == 0 || inst.NumArgs() == 0) {
ctx.Fail(inst, "has no native SPIR-V representation"); ctx.Fail(inst, "has no native SPIR-V representation");
} }
@@ -295,7 +303,7 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher, void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
const IR::Inst& inst) { const IR::Inst& inst) {
if (inst.GetOpcode() == IR::ValueOpcode::Phi) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
const auto type = ctx.TypeId(inst.GetType()); const auto type = TypeId(ctx.state, inst.GetType());
if (type == 0) { if (type == 0) {
ctx.Fail(inst, "cannot be loaded by the dispatcher"); ctx.Fail(inst, "cannot be loaded by the dispatcher");
} }
@@ -425,7 +433,7 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
} // namespace } // namespace
uint32_t ValueEmitContext::TypeId(IR::Type type) const { uint32_t TypeId(EmitterState& state, IR::Type type) {
switch (type) { switch (type) {
case IR::Type::U1: return TypeBool(state); case IR::Type::U1: return TypeBool(state);
case IR::Type::U8: case IR::Type::U8:
@@ -470,7 +478,7 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
return loaded->second.second; return loaded->second.second;
} }
const auto id = state.builder.AllocateId(); const auto id = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLoad, TypeId(inst->GetType()), id, found->second}); state.builder.AddFunction({spv::OpLoad, TypeId(state, inst->GetType()), id, found->second});
dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id}); dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
return id; return id;
} }
@@ -536,7 +544,7 @@ uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
} }
uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) { uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
const auto type = TypeId(inst.Arg(index).GetType()); const auto type = TypeId(state, inst.Arg(index).GetType());
const auto scope = ConstantU32(state, spv::ScopeSubgroup); const auto scope = ConstantU32(state, spv::ScopeSubgroup);
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
if (other_half == nullptr) { if (other_half == nullptr) {
@@ -574,7 +582,7 @@ uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
if (const auto found = definitions.find(&inst); found != definitions.end()) { if (const auto found = definitions.find(&inst); found != definitions.end()) {
if (found->second != value) { if (found->second != value) {
state.builder.AddFunction( state.builder.AddFunction(
{spv::OpCopyObject, TypeId(inst.GetType()), found->second, value}); {spv::OpCopyObject, TypeId(state, inst.GetType()), found->second, value});
} }
return found->second; return found->second;
} }
tests/ShaderRecompilerComputeTests.cpp
+39
View File
@@ -16687,6 +16687,44 @@ TestCase VectorVop3LshlrevB64Captured() {
return test; return test;
} }
TestCase VectorDynamicU64ShiftEdges() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorDynamicU64ShiftEdges";
test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u};
test.expected = test.initial;
auto& code = test.code;
for (const uint64_t source : {uint64_t{0}, uint64_t{1},
uint64_t{0x8000000000000000ull},
uint64_t{0xffffffffffffffffull},
uint64_t{0x0123456789abcdefull}}) {
AppendVMovLiteral(&code, 4, static_cast<u32>(source));
AppendVMovLiteral(&code, 5, static_cast<u32>(source >> 32u));
for (u32 index = 0; index < test.initial.size(); ++index) {
// A buffer load prevents the constant-shift specialization.
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 3, 30);
for (const bool left : {true, false}) {
AppendVop3(&code, left ? 0x2ff : 0x300, 33, Vgpr(3), Vgpr(4));
const auto amount = test.initial[index] & 63u;
const auto expected = left ? source << amount : source >> amount;
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 33, output);
AppendStoreVgpr(&code, 34, output + 1u);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64,
O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical", "OpSelect"};
test.forbidden_spirv = {"OpTypeInt 64"};
return test;
}
TestCase VectorVop3IntegerOps() { TestCase VectorVop3IntegerOps() {
using O = ShaderOpcode; using O = ShaderOpcode;
@@ -24745,6 +24783,7 @@ std::vector<TestCase> MakeCases() {
AddCase(ScalarBfeI32CapturedRawSignExtends); AddCase(ScalarBfeI32CapturedRawSignExtends);
AddCase(BitfieldExtractWidthPastEndEdges); AddCase(BitfieldExtractWidthPastEndEdges);
AddCase(Scalar64BitOps); AddCase(Scalar64BitOps);
AddCase(VectorDynamicU64ShiftEdges);
AddCase(ScalarConditionalMoveB64); AddCase(ScalarConditionalMoveB64);
AddCase(ScalarConditionalMoveB64PreservesMasks); AddCase(ScalarConditionalMoveB64PreservesMasks);
AddCase(ScalarAndn2B64SccUsesMaskShadow); AddCase(ScalarAndn2B64SccUsesMaskShadow);
tests/shaderCfgTests.cpp
+33
View File
@@ -8708,6 +8708,38 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
"dispatcher did not use one canonical U32x3 spill slot"); "dispatcher did not use one canonical U32x3 spill slot");
} }
void TestNewShaderRecompilerPlanningOnlyLoads() {
using namespace ShaderRecompiler;
const uint32_t shader[] = {EncodeSopp(0x01)};
auto options = MakeCompileOptions(ShaderType::Compute);
auto result = RecompileForTest(shader, options);
auto& program = result.program;
IR::IREmitter ir(program.blocks.front());
IR::MemoryInfo memory;
memory.kind = IR::ResourceKind::ScalarAddress;
memory.planning_only = true;
program.memory_info.push_back(memory);
const auto address = ir.Emit(IR::ValueOpcode::GetAddressResource,
{IR::Value(0x1000u), IR::Value(0u)});
const auto load = ir.Emit(IR::ValueOpcode::LoadAddressU32,
{address, IR::Value(0u), IR::Value(0u), IR::Value(true)});
load.Instruction()->SetFlags(IR::MemoryFlags{.index = 0});
memory.kind = IR::ResourceKind::ScalarBuffer;
program.memory_info.push_back(memory);
const auto buffer = ir.Emit(IR::ValueOpcode::GetBufferResource,
{IR::Value(0x2000u), IR::Value(0u),
IR::Value(16u), IR::Value(0u)});
const auto read = ir.Emit(IR::ValueOpcode::ReadConstBuffer,
{buffer, IR::Value(0u)});
read.Instruction()->SetFlags(IR::MemoryFlags{.index = 1});
ir.Emit(IR::ValueOpcode::ReferenceU32, {load});
ir.Emit(IR::ValueOpcode::ReferenceU32, {read});
const auto spirv = Spirv::EmitProgram(program, options.input_info);
CheckSpirvBinaryValidates(spirv);
Check(spirv == result.spirv,
"planning-only loads or references emitted GPU operands/instructions");
}
void TestNewShaderRecompilerU64PairTranslation() { void TestNewShaderRecompilerU64PairTranslation() {
using namespace ShaderRecompiler; using namespace ShaderRecompiler;
@@ -13017,6 +13049,7 @@ int main() {
TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry(); TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry();
TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerCfgIrreducibleDispatcher();
TestNewShaderRecompilerDispatcherSpillsU32x3(); TestNewShaderRecompilerDispatcherSpillsU32x3();
TestNewShaderRecompilerPlanningOnlyLoads();
TestNewShaderRecompilerU64PairTranslation(); TestNewShaderRecompilerU64PairTranslation();
TestComputeDispatchWaveSize(); TestComputeDispatchWaveSize();
TestNewShaderRecompilerBufferLoadsGuardedByExec(); TestNewShaderRecompilerBufferLoadsGuardedByExec();