mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-30 01:01:21 +00:00
shader: add BVH stack operations
This commit is contained in:
nmzik committed
1 parent
6ff6e7551d
commit
bfd4c34b88
18 files changed
+419
-3
No files matched your search
@@ -338,6 +338,7 @@ inline constexpr auto EmitDataConsume = EmitAppendConsume;
|
||||
uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
inline constexpr auto EmitImageQueryDimensions = EmitImage;
|
||||
inline constexpr auto EmitImageQueryLod = EmitImage;
|
||||
inline constexpr auto EmitImageRead = EmitImage;
|
||||
|
||||
@@ -120,6 +120,7 @@ struct EmitterState {
|
||||
uint32_t indirect_format_geometry_function = 0;
|
||||
std::map<std::array<uint32_t, 4>, uint32_t> indirect_buffer_load_functions;
|
||||
uint32_t bvh_intersect_function = 0;
|
||||
uint32_t bvh_stack_function = 0;
|
||||
uint32_t gds_variable = 0;
|
||||
uint32_t gds_length = 0;
|
||||
uint32_t push_constant_variable = 0;
|
||||
@@ -542,6 +543,7 @@ void EmitProgram(EmitterState& state);
|
||||
void DefineGetBdaPointer(EmitterState& state);
|
||||
void DefineIndirectBufferLoads(ValueEmitContext& ctx);
|
||||
void DefineBvhIntersect(EmitterState& state);
|
||||
void DefineTrinityFunctions(EmitterState& state);
|
||||
uint32_t GetBdaPointer(EmitterState& state, uint32_t address);
|
||||
|
||||
// These templates accept local lambdas from several emitter translation units.
|
||||
|
||||
@@ -676,6 +676,7 @@ void EmitProgram(EmitterState& state) {
|
||||
DefineGetBdaPointer(state);
|
||||
DefineIndirectBufferLoads(ctx);
|
||||
DefineBvhIntersect(state);
|
||||
DefineTrinityFunctions(state);
|
||||
if (program.info.uses_swizzle) {
|
||||
ctx.scratch_u32_variable = state.builder.AllocateId();
|
||||
if (state.lane_count == 2) {
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t Bits(EmitterState& s, uint32_t value, uint32_t offset, uint32_t count) {
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(s), result, value,
|
||||
ConstantU32(s, offset), ConstantU32(s, count));
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t SetBits(EmitterState& s, uint32_t value, uint32_t field, uint32_t offset, uint32_t count) {
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpBitFieldInsert, TypeU32(s), result, value, field,
|
||||
ConstantU32(s, offset), ConstantU32(s, count));
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Pair(EmitterState& s, uint32_t first, uint32_t second) {
|
||||
const auto result = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Pair(s), result, first, second);
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t StackIndex(EmitterState& s, uint32_t control, uint32_t ring) {
|
||||
return EmitAddU32(s, Bits(s, control, 15, 14),
|
||||
Binary(s, spv::OpShiftLeftLogical, TypeU32(s), ring, ConstantU32(s, 5)));
|
||||
}
|
||||
|
||||
uint32_t StackPush(EmitterState& s, const MemoryResourceAccess& memory,
|
||||
uint32_t control, uint32_t node, uint32_t capacity) {
|
||||
const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
|
||||
const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5);
|
||||
const auto index = StackIndex(s, control, ring);
|
||||
EmitIfCondition(s, EmitMemoryElementInBounds(s, memory, index), [&] {
|
||||
s.builder.AddFunction(spv::OpStore, EmitMemoryElementPointer(s, memory, index), node);
|
||||
});
|
||||
control = Binary(s, spv::OpBitwiseOr, u, control,
|
||||
Select(s, u, Binary(s, spv::OpIEqual, b, count, capacity), ConstantU32(s, 1u << 30), zero));
|
||||
control = Select(s, u, Binary(s, spv::OpIEqual, b, parent, capacity),
|
||||
Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control);
|
||||
control = SetBits(s, control, EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
|
||||
EmitAddU32(s, count, one), capacity), 0, 5);
|
||||
control = SetBits(s, control, EmitGlsl<GLSLstd450UMin, IR::Type::U32>(s,
|
||||
EmitAddU32(s, parent, one), capacity), 5, 5);
|
||||
const auto next = EmitAddU32(s, ring, one);
|
||||
return SetBits(s, control, Select(s, u, Binary(s, spv::OpIEqual, b, next, capacity), zero, next), 10, 5);
|
||||
}
|
||||
|
||||
uint32_t StackPop(EmitterState& s, const MemoryResourceAccess& memory,
|
||||
uint32_t control, uint32_t capacity) {
|
||||
const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
|
||||
const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5);
|
||||
const auto has_parent = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero);
|
||||
const auto returning = Binary(s, spv::OpLogicalAnd, b, has_parent,
|
||||
Binary(s, spv::OpIEqual, b, parent, zero));
|
||||
control = SetBits(s, control, Select(s, u, returning, one, zero), 31, 1);
|
||||
control = Select(s, u, returning,
|
||||
Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control);
|
||||
const auto overflow = Binary(s, spv::OpINotEqual, b, Bits(s, control, 30, 1), zero);
|
||||
const auto empty = Pair(s,
|
||||
Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, 0x5fffffff)),
|
||||
Select(s, u, overflow, ConstantU32(s, 0xffffffff), ConstantU32(s, 0xfffffffe)));
|
||||
return EmitValueOrDefaultIfCondition(s, Binary(s, spv::OpINotEqual, b, count, zero),
|
||||
TypeU32Pair(s), empty, [&] {
|
||||
control = SetBits(s, control, Binary(s, spv::OpISub, u, count, one), 0, 5);
|
||||
control = SetBits(s, control, Binary(s, spv::OpISub, u, parent, one), 5, 5);
|
||||
const auto previous = Select(s, u, Binary(s, spv::OpIEqual, b, ring, zero),
|
||||
Binary(s, spv::OpISub, u, capacity, one), Binary(s, spv::OpISub, u, ring, one));
|
||||
control = SetBits(s, control, previous, 10, 5);
|
||||
const auto index = StackIndex(s, control, previous);
|
||||
const auto node = EmitValueOrDefaultIfCondition(s,
|
||||
EmitMemoryElementInBounds(s, memory, index), u, zero, [&] {
|
||||
const auto value = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpLoad, u, value, EmitMemoryElementPointer(s, memory, index));
|
||||
return value;
|
||||
});
|
||||
const auto instance = Binary(s, spv::OpIEqual, b, Bits(s, node, 0, 3), ConstantU32(s, 6));
|
||||
control = EmitValueOrDefaultIfCondition(s, instance, u, control, [&] {
|
||||
const auto nested = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero);
|
||||
const auto discarded = Binary(s, spv::OpBitwiseOr, u,
|
||||
SetBits(s, control, Bits(s, control, 5, 5), 0, 5), ConstantU32(s, 1u << 30));
|
||||
return Binary(s, spv::OpBitwiseOr, u,
|
||||
SetBits(s, Select(s, u, nested, discarded, control), zero, 5, 5), ConstantU32(s, 1u << 29));
|
||||
});
|
||||
return Pair(s, control, node);
|
||||
});
|
||||
}
|
||||
|
||||
void DefineBvhStack(EmitterState& s) {
|
||||
if (!s.program.info.bvh_stack) return;
|
||||
const auto u = TypeU32(s), b = TypeBool(s), vec4 = TypeU32Vector(s, 4), pair = TypeU32Pair(s);
|
||||
const auto zero = ConstantU32(s, 0), one = ConstantU32(s, 1);
|
||||
const bool local = s.lds_storage_class == spv::StorageClassFunction;
|
||||
const bool buffer = s.lds_storage_class == spv::StorageClassStorageBuffer;
|
||||
std::vector<uint32_t> types {u, u, vec4, vec4, u};
|
||||
if (local) types.push_back(TypeU32ArrayPointer(s, spv::StorageClassFunction, LdsDwordCount(s)));
|
||||
if (buffer) types.push_back(u);
|
||||
const auto signature = s.builder.Type(spv::OpTypeFunction, pair, types);
|
||||
s.bvh_stack_function = s.builder.AllocateId();
|
||||
s.builder.AddName(s.bvh_stack_function, "trinity_bvh_stack");
|
||||
s.builder.AddFunction(spv::OpFunction, pair, s.bvh_stack_function, spv::FunctionControlMaskNone, signature);
|
||||
std::vector<uint32_t> args;
|
||||
for (const auto type: types) {
|
||||
args.push_back(s.builder.AllocateId());
|
||||
s.builder.AddFunction(spv::OpFunctionParameter, type, args.back());
|
||||
}
|
||||
const auto old_lds = s.lds_variable, old_base = s.lds_base_dwords;
|
||||
if (local) s.lds_variable = args.back();
|
||||
if (buffer) s.lds_base_dwords = args.back();
|
||||
const auto memory = PrepareMemoryResourceAccess(s, {.kind = IR::ResourceKind::Lds});
|
||||
const auto entry = s.builder.AllocateId();
|
||||
EmitLabel(s, entry);
|
||||
const auto last = args[1], capacity = args[4];
|
||||
const auto walkback = Binary(s, spv::OpINotEqual, b, Bits(s, last, 3, 29), ConstantU32(s, 0x1fffffff));
|
||||
const auto last4 = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpCompositeConstruct, vec4, last4, last, last, last, last);
|
||||
const auto found = [&](uint32_t values) {
|
||||
return Unary(s, spv::OpAny, b, Binary(s, spv::OpIEqual, TypeBoolVector(s, 4), values, last4));
|
||||
};
|
||||
const auto initial_valid = Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, walkback),
|
||||
Binary(s, spv::OpLogicalOr, b, found(args[2]), found(args[3])));
|
||||
const auto header = s.builder.AllocateId(), body = s.builder.AllocateId();
|
||||
const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpBranch, header);
|
||||
EmitLabel(s, header);
|
||||
const auto index = s.builder.AllocateId(), control = s.builder.AllocateId();
|
||||
const auto valid = s.builder.AllocateId(), skip_low = s.builder.AllocateId(), skip_high = s.builder.AllocateId();
|
||||
const auto index_phi = s.builder.AddDeferredPhi(u, index, 2);
|
||||
const auto control_phi = s.builder.AddDeferredPhi(u, control, 2);
|
||||
const auto valid_phi = s.builder.AddDeferredPhi(b, valid, 2);
|
||||
const auto low_phi = s.builder.AddDeferredPhi(b, skip_low, 2);
|
||||
const auto high_phi = s.builder.AddDeferredPhi(b, skip_high, 2);
|
||||
const auto more = Binary(s, spv::OpLogicalAnd, b, valid,
|
||||
Binary(s, spv::OpULessThan, b, index, ConstantU32(s, 8)));
|
||||
s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask);
|
||||
s.builder.AddFunction(spv::OpBranchConditional, more, body, merge);
|
||||
EmitLabel(s, body);
|
||||
const auto high = Binary(s, spv::OpUGreaterThanEqual, b, index, ConstantU32(s, 4));
|
||||
const auto high4 = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpCompositeConstruct, TypeBoolVector(s, 4), high4, high, high, high, high);
|
||||
const auto values = Select(s, vec4, high4, args[3], args[2]);
|
||||
const auto node = s.builder.AllocateId();
|
||||
s.builder.AddFunction(spv::OpVectorExtractDynamic, u, node, values, Bits(s, index, 0, 2));
|
||||
const auto sentinel = Binary(s, spv::OpIEqual, b, Bits(s, node, 3, 29), ConstantU32(s, 0x1fffffff));
|
||||
const auto enabled = Unary(s, spv::OpLogicalNot, b, Select(s, b, high, skip_high, skip_low));
|
||||
const auto marker = Binary(s, spv::OpLogicalAnd, b, sentinel, enabled);
|
||||
const auto next_low = Binary(s, spv::OpLogicalOr, b, skip_low,
|
||||
Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 1, 1), zero)));
|
||||
const auto next_high = Binary(s, spv::OpLogicalOr, b, skip_high,
|
||||
Binary(s, spv::OpLogicalAnd, b, high,
|
||||
Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 2, 1), zero))));
|
||||
const auto next_valid = Unary(s, spv::OpLogicalNot, b,
|
||||
Binary(s, spv::OpLogicalAnd, b, walkback, Binary(s, spv::OpIEqual, b, node, last)));
|
||||
const auto push = Binary(s, spv::OpLogicalAnd, b, next_valid,
|
||||
Unary(s, spv::OpLogicalNot, b,
|
||||
Binary(s, spv::OpLogicalOr, b, sentinel, Select(s, b, high, next_high, next_low))));
|
||||
const auto next_control = EmitValueOrDefaultIfCondition(s, push, u, control, [&] {
|
||||
return StackPush(s, memory, control, node, capacity);
|
||||
});
|
||||
s.builder.AddFunction(spv::OpBranch, cont);
|
||||
EmitLabel(s, cont);
|
||||
const auto next_index = Binary(s, spv::OpISub, u, index, one);
|
||||
s.builder.AddFunction(spv::OpBranch, header);
|
||||
s.builder.PatchDeferredPhi(index_phi, 0, ConstantU32(s, 7), entry);
|
||||
s.builder.PatchDeferredPhi(index_phi, 1, next_index, cont);
|
||||
s.builder.PatchDeferredPhi(control_phi, 0, args[0], entry);
|
||||
s.builder.PatchDeferredPhi(control_phi, 1, next_control, cont);
|
||||
s.builder.PatchDeferredPhi(valid_phi, 0, initial_valid, entry);
|
||||
s.builder.PatchDeferredPhi(valid_phi, 1, next_valid, cont);
|
||||
s.builder.PatchDeferredPhi(low_phi, 0, ConstantBool(s, false), entry);
|
||||
s.builder.PatchDeferredPhi(low_phi, 1, next_low, cont);
|
||||
s.builder.PatchDeferredPhi(high_phi, 0, ConstantBool(s, false), entry);
|
||||
s.builder.PatchDeferredPhi(high_phi, 1, next_high, cont);
|
||||
EmitLabel(s, merge);
|
||||
s.builder.AddFunction(spv::OpReturnValue, StackPop(s, memory, control, capacity));
|
||||
s.builder.AddFunction(spv::OpFunctionEnd);
|
||||
if (local) s.lds_variable = old_lds;
|
||||
if (buffer) s.lds_base_dwords = old_base;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void DefineTrinityFunctions(EmitterState& s) {
|
||||
DefineBvhStack(s);
|
||||
}
|
||||
|
||||
uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& s = ctx.state;
|
||||
const auto capacity = ctx.Memory(inst).offset;
|
||||
if (s.bvh_stack_function == 0 || capacity < 8 || capacity > 31)
|
||||
ctx.Fail(inst, "invalid BVH stack function or capacity");
|
||||
const std::array values {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)};
|
||||
return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 4), TypeU32Pair(s),
|
||||
ConstantU32CompositeZero(s, 2), [&] {
|
||||
const auto result = s.builder.AllocateId();
|
||||
std::vector<uint32_t> call {spv::OpFunctionCall, TypeU32Pair(s), result,
|
||||
s.bvh_stack_function, values[0], values[1], values[2], values[3], ConstantU32(s, capacity)};
|
||||
if (s.lds_storage_class == spv::StorageClassFunction) call.push_back(s.lds_variable);
|
||||
if (s.lds_storage_class == spv::StorageClassStorageBuffer) call.push_back(s.lds_base_dwords);
|
||||
s.builder.AddFunction(call);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -142,6 +142,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
|
||||
{0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
|
||||
{0xb2u, Opcode::DS_PERMUTE_B32, 1, 32}, {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32},
|
||||
{0xdeu, Opcode::DS_WRITE_B96, 3, 32}, {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
|
||||
{0xe1u, Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32, 1, 32},
|
||||
{0xfeu, Opcode::DS_READ_B96, 3, 32}, {0xffu, Opcode::DS_READ_B128, 4, 32},
|
||||
};
|
||||
|
||||
@@ -232,6 +233,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
|
||||
|
||||
uint32_t DsSourceCount(Opcode opcode) {
|
||||
switch (opcode) {
|
||||
case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return 3u;
|
||||
case Opcode::DS_WRITE2_B32:
|
||||
case Opcode::DS_WRITE2ST64_B32:
|
||||
case Opcode::DS_WRITE2_B64:
|
||||
@@ -473,6 +475,13 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
|
||||
if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
|
||||
SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
|
||||
}
|
||||
if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 &&
|
||||
(inst.gds || inst.offset < 8u || inst.offset > 31u || data1 > 248u)) {
|
||||
SetUnsupported(inst, Family::DS, opcode, "invalid BVH stack operands or capacity");
|
||||
}
|
||||
if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 && vdst == addr) {
|
||||
SetUnsupported(inst, Family::DS, opcode, "overlapping BVH stack outputs are not implemented");
|
||||
}
|
||||
if (inst.gds && inst.opcode == Opcode::DS_CMPST_RTN_B32) {
|
||||
SetUnsupported(inst, Family::DS, opcode, "DS compare-store return B32 GDS variant is not implemented");
|
||||
}
|
||||
|
||||
@@ -740,6 +740,7 @@ std::string InstructionToString(const Instruction& inst) {
|
||||
case Opcode::DS_READ2ST64_B64:
|
||||
case Opcode::DS_READ_B96:
|
||||
case Opcode::DS_READ_B128:
|
||||
case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32:
|
||||
case Opcode::DS_WRITE_B8:
|
||||
case Opcode::DS_WRITE_B16:
|
||||
case Opcode::DS_WRITE_B8_D16_HI:
|
||||
|
||||
@@ -685,6 +685,7 @@ enum class Opcode {
|
||||
DS_READ2ST64_B64,
|
||||
DS_READ_B96,
|
||||
DS_READ_B128,
|
||||
DS_BVH_STACK_PUSH8_POP1_RTN_B32,
|
||||
DS_WRITE_B8,
|
||||
DS_WRITE_B16,
|
||||
DS_WRITE_B8_D16_HI,
|
||||
|
||||
@@ -781,6 +781,15 @@ void Translator::WriteSharedU32(uint32_t width, IR::U32 address,
|
||||
}
|
||||
}
|
||||
|
||||
void Translator::DS_BVH_STACK(const Decoder::Instruction& inst) {
|
||||
const auto result = ir.Emit(IR::ValueOpcode::BvhStack,
|
||||
{ReadU32(inst.src0), ReadU32(inst.src1), ConstructU32x4(inst.src2, 4),
|
||||
ConstructU32x4(OffsetOperand(inst.src2, 4), 4), ir.GetExec()},
|
||||
AddMemoryInfo(MemoryInfoFromDecoded(inst), inst.pc));
|
||||
WriteOperand(inst.src0, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(0u)}));
|
||||
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(1u)}));
|
||||
}
|
||||
|
||||
void Translator::DS_READ(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto address = ReadU32(MemorySourceAt(inst, 0));
|
||||
@@ -1186,6 +1195,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataAppend);
|
||||
case Decoder::Opcode::DS_WRITE_ADDTID_B32: return DS_ADDTID(inst, true);
|
||||
case Decoder::Opcode::DS_READ_ADDTID_B32: return DS_ADDTID(inst, false);
|
||||
case Decoder::Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return DS_BVH_STACK(inst);
|
||||
case Decoder::Opcode::DS_READ2_B32:
|
||||
case Decoder::Opcode::DS_READ2ST64_B32:
|
||||
case Decoder::Opcode::DS_READ2_B64:
|
||||
|
||||
@@ -105,6 +105,7 @@ private:
|
||||
void WriteSharedU32(uint32_t width, IR::U32 address, const std::array<IR::Value, 4>& values,
|
||||
const IR::MemoryInfo& memory, uint32_t pc);
|
||||
void DS_READ(const Decoder::Instruction& inst);
|
||||
void DS_BVH_STACK(const Decoder::Instruction& inst);
|
||||
void DS_READ2(const Decoder::Instruction& inst);
|
||||
void DS_WRITE(const Decoder::Instruction& inst);
|
||||
void DS_WRITE2(const Decoder::Instruction& inst);
|
||||
|
||||
@@ -155,7 +155,7 @@ KYTY_IO_FIELDS(SampledResourcePair, a, b, c)
|
||||
KYTY_IO_FIELDS(StageInput, a, b, c, d, e)
|
||||
KYTY_IO_FIELDS(StageOutput, a, b, c, d)
|
||||
KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
|
||||
A, B, C, D, E, F, G, H, I, J, K)
|
||||
A, B, C, D, E, F, G, H, I, J, K, L)
|
||||
KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b)
|
||||
KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks, a, b, c)
|
||||
KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j, k,
|
||||
|
||||
@@ -543,6 +543,7 @@ struct ShaderInfo {
|
||||
bool uses_image_heap = false;
|
||||
bool uses_dispatch_threads = false;
|
||||
bool bvh = false;
|
||||
bool bvh_stack = false;
|
||||
bool subgroup_ballot = false;
|
||||
bool subgroup_vote = false;
|
||||
bool subgroup_barrier = false;
|
||||
|
||||
@@ -76,7 +76,8 @@ bool HasSideEffects(ValueOpcode opcode) {
|
||||
return true;
|
||||
}
|
||||
const auto shared_access = SharedAccessOf(opcode);
|
||||
if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic ||
|
||||
if (shared_access == SharedAccess::Write || shared_access == SharedAccess::ReadWrite ||
|
||||
shared_access == SharedAccess::Atomic ||
|
||||
shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) {
|
||||
return true;
|
||||
}
|
||||
@@ -156,6 +157,7 @@ uint32_t BufferComponentCount(ValueOpcode opcode) {
|
||||
|
||||
SharedAccess SharedAccessOf(ValueOpcode opcode) {
|
||||
switch (opcode) {
|
||||
case ValueOpcode::BvhStack: return SharedAccess::ReadWrite;
|
||||
case ValueOpcode::LoadSharedU8:
|
||||
case ValueOpcode::LoadSharedU16:
|
||||
case ValueOpcode::LoadSharedU32:
|
||||
|
||||
@@ -17,7 +17,7 @@ enum class ValueOpcode : uint16_t {
|
||||
};
|
||||
|
||||
enum class BufferAccess { None, Read, Write, Atomic };
|
||||
enum class SharedAccess { None, Read, Write, Atomic, Append, Consume };
|
||||
enum class SharedAccess { None, Read, Write, ReadWrite, Atomic, Append, Consume };
|
||||
enum class AddressAccess { None, Read, Write };
|
||||
enum class ImageAccess { None, Read, Write, Atomic };
|
||||
enum class ImageResourceClass { None, Sampled, Storage };
|
||||
|
||||
@@ -318,6 +318,7 @@ VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
|
||||
|
||||
VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
|
||||
VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1)
|
||||
VALUE_OPCODE(BvhStack, U32x2, U32, U32, U32x4, U32x4, U1)
|
||||
VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
|
||||
VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
|
||||
VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
|
||||
|
||||
@@ -515,6 +515,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
|
||||
case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break;
|
||||
case ValueOpcode::DataCacheInvalidate: info.coherent_buffers = true; break;
|
||||
case ValueOpcode::BvhIntersect: info.bvh = true; break;
|
||||
case ValueOpcode::BvhStack: info.bvh_stack = true; break;
|
||||
case ValueOpcode::ConditionRef:
|
||||
info.subgroup_vote |=
|
||||
inst.Flags<CFG::BranchCondition>() != CFG::BranchCondition::ScalarInstruction;
|
||||
|
||||
@@ -1,3 +1,132 @@
|
||||
TestCase TrinityStack(u32 wave_size, u32 capacity, bool storage = false) {
|
||||
TestCase test;
|
||||
static constexpr const char* names[2][3] = {
|
||||
{"TrinityStack32Capacity8", "TrinityStack32Capacity16", "TrinityStack32Capacity31"},
|
||||
{"TrinityStack64Capacity8", "TrinityStack64Capacity16", "TrinityStack64Capacity31"}};
|
||||
test.name = storage ? (wave_size == 32 ? "TrinityStackStorage32" : "TrinityStackStorage64")
|
||||
: names[wave_size == 64][capacity == 8 ? 0 : capacity == 16 ? 1 : 2];
|
||||
constexpr u32 Invalid = 0xffffffffu, Terminal = 0xfffffffeu;
|
||||
constexpr u32 Parent = 1u << 29, Overflow = 1u << 30, Return = 1u << 31;
|
||||
const auto state = [](u32 count, u32 parent, u32 ring, u32 flags = 0) {
|
||||
return count | (parent << 5) | (ring << 10) | flags;
|
||||
};
|
||||
test.initial.resize(wave_size * 32);
|
||||
struct Output { u32 control, node; std::array<u32, 8> lds; };
|
||||
std::vector<Output> outputs(wave_size);
|
||||
for (u32 lane = 0; lane < wave_size; ++lane) {
|
||||
const u32 which = lane % 16;
|
||||
const u32 base = which == 15 ? 0x3fff : (lane / 32) * capacity * 32 + lane % 32;
|
||||
auto* input = test.initial.data() + lane * 32;
|
||||
auto& out = outputs[lane];
|
||||
input[0] = base << 15;
|
||||
input[1] = Invalid;
|
||||
std::fill(input + 2, input + 10, Invalid);
|
||||
input[10] = base * 4;
|
||||
for (u32 i = 0; i < 8; ++i) out.lds[i] = 100 + i * 8;
|
||||
if (capacity == 8) out.lds[7] = 88;
|
||||
out.node = Terminal;
|
||||
out.control = 0;
|
||||
const auto children = [&] {
|
||||
for (u32 i = 0; i < 8; ++i) input[2 + i] = (i + 1) * 8;
|
||||
};
|
||||
switch (which) {
|
||||
case 0: break;
|
||||
case 1: input[0] |= Overflow; out.control = Overflow; out.node = Invalid; break;
|
||||
case 2:
|
||||
children(); out.control = state(7, 7, 7); out.node = 8;
|
||||
out.lds = {64, 56, 48, 40, 32, 24, 16, 8}; break;
|
||||
case 3:
|
||||
input[2] = 8; input[4] = 24; input[7] = 48; input[9] = 64;
|
||||
out.control = state(3, 3, 3); out.node = 8;
|
||||
out.lds[0] = 64; out.lds[1] = 48; out.lds[2] = 24; out.lds[3] = 8; break;
|
||||
case 4:
|
||||
case 14:
|
||||
input[0] |= state(capacity, capacity, 0, which == 14 ? Parent : 0);
|
||||
input[2] = 8; out.control = state(capacity - 1, capacity - 1, 0, Overflow);
|
||||
out.node = out.lds[0] = 8; break;
|
||||
case 5:
|
||||
input[0] |= state(1, 1, 0); out.control = state(0, 0, capacity - 1); out.node = 88; break;
|
||||
case 6:
|
||||
input[0] |= state(1, 0, 1, Parent);
|
||||
out.control = state(0, 31, 0, Return); out.node = 100; break;
|
||||
case 7:
|
||||
input[2] = 14; out.control = Parent; out.node = out.lds[0] = 14; break;
|
||||
case 8:
|
||||
input[0] |= state(3, 1, 3, Parent);
|
||||
out.lds[2] = 14;
|
||||
out.control = state(0, 0, 2, Parent | Overflow); out.node = 14; break;
|
||||
case 9:
|
||||
children(); input[1] = 24; out.control = state(4, 4, 4); out.node = 32;
|
||||
out.lds[0] = 64; out.lds[1] = 56; out.lds[2] = 48; out.lds[3] = 40; out.lds[4] = 32; break;
|
||||
case 10:
|
||||
children(); input[1] = 96; input[0] |= state(1, 1, 1);
|
||||
out.node = 100; break;
|
||||
case 11:
|
||||
children(); input[9] = 0xfffffffbu; out.control = state(3, 3, 3); out.node = 8;
|
||||
out.lds[0] = 32; out.lds[1] = 24; out.lds[2] = 16; out.lds[3] = 8; break;
|
||||
case 12: children(); input[9] = 0xfffffff9u; break;
|
||||
case 13:
|
||||
children(); input[5] = 0xfffffffdu; out.control = state(3, 3, 3); out.node = 40;
|
||||
out.lds[0] = 64; out.lds[1] = 56; out.lds[2] = 48; out.lds[3] = 40; break;
|
||||
case 15:
|
||||
input[0] |= state(1, 1, 1); out.node = 0; out.lds.fill(0); break;
|
||||
}
|
||||
out.control |= base << 15;
|
||||
if ((lane % 32) >= 16) {
|
||||
out.control = input[0]; out.node = input[2];
|
||||
for (u32 i = 0; i < 8; ++i) out.lds[i] = 100 + i * 8;
|
||||
if (capacity == 8) out.lds[7] = 88;
|
||||
if (which == 8) out.lds[2] = 14;
|
||||
if (which == 15) out.lds.fill(0);
|
||||
}
|
||||
}
|
||||
test.expected = test.initial;
|
||||
for (u32 lane = 0; lane < wave_size; ++lane) {
|
||||
test.expected[lane * 32 + 20] = outputs[lane].control;
|
||||
test.expected[lane * 32 + 21] = outputs[lane].node;
|
||||
std::copy(outputs[lane].lds.begin(), outputs[lane].lds.end(), test.expected.begin() + lane * 32 + 24);
|
||||
}
|
||||
auto& code = test.code;
|
||||
code.push_back(EncodeVop2(0x1a, 1, InlineU32(7), 0));
|
||||
code.insert(code.end(), {EncodeMubuf0(0x0d), EncodeMubuf1(8, 0, 1),
|
||||
EncodeMubuf0(0x0e, 8), EncodeMubuf1(16, 0, 1),
|
||||
EncodeMubuf0(0x0e, 24), EncodeMubuf1(20, 0, 1),
|
||||
EncodeMubuf0(0x0c, 40), EncodeMubuf1(2, 0, 1)});
|
||||
for (u32 i = 0; i < 8; ++i) {
|
||||
AppendVMovU32(&code, 3, 100 + i * 8);
|
||||
code.insert(code.end(), {EncodeDs0(0x0d, i * 128), EncodeDs1(0, 3, 2)});
|
||||
}
|
||||
AppendVMovU32(&code, 3, 88);
|
||||
code.insert(code.end(), {EncodeDs0(0x0d, (capacity - 1) * 128), EncodeDs1(0, 3, 2)});
|
||||
// Lane eight starts with a nested instance on top of its stack.
|
||||
AppendSMovLiteral(&code, 126, 0x01000100u);
|
||||
if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x01000100u);
|
||||
AppendVMovU32(&code, 3, 14);
|
||||
code.insert(code.end(), {EncodeDs0(0x0d, 256), EncodeDs1(0, 3, 2)});
|
||||
AppendSMovLiteral(&code, 126, 0x0000ffffu);
|
||||
if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x0000ffffu);
|
||||
code.insert(code.end(), {EncodeDs0(0xe1, capacity), EncodeDs1Ex(16, 16, 9, 8)});
|
||||
code.push_back(EncodeSop1(0x04, 126, 193));
|
||||
code.push_back(EncodeVop2(0x1a, 3, InlineU32(5), 0));
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 8, 3, 20);
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 16, 3, 21);
|
||||
for (u32 i = 0; i < 8; ++i) {
|
||||
code.insert(code.end(), {EncodeDs0(0x36, i * 128), EncodeDs1(24, 0, 2)});
|
||||
AppendStoreVgprAtLaneDwordOffset(&code, 24, 3, 24 + i);
|
||||
}
|
||||
AppendEnd(&code);
|
||||
test.compute_info.wave_size = wave_size;
|
||||
test.compute_info.threads_num[0] = wave_size;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
test.compute_info.thread_ids_num = 1;
|
||||
test.compute_info.lds_size_dwords = wave_size * capacity;
|
||||
test.compute_info.lds_storage = storage;
|
||||
test.has_compute_info = true;
|
||||
test.decoded_counts = {{"DS_BVH_STACK_PUSH8_POP1_RTN_B32", 1}};
|
||||
test.forbidden_spirv = {"OpControlBarrier", "OpMemoryBarrier", "get_bda_pointer"};
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase FlatStackApertures(u32 wave_size) {
|
||||
TestCase test;
|
||||
test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64";
|
||||
|
||||
@@ -1,4 +1,45 @@
|
||||
void TestTrinityStackInstruction() {
|
||||
namespace D = ShaderRecompiler::Decoder;
|
||||
const std::array code{0xdb840008u, 0x01010a09u,
|
||||
0xdb840010u, 0x02010a09u, EncodeSopp(0x01)};
|
||||
D::Program decoded;
|
||||
D::DecodeProgram(code, decoded);
|
||||
const auto& first = decoded.instructions.front();
|
||||
Check(first.opcode == D::Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 &&
|
||||
first.src_count == 3 && first.offset == 8 && first.dst.reg == 1 &&
|
||||
first.src0.reg == 9 && first.src1.reg == 10 && first.src2.reg == 1,
|
||||
"Trinity stack operands or capacity were not decoded");
|
||||
ShaderComputeInputInfo compute{};
|
||||
compute.lds_size_dwords = 1024;
|
||||
auto options = MakeCompileOptions(ShaderType::Compute);
|
||||
options.input_info.compute = &compute;
|
||||
const auto result = RecompileForTest(code, options);
|
||||
Check(CountSourceOccurrences(result.ir_dump, " = BvhStack ") == 2 &&
|
||||
result.program.info.uses_lds && result.program.info.bvh_stack &&
|
||||
!result.program.info.shared_int64_atomics,
|
||||
"unused stack results discarded LDS writes or selected 64-bit atomics");
|
||||
Check(SpirvInstructionOpcodeCount(result.spirv, 54u) == 2 &&
|
||||
SpirvInstructionOpcodeCount(result.spirv, 246u) == 1,
|
||||
"Trinity stack calls duplicated the function or unrolled its eight pushes");
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
compute.lds_storage = true;
|
||||
CheckSpirvBinaryValidates(RecompileForTest(code, options).spirv);
|
||||
options = MakeCompileOptions(ShaderType::Pixel);
|
||||
CheckSpirvBinaryValidates(RecompileForTest(code, options).spirv);
|
||||
for (const auto words : {std::array{0xdb860008u, 0x01010a09u},
|
||||
std::array{0xdb840007u, 0x01010a09u},
|
||||
std::array{0xdb840020u, 0x01010a09u},
|
||||
std::array{0xdb840008u, 0x01f90a09u},
|
||||
std::array{0xdb840008u, 0x09010a09u}}) {
|
||||
const std::array invalid{words[0], words[1], EncodeSopp(0x01)};
|
||||
D::DecodeProgram(invalid, decoded);
|
||||
Check(decoded.instructions[0].opcode == D::Opcode::UNSUPPORTED,
|
||||
"unsupported stack encoding was silently accepted");
|
||||
}
|
||||
}
|
||||
|
||||
void TestRayTracingInstructions() {
|
||||
TestTrinityStackInstruction();
|
||||
namespace D = ShaderRecompiler::Decoder;
|
||||
const std::array code{
|
||||
0xf1989f01u, 0x00081014u,
|
||||
|
||||
@@ -46807,6 +46807,10 @@ std::vector<TestCase> MakeCases() {
|
||||
AddCase(ScratchIsPrivatePerInvocation);
|
||||
cases.push_back(FlatStackApertures(32));
|
||||
cases.push_back(FlatStackApertures(64));
|
||||
for (const u32 wave : {32u, 64u}) {
|
||||
for (const u32 capacity : {8u, 16u, 31u}) cases.push_back(TrinityStack(wave, capacity));
|
||||
cases.push_back(TrinityStack(wave, 8, true));
|
||||
}
|
||||
cases.push_back(BvhIntersections(true, true, 1));
|
||||
cases.push_back(BvhIntersections(true, true, 0, true));
|
||||
for (bool barycentrics : {false, true}) {
|
||||
@@ -53346,6 +53350,10 @@ int main(int argc, char **argv) {
|
||||
#endif
|
||||
if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
for (const u32 wave : {32u, 64u}) {
|
||||
for (const u32 capacity : {8u, 16u, 31u}) RunCase(&vulkan, TrinityStack(wave, capacity));
|
||||
RunCase(&vulkan, TrinityStack(wave, 8, true));
|
||||
}
|
||||
RunCase(&vulkan, FlatStackApertures(32));
|
||||
RunCase(&vulkan, FlatStackApertures(64));
|
||||
RunCase(&vulkan, BvhIntersections(true, true, 1));
|
||||
|
||||
Reference in new issue
Block a user