diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index b4add49be..42825d9f8 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -338,6 +338,7 @@ inline constexpr auto EmitDataConsume = EmitAppendConsume; uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst); void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst); inline constexpr auto EmitImageQueryDimensions = EmitImage; inline constexpr auto EmitImageQueryLod = EmitImage; inline constexpr auto EmitImageRead = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 46d602444..5a0a1418f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -120,6 +120,7 @@ struct EmitterState { uint32_t indirect_format_geometry_function = 0; std::map, uint32_t> indirect_buffer_load_functions; uint32_t bvh_intersect_function = 0; + uint32_t bvh_stack_function = 0; uint32_t gds_variable = 0; uint32_t gds_length = 0; uint32_t push_constant_variable = 0; @@ -542,6 +543,7 @@ void EmitProgram(EmitterState& state); void DefineGetBdaPointer(EmitterState& state); void DefineIndirectBufferLoads(ValueEmitContext& ctx); void DefineBvhIntersect(EmitterState& state); +void DefineTrinityFunctions(EmitterState& state); uint32_t GetBdaPointer(EmitterState& state, uint32_t address); // These templates accept local lambdas from several emitter translation units. diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index d35e769e2..f598365ee 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -676,6 +676,7 @@ void EmitProgram(EmitterState& state) { DefineGetBdaPointer(state); DefineIndirectBufferLoads(ctx); DefineBvhIntersect(state); + DefineTrinityFunctions(state); if (program.info.uses_swizzle) { ctx.scratch_u32_variable = state.builder.AllocateId(); if (state.lane_count == 2) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp new file mode 100644 index 000000000..a952830c6 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp @@ -0,0 +1,207 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +uint32_t Bits(EmitterState& s, uint32_t value, uint32_t offset, uint32_t count) { + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpBitFieldUExtract, TypeU32(s), result, value, + ConstantU32(s, offset), ConstantU32(s, count)); + return result; +} + +uint32_t SetBits(EmitterState& s, uint32_t value, uint32_t field, uint32_t offset, uint32_t count) { + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpBitFieldInsert, TypeU32(s), result, value, field, + ConstantU32(s, offset), ConstantU32(s, count)); + return result; +} + +uint32_t Pair(EmitterState& s, uint32_t first, uint32_t second) { + const auto result = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeConstruct, TypeU32Pair(s), result, first, second); + return result; +} + +uint32_t StackIndex(EmitterState& s, uint32_t control, uint32_t ring) { + return EmitAddU32(s, Bits(s, control, 15, 14), + Binary(s, spv::OpShiftLeftLogical, TypeU32(s), ring, ConstantU32(s, 5))); +} + +uint32_t StackPush(EmitterState& s, const MemoryResourceAccess& memory, + uint32_t control, uint32_t node, uint32_t capacity) { + const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1); + const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5); + const auto index = StackIndex(s, control, ring); + EmitIfCondition(s, EmitMemoryElementInBounds(s, memory, index), [&] { + s.builder.AddFunction(spv::OpStore, EmitMemoryElementPointer(s, memory, index), node); + }); + control = Binary(s, spv::OpBitwiseOr, u, control, + Select(s, u, Binary(s, spv::OpIEqual, b, count, capacity), ConstantU32(s, 1u << 30), zero)); + control = Select(s, u, Binary(s, spv::OpIEqual, b, parent, capacity), + Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control); + control = SetBits(s, control, EmitGlsl(s, + EmitAddU32(s, count, one), capacity), 0, 5); + control = SetBits(s, control, EmitGlsl(s, + EmitAddU32(s, parent, one), capacity), 5, 5); + const auto next = EmitAddU32(s, ring, one); + return SetBits(s, control, Select(s, u, Binary(s, spv::OpIEqual, b, next, capacity), zero, next), 10, 5); +} + +uint32_t StackPop(EmitterState& s, const MemoryResourceAccess& memory, + uint32_t control, uint32_t capacity) { + const auto u = TypeU32(s), b = TypeBool(s), zero = ConstantU32(s, 0), one = ConstantU32(s, 1); + const auto count = Bits(s, control, 0, 5), parent = Bits(s, control, 5, 5), ring = Bits(s, control, 10, 5); + const auto has_parent = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero); + const auto returning = Binary(s, spv::OpLogicalAnd, b, has_parent, + Binary(s, spv::OpIEqual, b, parent, zero)); + control = SetBits(s, control, Select(s, u, returning, one, zero), 31, 1); + control = Select(s, u, returning, + Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, ~(1u << 29))), control); + const auto overflow = Binary(s, spv::OpINotEqual, b, Bits(s, control, 30, 1), zero); + const auto empty = Pair(s, + Binary(s, spv::OpBitwiseAnd, u, control, ConstantU32(s, 0x5fffffff)), + Select(s, u, overflow, ConstantU32(s, 0xffffffff), ConstantU32(s, 0xfffffffe))); + return EmitValueOrDefaultIfCondition(s, Binary(s, spv::OpINotEqual, b, count, zero), + TypeU32Pair(s), empty, [&] { + control = SetBits(s, control, Binary(s, spv::OpISub, u, count, one), 0, 5); + control = SetBits(s, control, Binary(s, spv::OpISub, u, parent, one), 5, 5); + const auto previous = Select(s, u, Binary(s, spv::OpIEqual, b, ring, zero), + Binary(s, spv::OpISub, u, capacity, one), Binary(s, spv::OpISub, u, ring, one)); + control = SetBits(s, control, previous, 10, 5); + const auto index = StackIndex(s, control, previous); + const auto node = EmitValueOrDefaultIfCondition(s, + EmitMemoryElementInBounds(s, memory, index), u, zero, [&] { + const auto value = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpLoad, u, value, EmitMemoryElementPointer(s, memory, index)); + return value; + }); + const auto instance = Binary(s, spv::OpIEqual, b, Bits(s, node, 0, 3), ConstantU32(s, 6)); + control = EmitValueOrDefaultIfCondition(s, instance, u, control, [&] { + const auto nested = Binary(s, spv::OpINotEqual, b, Bits(s, control, 29, 1), zero); + const auto discarded = Binary(s, spv::OpBitwiseOr, u, + SetBits(s, control, Bits(s, control, 5, 5), 0, 5), ConstantU32(s, 1u << 30)); + return Binary(s, spv::OpBitwiseOr, u, + SetBits(s, Select(s, u, nested, discarded, control), zero, 5, 5), ConstantU32(s, 1u << 29)); + }); + return Pair(s, control, node); + }); +} + +void DefineBvhStack(EmitterState& s) { + if (!s.program.info.bvh_stack) return; + const auto u = TypeU32(s), b = TypeBool(s), vec4 = TypeU32Vector(s, 4), pair = TypeU32Pair(s); + const auto zero = ConstantU32(s, 0), one = ConstantU32(s, 1); + const bool local = s.lds_storage_class == spv::StorageClassFunction; + const bool buffer = s.lds_storage_class == spv::StorageClassStorageBuffer; + std::vector types {u, u, vec4, vec4, u}; + if (local) types.push_back(TypeU32ArrayPointer(s, spv::StorageClassFunction, LdsDwordCount(s))); + if (buffer) types.push_back(u); + const auto signature = s.builder.Type(spv::OpTypeFunction, pair, types); + s.bvh_stack_function = s.builder.AllocateId(); + s.builder.AddName(s.bvh_stack_function, "trinity_bvh_stack"); + s.builder.AddFunction(spv::OpFunction, pair, s.bvh_stack_function, spv::FunctionControlMaskNone, signature); + std::vector args; + for (const auto type: types) { + args.push_back(s.builder.AllocateId()); + s.builder.AddFunction(spv::OpFunctionParameter, type, args.back()); + } + const auto old_lds = s.lds_variable, old_base = s.lds_base_dwords; + if (local) s.lds_variable = args.back(); + if (buffer) s.lds_base_dwords = args.back(); + const auto memory = PrepareMemoryResourceAccess(s, {.kind = IR::ResourceKind::Lds}); + const auto entry = s.builder.AllocateId(); + EmitLabel(s, entry); + const auto last = args[1], capacity = args[4]; + const auto walkback = Binary(s, spv::OpINotEqual, b, Bits(s, last, 3, 29), ConstantU32(s, 0x1fffffff)); + const auto last4 = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeConstruct, vec4, last4, last, last, last, last); + const auto found = [&](uint32_t values) { + return Unary(s, spv::OpAny, b, Binary(s, spv::OpIEqual, TypeBoolVector(s, 4), values, last4)); + }; + const auto initial_valid = Binary(s, spv::OpLogicalOr, b, Unary(s, spv::OpLogicalNot, b, walkback), + Binary(s, spv::OpLogicalOr, b, found(args[2]), found(args[3]))); + const auto header = s.builder.AllocateId(), body = s.builder.AllocateId(); + const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpBranch, header); + EmitLabel(s, header); + const auto index = s.builder.AllocateId(), control = s.builder.AllocateId(); + const auto valid = s.builder.AllocateId(), skip_low = s.builder.AllocateId(), skip_high = s.builder.AllocateId(); + const auto index_phi = s.builder.AddDeferredPhi(u, index, 2); + const auto control_phi = s.builder.AddDeferredPhi(u, control, 2); + const auto valid_phi = s.builder.AddDeferredPhi(b, valid, 2); + const auto low_phi = s.builder.AddDeferredPhi(b, skip_low, 2); + const auto high_phi = s.builder.AddDeferredPhi(b, skip_high, 2); + const auto more = Binary(s, spv::OpLogicalAnd, b, valid, + Binary(s, spv::OpULessThan, b, index, ConstantU32(s, 8))); + s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask); + s.builder.AddFunction(spv::OpBranchConditional, more, body, merge); + EmitLabel(s, body); + const auto high = Binary(s, spv::OpUGreaterThanEqual, b, index, ConstantU32(s, 4)); + const auto high4 = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeConstruct, TypeBoolVector(s, 4), high4, high, high, high, high); + const auto values = Select(s, vec4, high4, args[3], args[2]); + const auto node = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpVectorExtractDynamic, u, node, values, Bits(s, index, 0, 2)); + const auto sentinel = Binary(s, spv::OpIEqual, b, Bits(s, node, 3, 29), ConstantU32(s, 0x1fffffff)); + const auto enabled = Unary(s, spv::OpLogicalNot, b, Select(s, b, high, skip_high, skip_low)); + const auto marker = Binary(s, spv::OpLogicalAnd, b, sentinel, enabled); + const auto next_low = Binary(s, spv::OpLogicalOr, b, skip_low, + Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 1, 1), zero))); + const auto next_high = Binary(s, spv::OpLogicalOr, b, skip_high, + Binary(s, spv::OpLogicalAnd, b, high, + Binary(s, spv::OpLogicalAnd, b, marker, Binary(s, spv::OpIEqual, b, Bits(s, node, 2, 1), zero)))); + const auto next_valid = Unary(s, spv::OpLogicalNot, b, + Binary(s, spv::OpLogicalAnd, b, walkback, Binary(s, spv::OpIEqual, b, node, last))); + const auto push = Binary(s, spv::OpLogicalAnd, b, next_valid, + Unary(s, spv::OpLogicalNot, b, + Binary(s, spv::OpLogicalOr, b, sentinel, Select(s, b, high, next_high, next_low)))); + const auto next_control = EmitValueOrDefaultIfCondition(s, push, u, control, [&] { + return StackPush(s, memory, control, node, capacity); + }); + s.builder.AddFunction(spv::OpBranch, cont); + EmitLabel(s, cont); + const auto next_index = Binary(s, spv::OpISub, u, index, one); + s.builder.AddFunction(spv::OpBranch, header); + s.builder.PatchDeferredPhi(index_phi, 0, ConstantU32(s, 7), entry); + s.builder.PatchDeferredPhi(index_phi, 1, next_index, cont); + s.builder.PatchDeferredPhi(control_phi, 0, args[0], entry); + s.builder.PatchDeferredPhi(control_phi, 1, next_control, cont); + s.builder.PatchDeferredPhi(valid_phi, 0, initial_valid, entry); + s.builder.PatchDeferredPhi(valid_phi, 1, next_valid, cont); + s.builder.PatchDeferredPhi(low_phi, 0, ConstantBool(s, false), entry); + s.builder.PatchDeferredPhi(low_phi, 1, next_low, cont); + s.builder.PatchDeferredPhi(high_phi, 0, ConstantBool(s, false), entry); + s.builder.PatchDeferredPhi(high_phi, 1, next_high, cont); + EmitLabel(s, merge); + s.builder.AddFunction(spv::OpReturnValue, StackPop(s, memory, control, capacity)); + s.builder.AddFunction(spv::OpFunctionEnd); + if (local) s.lds_variable = old_lds; + if (buffer) s.lds_base_dwords = old_base; +} + +} // namespace + +void DefineTrinityFunctions(EmitterState& s) { + DefineBvhStack(s); +} + +uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& s = ctx.state; + const auto capacity = ctx.Memory(inst).offset; + if (s.bvh_stack_function == 0 || capacity < 8 || capacity > 31) + ctx.Fail(inst, "invalid BVH stack function or capacity"); + const std::array values {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}; + return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 4), TypeU32Pair(s), + ConstantU32CompositeZero(s, 2), [&] { + const auto result = s.builder.AllocateId(); + std::vector call {spv::OpFunctionCall, TypeU32Pair(s), result, + s.bvh_stack_function, values[0], values[1], values[2], values[3], ConstantU32(s, capacity)}; + if (s.lds_storage_class == spv::StorageClassFunction) call.push_back(s.lds_variable); + if (s.lds_storage_class == spv::StorageClassStorageBuffer) call.push_back(s.lds_base_dwords); + s.builder.AddFunction(call); + return result; + }); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 296ddc8ed..92d5cae47 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -142,6 +142,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32}, {0xb2u, Opcode::DS_PERMUTE_B32, 1, 32}, {0xb3u, Opcode::DS_BPERMUTE_B32, 1, 32}, {0xdeu, Opcode::DS_WRITE_B96, 3, 32}, {0xdfu, Opcode::DS_WRITE_B128, 4, 32}, + {0xe1u, Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32, 1, 32}, {0xfeu, Opcode::DS_READ_B96, 3, 32}, {0xffu, Opcode::DS_READ_B128, 4, 32}, }; @@ -232,6 +233,7 @@ bool IsDsAtomicOpcode(Opcode opcode) { uint32_t DsSourceCount(Opcode opcode) { switch (opcode) { + case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return 3u; case Opcode::DS_WRITE2_B32: case Opcode::DS_WRITE2ST64_B32: case Opcode::DS_WRITE2_B64: @@ -473,6 +475,13 @@ void DecodeDs(uint32_t pc, std::span code, uint32_t word_index, if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) { SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented"); } + if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 && + (inst.gds || inst.offset < 8u || inst.offset > 31u || data1 > 248u)) { + SetUnsupported(inst, Family::DS, opcode, "invalid BVH stack operands or capacity"); + } + if (inst.opcode == Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 && vdst == addr) { + SetUnsupported(inst, Family::DS, opcode, "overlapping BVH stack outputs are not implemented"); + } if (inst.gds && inst.opcode == Opcode::DS_CMPST_RTN_B32) { SetUnsupported(inst, Family::DS, opcode, "DS compare-store return B32 GDS variant is not implemented"); } diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index e00b5731e..16ad123d9 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -740,6 +740,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_READ2ST64_B64: case Opcode::DS_READ_B96: case Opcode::DS_READ_B128: + case Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: case Opcode::DS_WRITE_B8: case Opcode::DS_WRITE_B16: case Opcode::DS_WRITE_B8_D16_HI: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 6f34d647d..7fed56277 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -685,6 +685,7 @@ enum class Opcode { DS_READ2ST64_B64, DS_READ_B96, DS_READ_B128, + DS_BVH_STACK_PUSH8_POP1_RTN_B32, DS_WRITE_B8, DS_WRITE_B16, DS_WRITE_B8_D16_HI, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 16cd25489..b44980b98 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -781,6 +781,15 @@ void Translator::WriteSharedU32(uint32_t width, IR::U32 address, } } +void Translator::DS_BVH_STACK(const Decoder::Instruction& inst) { + const auto result = ir.Emit(IR::ValueOpcode::BvhStack, + {ReadU32(inst.src0), ReadU32(inst.src1), ConstructU32x4(inst.src2, 4), + ConstructU32x4(OffsetOperand(inst.src2, 4), 4), ir.GetExec()}, + AddMemoryInfo(MemoryInfoFromDecoded(inst), inst.pc)); + WriteOperand(inst.src0, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(0u)})); + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, {result, IR::Value(1u)})); +} + void Translator::DS_READ(const Decoder::Instruction& inst) { const auto memory = MemoryInfoFromDecoded(inst); const auto address = ReadU32(MemorySourceAt(inst, 0)); @@ -1186,6 +1195,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataAppend); case Decoder::Opcode::DS_WRITE_ADDTID_B32: return DS_ADDTID(inst, true); case Decoder::Opcode::DS_READ_ADDTID_B32: return DS_ADDTID(inst, false); + case Decoder::Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32: return DS_BVH_STACK(inst); case Decoder::Opcode::DS_READ2_B32: case Decoder::Opcode::DS_READ2ST64_B32: case Decoder::Opcode::DS_READ2_B64: diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index c30aa53fa..4f28a976f 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -105,6 +105,7 @@ private: void WriteSharedU32(uint32_t width, IR::U32 address, const std::array& values, const IR::MemoryInfo& memory, uint32_t pc); void DS_READ(const Decoder::Instruction& inst); + void DS_BVH_STACK(const Decoder::Instruction& inst); void DS_READ2(const Decoder::Instruction& inst); void DS_WRITE(const Decoder::Instruction& inst); void DS_WRITE2(const Decoder::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/Serialize.h b/src/graphics/shader/recompiler/ir/Serialize.h index e7de23560..0bbf4cf1e 100644 --- a/src/graphics/shader/recompiler/ir/Serialize.h +++ b/src/graphics/shader/recompiler/ir/Serialize.h @@ -155,7 +155,7 @@ KYTY_IO_FIELDS(SampledResourcePair, a, b, c) KYTY_IO_FIELDS(StageInput, a, b, c, d, e) KYTY_IO_FIELDS(StageOutput, a, b, c, d) KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, - A, B, C, D, E, F, G, H, I, J, K) + A, B, C, D, E, F, G, H, I, J, K, L) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks, a, b, c) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j, k, diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 6a5a06f3c..c0b647c7d 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -543,6 +543,7 @@ struct ShaderInfo { bool uses_image_heap = false; bool uses_dispatch_threads = false; bool bvh = false; + bool bvh_stack = false; bool subgroup_ballot = false; bool subgroup_vote = false; bool subgroup_barrier = false; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index 3beebab9b..e6c31a5ad 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -76,7 +76,8 @@ bool HasSideEffects(ValueOpcode opcode) { return true; } const auto shared_access = SharedAccessOf(opcode); - if (shared_access == SharedAccess::Write || shared_access == SharedAccess::Atomic || + if (shared_access == SharedAccess::Write || shared_access == SharedAccess::ReadWrite || + shared_access == SharedAccess::Atomic || shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) { return true; } @@ -156,6 +157,7 @@ uint32_t BufferComponentCount(ValueOpcode opcode) { SharedAccess SharedAccessOf(ValueOpcode opcode) { switch (opcode) { + case ValueOpcode::BvhStack: return SharedAccess::ReadWrite; case ValueOpcode::LoadSharedU8: case ValueOpcode::LoadSharedU16: case ValueOpcode::LoadSharedU32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index 0127a2dd0..7873f5885 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -17,7 +17,7 @@ enum class ValueOpcode : uint16_t { }; enum class BufferAccess { None, Read, Write, Atomic }; -enum class SharedAccess { None, Read, Write, Atomic, Append, Consume }; +enum class SharedAccess { None, Read, Write, ReadWrite, Atomic, Append, Consume }; enum class AddressAccess { None, Read, Write }; enum class ImageAccess { None, Read, Write, Atomic }; enum class ImageResourceClass { None, Sampled, Storage }; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 5cff6d0dd..033862b63 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -318,6 +318,7 @@ VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1) +VALUE_OPCODE(BvhStack, U32x2, U32, U32, U32x4, U32x4, U1) VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress) VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1) VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 54da499d7..c27f98d0d 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -515,6 +515,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break; case ValueOpcode::DataCacheInvalidate: info.coherent_buffers = true; break; case ValueOpcode::BvhIntersect: info.bvh = true; break; + case ValueOpcode::BvhStack: info.bvh_stack = true; break; case ValueOpcode::ConditionRef: info.subgroup_vote |= inst.Flags() != CFG::BranchCondition::ScalarInstruction; diff --git a/tests/ShaderRayTracingGpuTests.inc b/tests/ShaderRayTracingGpuTests.inc index f2bdb2a29..a38dc7ad1 100644 --- a/tests/ShaderRayTracingGpuTests.inc +++ b/tests/ShaderRayTracingGpuTests.inc @@ -1,3 +1,132 @@ +TestCase TrinityStack(u32 wave_size, u32 capacity, bool storage = false) { + TestCase test; + static constexpr const char* names[2][3] = { + {"TrinityStack32Capacity8", "TrinityStack32Capacity16", "TrinityStack32Capacity31"}, + {"TrinityStack64Capacity8", "TrinityStack64Capacity16", "TrinityStack64Capacity31"}}; + test.name = storage ? (wave_size == 32 ? "TrinityStackStorage32" : "TrinityStackStorage64") + : names[wave_size == 64][capacity == 8 ? 0 : capacity == 16 ? 1 : 2]; + constexpr u32 Invalid = 0xffffffffu, Terminal = 0xfffffffeu; + constexpr u32 Parent = 1u << 29, Overflow = 1u << 30, Return = 1u << 31; + const auto state = [](u32 count, u32 parent, u32 ring, u32 flags = 0) { + return count | (parent << 5) | (ring << 10) | flags; + }; + test.initial.resize(wave_size * 32); + struct Output { u32 control, node; std::array lds; }; + std::vector outputs(wave_size); + for (u32 lane = 0; lane < wave_size; ++lane) { + const u32 which = lane % 16; + const u32 base = which == 15 ? 0x3fff : (lane / 32) * capacity * 32 + lane % 32; + auto* input = test.initial.data() + lane * 32; + auto& out = outputs[lane]; + input[0] = base << 15; + input[1] = Invalid; + std::fill(input + 2, input + 10, Invalid); + input[10] = base * 4; + for (u32 i = 0; i < 8; ++i) out.lds[i] = 100 + i * 8; + if (capacity == 8) out.lds[7] = 88; + out.node = Terminal; + out.control = 0; + const auto children = [&] { + for (u32 i = 0; i < 8; ++i) input[2 + i] = (i + 1) * 8; + }; + switch (which) { + case 0: break; + case 1: input[0] |= Overflow; out.control = Overflow; out.node = Invalid; break; + case 2: + children(); out.control = state(7, 7, 7); out.node = 8; + out.lds = {64, 56, 48, 40, 32, 24, 16, 8}; break; + case 3: + input[2] = 8; input[4] = 24; input[7] = 48; input[9] = 64; + out.control = state(3, 3, 3); out.node = 8; + out.lds[0] = 64; out.lds[1] = 48; out.lds[2] = 24; out.lds[3] = 8; break; + case 4: + case 14: + input[0] |= state(capacity, capacity, 0, which == 14 ? Parent : 0); + input[2] = 8; out.control = state(capacity - 1, capacity - 1, 0, Overflow); + out.node = out.lds[0] = 8; break; + case 5: + input[0] |= state(1, 1, 0); out.control = state(0, 0, capacity - 1); out.node = 88; break; + case 6: + input[0] |= state(1, 0, 1, Parent); + out.control = state(0, 31, 0, Return); out.node = 100; break; + case 7: + input[2] = 14; out.control = Parent; out.node = out.lds[0] = 14; break; + case 8: + input[0] |= state(3, 1, 3, Parent); + out.lds[2] = 14; + out.control = state(0, 0, 2, Parent | Overflow); out.node = 14; break; + case 9: + children(); input[1] = 24; out.control = state(4, 4, 4); out.node = 32; + out.lds[0] = 64; out.lds[1] = 56; out.lds[2] = 48; out.lds[3] = 40; out.lds[4] = 32; break; + case 10: + children(); input[1] = 96; input[0] |= state(1, 1, 1); + out.node = 100; break; + case 11: + children(); input[9] = 0xfffffffbu; out.control = state(3, 3, 3); out.node = 8; + out.lds[0] = 32; out.lds[1] = 24; out.lds[2] = 16; out.lds[3] = 8; break; + case 12: children(); input[9] = 0xfffffff9u; break; + case 13: + children(); input[5] = 0xfffffffdu; out.control = state(3, 3, 3); out.node = 40; + out.lds[0] = 64; out.lds[1] = 56; out.lds[2] = 48; out.lds[3] = 40; break; + case 15: + input[0] |= state(1, 1, 1); out.node = 0; out.lds.fill(0); break; + } + out.control |= base << 15; + if ((lane % 32) >= 16) { + out.control = input[0]; out.node = input[2]; + for (u32 i = 0; i < 8; ++i) out.lds[i] = 100 + i * 8; + if (capacity == 8) out.lds[7] = 88; + if (which == 8) out.lds[2] = 14; + if (which == 15) out.lds.fill(0); + } + } + test.expected = test.initial; + for (u32 lane = 0; lane < wave_size; ++lane) { + test.expected[lane * 32 + 20] = outputs[lane].control; + test.expected[lane * 32 + 21] = outputs[lane].node; + std::copy(outputs[lane].lds.begin(), outputs[lane].lds.end(), test.expected.begin() + lane * 32 + 24); + } + auto& code = test.code; + code.push_back(EncodeVop2(0x1a, 1, InlineU32(7), 0)); + code.insert(code.end(), {EncodeMubuf0(0x0d), EncodeMubuf1(8, 0, 1), + EncodeMubuf0(0x0e, 8), EncodeMubuf1(16, 0, 1), + EncodeMubuf0(0x0e, 24), EncodeMubuf1(20, 0, 1), + EncodeMubuf0(0x0c, 40), EncodeMubuf1(2, 0, 1)}); + for (u32 i = 0; i < 8; ++i) { + AppendVMovU32(&code, 3, 100 + i * 8); + code.insert(code.end(), {EncodeDs0(0x0d, i * 128), EncodeDs1(0, 3, 2)}); + } + AppendVMovU32(&code, 3, 88); + code.insert(code.end(), {EncodeDs0(0x0d, (capacity - 1) * 128), EncodeDs1(0, 3, 2)}); + // Lane eight starts with a nested instance on top of its stack. + AppendSMovLiteral(&code, 126, 0x01000100u); + if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x01000100u); + AppendVMovU32(&code, 3, 14); + code.insert(code.end(), {EncodeDs0(0x0d, 256), EncodeDs1(0, 3, 2)}); + AppendSMovLiteral(&code, 126, 0x0000ffffu); + if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x0000ffffu); + code.insert(code.end(), {EncodeDs0(0xe1, capacity), EncodeDs1Ex(16, 16, 9, 8)}); + code.push_back(EncodeSop1(0x04, 126, 193)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(5), 0)); + AppendStoreVgprAtLaneDwordOffset(&code, 8, 3, 20); + AppendStoreVgprAtLaneDwordOffset(&code, 16, 3, 21); + for (u32 i = 0; i < 8; ++i) { + code.insert(code.end(), {EncodeDs0(0x36, i * 128), EncodeDs1(24, 0, 2)}); + AppendStoreVgprAtLaneDwordOffset(&code, 24, 3, 24 + i); + } + AppendEnd(&code); + test.compute_info.wave_size = wave_size; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.lds_size_dwords = wave_size * capacity; + test.compute_info.lds_storage = storage; + test.has_compute_info = true; + test.decoded_counts = {{"DS_BVH_STACK_PUSH8_POP1_RTN_B32", 1}}; + test.forbidden_spirv = {"OpControlBarrier", "OpMemoryBarrier", "get_bda_pointer"}; + return test; +} + TestCase FlatStackApertures(u32 wave_size) { TestCase test; test.name = wave_size == 32 ? "FlatStackApertures32" : "FlatStackApertures64"; diff --git a/tests/ShaderRayTracingTests.inc b/tests/ShaderRayTracingTests.inc index 79f403299..48eb09101 100644 --- a/tests/ShaderRayTracingTests.inc +++ b/tests/ShaderRayTracingTests.inc @@ -1,4 +1,45 @@ +void TestTrinityStackInstruction() { + namespace D = ShaderRecompiler::Decoder; + const std::array code{0xdb840008u, 0x01010a09u, + 0xdb840010u, 0x02010a09u, EncodeSopp(0x01)}; + D::Program decoded; + D::DecodeProgram(code, decoded); + const auto& first = decoded.instructions.front(); + Check(first.opcode == D::Opcode::DS_BVH_STACK_PUSH8_POP1_RTN_B32 && + first.src_count == 3 && first.offset == 8 && first.dst.reg == 1 && + first.src0.reg == 9 && first.src1.reg == 10 && first.src2.reg == 1, + "Trinity stack operands or capacity were not decoded"); + ShaderComputeInputInfo compute{}; + compute.lds_size_dwords = 1024; + auto options = MakeCompileOptions(ShaderType::Compute); + options.input_info.compute = &compute; + const auto result = RecompileForTest(code, options); + Check(CountSourceOccurrences(result.ir_dump, " = BvhStack ") == 2 && + result.program.info.uses_lds && result.program.info.bvh_stack && + !result.program.info.shared_int64_atomics, + "unused stack results discarded LDS writes or selected 64-bit atomics"); + Check(SpirvInstructionOpcodeCount(result.spirv, 54u) == 2 && + SpirvInstructionOpcodeCount(result.spirv, 246u) == 1, + "Trinity stack calls duplicated the function or unrolled its eight pushes"); + CheckSpirvBinaryValidates(result.spirv); + compute.lds_storage = true; + CheckSpirvBinaryValidates(RecompileForTest(code, options).spirv); + options = MakeCompileOptions(ShaderType::Pixel); + CheckSpirvBinaryValidates(RecompileForTest(code, options).spirv); + for (const auto words : {std::array{0xdb860008u, 0x01010a09u}, + std::array{0xdb840007u, 0x01010a09u}, + std::array{0xdb840020u, 0x01010a09u}, + std::array{0xdb840008u, 0x01f90a09u}, + std::array{0xdb840008u, 0x09010a09u}}) { + const std::array invalid{words[0], words[1], EncodeSopp(0x01)}; + D::DecodeProgram(invalid, decoded); + Check(decoded.instructions[0].opcode == D::Opcode::UNSUPPORTED, + "unsupported stack encoding was silently accepted"); + } +} + void TestRayTracingInstructions() { + TestTrinityStackInstruction(); namespace D = ShaderRecompiler::Decoder; const std::array code{ 0xf1989f01u, 0x00081014u, diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 0ec998f33..86f7d5537 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -46807,6 +46807,10 @@ std::vector MakeCases() { AddCase(ScratchIsPrivatePerInvocation); cases.push_back(FlatStackApertures(32)); cases.push_back(FlatStackApertures(64)); + for (const u32 wave : {32u, 64u}) { + for (const u32 capacity : {8u, 16u, 31u}) cases.push_back(TrinityStack(wave, capacity)); + cases.push_back(TrinityStack(wave, 8, true)); + } cases.push_back(BvhIntersections(true, true, 1)); cases.push_back(BvhIntersections(true, true, 0, true)); for (bool barycentrics : {false, true}) { @@ -53346,6 +53350,10 @@ int main(int argc, char **argv) { #endif if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) { VulkanHarness vulkan; + for (const u32 wave : {32u, 64u}) { + for (const u32 capacity : {8u, 16u, 31u}) RunCase(&vulkan, TrinityStack(wave, capacity)); + RunCase(&vulkan, TrinityStack(wave, 8, true)); + } RunCase(&vulkan, FlatStackApertures(32)); RunCase(&vulkan, FlatStackApertures(64)); RunCase(&vulkan, BvhIntersections(true, true, 1));