From 5eab0ec73855acdf776f5c0669911e2f5ddfb743 Mon Sep 17 00:00:00 2001 From: nmzik Date: Thu, 29 Oct 2026 18:19:40 +0100 Subject: [PATCH] shader: add BVH8 intersections and instance transforms --- .../backend/spirv/spirvEmitterBvh.cpp | 37 +- .../backend/spirv/spirvEmitterInstructions.h | 2 + .../backend/spirv/spirvEmitterInternal.h | 7 + .../backend/spirv/spirvEmitterProgram.cpp | 1 + .../backend/spirv/spirvEmitterTrinity.cpp | 1 + .../backend/spirv/spirvEmitterTrinityBvh.cpp | 511 ++++++++++++++++++ .../recompiler/frontend/decode/ImageOps.cpp | 32 ++ .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 29 +- .../frontend/translate/Translator.h | 1 + src/graphics/shader/recompiler/ir/Serialize.h | 2 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 1 + src/graphics/shader/recompiler/ir/Type.cpp | 1 + src/graphics/shader/recompiler/ir/Type.h | 1 + .../recompiler/ir/opcodes/ValueOpcodes.cpp | 1 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 2 + .../recompiler/ir/passes/ResourceTracking.cpp | 2 +- .../ir/passes/ShaderInfoCollection.cpp | 1 + tests/ShaderRayTracingGpuTests.inc | 303 +++++++++++ tests/ShaderRayTracingTests.inc | 41 ++ tests/ShaderRecompilerComputeTests.cpp | 2 + 21 files changed, 964 insertions(+), 15 deletions(-) create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinityBvh.cpp diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp index bdc49c800..992176150 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp @@ -30,16 +30,7 @@ uint32_t NodeWord(EmitterState& s, uint32_t block, uint32_t word) { uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary, Vec3 origin, Vec3 direction) { const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s); - const auto zero = ConstantF32Value(s, 0.0f); - const auto op = [&](spv::Op code, uint32_t x, uint32_t y) { - const auto value = Binary(s, code, f, x, y); - s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction); - return value; - }; const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); }; - const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); }; - const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); }; - const auto abs = [&](uint32_t x) { return EmitGlsl(s, x); }; const auto is_kind = [&](uint32_t n) { return cmp(spv::OpIEqual, kind, ConstantU32(s, n)); }; // The four triangles are (0,1,2), (1,3,2), (2,3,4), (2,4,0). const std::array vertices { @@ -56,6 +47,28 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary, NodeWord(s, block, Binary(s, spv::OpIAdd, u, first, ConstantU32(s, axis)))); } const auto flag = NodeWord(s, block, ConstantU32(s, 15)); + const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag, + Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8))); + return EmitBvhTriangle(s, positions, origin, direction, remap, + Binary(s, spv::OpIAdd, u, flag, kind), bary); +} + +} // namespace + +uint32_t EmitBvhTriangle(EmitterState& s, const std::array& positions, + Vec3 origin, Vec3 direction, uint32_t remap, + uint32_t triangle_id, uint32_t bary) { + const auto f = TypeF32(s), u = TypeU32(s), b = TypeBool(s); + const auto zero = ConstantF32Value(s, 0.0f); + const auto op = [&](spv::Op code, uint32_t x, uint32_t y) { + const auto value = Binary(s, code, f, x, y); + s.builder.AddAnnotation(spv::OpDecorate, value, spv::DecorationNoContraction); + return value; + }; + const auto cmp = [&](spv::Op code, uint32_t x, uint32_t y) { return Binary(s, code, b, x, y); }; + const auto either = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); }; + const auto both = [&](uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); }; + const auto abs = [&](uint32_t x) { return EmitGlsl(s, x); }; const auto y_largest = cmp(spv::OpFOrdLessThan, abs(direction[0]), abs(direction[1])); const auto z_largest = cmp(spv::OpFOrdLessThan, Select(s, f, y_largest, abs(direction[1]), abs(direction[0])), abs(direction[2])); @@ -112,8 +125,6 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary, cmp(spv::OpFOrdGreaterThan, projected[i][1], zero), right); missed = either(missed, both(cmp(spv::OpFOrdEqual, edge[i], zero), excluded)); } - const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag, - Binary(s, spv::OpIMul, u, kind, ConstantU32(s, 8))); const auto coordinate = [&](uint32_t shift) { const auto index = Binary(s, spv::OpBitwiseAnd, u, Binary(s, spv::OpShiftRightLogical, u, remap, ConstantU32(s, shift)), ConstantU32(s, 3)); @@ -124,11 +135,13 @@ uint32_t Triangle(EmitterState& s, uint32_t block, uint32_t kind, uint32_t bary, const std::array result { Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32(s, 0x7f800000), numerator)), Unary(s, spv::OpBitcast, u, Select(s, f, missed, ConstantF32Value(s, 1.0f), denominator)), - Select(s, u, bary, coordinate(0), Binary(s, spv::OpIAdd, u, flag, kind)), + Select(s, u, bary, coordinate(0), triangle_id), Select(s, u, bary, coordinate(2), Select(s, u, missed, ConstantU32(s, 0), ConstantU32(s, 1))) }; return Vector(s, TypeU32Vector(s, 4), result); } +namespace { + std::array Box(EmitterState& s, uint32_t first, uint32_t second, uint32_t half, uint32_t child) { const auto f = TypeF32(s); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 42825d9f8..82f9cd6c4 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -338,6 +338,8 @@ inline constexpr auto EmitDataConsume = EmitAppendConsume; uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst); void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBvh8Intersect(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBvh8Extract(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst); inline constexpr auto EmitImageQueryDimensions = EmitImage; inline constexpr auto EmitImageQueryLod = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 5a0a1418f..66af4fe9c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -120,6 +120,8 @@ struct EmitterState { uint32_t indirect_format_geometry_function = 0; std::map, uint32_t> indirect_buffer_load_functions; uint32_t bvh_intersect_function = 0; + uint32_t bvh8_result_type = 0; + uint32_t bvh8_intersect_function = 0; uint32_t bvh_stack_function = 0; uint32_t gds_variable = 0; uint32_t gds_length = 0; @@ -170,6 +172,7 @@ uint32_t TypeBoolVector(EmitterState& state, uint32_t components); uint32_t TypeU32(EmitterState& state); uint32_t TypeU64(EmitterState& state); uint32_t TypeU32Pair(EmitterState& state); +uint32_t TypeBvh8Result(EmitterState& state); uint32_t TypeI32(EmitterState& state); uint32_t TypeI32Pair(EmitterState& state); uint32_t TypeF32(EmitterState& state); @@ -543,6 +546,10 @@ void EmitProgram(EmitterState& state); void DefineGetBdaPointer(EmitterState& state); void DefineIndirectBufferLoads(ValueEmitContext& ctx); void DefineBvhIntersect(EmitterState& state); +void DefineTrinityBvhIntersect(EmitterState& state); +uint32_t EmitBvhTriangle(EmitterState& state, const std::array, 3>& positions, + std::array origin, std::array direction, + uint32_t remap, uint32_t triangle_id, uint32_t bary); void DefineTrinityFunctions(EmitterState& state); uint32_t GetBdaPointer(EmitterState& state, uint32_t address); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index f598365ee..bc04961bd 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -414,6 +414,7 @@ uint32_t TypeId(EmitterState& state, IR::Type type) { case IR::Type::U32: case IR::Type::F16: return TypeU32(state); case IR::Type::U64: return TypeU64(state); + case IR::Type::Bvh8Result: return TypeBvh8Result(state); case IR::Type::U32x2: return TypeU32Pair(state); case IR::Type::F32: return TypeF32(state); case IR::Type::F64: return TypeF64(state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp index a952830c6..1342d18ef 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinity.cpp @@ -184,6 +184,7 @@ void DefineBvhStack(EmitterState& s) { void DefineTrinityFunctions(EmitterState& s) { DefineBvhStack(s); + DefineTrinityBvhIntersect(s); } uint32_t EmitBvhStack(ValueEmitContext& ctx, const IR::Inst& inst) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinityBvh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinityBvh.cpp new file mode 100644 index 000000000..49175566c --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTrinityBvh.cpp @@ -0,0 +1,511 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h" + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +uint32_t TypeBvh8Result(EmitterState& s) { + if (s.bvh8_result_type == 0) { + const auto v = TypeU32Vector(s, 4); + s.bvh8_result_type = s.builder.Type(spv::OpTypeStruct, v, v, v, v, TypeU32(s)); + } + return s.bvh8_result_type; +} + +namespace { +using Vec3 = std::array; + +struct Bvh8Emitter { + EmitterState& s; + const uint32_t u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), w = TypeU64(s); + const uint32_t v4 = TypeU32Vector(s, 4), v3 = TypeF32Vector(s, 3); + uint32_t triangle_function = 0, boxes_function = 0, decompress_function = 0; + + uint32_t C(uint32_t x) { return ConstantU32(s, x); } + uint32_t And(uint32_t x, uint32_t y) { return Binary(s, spv::OpBitwiseAnd, u, x, y); } + uint32_t Or(uint32_t x, uint32_t y) { return Binary(s, spv::OpBitwiseOr, u, x, y); } + uint32_t Add(uint32_t x, uint32_t y) { return EmitAddU32(s, x, y); } + uint32_t Sub(uint32_t x, uint32_t y) { return Binary(s, spv::OpISub, u, x, y); } + uint32_t Shl(uint32_t x, uint32_t n) { return Binary(s, spv::OpShiftLeftLogical, u, x, C(n)); } + uint32_t Shr(uint32_t x, uint32_t n) { return Binary(s, spv::OpShiftRightLogical, u, x, C(n)); } + uint32_t Eq(uint32_t x, uint32_t y) { return Binary(s, spv::OpIEqual, b, x, y); } + uint32_t Both(uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalAnd, b, x, y); } + uint32_t Either(uint32_t x, uint32_t y) { return Binary(s, spv::OpLogicalOr, b, x, y); } + uint32_t Not(uint32_t x) { return Unary(s, spv::OpLogicalNot, b, x); } + uint32_t Bit(uint32_t x, uint32_t n) { return Not(Eq(And(x, C(1u << n)), C(0))); } + uint32_t Choose(uint32_t condition, uint32_t x, uint32_t y) { return Select(s, u, condition, x, y); } + uint32_t Float(uint32_t x) { return Unary(s, spv::OpBitcast, f, x); } + uint32_t Raw(uint32_t x) { return Unary(s, spv::OpBitcast, u, x); } + uint32_t Fop(spv::Op op, uint32_t x, uint32_t y) { return Binary(s, op, f, x, y); } + uint32_t Fcmp(spv::Op op, uint32_t x, uint32_t y) { return Binary(s, op, b, x, y); } + uint32_t Min(uint32_t x, uint32_t y) { return EmitGlsl(s, x, y); } + uint32_t Max(uint32_t x, uint32_t y) { return EmitGlsl(s, x, y); } + uint32_t Extract(uint32_t type, uint32_t value, uint32_t index) { + const auto id = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeExtract, type, id, value, index); + return id; + } + uint32_t Make(uint32_t type, std::span values) { + const auto id = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpCompositeConstruct, type, id, values); + return id; + } + uint32_t Make4(uint32_t x, uint32_t y, uint32_t z, uint32_t t) { + return Make(v4, std::array{x, y, z, t}); + } + Vec3 Unpack3(uint32_t value) { + return {Extract(f, value, 0), Extract(f, value, 1), Extract(f, value, 2)}; + } + uint32_t Word(uint32_t block, uint32_t index) { + const auto offset = Unary(s, spv::OpUConvert, w, Shl(index, 2)); + const auto address = Binary(s, spv::OpIAdd, w, block, offset); + const auto pointer = Unary(s, spv::OpConvertUToPtr, TypePhysicalU32Pointer(s), address); + const auto id = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpLoad, u, id, pointer, spv::MemoryAccessAlignedMask, 4u); + return id; + } + uint32_t Call(uint32_t type, uint32_t function, std::span args) { + const auto id = s.builder.AllocateId(); + std::vector call {spv::OpFunctionCall, type, id, function}; + call.insert(call.end(), args.begin(), args.end()); + s.builder.AddFunction(call); + return id; + } + std::vector Begin(uint32_t& function, const char* name, uint32_t result, + std::span types) { + function = s.builder.AllocateId(); + s.builder.AddName(function, name); + s.builder.AddFunction(spv::OpFunction, result, function, spv::FunctionControlMaskNone, + s.builder.Type(spv::OpTypeFunction, result, types)); + std::vector args; + for (auto type: types) { + args.push_back(s.builder.AllocateId()); + s.builder.AddFunction(spv::OpFunctionParameter, type, args.back()); + } + EmitLabel(s, s.builder.AllocateId()); + return args; + } + void End(uint32_t result) { + s.builder.AddFunction(spv::OpReturnValue, result); + s.builder.AddFunction(spv::OpFunctionEnd); + } + uint32_t Result(const std::array& dst, const Vec3& org, const Vec3& dir, + uint32_t instance) { + const std::array groups {Make4(dst[0], dst[1], dst[2], dst[3]), + Make4(dst[4], dst[5], dst[6], dst[7]), Make4(dst[8], dst[9], Raw(org[0]), Raw(org[1])), + Make4(Raw(org[2]), Raw(dir[0]), Raw(dir[1]), Raw(dir[2])), instance}; + return Make(TypeBvh8Result(s), groups); + } + uint32_t NormalizeFlags(uint32_t flags) { + flags = Choose(Bit(flags, 2), And(flags, C(~0x30u)), flags); + return Or(flags, Choose(Eq(And(flags, C(0x30)), C(0x30)), C(0x100), C(0))); + } + + void DefineTriangle() { + const auto a = Begin(triangle_function, "trinity_triangle", v4, + std::array{w, u, u, u, v3, v3}); + const auto block = a[0], slot = a[1], flags = a[2], descriptor = a[3]; + const auto origin = Unpack3(a[4]), direction = Unpack3(a[5]); + const auto flag_word = Word(block, C(15)); + const auto remap = Binary(s, spv::OpShiftRightLogical, u, flag_word, Shl(slot, 3)); + const auto bary = Bit(descriptor, 24); + const auto pointer_flags = Both(bary, Bit(descriptor, 23)); + const auto tri_flags = Choose(pointer_flags, remap, C(0)); + const auto procedural = Bit(tri_flags, 6); + const auto raw = EmitValueOrDefaultIfCondition(s, Not(procedural), v4, + Make4(C(0), C(0x3f800000), C(0x3f800000), C(0x3f800000)), [&] { + std::array positions; + const std::array vertices {slot, Choose(Eq(slot, C(0)), C(1), C(3)), C(2)}; + for (uint32_t vertex = 0; vertex < 3; ++vertex) + for (uint32_t axis = 0; axis < 3; ++axis) + positions[vertex][axis] = Float(Word(block, + Add(Binary(s, spv::OpIMul, u, vertices[vertex], C(3)), C(axis)))); + return EmitBvhTriangle(s, positions, origin, direction, remap, Add(flag_word, slot), bary); + }); + std::array result; + for (uint32_t i = 0; i < 4; ++i) result[i] = Extract(u, raw, i); + auto missed = Eq(result[0], C(0x7f800000)); + const auto flip = Both(Both(Not(procedural), Not(missed)), + Binary(s, spv::OpLogicalNotEqual, b, Bit(flags, 3), Bit(tri_flags, 5))); + for (uint32_t i = 0; i < 4; ++i) + result[i] = Binary(s, spv::OpBitwiseXor, u, result[i], + Choose(i < 2 ? flip : Both(flip, bary), C(0x80000000), C(0))); + const auto front = Fcmp(spv::OpFOrdGreaterThan, Float(result[1]), ConstantF32Value(s, 0.0f)); + const auto opaque = Both(Either(Bit(tri_flags, 7), Bit(flags, 0)), Not(Bit(flags, 1))); + missed = Either(missed, Either(Both(opaque, Bit(flags, 6)), Both(Not(opaque), Bit(flags, 7)))); + missed = Either(missed, Both(Both(Not(Bit(tri_flags, 4)), Not(procedural)), + Select(s, b, front, Bit(flags, 5), Bit(flags, 4)))); + missed = Either(missed, Select(s, b, procedural, Bit(flags, 9), Bit(flags, 8))); + const auto sign = [&](uint32_t value, uint32_t enabled) { + return Choose(pointer_flags, Or(And(value, C(0x7fffffff)), + Choose(enabled, C(0x80000000), C(0))), value); + }; + result[2] = sign(result[2], Either(Not(opaque), procedural)); + result[3] = sign(result[3], Not(opaque)); + const std::array miss {C(0x7f800000), C(0x3f800000), Choose(bary, C(0), C(~0u)), C(0)}; + for (uint32_t i = 0; i < 4; ++i) result[i] = Choose(missed, miss[i], result[i]); + End(Make(v4, result)); + } + + uint32_t Instance(uint32_t block, uint32_t flags, uint32_t mask, Vec3 org, Vec3 dir) { + const auto lo = Word(block, C(12)), hi = Word(block, C(13)); + const auto inst_flags = Shr(hi, 22); + auto merged = Or(And(Or(flags, inst_flags), C(0x3f0)), And(inst_flags, C(4))); + merged = Or(merged, And(Binary(s, spv::OpBitwiseXor, u, flags, inst_flags), C(8))); + merged = Or(merged, Choose(Either(Bit(flags, 0), Both(Bit(inst_flags, 0), Not(Bit(flags, 1)))), C(1), C(0))); + merged = Or(merged, Choose(Either(Bit(flags, 1), Both(Bit(inst_flags, 1), Not(Bit(flags, 0)))), C(2), C(0))); + const auto user = Word(block, C(15)); + std::array dst; + dst.fill(C(~0u)); + dst[2] = lo; + dst[3] = Or(And(hi, C(0x3fffff)), Shl(merged, 22)); + dst[6] = And(user, C(0xffffff)); + dst[7] = Choose(Eq(And(mask, Shr(user, 24)), C(0)), C(~0u), Word(block, C(14))); + Vec3 transformed_org, transformed_dir; + for (uint32_t axis = 0; axis < 3; ++axis) { + Vec3 row; + for (uint32_t i = 0; i < 3; ++i) row[i] = Float(Word(block, C(axis * 4 + i))); + const auto dot = [&](const Vec3& value) { + return Fop(spv::OpFAdd, Fop(spv::OpFAdd, Fop(spv::OpFMul, row[0], value[0]), + Fop(spv::OpFMul, row[1], value[1])), Fop(spv::OpFMul, row[2], value[2])); + }; + transformed_org[axis] = Fop(spv::OpFAdd, dot(org), Float(Word(block, C(axis * 4 + 3)))); + transformed_dir[axis] = dot(dir); + } + return Result(dst, transformed_org, transformed_dir, C(1)); + } + + void DefineDecompress(); + void DefineBoxes(); + uint32_t Boxes(uint32_t first, uint32_t second, uint32_t full, uint32_t compressed, + uint32_t flags, uint32_t descriptor, uint32_t extent, uint32_t org, uint32_t dir); + void DefineIntersect(); +}; + +void Bvh8Emitter::DefineDecompress() { + const auto a = Begin(decompress_function, "trinity_shared_exponent", f, std::array{u, u, b}); + const auto exponent = a[0], packed = a[1], minimum = a[2]; + const auto negative = Bit(packed, 17), mantissa = And(packed, C(0x1ffff)); + const auto outward = Eq(Choose(negative, C(1), C(0)), Choose(minimum, C(1), C(0))); + const auto nonzero = Not(Eq(mantissa, C(0))); + const auto highest = EmitGlsl(s, mantissa); + const auto shift = EmitGlsl(s, Sub(C(16), highest), exponent); + const auto decoded_exp = Choose(nonzero, Sub(exponent, shift), + Choose(Both(outward, Binary(s, spv::OpUGreaterThan, b, exponent, C(17))), Sub(exponent, C(17)), C(0))); + // Zero mantissas take a separate result, but their unused shifts must still be defined. + const auto amount = Add(C(7), Choose(nonzero, shift, C(0))); + const auto trail = Choose(Both(outward, Not(Eq(decoded_exp, C(255)))), + Sub(Binary(s, spv::OpShiftLeftLogical, u, C(1), amount), C(1)), C(0)); + const auto fraction = Choose(nonzero, + And(Or(Binary(s, spv::OpShiftLeftLogical, u, mantissa, amount), trail), C(0x7fffff)), + Choose(outward, C(0x7fffff), C(0))); + End(Float(Or(Or(fraction, Shl(decoded_exp, 23)), Choose(negative, C(0x80000000), C(0))))); +} + +void Bvh8Emitter::DefineBoxes() { + const auto result_type = s.builder.Type(spv::OpTypeStruct, v4, v4); + const auto a = Begin(boxes_function, "trinity_box_slot", result_type, + std::array{w, w, b, b, u, u, f, v3, v3}); + const auto first = a[0], second = a[1], full = a[2], compressed = a[3]; + const auto flags = a[4], descriptor = a[5], extent = a[6]; + const auto org = Unpack3(a[7]), dir = Unpack3(a[8]); + const auto zero = ConstantF32Value(s, 0.0f), infinity = ConstantF32(s, 0x7f800000); + Vec3 inverse; + for (uint32_t i = 0; i < 3; ++i) inverse[i] = Fop(spv::OpFDiv, ConstantF32Value(s, 1.0f), dir[i]); + const auto packed = [&](uint32_t offset, uint32_t width) { + const auto word = Shr(offset, 5), shift = And(offset, C(31)); + const auto lo = Binary(s, spv::OpShiftRightLogical, u, Word(first, word), shift); + const auto next = EmitGlsl(s, Add(word, C(1)), C(15)); + const auto hi = Binary(s, spv::OpShiftLeftLogical, u, Word(first, next), And(Sub(C(32), shift), C(31))); + return And(Or(lo, Choose(Eq(shift, C(0)), C(0), hi)), C((1u << width) - 1)); + }; + const auto format = Choose(compressed, C(0), Choose(full, C(1), C(2))); + const auto entry_label = s.current_label; + const auto header = s.builder.AllocateId(), body = s.builder.AllocateId(); + const auto cont = s.builder.AllocateId(), merge = s.builder.AllocateId(); + const auto initial_nodes = Make4(C(~0u), C(~0u), C(~0u), C(~0u)); + const auto initial_keys = Make4(C(0x7f800000), C(0x7f800000), C(0x7f800000), C(0x7f800000)); + const auto base = And(Word(first, C(0)), C(0x1fffffff)); + s.builder.AddFunction(spv::OpBranch, header); + EmitLabel(s, header); + const auto child = s.builder.AllocateId(), prefix = s.builder.AllocateId(); + const auto nodes = s.builder.AllocateId(), keys = s.builder.AllocateId(); + const auto child_phi = s.builder.AddDeferredPhi(u, child, 2); + const auto prefix_phi = s.builder.AddDeferredPhi(u, prefix, 2); + const auto nodes_phi = s.builder.AddDeferredPhi(v4, nodes, 2); + const auto keys_phi = s.builder.AddDeferredPhi(v4, keys, 2); + const auto more = Binary(s, spv::OpULessThan, b, child, C(4)); + s.builder.AddFunction(spv::OpLoopMerge, merge, cont, spv::LoopControlDontUnrollMask); + s.builder.AddFunction(spv::OpBranchConditional, more, body, merge); + EmitLabel(s, body); + const auto kind = packed(Add(C(29), Binary(s, spv::OpIMul, u, child, C(3))), 3); + const auto direct_node = Word(first, child); + const auto node = Choose(compressed, Or(Shl(prefix, 3), kind), direct_node); + const auto child_flags = Choose(compressed, + And(packed(Add(C(41), Shl(child, 2)), 4), Choose(Eq(child, C(3)), C(7), C(15))), C(0)); + const auto step = Choose(Eq(kind, C(2)), C(4), + Choose(Either(Either(Eq(kind, C(3)), Eq(kind, C(5))), Eq(kind, C(6))), C(2), C(1))); + const auto next_prefix = Add(prefix, step); + const auto bounds_type = s.builder.Type(spv::OpTypeStruct, v3, v3); + const auto box = EmitIndexSwitch(s, format, 3, bounds_type, [&](uint32_t encoding) { + std::array bounds; + if (encoding == 0) { + const auto yz = Word(first, C(2)); + const std::array exponents {Shr(Word(first, C(1)), 24), And(yz, C(255)), And(Shr(yz, 8), C(255))}; + for (uint32_t component = 0; component < 6; ++component) { + const auto offset = Add(C(80 + component * 18), Binary(s, spv::OpIMul, u, child, C(108))); + bounds[component] = Call(f, decompress_function, + std::array{exponents[component % 3], packed(offset, 18), ConstantBool(s, component < 3)}); + } + } else if (encoding == 1) { + for (uint32_t component = 0; component < 6; ++component) { + const auto word = Add(C(4 + component), Binary(s, spv::OpIMul, u, child, C(6))); + const auto block = Select(s, w, Binary(s, spv::OpULessThan, b, word, C(16)), first, second); + bounds[component] = Float(Word(block, And(word, C(15)))); + } + } else { + for (uint32_t pair = 0; pair < 3; ++pair) { + const auto word = Add(C(4 + pair), Binary(s, spv::OpIMul, u, child, C(3))); + const auto value = EmitGlsl(s, Word(first, word)); + bounds[pair * 2] = Extract(f, value, 0); + bounds[pair * 2 + 1] = Extract(f, value, 1); + } + } + return Make(bounds_type, std::array{Make(v3, std::span(bounds).first(3)), + Make(v3, std::span(bounds).last(3))}); + }); + const auto lo_bound = Unpack3(Extract(v3, box, 0)), hi_bound = Unpack3(Extract(v3, box, 1)); + Vec3 near, far; + auto valid = Not(Eq(And(child_flags, C(3)), C(3))); + for (uint32_t axis = 0; axis < 3; ++axis) { + valid = Both(valid, Fcmp(spv::OpFOrdLessThanEqual, lo_bound[axis], hi_bound[axis])); + const auto lo = Fop(spv::OpFMul, Fop(spv::OpFSub, lo_bound[axis], org[axis]), inverse[axis]); + const auto hi = Fop(spv::OpFMul, Fop(spv::OpFSub, hi_bound[axis], org[axis]), inverse[axis]); + const auto forward = Fcmp(spv::OpFOrdGreaterThanEqual, inverse[axis], zero); + near[axis] = Select(s, f, forward, lo, hi); + far[axis] = Select(s, f, forward, hi, lo); + } + const auto entry = Max(Max(near[0], near[1]), near[2]); + const auto exit = Min(Min(far[0], far[1]), far[2]); + const auto exit_bits = Raw(Select(s, f, Fcmp(spv::OpFOrdEqual, exit, zero), zero, exit)); + const auto grown = Float(EmitGlsl(s, + Add(exit_bits, And(Shr(descriptor, 23), C(255))), C(0x7f800000))); + valid = Both(valid, Both(Fcmp(spv::OpFOrdGreaterThanEqual, exit, zero), + Both(Fcmp(spv::OpFOrdLessThan, entry, extent), Fcmp(spv::OpFOrdLessThanEqual, entry, grown)))); + const auto opaque = Both(Either(Bit(child_flags, 0), Bit(flags, 0)), Not(Bit(flags, 1))); + const auto nonopaque = Both(Either(Bit(child_flags, 1), Bit(flags, 1)), Not(Bit(flags, 0))); + const auto culled = Either(Either(Both(opaque, Bit(flags, 6)), Both(nonopaque, Bit(flags, 7))), + Either(Both(Bit(child_flags, 2), Bit(flags, 8)), Both(Bit(child_flags, 3), Bit(flags, 9)))); + valid = Both(valid, Not(culled)); + const auto heuristic = And(Shr(descriptor, 21), C(3)); + auto key = Max(entry, zero); + key = Select(s, f, Eq(heuristic, C(1)), Unary(s, spv::OpFNegate, f, Fop(spv::OpFSub, exit, entry)), key); + key = Select(s, f, Eq(heuristic, C(2)), Fop(spv::OpFAdd, entry, exit), key); + const auto next_nodes = s.builder.AllocateId(), next_keys = s.builder.AllocateId(); + s.builder.AddFunction(spv::OpVectorInsertDynamic, v4, next_nodes, nodes, Choose(valid, node, C(~0u)), child); + s.builder.AddFunction(spv::OpVectorInsertDynamic, v4, next_keys, keys, Raw(Select(s, f, valid, key, infinity)), child); + s.builder.AddFunction(spv::OpBranch, cont); + EmitLabel(s, cont); + const auto next_child = Add(child, C(1)); + s.builder.AddFunction(spv::OpBranch, header); + s.builder.PatchDeferredPhi(child_phi, 0, C(0), entry_label); + s.builder.PatchDeferredPhi(child_phi, 1, next_child, cont); + s.builder.PatchDeferredPhi(prefix_phi, 0, base, entry_label); + s.builder.PatchDeferredPhi(prefix_phi, 1, next_prefix, cont); + s.builder.PatchDeferredPhi(nodes_phi, 0, initial_nodes, entry_label); + s.builder.PatchDeferredPhi(nodes_phi, 1, next_nodes, cont); + s.builder.PatchDeferredPhi(keys_phi, 0, initial_keys, entry_label); + s.builder.PatchDeferredPhi(keys_phi, 1, next_keys, cont); + EmitLabel(s, merge); + End(Make(result_type, std::array{nodes, keys})); +} + +uint32_t Bvh8Emitter::Boxes(uint32_t first, uint32_t second, uint32_t full, uint32_t compressed, + uint32_t flags, uint32_t descriptor, uint32_t extent, + uint32_t org, uint32_t dir) { + return Call(s.builder.Type(spv::OpTypeStruct, v4, v4), boxes_function, + std::array{first, second, full, compressed, flags, descriptor, extent, org, dir}); +} + +void Bvh8Emitter::DefineIntersect() { + const auto result_type = TypeBvh8Result(s); + const auto a = Begin(s.bvh8_intersect_function, "trinity_bvh_intersect", result_type, + std::array{v4, w, u, f, u, v3, v3}); + const auto descriptor = a[0], encoded_base = a[1], node = a[2], extent = a[3], mask = a[4]; + const auto org = Unpack3(a[5]), dir = Unpack3(a[6]); + std::array d; + for (uint32_t i = 0; i < 4; ++i) d[i] = Extract(u, descriptor, i); + const auto kind = And(node, C(7)); + const auto triangle = Binary(s, spv::OpULessThan, b, kind, C(2)); + const auto instance = Eq(kind, C(6)); + const auto full = Either(Eq(kind, C(2)), Eq(kind, C(5))); + const auto big = Either(Eq(kind, C(2)), Eq(kind, C(3))); + const auto compressed = Both(Not(full), Bit(d[3], 20)); + const auto flags = NormalizeFlags(Choose(Bit(d[3], 23), + Unary(s, spv::OpUConvert, u, Binary(s, spv::OpShiftRightLogical, w, encoded_base, C(54))), C(0))); + const auto base = Binary(s, spv::OpShiftLeftLogical, w, PackU64(s, d[0], And(d[1], C(255))), C(8)); + const auto lane_base = Binary(s, spv::OpShiftLeftLogical, w, + Binary(s, spv::OpBitwiseAnd, w, encoded_base, ConstantU64(s, (uint64_t{1} << 54) - 1)), C(3)); + const auto offset = Binary(s, spv::OpShiftLeftLogical, w, Unary(s, spv::OpUConvert, w, Shr(node, 3)), C(6)); + const auto address = Binary(s, spv::OpIAdd, w, base, Binary(s, spv::OpIAdd, w, lane_base, offset)); + const auto capacity = Binary(s, spv::OpIAdd, w, + PackU64(s, d[2], And(d[3], C(0x3ff))), ConstantU64(s, 1)); + const auto limit = Binary(s, spv::OpIAdd, w, base, + Binary(s, spv::OpShiftLeftLogical, w, capacity, C(6))); + auto valid = Both(Eq(Shr(d[3], 28), C(8)), + Eq(And(Unary(s, spv::OpUConvert, u, encoded_base), C(7)), C(0))); + valid = Both(valid, Both(Not(Eq(kind, C(7))), Either(Not(instance), Bit(d[3], 22)))); + valid = Both(valid, Not(Unary(s, spv::OpIsNan, b, extent))); + for (uint32_t i = 0; i < 3; ++i) { + valid = Both(valid, Not(Either(Unary(s, spv::OpIsNan, b, org[i]), Unary(s, spv::OpIsInf, b, org[i])))); + valid = Both(valid, Not(Unary(s, spv::OpIsNan, b, dir[i]))); + } + const auto opaque = Bit(flags, 0), nonopaque = Bit(flags, 1); + valid = Both(valid, Not(Either(Either(Both(opaque, Bit(flags, 6)), Both(nonopaque, Bit(flags, 7))), + Both(Bit(flags, 6), Bit(flags, 7))))); + const auto fetch_size = Select(s, w, full, ConstantU64(s, 128), ConstantU64(s, 64)); + const auto fetch = [&](uint32_t at, uint32_t enabled) { + const auto end = Binary(s, spv::OpIAdd, w, at, fetch_size); + const auto in_bounds = Both(Binary(s, spv::OpULessThanEqual, b, end, limit), + Binary(s, spv::OpULessThanEqual, b, end, ConstantU64(s, (uint64_t{1} << 48) - 1))); + return EmitValueOrDefaultIfCondition(s, Both(Both(valid, enabled), in_bounds), w, + ConstantU64(s, 0), [&] { return GetBdaPointer(s, at); }); + }; + const auto first = fetch(address, ConstantBool(s, true)); + const auto present = [&](uint32_t pointer) { return Not(Binary(s, spv::OpIEqual, b, pointer, ConstantU64(s, 0))); }; + const auto tail = [&](uint32_t at, uint32_t pointer) { + return EmitValueOrDefaultIfCondition(s, Both(full, present(pointer)), w, ConstantU64(s, 0), [&] { + return GetBdaPointer(s, Binary(s, spv::OpIAdd, w, at, ConstantU64(s, 64))); + }); + }; + const auto second = tail(address, first); + const auto available = Both(present(first), Either(Not(full), present(second))); + std::array invalid; + invalid.fill(C(~0u)); + for (uint32_t slot = 0; slot < 2; ++slot) { + invalid[slot * 4] = Choose(triangle, C(0x7f800000), C(~0u)); + invalid[slot * 4 + 1] = Choose(triangle, C(0x3f800000), C(~0u)); + invalid[slot * 4 + 3] = Choose(triangle, C(0), C(~0u)); + } + const auto default_result = Result(invalid, org, dir, C(0)); + const auto value = EmitValueOrDefaultIfCondition(s, Both(instance, available), result_type, default_result, [&] { + return Instance(first, flags, mask, org, dir); + }); + const auto result = EmitValueOrDefaultIfCondition(s, Both(Not(instance), valid), result_type, value, [&] { + const auto second_address = Binary(s, spv::OpIAdd, w, address, fetch_size); + const auto next_first = fetch(second_address, big); + const auto next_second = tail(second_address, next_first); + const auto next_available = Both(present(next_first), Either(Not(full), present(next_second))); + const auto triangles = EmitValueOrDefaultIfCondition(s, triangle, result_type, default_result, [&] { + auto dst = invalid; + for (uint32_t slot = 0; slot < 2; ++slot) { + const auto active = Both(available, slot == 0 ? ConstantBool(s, true) : Eq(kind, C(1))); + const auto absent = Both(available, Both(Eq(kind, C(0)), Bit(d[3], 24))); + const auto miss = Make4(C(0x7f800000), C(0x3f800000), + slot == 1 ? Choose(absent, C(0), C(~0u)) : C(~0u), C(0)); + const auto hit = EmitValueOrDefaultIfCondition(s, active, v4, miss, [&] { + return Call(v4, triangle_function, std::array{first, C(slot), flags, d[3], a[5], a[6]}); + }); + for (uint32_t i = 0; i < 4; ++i) dst[slot * 4 + i] = Extract(u, hit, i); + dst[8 + slot] = EmitValueOrDefaultIfCondition(s, active, u, C(~0u), [&] { return Word(first, C(14)); }); + } + return Result(dst, org, dir, C(0)); + }); + return EmitValueOrDefaultIfCondition(s, Not(triangle), result_type, triangles, [&] { + auto dst = invalid; + const auto slot_type = s.builder.Type(spv::OpTypeStruct, v4, v4); + const auto miss = Make(slot_type, std::array{Make4(C(~0u), C(~0u), C(~0u), C(~0u)), + Make4(C(0x7f800000), C(0x7f800000), C(0x7f800000), C(0x7f800000))}); + std::array keys; + for (uint32_t slot = 0; slot < 2; ++slot) { + const auto hit = EmitValueOrDefaultIfCondition(s, slot == 0 ? available : next_available, + slot_type, miss, [&] { + return Boxes(slot == 0 ? first : next_first, slot == 0 ? second : next_second, + full, compressed, flags, d[1], extent, a[5], a[6]); + }); + const auto indices = Extract(v4, hit, 0), depths = Extract(v4, hit, 1); + for (uint32_t i = 0; i < 4; ++i) { + dst[slot * 4 + i] = Extract(u, indices, i); + keys[slot * 4 + i] = Extract(u, depths, i); + } + } + const auto sort = Bit(d[1], 31), wide = Both(sort, Bit(d[3], 21)), tri_first = Bit(d[1], 20); + constexpr std::array, 19> network {{{0,1}, {2,3}, {0,2}, {1,3}, {1,2}, + {4,5}, {6,7}, {4,6}, {5,7}, {5,6}, {0,4}, {1,5}, {1,4}, {2,6}, {3,7}, {3,6}, {2,4}, {3,5}, {3,4}}}; + const auto unsorted = Make(slot_type, std::array{Make(v4, std::span(dst).first(4)), + Make(v4, std::span(dst).subspan(4, 4))}); + const auto sorted_result = EmitValueOrDefaultIfCondition(s, sort, slot_type, unsorted, [&] { + auto sorted_dst = dst; + for (size_t step = 0; step < network.size(); ++step) { + const auto [x, y] = network[step]; + const auto is_tri = [&](uint32_t index) { return Binary(s, spv::OpULessThan, b, And(index, C(7)), C(2)); }; + const auto xt = is_tri(sorted_dst[x]), yt = is_tri(sorted_dst[y]); + auto swap = Fcmp(spv::OpFOrdGreaterThan, Float(keys[x]), Float(keys[y])); + swap = Select(s, b, Both(tri_first, Binary(s, spv::OpLogicalNotEqual, b, xt, yt)), yt, swap); + if (step >= 10) swap = Both(wide, swap); + const auto key = keys[x], index = sorted_dst[x]; + const auto later = [&](uint32_t position) { + return std::ranges::any_of(std::span(network).subspan(step + 1), [=](auto pair) { + return pair.first == position || pair.second == position; + }); + }; + if (later(x)) keys[x] = Choose(swap, keys[y], key); + if (later(y)) keys[y] = Choose(swap, key, keys[y]); + sorted_dst[x] = Choose(swap, sorted_dst[y], index); + sorted_dst[y] = Choose(swap, index, sorted_dst[y]); + } + return Make(slot_type, std::array{Make(v4, std::span(sorted_dst).first(4)), + Make(v4, std::span(sorted_dst).subspan(4, 4))}); + }); + for (uint32_t slot = 0; slot < 2; ++slot) { + const auto indices = Extract(v4, sorted_result, slot); + for (uint32_t i = 0; i < 4; ++i) dst[slot * 4 + i] = Extract(u, indices, i); + } + return Result(dst, org, dir, C(0)); + }); + }); + End(result); +} + +} // namespace + +void DefineTrinityBvhIntersect(EmitterState& s) { + if (!s.program.info.bvh8) return; + Bvh8Emitter e {s}; + e.DefineDecompress(); + e.DefineTriangle(); + e.DefineBoxes(); + e.DefineIntersect(); +} + +uint32_t EmitBvh8Extract(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto index = inst.Arg(1).U32(); + if (index > 16) ctx.Fail(inst, "invalid BVH8 result component"); + const auto result = ctx.state.builder.AllocateId(); + if (index == 16) + ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), result, + ctx.Arg(inst, 0), 4u); + else + ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), result, + ctx.Arg(inst, 0), index / 4, index % 4); + return result; +} + +uint32_t EmitBvh8Intersect(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& s = ctx.state; + const auto* ray = ctx.ImageAddress(inst.Arg(1)); + if (ray == nullptr || ray->NumArgs() < 11 || s.bvh8_intersect_function == 0) + ctx.Fail(inst, "BVH8 intersection requires the complete ray and shared function"); + Bvh8Emitter e {s}; + const auto vector = [&](uint32_t first) { + return e.Make(e.v3, Vec3{e.Float(ctx.Arg(*ray, first)), e.Float(ctx.Arg(*ray, first + 1)), + e.Float(ctx.Arg(*ray, first + 2))}); + }; + const std::array args {ctx.Arg(inst, 0), PackU64(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1)), + ctx.Arg(*ray, 10), e.Float(ctx.Arg(*ray, 2)), ctx.Arg(*ray, 3), vector(4), vector(7)}; + const auto type = TypeBvh8Result(s); + return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), type, + s.builder.Constant(spv::OpConstantNull, type), [&] { + return e.Call(type, s.bvh8_intersect_function, args); + }); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 2996053a7..2133ea6bb 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -281,6 +281,7 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim case 0x09u: return Opcode::IMAGE_STORE_MIP; case 0x0eu: return Opcode::IMAGE_GET_RESINFO; case 0x60u: return Opcode::IMAGE_GET_LOD; + case 0xe5u: return Opcode::IMAGE_BVH8_INTERSECT_RAY; case 0xe6u: case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY; default: return Opcode::UNSUPPORTED; @@ -297,6 +298,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension, } switch (opcode) { + case 0xe5u: case 0xe6u: return 11u; case 0xe7u: return 12u; case 0x0eu: return 1u; @@ -427,6 +429,36 @@ void DecodeMimg(uint32_t pc, std::span code, uint32_t word_index "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf"); } + if (opcode == 0xe5u) { + inst.data_components = inst.data_dwords = 10; + if (a16 || d16 || !r128 || inst.dmask != 0xfu || + (nsa_dwords != 0u && nsa_dwords != 2u) || vdata > 246u || + (nsa_dwords == 0u && vaddr > 245u)) { + SetUnsupported(inst, Family::MIMG, opcode, "invalid BVH8 ray or destination encoding"); + } + if (nsa_dwords == 2u) { + const auto origin = inst.image_nsa_addr[3], direction = inst.image_nsa_addr[4]; + const auto node = inst.image_nsa_addr[5]; + if (origin > 253u || direction > 253u) + SetUnsupported(inst, Family::MIMG, opcode, "BVH8 ray registers exceed VGPR range"); + for (uint32_t i = 0; i < 3; ++i) { + inst.image_nsa_addr[3 + i] = origin + i; + inst.image_nsa_addr[6 + i] = direction + i; + } + inst.image_nsa_addr[9] = node; + } + const auto origin = nsa_dwords != 0 ? inst.image_nsa_addr[3] : vaddr + 4; + const auto direction = nsa_dwords != 0 ? inst.image_nsa_addr[6] : vaddr + 7; + bool overlapping = origin < direction + 3 && direction < origin + 3; + for (const auto output : {2u, 3u, 6u, 7u}) { + const auto reg = vdata + output; + overlapping |= (reg >= origin && reg < origin + 3) || + (reg >= direction && reg < direction + 3); + } + if (overlapping) + SetUnsupported(inst, Family::MIMG, opcode, "overlapping BVH8 instance outputs are unsupported"); + } + DecodeVectorGpr(vdata, inst.dst); DecodeVectorGpr(vaddr, inst.src0); DecodeScalarSource(srsrc * 4u, pc, inst.src1); diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 7fed56277..2b5754ed5 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -701,6 +701,7 @@ enum class Opcode { DS_WRITE_ADDTID_B32, DS_READ_ADDTID_B32, IMAGE_BVH_INTERSECT_RAY, + IMAGE_BVH8_INTERSECT_RAY, IMAGE_GET_RESINFO, IMAGE_GET_LOD, IMAGE_LOAD, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index b44980b98..3ac57a558 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -326,7 +326,8 @@ IR::Value Translator::MakeImageAddress(const Decoder::Instruction& inst, inst.image_sample_flags, inst.image_dimension, inst.image_address_components); EXIT_IF(count > components.size()); const auto nsa_components = - std::min(inst.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); + inst.opcode == Decoder::Opcode::IMAGE_BVH8_INTERSECT_RAY && inst.image_nsa_dwords != 0u + ? 10u : std::min(inst.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); for (uint32_t index = 0; index < count; index++) { if (index != 0u && index - 1u < nsa_components) { components[index] = @@ -948,6 +949,31 @@ void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) { {ReadU32(inst.src1), address, ir.GetExec()})); } +void Translator::IMAGE_BVH8(const Decoder::Instruction& inst) { + std::array previous; + for (uint32_t i = 0; i < previous.size(); ++i) + previous[i] = ReadU32(OffsetOperand(inst.dst, i)); + const auto result = ir.Emit(IR::ValueOpcode::Bvh8Intersect, + {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()}); + const auto get = [&](uint32_t i) { + return IR::U32(ir.Emit(IR::ValueOpcode::Bvh8Extract, {result, IR::Value(i)})); + }; + const auto instance = ir.INotEqual(get(16), IR::U32(IR::Value(0u))); + for (uint32_t i = 0; i < 10; ++i) { + auto value = get(i); + if (i == 0 || i == 1 || i == 4 || i == 5 || i == 8 || i == 9) + value = ir.Select(instance, previous[i], value); + WriteOperand(OffsetOperand(inst.dst, i), value); + } + for (uint32_t i = 0; i < 6; ++i) { + auto reg = inst.image_nsa_dwords != 0 ? inst.image_nsa_addr[3 + i] : inst.src0.reg + 4 + i; + auto target = inst.src0; + target.reg = reg; + // Non-instance results may alias the ray inputs. + WriteOperand(target, ir.Select(instance, get(10 + i), ReadU32(target))); + } +} + void Translator::EmitMemory(const Decoder::Instruction& inst) { switch (inst.opcode) { // Translated scalar loads have no cache of their own to invalidate. @@ -956,6 +982,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::BUFFER_GL1_INV: ir.Emit(IR::ValueOpcode::DataCacheInvalidate); return; + case Decoder::Opcode::IMAGE_BVH8_INTERSECT_RAY: return IMAGE_BVH8(inst); case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: { const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect, {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()}, diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h index 4f28a976f..e517a87a5 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Translator.h +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -105,6 +105,7 @@ private: void WriteSharedU32(uint32_t width, IR::U32 address, const std::array& values, const IR::MemoryInfo& memory, uint32_t pc); void DS_READ(const Decoder::Instruction& inst); + void IMAGE_BVH8(const Decoder::Instruction& inst); void DS_BVH_STACK(const Decoder::Instruction& inst); void DS_READ2(const Decoder::Instruction& inst); void DS_WRITE(const Decoder::Instruction& inst); diff --git a/src/graphics/shader/recompiler/ir/Serialize.h b/src/graphics/shader/recompiler/ir/Serialize.h index 0bbf4cf1e..95044f323 100644 --- a/src/graphics/shader/recompiler/ir/Serialize.h +++ b/src/graphics/shader/recompiler/ir/Serialize.h @@ -155,7 +155,7 @@ KYTY_IO_FIELDS(SampledResourcePair, a, b, c) KYTY_IO_FIELDS(StageInput, a, b, c, d, e) KYTY_IO_FIELDS(StageOutput, a, b, c, d) KYTY_IO_FIELDS(ShaderInfo, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, - A, B, C, D, E, F, G, H, I, J, K, L) + A, B, C, D, E, F, G, H, I, J, K, L, M) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows, a, b) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead::MaskRanks, a, b, c) KYTY_IO_FIELDS(DescriptorSource::IndirectDescriptor::SelectorRead, a, b, c, d, e, f, g, h, i, j, k, diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index c0b647c7d..4fce2eb53 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -543,6 +543,7 @@ struct ShaderInfo { bool uses_image_heap = false; bool uses_dispatch_threads = false; bool bvh = false; + bool bvh8 = false; bool bvh_stack = false; bool subgroup_ballot = false; bool subgroup_vote = false; diff --git a/src/graphics/shader/recompiler/ir/Type.cpp b/src/graphics/shader/recompiler/ir/Type.cpp index db8de73eb..12dbe17fc 100644 --- a/src/graphics/shader/recompiler/ir/Type.cpp +++ b/src/graphics/shader/recompiler/ir/Type.cpp @@ -31,6 +31,7 @@ std::string TypeName(Type type) { "SamplerResource", "ImageAddress", "F64", + "Bvh8Result", }; const auto bits = static_cast(type); if (bits == 0) { diff --git a/src/graphics/shader/recompiler/ir/Type.h b/src/graphics/shader/recompiler/ir/Type.h index dd97d53ae..cd53788a2 100644 --- a/src/graphics/shader/recompiler/ir/Type.h +++ b/src/graphics/shader/recompiler/ir/Type.h @@ -28,6 +28,7 @@ enum class Type : uint32_t { SamplerResource = 1u << 18u, ImageAddress = 1u << 19u, F64 = 1u << 20u, + Bvh8Result = 1u << 21u, }; constexpr Type operator|(Type lhs, Type rhs) { diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index e6c31a5ad..1e27642a4 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -44,6 +44,7 @@ constexpr Type AddressResource = Type::AddressResource; constexpr Type ImageResource = Type::ImageResource; constexpr Type SamplerResource = Type::SamplerResource; constexpr Type ImageAddress = Type::ImageAddress; +constexpr Type Bvh8Result = Type::Bvh8Result; constexpr std::array(ValueOpcode::Count)> MetaTable = {{ #define VALUE_OPCODE(name, ...) MakeMeta(#name __VA_OPT__(, ) __VA_ARGS__), diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 033862b63..0b750d687 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -318,6 +318,8 @@ VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) VALUE_OPCODE(BvhIntersect, U32x4, U32x4, ImageAddress, U1) +VALUE_OPCODE(Bvh8Intersect, Bvh8Result, U32x4, ImageAddress, U1) +VALUE_OPCODE(Bvh8Extract, U32, Bvh8Result, U32) VALUE_OPCODE(BvhStack, U32x2, U32, U32, U32x4, U32x4, U1) VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress) VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 2042ac987..6c4f16d2b 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -4182,7 +4182,7 @@ private: void Collect(Inst& inst) { const auto op = inst.GetOpcode(); - if (op == ValueOpcode::BvhIntersect) { + if (op == ValueOpcode::BvhIntersect || op == ValueOpcode::Bvh8Intersect) { m_info.uses_dma = true; return; } diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index c27f98d0d..4a54ae7c7 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -515,6 +515,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break; case ValueOpcode::DataCacheInvalidate: info.coherent_buffers = true; break; case ValueOpcode::BvhIntersect: info.bvh = true; break; + case ValueOpcode::Bvh8Intersect: info.bvh8 = true; break; case ValueOpcode::BvhStack: info.bvh_stack = true; break; case ValueOpcode::ConditionRef: info.subgroup_vote |= diff --git a/tests/ShaderRayTracingGpuTests.inc b/tests/ShaderRayTracingGpuTests.inc index a38dc7ad1..1976711e5 100644 --- a/tests/ShaderRayTracingGpuTests.inc +++ b/tests/ShaderRayTracingGpuTests.inc @@ -1,3 +1,306 @@ +TestCase TrinityBvhIntersections(bool compressed, bool bary, bool sorted, + bool wide_sort, u32 heuristic = 0, bool triangles_first = false, + u32 wave_size = 32, bool edges = false) { + TestCase test; + test.name = compressed ? (sorted ? "TrinityCompressedSorted" : "TrinityCompressedUnsorted") + : (sorted ? "TrinityFp32Sorted" : "TrinityFp32Unsorted"); + if (!bary) test.name = "TrinityTriangleIds"; + if (heuristic == 1) test.name = "TrinityLargestInterval"; + if (heuristic == 2) test.name = "TrinityClosestMidpoint"; + if (triangles_first) test.name = "TrinityTrianglesFirst"; + if (wave_size == 64) test.name = "TrinityBvhWave64"; + if (edges) test.name = compressed ? "TrinityCompressedEdges" : "TrinityFp32Edges"; + constexpr u32 NodeStart = 4096, OutputStart = 2048, GuestBase = 0x10000; + constexpr u32 Invalid = ~0u, Infinity = 0x7f800000, Flag = 0x8484, Shape = 0x12345678; + const auto bits = [](float x) { return std::bit_cast(x); }; + test.initial.resize(NodeStart + 256 * 16); + auto& data = test.initial; + const std::array vertices {-1,-1,2, 1,-1,2, 0,1,2, 2,1,2}; + for (u32 i = 0; i < vertices.size(); ++i) data[NodeStart + i] = bits(vertices[i]); + data[NodeStart + 14] = Shape; + data[NodeStart + 15] = Flag; + const std::array matrix {2,0,0,1, 0,3,0,-2, 0,0,4,3}; + for (u32 i = 0; i < matrix.size(); ++i) data[NodeStart + 16 + i] = bits(matrix[i]); + data[NodeStart + 28] = 0x1000; + data[NodeStart + 29] = 9u << 22; // Force opaque and flip facing. + data[NodeStart + 30] = 0x98; + data[NodeStart + 31] = 0x04123456; + const std::array near {5,1,3,7,4,2,6,8}; + const std::array length {1,8,3,5,7,2,4,6}; + const std::array children {128,138,171,190,256,266,299,318}; + const auto set_bits = [&](u32 base, u32 offset, u32 count, u32 value) { + for (u32 bit = 0; bit < count; ++bit) + data[base + (offset + bit) / 32] |= ((value >> bit) & 1) << ((offset + bit) % 32); + }; + for (u32 slot = 0; slot < 2; ++slot) { + const u32 base = NodeStart + (3 + slot * (compressed ? 1 : 2)) * 16; + if (compressed) { + set_bits(base, 0, 29, slot ? 32 : 16); + for (u32 i = 0; i < 4; ++i) set_bits(base, 29 + i * 3, 3, children[slot * 4 + i] & 7); + set_bits(base, 56, 8, 127); set_bits(base, 64, 8, 127); set_bits(base, 72, 8, 130); + } + for (u32 i = 0; i < 4; ++i) { + const u32 index = slot * 4 + i; + if (compressed) { + // Exact integer minima; positive maxima round outward within one quantization step. + const std::array planes {0x30000, 0x30000, u32(near[index]) * 8192, + 0x10000, 0x10000, u32(near[index] + length[index]) * 8192}; + for (u32 axis = 0; axis < 6; ++axis) set_bits(base, 80 + i * 108 + axis * 18, 18, planes[axis]); + } else { + data[base + i] = children[index]; + const std::array bounds {-1,-1,near[index], 1,1,near[index] + length[index]}; + for (u32 axis = 0; axis < 6; ++axis) data[base + 4 + i * 6 + axis] = bits(bounds[axis]); + } + } + } + if (compressed) { + std::copy_n(data.begin() + NodeStart + 3 * 16, 16, data.begin() + NodeStart + 10 * 16); + } else { + constexpr std::array lo {0x4500, 0x3c00, 0x4200, 0x4700}; + constexpr std::array hi {0x4600, 0x4880, 0x4600, 0x4a00}; + for (u32 i = 0; i < 4; ++i) { + data[NodeStart + 160 + i] = children[i]; + data[NodeStart + 164 + i * 3] = 0xbc00bc00; + data[NodeStart + 165 + i * 3] = 0x3c000000 | lo[i]; + data[NodeStart + 166 + i * 3] = (hi[i] << 16) | 0x3c00; + } + } + if (edges) { + // Slot zero ends exactly at the descriptor limit; slot one lies beyond it. + std::copy_n(data.begin() + NodeStart + 3 * 16, compressed ? 16 : 32, + data.begin() + NodeStart + (compressed ? 253 : 252) * 16); + for (u32 index : {12u, 13u}) + std::copy_n(data.begin() + NodeStart, 16, data.begin() + NodeStart + index * 16); + data[NodeStart + 12 * 16 + 15] = 0xc4c4; // Opaque procedural pair. + data[NodeStart + 13 * 16 + 15] = 0x9494; // Opaque, double-sided pair. + if (compressed) { + std::copy_n(data.begin() + NodeStart + 3 * 16, 16, data.begin() + NodeStart + 11 * 16); + set_bits(NodeStart + 11 * 16, 41, 4, 1); + set_bits(NodeStart + 11 * 16, 45, 4, 2); + set_bits(NodeStart + 11 * 16, 49, 4, 3); + } + } + std::vector> expected(wave_size); + for (u32 lane = 0; lane < wave_size; ++lane) { + const u32 which = lane % 16, input = lane * 16; + auto& out = expected[lane]; + out.fill(Invalid); + data[input + 2] = bits(20); data[input + 3] = 4; data[input + 9] = bits(1); + const auto miss = [&](u32 slot, bool invalid = false) { + out[slot * 4] = Infinity; out[slot * 4 + 1] = bits(1); + out[slot * 4 + 2] = !invalid && bary ? 0 : Invalid; out[slot * 4 + 3] = 0; + }; + const auto hit = [&](u32 slot, float u, float v) { + out[slot * 4] = bits(-8); out[slot * 4 + 1] = bits(-4); + out[slot * 4 + 2] = bary ? bits(u) : Flag + slot; + out[slot * 4 + 3] = bary ? bits(v) : 1; + }; + if (which <= 2 || which >= 9 && which <= 14) { + data[input + 10] = which == 1 || which == 2 ? 1 : 0; + miss(0); miss(1); + out[8] = Shape; + if (which == 1) { + data[input + 4] = bits(1); hit(1, 2, 1); out[9] = Shape; + } else { + hit(0, 1, 1); + if (which == 2) { miss(1); out[9] = Shape; } + } + if (which == 9 || which == 10 || which == 11) { + if (which == 9) data[input + 4] = 0x7fc00000; + if (which == 10) data[input + 10] = 0xfff8; + if (which == 11) data[input + 1] = 0x41u << 22; + miss(0, true); miss(1, true); out[8] = out[9] = Invalid; + } else if (which == 12) { + data[input + 1] = 0x100u << 22; miss(0); + } else if (which == 13) { + data[input + 1] = 8u << 22; out[0] = bits(8); out[1] = bits(4); + } else if (which == 14) { + data[input + 1] = 2u << 22; + if (bary) { out[2] = bits(-1); out[3] = bits(-1); } + } + } else if (which >= 3 && which <= 5) { + data[input + 10] = which == 3 ? (3 * 8 + (compressed ? 3 : 2)) + : which == 4 ? (3 * 8 + (compressed ? 4 : 5)) : (10 * 8 + 4); + std::array order {0,1,2,3,4,5,6,7}; + const auto compare = [&](u32 x, u32 y) { + const bool xt = (children[x] & 7) < 2, yt = (children[y] & 7) < 2; + if (triangles_first && xt != yt) return xt; + if (heuristic == 1) return length[x] > length[y]; + if (heuristic == 2) return 2 * near[x] + length[x] < 2 * near[y] + length[y]; + return near[x] < near[y]; + }; + if (sorted) { + if (wide_sort && which == 3) std::stable_sort(order.begin(), order.end(), compare); + else { + std::stable_sort(order.begin(), order.begin() + 4, compare); + std::stable_sort(order.begin() + 4, order.end(), compare); + } + } + for (u32 i = 0; i < (which == 3 ? 8u : 4u); ++i) out[i] = children[order[i]]; + } else if (which == 6 || which == 7 || which == 15) { + data[input + 10] = 14; + if (which == 15) { data[input] = 8; data[input + 10] = 6; } + data[input + 4] = bits(1); data[input + 5] = bits(2); data[input + 6] = bits(3); + if (which == 7) data[input + 3] = 2; + for (u32 i = 0; i < 10; ++i) out[i] = 0xabc00000 + i; + out[2] = 0x1000; out[3] = 9u << 22; out[6] = 0x123456; out[7] = which == 7 ? Invalid : 0x98; + } else data[input + 10] = 7; + if (edges) { + switch (which) { + case 0: + case 1: + case 2: + case 5: + if (which == 0) data[input] = 1; // Misaligned encoded base must not read across a block. + if (which == 1) data[input + 4] = Infinity; + if (which == 2) data[input + 2] = 0x7fc00000; + if (which == 5) { data[input + 1] = 0x2000; data[input + 10] = 0; } + miss(0, true); miss(1, true); out[8] = out[9] = Invalid; + break; + case 3: + data[input + 10] = compressed ? 253 * 8 + 3 : 252 * 8 + 2; + for (u32 i = 4; i < 8; ++i) out[i] = Invalid; + break; + case 4: + if (compressed) { + data[input + 10] = 11 * 8 + 4; data[input + 1] = 0x40u << 22; + out[0] = out[2] = Invalid; + } else { + data[input + 2] = bits(1); + for (u32 i = 0; i < 4; ++i) out[i] = Invalid; + } + break; + case 6: + data[input + 1] = 8u << 22; out[3] = 1u << 22; + break; + case 7: + data[input + 1] = 2u << 22; out[3] = 10u << 22; + break; + case 9: + case 10: + data[input + 10] = 12 * 8; + data[input + 4] = bits(100); + out[8] = Shape; miss(0); miss(1); + if (which == 9) { out[0] = bits(0); out[1] = bits(1); out[2] = bits(-1); out[3] = bits(1); } + else data[input + 1] = 0x200u << 22; + break; + case 11: + data[input + 1] = 0x10u << 22; miss(0); miss(1); out[8] = Shape; + break; + case 12: + case 13: + data[input + 1] = (which == 12 ? 0x14u : 0x10u) << 22; + if (which == 13) data[input + 10] = 13 * 8; + hit(0, 1, 1); + break; + case 14: + // The compressed +X bound rounds outward beyond exactly one. + if (compressed) { + data[input + 1] = 0; data[input + 4] = bits(1.0f + 0x1p-17f); + data[input + 10] = 3 * 8 + 4; + out.fill(Invalid); + for (u32 i = 0; i < 4; ++i) out[i] = children[i]; + } + break; + } + } + for (u32 i = 0; i < 6; ++i) out[10 + i] = data[input + 4 + i]; + if (which == 6 || which == 7 || which == 15) { + out[10] = bits(3); out[11] = bits(4); out[12] = bits(15); out[15] = bits(4); + } + if (lane % 32 >= 16) { + for (u32 i = 0; i < 10; ++i) out[i] = 0xabc00000 + i; + for (u32 i = 0; i < 6; ++i) out[10 + i] = data[input + 4 + i]; + } + } + test.expected = data; + for (u32 lane = 0; lane < wave_size; ++lane) + std::copy(expected[lane].begin(), expected[lane].end(), test.expected.begin() + OutputStart + lane * 16); + auto& code = test.code; + AppendSMovLiteral(&code, 8, GuestBase >> 8); + AppendSMovLiteral(&code, 9, (sorted ? 1u << 31 : 0) | (triangles_first ? 1u << 20 : 0) | (heuristic << 21)); + AppendSMovLiteral(&code, 10, edges ? 253 : 255); + AppendSMovLiteral(&code, 11, 0x80c00000 | (bary ? 1u << 24 : 0) | + (compressed ? 1u << 20 : 0) | (wide_sort ? 1u << 21 : 0)); + code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0)); + for (u32 offset : {0u,4u,8u}) + code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4), EncodeMubuf1(4 + offset, 0, 1)}); + for (u32 i = 0; i < 10; ++i) AppendVMovLiteral(&code, 32 + i, 0xabc00000 + i); + AppendSMovLiteral(&code, 126, 0x0000ffff); + if (wave_size == 64) AppendSMovLiteral(&code, 127, 0x0000ffff); + code.insert(code.end(), {EncodeMimg0(0xe5, 15, 2, false, 0, true), EncodeMimg1(32, 4, 2), + 0x08070605, 0x00000e0b}); + code.push_back(EncodeSop1(0x04, 126, 193)); + code.push_back(EncodeVop2(0x1a, 2, InlineU32(4), 0)); + for (u32 i = 0; i < 10; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 32 + i, 2, OutputStart + i); + for (u32 i = 0; i < 6; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 8 + i, 2, OutputStart + 10 + i); + AppendEnd(&code); + test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)}}; + test.compute_info.wave_size = wave_size; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + test.decoded_counts = {{"IMAGE_BVH8_INTERSECT_RAY", 1}}; + test.required_spirv = {"trinity_bvh_intersect"}; + return test; +} + +TestCase TrinityBvhAliases() { + auto test = TrinityBvhIntersections(false, true, false, false); + test.name = "TrinityBvhRegisterAliases"; + constexpr u32 OutputStart = 2048; + for (u32 lane = 0; lane < 32; ++lane) { + auto* out = test.expected.data() + OutputStart + lane * 16; + const auto* input = test.initial.data() + lane * 16; + const bool active = lane < 16, instance = active && (lane == 6 || lane == 7 || lane == 15); + if (!active) { + for (u32 i = 0; i < 6; ++i) out[4 + i] = input[i]; + } else if (instance) { + out[4] = input[0]; out[5] = input[1]; + out[8] = out[10]; out[9] = out[11]; + } else { + // A non-instance writes its shape IDs into registers formerly holding ray XY. + out[10] = out[8]; out[11] = out[9]; + } + } + auto& code = test.code; + code.clear(); + AppendSMovLiteral(&code, 8, 0x100); + AppendSMovLiteral(&code, 9, 0); + AppendSMovLiteral(&code, 10, 255); + AppendSMovLiteral(&code, 11, 0x81c00000); + code.push_back(EncodeVop2(0x1a, 50, InlineU32(6), 0)); + code.push_back(EncodeVop2(0x1a, 51, InlineU32(4), 0)); + for (u32 i = 0; i < 10; ++i) AppendVMovLiteral(&code, i, 0xabc00000 + i); + for (u32 offset : {0u, 4u, 8u}) + code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4), + EncodeMubuf1(4 + offset, 0, 50)}); + AppendSMovLiteral(&code, 126, 0x0000ffff); + code.insert(code.end(), {EncodeMimg0(0xe5, 15, 2, false, 0, true), EncodeMimg1(0, 4, 2), + 0x08070605, 0x00000e0b}); + code.push_back(EncodeSop1(0x04, 126, 193)); + for (u32 i = 0; i < 10; ++i) AppendStoreVgprAtLaneDwordOffset(&code, i, 51, OutputStart + i); + for (u32 i = 0; i < 6; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 8 + i, 51, OutputStart + 10 + i); + AppendEnd(&code); + return test; +} + +std::vector TrinityBvhCases() { + return {TrinityBvhIntersections(false, true, false, false), + TrinityBvhIntersections(false, true, true, true), + TrinityBvhIntersections(true, true, false, false), + TrinityBvhIntersections(true, true, true, false), + TrinityBvhIntersections(false, false, true, true), + TrinityBvhIntersections(false, true, true, true, 1), + TrinityBvhIntersections(false, true, true, true, 2), + TrinityBvhIntersections(true, true, true, true, 0, true), + TrinityBvhIntersections(true, true, true, true, 0, false, 64), + TrinityBvhIntersections(false, true, false, false, 0, false, 32, true), + TrinityBvhIntersections(true, true, false, false, 0, false, 32, true), + TrinityBvhAliases()}; +} + TestCase TrinityStack(u32 wave_size, u32 capacity, bool storage = false) { TestCase test; static constexpr const char* names[2][3] = { diff --git a/tests/ShaderRayTracingTests.inc b/tests/ShaderRayTracingTests.inc index 48eb09101..d9c924e32 100644 --- a/tests/ShaderRayTracingTests.inc +++ b/tests/ShaderRayTracingTests.inc @@ -38,7 +38,48 @@ void TestTrinityStackInstruction() { } } +void TestTrinityBvhInstruction() { + namespace D = ShaderRecompiler::Decoder; + const std::array code{0xf1949f05u, 0x00010000u, 0x0a030201u, 0x0000040du, + EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(0, 0, 0), + EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(4, 0, 0), + EncodeMubuf0(0x1d, 32, false), EncodeMubuf1(8, 0, 0), + EncodeMubuf0(0x1e, 40, false), EncodeMubuf1(10, 0, 0), + EncodeMubuf0(0x1d, 56, false), EncodeMubuf1(14, 0, 0), EncodeSopp(0x01)}; + D::Program decoded; + D::DecodeProgram(code, decoded); + const auto& inst = decoded.instructions.front(); + Check(inst.opcode == D::Opcode::IMAGE_BVH8_INTERSECT_RAY && inst.word_count == 4 && + inst.data_dwords == 10 && inst.image_address_components == 11 && + inst.image_nsa_addr[3] == 10 && inst.image_nsa_addr[5] == 12 && + inst.image_nsa_addr[6] == 13 && inst.image_nsa_addr[8] == 15 && + inst.image_nsa_addr[9] == 4 && inst.src1.reg == 4, + "Trinity BVH8 grouped ray registers were not decoded"); + const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute)); + Check(result.program.info.bvh8 && !result.program.info.bvh && + result.program.info.uses_dma && result.program.info.images.empty(), + "Trinity BVH8 did not use its separate raw-memory helper"); + Check(CountSourceOccurrences(result.ir_dump, " = Bvh8Intersect ") == 1, + "Trinity BVH8 result and ray writes duplicated the intersection"); + CheckSpirvBinaryValidates(result.spirv); + Check(SpirvInstructionOpcodeCount(result.spirv, 54u) == 6 && + SpirvInstructionOpcodeCount(result.spirv, 246u) == 1, + "Trinity BVH8 duplicated helpers or unrolled its four-child loop"); + CheckSpirvBinaryValidates(RecompileForTest(code, MakeCompileOptions(ShaderType::Pixel)).spirv); + for (const auto words : {std::array{0xf1949f05u, 0x40010000u, 0x0a030201u, 0x0000040du}, + std::array{0xf1949f05u, 0x0001f700u, 0x0a030201u, 0x0000040du}, + std::array{0xf1949f05u, 0x00010000u, 0xfe030201u, 0x0000040du}, + std::array{0xf1949f05u, 0x00010700u, 0x0a030201u, 0x0000040du}, + std::array{0xf1949f05u, 0x00010000u, 0x0a030201u, 0x0000040bu}}) { + const std::array invalid{words[0], words[1], words[2], words[3], EncodeSopp(0x01)}; + D::DecodeProgram(invalid, decoded); + Check(decoded.instructions[0].opcode == D::Opcode::UNSUPPORTED, + "Trinity BVH8 accepted truncated ray or destination registers"); + } +} + void TestRayTracingInstructions() { + TestTrinityBvhInstruction(); TestTrinityStackInstruction(); namespace D = ShaderRecompiler::Decoder; const std::array code{ diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 86f7d5537..c236e5108 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -46807,6 +46807,7 @@ std::vector MakeCases() { AddCase(ScratchIsPrivatePerInvocation); cases.push_back(FlatStackApertures(32)); cases.push_back(FlatStackApertures(64)); + for (auto& test : TrinityBvhCases()) cases.push_back(std::move(test)); for (const u32 wave : {32u, 64u}) { for (const u32 capacity : {8u, 16u, 31u}) cases.push_back(TrinityStack(wave, capacity)); cases.push_back(TrinityStack(wave, 8, true)); @@ -53350,6 +53351,7 @@ int main(int argc, char **argv) { #endif if (argc == 2 && std::strcmp(argv[1], "--ray-tracing-only") == 0) { VulkanHarness vulkan; + for (const auto& test : TrinityBvhCases()) RunCase(&vulkan, test); for (const u32 wave : {32u, 64u}) { for (const u32 capacity : {8u, 16u, 31u}) RunCase(&vulkan, TrinityStack(wave, capacity)); RunCase(&vulkan, TrinityStack(wave, 8, true));