diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index 0e63244da..cbfdb5b67 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -801,6 +801,8 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { prepared.buffers[index] = NativeStorageBuffer(m_context, prepared.buffer_sources[index], resource, buffer_offset); pack_memory_offset(index, buffer_offset); + prepared.shader_data[layout.MemorySizeDword() + index] = + static_cast(prepared.buffers[index].range); } if (layout.descriptor_counts[static_cast(BindingKind::FlattenedSrt)] != 0) { prepared.flattened_srt = NativeUpload(m_context, snapshot.flattened_srt); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index afe403111..fdf9a5404 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -112,6 +112,7 @@ struct EmitterState { uint32_t lane_half = 0; std::array storage_buffers {}; std::vector memory_byte_offsets; + std::vector memory_byte_sizes; uint32_t bda_pagetable_variable = 0; uint32_t fault_buffer_variable = 0; uint32_t bda_pointer_function = 0; @@ -414,7 +415,7 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR:: uint32_t SharedIndirectBufferResource(const EmitterState& state, uint32_t resource); -void EmitMemoryOffsets(EmitterState& state); +void EmitMemoryBindings(EmitterState& state); uint32_t LdsDwordCount(const EmitterState& state); void EnsureLdsStorage(EmitterState& state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 282277bcb..e8ad71d9a 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -154,9 +154,10 @@ uint32_t SharedIndirectBufferResource(const EmitterState& state, uint32_t resour return representative; } -void EmitMemoryOffsets(EmitterState& state) { +void EmitMemoryBindings(EmitterState& state) { const auto count = state.program.bindings.descriptor_counts[static_cast(IR::DescriptorBindingKind::Buffers)]; state.memory_byte_offsets.resize(count); + state.memory_byte_sizes.resize(count); std::vector dynamic_offsets(count); for (uint32_t resource = 0; resource < state.program.info.buffers.size(); ++resource) { if (SharedIndirectBufferResource(state, resource) == UINT32_MAX) continue; @@ -174,6 +175,8 @@ void EmitMemoryOffsets(EmitterState& state) { if (dynamic_offsets[i + lane]) continue; state.memory_byte_offsets[i + lane] = EmitNative( state, word, ConstantU32(state, lane * 8u), ConstantU32(state, 8u)); + state.memory_byte_sizes[i + lane] = + EmitShaderDataDwordLoad(state, state.program.bindings.MemorySizeDword() + i + lane); } } } @@ -257,22 +260,28 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, state.builder.RequireCapability(spv::CapabilityShaderNonUniform); state.builder.RequireCapability(spv::CapabilityStorageBufferArrayNonUniformIndexing); state.builder.AddAnnotation(spv::OpDecorate, access.object_pointer, spv::DecorationNonUniform); - const auto offsets_start = state.program.bindings.memory_offset_dword + - (state.program.bindings.UsesPushData() ? state.program.bindings.push_data_start_dword : 0u); + const auto& bindings = state.program.bindings; + const auto data_start = bindings.UsesPushData() ? bindings.push_data_start_dword : 0u; const auto word = EmitShaderDataDwordLoadAt(state, EmitAddU32( - state, ConstantU32(state, offsets_start), + state, ConstantU32(state, bindings.memory_offset_dword + data_start), Binary(state, spv::OpShiftRightLogical, TypeU32(state), array_index, ConstantU32(state, 2)))); const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), array_index, ConstantU32(state, 3)), ConstantU32(state, 3)); access.byte_offset = EmitNative( state, word, shift, ConstantU32(state, 8)); + access.length = EmitShaderDataDwordLoadAt(state, EmitAddU32( + state, ConstantU32(state, bindings.MemorySizeDword() + data_start), array_index)); } else { access.byte_offset = state.memory_byte_offsets[static_index]; + access.length = state.memory_byte_sizes[static_index]; + } + // storage_buffers[i] views the binding as 1 << i byte elements. + const auto shift = static_cast(&buffer - state.storage_buffers.data()); + if (shift != 0u) { + access.length = Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.length, + ConstantU32(state, shift)); } - access.length = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, - access.object_pointer, 0); return access; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index c664c603c..f70aead14 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -746,7 +746,7 @@ void EmitProgram(EmitterState& state) { state.lds_base_dwords = EmitBinaryU32(state, spv::OpIMul, index, ConstantU32(state, LdsDwordCount(state))); } - EmitMemoryOffsets(state); + EmitMemoryBindings(state); if (program.blocks.empty()) { EmitReturn(ctx); } else if (state.program.dispatcher_fallback) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 92c30c824..680919c35 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -495,10 +495,16 @@ struct Bindings { uint32_t dispatch_thread_dword = PushData::NoStart; uint32_t memory_offset_dword = 0; - [[nodiscard]] uint32_t ShaderDataDwords() const { + // Packed 8-bit binding offsets, then each binding's bound range in bytes: OpArrayLength + // reads 0 for ranges of 2 GiB or more on NVIDIA. + [[nodiscard]] uint32_t MemorySizeDword() const { return memory_offset_dword + (descriptor_counts[static_cast(DescriptorBindingKind::Buffers)] + 3u) / 4u; } + [[nodiscard]] uint32_t ShaderDataDwords() const { + return MemorySizeDword() + + descriptor_counts[static_cast(DescriptorBindingKind::Buffers)]; + } [[nodiscard]] bool UsesPushData() const { return push_data_start_dword != PushData::NoStart; } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index cdc8210b7..f79a53e22 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -17766,7 +17766,7 @@ public: } std::vector ReadBuffer(const char *shader_name, const Buffer &buffer, - size_t dword_count) { + size_t dword_count, size_t first_dword = 0) { if (!buffer.coherent) { vk::MappedMemoryRange range{}; range.sType = vk::StructureType::eMappedMemoryRange; @@ -17783,7 +17783,7 @@ public: m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data), "vkMapMemory"); std::vector ret(dword_count, 0); - std::memcpy(ret.data(), data, dword_count * sizeof(u32)); + std::memcpy(ret.data(), static_cast(data) + first_dword, dword_count * sizeof(u32)); m_device.unmapMemory(buffer.memory); return ret; } @@ -18013,6 +18013,7 @@ public: Require(test.name, "dispatch", info.range <= buffer.size, "storage buffer descriptor range exceeds backing buffer"); } + shader_data[layout.MemorySizeDword() + index] = static_cast(info.range); } vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; @@ -40606,6 +40607,50 @@ void CheckFiniteBufferSelection(VulkanHarness &vulkan) { } } +void CheckStorageBufferBindingOver2GiB(VulkanHarness &vulkan) { + // OpArrayLength reads 0 for ranges of 2 GiB or more on NVIDIA; bounds use the pushed binding size. + constexpr u32 bytes = 0x80000010u, end = bytes / 4u, last = end - 1u; + TestCase test; + test.name = "StorageBufferBindingOver2GiB"; + Require(test.name, "device limit", + vulkan.RuntimeContext().GetPhysicalDeviceProperties().limits.maxStorageBufferRange >= bytes, + "storage buffer range cannot cover 2 GiB"); + AppendVMovLiteral(&test.code, 4, last * 4u); + AppendVMovLiteral(&test.code, 6, end * 4u); + test.code.insert(test.code.end(), {EncodeMubuf0(0x0c), EncodeMubuf1(5, 0, 4), + EncodeMubuf0(0x0c), EncodeMubuf1(7, 0, 6), + EncodeMubuf0(0x1c), EncodeMubuf1(5, 0, 6), EncodeSopp(0x0c, 0)}); + AppendStoreVgpr(&test.code, 5, 4); + AppendStoreVgpr(&test.code, 7, 5); + AppendEnd(&test.code); + test.user_data = MakeStructuredStorageBufferData(0, bytes); + test.user_data[50] = 1u << 20u; + test.user_data[51] = 3u << 28u; + test.has_user_data = true; + test.forbidden_spirv = {"OpArrayLength"}; + const auto compiled = CompileCase(test, vulkan.SubgroupSize()); + for (const auto &value : compiled.resources.buffers) { + ShaderBufferResource descriptor{}; + std::copy_n(value.dwords.begin(), 4u, descriptor.fields); + test.storage_buffer_ranges.emplace_back(descriptor.Base48(), u32(descriptor.GetSize())); + } + VulkanHarness::Buffer buffer; + { + std::vector initial(end + 4u, 0u); + initial[5] = 0xdeadbeefu; + initial[last] = 0x44444444u; + initial[end] = 0xcafebabeu; + buffer = vulkan.CreateStorageBuffer(test.name, initial, initial.size()); + } + vulkan.Dispatch(test, compiled, buffer); + const auto loads = vulkan.ReadBuffer(test.name, buffer, 2u, 4u); + const auto canary = vulkan.ReadBuffer(test.name, buffer, 1u, end); + vulkan.DestroyBuffer(&buffer); + CompareWords(test, "a load at the last dword, a zero load and a dropped store at the 2 GiB range end", + {0x44444444u, 0u, 0xcafebabeu}, {loads[0], loads[1], canary[0]}); + std::printf("[compute] %-32s ok\n", test.name); +} + void CheckIndirectBufferStore(VulkanHarness &vulkan) { using namespace ShaderRecompiler::IR; TestCase test; @@ -40718,23 +40763,20 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { test.required_spirv = {"StorageBufferArrayNonUniformIndexing"}; test.forbidden_spirv.push_back("OpSwitch"); std::set nonuniform; - std::vector accessed_pointers; + std::vector stored_pointers; u32 prefix_extracts = 0; for (size_t offset = 5; offset < compiled.spirv.size();) { const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u); const auto opcode = static_cast(words[0] & 0xffffu); if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform) nonuniform.insert(words[1]); - if (opcode == spv::OpArrayLength) accessed_pointers.push_back(words[3]); - if (opcode == spv::OpStore) accessed_pointers.push_back(words[1]); + if (opcode == spv::OpStore) stored_pointers.push_back(words[1]); if (opcode == spv::OpBitFieldUExtract) ++prefix_extracts; offset += words.size(); } Require(test.name, "shared store and nonuniform pointers", - prefix_extracts == 1u && accessed_pointers.size() == 2u && - std::ranges::all_of(accessed_pointers, [&](u32 pointer) { - return nonuniform.contains(pointer); - }), + prefix_extracts == 1u && stored_pointers.size() == 1u && + nonuniform.contains(stored_pointers.front()), "indirect store retained static prefixes, was duplicated or lost a NonUniform pointer decoration"); } else if (single) { test.forbidden_spirv.push_back("OpSwitch"); @@ -51769,6 +51811,7 @@ int main(int argc, char **argv) { CheckGuardedWaterfallBufferRecords(vulkan); CheckGpuSelectedFormattedRecordStore(vulkan); CheckFiniteBufferSelection(vulkan); + CheckStorageBufferBindingOver2GiB(vulkan); CheckRuntimeBufferRecords(vulkan); CheckSharedIndirectBufferLoads(vulkan); CheckComputeThreadDimensions(vulkan); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 3e143d6f4..8054b7ee1 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -7464,7 +7464,7 @@ void TestNewShaderRecompilerMubufFormatTranslation() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() { +void TestNewShaderRecompilerFormattedStoreUsesRuntimeBindingSizeOnly() { const uint32_t shader[] = { EncodeMubuf0(0x04), EncodeMubuf1(0, 0, 1), // buffer_store_format_x @@ -7491,8 +7491,9 @@ void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() { CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); - Check((source.find("OpArrayLength") != std::string::npos), - "formatted store SPIR-V lacks runtime storage-buffer bounds check"); + Check(source.find("OpULessThan") != std::string::npos && + source.find("OpArrayLength") == std::string::npos, + "formatted store SPIR-V lacks a bounds check against its pushed binding size"); Check( !SpirvSourceHasInstructionUsing(source, "OpULessThan", "%uint_5"), "formatted store SPIR-V baked descriptor NumRecords into a store guard"); @@ -10638,19 +10639,18 @@ void TestNewShaderRecompilerBufferLoadsGuardedByExec() { const auto source = DisassembleSpirvBinary(result.spirv); const auto exec_branch = source.find("OpBranchConditional", 0); - const auto array_length = - source.find("OpArrayLength", 0); - const auto bounds_branch = source.find("OpBranchConditional", array_length); + const auto bounds = source.find("OpULessThan", 0); + const auto bounds_branch = source.find("OpBranchConditional", bounds); const auto element_access = source.find("OpAccessChain %_ptr_StorageBuffer_uint", 0); Check(exec_branch != std::string::npos, "buffer load SPIR-V lacks EXEC guard branch"); - Check(array_length != std::string::npos, - "buffer load SPIR-V lacks storage buffer array-length bounds check"); + Check(bounds != std::string::npos, + "buffer load SPIR-V lacks storage buffer bounds check"); Check(bounds_branch != std::string::npos, "buffer load SPIR-V lacks storage buffer bounds branch"); Check(element_access != std::string::npos, "buffer load SPIR-V lacks storage element access"); - Check(exec_branch < array_length, + Check(exec_branch < bounds, "buffer load bounds check was emitted outside EXEC guard"); Check(bounds_branch < element_access, "buffer load storage element pointer was formed before bounds guard"); @@ -10683,13 +10683,12 @@ void TestNewShaderRecompilerBufferAtomicsGuardedByBounds() { CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); - const auto array_length = - source.find("OpArrayLength", 0); - const auto bounds_branch = source.find("OpBranchConditional", array_length); + const auto bounds = source.find("OpULessThan", 0); + const auto bounds_branch = source.find("OpBranchConditional", bounds); const auto atomic = source.find(test.spirv, 0); const auto memory_barrier = source.find("OpMemoryBarrier", atomic); - Check(array_length != std::string::npos && + Check(bounds != std::string::npos && bounds_branch != std::string::npos && atomic != std::string::npos && memory_barrier != std::string::npos, @@ -10738,14 +10737,13 @@ void TestCapturedBufferAtomicsX2() { "64-bit buffer atomic storage view does not use eight-byte elements"); Check(CountSourceOccurrences(source, "Aliased") == 2u, "both storage-buffer views must declare that they alias"); - const auto array_length = - source.find("OpArrayLength", 0); - const auto bounds_branch = source.find("OpBranchConditional", array_length); + const auto bounds = source.find("OpULessThan", 0); + const auto bounds_branch = source.find("OpBranchConditional", bounds); const auto atomic = source.find(test.spirv_name, 0); const auto memory_barrier = source.find("OpMemoryBarrier", atomic); - Check(array_length != std::string::npos && + Check(bounds != std::string::npos && bounds_branch != std::string::npos && atomic != std::string::npos && bounds_branch < atomic, "64-bit buffer atomic was not guarded by storage-buffer bounds"); @@ -16577,7 +16575,6 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { .variables = 2, .loads = 9, .stores = 4, - .array_lengths = 2, .phis = 5, .labels = 34, .selection_merges = 10, @@ -16778,7 +16775,7 @@ int main() { TestNewShaderRecompilerNativeWideBufferIr(); TestNewShaderRecompilerScalarB64LaneTranslation(); TestNewShaderRecompilerMubufFormatTranslation(); - TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly(); + TestNewShaderRecompilerFormattedStoreUsesRuntimeBindingSizeOnly(); TestNewShaderRecompilerTypedBufferTranslation(); TestNewShaderRecompilerDsReadWrite2Translation(); TestNewShaderRecompilerDsWideAndAtomicTranslation();