shader: remove recompiler error-string propagation

This commit is contained in:
nmzik committed 2026-08-31 05:31:41 +04:00
1 parent 13533bd053
commit af8aecfdde
47 files changed
+2040 -3253

No files matched your search

@@ -895,9 +895,9 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti
EXIT_IF(!runtime); EXIT_IF(!runtime);
const auto& program = *runtime.program; const auto& program = *runtime.program;
const auto& snapshot = *runtime.resources; const auto& snapshot = *runtime.resources;
std::string error; if (!ShaderRecompiler::IR::ValidateResourceSpecialization(program, snapshot)) {
if (!ShaderRecompiler::IR::ValidateResourceSpecialization(program, snapshot, &error)) { EXIT("invalid native shader runtime snapshot: hash=0x%016" PRIx64 " stage=%u\n",
EXIT("invalid native shader runtime snapshot: %s\n", error.c_str()); program.shader_hash, static_cast<unsigned>(program.stage));
} }
PreparedBindings prepared; PreparedBindings prepared;
@@ -472,20 +472,14 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded
} // namespace } // namespace
bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options, CompileResult Recompile(std::span<const uint32_t> code, const CompileOptions& options) {
CompileResult& result, std::string* error) {
if (code.empty()) { if (code.empty()) {
if (error != nullptr) { EXIT("shader recompiler input is empty\n");
*error = "invalid shader recompiler input";
}
return false;
} }
if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex && if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
options.stage != ShaderType::Pixel) { options.stage != ShaderType::Pixel) {
if (error != nullptr) { EXIT("shader recompiler received unsupported stage %u\n",
*error = "shader recompiler supports compute, vertex, and pixel stages"; static_cast<unsigned>(options.stage));
}
return false;
} }
const auto compile_begin = std::chrono::steady_clock::now(); const auto compile_begin = std::chrono::steady_clock::now();
@@ -500,9 +494,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
static_cast<uint64_t>(code.size())); static_cast<uint64_t>(code.size()));
Decoder::Program decoded; Decoder::Program decoded;
if (!Decoder::DecodeProgram(code, decoded, error)) { Decoder::DecodeProgram(code, decoded);
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64 LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64
" elapsed_ms=%" PRIu64 "\n", " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash, GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -516,12 +508,9 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
} }
} }
CFG::Graph cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options), LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash); StageName(options.stage), options.shader_hash);
if (!CFG::BuildGraph(decoded, cfg, error)) { auto cfg = CFG::BuildGraph(decoded);
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64 LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
" loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", " loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash, GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -534,13 +523,12 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
dispatcher_reason = cfg.unsupported_reason; dispatcher_reason = cfg.unsupported_reason;
LogDispatcherFallback(options, cfg, "build", dispatcher_reason); LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
} else { } else {
std::string structure_error; const auto unstructured_cfg = cfg;
const auto unstructured_cfg = cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n", LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash); GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!CFG::Structurize(cfg, &structure_error)) { if (!CFG::Structurize(cfg)) {
dispatcher_fallback = true; dispatcher_fallback = true;
dispatcher_reason = structure_error; dispatcher_reason = cfg.unsupported_reason;
const auto failure_kind = cfg.failure_kind; const auto failure_kind = cfg.failure_kind;
const auto failure_block = cfg.failure_block; const auto failure_block = cfg.failure_block;
LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason); LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
@@ -548,7 +536,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
cfg.unsupported = true; cfg.unsupported = true;
cfg.failure_kind = failure_kind; cfg.failure_kind = failure_kind;
cfg.failure_block = failure_block; cfg.failure_block = failure_block;
cfg.unsupported_reason = structure_error; cfg.unsupported_reason = dispatcher_reason;
} else { } else {
LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options), LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
static_cast<uint64_t>(cfg.blocks.size())); static_cast<uint64_t>(cfg.blocks.size()));
@@ -560,14 +548,28 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms()); static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
} }
IR::Program ir;
const ShaderVertexInputInfo* vertex = nullptr; const ShaderVertexInputInfo* vertex = nullptr;
const ShaderPixelInputInfo* pixel = nullptr; const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr; const ShaderComputeInputInfo* compute = nullptr;
switch (options.stage) { switch (options.stage) {
case ShaderType::Vertex: vertex = options.input_info.vertex; break; case ShaderType::Vertex:
case ShaderType::Pixel: pixel = options.input_info.pixel; break; vertex = options.input_info.vertex;
case ShaderType::Compute: compute = options.input_info.compute; break; if (vertex == nullptr) {
EXIT("vertex shader recompilation has no input metadata\n");
}
break;
case ShaderType::Pixel:
pixel = options.input_info.pixel;
if (pixel == nullptr) {
EXIT("pixel shader recompilation has no input metadata\n");
}
break;
case ShaderType::Compute:
compute = options.input_info.compute;
if (compute == nullptr) {
EXIT("compute shader recompilation has no input metadata\n");
}
break;
default: break; default: break;
} }
EmbeddedFetchData embedded_fetch; EmbeddedFetchData embedded_fetch;
@@ -597,9 +599,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
}; };
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n", LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash); GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!Frontend::TranslateProgram(decoded, cfg, translate_options, ir, error)) { auto ir = Frontend::TranslateProgram(decoded, cfg, translate_options);
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64 LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
" elapsed_ms=%" PRIu64 "\n", " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash, GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -626,18 +626,9 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
lds_barriers.inserted_barriers); lds_barriers.inserted_barriers);
} }
} }
std::string srt_error; IR::BuildSrtPlan(ir);
if (!IR::BuildSrtPlan(ir, &srt_error)) {
LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str());
if (error != nullptr) {
*error = std::move(srt_error);
}
return false;
}
IR::EliminateDeadCode(ir.blocks); IR::EliminateDeadCode(ir.blocks);
if (!IR::TrackResources(ir, error)) { IR::TrackResources(ir);
return false;
}
IR::EliminateDeadCode(ir.blocks); IR::EliminateDeadCode(ir.blocks);
if (options.stage == ShaderType::Vertex) { if (options.stage == ShaderType::Vertex) {
ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr; ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
@@ -665,27 +656,20 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
runtime.read_memory = ReadZeroMemory; runtime.read_memory = ReadZeroMemory;
} }
runtime.userdata = options.read_memory_data; runtime.userdata = options.read_memory_data;
if (!IR::MaterializeResources(ir, runtime, resources, error)) { if (!IR::MaterializeResources(ir, runtime, resources)) {
return false; EXIT("shader resource materialization failed: stage=%s hash=0x%016" PRIx64 "\n",
StageName(options.stage), options.shader_hash);
} }
} }
if (!IR::SpecializeResources(ir, resources, error)) { IR::SpecializeResources(ir, resources);
return false;
}
IR::ShaderInfoOptions info_options; IR::ShaderInfoOptions info_options;
info_options.vertex = vertex; info_options.vertex = vertex;
info_options.pixel = pixel; info_options.pixel = pixel;
info_options.compute = compute; info_options.compute = compute;
if (!IR::CollectShaderInfo(ir, info_options, error)) { IR::CollectShaderInfo(ir, info_options);
return false; IR::AllocateBindings(ir, options.push_constant_offset);
} Spirv::AnalyzeProgramRequirements(ir);
if (!IR::AllocateBindings(ir, options.push_constant_offset, error)) {
return false;
}
if (!Spirv::AnalyzeProgramRequirements(ir, error)) {
return false;
}
std::string ir_dump; std::string ir_dump;
if (options.dump_ir) { if (options.dump_ir) {
ir_dump = MakeIrDump(cfg, ir); ir_dump = MakeIrDump(cfg, ir);
@@ -694,36 +678,22 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
} }
} }
std::vector<uint32_t> spirv;
std::string emit_error;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash); GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error)) { auto spirv = Spirv::EmitProgram(ir, resources, options.input_info);
LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str());
if (dispatcher_fallback && error != nullptr) {
*error = fmt::format("dispatcher fallback failed after {}: {}",
dispatcher_reason.c_str(), emit_error.c_str());
LOGF("%s dispatcher fallback emit failed: %s\n", GetDumpLabel(options), error->c_str());
} else if (error != nullptr) {
*error = emit_error;
}
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
" elapsed_ms=%" PRIu64 "\n", " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash, GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(spirv.size()), phase_ms()); static_cast<uint64_t>(spirv.size()), phase_ms());
CompileResult result;
result.spirv = std::move(spirv); result.spirv = std::move(spirv);
result.program = std::move(ir); result.program = std::move(ir);
result.resources = std::move(resources); result.resources = std::move(resources);
if (options.dump_ir) { if (options.dump_ir) {
result.decoded_dump = std::move(decoded_dump); result.decoded_dump = std::move(decoded_dump);
result.ir_dump = std::move(ir_dump); result.ir_dump = std::move(ir_dump);
} else {
result.decoded_dump.clear();
result.ir_dump.clear();
} }
return true; return result;
} }
} // namespace Libs::Graphics::ShaderRecompiler } // namespace Libs::Graphics::ShaderRecompiler
@@ -39,8 +39,8 @@ struct CompileResult {
IR::ResourceSnapshot resources; IR::ResourceSnapshot resources;
}; };
bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options, [[nodiscard]] CompileResult Recompile(std::span<const uint32_t> code,
CompileResult& result, std::string* error); const CompileOptions& options);
} // namespace Libs::Graphics::ShaderRecompiler } // namespace Libs::Graphics::ShaderRecompiler
@@ -1,19 +1,23 @@
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include <algorithm> #include <algorithm>
#include <array> #include <array>
namespace Libs::Graphics::ShaderRecompiler::Spirv { namespace Libs::Graphics::ShaderRecompiler::Spirv {
using ShaderError::Fail;
namespace { namespace {
bool ValidateNativeProgram(const IR::Program& program, std::string* error) { [[noreturn]] void Fail(const IR::Program& program, const char* reason) {
EXIT("SPIR-V validation failed: hash=0x%016" PRIx64 " stage=%u reason=%s\n",
program.shader_hash, static_cast<unsigned>(program.stage), reason);
std::abort();
}
void ValidateNativeProgram(const IR::Program& program) {
using Kind = IR::DescriptorBindingKind; using Kind = IR::DescriptorBindingKind;
constexpr auto KindCount = static_cast<size_t>(Kind::Count); constexpr auto KindCount = static_cast<size_t>(Kind::Count);
std::array<std::vector<uint32_t>, KindCount> expected; std::array<std::vector<uint32_t>, KindCount> expected;
@@ -36,13 +40,13 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
for (uint32_t i = 0; i < program.info.images.size(); i++) { for (uint32_t i = 0; i < program.info.images.size(); i++) {
const auto kind = IR::DescriptorBindingForImage(program.info.images[i]); const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
if (!kind.has_value()) { if (!kind.has_value()) {
return Fail(error, "native shader plan has an invalid image class"); Fail(program, "native shader plan has an invalid image class");
} }
present[static_cast<size_t>(*kind)] = true; present[static_cast<size_t>(*kind)] = true;
const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage; const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage;
const auto count = dynamic ? program.info.images[i].mip_count : 1u; const auto count = dynamic ? program.info.images[i].mip_count : 1u;
if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) { if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
return Fail(error, "native shader plan has an invalid image mip descriptor count"); Fail(program, "native shader plan has an invalid image mip descriptor count");
} }
expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(), expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(),
count, i); count, i);
@@ -58,11 +62,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
} }
const auto index = inst.Flags<IR::MemoryFlags>().index; const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.memory_info.size()) { if (index >= program.memory_info.size()) {
return Fail(error, "shared operation has invalid memory metadata"); Fail(program, "shared operation has invalid memory metadata");
} }
const auto kind = program.memory_info[index].kind; const auto kind = program.memory_info[index].kind;
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
return Fail(error, "shared operation has invalid resource kind"); Fail(program, "shared operation has invalid resource kind");
} }
uses_gds |= kind == IR::ResourceKind::Gds; uses_gds |= kind == IR::ResourceKind::Gds;
} }
@@ -91,13 +95,13 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
const auto kind = static_cast<size_t>(binding.kind); const auto kind = static_cast<size_t>(binding.kind);
if (kind >= KindCount || seen[kind] || !present[kind] || if (kind >= KindCount || seen[kind] || !present[kind] ||
binding.resources != expected[kind]) { binding.resources != expected[kind]) {
return Fail(error, "native descriptor groups do not match shader topology"); Fail(program, "native descriptor groups do not match shader topology");
} }
seen[kind] = true; seen[kind] = true;
} }
for (size_t i = 0; i < KindCount; i++) { for (size_t i = 0; i < KindCount; i++) {
if (present[i] != seen[i]) { if (present[i] != seen[i]) {
return Fail(error, "native shader plan is missing a required descriptor group"); Fail(program, "native shader plan is missing a required descriptor group");
} }
} }
const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)]; const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
@@ -114,7 +118,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
std::adjacent_find(program.bindings.user_data_registers.begin(), std::adjacent_find(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) != program.bindings.user_data_registers.end()) !=
program.bindings.user_data_registers.end()) { program.bindings.user_data_registers.end()) {
return Fail(error, "native user-data layout is inconsistent"); Fail(program, "native user-data layout is inconsistent");
} }
const auto planning_only_handle = [&](const IR::Inst& handle) { const auto planning_only_handle = [&](const IR::Inst& handle) {
@@ -139,7 +143,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
break; break;
} }
if (dense >= program.info.buffers.size()) { if (dense >= program.info.buffers.size()) {
return Fail(error, "typed buffer handle has an invalid dense resource"); Fail(program, "typed buffer handle has an invalid dense resource");
} }
break; break;
case IR::ValueOpcode::GetAddressResource: case IR::ValueOpcode::GetAddressResource:
@@ -147,29 +151,29 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
break; break;
} }
if (inst.NumArgs() != 2 || !program.info.uses_dma) { if (inst.NumArgs() != 2 || !program.info.uses_dma) {
return Fail(error, "typed address handle has invalid DMA metadata"); Fail(program, "typed address handle has invalid DMA metadata");
} }
break; break;
case IR::ValueOpcode::GetScratchResource: case IR::ValueOpcode::GetScratchResource:
if (inst.NumArgs() != 0 || program.scratch_dwords == 0) { if (inst.NumArgs() != 0 || program.scratch_dwords == 0) {
return Fail(error, "typed scratch handle has invalid shader metadata"); Fail(program, "typed scratch handle has invalid shader metadata");
} }
break; break;
case IR::ValueOpcode::GetImageResource: case IR::ValueOpcode::GetImageResource:
if (dense >= program.info.images.size()) { if (dense >= program.info.images.size()) {
return Fail(error, "typed image handle has an invalid dense resource"); Fail(program, "typed image handle has an invalid dense resource");
} }
break; break;
case IR::ValueOpcode::GetSamplerResource: case IR::ValueOpcode::GetSamplerResource:
if (dense >= program.info.samplers.size()) { if (dense >= program.info.samplers.size()) {
return Fail(error, "typed sampler handle has an invalid dense resource"); Fail(program, "typed sampler handle has an invalid dense resource");
} }
break; break;
case IR::ValueOpcode::ReadConst: { case IR::ValueOpcode::ReadConst: {
const auto slot = inst.Arg(1).Resolve(); const auto slot = inst.Arg(1).Resolve();
if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 || if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
slot.U32() >= program.srt_reads.size()) { slot.U32() >= program.srt_reads.size()) {
return Fail(error, "flattened SRT read has an invalid dense slot"); Fail(program, "flattened SRT read has an invalid dense slot");
} }
break; break;
} }
@@ -177,12 +181,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
} }
} }
} }
return true;
} }
} // namespace } // namespace
bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { void AnalyzeProgramRequirements(IR::Program& program) {
program.spirv_requirements.reset(); program.spirv_requirements.reset();
IR::SpirvRequirements requirements {}; IR::SpirvRequirements requirements {};
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; }; const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
@@ -192,30 +195,30 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
if (address_access != IR::AddressAccess::None) { if (address_access != IR::AddressAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index; const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, "address operation has invalid memory metadata"); Fail(program, "address operation has invalid memory metadata");
} }
if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) { if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
if (program.scratch_dwords == 0) { if (program.scratch_dwords == 0) {
return Fail(error, "scratch operation has no per-thread storage"); Fail(program, "scratch operation has no per-thread storage");
} }
requirements.function_scratch = true; requirements.function_scratch = true;
} else if (address_access == IR::AddressAccess::Write) { } else if (address_access == IR::AddressAccess::Write) {
return Fail(error, "writable FLAT/GLOBAL addresses require GPU ownership tracking"); Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
} }
} }
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index; const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, "buffer operation has invalid memory metadata"); Fail(program, "buffer operation has invalid memory metadata");
} }
const auto& memory = program.memory_info[memory_index]; const auto& memory = program.memory_info[memory_index];
if (memory.kind == IR::ResourceKind::Buffer) { if (memory.kind == IR::ResourceKind::Buffer) {
if (memory.resource >= program.info.buffers.size()) { if (memory.resource >= program.info.buffers.size()) {
return Fail(error, "buffer operation has invalid resource metadata"); Fail(program, "buffer operation has invalid resource metadata");
} }
if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
if (program.stage != ShaderType::Compute) { if (program.stage != ShaderType::Compute) {
return Fail(error, "buffer ADD_TID is only valid for compute shaders"); Fail(program, "buffer ADD_TID is only valid for compute shaders");
} }
requirements.subgroup_local_invocation_id = true; requirements.subgroup_local_invocation_id = true;
} }
@@ -225,11 +228,11 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
if (shared_access != IR::SharedAccess::None) { if (shared_access != IR::SharedAccess::None) {
const auto index = inst.Flags<IR::MemoryFlags>().index; const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.memory_info.size()) { if (index >= program.memory_info.size()) {
return Fail(error, "shared operation has invalid memory metadata"); Fail(program, "shared operation has invalid memory metadata");
} }
const auto kind = program.memory_info[index].kind; const auto kind = program.memory_info[index].kind;
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
return Fail(error, "shared operation has invalid resource kind"); Fail(program, "shared operation has invalid resource kind");
} }
if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) { if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
requirements.function_lds = true; requirements.function_lds = true;
@@ -283,7 +286,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
case IR::ValueOpcode::SetAttribute: { case IR::ValueOpcode::SetAttribute: {
const auto index = inst.Flags<IR::ExportFlags>().index; const auto index = inst.Flags<IR::ExportFlags>().index;
if (index >= program.export_info.size()) { if (index >= program.export_info.size()) {
return Fail(error, "attribute export has invalid metadata"); Fail(program, "attribute export has invalid metadata");
} }
if (program.stage == ShaderType::Pixel && if (program.stage == ShaderType::Pixel &&
program.export_info[index].vm) { program.export_info[index].vm) {
@@ -296,37 +299,30 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
} }
} }
program.spirv_requirements.emplace(requirements); program.spirv_requirements.emplace(requirements);
return true;
} }
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, std::vector<uint32_t> EmitProgram(const IR::Program& program,
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, const IR::ResourceSnapshot& resources,
std::string* error) { ShaderStageInputInfo input_info) {
using namespace Emitter; using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) { program.stage != ShaderType::Pixel) {
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders"); Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
return false;
} }
if (!program.srt_plan_complete || !program.resource_tracking_complete || if (!program.srt_plan_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete || !program.shader_info_complete || !program.binding_layout_complete ||
!program.spirv_requirements.has_value()) { !program.spirv_requirements.has_value()) {
SetError(error, "SPIR-V emitter requires a fully planned native shader program"); Fail(program, "SPIR-V emitter requires a fully planned native shader program");
return false;
} }
if (!IR::ValidateResourceSnapshot(program, resources, error)) { if (!IR::ValidateResourceSnapshot(program, resources)) {
return false; Fail(program, "resource snapshot is inconsistent with the native shader plan");
} }
if (!IR::ValidateResourceSpecialization(program, resources, error)) { if (!IR::ValidateResourceSpecialization(program, resources)) {
return false; Fail(program, "resource specialization is inconsistent with the native shader plan");
}
if (!ValidateNativeProgram(program, error)) {
return false;
}
if (!IR::ValidateProgram(program, true, error)) {
return false;
} }
ValidateNativeProgram(program);
IR::ValidateProgram(program, true);
EmitterState state(program, resources, input_info); EmitterState state(program, resources, input_info);
state.stage = program.stage; state.stage = program.stage;
state.wave_size = program.wave_size; state.wave_size = program.wave_size;
@@ -337,17 +333,13 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
AllocateInputVariables(state); AllocateInputVariables(state);
AllocateOutputVariables(state); AllocateOutputVariables(state);
DefineModule(state); DefineModule(state);
if (!EmitProgram(state, program, error)) { EmitProgram(state, program);
return false;
}
auto binary = state.builder.Build(); auto binary = state.builder.Build();
if (binary.empty()) { if (binary.empty()) {
SetError(error, "SPIR-V builder returned an empty module"); Fail(program, "SPIR-V builder returned an empty module");
return false;
} }
spirv = std::move(binary); return binary;
return true;
} }
} // namespace Libs::Graphics::ShaderRecompiler::Spirv } // namespace Libs::Graphics::ShaderRecompiler::Spirv
@@ -9,10 +9,11 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv { namespace Libs::Graphics::ShaderRecompiler::Spirv {
bool AnalyzeProgramRequirements(IR::Program& program, std::string* error); void AnalyzeProgramRequirements(IR::Program& program);
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, std::vector<uint32_t> EmitProgram(const IR::Program& program,
ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error); const IR::ResourceSnapshot& resources,
ShaderStageInputInfo input_info);
} // namespace Libs::Graphics::ShaderRecompiler::Spirv } // namespace Libs::Graphics::ShaderRecompiler::Spirv
@@ -37,12 +37,6 @@ bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
ShaderPixelParameterIsCustom(*state.input_info.pixel, attr); ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
} }
void SetError(std::string* error, const char* message) {
if (error != nullptr) {
*error = message;
}
}
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
uint32_t index) { uint32_t index) {
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) { return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
@@ -439,7 +439,8 @@ struct ValueEmitContext {
const IR::MemoryInfo& Memory(const IR::Inst& inst) const; const IR::MemoryInfo& Memory(const IR::Inst& inst) const;
const IR::ExportInfo& Export(const IR::Inst& inst) const; const IR::ExportInfo& Export(const IR::Inst& inst) const;
uint32_t Label(const IR::Block* block) const; uint32_t Label(const IR::Block* block) const;
void Fail(const IR::Inst& inst, const char* reason); [[noreturn]] void Fail(const char* reason) const;
[[noreturn]] void Fail(const IR::Inst& inst, const char* reason) const;
EmitterState& state; EmitterState& state;
const IR::Program& program; const IR::Program& program;
@@ -449,8 +450,6 @@ struct ValueEmitContext {
std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads; std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
const IR::Block* current_block = nullptr; const IR::Block* current_block = nullptr;
uint32_t scratch_u32_variable = 0; uint32_t scratch_u32_variable = 0;
bool failed = false;
std::string error;
}; };
enum class VertexInputScalarKind { Float, Sint, Uint }; enum class VertexInputScalarKind { Float, Sint, Uint };
@@ -613,8 +612,6 @@ uint32_t VertexParameterVectorOrScalarType(EmitterState& state, VertexInputScala
uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind, uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
uint32_t components); uint32_t components);
void SetError(std::string* error, const char* message);
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index); bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program); void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
@@ -864,7 +861,7 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst); bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error); void EmitProgram(EmitterState& state, const IR::Program& program);
void DefineGetBdaPointer(EmitterState& state); void DefineGetBdaPointer(EmitterState& state);
@@ -1,8 +1,9 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "common/assert.h"
#include <algorithm> #include <algorithm>
#include <bit> #include <bit>
#include <fmt/format.h>
#include <functional> #include <functional>
#include <optional> #include <optional>
#include <unordered_set> #include <unordered_set>
@@ -196,57 +197,50 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
} }
} }
bool EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) || if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
EmitValueImage(ctx, inst)) { EmitValueImage(ctx, inst)) {
return true; return;
} }
ctx.Fail(inst, "has no direct SPIR-V emitter"); ctx.Fail(inst, "has no direct SPIR-V emitter");
return false;
} }
bool EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured, void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
const IR::Inst& inst) { const IR::Inst& inst) {
if (inst.GetOpcode() == IR::ValueOpcode::Phi) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
const auto type = ctx.TypeId(inst.GetType()); const auto type = ctx.TypeId(inst.GetType());
if (type == 0 || inst.NumArgs() == 0) { if (type == 0 || inst.NumArgs() == 0) {
ctx.Fail(inst, "has no native SPIR-V representation"); ctx.Fail(inst, "has no native SPIR-V representation");
return false;
} }
for (size_t index = 0; index < inst.NumArgs(); index++) { for (size_t index = 0; index < inst.NumArgs(); index++) {
const auto* predecessor = inst.PhiBlock(index); const auto* predecessor = inst.PhiBlock(index);
if (predecessor == nullptr || !ctx.labels.contains(predecessor)) { if (predecessor == nullptr || !ctx.labels.contains(predecessor)) {
ctx.Fail(inst, "has a predecessor outside the structured function"); ctx.Fail(inst, "has a predecessor outside the structured function");
return false;
} }
} }
structured.deferred_phis.push_back( structured.deferred_phis.push_back(
{ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst}); {ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst});
return true; return;
} }
return EmitDirectInstruction(ctx, inst); EmitDirectInstruction(ctx, inst);
} }
bool EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher, void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
const IR::Inst& inst) { const IR::Inst& inst) {
if (inst.GetOpcode() == IR::ValueOpcode::Phi) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
const auto type = ctx.TypeId(inst.GetType()); const auto type = ctx.TypeId(inst.GetType());
if (type == 0) { if (type == 0) {
ctx.Fail(inst, "cannot be loaded by the dispatcher"); ctx.Fail(inst, "cannot be loaded by the dispatcher");
return false;
} }
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)}); {OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)});
return true; return;
}
if (!EmitDirectInstruction(ctx, inst)) {
return false;
} }
EmitDirectInstruction(ctx, inst);
if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) { if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))}); {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
} }
return true;
} }
template <typename EmitInstruction> template <typename EmitInstruction>
@@ -258,14 +252,11 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&&
if (inst.GetOpcode() == IR::ValueOpcode::Phi) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
if (emitted_non_phi) { if (emitted_non_phi) {
ctx.Fail(inst, "appears after a non-Phi instruction"); ctx.Fail(inst, "appears after a non-Phi instruction");
return;
} }
} else { } else {
emitted_non_phi = true; emitted_non_phi = true;
} }
if (!emit_instruction(inst)) { emit_instruction(inst);
return;
}
} }
} }
@@ -275,15 +266,10 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
const auto* predecessor = deferred.instruction->PhiBlock(index); const auto* predecessor = deferred.instruction->PhiBlock(index);
const auto found = structured.block_exit_labels.find(predecessor); const auto found = structured.block_exit_labels.find(predecessor);
if (found == structured.block_exit_labels.end()) { if (found == structured.block_exit_labels.end()) {
ctx.failed = true; ctx.Fail(*deferred.instruction, "has a predecessor that was not emitted");
ctx.error = "typed Phi predecessor was not emitted";
return;
} }
ctx.state.builder.PatchDeferredPhi( ctx.state.builder.PatchDeferredPhi(
deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second); deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second);
if (ctx.failed) {
return;
}
} }
} }
} }
@@ -291,20 +277,15 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
void EmitStructuredFunction(ValueEmitContext& ctx) { void EmitStructuredFunction(ValueEmitContext& ctx) {
StructuredFunctionState structured; StructuredFunctionState structured;
ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())}); ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) { for (size_t index = 0; index < ctx.program.blocks.size(); index++) {
const auto* block = ctx.program.blocks[index]; const auto* block = ctx.program.blocks[index];
EmitBlock(ctx, block, [&](const IR::Inst& inst) { EmitBlock(ctx, block, [&](const IR::Inst& inst) {
return EmitStructuredInstruction(ctx, structured, inst); EmitStructuredInstruction(ctx, structured, inst);
}); });
if (ctx.failed) {
break;
}
structured.block_exit_labels.emplace(block, ctx.state.current_label); structured.block_exit_labels.emplace(block, ctx.state.current_label);
EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]); EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]);
} }
if (!ctx.failed) { PatchStructuredPhis(ctx, structured);
PatchStructuredPhis(ctx, structured);
}
} }
void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) { void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
@@ -312,11 +293,8 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
const auto* entry = ctx.program.blocks.front(); const auto* entry = ctx.program.blocks.front();
state.builder.AddFunction({OpBranch, ctx.Label(entry)}); state.builder.AddFunction({OpBranch, ctx.Label(entry)});
EmitBlock(ctx, entry, [&](const IR::Inst& inst) { EmitBlock(ctx, entry, [&](const IR::Inst& inst) {
return EmitDispatcherInstruction(ctx, dispatcher, inst); EmitDispatcherInstruction(ctx, dispatcher, inst);
}); });
if (ctx.failed) {
return;
}
const auto initial_pc = const auto initial_pc =
EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front()); EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front());
const auto initial_parent = state.current_label; const auto initial_parent = state.current_label;
@@ -347,22 +325,16 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc, std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc,
ConstantU32(state, UINT32_MAX), dispatcher.select_label}; ConstantU32(state, UINT32_MAX), dispatcher.select_label};
for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) { for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) { EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) {
return EmitDispatcherInstruction(ctx, dispatcher, inst); EmitDispatcherInstruction(ctx, dispatcher, inst);
}); });
if (ctx.failed) {
break;
}
const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index], const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index],
ctx.program.block_info[index]); ctx.program.block_info[index]);
next_pc_words.push_back(selected); next_pc_words.push_back(selected);
next_pc_words.push_back(state.current_label); next_pc_words.push_back(state.current_label);
state.builder.AddFunction({OpBranch, dispatcher.after_switch_label}); state.builder.AddFunction({OpBranch, dispatcher.after_switch_label});
} }
if (ctx.failed) {
return;
}
EmitLabel(state, dispatcher.after_switch_label); EmitLabel(state, dispatcher.after_switch_label);
state.builder.AddFunction(next_pc_words); state.builder.AddFunction(next_pc_words);
state.builder.AddFunction({OpBranch, dispatcher.continue_label}); state.builder.AddFunction({OpBranch, dispatcher.continue_label});
@@ -408,9 +380,7 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
} }
const auto* inst = value.ResolveInstruction(); const auto* inst = value.ResolveInstruction();
if (inst == nullptr) { if (inst == nullptr) {
failed = true; Fail("direct SPIR-V emitter received a non-value argument");
error = "direct SPIR-V emitter received a non-value argument";
return ConstantU32(state, 0);
} }
if (dispatcher_spills != nullptr && current_block != nullptr && if (dispatcher_spills != nullptr && current_block != nullptr &&
inst->Parent() != current_block) { inst->Parent() != current_block) {
@@ -469,9 +439,7 @@ uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) { uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) {
const auto* inst = value.ResolveInstruction(); const auto* inst = value.ResolveInstruction();
if (inst == nullptr || inst->GetOpcode() != opcode) { if (inst == nullptr || inst->GetOpcode() != opcode) {
failed = true; Fail("typed resource handle has the wrong producer");
error = "typed resource handle has the wrong producer";
return 0;
} }
return inst->Flags<uint32_t>(); return inst->Flags<uint32_t>();
} }
@@ -479,9 +447,7 @@ uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode
const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) { const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) {
const auto* inst = value.ResolveInstruction(); const auto* inst = value.ResolveInstruction();
if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) { if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) {
failed = true; Fail("typed image address was not constructed by MakeImageAddress");
error = "typed image address was not constructed by MakeImageAddress";
return nullptr;
} }
return inst; return inst;
} }
@@ -498,12 +464,20 @@ uint32_t ValueEmitContext::Label(const IR::Block* block) const {
return labels.at(block); return labels.at(block);
} }
void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) { [[noreturn]] void ValueEmitContext::Fail(const char* reason) const {
failed = true; EXIT("SPIR-V emission failed: hash=0x%016" PRIx64 " stage=%u reason=%s\n",
error = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason); program.shader_hash, static_cast<unsigned>(program.stage), reason);
std::abort();
} }
bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error) { [[noreturn]] void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) const {
EXIT("SPIR-V emission failed: hash=0x%016" PRIx64 " stage=%u opcode=%s reason=%s\n",
program.shader_hash, static_cast<unsigned>(program.stage),
IR::ValueOpcodeName(inst.GetOpcode()), reason);
std::abort();
}
void EmitProgram(EmitterState& state, const IR::Program& program) {
ValueEmitContext ctx(state, program); ValueEmitContext ctx(state, program);
std::optional<DispatcherFunctionState> dispatcher; std::optional<DispatcherFunctionState> dispatcher;
if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) { if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
@@ -562,10 +536,6 @@ bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* e
dispatch.merge_label = state.builder.AllocateId(); dispatch.merge_label = state.builder.AllocateId();
ctx.dispatcher_spills = &dispatch.spills; ctx.dispatcher_spills = &dispatch.spills;
} }
if (ctx.failed) {
SetError(error, ctx.error.c_str());
return false;
}
DefineGetBdaPointer(state); DefineGetBdaPointer(state);
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
if (std::ranges::any_of(*block, [](const IR::Inst& inst) { if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
@@ -630,11 +600,6 @@ bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* e
EmitStructuredFunction(ctx); EmitStructuredFunction(ctx);
} }
state.builder.AddFunction({OpFunctionEnd}); state.builder.AddFunction({OpFunctionEnd});
if (ctx.failed) {
SetError(error, ctx.error.c_str());
return false;
}
return true;
} }
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
@@ -1,5 +1,7 @@
#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
#include "common/assert.h"
#include <algorithm> #include <algorithm>
#include <fmt/format.h> #include <fmt/format.h>
#include <iterator> #include <iterator>
@@ -24,19 +26,18 @@ struct SetpcTargetInfo {
std::vector<uint32_t> selector_target_pcs; std::vector<uint32_t> selector_target_pcs;
}; };
void SetError(std::string* error, const std::string& message) { void SetFailure(Graph& graph, FailureKind kind, uint32_t block_id, const std::string& message) {
if (error != nullptr) {
*error = message;
}
}
void SetFailure(Graph& graph, FailureKind kind, uint32_t block_id, const std::string& message,
std::string* error) {
graph.unsupported = true; graph.unsupported = true;
graph.failure_kind = kind; graph.failure_kind = kind;
graph.failure_block = block_id; graph.failure_block = block_id;
graph.unsupported_reason = message; graph.unsupported_reason = message;
SetError(error, message); }
[[noreturn]] void ExitBuildFailure(Graph& graph, FailureKind kind, uint32_t block_id,
const std::string& message) {
SetFailure(graph, kind, block_id, message);
EXIT("shader CFG build failed: %s", message.c_str());
std::abort();
} }
uint32_t InstructionEndPc(const Instruction& inst) { uint32_t InstructionEndPc(const Instruction& inst) {
@@ -1267,7 +1268,7 @@ bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t mer
return false; return false;
} }
bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) { bool CanonicalizeNaturalLoops(Graph& graph) {
const auto rewrite_budget = graph.blocks.size() * 2u + 16u; const auto rewrite_budget = graph.blocks.size() * 2u + 16u;
for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) { for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) {
bool changed = false; bool changed = false;
@@ -1322,19 +1323,18 @@ bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
} }
SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
"CFG loop canonicalization exceeded rewrite budget", error); "CFG loop canonicalization exceeded rewrite budget");
return false; return false;
} }
bool IsolateSemanticLoopHeaders(Graph& graph, std::string* error) { bool IsolateSemanticLoopHeaders(Graph& graph) {
const auto isolation_budget = graph.natural_loops.size() + 1u; const auto isolation_budget = graph.natural_loops.size() + 1u;
for (size_t isolation = 0; isolation < isolation_budget; isolation++) { for (size_t isolation = 0; isolation < isolation_budget; isolation++) {
const auto loop = std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(), const auto loop = std::find_if(
[&](const auto& value) { graph.natural_loops.begin(), graph.natural_loops.end(), [&](const auto& value) {
const auto* header = graph.FindBlock(value.header); const auto* header = graph.FindBlock(value.header);
return header != nullptr && return header != nullptr && header->inst_begin != header->inst_end;
header->inst_begin != header->inst_end; });
});
if (loop == graph.natural_loops.end()) { if (loop == graph.natural_loops.end()) {
return true; return true;
} }
@@ -1346,14 +1346,13 @@ bool IsolateSemanticLoopHeaders(Graph& graph, std::string* error) {
// displacing OpLoopMerge into a helper-created block. // displacing OpLoopMerge into a helper-created block.
if (!IsolateSemanticLoopHeader(graph, loop->header)) { if (!IsolateSemanticLoopHeader(graph, loop->header)) {
SetFailure(graph, FailureKind::StructuredControlFlow, loop->header, SetFailure(graph, FailureKind::StructuredControlFlow, loop->header,
fmt::format("failed to isolate semantic loop header {}", loop->header), fmt::format("failed to isolate semantic loop header {}", loop->header));
error);
return false; return false;
} }
} }
SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
"CFG semantic loop-header isolation exceeded rewrite budget", error); "CFG semantic loop-header isolation exceeded rewrite budget");
return false; return false;
} }
@@ -1453,7 +1452,7 @@ std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& head
return region; return region;
} }
bool SplitOneSelectionMerge(Graph& graph, std::string* error) { bool SplitOneSelectionMerge(Graph& graph) {
std::vector<uint32_t> loop_headers; std::vector<uint32_t> loop_headers;
loop_headers.reserve(graph.natural_loops.size()); loop_headers.reserve(graph.natural_loops.size());
for (const auto& loop: graph.natural_loops) { for (const auto& loop: graph.natural_loops) {
@@ -1501,8 +1500,7 @@ bool SplitOneSelectionMerge(Graph& graph, std::string* error) {
graph, FailureKind::StructuredControlFlow, block_id, graph, FailureKind::StructuredControlFlow, block_id,
fmt::format("selection header block {} has externally entered region block {}; " fmt::format("selection header block {} has externally entered region block {}; "
"semantic block cloning is disabled", "semantic block cloning is disabled",
block_id, *external), block_id, *external));
error);
return false; return false;
} }
const auto construct_blocks = DominatedBlocks(graph, block_id, merge); const auto construct_blocks = DominatedBlocks(graph, block_id, merge);
@@ -1514,11 +1512,11 @@ bool SplitOneSelectionMerge(Graph& graph, std::string* error) {
return false; return false;
} }
bool SplitSharedMergeBlocks(Graph& graph, std::string* error) { bool SplitSharedMergeBlocks(Graph& graph) {
const auto original_block_count = static_cast<uint32_t>(graph.blocks.size()); const auto original_block_count = static_cast<uint32_t>(graph.blocks.size());
const auto split_budget = std::max<uint32_t>(16u, original_block_count * 4u); const auto split_budget = std::max<uint32_t>(16u, original_block_count * 4u);
for (uint32_t splits = 0; splits < split_budget; splits++) { for (uint32_t splits = 0; splits < split_budget; splits++) {
if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, error)) { if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) {
return !graph.unsupported; return !graph.unsupported;
} }
RebuildPredecessors(graph); RebuildPredecessors(graph);
@@ -1528,8 +1526,7 @@ bool SplitSharedMergeBlocks(Graph& graph, std::string* error) {
fmt::format("CFG shared merge splitting exceeded budget: original_blocks={} " fmt::format("CFG shared merge splitting exceeded budget: original_blocks={} "
"current_blocks={} split_budget={}", "current_blocks={} split_budget={}",
original_block_count, static_cast<uint64_t>(graph.blocks.size()), original_block_count, static_cast<uint64_t>(graph.blocks.size()),
split_budget), split_budget));
error);
return false; return false;
} }
@@ -1890,12 +1887,11 @@ bool RouteSharedSelectionArm(Graph& graph, uint32_t route_variable) {
} // namespace } // namespace
bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error) { Graph BuildGraph(const Decoder::Program& program) {
graph = {}; Graph graph;
if (program.instructions.empty()) { if (program.instructions.empty()) {
SetError(error, "cannot build CFG for empty shader"); ExitBuildFailure(graph, FailureKind::InvalidLabel, UINT32_MAX,
return false; "cannot build CFG for empty shader");
} }
const auto first_pc = program.instructions.front().pc; const auto first_pc = program.instructions.front().pc;
@@ -1905,11 +1901,10 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
for (const auto& inst: program.instructions) { for (const auto& inst: program.instructions) {
instruction_pcs.insert(inst.pc); instruction_pcs.insert(inst.pc);
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetFailure(graph, FailureKind::UnsupportedInstruction, UINT32_MAX, ExitBuildFailure(
fmt::format("unsupported decoded instruction in CFG at pc 0x{:08x}: {}", graph, FailureKind::UnsupportedInstruction, UINT32_MAX,
inst.pc, Decoder::InstructionToString(inst).c_str()), fmt::format("unsupported decoded instruction in CFG at pc 0x{:08x}: {}", inst.pc,
error); Decoder::InstructionToString(inst).c_str()));
return false;
} }
} }
@@ -1923,11 +1918,9 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
const auto next_pc = InstructionEndPc(inst); const auto next_pc = InstructionEndPc(inst);
if (IsBranch(inst.opcode)) { if (IsBranch(inst.opcode)) {
if (!IsValidTarget(inst.branch_target, instruction_pcs, first_pc, end_pc)) { if (!IsValidTarget(inst.branch_target, instruction_pcs, first_pc, end_pc)) {
SetFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, ExitBuildFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}", inst.pc, fmt::format("branch at pc 0x{:08x} targets invalid pc 0x{:08x}",
inst.branch_target), inst.pc, inst.branch_target));
error);
return false;
} }
labels.insert(inst.branch_target); labels.insert(inst.branch_target);
if (next_pc <= end_pc) { if (next_pc <= end_pc) {
@@ -1936,21 +1929,19 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
} else if (inst.opcode == Opcode::S_SETPC_B64) { } else if (inst.opcode == Opcode::S_SETPC_B64) {
SetpcTargetInfo target_info; SetpcTargetInfo target_info;
if (!ResolveSetpcTargets(program, i, target_info)) { if (!ResolveSetpcTargets(program, i, target_info)) {
SetFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, ExitBuildFailure(
fmt::format("unsupported dynamic S_SETPC_B64 at pc 0x{:08x}", inst.pc), graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
error); fmt::format("unsupported dynamic S_SETPC_B64 at pc 0x{:08x}", inst.pc));
return false;
} }
const auto& target_pcs = target_info.indirect const auto& target_pcs = target_info.indirect
? target_info.target_pcs ? target_info.target_pcs
: std::vector<uint32_t> {target_info.target}; : std::vector<uint32_t> {target_info.target};
for (const auto target: target_pcs) { for (const auto target: target_pcs) {
if (!IsValidTarget(target, instruction_pcs, first_pc, end_pc)) { if (!IsValidTarget(target, instruction_pcs, first_pc, end_pc)) {
SetFailure(graph, FailureKind::InvalidBranchTarget, UINT32_MAX, ExitBuildFailure(
fmt::format("S_SETPC_B64 at pc 0x{:08x} targets invalid pc 0x{:08x}", graph, FailureKind::InvalidBranchTarget, UINT32_MAX,
inst.pc, target), fmt::format("S_SETPC_B64 at pc 0x{:08x} targets invalid pc 0x{:08x}",
error); inst.pc, target));
return false;
} }
labels.insert(target); labels.insert(target);
} }
@@ -1971,10 +1962,9 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
continue; continue;
} }
if (start != end_pc && !instruction_pcs.contains(start)) { if (start != end_pc && !instruction_pcs.contains(start)) {
SetFailure(graph, FailureKind::InvalidLabel, UINT32_MAX, ExitBuildFailure(
fmt::format("CFG label does not start on an instruction: 0x{:08x}", start), graph, FailureKind::InvalidLabel, UINT32_MAX,
error); fmt::format("CFG label does not start on an instruction: 0x{:08x}", start));
return false;
} }
BasicBlock block; BasicBlock block;
@@ -2046,11 +2036,10 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
block.terminator.true_block = pc_to_block.at(last.branch_target); block.terminator.true_block = pc_to_block.at(last.branch_target);
const auto fallthrough = pc_to_block.find(next_pc); const auto fallthrough = pc_to_block.find(next_pc);
if (fallthrough == pc_to_block.end()) { if (fallthrough == pc_to_block.end()) {
SetFailure(graph, FailureKind::MissingFallthrough, block.id, ExitBuildFailure(
fmt::format("conditional branch at pc 0x{:08x} has no fallthrough block", graph, FailureKind::MissingFallthrough, block.id,
last.pc), fmt::format("conditional branch at pc 0x{:08x} has no fallthrough block",
error); last.pc));
return false;
} }
block.terminator.false_block = fallthrough->second; block.terminator.false_block = fallthrough->second;
} else { } else {
@@ -2131,27 +2120,26 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
graph.unsupported = false; graph.unsupported = false;
} }
return true; return graph;
} }
namespace { namespace {
bool StructurizeImpl(Graph& graph, std::string* error) { bool StructurizeImpl(Graph& graph) {
if (graph.unsupported || graph.irreducible) { if (graph.unsupported || graph.irreducible) {
if (graph.unsupported_reason.empty()) { if (graph.unsupported_reason.empty()) {
graph.unsupported_reason = "unsupported CFG"; graph.unsupported_reason = "unsupported CFG";
} }
SetError(error, graph.unsupported_reason);
return false; return false;
} }
if (!CanonicalizeNaturalLoops(graph, error)) { if (!CanonicalizeNaturalLoops(graph)) {
return false; return false;
} }
if (!SplitSharedMergeBlocks(graph, error)) { if (!SplitSharedMergeBlocks(graph)) {
return false; return false;
} }
if (!IsolateSemanticLoopHeaders(graph, error)) { if (!IsolateSemanticLoopHeaders(graph)) {
return false; return false;
} }
ClearStructuredTerminators(graph); ClearStructuredTerminators(graph);
@@ -2164,8 +2152,7 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
graph, FailureKind::StructuredControlFlow, header, graph, FailureKind::StructuredControlFlow, header,
fmt::format( fmt::format(
"duplicate structured merge block {} for header {} (already used by header {})", "duplicate structured merge block {} for header {} (already used by header {})",
merge, header, it->second), merge, header, it->second));
error);
return false; return false;
} }
return true; return true;
@@ -2176,16 +2163,14 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
if (header == nullptr || loop.merge == UINT32_MAX || loop.continue_block == UINT32_MAX) { if (header == nullptr || loop.merge == UINT32_MAX || loop.continue_block == UINT32_MAX) {
SetFailure( SetFailure(
graph, FailureKind::StructuredControlFlow, loop.header, graph, FailureKind::StructuredControlFlow, loop.header,
fmt::format("loop at block {} has no structured merge/continue", loop.header), fmt::format("loop at block {} has no structured merge/continue", loop.header));
error);
return false; return false;
} }
if (header->inst_begin != header->inst_end || if (header->inst_begin != header->inst_end ||
header->terminator.kind != TerminatorKind::Branch) { header->terminator.kind != TerminatorKind::Branch) {
SetFailure(graph, FailureKind::StructuredControlFlow, loop.header, SetFailure(
fmt::format("loop header {} is not a dedicated empty control block", graph, FailureKind::StructuredControlFlow, loop.header,
loop.header), fmt::format("loop header {} is not a dedicated empty control block", loop.header));
error);
return false; return false;
} }
if (!reserve_merge_block(loop.header, loop.merge)) { if (!reserve_merge_block(loop.header, loop.merge)) {
@@ -2208,8 +2193,7 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
const auto merge = FindSelectionMerge(graph, block); const auto merge = FindSelectionMerge(graph, block);
if (merge == UINT32_MAX) { if (merge == UINT32_MAX) {
SetFailure(graph, FailureKind::StructuredControlFlow, block.id, SetFailure(graph, FailureKind::StructuredControlFlow, block.id,
fmt::format("conditional block {} has no structured merge", block.id), fmt::format("conditional block {} has no structured merge", block.id));
error);
return false; return false;
} }
@@ -2224,16 +2208,15 @@ bool StructurizeImpl(Graph& graph, std::string* error) {
} // namespace } // namespace
bool Structurize(Graph& graph, std::string* error) { bool Structurize(Graph& graph) {
Graph original = graph; Graph original = graph;
if (StructurizeImpl(graph, error)) { if (StructurizeImpl(graph)) {
return true; return true;
} }
Graph failed_graph = std::move(graph); Graph failed_graph = std::move(graph);
std::string failed_error = error != nullptr ? *error : std::string {}; graph = std::move(original);
graph = std::move(original); const auto route_budget = static_cast<uint32_t>(graph.blocks.size());
const auto route_budget = static_cast<uint32_t>(graph.blocks.size());
// Apply one route at a time and retry. Eagerly routing every matching diamond can // Apply one route at a time and retry. Eagerly routing every matching diamond can
// rewrite unrelated selections that were already structurally valid. // rewrite unrelated selections that were already structurally valid.
for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) { for (uint32_t route_variable = 0; route_variable < route_budget; route_variable++) {
@@ -2241,18 +2224,12 @@ bool Structurize(Graph& graph, std::string* error) {
break; break;
} }
Graph routed = graph; Graph routed = graph;
if (error != nullptr) { if (StructurizeImpl(routed)) {
error->clear();
}
if (StructurizeImpl(routed, error)) {
graph = std::move(routed); graph = std::move(routed);
return true; return true;
} }
} }
graph = std::move(failed_graph); graph = std::move(failed_graph);
if (error != nullptr) {
*error = std::move(failed_error);
}
return false; return false;
} }
@@ -108,8 +108,8 @@ struct Graph {
uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const; uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
}; };
bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error); Graph BuildGraph(const Decoder::Program& program);
bool Structurize(Graph& graph, std::string* error); bool Structurize(Graph& graph);
std::string BranchConditionToString(BranchCondition condition); std::string BranchConditionToString(BranchCondition condition);
std::string FailureKindToString(FailureKind kind); std::string FailureKindToString(FailureKind kind);
std::string GraphToString(const Graph& graph); std::string GraphToString(const Graph& graph);
@@ -357,8 +357,8 @@ void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
DecodeVectorGpr(vdata, inst.dst); DecodeVectorGpr(vdata, inst.dst);
DecodeVectorGpr(vaddr, inst.src0); DecodeVectorGpr(vaddr, inst.src0);
DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr); DecodeScalarSource(srsrc * 4u, pc, inst.src1);
DecodeScalarSource(ssamp * 4u, pc, inst.src2, nullptr); DecodeScalarSource(ssamp * 4u, pc, inst.src2);
inst.src_count = 3; inst.src_count = 3;
} }
@@ -241,11 +241,11 @@ void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented"); MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
} }
DecodeScalarDestination(sdst, pc, inst.dst, nullptr); DecodeScalarDestination(sdst, pc, inst.dst);
// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same // SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
// pair index; their descriptor operand consumes four SGPRs from that base. // pair index; their descriptor operand consumes four SGPRs from that base.
DecodeScalarSource(sbase * 2u, pc, inst.src0, nullptr); DecodeScalarSource(sbase * 2u, pc, inst.src0);
DecodeScalarSource(soffset, pc, inst.src1, nullptr); DecodeScalarSource(soffset, pc, inst.src1);
inst.src_count = 2; inst.src_count = 2;
} }
@@ -278,8 +278,8 @@ void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
DecodeVectorGpr(vdata, inst.dst); DecodeVectorGpr(vdata, inst.dst);
DecodeVectorGpr(vaddr, inst.src0); DecodeVectorGpr(vaddr, inst.src0);
DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr); DecodeScalarSource(srsrc * 4u, pc, inst.src1);
DecodeScalarSource(soffset, pc, inst.src2, nullptr); DecodeScalarSource(soffset, pc, inst.src2);
inst.src_count = 3; inst.src_count = 3;
} }
@@ -316,8 +316,8 @@ void DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
DecodeVectorGpr(vdata, inst.dst); DecodeVectorGpr(vdata, inst.dst);
DecodeVectorGpr(vaddr, inst.src0); DecodeVectorGpr(vaddr, inst.src0);
DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr); DecodeScalarSource(srsrc * 4u, pc, inst.src1);
DecodeScalarSource(soffset, pc, inst.src2, nullptr); DecodeScalarSource(soffset, pc, inst.src2);
inst.src_count = 3; inst.src_count = 3;
} }
@@ -364,7 +364,7 @@ void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
DecodeVectorGpr(addr + 1u, inst.src1); DecodeVectorGpr(addr + 1u, inst.src1);
inst.src_count = 2; inst.src_count = 2;
} else { } else {
DecodeScalarSource(saddr, pc, inst.src1, nullptr); DecodeScalarSource(saddr, pc, inst.src1);
inst.src_count = 2; inst.src_count = 2;
} }
} }
@@ -105,21 +105,18 @@ constexpr auto SOPK_OPS = Detail::MakeOpcodeTable<0x20>(SOPK_OPCODE_LIST);
constexpr auto SOPC_OPS = Detail::MakeOpcodeTable<0x80>(SOPC_OPCODE_LIST); constexpr auto SOPC_OPS = Detail::MakeOpcodeTable<0x80>(SOPC_OPCODE_LIST);
constexpr auto SOPP_OPS = Detail::MakeOpcodeTable<0x80>(SOPP_OPCODE_LIST); constexpr auto SOPP_OPS = Detail::MakeOpcodeTable<0x80>(SOPP_OPCODE_LIST);
bool DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction& inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) { Instruction& inst, uint32_t ssrc0, uint32_t ssrc1) {
if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) || DecodeScalarSource(ssrc0, pc, inst.src0);
!DecodeScalarSource(ssrc1, pc, inst.src1, error)) { DecodeScalarSource(ssrc1, pc, inst.src1);
return false;
}
inst.src_count = 2; inst.src_count = 2;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
} // namespace } // namespace
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 8u) & 0xffu; const uint32_t opcode = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu; const uint32_t ssrc0 = word & 0xffu;
@@ -134,35 +131,31 @@ bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented"); SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented");
return true; return;
} }
switch (inst.opcode) { switch (inst.opcode) {
case Opcode::S_GETPC_B64: case Opcode::S_GETPC_B64:
inst.src_count = 0; inst.src_count = 0;
return DecodeScalarDestination(sdst, pc, inst.dst, error); DecodeScalarDestination(sdst, pc, inst.dst);
return;
case Opcode::S_SETPC_B64: case Opcode::S_SETPC_B64:
inst.src_count = 1; inst.src_count = 1;
inst.dst.kind = OperandKind::Null; inst.dst.kind = OperandKind::Null;
if (!DecodeScalarSource(ssrc0, pc, inst.src0, error)) { DecodeScalarSource(ssrc0, pc, inst.src0);
return false;
}
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
default: break; default: break;
} }
if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) || DecodeScalarSource(ssrc0, pc, inst.src0);
!DecodeScalarDestination(sdst, pc, inst.dst, error)) { DecodeScalarDestination(sdst, pc, inst.dst);
return false;
}
inst.src_count = 1; inst.src_count = 1;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 23u) & 0x7fu; const uint32_t opcode = (word >> 23u) & 0x7fu;
const uint32_t ssrc1 = (word >> 8u) & 0xffu; const uint32_t ssrc1 = (word >> 8u) & 0xffu;
@@ -178,17 +171,15 @@ bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented"); SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented");
return true; return;
} }
if (!DecodeScalarDestination(sdst, pc, inst.dst, error)) { DecodeScalarDestination(sdst, pc, inst.dst);
return false; DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1);
}
return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
} }
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 23u) & 0x1fu; const uint32_t opcode = (word >> 23u) & 0x1fu;
const uint32_t sdst = (word >> 16u) & 0x7fu; const uint32_t sdst = (word >> 16u) & 0x7fu;
@@ -207,18 +198,18 @@ bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented"); SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented");
return true; return;
} }
switch (inst.opcode) { switch (inst.opcode) {
case Opcode::S_MOVK_I32: return DecodeScalarDestination(sdst, pc, inst.dst, error); case Opcode::S_MOVK_I32: DecodeScalarDestination(sdst, pc, inst.dst); return;
case Opcode::S_WAITCNT: { case Opcode::S_WAITCNT: {
const uint32_t waitcnt = word & 0xffffu; const uint32_t waitcnt = word & 0xffffu;
inst.dst.kind = OperandKind::Null; inst.dst.kind = OperandKind::Null;
inst.src0.signed_val = static_cast<int32_t>(waitcnt); inst.src0.signed_val = static_cast<int32_t>(waitcnt);
inst.src0.value = waitcnt; inst.src0.value = waitcnt;
inst.src_count = 1; inst.src_count = 1;
return true; return;
} }
case Opcode::S_SETREG_B32: case Opcode::S_SETREG_B32:
inst.dst.kind = OperandKind::Null; inst.dst.kind = OperandKind::Null;
@@ -226,26 +217,25 @@ bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
inst.src1.value = word & 0xffffu; inst.src1.value = word & 0xffffu;
inst.src1.signed_val = static_cast<int32_t>(imm); inst.src1.signed_val = static_cast<int32_t>(imm);
inst.src_count = 2; inst.src_count = 2;
return DecodeScalarSource(sdst, pc, inst.src0, error); DecodeScalarSource(sdst, pc, inst.src0);
return;
default: break; default: break;
} }
inst.src1 = inst.src0; inst.src1 = inst.src0;
if (!DecodeScalarSource(sdst, pc, inst.src0, error)) { DecodeScalarSource(sdst, pc, inst.src0);
return false;
}
if (inst.opcode == Opcode::S_ADD_I32 || inst.opcode == Opcode::S_MULK_I32) { if (inst.opcode == Opcode::S_ADD_I32 || inst.opcode == Opcode::S_MULK_I32) {
inst.src_count = 2; inst.src_count = 2;
return DecodeScalarDestination(sdst, pc, inst.dst, error); DecodeScalarDestination(sdst, pc, inst.dst);
return;
} }
inst.dst.kind = OperandKind::Scc; inst.dst.kind = OperandKind::Scc;
inst.src_count = 2; inst.src_count = 2;
return true;
} }
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t ssrc1 = (word >> 8u) & 0xffu; const uint32_t ssrc1 = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu; const uint32_t ssrc0 = word & 0xffu;
@@ -261,10 +251,10 @@ bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented"); SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented");
return true; return;
} }
return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error); DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1);
} }
void DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
@@ -5,14 +5,14 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder { namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
void DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction& inst); Instruction& inst);
@@ -1,5 +1,6 @@
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/frontend/decode/ExportOps.h" #include "graphics/shader/recompiler/frontend/decode/ExportOps.h"
#include "graphics/shader/recompiler/frontend/decode/ImageOps.h" #include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h" #include "graphics/shader/recompiler/frontend/decode/MemoryOps.h"
@@ -17,12 +18,6 @@ uint32_t FloatBits(float value) {
return std::bit_cast<uint32_t>(value); return std::bit_cast<uint32_t>(value);
} }
void SetError(std::string* error, const char* message) {
if (error != nullptr) {
*error = message;
}
}
bool HasLiteral(const Instruction& inst) { bool HasLiteral(const Instruction& inst) {
return inst.src0.kind == OperandKind::LiteralConstant || return inst.src0.kind == OperandKind::LiteralConstant ||
inst.src1.kind == OperandKind::LiteralConstant || inst.src1.kind == OperandKind::LiteralConstant ||
@@ -216,86 +211,76 @@ const char* ImageDimensionToString(ImageDimension dimension) {
} }
} }
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error) { void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand) {
operand = {}; operand = {};
if (code <= 105u) { if (code <= 105u) {
operand.kind = OperandKind::Sgpr; operand.kind = OperandKind::Sgpr;
operand.reg = code; operand.reg = code;
return true; return;
} }
if (code >= 128u && code <= 192u) { if (code >= 128u && code <= 192u) {
operand.kind = OperandKind::IntegerInlineConstant; operand.kind = OperandKind::IntegerInlineConstant;
operand.signed_val = static_cast<int32_t>(code - 128u); operand.signed_val = static_cast<int32_t>(code - 128u);
operand.value = static_cast<uint32_t>(operand.signed_val); operand.value = static_cast<uint32_t>(operand.signed_val);
return true; return;
} }
if (code >= 193u && code <= 208u) { if (code >= 193u && code <= 208u) {
operand.kind = OperandKind::IntegerInlineConstant; operand.kind = OperandKind::IntegerInlineConstant;
operand.signed_val = 192 - static_cast<int32_t>(code); operand.signed_val = 192 - static_cast<int32_t>(code);
operand.value = static_cast<uint32_t>(operand.signed_val); operand.value = static_cast<uint32_t>(operand.signed_val);
return true; return;
} }
if (code >= 240u && code <= 247u) { if (code >= 240u && code <= 247u) {
constexpr float values[] = {0.5f, -0.5f, 1.0f, -1.0f, 2.0f, -2.0f, 4.0f, -4.0f}; constexpr float values[] = {0.5f, -0.5f, 1.0f, -1.0f, 2.0f, -2.0f, 4.0f, -4.0f};
operand.kind = OperandKind::FloatInlineConstant; operand.kind = OperandKind::FloatInlineConstant;
operand.float_val = values[code - 240u]; operand.float_val = values[code - 240u];
operand.value = FloatBits(operand.float_val); operand.value = FloatBits(operand.float_val);
return true; return;
} }
if (code >= 256u && code <= 511u) { if (code >= 256u && code <= 511u) {
DecodeVectorGpr(code - 256u, operand); DecodeVectorGpr(code - 256u, operand);
return true; return;
} }
switch (code) { switch (code) {
case 106u: operand.kind = OperandKind::VccLo; return true; case 106u: operand.kind = OperandKind::VccLo; return;
case 107u: operand.kind = OperandKind::VccHi; return true; case 107u: operand.kind = OperandKind::VccHi; return;
case 124u: operand.kind = OperandKind::M0; return true; case 124u: operand.kind = OperandKind::M0; return;
case 125u: operand.kind = OperandKind::Null; return true; case 125u: operand.kind = OperandKind::Null; return;
case 126u: operand.kind = OperandKind::ExecLo; return true; case 126u: operand.kind = OperandKind::ExecLo; return;
case 127u: operand.kind = OperandKind::ExecHi; return true; case 127u: operand.kind = OperandKind::ExecHi; return;
case 239u: operand.kind = OperandKind::PopsExitingWaveId; return true; case 239u: operand.kind = OperandKind::PopsExitingWaveId; return;
case 248u: case 248u:
operand.kind = OperandKind::FloatInlineConstant; operand.kind = OperandKind::FloatInlineConstant;
operand.float_val = 0.15915494309189535f; operand.float_val = 0.15915494309189535f;
operand.value = FloatBits(operand.float_val); operand.value = FloatBits(operand.float_val);
return true; return;
case 251u: operand.kind = OperandKind::VccZ; return true; case 251u: operand.kind = OperandKind::VccZ; return;
case 252u: operand.kind = OperandKind::ExecZ; return true; case 252u: operand.kind = OperandKind::ExecZ; return;
case 253u: operand.kind = OperandKind::Scc; return true; case 253u: operand.kind = OperandKind::Scc; return;
case 255u: operand.kind = OperandKind::LiteralConstant; return true; case 255u: operand.kind = OperandKind::LiteralConstant; return;
default: default: EXIT("unsupported scalar source operand 0x%08x at pc 0x%08x", code, pc);
if (error != nullptr) {
*error = fmt::format("unsupported scalar source operand 0x{:08x} at pc 0x{:08x}",
code, pc);
}
return false;
} }
} }
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error) { void DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand) {
operand = {}; operand = {};
if (code <= 105u) { if (code <= 105u) {
operand.kind = OperandKind::Sgpr; operand.kind = OperandKind::Sgpr;
operand.reg = code; operand.reg = code;
return true; return;
} }
switch (code) { switch (code) {
case 106u: operand.kind = OperandKind::VccLo; return true; case 106u: operand.kind = OperandKind::VccLo; return;
case 107u: operand.kind = OperandKind::VccHi; return true; case 107u: operand.kind = OperandKind::VccHi; return;
case 124u: operand.kind = OperandKind::M0; return true; case 124u: operand.kind = OperandKind::M0; return;
case 125u: operand.kind = OperandKind::Null; return true; case 125u: operand.kind = OperandKind::Null; return;
case 126u: operand.kind = OperandKind::ExecLo; return true; case 126u: operand.kind = OperandKind::ExecLo; return;
case 127u: operand.kind = OperandKind::ExecHi; return true; case 127u: operand.kind = OperandKind::ExecHi; return;
default: default: EXIT("unsupported scalar destination operand 0x%08x at pc 0x%08x", code, pc);
if (error != nullptr) {
*error = fmt::format(
"unsupported scalar destination operand 0x{:08x} at pc 0x{:08x}", code, pc);
}
return false;
} }
} }
@@ -368,39 +353,33 @@ Family GetInstructionFamily(uint32_t word) {
} }
} }
bool DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst) {
std::string* error) {
const uint32_t pc = word_index * sizeof(uint32_t); const uint32_t pc = word_index * sizeof(uint32_t);
switch (GetInstructionFamily(code[word_index])) { switch (GetInstructionFamily(code[word_index])) {
case Family::SOP1: return DecodeSop1(pc, code, word_index, inst, error); case Family::SOP1: DecodeSop1(pc, code, word_index, inst); return;
case Family::SOP2: return DecodeSop2(pc, code, word_index, inst, error); case Family::SOP2: DecodeSop2(pc, code, word_index, inst); return;
case Family::SOPK: return DecodeSopk(pc, code, word_index, inst, error); case Family::SOPK: DecodeSopk(pc, code, word_index, inst); return;
case Family::SOPC: return DecodeSopc(pc, code, word_index, inst, error); case Family::SOPC: DecodeSopc(pc, code, word_index, inst); return;
case Family::SOPP: DecodeSopp(pc, code, word_index, inst); return true; case Family::SOPP: DecodeSopp(pc, code, word_index, inst); return;
case Family::VOP1: return DecodeVop1(pc, code, word_index, inst, error); case Family::VOP1: DecodeVop1(pc, code, word_index, inst); return;
case Family::VOP2: return DecodeVop2(pc, code, word_index, inst, error); case Family::VOP2: DecodeVop2(pc, code, word_index, inst); return;
case Family::VOP3: return DecodeVop3(pc, code, word_index, inst, error); case Family::VOP3: DecodeVop3(pc, code, word_index, inst); return;
case Family::VOP3P: return DecodeVop3p(pc, code, word_index, inst, error); case Family::VOP3P: DecodeVop3p(pc, code, word_index, inst); return;
case Family::VOPC: return DecodeVopc(pc, code, word_index, inst, error); case Family::VOPC: DecodeVopc(pc, code, word_index, inst); return;
case Family::VINTRP: DecodeVintrp(pc, code, word_index, inst); return true; case Family::VINTRP: DecodeVintrp(pc, code, word_index, inst); return;
case Family::SMEM: DecodeSmem(pc, code, word_index, inst); return true; case Family::SMEM: DecodeSmem(pc, code, word_index, inst); return;
case Family::MUBUF: DecodeMubuf(pc, code, word_index, inst); return true; case Family::MUBUF: DecodeMubuf(pc, code, word_index, inst); return;
case Family::MTBUF: DecodeMtbuf(pc, code, word_index, inst); return true; case Family::MTBUF: DecodeMtbuf(pc, code, word_index, inst); return;
case Family::FLAT: DecodeFlat(pc, code, word_index, inst); return true; case Family::FLAT: DecodeFlat(pc, code, word_index, inst); return;
case Family::DS: DecodeDs(pc, code, word_index, inst); return true; case Family::DS: DecodeDs(pc, code, word_index, inst); return;
case Family::MIMG: DecodeMimg(pc, code, word_index, inst); return true; case Family::MIMG: DecodeMimg(pc, code, word_index, inst); return;
case Family::EXP: DecodeExp(pc, code, word_index, inst); return true; case Family::EXP: DecodeExp(pc, code, word_index, inst); return;
default: default:
if (error != nullptr) { EXIT("unknown RDNA2 instruction family at pc 0x%08x, raw=0x%08x", pc, code[word_index]);
*error =
fmt::format("unknown RDNA2 instruction family at pc 0x{:08x}, raw=0x{:08x}", pc,
code[word_index]);
}
return false;
} }
} }
bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error) { void DecodeProgram(std::span<const uint32_t> code, Program& program) {
program.instructions.clear(); program.instructions.clear();
program.instructions.reserve(code.size()); program.instructions.reserve(code.size());
program.code = code; program.code = code;
@@ -408,10 +387,7 @@ bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string
std::vector<bool> branch_targets; std::vector<bool> branch_targets;
for (uint32_t word_index = 0; word_index < code.size();) { for (uint32_t word_index = 0; word_index < code.size();) {
program.instructions.emplace_back(); program.instructions.emplace_back();
if (!DecodeInstruction(code, word_index, program.instructions.back(), error)) { DecodeInstruction(code, word_index, program.instructions.back());
program.instructions.pop_back();
return false;
}
const auto& inst = program.instructions.back(); const auto& inst = program.instructions.back();
word_index += inst.word_count; word_index += inst.word_count;
@@ -425,12 +401,11 @@ bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string
} }
if (inst.opcode == Opcode::S_ENDPGM && if (inst.opcode == Opcode::S_ENDPGM &&
(word_index >= code.size() || branch_targets.empty() || !branch_targets[word_index])) { (word_index >= code.size() || branch_targets.empty() || !branch_targets[word_index])) {
return true; return;
} }
} }
SetError(error, "shader decode reached the code boundary before S_ENDPGM"); EXIT("shader decode reached the code boundary before S_ENDPGM");
return false;
} }
std::string OperandToString(const Operand& operand) { std::string OperandToString(const Operand& operand) {
@@ -704,12 +704,11 @@ struct Program {
// Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs. // Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.
Family GetInstructionFamily(uint32_t word); Family GetInstructionFamily(uint32_t word);
// The output object must be freshly initialized. // The output object must be freshly initialized.
bool DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst);
std::string* error); void DecodeProgram(std::span<const uint32_t> code, Program& program);
bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error);
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error); void DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand);
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error); void DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand);
void DecodeVectorGpr(uint32_t reg, Operand& operand); void DecodeVectorGpr(uint32_t reg, Operand& operand);
void ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst); void ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst);
void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index, void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index,
@@ -204,55 +204,52 @@ struct VopcOpcodeInfo {
}; };
constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = { constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
{0x00u, Opcode::V_CMP_F_F32}, {0x01u, Opcode::V_CMP_LT_F32}, {0x00u, Opcode::V_CMP_F_F32}, {0x01u, Opcode::V_CMP_LT_F32},
{0x02u, Opcode::V_CMP_EQ_F32}, {0x03u, Opcode::V_CMP_LE_F32}, {0x02u, Opcode::V_CMP_EQ_F32}, {0x03u, Opcode::V_CMP_LE_F32},
{0x04u, Opcode::V_CMP_GT_F32}, {0x05u, Opcode::V_CMP_LG_F32}, {0x04u, Opcode::V_CMP_GT_F32}, {0x05u, Opcode::V_CMP_LG_F32},
{0x06u, Opcode::V_CMP_GE_F32}, {0x07u, Opcode::V_CMP_O_F32}, {0x06u, Opcode::V_CMP_GE_F32}, {0x07u, Opcode::V_CMP_O_F32},
{0x08u, Opcode::V_CMP_U_F32}, {0x09u, Opcode::V_CMP_NGE_F32}, {0x08u, Opcode::V_CMP_U_F32}, {0x09u, Opcode::V_CMP_NGE_F32},
{0x0au, Opcode::V_CMP_NLG_F32}, {0x0bu, Opcode::V_CMP_NGT_F32}, {0x0au, Opcode::V_CMP_NLG_F32}, {0x0bu, Opcode::V_CMP_NGT_F32},
{0x0cu, Opcode::V_CMP_NLE_F32}, {0x0du, Opcode::V_CMP_NEQ_F32}, {0x0cu, Opcode::V_CMP_NLE_F32}, {0x0du, Opcode::V_CMP_NEQ_F32},
{0x0eu, Opcode::V_CMP_NLT_F32}, {0x0fu, Opcode::V_CMP_TRU_F32}, {0x0eu, Opcode::V_CMP_NLT_F32}, {0x0fu, Opcode::V_CMP_TRU_F32},
{0x11u, Opcode::V_CMPX_LT_F32}, {0x12u, Opcode::V_CMPX_EQ_F32}, {0x11u, Opcode::V_CMPX_LT_F32}, {0x12u, Opcode::V_CMPX_EQ_F32},
{0x13u, Opcode::V_CMPX_LE_F32}, {0x14u, Opcode::V_CMPX_GT_F32}, {0x13u, Opcode::V_CMPX_LE_F32}, {0x14u, Opcode::V_CMPX_GT_F32},
{0x15u, Opcode::V_CMPX_LG_F32}, {0x16u, Opcode::V_CMPX_GE_F32}, {0x15u, Opcode::V_CMPX_LG_F32}, {0x16u, Opcode::V_CMPX_GE_F32},
{0x19u, Opcode::V_CMPX_NGE_F32}, {0x1au, Opcode::V_CMPX_NLG_F32}, {0x19u, Opcode::V_CMPX_NGE_F32}, {0x1au, Opcode::V_CMPX_NLG_F32},
{0x1bu, Opcode::V_CMPX_NGT_F32}, {0x1cu, Opcode::V_CMPX_NLE_F32}, {0x1bu, Opcode::V_CMPX_NGT_F32}, {0x1cu, Opcode::V_CMPX_NLE_F32},
{0x1du, Opcode::V_CMPX_NEQ_F32}, {0x1eu, Opcode::V_CMPX_NLT_F32}, {0x1du, Opcode::V_CMPX_NEQ_F32}, {0x1eu, Opcode::V_CMPX_NLT_F32},
{0x80u, Opcode::V_CMP_F_I32}, {0x81u, Opcode::V_CMP_LT_I32}, {0x80u, Opcode::V_CMP_F_I32}, {0x81u, Opcode::V_CMP_LT_I32},
{0x82u, Opcode::V_CMP_EQ_I32}, {0x83u, Opcode::V_CMP_LE_I32}, {0x82u, Opcode::V_CMP_EQ_I32}, {0x83u, Opcode::V_CMP_LE_I32},
{0x84u, Opcode::V_CMP_GT_I32}, {0x85u, Opcode::V_CMP_NE_I32}, {0x84u, Opcode::V_CMP_GT_I32}, {0x85u, Opcode::V_CMP_NE_I32},
{0x86u, Opcode::V_CMP_GE_I32}, {0x87u, Opcode::V_CMP_T_I32}, {0x86u, Opcode::V_CMP_GE_I32}, {0x87u, Opcode::V_CMP_T_I32},
{0x88u, Opcode::V_CMP_CLASS_F32}, {0x89u, Opcode::V_CMP_LT_I16}, {0x88u, Opcode::V_CMP_CLASS_F32}, {0x89u, Opcode::V_CMP_LT_I16},
{0x8au, Opcode::V_CMP_EQ_I16}, {0x8bu, Opcode::V_CMP_LE_I16}, {0x8au, Opcode::V_CMP_EQ_I16}, {0x8bu, Opcode::V_CMP_LE_I16},
{0x8cu, Opcode::V_CMP_GT_I16}, {0x8du, Opcode::V_CMP_NE_I16}, {0x8cu, Opcode::V_CMP_GT_I16}, {0x8du, Opcode::V_CMP_NE_I16},
{0x8eu, Opcode::V_CMP_GE_I16}, {0x91u, Opcode::V_CMPX_LT_I32}, {0x8eu, Opcode::V_CMP_GE_I16}, {0x91u, Opcode::V_CMPX_LT_I32},
{0x92u, Opcode::V_CMPX_EQ_I32}, {0x93u, Opcode::V_CMPX_LE_I32}, {0x92u, Opcode::V_CMPX_EQ_I32}, {0x93u, Opcode::V_CMPX_LE_I32},
{0x94u, Opcode::V_CMPX_GT_I32}, {0x95u, Opcode::V_CMPX_NE_I32}, {0x94u, Opcode::V_CMPX_GT_I32}, {0x95u, Opcode::V_CMPX_NE_I32},
{0x96u, Opcode::V_CMPX_GE_I32}, {0x98u, Opcode::V_CMPX_CLASS_F32}, {0x96u, Opcode::V_CMPX_GE_I32}, {0x98u, Opcode::V_CMPX_CLASS_F32},
{0xa9u, Opcode::V_CMP_LT_U16}, {0xa9u, Opcode::V_CMP_LT_U16}, {0xaau, Opcode::V_CMP_EQ_U16},
{0xaau, Opcode::V_CMP_EQ_U16}, {0xabu, Opcode::V_CMP_LE_U16}, {0xabu, Opcode::V_CMP_LE_U16}, {0xacu, Opcode::V_CMP_GT_U16},
{0xacu, Opcode::V_CMP_GT_U16}, {0xadu, Opcode::V_CMP_NE_U16}, {0xadu, Opcode::V_CMP_NE_U16}, {0xaeu, Opcode::V_CMP_GE_U16},
{0xaeu, Opcode::V_CMP_GE_U16}, {0xbcu, Opcode::V_CMPX_GT_U16}, {0xbcu, Opcode::V_CMPX_GT_U16}, {0xc0u, Opcode::V_CMP_F_U32},
{0xc0u, Opcode::V_CMP_F_U32}, {0xc1u, Opcode::V_CMP_LT_U32}, {0xc2u, Opcode::V_CMP_EQ_U32},
{0xc1u, Opcode::V_CMP_LT_U32}, {0xc2u, Opcode::V_CMP_EQ_U32}, {0xc3u, Opcode::V_CMP_LE_U32}, {0xc4u, Opcode::V_CMP_GT_U32},
{0xc3u, Opcode::V_CMP_LE_U32}, {0xc4u, Opcode::V_CMP_GT_U32}, {0xc5u, Opcode::V_CMP_NE_U32}, {0xc6u, Opcode::V_CMP_GE_U32},
{0xc5u, Opcode::V_CMP_NE_U32}, {0xc6u, Opcode::V_CMP_GE_U32}, {0xc7u, Opcode::V_CMP_T_U32}, {0xa2u, Opcode::V_CMP_EQ_I64, false},
{0xc7u, Opcode::V_CMP_T_U32}, {0xa2u, Opcode::V_CMP_EQ_I64, false}, {0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd1u, Opcode::V_CMPX_LT_U32},
{0xb5u, Opcode::V_CMPX_NE_I64, false}, {0xd2u, Opcode::V_CMPX_EQ_U32}, {0xd3u, Opcode::V_CMPX_LE_U32},
{0xd1u, Opcode::V_CMPX_LT_U32}, {0xd2u, Opcode::V_CMPX_EQ_U32}, {0xd4u, Opcode::V_CMPX_GT_U32}, {0xd5u, Opcode::V_CMPX_NE_U32},
{0xd3u, Opcode::V_CMPX_LE_U32}, {0xd4u, Opcode::V_CMPX_GT_U32}, {0xd6u, Opcode::V_CMPX_GE_U32}, {0xe2u, Opcode::V_CMP_EQ_U64, false},
{0xd5u, Opcode::V_CMPX_NE_U32}, {0xd6u, Opcode::V_CMPX_GE_U32}, {0xe4u, Opcode::V_CMP_GT_U64, false}, {0xe5u, Opcode::V_CMP_NE_U64, false},
{0xe2u, Opcode::V_CMP_EQ_U64, false}, {0xe4u, Opcode::V_CMP_GT_U64, false}, {0xf5u, Opcode::V_CMPX_NE_U64, false}, {0xc9u, Opcode::V_CMP_LT_F16},
{0xe5u, Opcode::V_CMP_NE_U64, false}, {0xcau, Opcode::V_CMP_EQ_F16}, {0xcbu, Opcode::V_CMP_LE_F16},
{0xf5u, Opcode::V_CMPX_NE_U64, false}, {0xccu, Opcode::V_CMP_GT_F16}, {0xcdu, Opcode::V_CMP_LG_F16},
{0xc9u, Opcode::V_CMP_LT_F16}, {0xcau, Opcode::V_CMP_EQ_F16}, {0xceu, Opcode::V_CMP_GE_F16}, {0xedu, Opcode::V_CMP_NEQ_F16},
{0xcbu, Opcode::V_CMP_LE_F16}, {0xccu, Opcode::V_CMP_GT_F16}, {0xd9u, Opcode::V_CMPX_LT_F16}, {0xdau, Opcode::V_CMPX_EQ_F16},
{0xcdu, Opcode::V_CMP_LG_F16}, {0xceu, Opcode::V_CMP_GE_F16}, {0xdbu, Opcode::V_CMPX_LE_F16}, {0xdcu, Opcode::V_CMPX_GT_F16},
{0xedu, Opcode::V_CMP_NEQ_F16}, {0xd9u, Opcode::V_CMPX_LT_F16}, {0xdeu, Opcode::V_CMPX_GE_F16}, {0xfbu, Opcode::V_CMPX_NGT_F16},
{0xdau, Opcode::V_CMPX_EQ_F16}, {0xdbu, Opcode::V_CMPX_LE_F16}, {0xfdu, Opcode::V_CMPX_NEQ_F16}, {0xfeu, Opcode::V_CMPX_NLT_F16},
{0xdcu, Opcode::V_CMPX_GT_F16}, {0xdeu, Opcode::V_CMPX_GE_F16},
{0xfbu, Opcode::V_CMPX_NGT_F16}, {0xfdu, Opcode::V_CMPX_NEQ_F16},
{0xfeu, Opcode::V_CMPX_NLT_F16},
}; };
constexpr auto VOPC_OPS = Detail::MakeOpcodeTable<0x100>(VOPC_OPCODE_LIST); constexpr auto VOPC_OPS = Detail::MakeOpcodeTable<0x100>(VOPC_OPCODE_LIST);
@@ -641,8 +638,8 @@ bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
return true; return true;
} }
bool DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, Instruction& inst, std::string* error) { uint32_t opcode, uint32_t vdst, Instruction& inst) {
const auto modifier = code[word_index + 1u]; const auto modifier = code[word_index + 1u];
const auto src0 = modifier & 0xffu; const auto src0 = modifier & 0xffu;
const auto dst_sel = (modifier >> 8u) & 0x7u; const auto dst_sel = (modifier >> 8u) & 0x7u;
@@ -657,32 +654,27 @@ bool DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
SetRawWords(inst, code, word_index, 2); SetRawWords(inst, code, word_index, 2);
if (!ValidateVop1Sdwa(inst, opcode, modifier)) { if (!ValidateVop1Sdwa(inst, opcode, modifier)) {
return true; return;
} }
const bool scalar_dst = UsesScalarDestination(inst.opcode); const bool scalar_dst = UsesScalarDestination(inst.opcode);
if (scalar_dst) { if (scalar_dst) {
if (!DecodeScalarDestination(vdst, pc, inst.dst, error)) { DecodeScalarDestination(vdst, pc, inst.dst);
return false;
}
} else { } else {
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
} }
if (!DecodeScalarSource(src0 + (s0 == 0u ? 256u : 0u), pc, inst.src0, error)) { DecodeScalarSource(src0 + (s0 == 0u ? 256u : 0u), pc, inst.src0);
return false;
}
inst.dst.sdwa_sel = dst_sel; inst.dst.sdwa_sel = dst_sel;
inst.dst.sdwa_dst_unused = dst_u; inst.dst.sdwa_dst_unused = dst_u;
inst.dst.explicit_sdwa_dst = true; inst.dst.explicit_sdwa_dst = true;
inst.dst.clamp = clamp != 0u; inst.dst.clamp = clamp != 0u;
inst.dst.omod = omod; inst.dst.omod = omod;
inst.src0.sdwa_sel = src0_sel; inst.src0.sdwa_sel = src0_sel;
inst.src0.sdwa_sext = src0_sext != 0u; inst.src0.sdwa_sext = src0_sext != 0u;
inst.src0.negate = src0_neg != 0u; inst.src0.negate = src0_neg != 0u;
inst.src0.absolute = src0_abs != 0u; inst.src0.absolute = src0_abs != 0u;
inst.src_count = 1; inst.src_count = 1;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
void ApplyDppModifier(Operand& operand, uint32_t modifier) { void ApplyDppModifier(Operand& operand, uint32_t modifier) {
@@ -696,38 +688,33 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier) {
operand.dpp_row_mask = (modifier >> 28u) & 0xfu; operand.dpp_row_mask = (modifier >> 28u) & 0xfu;
} }
bool DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, Instruction& inst, std::string* error) { uint32_t opcode, uint32_t vdst, Instruction& inst) {
const auto modifier = code[word_index + 1u]; const auto modifier = code[word_index + 1u];
const auto src0 = modifier & 0xffu; const auto src0 = modifier & 0xffu;
SetRawWords(inst, code, word_index, 2); SetRawWords(inst, code, word_index, 2);
const bool scalar_dst = UsesScalarDestination(inst.opcode); const bool scalar_dst = UsesScalarDestination(inst.opcode);
if (scalar_dst) { if (scalar_dst) {
if (!DecodeScalarDestination(vdst, pc, inst.dst, error)) { DecodeScalarDestination(vdst, pc, inst.dst);
return false;
}
} else { } else {
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
} }
if (!DecodeScalarSource(src0 + 256u, pc, inst.src0, error)) { DecodeScalarSource(src0 + 256u, pc, inst.src0);
return false;
}
ApplyDppModifier(inst.src0, modifier); ApplyDppModifier(inst.src0, modifier);
inst.src_count = 1; inst.src_count = 1;
if (!IsVop1FloatSourceOpcode(inst.opcode) && (inst.src0.negate || inst.src0.absolute)) { if (!IsVop1FloatSourceOpcode(inst.opcode) && (inst.src0.negate || inst.src0.absolute)) {
SetUnsupported(inst, Family::VOP1, opcode, SetUnsupported(inst, Family::VOP1, opcode,
"VOP1 DPP integer source modifiers are not supported"); "VOP1 DPP integer source modifiers are not supported");
return true; return;
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
using Vop1ModifierDecodeFn = bool (*)(uint32_t pc, std::span<const uint32_t> code, using Vop1ModifierDecodeFn = void (*)(uint32_t pc, std::span<const uint32_t> code,
uint32_t word_index, uint32_t opcode, uint32_t vdst, uint32_t word_index, uint32_t opcode, uint32_t vdst,
Instruction& inst, std::string* error); Instruction& inst);
struct Vop1ModifierDecoder { struct Vop1ModifierDecoder {
uint32_t escape = 0; uint32_t escape = 0;
@@ -740,20 +727,14 @@ constexpr Vop1ModifierDecoder VOP1_MODIFIER_DECODERS[] = {
}; };
bool TryDecodeVop1Modifier(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, bool TryDecodeVop1Modifier(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t src0, uint32_t opcode, uint32_t vdst, Instruction& inst, uint32_t src0, uint32_t opcode, uint32_t vdst, Instruction& inst) {
std::string* error, bool* handled) {
if (handled != nullptr) {
*handled = false;
}
for (const auto& decoder: VOP1_MODIFIER_DECODERS) { for (const auto& decoder: VOP1_MODIFIER_DECODERS) {
if (decoder.escape == src0) { if (decoder.escape == src0) {
if (handled != nullptr) { decoder.decode(pc, code, word_index, opcode, vdst, inst);
*handled = true; return true;
}
return decoder.decode(pc, code, word_index, opcode, vdst, inst, error);
} }
} }
return true; return false;
} }
bool IsVop2FloatSourceOpcode(Opcode opcode) { bool IsVop2FloatSourceOpcode(Opcode opcode) {
@@ -1034,9 +1015,8 @@ bool ValidateVop2Sdwa(Instruction& inst, uint32_t opcode, const Vop2SdwaFields&
return true; return true;
} }
bool FinalizeVop2Instruction(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void FinalizeVop2Instruction(std::span<const uint32_t> code, uint32_t word_index,
Instruction& inst, std::string* error) { Instruction& inst) {
(void)pc;
switch (inst.opcode) { switch (inst.opcode) {
case Opcode::V_MADMK_F32: case Opcode::V_MADMK_F32:
case Opcode::V_FMAMK_F16: case Opcode::V_FMAMK_F16:
@@ -1067,55 +1047,48 @@ bool FinalizeVop2Instruction(uint32_t pc, std::span<const uint32_t> code, uint32
default: inst.src_count = 2; break; default: inst.src_count = 2; break;
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
bool DecodeVop2Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVop2Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction& inst, uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction& inst) {
std::string* error) {
const auto modifier = code[word_index + 1u]; const auto modifier = code[word_index + 1u];
const auto fields = DecodeVop2SdwaFields(modifier); const auto fields = DecodeVop2SdwaFields(modifier);
SetRawWords(inst, code, word_index, 2); SetRawWords(inst, code, word_index, 2);
if (!ValidateVop2Sdwa(inst, opcode, fields)) { if (!ValidateVop2Sdwa(inst, opcode, fields)) {
return true; return;
} }
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
if (!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, inst.src0, error) || DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, inst.src0);
!DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1, error)) { DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1);
return false;
}
inst.dst.sdwa_sel = fields.dst_sel; inst.dst.sdwa_sel = fields.dst_sel;
inst.dst.sdwa_dst_unused = fields.dst_u; inst.dst.sdwa_dst_unused = fields.dst_u;
inst.dst.explicit_sdwa_dst = true; inst.dst.explicit_sdwa_dst = true;
inst.dst.clamp = fields.clamp != 0u; inst.dst.clamp = fields.clamp != 0u;
inst.dst.omod = fields.omod; inst.dst.omod = fields.omod;
inst.src0.sdwa_sel = fields.src0_sel; inst.src0.sdwa_sel = fields.src0_sel;
inst.src0.sdwa_sext = fields.src0_sext != 0u; inst.src0.sdwa_sext = fields.src0_sext != 0u;
inst.src0.negate = fields.src0_neg != 0u; inst.src0.negate = fields.src0_neg != 0u;
inst.src0.absolute = fields.src0_abs != 0u; inst.src0.absolute = fields.src0_abs != 0u;
inst.src1.sdwa_sel = fields.src1_sel; inst.src1.sdwa_sel = fields.src1_sel;
inst.src1.sdwa_sext = fields.src1_sext != 0u; inst.src1.sdwa_sext = fields.src1_sext != 0u;
inst.src1.negate = fields.src1_neg != 0u; inst.src1.negate = fields.src1_neg != 0u;
inst.src1.absolute = fields.src1_abs != 0u; inst.src1.absolute = fields.src1_abs != 0u;
return FinalizeVop2Instruction(pc, code, word_index, inst, error); FinalizeVop2Instruction(code, word_index, inst);
} }
bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction& inst, uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction& inst) {
std::string* error) {
const auto modifier = code[word_index + 1u]; const auto modifier = code[word_index + 1u];
const auto src0 = modifier & 0xffu; const auto src0 = modifier & 0xffu;
SetRawWords(inst, code, word_index, 2); SetRawWords(inst, code, word_index, 2);
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
DecodeVectorGpr(vsrc1, inst.src1); DecodeVectorGpr(vsrc1, inst.src1);
if (!DecodeScalarSource(src0 + 256u, pc, inst.src0, error)) { DecodeScalarSource(src0 + 256u, pc, inst.src0);
return false;
}
ApplyDppModifier(inst.src0, modifier); ApplyDppModifier(inst.src0, modifier);
inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u; inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u;
inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u; inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16; const bool packed_fmac = inst.opcode == Opcode::V_PK_FMAC_F16;
if (packed_fmac) { if (packed_fmac) {
inst.src0.negate_hi = inst.src0.negate; inst.src0.negate_hi = inst.src0.negate;
@@ -1126,14 +1099,14 @@ bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
(inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute)) { (inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute)) {
SetUnsupported(inst, Family::VOP2, opcode, SetUnsupported(inst, Family::VOP2, opcode,
"VOP2 DPP integer source modifiers are not supported"); "VOP2 DPP integer source modifiers are not supported");
return true; return;
} }
return FinalizeVop2Instruction(pc, code, word_index, inst, error); FinalizeVop2Instruction(code, word_index, inst);
} }
using Vop2ModifierDecodeFn = bool (*)(uint32_t pc, std::span<const uint32_t> code, using Vop2ModifierDecodeFn = void (*)(uint32_t pc, std::span<const uint32_t> code,
uint32_t word_index, uint32_t opcode, uint32_t vdst, uint32_t word_index, uint32_t opcode, uint32_t vdst,
uint32_t vsrc1, Instruction& inst, std::string* error); uint32_t vsrc1, Instruction& inst);
struct Vop2ModifierDecoder { struct Vop2ModifierDecoder {
uint32_t escape = 0; uint32_t escape = 0;
@@ -1147,19 +1120,14 @@ constexpr Vop2ModifierDecoder VOP2_MODIFIER_DECODERS[] = {
bool TryDecodeVop2Modifier(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, bool TryDecodeVop2Modifier(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t src0, uint32_t opcode, uint32_t vdst, uint32_t vsrc1, uint32_t src0, uint32_t opcode, uint32_t vdst, uint32_t vsrc1,
Instruction& inst, std::string* error, bool* handled) { Instruction& inst) {
if (handled != nullptr) {
*handled = false;
}
for (const auto& decoder: VOP2_MODIFIER_DECODERS) { for (const auto& decoder: VOP2_MODIFIER_DECODERS) {
if (decoder.escape == src0) { if (decoder.escape == src0) {
if (handled != nullptr) { decoder.decode(pc, code, word_index, opcode, vdst, vsrc1, inst);
*handled = true; return true;
}
return decoder.decode(pc, code, word_index, opcode, vdst, vsrc1, inst, error);
} }
} }
return true; return false;
} }
struct VopcSdwaFields { struct VopcSdwaFields {
@@ -1200,31 +1168,29 @@ bool SupportsVopcSdwa(Opcode opcode) {
return opcode != Opcode::UNSUPPORTED; return opcode != Opcode::UNSUPPORTED;
} }
bool DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vsrc1, Instruction& inst, std::string* error) { uint32_t opcode, uint32_t vsrc1, Instruction& inst) {
const auto modifier = code[word_index + 1u]; const auto modifier = code[word_index + 1u];
const auto fields = DecodeVopcSdwaFields(modifier); const auto fields = DecodeVopcSdwaFields(modifier);
SetRawWords(inst, code, word_index, 2); SetRawWords(inst, code, word_index, 2);
if (fields.src0_sel > 6u || fields.src1_sel > 6u) { if (fields.src0_sel > 6u || fields.src1_sel > 6u) {
SetUnsupported(inst, Family::VOPC, opcode, "VOPC SDWA selector is invalid"); SetUnsupported(inst, Family::VOPC, opcode, "VOPC SDWA selector is invalid");
return true; return;
} }
if (!SupportsVopcSdwa(inst.opcode)) { if (!SupportsVopcSdwa(inst.opcode)) {
SetUnsupported(inst, Family::VOPC, opcode, SetUnsupported(inst, Family::VOPC, opcode,
"VOPC SDWA modifier is not supported for opcode"); "VOPC SDWA modifier is not supported for opcode");
return true; return;
} }
if (!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, inst.src0, error) || DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, inst.src0);
!DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1, error)) { DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1);
return false;
}
if (IsVopcCompareExec(inst.opcode)) { if (IsVopcCompareExec(inst.opcode)) {
inst.dst.kind = OperandKind::ExecLo; inst.dst.kind = OperandKind::ExecLo;
} else if (fields.sd == 0u) { } else if (fields.sd == 0u) {
inst.dst.kind = OperandKind::VccLo; inst.dst.kind = OperandKind::VccLo;
} else if (!DecodeScalarDestination(fields.sdst, pc, inst.dst, error)) { } else {
return false; DecodeScalarDestination(fields.sdst, pc, inst.dst);
} }
inst.src0.sdwa_sel = fields.src0_sel; inst.src0.sdwa_sel = fields.src0_sel;
inst.src0.sdwa_sext = fields.src0_sext != 0u; inst.src0.sdwa_sext = fields.src0_sext != 0u;
@@ -1236,34 +1202,30 @@ bool DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
inst.src1.absolute = fields.src1_abs != 0u; inst.src1.absolute = fields.src1_abs != 0u;
inst.src_count = 2; inst.src_count = 2;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
bool DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vsrc1, Instruction& inst, std::string* error) { uint32_t opcode, uint32_t vsrc1, Instruction& inst) {
const auto modifier = code[word_index + 1u]; const auto modifier = code[word_index + 1u];
const auto src0 = modifier & 0xffu; const auto src0 = modifier & 0xffu;
SetRawWords(inst, code, word_index, 2); SetRawWords(inst, code, word_index, 2);
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented"); SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented");
return true; return;
} }
const auto* info = Detail::FindOpcode(VOPC_OPS, opcode); const auto* info = Detail::FindOpcode(VOPC_OPS, opcode);
if (info == nullptr || !info->supports_dpp) { if (info == nullptr || !info->supports_dpp) {
SetUnsupported(inst, Family::VOPC, opcode, "VOPC DPP modifier is not supported for opcode"); SetUnsupported(inst, Family::VOPC, opcode, "VOPC DPP modifier is not supported for opcode");
return true; return;
} }
DecodeVectorGpr(vsrc1, inst.src1); DecodeVectorGpr(vsrc1, inst.src1);
if (!DecodeScalarSource(src0 + 256u, pc, inst.src0, error)) { DecodeScalarSource(src0 + 256u, pc, inst.src0);
return false;
}
inst.dst.kind = IsVopcCompareExec(inst.opcode) ? OperandKind::ExecLo : OperandKind::VccLo; inst.dst.kind = IsVopcCompareExec(inst.opcode) ? OperandKind::ExecLo : OperandKind::VccLo;
ApplyDppModifier(inst.src0, modifier); ApplyDppModifier(inst.src0, modifier);
inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u; inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u;
inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u; inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
inst.src_count = 2; inst.src_count = 2;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
uint32_t NativeVop3SourceCount(Opcode opcode) { uint32_t NativeVop3SourceCount(Opcode opcode) {
@@ -1560,8 +1522,8 @@ bool IsVopcCompareExec(Opcode opcode) {
} // namespace } // namespace
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 25u) & 0x3fu; const uint32_t opcode = (word >> 25u) & 0x3fu;
const uint32_t vdst = (word >> 17u) & 0xffu; const uint32_t vdst = (word >> 17u) & 0xffu;
@@ -1577,27 +1539,20 @@ bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::VOP2, opcode, "VOP2 opcode is not implemented"); SetUnsupported(inst, Family::VOP2, opcode, "VOP2 opcode is not implemented");
return true; return;
} }
bool modifier_handled = false; if (TryDecodeVop2Modifier(pc, code, word_index, src0, opcode, vdst, vsrc1, inst)) {
if (!TryDecodeVop2Modifier(pc, code, word_index, src0, opcode, vdst, vsrc1, inst, error, return;
&modifier_handled)) {
return false;
}
if (modifier_handled) {
return true;
} }
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
DecodeVectorGpr(vsrc1, inst.src1); DecodeVectorGpr(vsrc1, inst.src1);
if (!DecodeScalarSource(src0, pc, inst.src0, error)) { DecodeScalarSource(src0, pc, inst.src0);
return false; FinalizeVop2Instruction(code, word_index, inst);
}
return FinalizeVop2Instruction(pc, code, word_index, inst, error);
} }
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 9u) & 0xffu; const uint32_t opcode = (word >> 9u) & 0xffu;
const uint32_t src0 = word & 0x1ffu; const uint32_t src0 = word & 0x1ffu;
@@ -1612,39 +1567,29 @@ bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::VOP1, opcode, "VOP1 opcode is not implemented"); SetUnsupported(inst, Family::VOP1, opcode, "VOP1 opcode is not implemented");
return true; return;
} }
if (inst.opcode == Opcode::V_NOP) { if (inst.opcode == Opcode::V_NOP) {
inst.dst.kind = OperandKind::Null; inst.dst.kind = OperandKind::Null;
inst.src_count = 0; inst.src_count = 0;
return true; return;
} }
bool modifier_handled = false; if (TryDecodeVop1Modifier(pc, code, word_index, src0, opcode, vdst, inst)) {
if (!TryDecodeVop1Modifier(pc, code, word_index, src0, opcode, vdst, inst, error, return;
&modifier_handled)) {
return false;
}
if (modifier_handled) {
return true;
} }
const bool scalar_dst = UsesScalarDestination(inst.opcode); const bool scalar_dst = UsesScalarDestination(inst.opcode);
if (scalar_dst) { if (scalar_dst) {
if (!DecodeScalarDestination(vdst, pc, inst.dst, error)) { DecodeScalarDestination(vdst, pc, inst.dst);
return false;
}
} else { } else {
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
} }
if (!DecodeScalarSource(src0, pc, inst.src0, error)) { DecodeScalarSource(src0, pc, inst.src0);
return false;
}
inst.src_count = 1; inst.src_count = 1;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word = code[word_index]; const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 17u) & 0xffu; const uint32_t opcode = (word >> 17u) & 0xffu;
const uint32_t src0 = word & 0x1ffu; const uint32_t src0 = word & 0x1ffu;
@@ -1659,25 +1604,22 @@ bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
SetRawWords(inst, code, word_index, 1); SetRawWords(inst, code, word_index, 1);
switch (src0) { switch (src0) {
case 249u: return DecodeVopcSdwa(pc, code, word_index, opcode, vsrc1, inst, error); case 249u: DecodeVopcSdwa(pc, code, word_index, opcode, vsrc1, inst); return;
case 250u: return DecodeVopcDpp(pc, code, word_index, opcode, vsrc1, inst, error); case 250u: DecodeVopcDpp(pc, code, word_index, opcode, vsrc1, inst); return;
default: break; default: break;
} }
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented"); SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented");
return true; return;
} }
DecodeVectorGpr(vsrc1, inst.src1); DecodeVectorGpr(vsrc1, inst.src1);
if (!DecodeScalarSource(src0, pc, inst.src0, error)) { DecodeScalarSource(src0, pc, inst.src0);
return false;
}
inst.src_count = 2; inst.src_count = 2;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error) { Instruction& inst) {
const uint32_t word0 = code[word_index]; const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u]; const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = (word0 >> 16u) & 0x3ffu; const uint32_t opcode = (word0 >> 16u) & 0x3ffu;
@@ -1728,29 +1670,26 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
reason = "VOP3 mad-mix clamp/omod is not implemented"; reason = "VOP3 mad-mix clamp/omod is not implemented";
} }
SetUnsupported(inst, Family::VOP3, opcode, reason); SetUnsupported(inst, Family::VOP3, opcode, reason);
return true; return;
} }
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::VOP3, opcode, "VOP3 opcode is not implemented"); SetUnsupported(inst, Family::VOP3, opcode, "VOP3 opcode is not implemented");
return true; return;
} }
if (inst.opcode == Opcode::V_NOP) { if (inst.opcode == Opcode::V_NOP) {
inst.dst.kind = OperandKind::Null; inst.dst.kind = OperandKind::Null;
inst.src_count = 0; inst.src_count = 0;
return true; return;
} }
bool dst_ok = true;
if (compare_exec) { if (compare_exec) {
inst.dst.kind = OperandKind::ExecLo; inst.dst.kind = OperandKind::ExecLo;
} else if (scalar_dst) { } else if (scalar_dst) {
// VOP3A uses VDST for VOPC and the scalar-destination lane-read opcodes. // VOP3A uses VDST for VOPC and the scalar-destination lane-read opcodes.
dst_ok = DecodeScalarDestination(vdst, pc, inst.dst, error); DecodeScalarDestination(vdst, pc, inst.dst);
} else { } else {
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
} }
if (!dst_ok || !DecodeScalarSource(src0, pc, inst.src0, error)) { DecodeScalarSource(src0, pc, inst.src0);
return false;
}
inst.dst.clamp = native_clamp_modifier && clamp != 0u; inst.dst.clamp = native_clamp_modifier && clamp != 0u;
inst.dst.omod = native_result_modifiers ? omod : 0u; inst.dst.omod = native_result_modifiers ? omod : 0u;
if (permlane) { if (permlane) {
@@ -1758,15 +1697,13 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
inst.dst.op_sel_hi = (op_sel & 0x2u) != 0u; inst.dst.op_sel_hi = (op_sel & 0x2u) != 0u;
} }
if (vop3_vopc) { if (vop3_vopc) {
if (!DecodeScalarSource(src1, pc, inst.src1, error)) { DecodeScalarSource(src1, pc, inst.src1);
return false;
}
inst.src_count = 2; inst.src_count = 2;
if (native_source_modifiers) { if (native_source_modifiers) {
ApplyNativeVop3SourceModifiers(inst, abs, neg); ApplyNativeVop3SourceModifiers(inst, abs, neg);
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
} }
if (IsVop3EncodedVop1(opcode)) { if (IsVop3EncodedVop1(opcode)) {
inst.src_count = 1; inst.src_count = 1;
@@ -1774,45 +1711,35 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
ApplyNativeVop3SourceModifiers(inst, abs, neg); ApplyNativeVop3SourceModifiers(inst, abs, neg);
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
} }
if (carry_in_out) { if (carry_in_out) {
if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) || DecodeScalarDestination(sdst, pc, inst.dst2);
!DecodeScalarSource(src1, pc, inst.src1, error) || DecodeScalarSource(src1, pc, inst.src1);
!DecodeScalarSource(src2, pc, inst.src2, error)) { DecodeScalarSource(src2, pc, inst.src2);
return false;
}
inst.src_count = 3; inst.src_count = 3;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
} }
if (vop3b_carry_out) { if (vop3b_carry_out) {
if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) || DecodeScalarDestination(sdst, pc, inst.dst2);
!DecodeScalarSource(src1, pc, inst.src1, error)) { DecodeScalarSource(src1, pc, inst.src1);
return false;
}
inst.src_count = 2; inst.src_count = 2;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
} }
if (vop3b_mad_u64) { if (vop3b_mad_u64) {
if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) || DecodeScalarDestination(sdst, pc, inst.dst2);
!DecodeScalarSource(src1, pc, inst.src1, error) || DecodeScalarSource(src1, pc, inst.src1);
!DecodeScalarSource(src2, pc, inst.src2, error)) { DecodeScalarSource(src2, pc, inst.src2);
return false;
}
inst.src_count = 3; inst.src_count = 3;
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
} }
if (IsVop3EncodedVop2(opcode)) { if (IsVop3EncodedVop2(opcode)) {
if (!DecodeScalarSource(src1, pc, inst.src1, error)) { DecodeScalarSource(src1, pc, inst.src1);
return false;
}
if (inst.opcode == Opcode::V_CNDMASK_B32) { if (inst.opcode == Opcode::V_CNDMASK_B32) {
if (!DecodeScalarSource(src2, pc, inst.src2, error)) { DecodeScalarSource(src2, pc, inst.src2);
return false;
}
inst.src_count = 3; inst.src_count = 3;
} else { } else {
inst.src_count = 2; inst.src_count = 2;
@@ -1821,14 +1748,12 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
ApplyNativeVop3SourceModifiers(inst, abs, neg); ApplyNativeVop3SourceModifiers(inst, abs, neg);
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true; return;
}
if (!DecodeScalarSource(src1, pc, inst.src1, error)) {
return false;
} }
DecodeScalarSource(src1, pc, inst.src1);
inst.src_count = NativeVop3SourceCount(inst.opcode); inst.src_count = NativeVop3SourceCount(inst.opcode);
if (inst.src_count > 2u && !DecodeScalarSource(src2, pc, inst.src2, error)) { if (inst.src_count > 2u) {
return false; DecodeScalarSource(src2, pc, inst.src2);
} }
if (mad_mix) { if (mad_mix) {
ApplyNativeVop3TernaryModifiers(inst, op_sel, abs, neg); ApplyNativeVop3TernaryModifiers(inst, op_sel, abs, neg);
@@ -1844,11 +1769,10 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
ApplyNativeVop3SourceModifiers(inst, abs, neg); ApplyNativeVop3SourceModifiers(inst, abs, neg);
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction& inst, std::string* error) { Instruction& inst) {
const uint32_t word0 = code[word_index]; const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u]; const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = (word0 >> 16u) & 0x7fu; const uint32_t opcode = (word0 >> 16u) & 0x7fu;
@@ -1872,16 +1796,14 @@ bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
if (inst.opcode == Opcode::UNSUPPORTED) { if (inst.opcode == Opcode::UNSUPPORTED) {
SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P opcode is not implemented"); SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P opcode is not implemented");
return true; return;
} }
inst.src_count = Vop3pSourceCount(inst.opcode); inst.src_count = Vop3pSourceCount(inst.opcode);
DecodeVectorGpr(vdst, inst.dst); DecodeVectorGpr(vdst, inst.dst);
if (!DecodeScalarSource(src0, pc, inst.src0, error) || DecodeScalarSource(src0, pc, inst.src0);
!DecodeScalarSource(src1, pc, inst.src1, error)) { DecodeScalarSource(src1, pc, inst.src1);
return false; if (inst.src_count > 2u) {
} DecodeScalarSource(src2, pc, inst.src2);
if (inst.src_count > 2u && !DecodeScalarSource(src2, pc, inst.src2, error)) {
return false;
} }
if (inst.opcode == Opcode::V_FMA_F32) { if (inst.opcode == Opcode::V_FMA_F32) {
inst.dst.clamp = clamp != 0u; inst.dst.clamp = clamp != 0u;
@@ -1891,7 +1813,7 @@ bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
inst.dst.clamp = clamp != 0u; inst.dst.clamp = clamp != 0u;
} else if (clamp != 0u) { } else if (clamp != 0u) {
SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P integer clamp is not implemented"); SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P integer clamp is not implemented");
return true; return;
} }
ApplyVop3pSourceModifiers(inst, op_sel, op_sel_hi, neg, neg_hi); ApplyVop3pSourceModifiers(inst, op_sel, op_sel_hi, neg, neg_hi);
if (inst.opcode == Opcode::V_MAD_MIXLO_F16) { if (inst.opcode == Opcode::V_MAD_MIXLO_F16) {
@@ -1904,7 +1826,6 @@ bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
ApplyVop3pMixAbsModifiers(inst); ApplyVop3pMixAbsModifiers(inst);
} }
ReadLiteralOperands(code, word_index, inst); ReadLiteralOperands(code, word_index, inst);
return true;
} }
void DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
@@ -5,16 +5,16 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder { namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst, void DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
std::string* error); Instruction& inst);
bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction& inst, std::string* error); Instruction& inst);
void DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, void DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction& inst); Instruction& inst);
@@ -1,9 +1,8 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/frontend/translate/Translator.h" #include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include "graphics/shader/shader.h"
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
@@ -47,14 +46,13 @@ IR::ExportFlags Translator::AddExportInfo(const Decoder::Instruction& inst) {
return {.index = index, .pc = inst.pc}; return {.index = index, .pc = inst.pc};
} }
bool Translator::TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, void Translator::TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
uint32_t component_count) { uint32_t component_count) {
for (uint32_t component = 0; component < component_count; component++) { for (uint32_t component = 0; component < component_count; component++) {
const auto value = const auto value =
ir.Emit(IR::ValueOpcode::GetAttribute, {IR::Value(attribute), IR::Value(component)}); ir.Emit(IR::ValueOpcode::GetAttribute, {IR::Value(attribute), IR::Value(component)});
WriteOperand(OffsetOperand(inst.dst, component), value); WriteOperand(OffsetOperand(inst.dst, component), value);
} }
return true;
} }
void Translator::V_INTERP_P1_F32() {} void Translator::V_INTERP_P1_F32() {}
@@ -65,25 +63,20 @@ void Translator::V_INTERP_P2_F32(const Decoder::Instruction& inst) {
WriteOperand(inst.dst, value); WriteOperand(inst.dst, value);
} }
bool Translator::V_INTERP_MOV_F32(const Decoder::Instruction& inst, std::string* error) { void Translator::V_INTERP_MOV_F32(const Decoder::Instruction& inst) {
if (inst.src0.value >= 3u) { if (inst.src0.value >= 3u) {
return ShaderError::Fail(error, EXIT("v_interp_mov_f32 mode %u is reserved at pc 0x%08x", inst.src0.value, inst.pc);
fmt::format("v_interp_mov_f32 mode {} is reserved at pc 0x{:08x}",
inst.src0.value, inst.pc));
} }
const auto value = ir.Emit( const auto value = ir.Emit(
IR::ValueOpcode::GetInterpolationParameter, IR::ValueOpcode::GetInterpolationParameter,
{IR::Value(inst.src1.value), IR::Value(inst.src2.value), IR::Value(inst.src0.value)}); {IR::Value(inst.src1.value), IR::Value(inst.src2.value), IR::Value(inst.src0.value)});
WriteOperand(inst.dst, value); WriteOperand(inst.dst, value);
return true;
} }
bool Translator::EXP(const Decoder::Instruction& inst, std::string* error) { void Translator::EXP(const Decoder::Instruction& inst) {
uint32_t index = 0; uint32_t index = 0;
if (ExportTargetKindFromTarget(inst.exp.target, index) == IR::ExportTargetKind::Unknown) { if (ExportTargetKindFromTarget(inst.exp.target, index) == IR::ExportTargetKind::Unknown) {
return ShaderError::Fail(error, EXIT("unsupported EXP target 0x%02x at pc 0x%08x", inst.exp.target, inst.pc);
fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}",
inst.exp.target, inst.pc));
} }
std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u), std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u),
IR::Value(0u)}; IR::Value(0u)};
@@ -93,14 +86,13 @@ bool Translator::EXP(const Decoder::Instruction& inst, std::string* error) {
const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
{components[0], components[1], components[2], components[3]}); {components[0], components[1], components[2], components[3]});
ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst)); ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
return true;
} }
bool Translator::EmitInterpolation(const Decoder::Instruction& inst, std::string* error) { bool Translator::EmitInterpolation(const Decoder::Instruction& inst) {
switch (inst.opcode) { switch (inst.opcode) {
case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return true; case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return true;
case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return true; case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return true;
case Decoder::Opcode::V_INTERP_MOV_F32: return V_INTERP_MOV_F32(inst, error); case Decoder::Opcode::V_INTERP_MOV_F32: V_INTERP_MOV_F32(inst); return true;
default: return false; default: return false;
} }
} }
@@ -1,3 +1,4 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/frontend/translate/Translator.h" #include "graphics/shader/recompiler/frontend/translate/Translator.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
@@ -163,12 +164,12 @@ void Translator::S_INST_PREFETCH() {
ir.Emit(IR::ValueOpcode::InstPrefetch); ir.Emit(IR::ValueOpcode::InstPrefetch);
} }
bool Translator::S_GETPC_B64(const Decoder::Instruction& inst, std::string* error) { void Translator::S_GETPC_B64(const Decoder::Instruction& inst) {
const auto base = IR::U64(ir.Emit(IR::ValueOpcode::GetShaderBase)); const auto base = IR::U64(ir.Emit(IR::ValueOpcode::GetShaderBase));
const auto address = IR::U64( const auto address = IR::U64(
ir.Emit(IR::ValueOpcode::IAdd64, {base, IR::Value(static_cast<uint64_t>(inst.pc) + 4u)})); ir.Emit(IR::ValueOpcode::IAdd64, {base, IR::Value(static_cast<uint64_t>(inst.pc) + 4u)}));
if (inst.dst.kind == Decoder::OperandKind::Null) { if (inst.dst.kind == Decoder::OperandKind::Null) {
return true; return;
} }
auto high = PlainOperand(inst.dst); auto high = PlainOperand(inst.dst);
switch (inst.dst.kind) { switch (inst.dst.kind) {
@@ -179,25 +180,19 @@ bool Translator::S_GETPC_B64(const Decoder::Instruction& inst, std::string* erro
high.kind = Decoder::OperandKind::VccLo; high.kind = Decoder::OperandKind::VccLo;
high.reg = 0u; high.reg = 0u;
} else { } else {
if (error != nullptr) { EXIT("S_GETPC_B64 destination does not name a valid scalar pair at pc 0x%08x",
*error = "S_GETPC_B64 destination does not name a valid scalar pair"; inst.pc);
}
return false;
} }
break; break;
case Decoder::OperandKind::VccLo: high.kind = Decoder::OperandKind::VccHi; break; case Decoder::OperandKind::VccLo: high.kind = Decoder::OperandKind::VccHi; break;
case Decoder::OperandKind::M0: high.kind = Decoder::OperandKind::Null; break; case Decoder::OperandKind::M0: high.kind = Decoder::OperandKind::Null; break;
case Decoder::OperandKind::ExecLo: high.kind = Decoder::OperandKind::ExecHi; break; case Decoder::OperandKind::ExecLo: high.kind = Decoder::OperandKind::ExecHi; break;
default: default:
if (error != nullptr) { EXIT("S_GETPC_B64 destination does not name a valid scalar pair at pc 0x%08x", inst.pc);
*error = "S_GETPC_B64 destination does not name a valid scalar pair";
}
return false;
} }
const auto words = ExtractU64(address); const auto words = ExtractU64(address);
WriteOperand(inst.dst, words[0]); WriteOperand(inst.dst, words[0]);
WriteOperand(high, words[1]); WriteOperand(high, words[1]);
return true;
} }
void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) { void Translator::S_CSELECT_B32(const Decoder::Instruction& inst) {
@@ -213,8 +208,8 @@ void Translator::S_CSELECT_B64(const Decoder::Instruction& inst) {
} }
void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) { void Translator::MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers) {
if (inst.src0.kind == Decoder::OperandKind::Sgpr && inst.dst.kind == Decoder::OperandKind::Sgpr && if (inst.src0.kind == Decoder::OperandKind::Sgpr &&
inst.src0.reg == inst.dst.reg) { inst.dst.kind == Decoder::OperandKind::Sgpr && inst.src0.reg == inst.dst.reg) {
return; return;
} }
if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) { if (IsExecOrVcc(inst.src0) && IsExecOrVcc(inst.dst)) {
@@ -242,21 +237,15 @@ void Translator::S_WQM_B64(const Decoder::Instruction& inst) {
ir.SetScc(ScalarU64NonZero(result)); ir.SetScc(ScalarU64NonZero(result));
} }
bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error) { void Translator::V_MOVRELS_B32(const Decoder::Instruction& inst) {
if (inst.dst.kind != Decoder::OperandKind::Vgpr || if (inst.dst.kind != Decoder::OperandKind::Vgpr ||
inst.src0.kind != Decoder::OperandKind::Vgpr) { inst.src0.kind != Decoder::OperandKind::Vgpr) {
if (error != nullptr) { EXIT("V_MOVRELS_B32 requires VGPR source and destination at pc 0x%08x", inst.pc);
*error = "V_MOVRELS_B32 requires VGPR source and destination";
}
return false;
} }
if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp ||
inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute || inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute ||
inst.src0.dpp) { inst.src0.dpp) {
if (error != nullptr) { EXIT("V_MOVRELS_B32 modifiers are not implemented at pc 0x%08x", inst.pc);
*error = "V_MOVRELS_B32 modifiers are not implemented";
}
return false;
} }
const auto base = inst.src0.reg; const auto base = inst.src0.reg;
const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)), const auto m0 = ir.BitwiseAnd(ReadU32(ConditionOperand(Decoder::OperandKind::M0)),
@@ -267,23 +256,16 @@ bool Translator::V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* er
selected = ir.Select(match, ir.GetVectorReg(static_cast<IR::VectorReg>(index)), selected); selected = ir.Select(match, ir.GetVectorReg(static_cast<IR::VectorReg>(index)), selected);
} }
WriteOperand(DestinationOperand(inst), selected); WriteOperand(DestinationOperand(inst), selected);
return true;
} }
bool Translator::V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* error) { void Translator::V_MOVRELD_B32(const Decoder::Instruction& inst) {
if (inst.dst.kind != Decoder::OperandKind::Vgpr) { if (inst.dst.kind != Decoder::OperandKind::Vgpr) {
if (error != nullptr) { EXIT("V_MOVRELD_B32 requires VGPR destination at pc 0x%08x", inst.pc);
*error = "V_MOVRELD_B32 requires VGPR destination";
}
return false;
} }
if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp || if (inst.dst.sdwa_sel != 6u || inst.dst.omod != 0u || inst.dst.clamp ||
inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute || inst.src0.sdwa_sel != 6u || inst.src0.sdwa_sext || inst.src0.negate || inst.src0.absolute ||
inst.src0.dpp) { inst.src0.dpp) {
if (error != nullptr) { EXIT("V_MOVRELD_B32 modifiers are not implemented at pc 0x%08x", inst.pc);
*error = "V_MOVRELD_B32 modifiers are not implemented";
}
return false;
} }
const auto base = inst.dst.reg; const auto base = inst.dst.reg;
const auto value = ReadU32(inst.src0); const auto value = ReadU32(inst.src0);
@@ -295,7 +277,6 @@ bool Translator::V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* er
const auto write = ir.LogicalAnd(ir.GetExec(), match); const auto write = ir.LogicalAnd(ir.GetExec(), match);
ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg))); ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg)));
} }
return true;
} }
void Translator::V_READFIRSTLANE_B32(const Decoder::Instruction& inst) { void Translator::V_READFIRSTLANE_B32(const Decoder::Instruction& inst) {
@@ -1,23 +1,18 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/frontend/translate/Translator.h" #include "graphics/shader/recompiler/frontend/translate/Translator.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
bool Translator::TranslateInstruction(const Decoder::Instruction& inst, std::string* error) { void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
current_opcode = inst.opcode; current_opcode = inst.opcode;
current_pc = inst.pc; current_pc = inst.pc;
switch (inst.opcode) { switch (inst.opcode) {
case Decoder::Opcode::UNKNOWN: case Decoder::Opcode::UNKNOWN:
case Decoder::Opcode::COUNT: case Decoder::Opcode::COUNT:
if (error != nullptr) { EXIT("decoded opcode has no IR translation at pc 0x%08x", inst.pc);
*error = "decoded opcode has no IR translation";
}
return false;
case Decoder::Opcode::UNSUPPORTED: case Decoder::Opcode::UNSUPPORTED:
if (error != nullptr) { EXIT("unsupported decoded instruction: %s", Decoder::InstructionToString(inst).c_str());
*error = "unsupported decoded instruction: " + Decoder::InstructionToString(inst);
}
return false;
default: break; default: break;
} }
@@ -27,27 +22,30 @@ bool Translator::TranslateInstruction(const Decoder::Instruction& inst, std::str
case Decoder::Family::SOP2: case Decoder::Family::SOP2:
case Decoder::Family::SOPK: case Decoder::Family::SOPK:
case Decoder::Family::SOPC: case Decoder::Family::SOPC:
case Decoder::Family::SOPP: translated = EmitScalar(inst, error); break; case Decoder::Family::SOPP: translated = EmitScalar(inst); break;
case Decoder::Family::VOP1: case Decoder::Family::VOP1:
case Decoder::Family::VOP2: case Decoder::Family::VOP2:
case Decoder::Family::VOP3: case Decoder::Family::VOP3:
case Decoder::Family::VOP3P: case Decoder::Family::VOP3P:
case Decoder::Family::VOPC: translated = EmitVector(inst, error); break; case Decoder::Family::VOPC: translated = EmitVector(inst); break;
case Decoder::Family::SMEM: case Decoder::Family::SMEM:
case Decoder::Family::MUBUF: case Decoder::Family::MUBUF:
case Decoder::Family::MTBUF: case Decoder::Family::MTBUF:
case Decoder::Family::FLAT: case Decoder::Family::FLAT:
case Decoder::Family::DS: case Decoder::Family::DS:
case Decoder::Family::MIMG: translated = EmitMemory(inst, error); break; case Decoder::Family::MIMG: translated = EmitMemory(inst); break;
case Decoder::Family::VINTRP: translated = EmitInterpolation(inst, error); break; case Decoder::Family::VINTRP: translated = EmitInterpolation(inst); break;
case Decoder::Family::EXP: translated = EXP(inst, error); break; case Decoder::Family::EXP:
EXP(inst);
translated = true;
break;
default: break; default: break;
} }
if (!translated && error != nullptr && error->empty()) { if (!translated) {
*error = "decoded opcode has no IR translation"; EXIT("opcode %s at pc 0x%08x has no IR translation",
Decoder::InstructionToString(inst).c_str(), inst.pc);
} }
return translated;
} }
} // namespace Libs::Graphics::ShaderRecompiler::Frontend } // namespace Libs::Graphics::ShaderRecompiler::Frontend
@@ -1,9 +1,7 @@
#include "common/magicEnum.h"
#include "graphics/shader/recompiler/frontend/translate/Translator.h" #include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
@@ -890,7 +888,7 @@ bool Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
return true; return true;
} }
bool Translator::EmitMemory(const Decoder::Instruction& inst, std::string* error) { bool Translator::EmitMemory(const Decoder::Instruction& inst) {
switch (inst.opcode) { switch (inst.opcode) {
case Decoder::Opcode::S_LOAD_DWORD: case Decoder::Opcode::S_LOAD_DWORD:
case Decoder::Opcode::S_LOAD_DWORDX2: case Decoder::Opcode::S_LOAD_DWORDX2:
@@ -1080,12 +1078,7 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst, std::string* error
case Decoder::Opcode::DS_WRITE_B64: case Decoder::Opcode::DS_WRITE_B64:
case Decoder::Opcode::DS_WRITE_B96: case Decoder::Opcode::DS_WRITE_B96:
case Decoder::Opcode::DS_WRITE_B128: return DS_WRITE(inst); case Decoder::Opcode::DS_WRITE_B128: return DS_WRITE(inst);
default: default: return false;
if (error != nullptr) {
*error = fmt::format("memory-family opcode has no specialized IR translation: {}",
magic_enum::enum_name(inst.opcode));
}
return false;
} }
} }
@@ -2,14 +2,14 @@
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
bool Translator::EmitScalar(const Decoder::Instruction& inst, std::string* error) { bool Translator::EmitScalar(const Decoder::Instruction& inst) {
using O = Decoder::Opcode; using O = Decoder::Opcode;
switch (inst.opcode) { switch (inst.opcode) {
case O::S_MOV_B32: case O::S_MOV_B32:
case O::S_MOVK_I32: MOV_B32(inst, false); return true; case O::S_MOVK_I32: MOV_B32(inst, false); return true;
case O::S_MOV_B64: S_MOV_B64(inst); return true; case O::S_MOV_B64: S_MOV_B64(inst); return true;
case O::S_WQM_B64: S_WQM_B64(inst); return true; case O::S_WQM_B64: S_WQM_B64(inst); return true;
case O::S_GETPC_B64: return S_GETPC_B64(inst, error); case O::S_GETPC_B64: S_GETPC_B64(inst); return true;
case O::S_SETPC_B64: return true; case O::S_SETPC_B64: return true;
case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true; case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true;
case O::S_CSELECT_B64: S_CSELECT_B64(inst); return true; case O::S_CSELECT_B64: S_CSELECT_B64(inst); return true;
@@ -1,16 +1,14 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/frontend/translate/Translator.h" #include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include "graphics/shader/shader.h" #include "graphics/shader/shader.h"
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <fmt/format.h>
#include <unordered_map> #include <unordered_map>
#include <utility> #include <utility>
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
using ShaderError::Fail;
Decoder::Operand Translator::SourceAt(const Decoder::Instruction& inst, uint32_t index) { Decoder::Operand Translator::SourceAt(const Decoder::Instruction& inst, uint32_t index) {
switch (index) { switch (index) {
case 0: return inst.src0; case 0: return inst.src0;
@@ -396,8 +394,7 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
} }
} }
if (type == IR::Type::U16) { if (type == IR::Type::U16) {
Write16Bits(operand, Write16Bits(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {IR::U16(value)})));
IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {IR::U16(value)})));
return; return;
} }
if (type == IR::Type::F16) { if (type == IR::Type::F16) {
@@ -811,11 +808,10 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value)); WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
} }
bool Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) {
std::string* error) {
if (source.terminator.goto_value >= 0) { if (source.terminator.goto_value >= 0) {
if (source.terminator.goto_variable == UINT32_MAX) { if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error, fmt::format("block {} sets an invalid goto variable", source.id)); EXIT("block %u sets an invalid goto variable", source.id);
} }
ir.SetGotoVariable(source.terminator.goto_variable, ir.SetGotoVariable(source.terminator.goto_variable,
IR::U1(IR::Value(source.terminator.goto_value != 0))); IR::U1(IR::Value(source.terminator.goto_value != 0)));
@@ -827,13 +823,13 @@ bool Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo
info.indirect_target = info.indirect_target =
ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr)); ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr));
} else { } else {
return Fail(error, fmt::format("block {} has no indirect branch selector", source.id)); EXIT("block %u has no indirect branch selector", source.id);
} }
ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target}); ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target});
return true; return;
} }
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) { if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
return true; return;
} }
// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive // EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
// invocations leave the region without reconstructing a host-subgroup mask. // invocations leave the region without reconstructing a host-subgroup mask.
@@ -848,17 +844,15 @@ bool Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break; case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
case CFG::BranchCondition::GotoVariable: case CFG::BranchCondition::GotoVariable:
if (source.terminator.goto_variable == UINT32_MAX) { if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error, EXIT("block %u reads an invalid goto variable", source.id);
fmt::format("block {} reads an invalid goto variable", source.id));
} }
condition = ir.GetGotoVariable(source.terminator.goto_variable); condition = ir.GetGotoVariable(source.terminator.goto_variable);
break; break;
case CFG::BranchCondition::Unknown: case CFG::BranchCondition::Unknown:
return Fail(error, fmt::format("block {} has an unknown branch condition", source.id)); EXIT("block %u has an unknown branch condition", source.id);
} }
info.condition = condition; info.condition = condition;
ir.Emit(IR::ValueOpcode::Reference, {condition}); ir.Emit(IR::ValueOpcode::Reference, {condition});
return true;
} }
namespace { namespace {
@@ -980,47 +974,42 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
} }
} }
bool ValidateTranslateOptions(const TranslateOptions& options, std::string* error) { void ValidateTranslateOptions(const TranslateOptions& options) {
if (options.wave_size != 32u && options.wave_size != 64u) { if (options.wave_size != 32u && options.wave_size != 64u) {
return Fail(error, "shader translation requires wave32 or wave64"); EXIT("shader translation requires wave32 or wave64, got %u", options.wave_size);
} }
switch (options.stage) { switch (options.stage) {
case ShaderType::Vertex: case ShaderType::Vertex:
return options.vertex != nullptr || if (options.vertex == nullptr) {
Fail(error, "vertex shader translation has no vertex input metadata"); EXIT("vertex shader translation has no vertex input metadata");
}
return;
case ShaderType::Pixel: case ShaderType::Pixel:
return options.pixel != nullptr || if (options.pixel == nullptr) {
Fail(error, "pixel shader translation has no pixel input metadata"); EXIT("pixel shader translation has no pixel input metadata");
}
return;
case ShaderType::Compute: case ShaderType::Compute:
return options.compute != nullptr || if (options.compute == nullptr) {
Fail(error, "compute shader translation has no compute input metadata"); EXIT("compute shader translation has no compute input metadata");
default: return Fail(error, "shader translation has an unsupported stage"); }
return;
default:
EXIT("shader translation has unsupported stage %u",
static_cast<uint32_t>(options.stage));
} }
} }
} // namespace } // namespace
bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
const TranslateOptions& options, IR::Program& result, std::string* error) { const TranslateOptions& options) {
result = {}; ValidateTranslateOptions(options);
struct ResetOnFailure {
IR::Program& program;
bool complete = false;
~ResetOnFailure() {
if (!complete) {
program = {};
}
}
} reset_on_failure {result};
if (!ValidateTranslateOptions(options, error)) {
return false;
}
if (cfg.blocks.empty()) { if (cfg.blocks.empty()) {
return Fail(error, "cannot translate an empty CFG"); EXIT("cannot translate an empty CFG");
} }
IR::Program result;
result.stage = options.stage; result.stage = options.stage;
result.wave_size = options.wave_size; result.wave_size = options.wave_size;
result.shader_hash = options.shader_hash; result.shader_hash = options.shader_hash;
@@ -1038,8 +1027,8 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
for (const auto& cfg_block: cfg.blocks) { for (const auto& cfg_block: cfg.blocks) {
for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) { for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
if (index >= decoded.instructions.size()) { if (index >= decoded.instructions.size()) {
return Fail(error, EXIT("CFG block %u references instruction %u outside decoded program of size %zu",
"CFG block references an instruction outside the decoded program"); cfg_block.id, index, decoded.instructions.size());
} }
const auto& instruction = decoded.instructions[index]; const auto& instruction = decoded.instructions[index];
if (IsCodeTableLoad(cfg, instruction.pc)) { if (IsCodeTableLoad(cfg, instruction.pc)) {
@@ -1056,7 +1045,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
return block.id; return block.id;
})->id; })->id;
if (max_id == UINT32_MAX) { if (max_id == UINT32_MAX) {
return Fail(error, "cannot allocate a typed entry block id"); EXIT("cannot allocate a typed entry block id");
} }
CFG::Terminator terminator; CFG::Terminator terminator;
terminator.kind = CFG::TerminatorKind::Branch; terminator.kind = CFG::TerminatorKind::Branch;
@@ -1070,7 +1059,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
block_indices.reserve(cfg.blocks.size()); block_indices.reserve(cfg.blocks.size());
for (const auto& source_block: cfg.blocks) { for (const auto& source_block: cfg.blocks) {
if (!block_indices.emplace(source_block.id, result.blocks.size()).second) { if (!block_indices.emplace(source_block.id, result.blocks.size()).second) {
return Fail(error, fmt::format("CFG contains duplicate block id {}", source_block.id)); EXIT("CFG contains duplicate block id %u", source_block.id);
} }
result.block_storage.push_back(std::make_unique<IR::Block>()); result.block_storage.push_back(std::make_unique<IR::Block>());
result.blocks.push_back(result.block_storage.back().get()); result.blocks.push_back(result.block_storage.back().get());
@@ -1082,8 +1071,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
for (const auto successor: source_block.successors) { for (const auto successor: source_block.successors) {
const auto target = block_indices.find(successor); const auto target = block_indices.find(successor);
if (target == block_indices.end()) { if (target == block_indices.end()) {
return Fail(error, fmt::format("CFG block {} has unknown successor {}", EXIT("CFG block %u has unknown successor %u", source_block.id, successor);
source_block.id, successor));
} }
result.blocks[source_index]->AddBranch(result.blocks[target->second]); result.blocks[source_index]->AddBranch(result.blocks[target->second]);
} }
@@ -1182,7 +1170,7 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
} }
} }
for (const auto& cfg_block: cfg.blocks) { for (const auto& cfg_block: cfg.blocks) {
const auto typed_index = block_indices.at(cfg_block.id); const auto typed_index = block_indices.at(cfg_block.id);
Translator translator(result, result.blocks[typed_index], vector_limit, options.wave_size); Translator translator(result, result.blocks[typed_index], vector_limit, options.wave_size);
for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) { for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
const auto& instruction = decoded.instructions[index]; const auto& instruction = decoded.instructions[index];
@@ -1198,44 +1186,27 @@ bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
instruction.data_dwords == embedded->components && instruction.data_dwords == embedded->components &&
instruction.dst.kind == Decoder::OperandKind::Vgpr) { instruction.dst.kind == Decoder::OperandKind::Vgpr) {
if (options.vertex == nullptr) { if (options.vertex == nullptr) {
return Fail(error, "embedded vertex fetch plan has no vertex input metadata"); EXIT("embedded vertex fetch plan has no vertex input metadata");
} }
const auto resource = ResolveEmbeddedFetchResource(*options.vertex, *embedded); const auto resource = ResolveEmbeddedFetchResource(*options.vertex, *embedded);
if (resource < 0 || resource >= options.vertex->resources_num) { if (resource < 0 || resource >= options.vertex->resources_num) {
return Fail( EXIT("embedded vertex fetch at 0x%08x has no resource for attribute %d",
error, instruction.pc, embedded->attrib_id);
fmt::format(
"embedded vertex fetch at 0x{:08x} has no resource for attribute {}",
instruction.pc, embedded->attrib_id));
} }
result.info.vertex_fetch_components[static_cast<size_t>(resource)] = result.info.vertex_fetch_components[static_cast<size_t>(resource)] =
static_cast<uint8_t>(std::max<uint32_t>( static_cast<uint8_t>(std::max<uint32_t>(
result.info.vertex_fetch_components[static_cast<size_t>(resource)], result.info.vertex_fetch_components[static_cast<size_t>(resource)],
embedded->components)); embedded->components));
if (!translator.TranslateEmbeddedFetch(instruction, static_cast<uint32_t>(resource), translator.TranslateEmbeddedFetch(instruction, static_cast<uint32_t>(resource),
embedded->components)) { embedded->components);
return false;
}
continue; continue;
} }
if (!translator.TranslateInstruction(instruction, error)) { translator.TranslateInstruction(instruction);
if (error == nullptr || error->empty()) {
return Fail(error, fmt::format("opcode {} at 0x{:08x} has no IR translation",
magic_enum::enum_name(instruction.opcode),
instruction.pc));
}
return false;
}
}
if (!translator.AddBranchCondition(cfg_block, result.block_info[typed_index], error)) {
return false;
} }
translator.AddBranchCondition(cfg_block, result.block_info[typed_index]);
} }
if (!IR::ValidateProgram(result, false, error)) { IR::ValidateProgram(result, false);
return false; return result;
}
reset_on_failure.complete = true;
return true;
} }
} // namespace Libs::Graphics::ShaderRecompiler::Frontend } // namespace Libs::Graphics::ShaderRecompiler::Frontend
@@ -38,7 +38,7 @@ struct TranslateOptions {
const EmbeddedFetchPlan* embedded_fetch = nullptr; const EmbeddedFetchPlan* embedded_fetch = nullptr;
}; };
bool TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph& cfg,
const TranslateOptions& options, IR::Program& result, std::string* error); const TranslateOptions& options);
} // namespace Libs::Graphics::ShaderRecompiler::Frontend } // namespace Libs::Graphics::ShaderRecompiler::Frontend
@@ -13,10 +13,10 @@ public:
: program(program), ir(block), current_vector_limit(vector_limit), : program(program), ir(block), current_vector_limit(vector_limit),
current_wave_size(wave_size) {} current_wave_size(wave_size) {}
bool TranslateInstruction(const Decoder::Instruction& inst, std::string* error); void TranslateInstruction(const Decoder::Instruction& inst);
bool TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute, void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
uint32_t component_count); uint32_t component_count);
bool AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info, std::string* error); void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info);
private: private:
struct ScalarU64 { struct ScalarU64 {
@@ -194,26 +194,26 @@ private:
bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign); bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode, bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary); IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary);
bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type, bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
bool reverse, bool mask_shift_count, bool update_scc); bool reverse, bool mask_shift_count, bool update_scc);
bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
bool negate_rhs, bool negate_result, bool update_scc); bool negate_rhs, bool negate_result, bool update_scc);
bool V_AND_OR_B32(const Decoder::Instruction& inst); bool V_AND_OR_B32(const Decoder::Instruction& inst);
bool V_OR3_B32(const Decoder::Instruction& inst); bool V_OR3_B32(const Decoder::Instruction& inst);
bool V_XOR3_B32(const Decoder::Instruction& inst); bool V_XOR3_B32(const Decoder::Instruction& inst);
bool S_FF1_I32_B64(const Decoder::Instruction& inst); bool S_FF1_I32_B64(const Decoder::Instruction& inst);
bool V_FFBH_32(const Decoder::Instruction& inst, bool sign); bool V_FFBH_32(const Decoder::Instruction& inst, bool sign);
bool S_FLBIT_I32_B64(const Decoder::Instruction& inst); bool S_FLBIT_I32_B64(const Decoder::Instruction& inst);
bool Integer24(const Decoder::Instruction& inst, bool sign, bool addend); bool Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
bool V_MAD_U64_U32(const Decoder::Instruction& inst); bool V_MAD_U64_U32(const Decoder::Instruction& inst);
bool V_SAD_U32(const Decoder::Instruction& inst); bool V_SAD_U32(const Decoder::Instruction& inst);
bool V_ADD3_U32(const Decoder::Instruction& inst); bool V_ADD3_U32(const Decoder::Instruction& inst);
bool S_BITSET_B32(const Decoder::Instruction& inst, bool set); bool S_BITSET_B32(const Decoder::Instruction& inst, bool set);
bool V_BCNT_U32_B32(const Decoder::Instruction& inst); bool V_BCNT_U32_B32(const Decoder::Instruction& inst);
bool V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low); bool V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
bool S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst); bool S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
bool S_QUADMASK_B64(const Decoder::Instruction& inst); bool S_QUADMASK_B64(const Decoder::Instruction& inst);
bool BFM_B32(const Decoder::Instruction& inst); bool BFM_B32(const Decoder::Instruction& inst);
IR::U32 RightMask32(IR::U32 count); IR::U32 RightMask32(IR::U32 count);
IR::U64 RightMask64(IR::U32 count); IR::U64 RightMask64(IR::U32 count);
bool S_BFM_B64(const Decoder::Instruction& inst); bool S_BFM_B64(const Decoder::Instruction& inst);
@@ -246,27 +246,27 @@ private:
void S_SENDMSG(); void S_SENDMSG();
void S_TTRACEDATA(); void S_TTRACEDATA();
void S_INST_PREFETCH(); void S_INST_PREFETCH();
bool S_GETPC_B64(const Decoder::Instruction& inst, std::string* error); void S_GETPC_B64(const Decoder::Instruction& inst);
void S_CSELECT_B32(const Decoder::Instruction& inst); void S_CSELECT_B32(const Decoder::Instruction& inst);
void S_CSELECT_B64(const Decoder::Instruction& inst); void S_CSELECT_B64(const Decoder::Instruction& inst);
void MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers); void MOV_B32(const Decoder::Instruction& inst, bool apply_float_modifiers);
void S_MOV_B64(const Decoder::Instruction& inst); void S_MOV_B64(const Decoder::Instruction& inst);
void S_WQM_B64(const Decoder::Instruction& inst); void S_WQM_B64(const Decoder::Instruction& inst);
bool V_MOVRELS_B32(const Decoder::Instruction& inst, std::string* error); void V_MOVRELS_B32(const Decoder::Instruction& inst);
bool V_MOVRELD_B32(const Decoder::Instruction& inst, std::string* error); void V_MOVRELD_B32(const Decoder::Instruction& inst);
void V_READFIRSTLANE_B32(const Decoder::Instruction& inst); void V_READFIRSTLANE_B32(const Decoder::Instruction& inst);
void V_READLANE_B32(const Decoder::Instruction& inst); void V_READLANE_B32(const Decoder::Instruction& inst);
void V_WRITELANE_B32(const Decoder::Instruction& inst); void V_WRITELANE_B32(const Decoder::Instruction& inst);
void V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16); void V_PERMLANE16_B32(const Decoder::Instruction& inst, bool x16);
void V_INTERP_P1_F32(); void V_INTERP_P1_F32();
void V_INTERP_P2_F32(const Decoder::Instruction& inst); void V_INTERP_P2_F32(const Decoder::Instruction& inst);
bool V_INTERP_MOV_F32(const Decoder::Instruction& inst, std::string* error); void V_INTERP_MOV_F32(const Decoder::Instruction& inst);
bool EXP(const Decoder::Instruction& inst, std::string* error); void EXP(const Decoder::Instruction& inst);
bool EmitScalar(const Decoder::Instruction& inst, std::string* error); bool EmitScalar(const Decoder::Instruction& inst);
bool EmitVector(const Decoder::Instruction& inst, std::string* error); bool EmitVector(const Decoder::Instruction& inst);
bool EmitInterpolation(const Decoder::Instruction& inst, std::string* error); bool EmitInterpolation(const Decoder::Instruction& inst);
bool EmitMemory(const Decoder::Instruction& inst, std::string* error); bool EmitMemory(const Decoder::Instruction& inst);
IR::Program& program; IR::Program& program;
IR::IREmitter ir; IR::IREmitter ir;
@@ -2,7 +2,7 @@
namespace Libs::Graphics::ShaderRecompiler::Frontend { namespace Libs::Graphics::ShaderRecompiler::Frontend {
bool Translator::EmitVector(const Decoder::Instruction& inst, std::string* error) { bool Translator::EmitVector(const Decoder::Instruction& inst) {
using O = Decoder::Opcode; using O = Decoder::Opcode;
switch (inst.opcode) { switch (inst.opcode) {
case O::V_NOP: return true; case O::V_NOP: return true;
@@ -13,8 +13,8 @@ bool Translator::EmitVector(const Decoder::Instruction& inst, std::string* error
case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true); return true; case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true); return true;
case O::V_MOV_B32: MOV_B32(inst, true); return true; case O::V_MOV_B32: MOV_B32(inst, true); return true;
case O::V_MOVRELS_B32: return V_MOVRELS_B32(inst, error); case O::V_MOVRELS_B32: V_MOVRELS_B32(inst); return true;
case O::V_MOVRELD_B32: return V_MOVRELD_B32(inst, error); case O::V_MOVRELD_B32: V_MOVRELD_B32(inst); return true;
case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return true; case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return true;
case O::V_READLANE_B32: V_READLANE_B32(inst); return true; case O::V_READLANE_B32: V_READLANE_B32(inst); return true;
case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return true; case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return true;
+90 -94
View File
@@ -1,7 +1,6 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include <fmt/format.h> #include <fmt/format.h>
#include <map> #include <map>
#include <new> #include <new>
@@ -11,10 +10,13 @@
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
using ShaderError::Fail;
namespace { namespace {
[[noreturn]] void Fail(std::string_view message) {
EXIT("shader IR validation failed: %s", std::string(message).c_str());
std::abort();
}
bool IsRegisterStatePseudo(ValueOpcode opcode) { bool IsRegisterStatePseudo(ValueOpcode opcode) {
switch (opcode) { switch (opcode) {
case ValueOpcode::GetThreadBitScalarRegister: case ValueOpcode::GetThreadBitScalarRegister:
@@ -153,10 +155,10 @@ Value ResolveInvariantPhi(const Program& program, Value value) {
return invariant; return invariant;
} }
bool ValidateProgram(const Program& program, bool require_ssa, std::string* error) { void ValidateProgram(const Program& program, bool require_ssa) {
if (program.blocks.size() != program.block_info.size() || if (program.blocks.size() != program.block_info.size() ||
program.blocks.size() != program.block_storage.size()) { program.blocks.size() != program.block_storage.size()) {
return Fail(error, "value IR block storage is inconsistent"); return Fail("value IR block storage is inconsistent");
} }
std::unordered_set<const Block*> blocks; std::unordered_set<const Block*> blocks;
std::unordered_map<const Block*, size_t> block_indices; std::unordered_map<const Block*, size_t> block_indices;
@@ -166,26 +168,26 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
const auto* block = program.blocks[block_index]; const auto* block = program.blocks[block_index];
if (block == nullptr || program.block_storage[block_index] == nullptr || if (block == nullptr || program.block_storage[block_index] == nullptr ||
block != program.block_storage[block_index].get()) { block != program.block_storage[block_index].get()) {
return Fail(error, "value IR block pointer is inconsistent"); return Fail("value IR block pointer is inconsistent");
} }
if (!blocks.insert(block).second) { if (!blocks.insert(block).second) {
return Fail(error, "value IR block pointer is duplicated"); return Fail("value IR block pointer is duplicated");
} }
block_indices.emplace(block, block_index); block_indices.emplace(block, block_index);
if (program.block_info[block_index].id == UINT32_MAX) { if (program.block_info[block_index].id == UINT32_MAX) {
return Fail(error, "value IR block uses the reserved exit id"); return Fail("value IR block uses the reserved exit id");
} }
if (!blocks_by_id.emplace(program.block_info[block_index].id, block).second) { if (!blocks_by_id.emplace(program.block_info[block_index].id, block).second) {
return Fail(error, "value IR block id is duplicated"); return Fail("value IR block id is duplicated");
} }
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (!instructions.insert(&inst).second) { if (!instructions.insert(&inst).second) {
return Fail(error, "value IR instruction is duplicated"); return Fail("value IR instruction is duplicated");
} }
} }
} }
if (!program.blocks.empty() && !program.blocks.front()->ImmPredecessors().empty()) { if (!program.blocks.empty() && !program.blocks.front()->ImmPredecessors().empty()) {
return Fail(error, "value IR entry block has a predecessor"); return Fail("value IR entry block has a predecessor");
} }
for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) {
@@ -193,28 +195,28 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
std::unordered_set<const Block*> predecessors; std::unordered_set<const Block*> predecessors;
for (const auto* predecessor: block->ImmPredecessors()) { for (const auto* predecessor: block->ImmPredecessors()) {
if (predecessor == nullptr || !blocks.contains(predecessor)) { if (predecessor == nullptr || !blocks.contains(predecessor)) {
return Fail(error, "value IR block has a foreign predecessor"); return Fail("value IR block has a foreign predecessor");
} }
if (!predecessors.insert(predecessor).second) { if (!predecessors.insert(predecessor).second) {
return Fail(error, "value IR block predecessor is duplicated"); return Fail("value IR block predecessor is duplicated");
} }
if (std::ranges::find(predecessor->ImmSuccessors(), block) == if (std::ranges::find(predecessor->ImmSuccessors(), block) ==
predecessor->ImmSuccessors().end()) { predecessor->ImmSuccessors().end()) {
return Fail(error, "value IR predecessor edge is not reciprocal"); return Fail("value IR predecessor edge is not reciprocal");
} }
} }
std::unordered_set<const Block*> successors; std::unordered_set<const Block*> successors;
for (const auto* successor: block->ImmSuccessors()) { for (const auto* successor: block->ImmSuccessors()) {
if (successor == nullptr || !blocks.contains(successor)) { if (successor == nullptr || !blocks.contains(successor)) {
return Fail(error, "value IR block has a foreign successor"); return Fail("value IR block has a foreign successor");
} }
if (!successors.insert(successor).second) { if (!successors.insert(successor).second) {
return Fail(error, "value IR block successor is duplicated"); return Fail("value IR block successor is duplicated");
} }
if (std::ranges::find(successor->ImmPredecessors(), block) == if (std::ranges::find(successor->ImmPredecessors(), block) ==
successor->ImmPredecessors().end()) { successor->ImmPredecessors().end()) {
return Fail(error, "value IR successor edge is not reciprocal"); return Fail("value IR successor edge is not reciprocal");
} }
} }
@@ -238,41 +240,40 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
switch (terminator.kind) { switch (terminator.kind) {
case CFG::TerminatorKind::Branch: case CFG::TerminatorKind::Branch:
if (!add_target(terminator.true_block)) { if (!add_target(terminator.true_block)) {
return Fail(error, "value IR branch target is missing"); return Fail("value IR branch target is missing");
} }
break; break;
case CFG::TerminatorKind::ConditionalBranch: case CFG::TerminatorKind::ConditionalBranch:
if (!add_target(terminator.true_block) || !add_target(terminator.false_block)) { if (!add_target(terminator.true_block) || !add_target(terminator.false_block)) {
return Fail(error, "value IR conditional branch target is missing"); return Fail("value IR conditional branch target is missing");
} }
if (!validate_control_value(program.block_info[block_index].condition, Type::U1)) { if (!validate_control_value(program.block_info[block_index].condition, Type::U1)) {
return Fail(error, "value IR conditional branch condition is invalid"); return Fail("value IR conditional branch condition is invalid");
} }
break; break;
case CFG::TerminatorKind::IndirectBranch: { case CFG::TerminatorKind::IndirectBranch: {
if (!validate_control_value(program.block_info[block_index].indirect_target, if (!validate_control_value(program.block_info[block_index].indirect_target,
Type::U32)) { Type::U32)) {
return Fail(error, "value IR indirect branch selector is invalid"); return Fail("value IR indirect branch selector is invalid");
} }
std::unordered_set<uint32_t> indirect_targets; std::unordered_set<uint32_t> indirect_targets;
for (const auto target: terminator.indirect_targets) { for (const auto target: terminator.indirect_targets) {
if (!indirect_targets.insert(target).second) { if (!indirect_targets.insert(target).second) {
return Fail(error, "value IR indirect branch target is duplicated"); return Fail("value IR indirect branch target is duplicated");
} }
if (!add_target(target)) { if (!add_target(target)) {
return Fail(error, "value IR indirect branch target is missing"); return Fail("value IR indirect branch target is missing");
} }
} }
if (terminator.indirect_selector_values.size() != if (terminator.indirect_selector_values.size() !=
terminator.indirect_selector_targets.size()) { terminator.indirect_selector_targets.size()) {
return Fail(error, "value IR indirect selector table is inconsistent"); return Fail("value IR indirect selector table is inconsistent");
} }
for (const auto target: terminator.indirect_selector_targets) { for (const auto target: terminator.indirect_selector_targets) {
const auto found = blocks_by_id.find(target); const auto found = blocks_by_id.find(target);
if (found == blocks_by_id.end() || if (found == blocks_by_id.end() ||
!expected_successors.contains(found->second)) { !expected_successors.contains(found->second)) {
return Fail(error, return Fail("value IR indirect selector target is not a CFG successor");
"value IR indirect selector target is not a CFG successor");
} }
} }
break; break;
@@ -284,73 +285,73 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
!blocks_by_id.contains(terminator.merge_block)) || !blocks_by_id.contains(terminator.merge_block)) ||
(terminator.continue_block != UINT32_MAX && (terminator.continue_block != UINT32_MAX &&
!blocks_by_id.contains(terminator.continue_block))) { !blocks_by_id.contains(terminator.continue_block))) {
return Fail(error, "value IR structured control target is missing"); return Fail("value IR structured control target is missing");
} }
if (successors != expected_successors) { if (successors != expected_successors) {
return Fail(error, "value IR terminator and successor graph disagree"); return Fail("value IR terminator and successor graph disagree");
} }
bool saw_non_phi = false; bool saw_non_phi = false;
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (inst.Parent() != block) { if (inst.Parent() != block) {
return Fail(error, "value IR instruction has the wrong parent block"); return Fail("value IR instruction has the wrong parent block");
} }
if (inst.GetOpcode() == ValueOpcode::Phi) { if (inst.GetOpcode() == ValueOpcode::Phi) {
if (saw_non_phi) { if (saw_non_phi) {
return Fail(error, "value IR Phi appears after a non-Phi instruction"); return Fail("value IR Phi appears after a non-Phi instruction");
} }
if (inst.NumPhiBlocks() != inst.NumArgs()) { if (inst.NumPhiBlocks() != inst.NumArgs()) {
return Fail(error, "value IR Phi parent table is inconsistent"); return Fail("value IR Phi parent table is inconsistent");
} }
if (inst.NumArgs() == 0) { if (inst.NumArgs() == 0) {
return Fail(error, "value IR Phi has no incoming values"); return Fail("value IR Phi has no incoming values");
} }
std::unordered_set<const Block*> incoming_blocks; std::unordered_set<const Block*> incoming_blocks;
for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) { for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) {
const auto* predecessor = inst.PhiBlock(arg_index); const auto* predecessor = inst.PhiBlock(arg_index);
if (predecessor == nullptr || !blocks.contains(predecessor) || if (predecessor == nullptr || !blocks.contains(predecessor) ||
!predecessors.contains(predecessor)) { !predecessors.contains(predecessor)) {
return Fail(error, "value IR Phi has a foreign or non-predecessor parent"); return Fail("value IR Phi has a foreign or non-predecessor parent");
} }
if (!incoming_blocks.insert(predecessor).second) { if (!incoming_blocks.insert(predecessor).second) {
return Fail(error, "value IR Phi parent is duplicated"); return Fail("value IR Phi parent is duplicated");
} }
if (inst.Arg(arg_index).GetType() != inst.GetType()) { if (inst.Arg(arg_index).GetType() != inst.GetType()) {
return Fail(error, "value IR Phi incoming type does not match its result"); return Fail("value IR Phi incoming type does not match its result");
} }
} }
if (incoming_blocks != predecessors) { if (incoming_blocks != predecessors) {
return Fail(error, "value IR Phi does not cover every predecessor"); return Fail("value IR Phi does not cover every predecessor");
} }
} else { } else {
saw_non_phi = true; saw_non_phi = true;
} }
if (require_ssa && IsRegisterStatePseudo(inst.GetOpcode())) { if (require_ssa && IsRegisterStatePseudo(inst.GetOpcode())) {
return Fail(error, fmt::format("register-state pseudo {} survived SSA rewrite", return Fail(fmt::format("register-state pseudo {} survived SSA rewrite",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
if (inst.GetOpcode() != ValueOpcode::Phi && inst.GetOpcode() != ValueOpcode::Identity && if (inst.GetOpcode() != ValueOpcode::Phi && inst.GetOpcode() != ValueOpcode::Identity &&
inst.GetType() == Type::Opaque) { inst.GetType() == Type::Opaque) {
return Fail(error, fmt::format("untyped opcode {} survived translation", return Fail(fmt::format("untyped opcode {} survived translation",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const bool fixed_signature = const bool fixed_signature =
inst.GetOpcode() != ValueOpcode::Phi && inst.GetOpcode() != ValueOpcode::Identity; inst.GetOpcode() != ValueOpcode::Phi && inst.GetOpcode() != ValueOpcode::Identity;
if (fixed_signature && inst.NumArgs() != NumArgsOf(inst.GetOpcode())) { if (fixed_signature && inst.NumArgs() != NumArgsOf(inst.GetOpcode())) {
return Fail(error, fmt::format("{} has {} arguments, expected {}", return Fail(fmt::format("{} has {} arguments, expected {}",
ValueOpcodeName(inst.GetOpcode()), inst.NumArgs(), ValueOpcodeName(inst.GetOpcode()), inst.NumArgs(),
NumArgsOf(inst.GetOpcode()))); NumArgsOf(inst.GetOpcode())));
} }
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
const auto memory_index = inst.Flags<MemoryFlags>().index; const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index", return Fail(fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const auto& memory = program.memory_info[memory_index]; const auto& memory = program.memory_info[memory_index];
if (memory.kind != ResourceKind::ScalarBuffer) { if (memory.kind != ResourceKind::ScalarBuffer) {
return Fail(error, fmt::format("{} has an invalid scalar-memory resource kind", return Fail(fmt::format("{} has an invalid scalar-memory resource kind",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const bool valid_group_width = const bool valid_group_width =
memory.component_count == 1u || memory.component_count == 2u || memory.component_count == 1u || memory.component_count == 2u ||
@@ -358,23 +359,23 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
memory.component_count == 16u; memory.component_count == 16u;
if (memory.data_bits != 32u || memory.data_dwords != 1u || !valid_group_width || if (memory.data_bits != 32u || memory.data_dwords != 1u || !valid_group_width ||
memory.component_index >= memory.component_count) { memory.component_index >= memory.component_count) {
return Fail(error, fmt::format("{} has inconsistent scalar-memory metadata", return Fail(fmt::format("{} has inconsistent scalar-memory metadata",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
} }
const auto address_info = AddressOpcodeInfoOf(inst.GetOpcode()); const auto address_info = AddressOpcodeInfoOf(inst.GetOpcode());
if (address_info.access != AddressAccess::None) { if (address_info.access != AddressAccess::None) {
const auto memory_index = inst.Flags<MemoryFlags>().index; const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index", return Fail(fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const auto& memory = program.memory_info[memory_index]; const auto& memory = program.memory_info[memory_index];
if (!IsAddressResourceKind(memory.kind) || if (!IsAddressResourceKind(memory.kind) ||
(memory.kind == ResourceKind::ScalarAddress && (memory.kind == ResourceKind::ScalarAddress &&
inst.GetOpcode() != ValueOpcode::LoadAddressU32)) { inst.GetOpcode() != ValueOpcode::LoadAddressU32)) {
return Fail(error, fmt::format("{} has an invalid address resource kind", return Fail(fmt::format("{} has an invalid address resource kind",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const bool scalar_address = memory.kind == ResourceKind::ScalarAddress; const bool scalar_address = memory.kind == ResourceKind::ScalarAddress;
const bool valid_group_width = const bool valid_group_width =
@@ -386,50 +387,50 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
if (memory.data_bits != address_info.data_bits || memory.data_dwords != 1u || if (memory.data_bits != address_info.data_bits || memory.data_dwords != 1u ||
!valid_group_width || memory.component_index >= memory.component_count || !valid_group_width || memory.component_index >= memory.component_count ||
memory.sampler != 0u) { memory.sampler != 0u) {
return Fail(error, fmt::format("{} has inconsistent address-memory metadata", return Fail(fmt::format("{} has inconsistent address-memory metadata",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
} }
const auto buffer_components = BufferComponentCount(inst.GetOpcode()); const auto buffer_components = BufferComponentCount(inst.GetOpcode());
if (buffer_components != 0u) { if (buffer_components != 0u) {
const auto memory_index = inst.Flags<MemoryFlags>().index; const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index", return Fail(fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const auto& memory = program.memory_info[memory_index]; const auto& memory = program.memory_info[memory_index];
if (memory.kind != ResourceKind::Buffer && if (memory.kind != ResourceKind::Buffer &&
memory.kind != ResourceKind::ScalarBuffer) { memory.kind != ResourceKind::ScalarBuffer) {
return Fail(error, fmt::format("{} has a non-buffer resource kind", return Fail(fmt::format("{} has a non-buffer resource kind",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
if (buffer_components > 1u && if (buffer_components > 1u &&
(memory.kind != ResourceKind::Buffer || memory.data_bits != 32u || (memory.kind != ResourceKind::Buffer || memory.data_bits != 32u ||
memory.data_dwords != buffer_components || memory.component_index != 0u)) { memory.data_dwords != buffer_components || memory.component_index != 0u)) {
return Fail(error, fmt::format("{} has inconsistent native-wide metadata", return Fail(fmt::format("{} has inconsistent native-wide metadata",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
if (buffer_components == 1u && if (buffer_components == 1u &&
(inst.GetOpcode() == ValueOpcode::LoadBufferU32 || (inst.GetOpcode() == ValueOpcode::LoadBufferU32 ||
inst.GetOpcode() == ValueOpcode::StoreBufferU32) && inst.GetOpcode() == ValueOpcode::StoreBufferU32) &&
memory.data_dwords != 1u) { memory.data_dwords != 1u) {
return Fail(error, fmt::format("{} retains scalar-sibling width metadata", return Fail(fmt::format("{} retains scalar-sibling width metadata",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
} }
const auto shared_components = SharedComponentCount(inst.GetOpcode()); const auto shared_components = SharedComponentCount(inst.GetOpcode());
if (shared_components != 0u) { if (shared_components != 0u) {
const auto memory_index = inst.Flags<MemoryFlags>().index; const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index", return Fail(fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const auto& memory = program.memory_info[memory_index]; const auto& memory = program.memory_info[memory_index];
if ((memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) || if ((memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) ||
memory.resource != 0u || memory.sampler != 0u || memory.component_count == 0u || memory.resource != 0u || memory.sampler != 0u || memory.component_count == 0u ||
memory.component_index >= memory.component_count) { memory.component_index >= memory.component_count) {
return Fail(error, fmt::format("{} has invalid shared-memory metadata", return Fail(fmt::format("{} has invalid shared-memory metadata",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
uint32_t expected_bits = 32u; uint32_t expected_bits = 32u;
if (inst.GetOpcode() == ValueOpcode::LoadSharedU8 || if (inst.GetOpcode() == ValueOpcode::LoadSharedU8 ||
@@ -441,26 +442,26 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
} }
if (memory.data_bits != expected_bits || memory.data_dwords != shared_components || if (memory.data_bits != expected_bits || memory.data_dwords != shared_components ||
(shared_components > 1u && memory.component_index != 0u)) { (shared_components > 1u && memory.component_index != 0u)) {
return Fail(error, fmt::format("{} has inconsistent shared-memory width", return Fail(fmt::format("{} has inconsistent shared-memory width",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
} }
const auto image_info = ImageOpcodeInfoOf(inst.GetOpcode()); const auto image_info = ImageOpcodeInfoOf(inst.GetOpcode());
if (image_info.access != ImageAccess::None) { if (image_info.access != ImageAccess::None) {
const auto memory_index = inst.Flags<MemoryFlags>().index; const auto memory_index = inst.Flags<MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) { if (memory_index >= program.memory_info.size()) {
return Fail(error, fmt::format("{} has an invalid memory-info index", return Fail(fmt::format("{} has an invalid memory-info index",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
const auto& memory = program.memory_info[memory_index]; const auto& memory = program.memory_info[memory_index];
if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) { if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) {
return Fail(error, fmt::format("{} has invalid image-memory metadata", return Fail(fmt::format("{} has invalid image-memory metadata",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
} }
if (inst.GetOpcode() == ValueOpcode::SetAttribute && if (inst.GetOpcode() == ValueOpcode::SetAttribute &&
inst.Flags<ExportFlags>().index >= program.export_info.size()) { inst.Flags<ExportFlags>().index >= program.export_info.size()) {
return Fail(error, "SetAttribute has an invalid export-info index"); return Fail("SetAttribute has an invalid export-info index");
} }
uint32_t composite_components = 0u; uint32_t composite_components = 0u;
switch (inst.GetOpcode()) { switch (inst.GetOpcode()) {
@@ -473,25 +474,24 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
if (composite_components != 0u && if (composite_components != 0u &&
(!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32 || (!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32 ||
inst.Arg(1).U32() >= composite_components)) { inst.Arg(1).U32() >= composite_components)) {
return Fail(error, fmt::format("{} has an invalid component index", return Fail(fmt::format("{} has an invalid component index",
ValueOpcodeName(inst.GetOpcode()))); ValueOpcodeName(inst.GetOpcode())));
} }
for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) { for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) {
const auto arg = inst.Arg(arg_index); const auto arg = inst.Arg(arg_index);
if (arg.IsEmpty()) { if (arg.IsEmpty()) {
return Fail(error, fmt::format("{} has an empty argument", return Fail(
ValueOpcodeName(inst.GetOpcode()))); fmt::format("{} has an empty argument", ValueOpcodeName(inst.GetOpcode())));
} }
if (fixed_signature && arg.GetType() != ArgTypeOf(inst.GetOpcode(), arg_index)) { if (fixed_signature && arg.GetType() != ArgTypeOf(inst.GetOpcode(), arg_index)) {
return Fail(error, return Fail(fmt::format("{} argument {} has type {}, expected {}",
fmt::format("{} argument {} has type {}, expected {}",
ValueOpcodeName(inst.GetOpcode()), arg_index, ValueOpcodeName(inst.GetOpcode()), arg_index,
TypeName(arg.GetType()), TypeName(arg.GetType()),
TypeName(ArgTypeOf(inst.GetOpcode(), arg_index)))); TypeName(ArgTypeOf(inst.GetOpcode(), arg_index))));
} }
if (const auto* definition = arg.TryInstruction(); if (const auto* definition = arg.TryInstruction();
definition != nullptr && !instructions.contains(definition)) { definition != nullptr && !instructions.contains(definition)) {
return Fail(error, "value IR argument has a foreign definition"); return Fail("value IR argument has a foreign definition");
} }
if (const auto* definition = arg.TryInstruction(); definition != nullptr) { if (const auto* definition = arg.TryInstruction(); definition != nullptr) {
const auto& uses = definition->Uses(); const auto& uses = definition->Uses();
@@ -499,8 +499,7 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
return candidate.user == &inst && candidate.operand == arg_index; return candidate.user == &inst && candidate.operand == arg_index;
}); });
if (use == uses.end()) { if (use == uses.end()) {
return Fail(error, return Fail(fmt::format("{} argument {} is absent from {} reverse uses",
fmt::format("{} argument {} is absent from {} reverse uses",
ValueOpcodeName(inst.GetOpcode()), arg_index, ValueOpcodeName(inst.GetOpcode()), arg_index,
ValueOpcodeName(definition->GetOpcode()))); ValueOpcodeName(definition->GetOpcode())));
} }
@@ -510,7 +509,7 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
} }
if (program.blocks.empty()) { if (program.blocks.empty()) {
return true; return;
} }
std::vector<bool> reachable(program.blocks.size(), false); std::vector<bool> reachable(program.blocks.size(), false);
@@ -527,7 +526,7 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
} }
} }
if (!std::ranges::all_of(reachable, [](bool value) { return value; })) { if (!std::ranges::all_of(reachable, [](bool value) { return value; })) {
return Fail(error, "value IR contains an unreachable block"); return Fail("value IR contains an unreachable block");
} }
std::vector<std::vector<bool>> dominators(program.blocks.size(), std::vector<std::vector<bool>> dominators(program.blocks.size(),
@@ -581,28 +580,25 @@ bool ValidateProgram(const Program& program, bool require_ssa, std::string* erro
const auto* predecessor = inst.PhiBlock(arg_index); const auto* predecessor = inst.PhiBlock(arg_index);
if (definition->Parent() != predecessor && if (definition->Parent() != predecessor &&
!dominates(definition->Parent(), predecessor)) { !dominates(definition->Parent(), predecessor)) {
return Fail(error, return Fail("value IR Phi incoming definition does not dominate its edge");
"value IR Phi incoming definition does not dominate its edge");
} }
} else if (definition->Parent() == block) { } else if (definition->Parent() == block) {
if (instruction_positions.at(definition) >= instruction_positions.at(&inst)) { if (instruction_positions.at(definition) >= instruction_positions.at(&inst)) {
return Fail(error, return Fail("value IR instruction uses a same-block definition before it");
"value IR instruction uses a same-block definition before it");
} }
} else if (!dominates(definition->Parent(), block)) { } else if (!dominates(definition->Parent(), block)) {
return Fail(error, "value IR instruction definition does not dominate its use"); return Fail("value IR instruction definition does not dominate its use");
} }
} }
} }
const auto& info = program.block_info[block_index]; const auto& info = program.block_info[block_index];
if (!info.condition.IsEmpty() && !control_dominates(info.condition, block)) { if (!info.condition.IsEmpty() && !control_dominates(info.condition, block)) {
return Fail(error, "value IR branch condition definition does not dominate its use"); return Fail("value IR branch condition definition does not dominate its use");
} }
if (!info.indirect_target.IsEmpty() && !control_dominates(info.indirect_target, block)) { if (!info.indirect_target.IsEmpty() && !control_dominates(info.indirect_target, block)) {
return Fail(error, "value IR indirect selector definition does not dominate its use"); return Fail("value IR indirect selector definition does not dominate its use");
} }
} }
return true;
} }
void ResolveControlFlowIdentities(Program& program) { void ResolveControlFlowIdentities(Program& program) {
+1 -2
View File
@@ -477,7 +477,6 @@ struct DescriptorSource {
struct SrtRead { struct SrtRead {
Value value; Value value;
uint32_t flat_offset = 0; uint32_t flat_offset = 0;
uint32_t use_pc = 0;
bool operator==(const SrtRead& other) const = default; bool operator==(const SrtRead& other) const = default;
}; };
@@ -523,7 +522,7 @@ struct Program {
std::string ProgramToString(const Program& program); std::string ProgramToString(const Program& program);
bool ValidateProgram(const Program& program, bool require_ssa, std::string* error); void ValidateProgram(const Program& program, bool require_ssa);
void ResolveControlFlowIdentities(Program& program); void ResolveControlFlowIdentities(Program& program);
bool EquivalentValue(const Program& program, Value left, Value right); bool EquivalentValue(const Program& program, Value left, Value right);
Value ResolveInvariantPhi(const Program& program, Value value); Value ResolveInvariantPhi(const Program& program, Value value);
@@ -1,5 +1,6 @@
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <algorithm> #include <algorithm>
@@ -9,12 +10,17 @@
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
namespace { namespace {
constexpr auto FirstImageBinding = DescriptorBindingKind::Sampled1D; constexpr auto FirstImageBinding = DescriptorBindingKind::Sampled1D;
constexpr uint32_t ImageBindingCount = constexpr uint32_t ImageBindingCount =
static_cast<uint32_t>(DescriptorBindingKind::StorageAtomic3D) - static_cast<uint32_t>(DescriptorBindingKind::StorageAtomic3D) -
static_cast<uint32_t>(FirstImageBinding) + 1u; static_cast<uint32_t>(FirstImageBinding) + 1u;
bool CollectUserData(const Program& program, std::vector<uint32_t>& result) { [[noreturn]] void BindingFail(const char* message) {
EXIT("shader binding layout failed: %s", message);
std::abort();
}
std::vector<uint32_t> CollectUserData(const Program& program) {
std::array<bool, NumScalarRegs> registers {}; std::array<bool, NumScalarRegs> registers {};
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
@@ -22,22 +28,22 @@ bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
continue; continue;
} }
if (inst.Arg(0).GetType() != Type::ScalarReg) { if (inst.Arg(0).GetType() != Type::ScalarReg) {
return false; BindingFail("typed shader contains an invalid user-data register");
} }
const auto index = RegIndex(inst.Arg(0).ScalarRegister()); const auto index = RegIndex(inst.Arg(0).ScalarRegister());
if (index >= NumScalarRegs) { if (index >= NumScalarRegs) {
return false; BindingFail("typed shader contains an invalid user-data register");
} }
registers[index] = true; registers[index] = true;
} }
} }
result.clear(); std::vector<uint32_t> result;
for (uint32_t index = 0; index < registers.size(); index++) { for (uint32_t index = 0; index < registers.size(); index++) {
if (registers[index]) { if (registers[index]) {
result.push_back(index); result.push_back(index);
} }
} }
return true; return result;
} }
void AddBinding(BindingLayout& layout, DescriptorBindingKind kind, void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
@@ -45,8 +51,8 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
layout.descriptors.push_back({kind, std::move(resources)}); layout.descriptors.push_back({kind, std::move(resources)});
} }
bool UsesGds(const Program& program, bool& uses_gds) { bool UsesGds(const Program& program) {
uses_gds = false; bool uses_gds = false;
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) { if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) {
@@ -54,44 +60,36 @@ bool UsesGds(const Program& program, bool& uses_gds) {
} }
const auto index = inst.Flags<MemoryFlags>().index; const auto index = inst.Flags<MemoryFlags>().index;
if (index >= program.memory_info.size()) { if (index >= program.memory_info.size()) {
return false; BindingFail("typed shader contains invalid shared-memory metadata");
} }
const auto kind = program.memory_info[index].kind; const auto kind = program.memory_info[index].kind;
if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) { if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) {
return false; BindingFail("typed shader contains invalid shared-memory metadata");
} }
uses_gds |= kind == ResourceKind::Gds; uses_gds |= kind == ResourceKind::Gds;
} }
} }
return true; return uses_gds;
} }
} // namespace } // namespace
bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::string* error) { void AllocateBindings(Program& program, uint32_t push_constant_offset) {
if (!program.shader_info_complete || program.binding_layout_complete) { if (!program.shader_info_complete || program.binding_layout_complete) {
if (error != nullptr) { EXIT("shader binding layout failed: %s", !program.shader_info_complete
*error = !program.shader_info_complete ? "shader info is not ready" ? "shader info is not ready"
: "binding layout already allocated"; : "binding layout already allocated");
}
return false;
} }
if (push_constant_offset > NativePushConstantSize || if (push_constant_offset > NativePushConstantSize ||
push_constant_offset % sizeof(uint32_t) != 0) { push_constant_offset % sizeof(uint32_t) != 0) {
if (error != nullptr) { EXIT("shader binding layout failed: push-constant offset %u exceeds the Vulkan minimum "
*error = "push-constant offset exceeds the Vulkan minimum guarantee or is unaligned"; "guarantee or is unaligned",
} push_constant_offset);
return false;
} }
BindingLayout next; BindingLayout next;
next.push_constant_offset = push_constant_offset; next.push_constant_offset = push_constant_offset;
if (!CollectUserData(program, next.user_data_registers)) { next.user_data_registers = CollectUserData(program);
if (error != nullptr) {
*error = "typed shader contains an invalid user-data register";
}
return false;
}
next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size()); next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
next.memory_offset_count = static_cast<uint32_t>(program.info.buffers.size()); next.memory_offset_count = static_cast<uint32_t>(program.info.buffers.size());
@@ -107,26 +105,18 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
for (uint32_t i = 0; i < program.info.images.size(); i++) { for (uint32_t i = 0; i < program.info.images.size(); i++) {
const auto kind = DescriptorBindingForImage(program.info.images[i]); const auto kind = DescriptorBindingForImage(program.info.images[i]);
if (!kind.has_value()) { if (!kind.has_value()) {
if (error != nullptr) { EXIT("shader binding layout failed: image %u has an invalid binding class", i);
*error = "shader info contains an invalid image binding class";
}
return false;
} }
const auto group = static_cast<uint32_t>(*kind) - static_cast<uint32_t>(FirstImageBinding); const auto group = static_cast<uint32_t>(*kind) - static_cast<uint32_t>(FirstImageBinding);
if (group >= image_groups.size()) { if (group >= image_groups.size()) {
if (error != nullptr) { EXIT("shader binding layout failed: image %u has an unmapped binding class", i);
*error = "shader info contains an unmapped image binding class";
}
return false;
} }
auto& resources = image_groups[group]; auto& resources = image_groups[group];
const auto dynamic = program.info.images[i].mip_mode == ImageMipMode::DynamicStorage; const auto dynamic = program.info.images[i].mip_mode == ImageMipMode::DynamicStorage;
const auto count = dynamic ? program.info.images[i].mip_count : 1u; const auto count = dynamic ? program.info.images[i].mip_count : 1u;
if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) { if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
if (error != nullptr) { EXIT("shader binding layout failed: image %u has invalid specialized mip count %u", i,
*error = "image has an invalid specialized mip descriptor count"; program.info.images[i].mip_count);
}
return false;
} }
resources.insert(resources.end(), count, i); resources.insert(resources.end(), count, i);
} }
@@ -146,14 +136,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
} }
AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources)); AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
} }
bool uses_gds = false; if (UsesGds(program)) {
if (!UsesGds(program, uses_gds)) {
if (error != nullptr) {
*error = "typed shader contains invalid shared-memory metadata";
}
return false;
}
if (uses_gds) {
AddBinding(next, DescriptorBindingKind::Gds); AddBinding(next, DescriptorBindingKind::Gds);
} }
if (program.info.uses_dma) { if (program.info.uses_dma) {
@@ -178,7 +161,6 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
program.bindings = std::move(next); program.bindings = std::move(next);
program.binding_layout_complete = true; program.binding_layout_complete = true;
return true;
} }
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind) { const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind) {
@@ -5,7 +5,7 @@
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::string* error); void AllocateBindings(Program& program, uint32_t push_constant_offset);
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind); const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
@@ -1,5 +1,6 @@
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "common/assert.h"
#include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/gpu_format.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shaderBindings.h" #include "graphics/shader/shaderBindings.h"
@@ -18,6 +19,11 @@ namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull; constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
constexpr uint64_t MaxIndirectImageProbes = 65536u; constexpr uint64_t MaxIndirectImageProbes = 65536u;
[[noreturn]] void SpecializationFail(std::string_view message) {
EXIT("shader resource specialization failed: %s", std::string(message).c_str());
std::abort();
}
Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor, Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor,
Decoder::ImageDimension requested) { Decoder::ImageDimension requested) {
const bool is_array = requested == Decoder::ImageDimension::Dim1DArray || const bool is_array = requested == Decoder::ImageDimension::Dim1DArray ||
@@ -79,7 +85,7 @@ uint32_t DescriptorImageSwizzle(const DescriptorValue& descriptor) {
Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) { Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) {
return Prospero::RemapTextureFormat(format) != format ? format return Prospero::RemapTextureFormat(format) != format ? format
: Prospero::BufferFormat::kInvalid; : Prospero::BufferFormat::kInvalid;
} }
bool DescriptorIsCube(const DescriptorValue& descriptor) { bool DescriptorIsCube(const DescriptorValue& descriptor) {
@@ -153,8 +159,7 @@ bool ReadSpecializationWord(const SrtRuntime& runtime, uint64_t address, uint32_
} }
bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynamic_offset, bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynamic_offset,
uint32_t immediate_offset, const SrtRuntime& runtime, uint32_t& word, uint32_t immediate_offset, const SrtRuntime& runtime, uint32_t& word) {
std::string* error, uint32_t use_pc) {
const auto byte_offset = static_cast<uint64_t>(dynamic_offset) + immediate_offset; const auto byte_offset = static_cast<uint64_t>(dynamic_offset) + immediate_offset;
const auto aligned = byte_offset & ~uint64_t {3}; const auto aligned = byte_offset & ~uint64_t {3};
const auto size = ScalarBufferSize(descriptor); const auto size = ScalarBufferSize(descriptor);
@@ -164,20 +169,10 @@ bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynam
} }
const auto base = descriptor.Base48() & ~uint64_t {3}; const auto base = descriptor.Base48() & ~uint64_t {3};
if (aligned > AddressMask - base) { if (aligned > AddressMask - base) {
if (error != nullptr) {
*error = fmt::format("indirect image scalar read at pc 0x{:08x} exceeds the "
"48-bit address space",
use_pc);
}
return false; return false;
} }
const auto address = base + aligned; const auto address = base + aligned;
if (!ReadSpecializationWord(runtime, address, word)) { if (!ReadSpecializationWord(runtime, address, word)) {
if (error != nullptr) {
*error = fmt::format("indirect image scalar read at pc 0x{:08x} failed at "
"0x{:016x}",
use_pc, address);
}
return false; return false;
} }
return true; return true;
@@ -186,23 +181,14 @@ bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynam
bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
const DescriptorValue& material_value, const DescriptorValue& material_value,
const DescriptorValue& heap_value, const SrtRuntime& runtime, const DescriptorValue& heap_value, const SrtRuntime& runtime,
ResourceSnapshot::IndirectImage& result, std::string* error, ResourceSnapshot::IndirectImage& result) {
uint32_t use_pc) {
ShaderBufferResource material; ShaderBufferResource material;
ShaderBufferResource heap; ShaderBufferResource heap;
if (!DecodeBufferDescriptor(material_value, material) || if (!DecodeBufferDescriptor(material_value, material) ||
!DecodeBufferDescriptor(heap_value, heap)) { !DecodeBufferDescriptor(heap_value, heap)) {
if (error != nullptr) {
*error = fmt::format("indirect image tables at pc 0x{:08x} have invalid descriptors",
use_pc);
}
return false; return false;
} }
if (material.Stride() != indirect.selector_stride) { if (material.Stride() != indirect.selector_stride) {
if (error != nullptr) {
*error =
fmt::format("indirect image material table at pc 0x{:08x} changed stride", use_pc);
}
return false; return false;
} }
@@ -215,11 +201,6 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
const auto limit = std::min<uint64_t>(UINT32_MAX, size + 3u); const auto limit = std::min<uint64_t>(UINT32_MAX, size + 3u);
const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u; const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
if (probe_count > MaxIndirectImageProbes) { if (probe_count > MaxIndirectImageProbes) {
if (error != nullptr) {
*error = fmt::format("indirect image material table at pc 0x{:08x} requires {} "
"bounded probes",
use_pc, probe_count);
}
return false; return false;
} }
@@ -227,8 +208,7 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
std::unordered_set<uint32_t> seen {0u}; std::unordered_set<uint32_t> seen {0u};
for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) { for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
uint32_t key = 0; uint32_t key = 0;
if (!ReadScalarBufferWord(material, static_cast<uint32_t>(offset), 0u, runtime, key, error, if (!ReadScalarBufferWord(material, static_cast<uint32_t>(offset), 0u, runtime, key)) {
use_pc)) {
return false; return false;
} }
if (seen.insert(key).second) { if (seen.insert(key).second) {
@@ -248,7 +228,7 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
const auto heap_offset = key << 5u; const auto heap_offset = key << 5u;
for (uint32_t dword = 0; dword < candidate.dword_count; dword++) { for (uint32_t dword = 0; dword < candidate.dword_count; dword++) {
if (!ReadScalarBufferWord(heap, heap_offset, dword * sizeof(uint32_t), runtime, if (!ReadScalarBufferWord(heap, heap_offset, dword * sizeof(uint32_t), runtime,
candidate.dwords[dword], error, use_pc)) { candidate.dwords[dword])) {
return false; return false;
} }
} }
@@ -281,35 +261,22 @@ size_t FlattenedRuntimeDwords(const Program& program) {
} // namespace } // namespace
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot, bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot) {
std::string* error) {
if (!program.resource_tracking_complete) { if (!program.resource_tracking_complete) {
if (error != nullptr) {
*error = "shader resources were not tracked";
}
return false; return false;
} }
if (snapshot.buffers.size() != program.info.buffers.size() || if (snapshot.buffers.size() != program.info.buffers.size() ||
snapshot.images.size() != program.info.images.size() || snapshot.images.size() != program.info.images.size() ||
snapshot.samplers.size() != program.info.samplers.size()) { snapshot.samplers.size() != program.info.samplers.size()) {
if (error != nullptr) {
*error = "resource snapshot does not match dense shader topology";
}
return false; return false;
} }
if (snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) { if (snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) {
if (error != nullptr) {
*error = "flattened SRT snapshot does not match the shader plan";
}
return false; return false;
} }
if (program.binding_layout_complete) { if (program.binding_layout_complete) {
for (const auto reg: program.bindings.user_data_registers) { for (const auto reg: program.bindings.user_data_registers) {
if (reg < program.user_data_base || if (reg < program.user_data_base ||
reg - program.user_data_base >= snapshot.user_data.size()) { reg - program.user_data_base >= snapshot.user_data.size()) {
if (error != nullptr) {
*error = fmt::format("runtime snapshot is missing user SGPR {}", reg);
}
return false; return false;
} }
} }
@@ -317,19 +284,12 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
for (uint32_t i = 0; i < program.info.buffers.size(); i++) { for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto alias = program.info.buffers[i].image_alias; const auto alias = program.info.buffers[i].image_alias;
if (alias != BufferResource::NoImageAlias && alias >= program.info.images.size()) { if (alias != BufferResource::NoImageAlias && alias >= program.info.images.size()) {
if (error != nullptr) {
*error = fmt::format("buffer resource {} has invalid image alias {}", i, alias);
}
return false; return false;
} }
} }
const auto CheckWidth = [&](const auto& values, uint32_t width, const char* kind) { const auto CheckWidth = [&](const auto& values, uint32_t width) {
for (uint32_t i = 0; i < values.size(); i++) { for (uint32_t i = 0; i < values.size(); i++) {
if (values[i].dword_count != width) { if (values[i].dword_count != width) {
if (error != nullptr) {
*error = fmt::format("{} descriptor {} has {} dwords", kind, i,
values[i].dword_count);
}
return false; return false;
} }
} }
@@ -345,56 +305,37 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
table.keys.empty() || table.keys.size() != table.candidates.size() || table.keys.empty() || table.keys.size() != table.candidates.size() ||
table.capacity < table.keys.size() || table.descriptors.empty() || table.capacity < table.keys.size() || table.descriptors.empty() ||
!indirect_resources.insert(table.resource).second) { !indirect_resources.insert(table.resource).second) {
if (error != nullptr) {
*error = "indirect image snapshot does not match the shader plan";
}
return false; return false;
} }
for (const auto candidate: table.candidates) { for (const auto candidate: table.candidates) {
if (candidate >= table.descriptors.size()) { if (candidate >= table.descriptors.size()) {
if (error != nullptr) {
*error = "indirect image snapshot has an invalid candidate";
}
return false; return false;
} }
} }
if (snapshot.images[table.resource] != table.descriptors[table.candidates[0]]) { if (snapshot.images[table.resource] != table.descriptors[table.candidates[0]]) {
if (error != nullptr) {
*error = "indirect image snapshot has an inconsistent root descriptor";
}
return false; return false;
} }
} }
return CheckWidth(snapshot.buffers, 4, "buffer") && CheckWidth(snapshot.images, 8, "image") && return CheckWidth(snapshot.buffers, 4) && CheckWidth(snapshot.images, 8) &&
CheckWidth(snapshot.samplers, 4, "sampler"); CheckWidth(snapshot.samplers, 4);
} }
bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot, bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot) {
std::string* error) { if (!ValidateResourceSnapshot(program, snapshot)) {
if (!ValidateResourceSnapshot(program, snapshot, error)) {
return false; return false;
} }
if (!snapshot.indirect_images.empty()) { if (!snapshot.indirect_images.empty()) {
if (error != nullptr) {
*error = "indirect image snapshot was not specialized";
}
return false; return false;
} }
for (uint32_t i = 0; i < program.info.buffers.size(); i++) { for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto& buffer = program.info.buffers[i]; const auto& buffer = program.info.buffers[i];
ShaderBufferResource descriptor; ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(snapshot.buffers[i], descriptor)) { if (!DecodeBufferDescriptor(snapshot.buffers[i], descriptor)) {
if (error != nullptr) {
*error = fmt::format("buffer descriptor {} has invalid width", i);
}
return false; return false;
} }
if (buffer.packed_stride != descriptor.PackedStride() || if (buffer.packed_stride != descriptor.PackedStride() ||
buffer.descriptor_format != descriptor.Format() || buffer.descriptor_format != descriptor.Format() ||
buffer.descriptor_swizzle != descriptor.DstSelXYZW()) { buffer.descriptor_swizzle != descriptor.DstSelXYZW()) {
if (error != nullptr) {
*error = fmt::format("buffer descriptor {} no longer matches specialization", i);
}
return false; return false;
} }
} }
@@ -403,9 +344,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
const auto& descriptor = snapshot.images[i]; const auto& descriptor = snapshot.images[i];
const auto mip_count = StorageMipCount(image, descriptor); const auto mip_count = StorageMipCount(image, descriptor);
if (mip_count == 0u || mip_count != image.mip_count) { if (mip_count == 0u || mip_count != image.mip_count) {
if (error != nullptr) {
*error = fmt::format("storage image descriptor {} changed mip range", i);
}
return false; return false;
} }
if (NullImageDescriptor(descriptor)) { if (NullImageDescriptor(descriptor)) {
@@ -419,10 +357,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
program.info.images[root].conversion_format != image.conversion_format || program.info.images[root].conversion_format != image.conversion_format ||
program.info.images[root].shader_swizzle != image.shader_swizzle || program.info.images[root].shader_swizzle != image.shader_swizzle ||
program.info.images[root].cube != image.cube) { program.info.images[root].cube != image.cube) {
if (error != nullptr) {
*error = fmt::format(
"null indirect image descriptor {} changed binding class", i);
}
return false; return false;
} }
continue; continue;
@@ -434,10 +368,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
} }
if (image.dimension != Decoder::ImageDimension::Dim2D || image.cube || if (image.dimension != Decoder::ImageDimension::Dim2D || image.cube ||
!canonical_kind) { !canonical_kind) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} no longer matches canonical null specialization", i);
}
return false; return false;
} }
continue; continue;
@@ -445,14 +375,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
const auto dimension = DescriptorDimension(descriptor, image.dimension); const auto dimension = DescriptorDimension(descriptor, image.dimension);
if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension || if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension ||
DescriptorIsCube(descriptor) != image.cube) { DescriptorIsCube(descriptor) != image.cube) {
if (error != nullptr) {
*error =
fmt::format("image descriptor {} no longer matches specialized dimension: "
"{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
descriptor.dwords[6], descriptor.dwords[7]);
}
return false; return false;
} }
if (image.kind == ResourceKind::Image || image.kind == ResourceKind::ImageUint || if (image.kind == ResourceKind::Image || image.kind == ResourceKind::ImageUint ||
@@ -463,49 +385,25 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
const auto format = const auto format =
static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu); static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
if (image.atomic && format != Prospero::BufferFormat::k32UInt) { if (image.atomic && format != Prospero::BufferFormat::k32UInt) {
if (error != nullptr) {
*error = fmt::format(
"atomic image descriptor {} changed to unsupported format {}", i,
static_cast<uint32_t>(format));
}
return false; return false;
} }
const auto conversion_format = ImageConversionFormat(format); const auto conversion_format = ImageConversionFormat(format);
if (image.conversion_format != conversion_format) { if (image.conversion_format != conversion_format) {
if (error != nullptr) {
*error = fmt::format("image descriptor {} changed format conversion", i);
}
return false; return false;
} }
if ((storage || conversion_format != Prospero::BufferFormat::kInvalid) && if ((storage || conversion_format != Prospero::BufferFormat::kInvalid) &&
image.shader_swizzle != DescriptorImageSwizzle(descriptor)) { image.shader_swizzle != DescriptorImageSwizzle(descriptor)) {
if (error != nullptr) {
*error = fmt::format("image descriptor {} changed swizzle", i);
}
return false; return false;
} }
const bool raw_sint_storage = const bool raw_sint_storage = storage && format == Prospero::BufferFormat::k32SInt &&
storage && format == Prospero::BufferFormat::k32SInt && image.written && image.written && !image.read && !image.atomic;
!image.read && !image.atomic; const bool uint_descriptor = Prospero::IsUintTextureFormat(format) || raw_sint_storage;
const bool uint_descriptor = const auto uint_program = image.kind == ResourceKind::ImageUint ||
Prospero::IsUintTextureFormat(format) || raw_sint_storage; image.kind == ResourceKind::StorageImageUint;
const auto uint_program = image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImageUint;
if (uint_descriptor != uint_program && !(image.atomic && uint_program)) { if (uint_descriptor != uint_program && !(image.atomic && uint_program)) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} no longer matches specialized format: "
"{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
descriptor.dwords[6], descriptor.dwords[7]);
}
return false; return false;
} }
if (image.depth_compare && uint_program) { if (image.depth_compare && uint_program) {
if (error != nullptr) {
*error = fmt::format("integer image descriptor {} uses depth comparison", i);
}
return false; return false;
} }
} }
@@ -514,11 +412,8 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
} }
bool MaterializeResources(const Program& program, const SrtRuntime& runtime, bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot& snapshot, std::string* error) { ResourceSnapshot& snapshot) {
if (!program.resource_tracking_complete) { if (!program.resource_tracking_complete) {
if (error != nullptr) {
*error = "shader resources were not tracked";
}
return false; return false;
} }
@@ -527,17 +422,12 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u + requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u +
program.info.samplers.size()); program.info.samplers.size());
for (const auto& buffer: program.info.buffers) { for (const auto& buffer: program.info.buffers) {
requests.push_back({buffer.source, buffer.first_use_pc}); requests.push_back({buffer.source});
} }
for (const auto& image: program.info.images) { for (const auto& image: program.info.images) {
const auto* source = Source(program, image.source); const auto* source = Source(program, image.source);
if (source != nullptr && source->indirect_image.has_value()) { if (source != nullptr && source->indirect_image.has_value()) {
if (runtime.read_specialization_memory == nullptr) { if (runtime.read_specialization_memory == nullptr) {
if (error != nullptr) {
*error = fmt::format("indirect image table at pc 0x{:08x} has no clean "
"memory reader",
image.first_use_pc);
}
return false; return false;
} }
MarkCleanFlatSlots(program, Source(program, source->indirect_image->material_source), MarkCleanFlatSlots(program, Source(program, source->indirect_image->material_source),
@@ -545,16 +435,16 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
MarkCleanFlatSlots(program, Source(program, source->indirect_image->heap_source), MarkCleanFlatSlots(program, Source(program, source->indirect_image->heap_source),
clean_flat_slots); clean_flat_slots);
} else { } else {
requests.push_back({image.source, image.first_use_pc}); requests.push_back({image.source});
} }
} }
for (const auto& sampler: program.info.samplers) { for (const auto& sampler: program.info.samplers) {
requests.push_back({sampler.source, sampler.first_use_pc}); requests.push_back({sampler.source});
} }
std::vector<DescriptorValue> values; std::vector<DescriptorValue> values;
std::vector<uint32_t> flattened_srt; std::vector<uint32_t> flattened_srt;
if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, clean_flat_slots, if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt,
error)) { clean_flat_slots)) {
return false; return false;
} }
@@ -581,20 +471,19 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
continue; continue;
} }
const std::array requests { const std::array requests {
DescriptorSourceRequest {source->indirect_image->material_source, DescriptorSourceRequest {source->indirect_image->material_source},
image.first_use_pc}, DescriptorSourceRequest {source->indirect_image->heap_source}};
DescriptorSourceRequest {source->indirect_image->heap_source, image.first_use_pc}};
SrtRuntime clean_runtime = runtime; SrtRuntime clean_runtime = runtime;
clean_runtime.read_memory = runtime.read_specialization_memory; clean_runtime.read_memory = runtime.read_specialization_memory;
std::vector<DescriptorValue> tables; std::vector<DescriptorValue> tables;
if (!EvaluateDescriptorSources(program, requests, clean_runtime, tables, error)) { if (!EvaluateDescriptorSources(program, requests, clean_runtime, tables)) {
return false; return false;
} }
const auto& material = tables[0]; const auto& material = tables[0];
const auto& heap = tables[1]; const auto& heap = tables[1];
ResourceSnapshot::IndirectImage table; ResourceSnapshot::IndirectImage table;
if (!MaterializeIndirectImage(*source->indirect_image, material, heap, runtime, table, if (!MaterializeIndirectImage(*source->indirect_image, material, heap, runtime,
error, image.first_use_pc)) { table)) {
return false; return false;
} }
if (image.indirect_root == ImageResource::NoIndirectImage) { if (image.indirect_root == ImageResource::NoIndirectImage) {
@@ -608,20 +497,12 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
} }
if (table.keys.size() > image.indirect_mapping_capacity || if (table.keys.size() > image.indirect_mapping_capacity ||
table.descriptors.size() > image.indirect_resources.size()) { table.descriptors.size() > image.indirect_resources.size()) {
if (error != nullptr) {
*error = fmt::format(
"indirect image table at pc 0x{:08x} changed candidate topology",
image.first_use_pc);
}
return false; return false;
} }
for (uint32_t candidate = 0; candidate < image.indirect_resources.size(); candidate++) { for (uint32_t candidate = 0; candidate < image.indirect_resources.size(); candidate++) {
const auto resource = image.indirect_resources[candidate]; const auto resource = image.indirect_resources[candidate];
if (resource >= program.info.images.size() || if (resource >= program.info.images.size() ||
program.info.images[resource].indirect_root != image_index) { program.info.images[resource].indirect_root != image_index) {
if (error != nullptr) {
*error = "indirect image specialization has an invalid candidate resource";
}
return false; return false;
} }
next.images[resource] = next.images[resource] =
@@ -649,31 +530,26 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
} }
} }
if (std::ranges::find(image_written, uint8_t {0}) != image_written.end()) { if (std::ranges::find(image_written, uint8_t {0}) != image_written.end()) {
if (error != nullptr) {
*error = "indirect image specialization left an unmaterialized candidate";
}
return false; return false;
} }
next.samplers.assign(cursor, cursor + program.info.samplers.size()); next.samplers.assign(cursor, cursor + program.info.samplers.size());
next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end()); next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
if (!ValidateResourceSnapshot(program, next, error)) { if (!ValidateResourceSnapshot(program, next)) {
return false; return false;
} }
snapshot = std::move(next); snapshot = std::move(next);
return true; return true;
} }
bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::string* error) { void SpecializeResources(Program& program, ResourceSnapshot& snapshot) {
if (!program.resource_tracking_complete || program.shader_info_complete || if (!program.resource_tracking_complete || program.shader_info_complete ||
program.binding_layout_complete) { program.binding_layout_complete) {
if (error != nullptr) { return SpecializationFail(!program.resource_tracking_complete
*error = !program.resource_tracking_complete ? "shader resources were not tracked" ? "shader resources were not tracked"
: "resource specialization is too late"; : "resource specialization is too late");
}
return false;
} }
if (!ValidateResourceSnapshot(program, snapshot, error)) { if (!ValidateResourceSnapshot(program, snapshot)) {
return false; return SpecializationFail("invalid resource snapshot");
} }
auto next = program.info; auto next = program.info;
@@ -681,10 +557,8 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
for (const auto& table: snapshot.indirect_images) { for (const auto& table: snapshot.indirect_images) {
if (table.resource >= next.images.size() || table.descriptors.size() < 2u || if (table.resource >= next.images.size() || table.descriptors.size() < 2u ||
next.images.size() + table.descriptors.size() - 1u > ShaderInfo::MaxImages) { next.images.size() + table.descriptors.size() - 1u > ShaderInfo::MaxImages) {
if (error != nullptr) { return SpecializationFail(
*error = "indirect image candidates exceed the dense image resource limit"; "indirect image candidates exceed the dense image resource limit");
}
return false;
} }
const auto root_image = next.images[table.resource]; const auto root_image = next.images[table.resource];
std::vector<uint32_t> resources(table.descriptors.size()); std::vector<uint32_t> resources(table.descriptors.size());
@@ -720,10 +594,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
for (uint32_t i = 0; i < next.buffers.size(); i++) { for (uint32_t i = 0; i < next.buffers.size(); i++) {
ShaderBufferResource descriptor; ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(next_snapshot.buffers[i], descriptor)) { if (!DecodeBufferDescriptor(next_snapshot.buffers[i], descriptor)) {
if (error != nullptr) { return SpecializationFail(fmt::format("buffer descriptor {} has invalid width", i));
*error = fmt::format("buffer descriptor {} has invalid width", i);
}
return false;
} }
next.buffers[i].packed_stride = descriptor.PackedStride(); next.buffers[i].packed_stride = descriptor.PackedStride();
next.buffers[i].descriptor_format = descriptor.Format(); next.buffers[i].descriptor_format = descriptor.Format();
@@ -734,10 +605,8 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
auto& image = next.images[i]; auto& image = next.images[i];
image.mip_count = StorageMipCount(image, descriptor); image.mip_count = StorageMipCount(image, descriptor);
if (image.mip_count == 0u) { if (image.mip_count == 0u) {
if (error != nullptr) { return SpecializationFail(
*error = fmt::format("storage image descriptor {} has an invalid mip range", i); fmt::format("storage image descriptor {} has an invalid mip range", i));
}
return false;
} }
if (NullImageDescriptor(descriptor)) { if (NullImageDescriptor(descriptor)) {
image.dimension = Decoder::ImageDimension::Dim2D; image.dimension = Decoder::ImageDimension::Dim2D;
@@ -755,37 +624,31 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
} }
const auto descriptor_dimension = DescriptorDimension(descriptor, image.dimension); const auto descriptor_dimension = DescriptorDimension(descriptor, image.dimension);
if (descriptor_dimension == Decoder::ImageDimension::Unknown) { if (descriptor_dimension == Decoder::ImageDimension::Unknown) {
if (error != nullptr) { return SpecializationFail(fmt::format(
*error = fmt::format( "image descriptor {} has unsupported type {}: {:08x},{:08x},{:08x},{:08x},"
"image descriptor {} has unsupported type {}: {:08x},{:08x},{:08x},{:08x}," "{:08x},{:08x},{:08x},{:08x}",
"{:08x},{:08x},{:08x},{:08x}", i, (descriptor.dwords[3] >> 28u) & 0xfu, descriptor.dwords[0], descriptor.dwords[1],
i, (descriptor.dwords[3] >> 28u) & 0xfu, descriptor.dwords[0], descriptor.dwords[2], descriptor.dwords[3], descriptor.dwords[4],
descriptor.dwords[1], descriptor.dwords[2], descriptor.dwords[3], descriptor.dwords[5], descriptor.dwords[6], descriptor.dwords[7]));
descriptor.dwords[4], descriptor.dwords[5], descriptor.dwords[6],
descriptor.dwords[7]);
}
return false;
} }
image.dimension = descriptor_dimension; image.dimension = descriptor_dimension;
image.cube = DescriptorIsCube(descriptor); image.cube = DescriptorIsCube(descriptor);
const auto format = const auto format =
static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu); static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
if (image.atomic && format != Prospero::BufferFormat::k32UInt) { if (image.atomic && format != Prospero::BufferFormat::k32UInt) {
if (error != nullptr) { return SpecializationFail(
*error = fmt::format("atomic image descriptor {} uses unsupported format {}", i, fmt::format("atomic image descriptor {} uses unsupported format {}", i,
static_cast<uint32_t>(format)); static_cast<uint32_t>(format)));
}
return false;
} }
const bool storage = image.kind == ResourceKind::StorageImage || const bool storage = image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint; image.kind == ResourceKind::StorageImageUint;
image.conversion_format = ImageConversionFormat(format); image.conversion_format = ImageConversionFormat(format);
if (storage || image.conversion_format != Prospero::BufferFormat::kInvalid) { if (storage || image.conversion_format != Prospero::BufferFormat::kInvalid) {
image.shader_swizzle = DescriptorImageSwizzle(descriptor); image.shader_swizzle = DescriptorImageSwizzle(descriptor);
} }
const bool raw_sint_storage = storage && format == Prospero::BufferFormat::k32SInt && const bool raw_sint_storage = storage && format == Prospero::BufferFormat::k32SInt &&
image.written && !image.read && !image.atomic; image.written && !image.read && !image.atomic;
const bool uint_image = Prospero::IsUintTextureFormat(format) || raw_sint_storage; const bool uint_image = Prospero::IsUintTextureFormat(format) || raw_sint_storage;
if (uint_image) { if (uint_image) {
switch (image.kind) { switch (image.kind) {
case ResourceKind::Image: image.kind = ResourceKind::ImageUint; break; case ResourceKind::Image: image.kind = ResourceKind::ImageUint; break;
@@ -803,19 +666,13 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
static_cast<size_t>(root.indirect_mapping_offset) + 1u + static_cast<size_t>(root.indirect_mapping_offset) + 1u +
static_cast<size_t>(root.indirect_mapping_capacity) * 2u > static_cast<size_t>(root.indirect_mapping_capacity) * 2u >
next_snapshot.flattened_srt.size()) { next_snapshot.flattened_srt.size()) {
if (error != nullptr) { return SpecializationFail("indirect image specialization has an invalid key mapping");
*error = "indirect image specialization has an invalid key mapping";
}
return false;
} }
uint32_t exemplar = ImageResource::NoIndirectImage; uint32_t exemplar = ImageResource::NoIndirectImage;
for (const auto resource: root.indirect_resources) { for (const auto resource: root.indirect_resources) {
if (resource >= next.images.size() || if (resource >= next.images.size() ||
next.images[resource].indirect_root != root_index) { next.images[resource].indirect_root != root_index) {
if (error != nullptr) { return SpecializationFail("indirect image specialization has an invalid candidate");
*error = "indirect image specialization has an invalid candidate";
}
return false;
} }
if (!NullImageDescriptor(next_snapshot.images[resource])) { if (!NullImageDescriptor(next_snapshot.images[resource])) {
exemplar = resource; exemplar = resource;
@@ -823,10 +680,7 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
} }
} }
if (exemplar == ImageResource::NoIndirectImage) { if (exemplar == ImageResource::NoIndirectImage) {
if (error != nullptr) { return SpecializationFail("indirect image specialization has no typed candidate");
*error = "indirect image specialization has no typed candidate";
}
return false;
} }
const auto& image_class = next.images[exemplar]; const auto& image_class = next.images[exemplar];
for (uint32_t candidate = 0; candidate < next.images.size(); candidate++) { for (uint32_t candidate = 0; candidate < next.images.size(); candidate++) {
@@ -849,12 +703,9 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
image.shader_swizzle != image_class.shader_swizzle || image.shader_swizzle != image_class.shader_swizzle ||
image.depth_compare != image_class.depth_compare || image.depth_compare != image_class.depth_compare ||
image.cube != image_class.cube) { image.cube != image_class.cube) {
if (error != nullptr) { return SpecializationFail(
*error = fmt::format( fmt::format("indirect image table at pc 0x{:08x} has incompatible candidates",
"indirect image table at pc 0x{:08x} has incompatible candidates", root.first_use_pc));
root.first_use_pc);
}
return false;
} }
} }
} }
@@ -867,11 +718,8 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
std::vector<SamplerUsage> sampler_usage(next.samplers.size()); std::vector<SamplerUsage> sampler_usage(next.samplers.size());
for (const auto& pair: next.sampled_pairs) { for (const auto& pair: next.sampled_pairs) {
if (pair.image >= next.images.size() || pair.sampler >= next.samplers.size()) { if (pair.image >= next.images.size() || pair.sampler >= next.samplers.size()) {
if (error != nullptr) { return SpecializationFail(
*error = fmt::format("sampled resource pair at pc 0x{:08x} is invalid", fmt::format("sampled resource pair at pc 0x{:08x} is invalid", pair.first_use_pc));
pair.first_use_pc);
}
return false;
} }
auto& usage = sampler_usage[pair.sampler]; auto& usage = sampler_usage[pair.sampler];
if (next.images[pair.image].conversion_format != Prospero::BufferFormat::kInvalid) { if (next.images[pair.image].conversion_format != Prospero::BufferFormat::kInvalid) {
@@ -891,10 +739,8 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
continue; continue;
} }
if (next.samplers.size() >= ShaderInfo::MaxSamplers) { if (next.samplers.size() >= ShaderInfo::MaxSamplers) {
if (error != nullptr) { return SpecializationFail(
*error = "point-filter sampler variants exceed the dense sampler resource limit"; "point-filter sampler variants exceed the dense sampler resource limit");
}
return false;
} }
point_sampler[i] = static_cast<uint32_t>(next.samplers.size()); point_sampler[i] = static_cast<uint32_t>(next.samplers.size());
auto sampler = next.samplers[i]; auto sampler = next.samplers[i];
@@ -915,19 +761,13 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
} }
const auto index = inst.Flags<MemoryFlags>().index; const auto index = inst.Flags<MemoryFlags>().index;
if (index >= memory_info.size()) { if (index >= memory_info.size()) {
if (error != nullptr) { return SpecializationFail(
*error = fmt::format("typed image instruction has invalid memory metadata {}", fmt::format("typed image instruction has invalid memory metadata {}", index));
index);
}
return false;
} }
auto& memory = memory_info[index]; auto& memory = memory_info[index];
if (memory.resource >= next.images.size()) { if (memory.resource >= next.images.size()) {
if (error != nullptr) { return SpecializationFail(fmt::format(
*error = fmt::format("typed image instruction has invalid resource {}", "typed image instruction has invalid resource {}", memory.resource));
memory.resource);
}
return false;
} }
const auto& image = next.images[memory.resource]; const auto& image = next.images[memory.resource];
if (image_opcode.needs_sampler && if (image_opcode.needs_sampler &&
@@ -937,22 +777,18 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
} }
if (image.indirect_root == memory.resource && if (image.indirect_root == memory.resource &&
inst.GetOpcode() != ValueOpcode::ImageSampleRaw) { inst.GetOpcode() != ValueOpcode::ImageSampleRaw) {
if (error != nullptr) { return SpecializationFail(
*error = fmt::format( fmt::format("indirect image table at pc 0x{:08x} is used by unsupported {}",
"indirect image table at pc 0x{:08x} is used by unsupported {}", inst.Flags<MemoryFlags>().pc, ValueOpcodeName(inst.GetOpcode())));
inst.Flags<MemoryFlags>().pc, ValueOpcodeName(inst.GetOpcode()));
}
return false;
} }
memory.kind = image.kind; memory.kind = image.kind;
memory.image_dimension = image.dimension; memory.image_dimension = image.dimension;
memory.image_cube = image.cube; memory.image_cube = image.cube;
} }
} }
program.info = std::move(next); program.info = std::move(next);
program.memory_info = std::move(memory_info); program.memory_info = std::move(memory_info);
snapshot = std::move(next_snapshot); snapshot = std::move(next_snapshot);
return true;
} }
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -22,19 +22,17 @@ struct ResourceSnapshot {
std::vector<IndirectImage> indirect_images; std::vector<IndirectImage> indirect_images;
}; };
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot, bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot);
std::string* error); bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot);
bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot,
std::string* error);
// Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot. // Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot.
// On failure the destination is unchanged. // On failure the destination is unchanged.
bool MaterializeResources(const Program& program, const SrtRuntime& runtime, bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot& snapshot, std::string* error); ResourceSnapshot& snapshot);
// Applies runtime descriptor shape/format facts to a copied dense topology before layout and // Applies runtime descriptor shape/format facts to a copied dense topology before layout and
// emission. On failure the program is unchanged. // emission. Invalid specializations are fatal.
bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::string* error); void SpecializeResources(Program& program, ResourceSnapshot& snapshot);
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -1,8 +1,8 @@
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "graphics/shader/shader.h"
#include <algorithm> #include <algorithm>
#include <fmt/format.h> #include <fmt/format.h>
@@ -12,7 +12,7 @@
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
namespace { namespace {
constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u); constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
constexpr uint32_t SamplerDword3ReservedMask = 0x3ffff000u; constexpr uint32_t SamplerDword3ReservedMask = 0x3ffff000u;
uint32_t PossibleU32Bits(Value value) { uint32_t PossibleU32Bits(Value value) {
@@ -101,19 +101,17 @@ public:
m_info.uses_dma = false; m_info.uses_dma = false;
} }
bool Run(std::string* error) { void Run() {
if (m_program.resource_tracking_complete) { if (m_program.resource_tracking_complete) {
return Fail(0, error, "resources already tracked"); Fail(0, "resources already tracked");
} }
if (!m_program.srt_plan_complete) { if (!m_program.srt_plan_complete) {
return Fail(0, error, "SRT plan is not ready"); Fail(0, "SRT plan is not ready");
} }
PlanIndirectImages(); PlanIndirectImages();
for (auto* block: m_program.blocks) { for (auto* block: m_program.blocks) {
for (auto& inst: *block) { for (auto& inst: *block) {
if (!Collect(inst, error)) { Collect(inst);
return false;
}
} }
} }
LinkImageAliases(); LinkImageAliases();
@@ -145,10 +143,9 @@ public:
return std::ranges::find(plan.reads, inst) != plan.reads.end(); return std::ranges::find(plan.reads, inst) != plan.reads.end();
}); });
}); });
m_program.descriptor_sources = std::move(m_sources); m_program.descriptor_sources = std::move(m_sources);
m_program.info = std::move(m_info); m_program.info = std::move(m_info);
m_program.resource_tracking_complete = true; m_program.resource_tracking_complete = true;
return true;
} }
private: private:
@@ -173,18 +170,19 @@ private:
std::array<const Inst*, 8> reads {}; std::array<const Inst*, 8> reads {};
}; };
bool Fail(uint32_t pc, std::string* error, const std::string& reason) const { [[noreturn]] void Fail(uint32_t pc, const std::string& reason) const {
return ShaderError::Fail( const auto message =
error, fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}", fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}",
m_program.shader_hash, StageName(m_program.stage), pc, reason)); m_program.shader_hash, StageName(m_program.stage), pc, reason);
EXIT("%s", message.c_str());
std::abort();
} }
bool MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust, void MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust,
DescriptorSource& descriptor, uint32_t pc, std::string* error) const { DescriptorSource& descriptor, uint32_t pc) const {
if (handle.NumArgs() != width) { if (handle.NumArgs() != width) {
return Fail(pc, error, Fail(pc, fmt::format("{} has {} descriptor dwords, expected {}",
fmt::format("{} has {} descriptor dwords, expected {}", ValueOpcodeName(handle.GetOpcode()), handle.NumArgs(), width));
ValueOpcodeName(handle.GetOpcode()), handle.NumArgs(), width));
} }
descriptor.dword_count = width; descriptor.dword_count = width;
for (uint32_t i = 0; i < width; i++) { for (uint32_t i = 0; i < width; i++) {
@@ -199,18 +197,15 @@ private:
// Border color and its table index are unused unless a clamp axis selects border mode. // Border color and its table index are unused unless a clamp axis selects border mode.
descriptor.dwords[3] = Value(0u); descriptor.dwords[3] = Value(0u);
} }
return true;
} }
bool ValidateSource(const DescriptorSource& descriptor, std::string& reason, bool ValidateSource(const DescriptorSource& descriptor, uint32_t& bad_dword) const {
uint32_t& bad_dword) const {
for (uint32_t i = 0; i < descriptor.dword_count; i++) { for (uint32_t i = 0; i < descriptor.dword_count; i++) {
bad_dword = i; bad_dword = i;
if (descriptor.dwords[i].Resolve().GetType() != Type::U32) { if (descriptor.dwords[i].Resolve().GetType() != Type::U32) {
reason = "is not U32";
return false; return false;
} }
if (!ValidateRuntimeValue(m_program, descriptor.dwords[i], reason)) { if (!ValidateRuntimeValue(m_program, descriptor.dwords[i])) {
return false; return false;
} }
} }
@@ -286,13 +281,12 @@ private:
bool MakeRuntimeBufferSource(const Inst& handle, uint32_t pc, uint32_t& source, bool MakeRuntimeBufferSource(const Inst& handle, uint32_t pc, uint32_t& source,
DescriptorSource& descriptor) { DescriptorSource& descriptor) {
if (handle.GetOpcode() != ValueOpcode::GetBufferResource || if (handle.GetOpcode() != ValueOpcode::GetBufferResource) {
!MakeSource(handle, 4u, false, false, descriptor, pc, nullptr)) {
return false; return false;
} }
std::string reason; MakeSource(handle, 4u, false, false, descriptor, pc);
uint32_t bad_dword = 0; uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) { if (!ValidateSource(descriptor, bad_dword)) {
return false; return false;
} }
source = InternSource(descriptor); source = InternSource(descriptor);
@@ -469,38 +463,30 @@ private:
} }
} }
bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle, void GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle,
uint32_t& source, std::string* error, bool sampler = false, uint32_t& source, bool sampler = false, bool sample_adjust = false) {
bool sample_adjust = false) {
handle = value.Resolve().TryInstruction(); handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != expected) { if (handle == nullptr || handle->GetOpcode() != expected) {
return Fail(pc, error, Fail(pc, fmt::format("memory operation requires {}", ValueOpcodeName(expected)));
fmt::format("memory operation requires {}", ValueOpcodeName(expected)));
} }
DescriptorSource descriptor; DescriptorSource descriptor;
if (!MakeSource(*handle, width, sampler, sample_adjust, descriptor, pc, error)) { MakeSource(*handle, width, sampler, sample_adjust, descriptor, pc);
return false; uint32_t bad_dword = 0;
} if (!ValidateSource(descriptor, bad_dword)) {
std::string reason; Fail(pc, fmt::format("{} dword {} is not a valid runtime value",
uint32_t bad_dword = 0; ValueOpcodeName(expected), bad_dword));
if (!ValidateSource(descriptor, reason, bad_dword)) {
return Fail(
pc, error,
fmt::format("{} dword {} {}", ValueOpcodeName(expected), bad_dword, reason));
} }
source = InternSource(descriptor); source = InternSource(descriptor);
return true;
} }
bool ValidateAddressHandle(Value value, uint32_t pc, std::string* error) const { void ValidateAddressHandle(Value value, uint32_t pc) const {
const auto* handle = value.Resolve().TryInstruction(); const auto* handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) { if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) {
return Fail(pc, error, "address operation requires GetAddressResource"); Fail(pc, "address operation requires GetAddressResource");
} }
if (handle->NumArgs() != 2) { if (handle->NumArgs() != 2) {
return Fail(pc, error, "GetAddressResource must have two address dwords"); Fail(pc, "GetAddressResource must have two address dwords");
} }
return true;
} }
uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) { uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
@@ -540,7 +526,7 @@ private:
const auto mip = MipMode(memory); const auto mip = MipMode(memory);
const bool depth = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0; const bool depth = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0;
for (uint32_t i = 0; i < m_info.images.size(); i++) { for (uint32_t i = 0; i < m_info.images.size(); i++) {
auto& image = m_info.images[i]; auto& image = m_info.images[i];
const bool compatible_kind = image.kind == memory.kind || (IsStorageImage(image.kind) && const bool compatible_kind = image.kind == memory.kind || (IsStorageImage(image.kind) &&
IsStorageImage(memory.kind)); IsStorageImage(memory.kind));
if (image.source == source && compatible_kind && if (image.source == source && compatible_kind &&
@@ -593,155 +579,139 @@ private:
return static_cast<uint32_t>(m_info.samplers.size() - 1); return static_cast<uint32_t>(m_info.samplers.size() - 1);
} }
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) { void AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc) {
for (auto& pair: m_info.sampled_pairs) { for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) { if (pair.image == image && pair.sampler == sampler) {
pair.first_use_pc = std::min(pair.first_use_pc, pc); pair.first_use_pc = std::min(pair.first_use_pc, pc);
return true; return;
} }
} }
if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) { if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) {
return Fail(pc, error, "sampled image/sampler pair limit exceeded"); Fail(pc, "sampled image/sampler pair limit exceeded");
} }
m_info.sampled_pairs.push_back({image, sampler, pc}); m_info.sampled_pairs.push_back({image, sampler, pc});
return true;
} }
bool AddHandlePatch(Inst* handle, uint32_t resource, uint32_t pc, std::string* error) { void AddHandlePatch(Inst* handle, uint32_t resource, uint32_t pc) {
for (const auto& patch: m_handle_patches) { for (const auto& patch: m_handle_patches) {
if (patch.handle == handle) { if (patch.handle == handle) {
return patch.resource == resource || if (patch.resource != resource) {
Fail(pc, error, Fail(pc, fmt::format("{} is reused with incompatible resource classes",
fmt::format("{} is reused with incompatible resource classes", ValueOpcodeName(handle->GetOpcode())));
ValueOpcodeName(handle->GetOpcode()))); }
return;
} }
} }
m_handle_patches.push_back({handle, resource}); m_handle_patches.push_back({handle, resource});
return true;
} }
bool AddMemoryPatch(uint32_t index, uint32_t resource, uint32_t sampler, bool has_sampler, void AddMemoryPatch(uint32_t index, uint32_t resource, uint32_t sampler, bool has_sampler,
uint32_t pc, std::string* error) { uint32_t pc) {
for (auto& patch: m_memory_patches) { for (auto& patch: m_memory_patches) {
if (patch.index != index) { if (patch.index != index) {
continue; continue;
} }
if (patch.resource != resource || if (patch.resource != resource ||
(has_sampler && patch.has_sampler && patch.sampler != sampler)) { (has_sampler && patch.has_sampler && patch.sampler != sampler)) {
return Fail(pc, error, "memory metadata is reused with incompatible resources"); Fail(pc, "memory metadata is reused with incompatible resources");
} }
if (has_sampler) { if (has_sampler) {
patch.sampler = sampler; patch.sampler = sampler;
patch.has_sampler = true; patch.has_sampler = true;
} }
return true; return;
} }
m_memory_patches.push_back({index, resource, sampler, has_sampler}); m_memory_patches.push_back({index, resource, sampler, has_sampler});
return true;
} }
bool Collect(Inst& inst, std::string* error) { void Collect(Inst& inst) {
const auto op = inst.GetOpcode(); const auto op = inst.GetOpcode();
const auto buffer = BufferAccessOf(op); const auto buffer = BufferAccessOf(op);
const auto address_info = AddressOpcodeInfoOf(op); const auto address_info = AddressOpcodeInfoOf(op);
const auto image_info = ImageOpcodeInfoOf(op); const auto image_info = ImageOpcodeInfoOf(op);
if (buffer == BufferAccess::None && address_info.access == AddressAccess::None && if (buffer == BufferAccess::None && address_info.access == AddressAccess::None &&
image_info.access == ImageAccess::None) { image_info.access == ImageAccess::None) {
return true; return;
} }
const auto flags = inst.Flags<MemoryFlags>(); const auto flags = inst.Flags<MemoryFlags>();
if (flags.index >= m_program.memory_info.size()) { if (flags.index >= m_program.memory_info.size()) {
return Fail(flags.pc, error, Fail(flags.pc, fmt::format("memory metadata index {} is out of range", flags.index));
fmt::format("memory metadata index {} is out of range", flags.index));
} }
if (inst.NumArgs() == 0) { if (inst.NumArgs() == 0) {
return Fail(flags.pc, error, "memory operation has no resource handle"); Fail(flags.pc, "memory operation has no resource handle");
} }
const auto& memory = m_program.memory_info[flags.index]; const auto& memory = m_program.memory_info[flags.index];
if (memory.planning_only || IsIndirectPlanningMemory(flags.index)) { if (memory.planning_only || IsIndirectPlanningMemory(flags.index)) {
return true; return;
} }
Inst* handle = nullptr; Inst* handle = nullptr;
uint32_t source = 0; uint32_t source = 0;
uint32_t resource = 0; uint32_t resource = 0;
if (buffer != BufferAccess::None) { if (buffer != BufferAccess::None) {
if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source, GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source);
error)) {
return false;
}
resource = AddBuffer(source, memory, op, flags.pc); resource = AddBuffer(source, memory, op, flags.pc);
if (resource == UINT32_MAX) { if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "buffer resource limit exceeded"); Fail(flags.pc, "buffer resource limit exceeded");
} }
return AddHandlePatch(handle, resource, flags.pc, error) && AddHandlePatch(handle, resource, flags.pc);
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error); AddMemoryPatch(flags.index, resource, 0, false, flags.pc);
return;
} }
if (address_info.access != AddressAccess::None) { if (address_info.access != AddressAccess::None) {
if (!IsAddressResourceKind(memory.kind)) { if (!IsAddressResourceKind(memory.kind)) {
return Fail(flags.pc, error, "address operation has invalid resource kind"); Fail(flags.pc, "address operation has invalid resource kind");
} }
if (memory.kind == ResourceKind::Scratch) { if (memory.kind == ResourceKind::Scratch) {
handle = inst.Arg(0).Resolve().TryInstruction(); handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource || if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource ||
handle->NumArgs() != 0) { handle->NumArgs() != 0) {
return Fail(flags.pc, error, "scratch operation requires GetScratchResource"); Fail(flags.pc, "scratch operation requires GetScratchResource");
} }
if (m_program.scratch_dwords == 0) { if (m_program.scratch_dwords == 0) {
return Fail(flags.pc, error, Fail(flags.pc, "scratch operation requires a nonzero AGC per-thread size");
"scratch operation requires a nonzero AGC per-thread size");
} }
return true; return;
}
if (!ValidateAddressHandle(inst.Arg(0), flags.pc, error)) {
return false;
} }
ValidateAddressHandle(inst.Arg(0), flags.pc);
m_info.uses_dma = true; m_info.uses_dma = true;
return true; return;
} }
if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) { if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) {
return Fail(flags.pc, error, "image operation has invalid resource kind"); Fail(flags.pc, "image operation has invalid resource kind");
} }
handle = inst.Arg(0).Resolve().TryInstruction(); handle = inst.Arg(0).Resolve().TryInstruction();
const auto* indirect = handle != nullptr ? FindIndirectImage(*handle) : nullptr; const auto* indirect = handle != nullptr ? FindIndirectImage(*handle) : nullptr;
if (indirect != nullptr) { if (indirect != nullptr) {
source = indirect->source; source = indirect->source;
} else if (!GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, } else {
source, error)) { GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, source);
return false;
} }
resource = AddImage(source, memory, op, flags.pc); resource = AddImage(source, memory, op, flags.pc);
if (resource == UINT32_MAX) { if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "image resource limit exceeded"); Fail(flags.pc, "image resource limit exceeded");
}
if (!AddHandlePatch(handle, resource, flags.pc, error)) {
return false;
} }
AddHandlePatch(handle, resource, flags.pc);
uint32_t sampler = 0; uint32_t sampler = 0;
if (image_info.needs_sampler) { if (image_info.needs_sampler) {
if (inst.NumArgs() < 2) { if (inst.NumArgs() < 2) {
return Fail(flags.pc, error, "sampled image operation has no sampler handle"); Fail(flags.pc, "sampled image operation has no sampler handle");
} }
Inst* sampler_handle = nullptr; Inst* sampler_handle = nullptr;
uint32_t sampler_source = 0; uint32_t sampler_source = 0;
const bool sample_adjust = const bool sample_adjust =
(memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0; (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0;
if (!GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, sampler_handle,
sampler_handle, sampler_source, error, true, sample_adjust)) { sampler_source, true, sample_adjust);
return false;
}
sampler = AddSampler(sampler_source, flags.pc); sampler = AddSampler(sampler_source, flags.pc);
if (sampler == UINT32_MAX) { if (sampler == UINT32_MAX) {
return Fail(flags.pc, error, "sampler resource limit exceeded"); Fail(flags.pc, "sampler resource limit exceeded");
}
if (!AddHandlePatch(sampler_handle, sampler, flags.pc, error) ||
!AddSampledPair(resource, sampler, flags.pc, error)) {
return false;
} }
AddHandlePatch(sampler_handle, sampler, flags.pc);
AddSampledPair(resource, sampler, flags.pc);
} }
return AddMemoryPatch(flags.index, resource, sampler, image_info.needs_sampler, flags.pc, AddMemoryPatch(flags.index, resource, sampler, image_info.needs_sampler, flags.pc);
error);
} }
const DescriptorSource* Source(uint32_t source) const { const DescriptorSource* Source(uint32_t source) const {
@@ -783,8 +753,8 @@ private:
} // namespace } // namespace
bool TrackResources(Program& program, std::string* error) { void TrackResources(Program& program) {
return Tracker(program).Run(error); Tracker(program).Run();
} }
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -7,7 +7,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in // Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
// descriptor_sources, then writes dense indices to handle flags and MemoryInfo. // descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
bool TrackResources(Program& program, std::string* error); void TrackResources(Program& program);
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -1,5 +1,6 @@
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" #include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <algorithm> #include <algorithm>
@@ -8,6 +9,11 @@
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
namespace { namespace {
[[noreturn]] void Fail(std::string_view message) {
EXIT("shader info collection failed: %s", std::string(message).c_str());
std::abort();
}
void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components, void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components,
std::string name, bool per_vertex = false) { std::string name, bool per_vertex = false) {
const auto input = std::find_if(info.inputs.begin(), info.inputs.end(), [=](const auto& value) { const auto input = std::find_if(info.inputs.begin(), info.inputs.end(), [=](const auto& value) {
@@ -34,38 +40,38 @@ void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t
} }
} }
bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) { void ValidateOptions(const Program& program, const ShaderInfoOptions& options) {
auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
switch (program.stage) { switch (program.stage) {
case ShaderType::Vertex: case ShaderType::Vertex:
if (options.vertex == nullptr) {
return Fail("vertex shader has no input metadata");
}
if (options.vertex->resources_num < 0 || if (options.vertex->resources_num < 0 ||
options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) { options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) {
return Fail("vertex resource count is out of range"); return Fail("vertex resource count is out of range");
} }
return true; return;
case ShaderType::Pixel: case ShaderType::Pixel:
return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) || if (options.pixel == nullptr) {
Fail("pixel input count is out of range"); return Fail("pixel shader has no input metadata");
}
if (options.pixel->input_num > std::size(options.pixel->interpolator_settings)) {
return Fail("pixel input count is out of range");
}
return;
case ShaderType::Compute: case ShaderType::Compute:
return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) || if (options.compute == nullptr) {
Fail("compute thread ID count is out of range"); return Fail("compute shader has no input metadata");
}
if (options.compute->thread_ids_num < 0 || options.compute->thread_ids_num > 3) {
return Fail("compute thread ID count is out of range");
}
return;
default: return Fail("unsupported shader stage for info collection"); default: return Fail("unsupported shader stage for info collection");
} }
} }
bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& options, void ValidateValueReferences(const Program& program, const ShaderInfoOptions& options) {
std::string* error) {
const auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
switch (inst.GetOpcode()) { switch (inst.GetOpcode()) {
@@ -148,7 +154,6 @@ bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& op
} }
} }
} }
return true;
} }
void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex, void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex,
@@ -269,14 +274,8 @@ void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
} }
} }
bool CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex, void CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex,
const ShaderPixelInputInfo* pixel, ShaderInfo& info, std::string* error) { const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
auto Fail = [&](std::string message) {
if (error != nullptr) {
*error = std::move(message);
}
return false;
};
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::SetAttribute) { if (inst.GetOpcode() != ValueOpcode::SetAttribute) {
@@ -347,31 +346,23 @@ bool CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex,
} }
} }
} }
return true;
} }
} // namespace } // namespace
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) { void CollectShaderInfo(Program& program, const ShaderInfoOptions& options) {
if (!program.resource_tracking_complete || program.shader_info_complete) { if (!program.resource_tracking_complete || program.shader_info_complete) {
if (error != nullptr) { return Fail(!program.resource_tracking_complete ? "shader resources were not tracked"
*error = !program.resource_tracking_complete ? "shader resources were not tracked" : "shader info already collected");
: "shader info already collected";
}
return false;
}
if (!ValidateOptions(program, options, error)) {
return false;
}
if (!ValidateValueReferences(program, options, error)) {
return false;
} }
ValidateOptions(program, options);
ValidateValueReferences(program, options);
auto next = program.info; auto next = program.info;
next.inputs.clear(); next.inputs.clear();
next.outputs.clear(); next.outputs.clear();
next.has_bitwise_xor = std::any_of( next.has_bitwise_xor =
program.blocks.begin(), program.blocks.end(), [](const auto* block) { std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto* block) {
return std::any_of(block->begin(), block->end(), [](const auto& inst) { return std::any_of(block->begin(), block->end(), [](const auto& inst) {
return inst.GetOpcode() == ValueOpcode::BitwiseXor32; return inst.GetOpcode() == ValueOpcode::BitwiseXor32;
}); });
@@ -380,15 +371,12 @@ bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break; case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break;
case ShaderType::Pixel: CollectPixelInputs(program, options.pixel, next); break; case ShaderType::Pixel: CollectPixelInputs(program, options.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(options.compute, next); break; case ShaderType::Compute: CollectComputeInputs(options.compute, next); break;
default: return false; default: return Fail("unsupported shader stage for info collection");
} }
CollectBuiltinInputs(program, next); CollectBuiltinInputs(program, next);
if (!CollectOutputs(program, options.vertex, options.pixel, next, error)) { CollectOutputs(program, options.vertex, options.pixel, next);
return false;
}
program.info = std::move(next); program.info = std::move(next);
program.shader_info_complete = true; program.shader_info_complete = true;
return true;
} }
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -13,7 +13,7 @@ struct ShaderInfoOptions {
// Completes the immutable shader interface after resource tracking. On failure Program::info and // Completes the immutable shader interface after resource tracking. On failure Program::info and
// all completion state remain unchanged. // all completion state remain unchanged.
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error); void CollectShaderInfo(Program& program, const ShaderInfoOptions& options);
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -1,7 +1,7 @@
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h"
#include <algorithm> #include <algorithm>
#include <bit> #include <bit>
@@ -120,10 +120,10 @@ class RuntimeValidator {
public: public:
explicit RuntimeValidator(const Program& program): m_program(program) {} explicit RuntimeValidator(const Program& program): m_program(program) {}
bool Run(Value value, std::string& reason) { return Validate(value, reason); } bool Run(Value value) { return Validate(value); }
private: private:
bool Validate(Value value, std::string& reason) { bool Validate(Value value) {
value = value.Resolve(); value = value.Resolve();
const auto* inst = value.TryInstruction(); const auto* inst = value.TryInstruction();
if (inst == nullptr) { if (inst == nullptr) {
@@ -133,14 +133,10 @@ private:
case Type::U16: case Type::U16:
case Type::U32: case Type::U32:
case Type::U64: return true; case Type::U64: return true;
default: default: return false;
reason = fmt::format("contains a non-integer immediate of type {}",
TypeName(value.GetType()));
return false;
} }
} }
if (!m_visiting.insert(inst).second) { if (!m_visiting.insert(inst).second) {
reason = fmt::format("contains a cyclic {} value", ValueOpcodeName(inst->GetOpcode()));
return false; return false;
} }
const auto finish = [&](bool valid) { const auto finish = [&](bool valid) {
@@ -151,25 +147,21 @@ private:
if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU8 || if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU8 ||
op == ValueOpcode::UndefU16 || op == ValueOpcode::UndefU32 || op == ValueOpcode::UndefU16 || op == ValueOpcode::UndefU32 ||
op == ValueOpcode::UndefU64 || op == ValueOpcode::Void) { op == ValueOpcode::UndefU64 || op == ValueOpcode::Void) {
reason = fmt::format("contains {}", ValueOpcodeName(op));
return finish(false); return finish(false);
} }
if (op == ValueOpcode::GetUserData) { if (op == ValueOpcode::GetUserData) {
if (inst->NumArgs() != 1 || inst->Arg(0).GetType() != Type::ScalarReg) { if (inst->NumArgs() != 1 || inst->Arg(0).GetType() != Type::ScalarReg) {
reason = "contains a malformed GetUserData";
return finish(false); return finish(false);
} }
const auto reg = RegIndex(inst->Arg(0).ScalarRegister()); const auto reg = RegIndex(inst->Arg(0).ScalarRegister());
if (reg < m_program.user_data_base || if (reg < m_program.user_data_base ||
reg - m_program.user_data_base >= m_program.user_data_count) { reg - m_program.user_data_base >= m_program.user_data_count) {
reason = fmt::format("references unavailable user SGPR {}", reg);
return finish(false); return finish(false);
} }
return finish(true); return finish(true);
} }
if (op == ValueOpcode::GetShaderBase) { if (op == ValueOpcode::GetShaderBase) {
if (inst->NumArgs() != 0) { if (inst->NumArgs() != 0) {
reason = "contains a malformed GetShaderBase";
return finish(false); return finish(false);
} }
return finish(true); return finish(true);
@@ -177,14 +169,12 @@ private:
if (op == ValueOpcode::Phi) { if (op == ValueOpcode::Phi) {
const auto invariant = ResolveInvariantPhi(m_program, value); const auto invariant = ResolveInvariantPhi(m_program, value);
if (invariant.IsEmpty()) { if (invariant.IsEmpty()) {
reason = "contains a control-dependent phi";
return finish(false); return finish(false);
} }
return finish(Validate(invariant, reason)); return finish(Validate(invariant));
} }
if (op == ValueOpcode::GetSrtResource) { if (op == ValueOpcode::GetSrtResource) {
if (inst->NumArgs() != 0) { if (inst->NumArgs() != 0) {
reason = "contains a malformed GetSrtResource";
return finish(false); return finish(false);
} }
return finish(true); return finish(true);
@@ -196,7 +186,6 @@ private:
ValueOpcode::GetSrtResource || ValueOpcode::GetSrtResource ||
!slot.IsImmediate() || slot.GetType() != Type::U32 || !slot.IsImmediate() || slot.GetType() != Type::U32 ||
slot.U32() >= m_program.srt_reads.size()) { slot.U32() >= m_program.srt_reads.size()) {
reason = "contains a malformed flattened SRT read";
return finish(false); return finish(false);
} }
} else if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) { } else if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
@@ -206,13 +195,11 @@ private:
const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr; const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr;
if (!IsRawRead(m_program, *inst) || handle == nullptr || if (!IsRawRead(m_program, *inst) || handle == nullptr ||
handle->GetOpcode() != expected) { handle->GetOpcode() != expected) {
reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op));
return finish(false); return finish(false);
} }
} else if (op == ValueOpcode::CompositeExtractU64) { } else if (op == ValueOpcode::CompositeExtractU64) {
const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {}; const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) { if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) {
reason = "contains an invalid U64 component index";
return finish(false); return finish(false);
} }
} else if (op == ValueOpcode::CompositeExtractU32x2) { } else if (op == ValueOpcode::CompositeExtractU32x2) {
@@ -222,7 +209,6 @@ private:
index.U32() >= 2u || index.U32() >= 2u ||
(source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 && (source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 &&
source->GetOpcode() != ValueOpcode::IAddCarry32)) { source->GetOpcode() != ValueOpcode::IAddCarry32)) {
reason = "contains an unsupported composite runtime source";
return finish(false); return finish(false);
} }
} }
@@ -233,17 +219,14 @@ private:
: op == ValueOpcode::GetSamplerResource ? 4u : op == ValueOpcode::GetSamplerResource ? 4u
: 2u; : 2u;
if (inst->NumArgs() != expected) { if (inst->NumArgs() != expected) {
reason = fmt::format("contains a malformed {}", ValueOpcodeName(op));
return finish(false); return finish(false);
} }
} else if (op != ValueOpcode::ReadConst && op != ValueOpcode::ReadConstBuffer && } else if (op != ValueOpcode::ReadConst && op != ValueOpcode::ReadConstBuffer &&
op != ValueOpcode::LoadAddressU32 && !IsRuntimeIntegerOp(op)) { op != ValueOpcode::LoadAddressU32 && !IsRuntimeIntegerOp(op)) {
reason =
fmt::format("contains unsupported or control-dependent {}", ValueOpcodeName(op));
return finish(false); return finish(false);
} }
for (size_t index = 0; index < inst->NumArgs(); index++) { for (size_t index = 0; index < inst->NumArgs(); index++) {
if (!Validate(inst->Arg(index), reason)) { if (!Validate(inst->Arg(index))) {
return finish(false); return finish(false);
} }
} }
@@ -258,7 +241,7 @@ class PlanBuilder {
public: public:
explicit PlanBuilder(Program& program): m_program(program) {} explicit PlanBuilder(Program& program): m_program(program) {}
bool Run(std::string* error) { void Run() {
m_program.srt_reads.clear(); m_program.srt_reads.clear();
m_program.dynamic_reads.clear(); m_program.dynamic_reads.clear();
for (auto* block: m_program.blocks) { for (auto* block: m_program.blocks) {
@@ -273,34 +256,29 @@ public:
(op == ValueOpcode::ReadConstBuffer && (op == ValueOpcode::ReadConstBuffer &&
kind == ResourceKind::ScalarAddress); kind == ResourceKind::ScalarAddress);
if (crosswired) { if (crosswired) {
return Fail(flags.pc, error, Fail(flags.pc,
fmt::format("{} has incompatible scalar memory metadata", fmt::format("{} has incompatible scalar memory metadata",
ValueOpcodeName(op))); ValueOpcodeName(op)));
} }
} }
} }
if (IsDescriptorHandle(inst.GetOpcode())) { if (IsDescriptorHandle(inst.GetOpcode())) {
for (size_t index = 0; index < inst.NumArgs(); index++) { for (size_t index = 0; index < inst.NumArgs(); index++) {
if (!Collect(inst.Arg(index), 0, error)) { Collect(inst.Arg(index), 0);
return false;
}
} }
} }
} }
} }
for (auto* block: m_program.blocks) { for (auto* block: m_program.blocks) {
for (auto& inst: *block) { for (auto& inst: *block) {
std::string reason;
if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) && if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) &&
inst.Arg(1).Resolve().IsImmediate() && inst.Arg(1).Resolve().IsImmediate() &&
ValidateRuntimeValue(m_program, Value(&inst), reason) && ValidateRuntimeValue(m_program, Value(&inst))) {
!Collect(Value(&inst), inst.Flags<MemoryFlags>().pc, error)) { Collect(Value(&inst), inst.Flags<MemoryFlags>().pc);
return false;
} }
} }
} }
PatchReads(); PatchReads();
return true;
} }
private: private:
@@ -310,18 +288,20 @@ private:
bool keep = false; bool keep = false;
}; };
bool Fail(uint32_t pc, std::string* error, const std::string& message) const { [[noreturn]] void Fail(uint32_t pc, const std::string& message) const {
return ShaderError::Fail(error, Diagnostic(m_program, pc, message)); const auto diagnostic = Diagnostic(m_program, pc, message);
EXIT("shader SRT planning failed: %s", diagnostic.c_str());
std::abort();
} }
bool Collect(Value value, uint32_t use_pc, std::string* error) { void Collect(Value value, uint32_t use_pc) {
value = value.Resolve(); value = value.Resolve();
if (value.IsImmediate()) { if (value.IsImmediate()) {
return true; return;
} }
auto* inst = value.TryInstruction(); auto* inst = value.TryInstruction();
if (inst == nullptr) { if (inst == nullptr) {
return Fail(use_pc, error, "invalid typed planning value"); Fail(use_pc, "invalid typed planning value");
} }
const auto cycle = std::ranges::find(m_visiting, inst); const auto cycle = std::ranges::find(m_visiting, inst);
if (cycle != m_visiting.end()) { if (cycle != m_visiting.end()) {
@@ -329,44 +309,40 @@ private:
return value->GetOpcode() == ValueOpcode::Phi; return value->GetOpcode() == ValueOpcode::Phi;
}); });
if (contains_phi) { if (contains_phi) {
return true; return;
} }
return Fail(use_pc, error, Fail(use_pc, fmt::format("cyclic typed planning value {} without a phi",
fmt::format("cyclic typed planning value {} without a phi", ValueOpcodeName(inst->GetOpcode())));
ValueOpcodeName(inst->GetOpcode())));
} }
if (std::ranges::find(m_visited, inst) != m_visited.end()) { if (std::ranges::find(m_visited, inst) != m_visited.end()) {
return true; return;
} }
m_visiting.push_back(inst); m_visiting.push_back(inst);
for (size_t index = 0; index < inst->NumArgs(); index++) { for (size_t index = 0; index < inst->NumArgs(); index++) {
if (!Collect(inst->Arg(index), use_pc, error)) { Collect(inst->Arg(index), use_pc);
return false;
}
} }
m_visiting.pop_back(); m_visiting.pop_back();
m_visited.push_back(inst); m_visited.push_back(inst);
if (!IsRawRead(m_program, *inst)) { if (!IsRawRead(m_program, *inst)) {
return true; return;
} }
const auto offset = inst->Arg(1).Resolve(); const auto offset = inst->Arg(1).Resolve();
if (!offset.IsImmediate() || offset.GetType() != Type::U32) { if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
if (std::ranges::find(m_program.dynamic_reads, value) == m_program.dynamic_reads.end()) { if (std::ranges::find(m_program.dynamic_reads, value) ==
m_program.dynamic_reads.end()) {
m_program.dynamic_reads.push_back(value); m_program.dynamic_reads.push_back(value);
} }
return true; return;
} }
use_pc = inst->Flags<MemoryFlags>().pc;
for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) { for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) {
if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) { if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) {
m_patches.push_back({inst, slot, false}); m_patches.push_back({inst, slot, false});
return true; return;
} }
} }
const auto slot = static_cast<uint32_t>(m_program.srt_reads.size()); const auto slot = static_cast<uint32_t>(m_program.srt_reads.size());
m_program.srt_reads.push_back({value, slot, use_pc}); m_program.srt_reads.push_back({value, slot});
m_patches.push_back({inst, slot, true}); m_patches.push_back({inst, slot, true});
return true;
} }
void PatchReads() { void PatchReads() {
@@ -411,14 +387,12 @@ class Evaluator {
public: public:
Evaluator(const Program& program, const SrtRuntime& runtime, Evaluator(const Program& program, const SrtRuntime& runtime,
std::span<const uint8_t> clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr) std::span<const uint8_t> clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr)
: m_program(program), m_runtime(runtime), : m_program(program), m_runtime(runtime), m_clean_flat_slots(clean_flat_slots),
m_clean_flat_slots(clean_flat_slots), m_clean_evaluator(clean_evaluator) {} m_clean_evaluator(clean_evaluator) {}
void SetUsePc(uint32_t pc) { m_use_pc = pc; } bool Evaluate(Value value, uint32_t& result) {
bool Evaluate(Value value, uint32_t& result, std::string* error) {
uint64_t wide = 0; uint64_t wide = 0;
if (!EvaluateWide(value, wide, error)) { if (!EvaluateWide(value, wide)) {
return false; return false;
} }
result = static_cast<uint32_t>(wide); result = static_cast<uint32_t>(wide);
@@ -426,11 +400,7 @@ public:
} }
private: private:
bool Fail(std::string* error, const std::string& message) const { bool EvaluateWide(Value value, uint64_t& result) {
return ShaderError::Fail(error, Diagnostic(m_program, m_use_pc, message));
}
bool EvaluateWide(Value value, uint64_t& result, std::string* error) {
value = value.Resolve(); value = value.Resolve();
if (value.IsImmediate()) { if (value.IsImmediate()) {
switch (value.GetType()) { switch (value.GetType()) {
@@ -439,23 +409,23 @@ private:
case Type::U16: result = value.U16(); return true; case Type::U16: result = value.U16(); return true;
case Type::U32: result = value.U32(); return true; case Type::U32: result = value.U32(); return true;
case Type::U64: result = value.U64(); return true; case Type::U64: result = value.U64(); return true;
default: return Fail(error, "non-integer immediate in runtime expression"); default: return false;
} }
} }
auto* inst = value.TryInstruction(); auto* inst = value.TryInstruction();
if (inst == nullptr) { if (inst == nullptr) {
return Fail(error, "invalid typed runtime value"); return false;
} }
if (const auto found = m_cache.find(inst); found != m_cache.end()) { if (const auto found = m_cache.find(inst); found != m_cache.end()) {
result = found->second; result = found->second;
return true; return true;
} }
if (std::ranges::find(m_visiting, inst) != m_visiting.end()) { if (std::ranges::find(m_visiting, inst) != m_visiting.end()) {
return Fail(error, "cyclic typed runtime value"); return false;
} }
m_visiting.push_back(inst); m_visiting.push_back(inst);
uint64_t out = 0; uint64_t out = 0;
if (!EvaluateInst(*inst, out, error)) { if (!EvaluateInst(*inst, out)) {
return false; return false;
} }
m_visiting.pop_back(); m_visiting.pop_back();
@@ -464,28 +434,27 @@ private:
return true; return true;
} }
bool Arg(const Inst& inst, size_t index, uint64_t& result, std::string* error) { bool Arg(const Inst& inst, size_t index, uint64_t& result) {
return EvaluateWide(inst.Arg(index), result, error); return EvaluateWide(inst.Arg(index), result);
} }
bool EvaluatePhi(const Inst& inst, uint64_t& result, std::string* error) { bool EvaluatePhi(const Inst& inst, uint64_t& result) {
const auto value = ResolveInvariantPhi(m_program, Value(const_cast<Inst*>(&inst))); const auto value = ResolveInvariantPhi(m_program, Value(const_cast<Inst*>(&inst)));
return !value.IsEmpty() ? EvaluateWide(value, result, error) return !value.IsEmpty() && EvaluateWide(value, result);
: Fail(error, "typed phi has runtime-dependent values");
} }
bool EvaluateExtract(const Inst& inst, uint64_t& result, std::string* error) { bool EvaluateExtract(const Inst& inst, uint64_t& result) {
const auto index = inst.Arg(1).Resolve(); const auto index = inst.Arg(1).Resolve();
if (!index.IsImmediate() || index.GetType() != Type::U32) { if (!index.IsImmediate() || index.GetType() != Type::U32) {
return Fail(error, "dynamic composite extract in runtime expression"); return false;
} }
const auto component = index.U32(); const auto component = index.U32();
if (component >= 2u) { if (component >= 2u) {
return Fail(error, "unsupported composite runtime source"); return false;
} }
if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) { if (inst.GetOpcode() == ValueOpcode::CompositeExtractU64) {
uint64_t packed = 0; uint64_t packed = 0;
if (!Arg(inst, 0, packed, error)) { if (!Arg(inst, 0, packed)) {
return false; return false;
} }
result = static_cast<uint32_t>(packed >> (component * 32u)); result = static_cast<uint32_t>(packed >> (component * 32u));
@@ -493,15 +462,15 @@ private:
} }
const auto* source = inst.Arg(0).ResolveInstruction(); const auto* source = inst.Arg(0).ResolveInstruction();
if (source == nullptr) { if (source == nullptr) {
return Fail(error, "unsupported composite runtime source"); return false;
} }
if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
return EvaluateWide(source->Arg(component), result, error); return EvaluateWide(source->Arg(component), result);
} }
if (source->GetOpcode() == ValueOpcode::IAddCarry32) { if (source->GetOpcode() == ValueOpcode::IAddCarry32) {
uint64_t lhs = 0; uint64_t lhs = 0;
uint64_t rhs = 0; uint64_t rhs = 0;
if (!Arg(*source, 0, lhs, error) || !Arg(*source, 1, rhs, error)) { if (!Arg(*source, 0, lhs) || !Arg(*source, 1, rhs)) {
return false; return false;
} }
const auto sum = const auto sum =
@@ -510,24 +479,23 @@ private:
component == 0u ? static_cast<uint32_t>(sum) : static_cast<uint32_t>(sum >> 32u); component == 0u ? static_cast<uint32_t>(sum) : static_cast<uint32_t>(sum >> 32u);
return true; return true;
} }
return Fail(error, "unsupported composite runtime source"); return false;
} }
bool EvaluateRawRead(const Inst& inst, uint64_t& result, std::string* error) { bool EvaluateRawRead(const Inst& inst, uint64_t& result) {
const auto flags = inst.Flags<MemoryFlags>(); const auto flags = inst.Flags<MemoryFlags>();
if (flags.index >= m_program.memory_info.size()) { if (flags.index >= m_program.memory_info.size()) {
return Fail(error, "raw scalar read has invalid metadata"); return false;
} }
const auto& mem = m_program.memory_info[flags.index]; const auto& mem = m_program.memory_info[flags.index];
const auto* handle = inst.Arg(0).ResolveInstruction(); const auto* handle = inst.Arg(0).ResolveInstruction();
if (handle == nullptr) { if (handle == nullptr) {
return Fail(error, "raw scalar read has no descriptor handle"); return false;
} }
uint64_t low = 0; uint64_t low = 0;
uint64_t high = 0; uint64_t high = 0;
uint64_t offset = 0; uint64_t offset = 0;
if (!Arg(*handle, 0, low, error) || !Arg(*handle, 1, high, error) || if (!Arg(*handle, 0, low) || !Arg(*handle, 1, high) || !Arg(inst, 1, offset)) {
!Arg(inst, 1, offset, error)) {
return false; return false;
} }
const auto base = ((high << 32u) | static_cast<uint32_t>(low)) & AddressMask; const auto base = ((high << 32u) | static_cast<uint32_t>(low)) & AddressMask;
@@ -536,12 +504,11 @@ private:
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
uint64_t records = 0; uint64_t records = 0;
uint64_t word3 = 0; uint64_t word3 = 0;
if (handle->NumArgs() != 4u || !Arg(*handle, 2, records, error) || if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) {
!Arg(*handle, 3, word3, error)) { return false;
return Fail(error, "constant-buffer descriptor has invalid width");
} }
if (immediate < 0) { if (immediate < 0) {
return Fail(error, "constant-buffer read has a negative immediate offset"); return false;
} }
const auto byte_offset = const auto byte_offset =
static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset); static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
@@ -551,21 +518,20 @@ private:
? static_cast<uint64_t>(static_cast<uint32_t>(records)) ? static_cast<uint64_t>(static_cast<uint32_t>(records))
: static_cast<uint64_t>(stride) * static_cast<uint32_t>(records); : static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
if (aligned > size || size - aligned < sizeof(uint32_t)) { if (aligned > size || size - aligned < sizeof(uint32_t)) {
return Fail( return false;
error, fmt::format("constant-buffer offset {} exceeds size {}", aligned, size));
} }
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
} else { } else {
const auto relative = (immediate & ~int64_t {3}) + const auto relative = (immediate & ~int64_t {3}) +
static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u); static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) { if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) {
return Fail(error, "raw scalar read is outside the 48-bit address space"); return false;
} }
} }
uint32_t word = 0; uint32_t word = 0;
if (m_runtime.read_memory != nullptr) { if (m_runtime.read_memory != nullptr) {
if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) { if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) {
return Fail(error, fmt::format("constant read failed at 0x{:016x}", address)); return false;
} }
} else { } else {
std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word)); std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
@@ -574,28 +540,28 @@ private:
return true; return true;
} }
bool EvaluateInst(const Inst& inst, uint64_t& result, std::string* error) { bool EvaluateInst(const Inst& inst, uint64_t& result) {
uint64_t a = 0; uint64_t a = 0;
uint64_t b = 0; uint64_t b = 0;
uint64_t c = 0; uint64_t c = 0;
const auto binary = [&]() { return Arg(inst, 0, a, error) && Arg(inst, 1, b, error); }; const auto binary = [&]() { return Arg(inst, 0, a) && Arg(inst, 1, b); };
const auto ternary = [&]() { const auto ternary = [&]() {
return Arg(inst, 0, a, error) && Arg(inst, 1, b, error) && Arg(inst, 2, c, error); return Arg(inst, 0, a) && Arg(inst, 1, b) && Arg(inst, 2, c);
}; };
switch (inst.GetOpcode()) { switch (inst.GetOpcode()) {
case ValueOpcode::GetUserData: { case ValueOpcode::GetUserData: {
const auto reg = RegIndex(inst.Arg(0).ScalarRegister()); const auto reg = RegIndex(inst.Arg(0).ScalarRegister());
if (reg < m_program.user_data_base || if (reg < m_program.user_data_base ||
reg - m_program.user_data_base >= m_runtime.user_data.size()) { reg - m_program.user_data_base >= m_runtime.user_data.size()) {
return Fail(error, fmt::format("user SGPR {} is unavailable", reg)); return false;
} }
result = m_runtime.user_data[reg - m_program.user_data_base]; result = m_runtime.user_data[reg - m_program.user_data_base];
return true; return true;
} }
case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
case ValueOpcode::Phi: return EvaluatePhi(inst, result, error); case ValueOpcode::Phi: return EvaluatePhi(inst, result);
case ValueOpcode::CompositeExtractU64: case ValueOpcode::CompositeExtractU64:
case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result, error); case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result);
case ValueOpcode::CompositeConstructU64: case ValueOpcode::CompositeConstructU64:
if (!binary()) { if (!binary()) {
return false; return false;
@@ -607,20 +573,19 @@ private:
const auto slot = inst.Arg(1).Resolve(); const auto slot = inst.Arg(1).Resolve();
if (!slot.IsImmediate() || slot.GetType() != Type::U32 || if (!slot.IsImmediate() || slot.GetType() != Type::U32 ||
slot.U32() >= m_program.srt_reads.size()) { slot.U32() >= m_program.srt_reads.size()) {
return Fail(error, "invalid flattened SRT index"); return false;
} }
if (slot.U32() < m_clean_flat_slots.size() && if (slot.U32() < m_clean_flat_slots.size() &&
m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) { m_clean_flat_slots[slot.U32()] != 0u && m_clean_evaluator != nullptr) {
m_clean_evaluator->SetUsePc(m_use_pc);
return m_clean_evaluator->EvaluateWide(m_program.srt_reads[slot.U32()].value, return m_clean_evaluator->EvaluateWide(m_program.srt_reads[slot.U32()].value,
result, error); result);
} }
return EvaluateWide(m_program.srt_reads[slot.U32()].value, result, error); return EvaluateWide(m_program.srt_reads[slot.U32()].value, result);
} }
case ValueOpcode::LoadAddressU32: case ValueOpcode::LoadAddressU32:
case ValueOpcode::ReadConstBuffer: case ValueOpcode::ReadConstBuffer:
if (IsRawRead(m_program, inst)) { if (IsRawRead(m_program, inst)) {
return EvaluateRawRead(inst, result, error); return EvaluateRawRead(inst, result);
} }
break; break;
case ValueOpcode::IAdd32: case ValueOpcode::IAdd32:
@@ -690,7 +655,7 @@ private:
} }
return false; return false;
case ValueOpcode::BitwiseNot32: case ValueOpcode::BitwiseNot32:
if (Arg(inst, 0, a, error)) { if (Arg(inst, 0, a)) {
result = ~static_cast<uint32_t>(a); result = ~static_cast<uint32_t>(a);
return true; return true;
} }
@@ -737,7 +702,7 @@ private:
const auto offset = static_cast<uint32_t>(b); const auto offset = static_cast<uint32_t>(b);
const auto width = static_cast<uint32_t>(c); const auto width = static_cast<uint32_t>(c);
if (offset > 32u || width > 32u - offset) { if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid unsigned bit-field range"); return false;
} }
const auto mask = width == 32u ? UINT32_MAX const auto mask = width == 32u ? UINT32_MAX
: width == 0u ? 0u : width == 0u ? 0u
@@ -751,7 +716,7 @@ private:
const auto offset = static_cast<uint32_t>(b); const auto offset = static_cast<uint32_t>(b);
const auto width = static_cast<uint32_t>(c); const auto width = static_cast<uint32_t>(c);
if (offset > 32u || width > 32u - offset) { if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid signed bit-field range"); return false;
} }
if (width == 0u) { if (width == 0u) {
result = 0; result = 0;
@@ -768,13 +733,13 @@ private:
return false; return false;
case ValueOpcode::BitFieldInsert: { case ValueOpcode::BitFieldInsert: {
uint64_t d = 0; uint64_t d = 0;
if (!ternary() || !Arg(inst, 3, d, error)) { if (!ternary() || !Arg(inst, 3, d)) {
return false; return false;
} }
const auto offset = static_cast<uint32_t>(c); const auto offset = static_cast<uint32_t>(c);
const auto width = static_cast<uint32_t>(d); const auto width = static_cast<uint32_t>(d);
if (offset > 32u || width > 32u - offset) { if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid inserted bit-field range"); return false;
} }
if (width == 0u) { if (width == 0u) {
result = static_cast<uint32_t>(a); result = static_cast<uint32_t>(a);
@@ -836,7 +801,7 @@ private:
} }
return false; return false;
case ValueOpcode::LogicalNot: case ValueOpcode::LogicalNot:
if (Arg(inst, 0, a, error)) { if (Arg(inst, 0, a)) {
result = a == 0u; result = a == 0u;
return true; return true;
} }
@@ -845,18 +810,16 @@ private:
case ValueOpcode::UndefU8: case ValueOpcode::UndefU8:
case ValueOpcode::UndefU16: case ValueOpcode::UndefU16:
case ValueOpcode::UndefU32: case ValueOpcode::UndefU32:
case ValueOpcode::UndefU64: return Fail(error, "undefined typed runtime value"); case ValueOpcode::UndefU64: return false;
default: break; default: break;
} }
return Fail(error, fmt::format("unsupported typed runtime opcode {}", return false;
ValueOpcodeName(inst.GetOpcode())));
} }
const Program& m_program; const Program& m_program;
const SrtRuntime& m_runtime; const SrtRuntime& m_runtime;
std::span<const uint8_t> m_clean_flat_slots; std::span<const uint8_t> m_clean_flat_slots;
Evaluator* m_clean_evaluator = nullptr; Evaluator* m_clean_evaluator = nullptr;
uint32_t m_use_pc = 0;
std::unordered_map<const Inst*, uint64_t> m_cache; std::unordered_map<const Inst*, uint64_t> m_cache;
std::vector<const Inst*> m_visiting; std::vector<const Inst*> m_visiting;
}; };
@@ -872,18 +835,12 @@ bool EvaluateRuntimeSourcesImpl(const Program& program
std::span<const DescriptorSourceRequest> requests, std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results, const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, bool evaluate_flat, std::vector<uint32_t>& flat, bool evaluate_flat,
std::span<const uint8_t> clean_flat_slots, std::string* error) { std::span<const uint8_t> clean_flat_slots) {
if (!program.srt_plan_complete) { if (!program.srt_plan_complete) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "typed SRT plan is not ready");
}
return false; return false;
} }
if (std::ranges::any_of(clean_flat_slots, [](uint8_t clean) { return clean != 0u; }) && if (std::ranges::any_of(clean_flat_slots, [](uint8_t clean) { return clean != 0u; }) &&
runtime.read_specialization_memory == nullptr) { runtime.read_specialization_memory == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "clean flattened SRT read has no memory reader");
}
return false; return false;
} }
SrtRuntime clean_runtime = runtime; SrtRuntime clean_runtime = runtime;
@@ -895,17 +852,12 @@ bool EvaluateRuntimeSourcesImpl(const Program& program
for (const auto& request: requests) { for (const auto& request: requests) {
const auto* source = Source(program, request.source); const auto* source = Source(program, request.source);
if (source == nullptr) { if (source == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, request.use_pc,
fmt::format("invalid descriptor source {}", request.source));
}
return false; return false;
} }
evaluator.SetUsePc(request.use_pc);
DescriptorValue value; DescriptorValue value;
value.dword_count = source->dword_count; value.dword_count = source->dword_count;
for (uint32_t index = 0; index < source->dword_count; index++) { for (uint32_t index = 0; index < source->dword_count; index++) {
if (!evaluator.Evaluate(source->dwords[index], value.dwords[index], error)) { if (!evaluator.Evaluate(source->dwords[index], value.dwords[index])) {
return false; return false;
} }
} }
@@ -918,9 +870,8 @@ bool EvaluateRuntimeSourcesImpl(const Program& program
const bool clean = read.flat_offset < clean_flat_slots.size() && const bool clean = read.flat_offset < clean_flat_slots.size() &&
clean_flat_slots[read.flat_offset] != 0u; clean_flat_slots[read.flat_offset] != 0u;
auto& selected = clean ? clean_evaluator : evaluator; auto& selected = clean ? clean_evaluator : evaluator;
selected.SetUsePc(read.use_pc);
if (read.flat_offset >= flattened.size() || if (read.flat_offset >= flattened.size() ||
!selected.Evaluate(read.value, flattened[read.flat_offset], error)) { !selected.Evaluate(read.value, flattened[read.flat_offset])) {
return false; return false;
} }
} }
@@ -934,31 +885,24 @@ bool EvaluateRuntimeSourcesImpl(const Program& program
} // namespace } // namespace
bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason) { bool ValidateRuntimeValue(const Program& program, Value value) {
return RuntimeValidator(program).Run(value, reason); return RuntimeValidator(program).Run(value);
} }
bool BuildSrtPlan(Program& program, std::string* error) { void BuildSrtPlan(Program& program) {
if (program.resource_tracking_complete) { if (program.resource_tracking_complete) {
if (error != nullptr) { EXIT("shader SRT planning failed: cannot rebuild SRT after resource tracking");
*error = "cannot rebuild SRT after resource tracking";
}
return false;
} }
program.srt_plan_complete = false; program.srt_plan_complete = false;
if (!PlanBuilder(program).Run(error)) { PlanBuilder(program).Run();
return false;
}
program.srt_plan_complete = true; program.srt_plan_complete = true;
return true;
} }
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, bool EvaluateDescriptorSource(const Program& program, uint32_t source, const SrtRuntime& runtime,
const SrtRuntime& runtime, DescriptorValue& result, DescriptorValue& result) {
std::string* error) { const DescriptorSourceRequest request {source};
const DescriptorSourceRequest request {source, use_pc};
std::vector<DescriptorValue> results; std::vector<DescriptorValue> results;
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) { if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results)) {
return false; return false;
} }
result = results.front(); result = results.front();
@@ -967,26 +911,23 @@ bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t
bool EvaluateDescriptorSources(const Program& program, bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests, std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results, const SrtRuntime& runtime, std::vector<DescriptorValue>& results) {
std::string* error) {
std::vector<uint32_t> ignored; std::vector<uint32_t> ignored;
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, {}, return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, {});
error);
} }
bool EvaluateRuntimeSources(const Program& program, bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests, std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results, const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::span<const uint8_t> clean_flat_slots, std::vector<uint32_t>& flat,
std::string* error) { std::span<const uint8_t> clean_flat_slots) {
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true,
clean_flat_slots, error); clean_flat_slots);
} }
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat, bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat) {
std::string* error) {
std::vector<DescriptorValue> ignored; std::vector<DescriptorValue> ignored;
return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, {}, error); return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, {});
} }
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -13,43 +13,38 @@ using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* val
struct SrtRuntime { struct SrtRuntime {
std::span<const uint32_t> user_data; std::span<const uint32_t> user_data;
uint64_t shader_base = 0; uint64_t shader_base = 0;
SrtMemoryReader read_memory = nullptr; SrtMemoryReader read_memory = nullptr;
void* userdata = nullptr; void* userdata = nullptr;
SrtMemoryReader read_specialization_memory = nullptr; SrtMemoryReader read_specialization_memory = nullptr;
}; };
struct DescriptorSourceRequest { struct DescriptorSourceRequest {
uint32_t source = 0; uint32_t source = 0;
uint32_t use_pc = 0;
}; };
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots; // Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
// dynamic offsets remain explicit and are never assigned a fake slot. // dynamic offsets remain explicit and are never assigned a fake slot.
bool BuildSrtPlan(Program& program, std::string* error); void BuildSrtPlan(Program& program);
bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason); bool ValidateRuntimeValue(const Program& program, Value value);
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, bool EvaluateDescriptorSource(const Program& program, uint32_t source, const SrtRuntime& runtime,
const SrtRuntime& runtime, DescriptorValue& result, DescriptorValue& result);
std::string* error);
// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by // Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by
// several descriptors are memoized once across the batch. // several descriptors are memoized once across the batch.
bool EvaluateDescriptorSources(const Program& program, bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests, std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results, const SrtRuntime& runtime, std::vector<DescriptorValue>& results);
std::string* error);
// Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk. // Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk.
// On failure neither destination is changed. // On failure neither destination is changed.
bool EvaluateRuntimeSources(const Program& program, bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests, std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results, const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::span<const uint8_t> clean_flat_slots, std::vector<uint32_t>& flat, std::span<const uint8_t> clean_flat_slots);
std::string* error);
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat, bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat);
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
+11 -18
View File
@@ -841,23 +841,22 @@ static void ApplyPixelOutputs(ShaderPixelInputInfo& info,
} }
} }
static bool LogPermutationMismatch(const ShaderRecompiler::IR::Program& program, static bool LogPermutationMismatch(const ShaderRecompiler::IR::Program* program,
const char* stage, const std::string& error) { const char* stage) {
static std::atomic<uint32_t> log_count {0}; static std::atomic<uint32_t> log_count {0};
if (Config::GraphicsDebugDumpEnabled() && if (Config::GraphicsDebugDumpEnabled() &&
log_count.fetch_add(1, std::memory_order_relaxed) < 64) { log_count.fetch_add(1, std::memory_order_relaxed) < 64) {
LOGF("ShaderProgramCache native runtime mismatch %s shader=0x%016" PRIx64 ": %s\n", LOGF("ShaderProgramCache native runtime mismatch %s shader=0x%016" PRIx64 "\n",
stage, program.shader_hash, error.c_str()); stage, program != nullptr ? program->shader_hash : 0);
} }
return false; return false;
} }
bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Program>& program, bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Program>& program,
const ShaderParams& params, ShaderVertexInputInfo& info) { const ShaderParams& params, ShaderVertexInputInfo& info) {
std::string error; if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage,
if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, &error,
ReadShaderGuestMemory)) { ReadShaderGuestMemory)) {
return LogPermutationMismatch(*program, "VS", error); return LogPermutationMismatch(program.get(), "VS");
} }
ApplyVertexOutputs(info, *program); ApplyVertexOutputs(info, *program);
return true; return true;
@@ -865,10 +864,9 @@ bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Progra
bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Program>& program, bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Program>& program,
const ShaderParams& params, ShaderPixelInputInfo& info) { const ShaderParams& params, ShaderPixelInputInfo& info) {
std::string error; if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage,
if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, &error,
ReadShaderGuestMemory)) { ReadShaderGuestMemory)) {
return LogPermutationMismatch(*program, "PS", error); return LogPermutationMismatch(program.get(), "PS");
} }
ApplyPixelOutputs(info, *program); ApplyPixelOutputs(info, *program);
return true; return true;
@@ -876,10 +874,9 @@ bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Progra
bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Program>& program, bool MaterializeProgram(const std::shared_ptr<const ShaderRecompiler::IR::Program>& program,
const ShaderParams& params, ShaderComputeInputInfo& info) { const ShaderParams& params, ShaderComputeInputInfo& info) {
std::string error; if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage,
if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, &error,
ReadShaderGuestMemory)) { ReadShaderGuestMemory)) {
return LogPermutationMismatch(*program, "CS", error); return LogPermutationMismatch(program.get(), "CS");
} }
return true; return true;
} }
@@ -1252,11 +1249,7 @@ static vk::ShaderModule CompileModule(vk::Device device, const ShaderParams& par
ShaderStageRuntime& stage) { ShaderStageRuntime& stage) {
const auto* stage_name = ShaderStageName(options.stage); const auto* stage_name = ShaderStageName(options.stage);
const auto* label = ShaderStageLabel(options.stage); const auto* label = ShaderStageLabel(options.stage);
ShaderRecompiler::CompileResult result; auto result = ShaderRecompiler::Recompile(params.code, options);
std::string error;
if (!ShaderRecompiler::TryRecompile(params.code, options, result, &error)) {
ExitShaderRecompilerFailure(label, options.shader_hash, error.c_str());
}
DumpShaderRecompilerOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump); DumpShaderRecompilerOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump);
if (!SpirvValidateBinary(label, options.shader_hash, result.spirv)) { if (!SpirvValidateBinary(label, options.shader_hash, result.spirv)) {
+1 -1
View File
@@ -57,7 +57,7 @@ using ShaderSpecializationMemoryReader = bool (*)(void* userdata, uint64_t addre
bool ShaderMaterializeStageRuntime( bool ShaderMaterializeStageRuntime(
std::shared_ptr<const ShaderRecompiler::IR::Program> program, std::shared_ptr<const ShaderRecompiler::IR::Program> program,
std::span<const uint32_t> user_data, uint64_t shader_base, ShaderStageRuntime& stage, std::span<const uint32_t> user_data, uint64_t shader_base, ShaderStageRuntime& stage,
std::string* error, ShaderSpecializationMemoryReader read_specialization_memory = nullptr, ShaderSpecializationMemoryReader read_specialization_memory = nullptr,
void* read_memory_data = nullptr); void* read_memory_data = nullptr);
constexpr uint32_t DstSel(uint32_t x, uint32_t y = 0, uint32_t z = 0, uint32_t w = 0) { constexpr uint32_t DstSel(uint32_t x, uint32_t y = 0, uint32_t z = 0, uint32_t w = 0) {
+3 -6
View File
@@ -7,13 +7,10 @@ namespace Libs::Graphics {
bool ShaderMaterializeStageRuntime(std::shared_ptr<const ShaderRecompiler::IR::Program> program, bool ShaderMaterializeStageRuntime(std::shared_ptr<const ShaderRecompiler::IR::Program> program,
std::span<const uint32_t> user_data, uint64_t shader_base, std::span<const uint32_t> user_data, uint64_t shader_base,
ShaderStageRuntime& stage, std::string* error, ShaderStageRuntime& stage,
ShaderSpecializationMemoryReader read_specialization_memory, ShaderSpecializationMemoryReader read_specialization_memory,
void* read_memory_data) { void* read_memory_data) {
if (program == nullptr) { if (program == nullptr) {
if (error != nullptr) {
*error = "missing native shader plan";
}
return false; return false;
} }
ShaderRecompiler::IR::SrtRuntime runtime; ShaderRecompiler::IR::SrtRuntime runtime;
@@ -22,8 +19,8 @@ bool ShaderMaterializeStageRuntime(std::shared_ptr<const ShaderRecompiler::IR::P
runtime.read_specialization_memory = read_specialization_memory; runtime.read_specialization_memory = read_specialization_memory;
runtime.userdata = read_memory_data; runtime.userdata = read_memory_data;
ShaderRecompiler::IR::ResourceSnapshot snapshot; ShaderRecompiler::IR::ResourceSnapshot snapshot;
if (!ShaderRecompiler::IR::MaterializeResources(*program, runtime, snapshot, error) || if (!ShaderRecompiler::IR::MaterializeResources(*program, runtime, snapshot) ||
!ShaderRecompiler::IR::ValidateResourceSpecialization(*program, snapshot, error)) { !ShaderRecompiler::IR::ValidateResourceSpecialization(*program, snapshot)) {
return false; return false;
} }
auto resources = auto resources =
+164 -192
View File
@@ -29,6 +29,19 @@ void Check(bool condition, const char *message) {
} }
} }
template <typename F>
void CheckFatal(F &&function, std::string_view expected, const char *message) {
try {
function();
} catch (const std::runtime_error &error) {
Check(std::string_view(error.what()).find(expected) !=
std::string_view::npos,
message);
return;
}
Check(false, message);
}
struct Fixture { struct Fixture {
Program program; Program program;
Block *block = nullptr; Block *block = nullptr;
@@ -75,8 +88,7 @@ struct Fixture {
} }
MemoryFlags AddMemory(MemoryInfo memory, uint32_t pc) { MemoryFlags AddMemory(MemoryInfo memory, uint32_t pc) {
const auto index = const auto index = static_cast<uint32_t>(program.memory_info.size());
static_cast<uint32_t>(program.memory_info.size());
program.memory_info.push_back(memory); program.memory_info.push_back(memory);
return {index, pc}; return {index, pc};
} }
@@ -113,10 +125,8 @@ struct Fixture {
} }
void PlanAndTrack() { void PlanAndTrack() {
std::string error; BuildSrtPlan(program);
if (!BuildSrtPlan(program, &error) || !TrackResources(program, &error)) { TrackResources(program);
throw std::runtime_error(error);
}
} }
}; };
@@ -241,10 +251,7 @@ void TestInvariantIndirectImageMaterialization() {
auto fixture = MakeIndirectImageFixture(false); auto fixture = MakeIndirectImageFixture(false);
fixture->PlanAndTrack(); fixture->PlanAndTrack();
EliminateDeadCode(fixture->program.blocks); EliminateDeadCode(fixture->program.blocks);
std::string validation_error; ValidateProgram(fixture->program, true);
Check(
ValidateProgram(fixture->program, true, &validation_error),
"post-tracking dead-code elimination invalidated descriptor provenance");
Check(fixture->program.info.buffers.size() == 1 && Check(fixture->program.info.buffers.size() == 1 &&
fixture->program.info.images.size() == 1 && fixture->program.info.images.size() == 1 &&
@@ -291,7 +298,6 @@ void TestInvariantIndirectImageMaterialization() {
.userdata = &memory, .userdata = &memory,
.read_specialization_memory = ReadLinearTestMemory}; .read_specialization_memory = ReadLinearTestMemory};
ResourceSnapshot snapshot; ResourceSnapshot snapshot;
std::string error;
const auto same_snapshot = [](const ResourceSnapshot &lhs, const auto same_snapshot = [](const ResourceSnapshot &lhs,
const ResourceSnapshot &rhs) { const ResourceSnapshot &rhs) {
return lhs.buffers == rhs.buffers && lhs.images == rhs.images && return lhs.buffers == rhs.buffers && lhs.images == rhs.images &&
@@ -300,7 +306,7 @@ void TestInvariantIndirectImageMaterialization() {
lhs.user_data == rhs.user_data && lhs.user_data == rhs.user_data &&
lhs.indirect_images.empty() == rhs.indirect_images.empty(); lhs.indirect_images.empty() == rhs.indirect_images.empty();
}; };
Check(MaterializeResources(fixture->program, runtime, snapshot, &error) && Check(MaterializeResources(fixture->program, runtime, snapshot) &&
snapshot.images.size() == 1 && snapshot.images.size() == 1 &&
std::equal(image_descriptor.begin(), image_descriptor.end(), std::equal(image_descriptor.begin(), image_descriptor.end(),
snapshot.images[0].dwords.begin()), snapshot.images[0].dwords.begin()),
@@ -308,8 +314,7 @@ void TestInvariantIndirectImageMaterialization() {
const auto prior_snapshot = snapshot; const auto prior_snapshot = snapshot;
memory.fail_address = 0x1004u; memory.fail_address = 0x1004u;
Check(!MaterializeResources(fixture->program, runtime, snapshot, &error) && Check(!MaterializeResources(fixture->program, runtime, snapshot) &&
error.find("scalar read") != std::string::npos &&
same_snapshot(snapshot, prior_snapshot), same_snapshot(snapshot, prior_snapshot),
"rejected planning memory read mutated the snapshot"); "rejected planning memory read mutated the snapshot");
memory.fail_address = UINT64_MAX; memory.fail_address = UINT64_MAX;
@@ -325,12 +330,11 @@ void TestInvariantIndirectImageMaterialization() {
memory.words[(0x2020u - memory.base) / 4u + 3u] = memory.words[(0x2020u - memory.base) / 4u + 3u] =
image_descriptor[3] ^ (1u << 28u); image_descriptor[3] ^ (1u << 28u);
ResourceSnapshot null_snapshot; ResourceSnapshot null_snapshot;
Check( Check(MaterializeResources(fixture->program, runtime, null_snapshot) &&
MaterializeResources(fixture->program, runtime, null_snapshot, &error) && null_snapshot.indirect_images.empty() &&
null_snapshot.indirect_images.empty() && std::ranges::all_of(null_snapshot.images[0].dwords,
std::ranges::all_of(null_snapshot.images[0].dwords, [](uint32_t dword) { return dword == 0u; }),
[](uint32_t dword) { return dword == 0u; }), "stale typed null image descriptors were not canonicalized");
"stale typed null image descriptors were not canonicalized");
for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) {
memory.words[(0x2000u - memory.base) / 4u + dword] = memory.words[(0x2000u - memory.base) / 4u + dword] =
@@ -341,13 +345,13 @@ void TestInvariantIndirectImageMaterialization() {
memory.words[(0x2020u - memory.base) / 4u] ^= 1u; memory.words[(0x2020u - memory.base) / 4u] ^= 1u;
memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u; memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u;
ResourceSnapshot dynamic_snapshot; ResourceSnapshot dynamic_snapshot;
Check(MaterializeResources(fixture->program, runtime, dynamic_snapshot, Check(MaterializeResources(fixture->program, runtime, dynamic_snapshot) &&
&error) &&
dynamic_snapshot.images.size() == 1 && dynamic_snapshot.images.size() == 1 &&
dynamic_snapshot.indirect_images.size() == 1 && dynamic_snapshot.indirect_images.size() == 1 &&
dynamic_snapshot.indirect_images[0].descriptors.size() == 2 && dynamic_snapshot.indirect_images[0].descriptors.size() == 2,
SpecializeResources(fixture->program, dynamic_snapshot, &error) && "dynamic indirect image table did not materialize");
fixture->program.info.images.size() == 2 && SpecializeResources(fixture->program, dynamic_snapshot);
Check(fixture->program.info.images.size() == 2 &&
fixture->program.info.images[0].indirect_root == 0 && fixture->program.info.images[0].indirect_root == 0 &&
fixture->program.info.images[0].indirect_mapping_capacity != 0 && fixture->program.info.images[0].indirect_mapping_capacity != 0 &&
fixture->program.info.images[0].indirect_resources.size() == 2 && fixture->program.info.images[0].indirect_resources.size() == 2 &&
@@ -364,27 +368,21 @@ void TestInvariantIndirectImageMaterialization() {
memory.words[(0x2000u - memory.base) / 4u] += 0x100u; memory.words[(0x2000u - memory.base) / 4u] += 0x100u;
memory.words[(0x2020u - memory.base) / 4u] += 0x101u; memory.words[(0x2020u - memory.base) / 4u] += 0x101u;
ResourceSnapshot rebound_snapshot; ResourceSnapshot rebound_snapshot;
Check(MaterializeResources(fixture->program, runtime, rebound_snapshot, Check(MaterializeResources(fixture->program, runtime, rebound_snapshot) &&
&error) && ValidateResourceSpecialization(fixture->program, rebound_snapshot),
ValidateResourceSpecialization(fixture->program, rebound_snapshot,
&error),
"stable indirect key mapping did not accept changed image addresses"); "stable indirect key mapping did not accept changed image addresses");
memory.words[(0x2020u - memory.base) / 4u] = memory.words[(0x2020u - memory.base) / 4u] =
memory.words[(0x2000u - memory.base) / 4u]; memory.words[(0x2000u - memory.base) / 4u];
Check(MaterializeResources(fixture->program, runtime, rebound_snapshot, Check(MaterializeResources(fixture->program, runtime, rebound_snapshot) &&
&error) && ValidateResourceSpecialization(fixture->program, rebound_snapshot),
ValidateResourceSpecialization(fixture->program, rebound_snapshot,
&error),
"runtime indirect key mapping did not accept collapsed candidates"); "runtime indirect key mapping did not accept collapsed candidates");
const auto collapsed_snapshot = rebound_snapshot; const auto collapsed_snapshot = rebound_snapshot;
ResourceSnapshot capacity_snapshot; ResourceSnapshot capacity_snapshot;
for (const uint32_t records : {1u, 3u}) { for (const uint32_t records : {1u, 3u}) {
user_data[2] = records; user_data[2] = records;
Check( Check(
MaterializeResources(fixture->program, runtime, capacity_snapshot, MaterializeResources(fixture->program, runtime, capacity_snapshot) &&
&error) && ValidateResourceSpecialization(fixture->program, capacity_snapshot),
ValidateResourceSpecialization(fixture->program, capacity_snapshot,
&error),
"runtime indirect key mapping rejected a fitting material-table size"); "runtime indirect key mapping rejected a fitting material-table size");
} }
user_data[2] = 2u; user_data[2] = 2u;
@@ -398,9 +396,7 @@ void TestInvariantIndirectImageMaterialization() {
} }
memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u; memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u;
Check( Check(
!MaterializeResources(fixture->program, runtime, rebound_snapshot, !MaterializeResources(fixture->program, runtime, rebound_snapshot) &&
&error) &&
error.find("candidate topology") != std::string::npos &&
same_snapshot(rebound_snapshot, collapsed_snapshot), same_snapshot(rebound_snapshot, collapsed_snapshot),
"larger indirect candidate topology reused or mutated a cached snapshot"); "larger indirect candidate topology reused or mutated a cached snapshot");
@@ -418,26 +414,26 @@ void TestInvariantIndirectImageMaterialization() {
.read_specialization_memory = .read_specialization_memory =
ReadLinearTestMemory}; ReadLinearTestMemory};
Check(!MaterializeResources(memory_backed->program, memory_backed_runtime, Check(!MaterializeResources(memory_backed->program, memory_backed_runtime,
snapshot, &error) && snapshot) &&
error.find("constant read failed") != std::string::npos &&
same_snapshot(snapshot, prior_snapshot), same_snapshot(snapshot, prior_snapshot),
"rejected indirect table descriptor read mutated the snapshot"); "rejected indirect table descriptor read mutated the snapshot");
memory.fail_address = UINT64_MAX; memory.fail_address = UINT64_MAX;
auto malformed = MakeIndirectImageFixture(true); auto malformed = MakeIndirectImageFixture(true);
Check(BuildSrtPlan(malformed->program, &error) && BuildSrtPlan(malformed->program);
!TrackResources(malformed->program, &error) && CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value",
error.find("ReadFirstLane") != std::string::npos && "malformed indirect image pattern was accepted");
!malformed->program.resource_tracking_complete && Check(!malformed->program.resource_tracking_complete &&
malformed->program.info.images.empty() && malformed->program.info.images.empty() &&
malformed->program.descriptor_sources.empty(), malformed->program.descriptor_sources.empty(),
"malformed indirect image pattern was partially accepted"); "malformed indirect image pattern was partially accepted");
auto wrapped_immediate = MakeIndirectImageFixture(false, 4u); auto wrapped_immediate = MakeIndirectImageFixture(false, 4u);
Check(BuildSrtPlan(wrapped_immediate->program, &error) && BuildSrtPlan(wrapped_immediate->program);
!TrackResources(wrapped_immediate->program, &error) && CheckFatal([&] { TrackResources(wrapped_immediate->program); },
error.find("ReadFirstLane") != std::string::npos && "not a valid runtime value",
!wrapped_immediate->program.resource_tracking_complete, "wrapped scalar immediate entered the invariant image proof");
Check(!wrapped_immediate->program.resource_tracking_complete,
"wrapped scalar immediate entered the invariant image proof"); "wrapped scalar immediate entered the invariant image proof");
} }
@@ -494,16 +490,12 @@ void TestDenseBufferTracking() {
second.Instruction()->Flags<uint32_t>() == 1, second.Instruction()->Flags<uint32_t>() == 1,
"typed handles were not assigned dense indices"); "typed handles were not assigned dense indices");
Check(fixture.program.memory_info[load_flags.index].resource == 0 && Check(fixture.program.memory_info[load_flags.index].resource == 0 &&
fixture.program.memory_info[store_flags.index].resource == fixture.program.memory_info[store_flags.index].resource == 0 &&
0 && fixture.program.memory_info[other_flags.index].resource == 1,
fixture.program.memory_info[other_flags.index].resource ==
1,
"typed memory metadata was not patched to dense indices"); "typed memory metadata was not patched to dense indices");
std::string error; CheckFatal([&] { TrackResources(fixture.program); }, "already tracked",
Check(!TrackResources(fixture.program, &error) && "resource tracking allowed a second mutation pass");
error.find("already tracked") != std::string::npos,
"resource tracking allowed a second mutation pass");
} }
void TestScalarAndVectorBufferAlias() { void TestScalarAndVectorBufferAlias() {
@@ -531,8 +523,7 @@ void TestScalarAndVectorBufferAlias() {
fixture.program.info.buffers[0].scalar, fixture.program.info.buffers[0].scalar,
"typed scalar and vector uses of one descriptor were split"); "typed scalar and vector uses of one descriptor were split");
Check(fixture.program.memory_info[scalar_flags.index].resource == 0 && Check(fixture.program.memory_info[scalar_flags.index].resource == 0 &&
fixture.program.memory_info[vector_flags.index].resource == fixture.program.memory_info[vector_flags.index].resource == 0,
0,
"scalar/vector alias did not share a dense index"); "scalar/vector alias did not share a dense index");
} }
@@ -552,16 +543,13 @@ void TestRuntimeUnsignedMinDescriptor() {
std::array<uint32_t, 1> user_data{0xffffffffu}; std::array<uint32_t, 1> user_data{0xffffffffu};
SrtRuntime runtime{.user_data = user_data}; SrtRuntime runtime{.user_data = user_data};
DescriptorValue value; DescriptorValue value;
std::string error;
const auto source = fixture.program.info.buffers[0].source; const auto source = fixture.program.info.buffers[0].source;
Check(EvaluateDescriptorSource(fixture.program, source, 0x330, runtime, value, Check(EvaluateDescriptorSource(fixture.program, source, runtime, value) &&
&error) &&
value.dwords[3] == 0x100u, value.dwords[3] == 0x100u,
"runtime descriptor unsigned minimum did not clamp its first operand"); "runtime descriptor unsigned minimum did not clamp its first operand");
user_data[0] = 0x80u; user_data[0] = 0x80u;
Check( Check(
EvaluateDescriptorSource(fixture.program, source, 0x330, runtime, value, EvaluateDescriptorSource(fixture.program, source, runtime, value) &&
&error) &&
value.dwords[3] == 0x80u, value.dwords[3] == 0x80u,
"runtime descriptor unsigned minimum did not preserve its first operand"); "runtime descriptor unsigned minimum did not preserve its first operand");
} }
@@ -624,9 +612,7 @@ void TestImagesSamplersAndAliases() {
repeated.second.Instruction()->Flags<uint32_t>() == 0, repeated.second.Instruction()->Flags<uint32_t>() == 0,
"unused sampler border colors prevented source interning"); "unused sampler border colors prevented source interning");
const auto sampler_source = fixture.program.info.samplers[0].source; const auto sampler_source = fixture.program.info.samplers[0].source;
Check(fixture.program.descriptor_sources[sampler_source] Check(fixture.program.descriptor_sources[sampler_source].dwords[3].U32() == 0,
.dwords[3]
.U32() == 0,
"unused sampler border color was not canonicalized"); "unused sampler border color was not canonicalized");
Check(fixture.program.info.buffers[0].image_alias == 0, Check(fixture.program.info.buffers[0].image_alias == 0,
"buffer/image descriptor alias was not linked"); "buffer/image descriptor alias was not linked");
@@ -673,9 +659,7 @@ void TestSampleAdjustSamplerScratch() {
std::array<uint32_t, 4> user_data{4u, 1u, 2u, 0x80000abcu}; std::array<uint32_t, 4> user_data{4u, 1u, 2u, 0x80000abcu};
SrtRuntime runtime{.user_data = user_data}; SrtRuntime runtime{.user_data = user_data};
DescriptorValue descriptor; DescriptorValue descriptor;
std::string error; Check(EvaluateDescriptorSource(fixture.program, source, runtime, descriptor) &&
Check(EvaluateDescriptorSource(fixture.program, source, 0x1ec, runtime,
descriptor, &error) &&
descriptor.dwords[3] == 0x80000abcu, descriptor.dwords[3] == 0x80000abcu,
"SampleAdjust canonicalization lost sampler border fields"); "SampleAdjust canonicalization lost sampler border fields");
@@ -704,11 +688,9 @@ void TestSampleAdjustSamplerScratch() {
rejected.Emit(ValueOpcode::ImageSampleRaw, rejected.Emit(ValueOpcode::ImageSampleRaw,
{rejected_image, rejected_sampler, rejected.ImageAddress()}, {rejected_image, rejected_sampler, rejected.ImageAddress()},
rejected.AddMemory(rejected_memory, 0x200)); rejected.AddMemory(rejected_memory, 0x200));
std::string rejected_error; BuildSrtPlan(rejected.program);
Check(BuildSrtPlan(rejected.program, &rejected_error) && CheckFatal([&] { TrackResources(rejected.program); },
!TrackResources(rejected.program, &rejected_error) && "not a valid runtime value", message);
rejected_error.find("WqmMask") != std::string::npos,
message);
}; };
CheckRejected(0u, 12u, CheckRejected(0u, 12u,
"ordinary sampling accepted SampleAdjust reserved scratch"); "ordinary sampling accepted SampleAdjust reserved scratch");
@@ -778,15 +760,13 @@ void TestDynamicStorageMipTracking() {
descriptor.dword_count = 8; descriptor.dword_count = 8;
ResourceSnapshot snapshot; ResourceSnapshot snapshot;
snapshot.images.assign(images.size(), descriptor); snapshot.images.assign(images.size(), descriptor);
std::string error; SpecializeResources(fixture.program, snapshot);
Check(SpecializeResources(fixture.program, snapshot, &error) && Check(fixture.program.info.images[1].mip_count == 3 &&
fixture.program.info.images[1].mip_count == 3 && ValidateResourceSpecialization(fixture.program, snapshot),
ValidateResourceSpecialization(fixture.program, snapshot, &error),
"base-1 through last-3 dynamic storage range was not specialized"); "base-1 through last-3 dynamic storage range was not specialized");
ShaderComputeInputInfo compute{}; ShaderComputeInputInfo compute{};
Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && CollectShaderInfo(fixture.program, {.compute = &compute});
AllocateBindings(fixture.program, 0, &error), AllocateBindings(fixture.program, 0);
"dynamic storage mip bindings were not allocated");
const auto *storage_binding = const auto *storage_binding =
FindBinding(fixture.program.bindings, DescriptorBindingKind::Storage2D); FindBinding(fixture.program.bindings, DescriptorBindingKind::Storage2D);
Check(storage_binding != nullptr && Check(storage_binding != nullptr &&
@@ -805,19 +785,19 @@ void TestDynamicStorageMipTracking() {
DescriptorValue null_descriptor{}; DescriptorValue null_descriptor{};
null_descriptor.dword_count = 8; null_descriptor.dword_count = 8;
null_snapshot.images.push_back(null_descriptor); null_snapshot.images.push_back(null_descriptor);
Check(SpecializeResources(null_program, null_snapshot, &error) && SpecializeResources(null_program, null_snapshot);
null_program.info.images[0].mip_count == 1 && Check(null_program.info.images[0].mip_count == 1 &&
ValidateResourceSpecialization(null_program, null_snapshot, &error), ValidateResourceSpecialization(null_program, null_snapshot),
"canonical null dynamic storage image did not retain one descriptor"); "canonical null dynamic storage image did not retain one descriptor");
snapshot.images[1].dwords[3] = snapshot.images[1].dwords[3] =
(snapshot.images[1].dwords[3] & ~(0xfu << 16u)) | (2u << 16u); (snapshot.images[1].dwords[3] & ~(0xfu << 16u)) | (2u << 16u);
Check(!ValidateResourceSpecialization(fixture.program, snapshot, &error), Check(!ValidateResourceSpecialization(fixture.program, snapshot),
"a changed dynamic storage mip count reused the specialization"); "a changed dynamic storage mip count reused the specialization");
snapshot.images[1].dwords[3] = snapshot.images[1].dwords[3] =
(snapshot.images[1].dwords[3] & ~((0xfu << 12u) | (0xfu << 16u))) | (snapshot.images[1].dwords[3] & ~((0xfu << 12u) | (0xfu << 16u))) |
(4u << 12u) | (3u << 16u); (4u << 12u) | (3u << 16u);
Check(!ValidateResourceSpecialization(fixture.program, snapshot, &error), Check(!ValidateResourceSpecialization(fixture.program, snapshot),
"an inverted dynamic storage mip range was accepted"); "an inverted dynamic storage mip range was accepted");
} }
@@ -861,11 +841,9 @@ void TestSrtFlatteningAndRuntimeMemoization() {
.userdata = &memory}; .userdata = &memory};
std::vector<DescriptorValue> descriptors; std::vector<DescriptorValue> descriptors;
std::vector<uint32_t> flat; std::vector<uint32_t> flat;
const DescriptorSourceRequest request{fixture.program.info.buffers[0].source, const DescriptorSourceRequest request{fixture.program.info.buffers[0].source};
12};
std::string error;
Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime,
descriptors, flat, {}, &error), descriptors, flat, {}),
"typed runtime source evaluation failed"); "typed runtime source evaluation failed");
Check(descriptors.size() == 1 && descriptors[0].dwords[0] == 0xdeadbeefu && Check(descriptors.size() == 1 && descriptors[0].dwords[0] == 0xdeadbeefu &&
flat == std::vector<uint32_t>{0xdeadbeefu} && memory.reads == 1, flat == std::vector<uint32_t>{0xdeadbeefu} && memory.reads == 1,
@@ -876,16 +854,16 @@ void TestSrtFlatteningAndRuntimeMemoization() {
descriptors = {{{1u}, 1u}}; descriptors = {{{1u}, 1u}};
flat = {2u}; flat = {2u};
Check(!EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, Check(!EvaluateRuntimeSources(fixture.program, std::span{&request, 1},
runtime, descriptors, flat, {}, &error) && runtime, descriptors, flat, {}) &&
descriptors == std::vector<DescriptorValue>{{{1u}, 1u}} && descriptors == std::vector<DescriptorValue>{{{1u}, 1u}} &&
flat == std::vector<uint32_t>{2u}, flat == std::vector<uint32_t>{2u},
"runtime evaluation failure was not transactional"); "runtime evaluation failure was not transactional");
ShaderComputeInputInfo compute{}; ShaderComputeInputInfo compute{};
Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && CollectShaderInfo(fixture.program, {.compute = &compute});
AllocateBindings(fixture.program, 0, &error) && AllocateBindings(fixture.program, 0);
FindBinding(fixture.program.bindings, Check(FindBinding(fixture.program.bindings,
DescriptorBindingKind::FlattenedSrt) != nullptr, DescriptorBindingKind::FlattenedSrt) != nullptr,
"flattened typed SRT reads did not receive a binding"); "flattened typed SRT reads did not receive a binding");
} }
@@ -919,18 +897,16 @@ void TestDynamicSrtReadRemainsExplicit() {
.read_memory = ReadTestMemory, .read_memory = ReadTestMemory,
.userdata = &memory}; .userdata = &memory};
DescriptorValue value; DescriptorValue value;
std::string error;
Check(EvaluateDescriptorSource(fixture.program, Check(EvaluateDescriptorSource(fixture.program,
fixture.program.info.buffers[0].source, 8, fixture.program.info.buffers[0].source, runtime, value) &&
runtime, value, &error) &&
value.dwords[0] == 0xabcdef01u && memory.reads == 1, value.dwords[0] == 0xabcdef01u && memory.reads == 1,
"dynamic typed scalar descriptor source was not evaluated"); "dynamic typed scalar descriptor source was not evaluated");
ShaderComputeInputInfo compute{}; ShaderComputeInputInfo compute{};
Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && CollectShaderInfo(fixture.program, {.compute = &compute});
AllocateBindings(fixture.program, 0, &error) && AllocateBindings(fixture.program, 0);
FindBinding(fixture.program.bindings, Check(FindBinding(fixture.program.bindings,
DescriptorBindingKind::FlattenedSrt) == nullptr && DescriptorBindingKind::FlattenedSrt) == nullptr &&
FindBinding(fixture.program.bindings, FindBinding(fixture.program.bindings,
DescriptorBindingKind::BdaPagetable) != nullptr && DescriptorBindingKind::BdaPagetable) != nullptr &&
FindBinding(fixture.program.bindings, FindBinding(fixture.program.bindings,
@@ -966,12 +942,10 @@ void TestPhiValidation() {
{handle, Value(0u), Value(0u), Value(0u), Value(true)}, {handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(memory, 20), merge); fixture.AddMemory(memory, 20), merge);
std::string error; BuildSrtPlan(fixture.program);
Check(BuildSrtPlan(fixture.program, &error), CheckFatal([&] { TrackResources(fixture.program); }, "not a valid runtime value",
"SRT planning rejected a well-formed typed phi"); "control-dependent descriptor phi was accepted");
Check(!TrackResources(fixture.program, &error) && Check(!fixture.program.resource_tracking_complete &&
error.find("control-dependent phi") != std::string::npos &&
!fixture.program.resource_tracking_complete &&
fixture.program.info.buffers.empty() && fixture.program.info.buffers.empty() &&
fixture.program.descriptor_sources.empty(), fixture.program.descriptor_sources.empty(),
"control-dependent descriptor phi was not rejected transactionally"); "control-dependent descriptor phi was not rejected transactionally");
@@ -994,9 +968,7 @@ void TestLoopCycleEnteredThroughRuntimeValue() {
{carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12}, {carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12},
loop); loop);
std::string error; BuildSrtPlan(fixture.program);
Check(BuildSrtPlan(fixture.program, &error),
"SRT planning rejected a valid loop entered through a runtime value");
} }
void TestInvariantLoopPhi() { void TestInvariantLoopPhi() {
@@ -1023,10 +995,8 @@ void TestInvariantLoopPhi() {
std::array<uint32_t, 1> user_data{0x12345678u}; std::array<uint32_t, 1> user_data{0x12345678u};
SrtRuntime runtime{.user_data = user_data}; SrtRuntime runtime{.user_data = user_data};
DescriptorValue descriptor; DescriptorValue descriptor;
std::string error;
Check(EvaluateDescriptorSource(fixture.program, Check(EvaluateDescriptorSource(fixture.program,
fixture.program.info.buffers[0].source, 4, fixture.program.info.buffers[0].source, runtime, descriptor) &&
runtime, descriptor, &error) &&
descriptor.dwords[0] == user_data[0], descriptor.dwords[0] == user_data[0],
"loop-invariant descriptor phi was not evaluated through typed SSA"); "loop-invariant descriptor phi was not evaluated through typed SSA");
} }
@@ -1057,11 +1027,9 @@ void TestDmaAddressMaterialization() {
std::array<uint32_t, 2> user_data{0x2008u, 0u}; std::array<uint32_t, 2> user_data{0x2008u, 0u};
SrtRuntime runtime{.user_data = user_data}; SrtRuntime runtime{.user_data = user_data};
ResourceSnapshot snapshot; ResourceSnapshot snapshot;
std::string error; Check(MaterializeResources(fixture.program, runtime, snapshot),
Check(MaterializeResources(fixture.program, runtime, snapshot, &error),
"DMA shader resources did not materialize"); "DMA shader resources did not materialize");
Check(SpecializeResources(fixture.program, snapshot, &error), SpecializeResources(fixture.program, snapshot);
"DMA shader resources were not specialized");
} }
void TestDynamicFlatAddressesUseDma() { void TestDynamicFlatAddressesUseDma() {
@@ -1089,8 +1057,7 @@ void TestDynamicFlatAddressesUseDma() {
std::array<uint32_t, 3> user_data{0x23456780u, 1u, 1u}; std::array<uint32_t, 3> user_data{0x23456780u, 1u, 1u};
SrtRuntime runtime{.user_data = user_data}; SrtRuntime runtime{.user_data = user_data};
ResourceSnapshot snapshot; ResourceSnapshot snapshot;
std::string error; Check(MaterializeResources(fixture.program, runtime, snapshot),
Check(MaterializeResources(fixture.program, runtime, snapshot, &error),
"exec-masked FLAT shader resources did not materialize"); "exec-masked FLAT shader resources did not materialize");
Fixture mismatch; Fixture mismatch;
@@ -1137,15 +1104,15 @@ void TestBufferSwizzleSpecialization() {
(1u << 24u)}; (1u << 24u)};
SrtRuntime runtime{.user_data = user_data}; SrtRuntime runtime{.user_data = user_data};
ResourceSnapshot snapshot; ResourceSnapshot snapshot;
std::string error; Check(MaterializeResources(fixture.program, runtime, snapshot),
Check(MaterializeResources(fixture.program, runtime, snapshot, &error) && "buffer resources did not materialize");
SpecializeResources(fixture.program, snapshot, &error) && SpecializeResources(fixture.program, snapshot);
fixture.program.info.buffers[0].descriptor_swizzle == swizzle && Check(fixture.program.info.buffers[0].descriptor_swizzle == swizzle &&
ValidateResourceSpecialization(fixture.program, snapshot, &error), ValidateResourceSpecialization(fixture.program, snapshot),
"buffer destination selectors were not specialized"); "buffer destination selectors were not specialized");
snapshot.buffers[0].dwords[3] ^= 1u << 9u; snapshot.buffers[0].dwords[3] ^= 1u << 9u;
Check(!ValidateResourceSpecialization(fixture.program, snapshot, &error), Check(!ValidateResourceSpecialization(fixture.program, snapshot),
"buffer swizzle change did not invalidate specialization"); "buffer swizzle change did not invalidate specialization");
} }
@@ -1171,17 +1138,14 @@ void TestShaderInfoAndBindingLayout() {
ShaderComputeInputInfo compute{}; ShaderComputeInputInfo compute{};
compute.dispatch_thread_dimensions = true; compute.dispatch_thread_dimensions = true;
std::string error; CollectShaderInfo(fixture.program, {.compute = &compute});
Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error),
"typed shader info collection failed");
Check(fixture.program.info.has_bitwise_xor && Check(fixture.program.info.has_bitwise_xor &&
!fixture.program.info.inputs.empty() && !fixture.program.info.inputs.empty() &&
fixture.program.info.inputs[0].kind == fixture.program.info.inputs[0].kind ==
StageInputKind::GlobalInvocationId, StageInputKind::GlobalInvocationId,
"typed shader values were not reflected in shader info"); "typed shader values were not reflected in shader info");
Check(AllocateBindings(fixture.program, 0, &error), AllocateBindings(fixture.program, 0);
"typed binding allocation failed");
Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::Buffers) != Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::Buffers) !=
nullptr && nullptr &&
FindBinding(fixture.program.bindings, DescriptorBindingKind::Gds) != FindBinding(fixture.program.bindings, DescriptorBindingKind::Gds) !=
@@ -1211,53 +1175,57 @@ void TestGraphicsPushConstantLayout() {
} }
fixture.program.shader_info_complete = true; fixture.program.shader_info_complete = true;
}; };
std::string error;
Fixture vertex(ShaderType::Vertex); Fixture vertex(ShaderType::Vertex);
AddUserData(vertex, 9); AddUserData(vertex, 9);
Check(AllocateBindings(vertex.program, 0, &error) && AllocateBindings(vertex.program, 0);
vertex.program.bindings.push_constant_offset == 0 && Check(vertex.program.bindings.push_constant_offset == 0 &&
vertex.program.bindings.push_constant_size == 9 * sizeof(uint32_t), vertex.program.bindings.push_constant_size == 9 * sizeof(uint32_t),
"vertex shader was not placed at the start of the graphics push bank"); "vertex shader was not placed at the start of the graphics push bank");
const auto pixel_offset = vertex.program.bindings.push_constant_size; const auto pixel_offset = vertex.program.bindings.push_constant_size;
Fixture pixel(ShaderType::Pixel); Fixture pixel(ShaderType::Pixel);
AddUserData(pixel, 4); AddUserData(pixel, 4);
Check(AllocateBindings(pixel.program, pixel_offset, &error) && AllocateBindings(pixel.program, pixel_offset);
pixel.program.bindings.push_constant_offset == pixel_offset && Check(
pixel.program.bindings.push_constant_size == 4 * sizeof(uint32_t) && pixel.program.bindings.push_constant_offset == pixel_offset &&
FindBinding(pixel.program.bindings, DescriptorBindingKind::UserData) == pixel.program.bindings.push_constant_size == 4 * sizeof(uint32_t) &&
nullptr, FindBinding(pixel.program.bindings,
"pixel shader did not follow the vertex data in the graphics push bank"); DescriptorBindingKind::UserData) == nullptr,
"pixel shader did not follow the vertex data in the graphics push bank");
Fixture edge(ShaderType::Pixel); Fixture edge(ShaderType::Pixel);
AddUserData(edge, 1); AddUserData(edge, 1);
Check(AllocateBindings(edge.program, NativePushConstantSize - sizeof(uint32_t), AllocateBindings(edge.program, NativePushConstantSize - sizeof(uint32_t));
&error) && Check(edge.program.bindings.push_constant_size == sizeof(uint32_t),
edge.program.bindings.push_constant_size == sizeof(uint32_t),
"last aligned push-constant dword did not fit in the graphics bank"); "last aligned push-constant dword did not fit in the graphics bank");
Fixture spill(ShaderType::Pixel); Fixture spill(ShaderType::Pixel);
AddUserData(spill, 32); AddUserData(spill, 32);
Check(AllocateBindings(spill.program, pixel_offset, &error) && AllocateBindings(spill.program, pixel_offset);
spill.program.bindings.push_constant_size == 0 && Check(
FindBinding(spill.program.bindings, DescriptorBindingKind::UserData) != spill.program.bindings.push_constant_size == 0 &&
nullptr, FindBinding(spill.program.bindings,
"pixel shader overlapping the vertex push data did not spill to storage"); DescriptorBindingKind::UserData) != nullptr,
"pixel shader overlapping the vertex push data did not spill to storage");
Fixture full(ShaderType::Pixel); Fixture full(ShaderType::Pixel);
AddUserData(full, 1); AddUserData(full, 1);
Check(AllocateBindings(full.program, NativePushConstantSize, &error) && AllocateBindings(full.program, NativePushConstantSize);
full.program.bindings.push_constant_size == 0 && Check(full.program.bindings.push_constant_size == 0 &&
FindBinding(full.program.bindings, DescriptorBindingKind::UserData) != FindBinding(full.program.bindings,
nullptr, DescriptorBindingKind::UserData) != nullptr,
"full graphics push bank did not spill pixel user data to storage"); "full graphics push bank did not spill pixel user data to storage");
Fixture invalid(ShaderType::Pixel); Fixture invalid(ShaderType::Pixel);
AddUserData(invalid, 1); AddUserData(invalid, 1);
Check(!AllocateBindings(invalid.program, CheckFatal(
NativePushConstantSize + sizeof(uint32_t), &error) && [&] {
!invalid.program.binding_layout_complete, AllocateBindings(invalid.program,
NativePushConstantSize + sizeof(uint32_t));
},
"push-constant offset",
"push-constant placement beyond the graphics bank was accepted");
Check(!invalid.program.binding_layout_complete,
"push-constant placement beyond the graphics bank was accepted"); "push-constant placement beyond the graphics bank was accepted");
} }
@@ -1273,12 +1241,11 @@ void TestResourceLimitIsTransactional() {
{handle, Value(0u), Value(0u), Value(0u), Value(true)}, {handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(memory, index * 4u)); fixture.AddMemory(memory, index * 4u));
} }
std::string error; BuildSrtPlan(fixture.program);
Check(BuildSrtPlan(fixture.program, &error), CheckFatal([&] { TrackResources(fixture.program); },
"SRT plan failed before resource-limit test"); "buffer resource limit exceeded",
Check(!TrackResources(fixture.program, &error) && "resource-limit failure was not reported");
error.find("buffer resource limit exceeded") != std::string::npos && Check(!fixture.program.resource_tracking_complete &&
!fixture.program.resource_tracking_complete &&
fixture.program.info.buffers.empty() && fixture.program.info.buffers.empty() &&
fixture.program.descriptor_sources.empty(), fixture.program.descriptor_sources.empty(),
"resource-limit failure partially mutated typed resource state"); "resource-limit failure partially mutated typed resource state");
@@ -1293,12 +1260,11 @@ void TestMalformedMemoryKindsRejected() {
fixture.Emit(ValueOpcode::StoreAddressU32, fixture.Emit(ValueOpcode::StoreAddressU32,
{address, Value(0u), Value(0u), Value(1u), Value(true)}, {address, Value(0u), Value(0u), Value(1u), Value(true)},
fixture.AddMemory(memory, 4)); fixture.AddMemory(memory, 4));
std::string error; BuildSrtPlan(fixture.program);
Check(BuildSrtPlan(fixture.program, &error) && CheckFatal(
!TrackResources(fixture.program, &error) && [&] { TrackResources(fixture.program); },
error.find("address operation has invalid resource kind") != "address operation has invalid resource kind",
std::string::npos, "resource tracking accepted an address opcode with buffer metadata");
"resource tracking accepted an address opcode with buffer metadata");
} }
{ {
Fixture fixture; Fixture fixture;
@@ -1311,12 +1277,11 @@ void TestMalformedMemoryKindsRejected() {
fixture.Emit(ValueOpcode::ImageRead, fixture.Emit(ValueOpcode::ImageRead,
{image, fixture.ImageAddress(), Value(true)}, {image, fixture.ImageAddress(), Value(true)},
fixture.AddMemory(memory, 8)); fixture.AddMemory(memory, 8));
std::string error; BuildSrtPlan(fixture.program);
Check(BuildSrtPlan(fixture.program, &error) && CheckFatal(
!TrackResources(fixture.program, &error) && [&] { TrackResources(fixture.program); },
error.find("image operation has invalid resource kind") != "image operation has invalid resource kind",
std::string::npos, "resource tracking accepted an image opcode with address metadata");
"resource tracking accepted an image opcode with address metadata");
} }
{ {
Fixture fixture; Fixture fixture;
@@ -1329,12 +1294,11 @@ void TestMalformedMemoryKindsRejected() {
fixture.Emit(ValueOpcode::ImageRead, fixture.Emit(ValueOpcode::ImageRead,
{image, fixture.ImageAddress(), Value(true)}, {image, fixture.ImageAddress(), Value(true)},
fixture.AddMemory(memory, 12)); fixture.AddMemory(memory, 12));
std::string error; BuildSrtPlan(fixture.program);
Check(BuildSrtPlan(fixture.program, &error) && CheckFatal(
!TrackResources(fixture.program, &error) && [&] { TrackResources(fixture.program); },
error.find("image operation has invalid resource kind") != "image operation has invalid resource kind",
std::string::npos, "resource tracking accepted a sampled read with storage metadata");
"resource tracking accepted a sampled read with storage metadata");
} }
{ {
Fixture fixture; Fixture fixture;
@@ -1347,12 +1311,10 @@ void TestMalformedMemoryKindsRejected() {
fixture.Emit(ValueOpcode::ImageAtomicIAdd32, fixture.Emit(ValueOpcode::ImageAtomicIAdd32,
{image, fixture.ImageAddress(), Value(1u), Value(true)}, {image, fixture.ImageAddress(), Value(1u), Value(true)},
fixture.AddMemory(memory, 16)); fixture.AddMemory(memory, 16));
std::string error; BuildSrtPlan(fixture.program);
Check( CheckFatal(
BuildSrtPlan(fixture.program, &error) && [&] { TrackResources(fixture.program); },
!TrackResources(fixture.program, &error) && "image operation has invalid resource kind",
error.find("image operation has invalid resource kind") !=
std::string::npos,
"resource tracking accepted a uint atomic with float storage metadata"); "resource tracking accepted a uint atomic with float storage metadata");
} }
} }
@@ -1399,14 +1361,24 @@ int main() {
// target links only fmt; keep assertion failures observable without widening // target links only fmt; keep assertion failures observable without widening
// its focused build manifest. // its focused build manifest.
namespace Common { namespace Common {
int DbgExitHandler(const char *, int, std::string_view text) {
throw std::runtime_error(std::string(text));
}
int DbgExitHandler(const char *, int, fmt::text_style, std::string_view text) {
throw std::runtime_error(std::string(text));
}
int DbgExitIfHandler(const char *expression, const char *file, int line) { int DbgExitIfHandler(const char *expression, const char *file, int line) {
throw std::runtime_error(std::string("typed IR assertion: ") + expression + throw std::runtime_error(std::string("typed IR assertion: ") + expression +
" at " + file + ':' + std::to_string(line)); " at " + file + ':' + std::to_string(line));
} }
int DbgNotImplementedHandler(const char *expression, const char *file, int line) { int DbgNotImplementedHandler(const char *expression, const char *file,
throw std::runtime_error(std::string("typed IR not implemented: ") + expression + int line) {
" at " + file + ':' + std::to_string(line)); throw std::runtime_error(std::string("typed IR not implemented: ") +
expression + " at " + file + ':' +
std::to_string(line));
} }
void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); } void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
@@ -1415,8 +1387,8 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
// Keep this focused standalone target self-contained by amalgamating its small // Keep this focused standalone target self-contained by amalgamating its small
// typed-IR implementation set. // typed-IR implementation set.
#include "graphics/shader/recompiler/ir/Block.cpp" #include "graphics/shader/recompiler/ir/Block.cpp"
#include "graphics/shader/recompiler/ir/Program.cpp"
#include "graphics/shader/recompiler/ir/Type.cpp" #include "graphics/shader/recompiler/ir/Type.cpp"
#include "graphics/shader/recompiler/ir/Value.cpp" #include "graphics/shader/recompiler/ir/Value.cpp"
#include "graphics/shader/recompiler/ir/Program.cpp"
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
+54 -59
View File
@@ -20,6 +20,19 @@ void Check(bool condition, const char *message) {
} }
} }
template <typename F>
void CheckFatal(F &&function, std::string_view expected, const char *message) {
try {
function();
} catch (const std::runtime_error &error) {
Check(std::string_view(error.what()).find(expected) !=
std::string_view::npos,
message);
return;
}
Check(false, message);
}
struct Fixture { struct Fixture {
Program program; Program program;
@@ -58,10 +71,7 @@ struct Fixture {
return static_cast<uint32_t>(program.memory_info.size() - 1u); return static_cast<uint32_t>(program.memory_info.size() - 1u);
} }
void Plan() { void Plan() { BuildSrtPlan(program); }
std::string error;
Check(BuildSrtPlan(program, &error), error.c_str());
}
}; };
struct TestMemory { struct TestMemory {
@@ -112,8 +122,7 @@ void TestImmediateFlatteningAndGvn() {
TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}}; TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}};
SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
std::vector<uint32_t> flat; std::vector<uint32_t> flat;
std::string error; Check(WalkSrt(fixture.program, runtime, flat), "flattened SRT walk failed");
Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str());
Check(flat == std::vector<uint32_t>{0xfeedbeefu} && memory_image.reads == 1, Check(flat == std::vector<uint32_t>{0xfeedbeefu} && memory_image.reads == 1,
"flattened SRT did not evaluate its canonical read once"); "flattened SRT did not evaluate its canonical read once");
} }
@@ -130,8 +139,7 @@ void TestRawScalarComponentAlignment() {
TestMemory memory_image{{{0x1000u, 0x12345678u}}}; TestMemory memory_image{{{0x1000u, 0x12345678u}}};
SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
std::vector<uint32_t> flat; std::vector<uint32_t> flat;
std::string error; Check(WalkSrt(fixture.program, runtime, flat), "raw scalar SRT walk failed");
Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str());
Check( Check(
flat == std::vector<uint32_t>{0x12345678u} && memory_image.reads == 1, flat == std::vector<uint32_t>{0x12345678u} && memory_image.reads == 1,
"raw scalar base, immediate, and offset were not aligned independently"); "raw scalar base, immediate, and offset were not aligned independently");
@@ -141,11 +149,9 @@ void TestScalarMemoryDomainMismatchFails() {
Fixture raw; Fixture raw;
const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer); const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer);
RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory); RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory);
std::string error; CheckFatal([&] { BuildSrtPlan(raw.program); },
Check(!BuildSrtPlan(raw.program, &error) && "incompatible scalar memory metadata",
error.find("incompatible scalar memory metadata") != "raw scalar load accepted descriptor-buffer metadata");
std::string::npos,
"raw scalar load accepted descriptor-buffer metadata");
Fixture buffer; Fixture buffer;
const auto buffer_memory = buffer.AddMemory(ResourceKind::ScalarAddress); const auto buffer_memory = buffer.AddMemory(ResourceKind::ScalarAddress);
@@ -154,11 +160,9 @@ void TestScalarMemoryDomainMismatchFails() {
{Value(0x1000u), Value(0u), Value(16u), Value(0u)}); {Value(0x1000u), Value(0u), Value(16u), Value(0u)});
buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)}, buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)},
buffer_memory); buffer_memory);
error.clear(); CheckFatal([&] { BuildSrtPlan(buffer.program); },
Check(!BuildSrtPlan(buffer.program, &error) && "incompatible scalar memory metadata",
error.find("incompatible scalar memory metadata") != "descriptor scalar load accepted raw-address metadata");
std::string::npos,
"descriptor scalar load accepted raw-address metadata");
} }
void TestDynamicReadRemainsTyped() { void TestDynamicReadRemainsTyped() {
@@ -191,8 +195,7 @@ void TestNestedSrtWalk() {
TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}}; TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}};
SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
std::vector<uint32_t> flat; std::vector<uint32_t> flat;
std::string error; Check(WalkSrt(fixture.program, runtime, flat), "nested SRT walk failed");
Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str());
Check(flat == std::vector<uint32_t>({0x2000u, 0xabcdef01u}), Check(flat == std::vector<uint32_t>({0x2000u, 0xabcdef01u}),
"nested typed SRT reads were not evaluated in dependency order"); "nested typed SRT reads were not evaluated in dependency order");
} }
@@ -215,10 +218,8 @@ void TestShaderBaseAndUserData() {
SrtRuntime runtime{.user_data = user_data, SrtRuntime runtime{.user_data = user_data,
.shader_base = 0x12345678abcdef00ull}; .shader_base = 0x12345678abcdef00ull};
DescriptorValue result; DescriptorValue result;
std::string error; Check(EvaluateDescriptorSource(fixture.program, 0, runtime, result),
Check(EvaluateDescriptorSource(fixture.program, 0, 0x90, runtime, result, "shader-relative descriptor evaluation failed");
&error),
error.c_str());
Check(result.dword_count == 3 && result.dwords[0] == 0xabcdef00u && Check(result.dword_count == 3 && result.dwords[0] == 0xabcdef00u &&
result.dwords[1] == 0x12345678u && result.dwords[2] == 0x24u, result.dwords[1] == 0x12345678u && result.dwords[2] == 0x24u,
"shader-relative typed descriptor expression evaluated incorrectly"); "shader-relative typed descriptor expression evaluated incorrectly");
@@ -242,9 +243,8 @@ void TestCarryAndBitFields() {
{.dwords = {low, high, inserted, sign}, .dword_count = 4}); {.dwords = {low, high, inserted, sign}, .dword_count = 4});
DescriptorValue result; DescriptorValue result;
std::string error; Check(EvaluateDescriptorSource(fixture.program, 0, {}, result),
Check(EvaluateDescriptorSource(fixture.program, 0, 0, {}, result, &error), "carry and bit-field descriptor evaluation failed");
error.c_str());
Check(result.dwords[0] == 1u && result.dwords[1] == 1u && Check(result.dwords[0] == 1u && result.dwords[1] == 1u &&
result.dwords[2] == 0x89abcdefu && result.dwords[3] == 0xffffffffu, result.dwords[2] == 0x89abcdefu && result.dwords[3] == 0xffffffffu,
"typed carry or bit-field runtime evaluation is incorrect"); "typed carry or bit-field runtime evaluation is incorrect");
@@ -267,17 +267,13 @@ void TestInvariantAndDivergentPhi() {
{.dwords = {Value(&divergent)}, .dword_count = 1}); {.dwords = {Value(&divergent)}, .dword_count = 1});
DescriptorValue result; DescriptorValue result;
std::string error; Check(EvaluateDescriptorSource(fixture.program, 0, {}, result) &&
Check( result.dwords[0] == 7u,
EvaluateDescriptorSource(fixture.program, 0, 0x100, {}, result, &error) && "loop-invariant typed phi was rejected");
result.dwords[0] == 7u,
"loop-invariant typed phi was rejected");
result.dword_count = 4; result.dword_count = 4;
result.dwords[0] = 0xdeadbeefu; result.dwords[0] = 0xdeadbeefu;
Check(!EvaluateDescriptorSource(fixture.program, 1, 0x104, {}, result, Check(!EvaluateDescriptorSource(fixture.program, 1, {}, result) &&
&error) && result.dword_count == 4 && result.dwords[0] == 0xdeadbeefu,
result.dword_count == 4 && result.dwords[0] == 0xdeadbeefu &&
error.find("runtime-dependent") != std::string::npos,
"divergent phi did not fail transactionally"); "divergent phi did not fail transactionally");
} }
@@ -318,8 +314,7 @@ void TestRuntime64BitDescriptorOps() {
{.dwords = {low, high}, .dword_count = 2}); {.dwords = {low, high}, .dword_count = 2});
DescriptorValue result; DescriptorValue result;
std::string error; Check(EvaluateDescriptorSource(fixture.program, 0, {}, result) &&
Check(EvaluateDescriptorSource(fixture.program, 0, 0, {}, result, &error) &&
result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u, result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1235u,
"64-bit typed descriptor arithmetic evaluation is incorrect"); "64-bit typed descriptor arithmetic evaluation is incorrect");
} }
@@ -339,10 +334,9 @@ void TestConstantBufferBounds() {
TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}}; TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}};
SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
std::vector<uint32_t> flat; std::vector<uint32_t> flat;
std::string error; Check(WalkSrt(fixture.program, runtime, flat) &&
Check(WalkSrt(fixture.program, runtime, flat, &error) &&
flat == std::vector<uint32_t>{0xa5a5a5a5u}, flat == std::vector<uint32_t>{0xa5a5a5a5u},
error.c_str()); "constant-buffer SRT walk failed");
Fixture overflow; Fixture overflow;
const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer); const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer);
@@ -356,9 +350,8 @@ void TestConstantBufferBounds() {
{overflow_read, Value(0u), Value(16u), Value(0u)}); {overflow_read, Value(0u), Value(16u), Value(0u)});
overflow.Plan(); overflow.Plan();
flat = {0x55u}; flat = {0x55u};
Check(!WalkSrt(overflow.program, runtime, flat, &error) && Check(!WalkSrt(overflow.program, runtime, flat) &&
flat == std::vector<uint32_t>{0x55u} && flat == std::vector<uint32_t>{0x55u},
error.find("exceeds size") != std::string::npos,
"out-of-bounds constant-buffer walk was not transactional"); "out-of-bounds constant-buffer walk was not transactional");
} }
@@ -420,10 +413,9 @@ void TestControlFlowValueSurvivesReadLaneFolding() {
Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return; Libs::Graphics::ShaderRecompiler::CFG::TerminatorKind::Return;
const auto undef = fixture.Emit(ValueOpcode::UndefU32); const auto undef = fixture.Emit(ValueOpcode::UndefU32);
const auto write = fixture.Emit( const auto write =
ValueOpcode::WriteLane, {undef, Value(42u), Value(5u)}); fixture.Emit(ValueOpcode::WriteLane, {undef, Value(42u), Value(5u)});
const auto read = const auto read = fixture.Emit(ValueOpcode::ReadLane, {write, Value(5u)});
fixture.Emit(ValueOpcode::ReadLane, {write, Value(5u)});
entry_info.condition = entry_info.condition =
fixture.Emit(ValueOpcode::IEqual32, {read, Value(42u)}); fixture.Emit(ValueOpcode::IEqual32, {read, Value(42u)});
fixture.Emit(ValueOpcode::Reference, {entry_info.condition}); fixture.Emit(ValueOpcode::Reference, {entry_info.condition});
@@ -435,11 +427,9 @@ void TestControlFlowValueSurvivesReadLaneFolding() {
RemoveIdentities(fixture.program.blocks); RemoveIdentities(fixture.program.blocks);
EliminateDeadCode(fixture.program.blocks); EliminateDeadCode(fixture.program.blocks);
std::string error; Check(entry_info.condition == Value(true),
Check(entry_info.condition == Value(true) && "folded branch condition did not survive identity removal");
ValidateProgram(fixture.program, true, &error), ValidateProgram(fixture.program, true);
error.empty() ? "folded branch condition did not survive identity removal"
: error.c_str());
} }
void TestUndefinedRuntimeValueFails() { void TestUndefinedRuntimeValueFails() {
@@ -450,11 +440,8 @@ void TestUndefinedRuntimeValueFails() {
{.dwords = {undef}, .dword_count = 1}); {.dwords = {undef}, .dword_count = 1});
DescriptorValue result; DescriptorValue result;
result.dword_count = 3; result.dword_count = 3;
std::string error; Check(!EvaluateDescriptorSource(fixture.program, 0, {}, result) &&
Check(!EvaluateDescriptorSource(fixture.program, 0, 0x200, {}, result, result.dword_count == 3,
&error) &&
result.dword_count == 3 &&
error.find("undefined typed runtime value") != std::string::npos,
"undefined typed descriptor source did not fail transactionally"); "undefined typed descriptor source did not fail transactionally");
} }
@@ -464,6 +451,14 @@ namespace Common {
int DbgExitIfHandler(const char *, const char *, int) { return 1; } int DbgExitIfHandler(const char *, const char *, int) { return 1; }
int DbgExitHandler(const char *, int, std::string_view text) {
throw std::runtime_error(std::string(text));
}
int DbgExitHandler(const char *, int, fmt::text_style, std::string_view text) {
throw std::runtime_error(std::string(text));
}
void DbgExit(int) { std::abort(); } void DbgExit(int) { std::abort(); }
} // namespace Common } // namespace Common
@@ -496,9 +491,9 @@ int main() {
// Keep this focused standalone target self-contained by amalgamating its small // Keep this focused standalone target self-contained by amalgamating its small
// typed-IR implementation set. // typed-IR implementation set.
#include "../src/graphics/shader/recompiler/ir/Block.cpp" #include "../src/graphics/shader/recompiler/ir/Block.cpp"
#include "../src/graphics/shader/recompiler/ir/Program.cpp"
#include "../src/graphics/shader/recompiler/ir/Type.cpp" #include "../src/graphics/shader/recompiler/ir/Type.cpp"
#include "../src/graphics/shader/recompiler/ir/Value.cpp" #include "../src/graphics/shader/recompiler/ir/Value.cpp"
#include "../src/graphics/shader/recompiler/ir/Program.cpp"
#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" #include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" #include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
+92 -93
View File
@@ -85,6 +85,12 @@
#include <string> #include <string>
#include <vector> #include <vector>
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
#include <cerrno>
#include <sys/wait.h>
#include <unistd.h>
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#ifndef NOMINMAX #ifndef NOMINMAX
#define NOMINMAX #define NOMINMAX
@@ -907,6 +913,31 @@ void Require(const char *shader_name, const char *stage, bool value,
} }
} }
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
template <typename Function>
void ExpectFatal(const char *name, Function function) {
const pid_t pid = ::fork();
Require(name, "fatal subprocess", pid >= 0,
"fork failed while starting fatal shader test");
if (pid == 0) {
function();
::_exit(0);
}
int status = 0;
pid_t waited = -1;
do {
waited = ::waitpid(pid, &status, 0);
} while (waited < 0 && errno == EINTR);
Require(name, "fatal subprocess", waited == pid,
"waitpid failed while collecting fatal shader test");
Require(name, "fatal exit",
WIFEXITED(status) && WEXITSTATUS(status) == (321 & 0xff),
"fatal shader path did not terminate with status 321");
std::printf("[host] %-32s ok\n", name);
}
#endif
void CheckLeastRecentlyUsedCacheOrdering() { void CheckLeastRecentlyUsedCacheOrdering() {
Common::LeastRecentlyUsedCache<uint32_t, uint64_t> cache; Common::LeastRecentlyUsedCache<uint32_t, uint64_t> cache;
const auto first = cache.Insert(1, 1); const auto first = cache.Insert(1, 1);
@@ -979,7 +1010,6 @@ struct TestCase {
std::vector<std::pair<std::string, size_t>> decoded_counts; std::vector<std::pair<std::string, size_t>> decoded_counts;
std::vector<std::pair<std::string, size_t>> ir_counts; std::vector<std::pair<std::string, size_t>> ir_counts;
u32 expected_storage_mip_descriptors = 0; u32 expected_storage_mip_descriptors = 0;
std::string expected_compile_error;
}; };
struct GraphicsCase { struct GraphicsCase {
@@ -1187,17 +1217,7 @@ CompiledShader CompileCase(const TestCase &test) {
options.wave_size = test.compute_info.wave_size; options.wave_size = test.compute_info.wave_size;
} }
ShaderRecompiler::CompileResult result; auto result = ShaderRecompiler::Recompile(test.code, options);
std::string error;
if (!ShaderRecompiler::TryRecompile(test.code, options, result, &error)) {
if (!test.expected_compile_error.empty() &&
error.find(test.expected_compile_error) != std::string::npos) {
return {};
}
Fail(test.name, "decode/IR", error.c_str());
}
Require(test.name, "decode/IR", test.expected_compile_error.empty(),
"shader unexpectedly compiled despite the required rejection");
for (const auto &[text, expected] : test.decoded_counts) { for (const auto &[text, expected] : test.decoded_counts) {
const auto actual = CountText(result.decoded_dump, text); const auto actual = CountText(result.decoded_dump, text);
Require(test.name, "decoded RDNA2", actual == expected, Require(test.name, "decoded RDNA2", actual == expected,
@@ -1245,9 +1265,7 @@ CompiledShader CompileCase(const TestCase &test) {
} }
result.program.bindings = {}; result.program.bindings = {};
result.program.binding_layout_complete = false; result.program.binding_layout_complete = false;
if (!ShaderRecompiler::IR::AllocateBindings(result.program, 0, &error)) { ShaderRecompiler::IR::AllocateBindings(result.program, 0);
Fail(test.name, "binding layout", error.c_str());
}
const auto *shader_data = ShaderRecompiler::IR::FindBinding( const auto *shader_data = ShaderRecompiler::IR::FindBinding(
result.program.bindings, result.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::UserData); ShaderRecompiler::IR::DescriptorBindingKind::UserData);
@@ -1256,13 +1274,8 @@ CompiledShader CompileCase(const TestCase &test) {
result.program.bindings.push_constant_size == 0 && result.program.bindings.push_constant_size == 0 &&
shader_data != nullptr, shader_data != nullptr,
"oversized shader data did not use its storage fallback"); "oversized shader data did not use its storage fallback");
std::vector<u32> storage_spirv; result.spirv = ShaderRecompiler::Spirv::EmitProgram(
if (!ShaderRecompiler::Spirv::EmitProgram(result.program, result.resources, result.program, result.resources, options.input_info);
options.input_info, storage_spirv,
&error)) {
Fail(test.name, "SPIR-V emit", error.c_str());
}
result.spirv = std::move(storage_spirv);
} }
Require(test.name, "SPIR-V emit", !result.spirv.empty(), Require(test.name, "SPIR-V emit", !result.spirv.empty(),
"recompiler returned empty SPIR-V"); "recompiler returned empty SPIR-V");
@@ -1338,12 +1351,7 @@ CompiledShader CompileFragmentCase(const GraphicsCase &test) {
options.input_info.pixel = &pixel_info; options.input_info.pixel = &pixel_info;
options.user_data = user_data.data(); options.user_data = user_data.data();
ShaderRecompiler::CompileResult result; auto result = ShaderRecompiler::Recompile(test.fragment_code, options);
std::string error;
if (!ShaderRecompiler::TryRecompile(test.fragment_code, options, result,
&error)) {
Fail(test.name, "decode/IR", error.c_str());
}
Require(test.name, "SPIR-V emit", !result.spirv.empty(), Require(test.name, "SPIR-V emit", !result.spirv.empty(),
"recompiler returned empty SPIR-V"); "recompiler returned empty SPIR-V");
ValidateSpirv(test.name, result.spirv); ValidateSpirv(test.name, result.spirv);
@@ -7579,10 +7587,7 @@ public:
const auto allocate_bindings = const auto allocate_bindings =
[&](ShaderRecompiler::IR::Program &program) { [&](ShaderRecompiler::IR::Program &program) {
program.shader_info_complete = true; program.shader_info_complete = true;
std::string error; ShaderRecompiler::IR::AllocateBindings(program, 0);
Require(name, "binding layout",
ShaderRecompiler::IR::AllocateBindings(program, 0, &error),
error.c_str());
}; };
constexpr auto stencil_format = Prospero::BufferFormat::k8UInt; constexpr auto stencil_format = Prospero::BufferFormat::k8UInt;
@@ -7616,11 +7621,7 @@ public:
ShaderRecompiler::IR::DescriptorValue null_descriptor{}; ShaderRecompiler::IR::DescriptorValue null_descriptor{};
null_descriptor.dword_count = 8; null_descriptor.dword_count = 8;
null_snapshot.images.assign(3, null_descriptor); null_snapshot.images.assign(3, null_descriptor);
std::string null_error; ShaderRecompiler::IR::SpecializeResources(null_program, null_snapshot);
Require(name, "null specialization",
ShaderRecompiler::IR::SpecializeResources(
null_program, null_snapshot, &null_error),
null_error.c_str());
allocate_bindings(null_program); allocate_bindings(null_program);
ShaderStageRuntime null_runtime{ ShaderStageRuntime null_runtime{
std::make_shared<const ShaderRecompiler::IR::Program>( std::make_shared<const ShaderRecompiler::IR::Program>(
@@ -11657,10 +11658,6 @@ void CompareGraphicsWords(const GraphicsCase &test,
void RunCase(VulkanHarness *vulkan, const TestCase &test) { void RunCase(VulkanHarness *vulkan, const TestCase &test) {
auto compiled = CompileCase(test); auto compiled = CompileCase(test);
if (!test.expected_compile_error.empty()) {
std::printf("[compute] %-32s ok\n", test.name);
return;
}
if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) { if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) {
Require(test.name, "resource specialization", Require(test.name, "resource specialization",
!compiled.program.info.images.empty() && !compiled.program.info.images.empty() &&
@@ -18506,8 +18503,6 @@ TestCase FlatStoreVariants() {
{O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT, {O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT,
O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2, O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2,
O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}}; O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}};
test.expected_compile_error =
"writable FLAT/GLOBAL addresses require GPU ownership tracking";
return test; return test;
} }
@@ -20314,9 +20309,7 @@ void CheckIndirectImageKeySwitch() {
program.info.samplers.push_back({1u, 0x10f0u}); program.info.samplers.push_back({1u, 0x10f0u});
program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u}); program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u});
std::string error; AllocateBindings(program, 0);
Require(name, "binding layout", AllocateBindings(program, 0, &error),
error.c_str());
ResourceSnapshot snapshot{}; ResourceSnapshot snapshot{};
DescriptorValue descriptor{}; DescriptorValue descriptor{};
descriptor.dword_count = 8; descriptor.dword_count = 8;
@@ -20340,14 +20333,9 @@ void CheckIndirectImageKeySwitch() {
snapshot.samplers.push_back(sampler_descriptor); snapshot.samplers.push_back(sampler_descriptor);
ShaderComputeInputInfo compute{}; ShaderComputeInputInfo compute{};
std::vector<u32> spirv; ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program);
Require(name, "SPIR-V requirements", auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, snapshot,
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, &error), {.compute = &compute});
error.c_str());
Require(name, "SPIR-V emit",
ShaderRecompiler::Spirv::EmitProgram(
program, snapshot, {.compute = &compute}, spirv, &error),
error.c_str());
ValidateSpirv(name, spirv); ValidateSpirv(name, spirv);
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2);
std::string text; std::string text;
@@ -21314,7 +21302,6 @@ std::vector<TestCase> MakeCases() {
AddCase(GlobalSignedImmediateRebasesBeforeSaddr); AddCase(GlobalSignedImmediateRebasesBeforeSaddr);
AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb); AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb);
AddCase(ScratchIsPrivatePerInvocation); AddCase(ScratchIsPrivatePerInvocation);
AddCase(FlatStoreVariants);
AddCase(DsReadWriteVariants); AddCase(DsReadWriteVariants);
AddCase(DsWriteB16D16HiCapturedUsesHighHalf); AddCase(DsWriteB16D16HiCapturedUsesHighHalf);
AddCase(DsReadU16D16CapturedPreservesHighHalf); AddCase(DsReadU16D16CapturedPreservesHighHalf);
@@ -21433,7 +21420,7 @@ void CheckPs5GameExampleImageClearRuntimeShape() {
compute.thread_ids_num = 1; compute.thread_ids_num = 1;
compute.workgroup_register = 8; compute.workgroup_register = 8;
const auto Compile = [&](const char *stage, const std::vector<u32> &code) { const auto Compile = [&](const std::vector<u32> &code) {
ShaderRecompiler::CompileOptions options; ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Compute; options.stage = ShaderType::Compute;
options.wave_size = 64; options.wave_size = 64;
@@ -21442,17 +21429,13 @@ void CheckPs5GameExampleImageClearRuntimeShape() {
options.user_data = user_data.data(); options.user_data = user_data.data();
options.input_info.compute = &compute; options.input_info.compute = &compute;
options.dump_ir = false; options.dump_ir = false;
ShaderRecompiler::CompileResult result; auto result = ShaderRecompiler::Recompile(code, options);
std::string error;
Require("Ps5GameExampleImageClear", stage,
ShaderRecompiler::TryRecompile(code, options, result, &error),
error);
ValidateSpirv("Ps5GameExampleImageClear", result.spirv); ValidateSpirv("Ps5GameExampleImageClear", result.spirv);
return result; return result;
}; };
const auto code = MakeCode(); const auto code = MakeCode();
auto positive = Compile("exact Prospero kernel", code); auto positive = Compile(code);
compute.stage.program = std::make_shared<const ShaderRecompiler::IR::Program>( compute.stage.program = std::make_shared<const ShaderRecompiler::IR::Program>(
std::move(positive.program)); std::move(positive.program));
@@ -21543,11 +21526,7 @@ void CheckEmbeddedFetchVertexOffset() {
options.user_data = user_data.data(); options.user_data = user_data.data();
options.input_info.vertex = &vertex; options.input_info.vertex = &vertex;
ShaderRecompiler::CompileResult result; auto result = ShaderRecompiler::Recompile(code, options);
std::string error;
Require(name, "compile",
ShaderRecompiler::TryRecompile(code, options, result, &error),
error);
Require(name, "fetch rewrite", Require(name, "fetch rewrite",
result.program.info.vertex_fetch_components[0] == 4, result.program.info.vertex_fetch_components[0] == 4,
"encoded fetch sequence was not recognized and rewritten"); "encoded fetch sequence was not recognized and rewritten");
@@ -23477,12 +23456,8 @@ void CheckImageSamplerSpecialization() {
ShaderRecompiler::IR::DescriptorValue sampler_descriptor{}; ShaderRecompiler::IR::DescriptorValue sampler_descriptor{};
sampler_descriptor.dword_count = 4; sampler_descriptor.dword_count = 4;
mixed_sampler_snapshot.samplers.push_back(sampler_descriptor); mixed_sampler_snapshot.samplers.push_back(sampler_descriptor);
std::string specialization_error; ShaderRecompiler::IR::SpecializeResources(mixed_sampler_program,
Require("ImageSamplerSpecialization", "mixed sampler specialization", mixed_sampler_snapshot);
ShaderRecompiler::IR::SpecializeResources(mixed_sampler_program,
mixed_sampler_snapshot,
&specialization_error),
specialization_error);
Require("ImageSamplerSpecialization", "mixed sampler variant", Require("ImageSamplerSpecialization", "mixed sampler variant",
mixed_sampler_program.info.samplers.size() == 2u && mixed_sampler_program.info.samplers.size() == 2u &&
!mixed_sampler_program.info.samplers[0].force_point_filtering && !mixed_sampler_program.info.samplers[0].force_point_filtering &&
@@ -23493,25 +23468,45 @@ void CheckImageSamplerSpecialization() {
"a shared native/bit-packed sampler was not split into point and " "a shared native/bit-packed sampler was not split into point and "
"native variants"); "native variants");
ShaderRecompiler::IR::Program packed_atomic_program;
packed_atomic_program.resource_tracking_complete = true;
auto packed_atomic_image = sampled_image;
packed_atomic_image.kind =
ShaderRecompiler::IR::ResourceKind::StorageImageUint;
packed_atomic_image.written = true;
packed_atomic_image.atomic = true;
packed_atomic_program.info.images.push_back(packed_atomic_image);
ShaderRecompiler::IR::ResourceSnapshot packed_atomic_snapshot;
packed_atomic_snapshot.images.push_back(packed_image_descriptor);
std::string atomic_error;
Require("ImageSamplerSpecialization", "packed atomic rejection",
!ShaderRecompiler::IR::SpecializeResources(
packed_atomic_program, packed_atomic_snapshot, &atomic_error) &&
atomic_error.find("unsupported format 34") != std::string::npos,
"a non-atomic PS5 packed format entered the typed atomic path");
std::printf("[host] %-32s ok\n", "ImageSpecializationPipelineId"); std::printf("[host] %-32s ok\n", "ImageSpecializationPipelineId");
} }
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
void CheckShaderRecompilerFatalContracts() {
ExpectFatal("WritableFlatStoreRejection", [] {
const auto test = FlatStoreVariants();
(void)CompileCase(test);
});
ExpectFatal("PackedAtomicImageRejection", [] {
ShaderRecompiler::IR::Program program;
program.resource_tracking_complete = true;
ShaderRecompiler::IR::ImageResource image;
image.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint;
image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
image.read = true;
image.written = true;
image.atomic = true;
program.info.images.push_back(image);
ShaderRecompiler::IR::DescriptorValue descriptor{};
descriptor.dword_count = 8;
descriptor.dwords[0] = 0x2000u;
descriptor.dwords[1] =
static_cast<uint32_t>(Prospero::BufferFormat::k11_11_10UInt) << 20u;
descriptor.dwords[2] = 3u | (3u << 14u);
descriptor.dwords[3] =
DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
ShaderRecompiler::IR::ResourceSnapshot snapshot;
snapshot.images.push_back(descriptor);
ShaderRecompiler::IR::SpecializeResources(program, snapshot);
});
}
#endif
void CheckStorageTextureVolumeUploadLayout() { void CheckStorageTextureVolumeUploadLayout() {
constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float; constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float;
constexpr uint32_t width = 33; constexpr uint32_t width = 33;
@@ -24232,10 +24227,7 @@ void CheckEmbeddedFetchLaneSpill() {
options.user_data = user_data.data(); options.user_data = user_data.data();
options.input_info.vertex = &vertex; options.input_info.vertex = &vertex;
ShaderRecompiler::CompileResult result; auto result = ShaderRecompiler::Recompile(code, options);
std::string error;
Require("EmbeddedFetchLaneSpill", "compile",
ShaderRecompiler::TryRecompile(code, options, result, &error), error);
Require("EmbeddedFetchLaneSpill", "fetch rewrite", Require("EmbeddedFetchLaneSpill", "fetch rewrite",
result.program.info.vertex_fetch_components[0] == 4, result.program.info.vertex_fetch_components[0] == 4,
"lane-spilled fetch descriptor was not recognized and rewritten"); "lane-spilled fetch descriptor was not recognized and rewritten");
@@ -25041,6 +25033,12 @@ int main(int argc, char **argv) {
std::setvbuf(stdout, nullptr, _IONBF, 0); std::setvbuf(stdout, nullptr, _IONBF, 0);
EnsureConfigInitialized(); EnsureConfigInitialized();
CheckLeastRecentlyUsedCacheOrdering(); CheckLeastRecentlyUsedCacheOrdering();
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
if (argc == 2 && std::strcmp(argv[1], "--shader-fatal-only") == 0) {
CheckShaderRecompilerFatalContracts();
return 0;
}
#endif
if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) { if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) {
CheckClipControlDepthClipState(); CheckClipControlDepthClipState();
return 0; return 0;
@@ -25301,6 +25299,7 @@ int main(int argc, char **argv) {
std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); std::fprintf(stderr, "unknown test selector: %s\n", argv[1]);
return 2; return 2;
} }
CheckShaderRecompilerFatalContracts();
VulkanHarness vulkan; VulkanHarness vulkan;
#endif #endif
CheckClipControlDepthClipState(); CheckClipControlDepthClipState();
+49 -7
View File
@@ -52,7 +52,7 @@ SrtProgram(uint64_t address) {
ValueOpcode::LoadAddressU32, ValueOpcode::LoadAddressU32,
{Value(&handle), Value(0u), Value(0u), Value(true)}); {Value(&handle), Value(0u), Value(0u), Value(true)});
raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40}); raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40});
program.srt_reads.push_back({Value(&raw), 0, 0x40}); program.srt_reads.push_back({Value(&raw), 0});
auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource); auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource);
auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst, auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst,
@@ -77,17 +77,38 @@ UnbasedFlatProgram() {
return std::make_shared<const Program>(std::move(program)); return std::make_shared<const Program>(std::move(program));
} }
std::shared_ptr<const Libs::Graphics::ShaderRecompiler::IR::Program>
UserDataBufferProgram() {
using namespace Libs::Graphics::ShaderRecompiler::IR;
Program program;
program.stage = Libs::Graphics::ShaderType::Compute;
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
auto &value_block = AddValueBlock(program);
auto &user_data = value_block.AppendNewInst(
ValueOpcode::GetUserData, {Value(static_cast<ScalarReg>(0))});
DescriptorSource source;
source.dwords[0] = Value(&user_data);
source.dwords[1] = Value(0u);
source.dwords[2] = Value(0u);
source.dwords[3] = Value(0u);
source.dword_count = 4;
program.descriptor_sources.push_back(source);
program.info.buffers.push_back({.source = 0});
return std::make_shared<const Program>(std::move(program));
}
void TestMappedSrtUsesDirectReaderByDefault() { void TestMappedSrtUsesDirectReaderByDefault() {
using namespace Libs::Graphics; using namespace Libs::Graphics;
const uint32_t dword = 0x12345678; const uint32_t dword = 0x12345678;
auto cached_program = SrtProgram(reinterpret_cast<uint64_t>(&dword)); auto cached_program = SrtProgram(reinterpret_cast<uint64_t>(&dword));
ShaderStageRuntime stage; ShaderStageRuntime stage;
std::string error;
uint32_t specialization_reads = 0; uint32_t specialization_reads = 0;
Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error, Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage,
RejectSpecializationRead, RejectSpecializationRead,
&specialization_reads), &specialization_reads),
error.c_str()); "mapped SRT stage materialization failed");
Check(specialization_reads == 0, Check(specialization_reads == 0,
"ordinary SRT read used the specialization reader"); "ordinary SRT read used the specialization reader");
Check(stage.program == cached_program && stage.resources != nullptr, Check(stage.program == cached_program && stage.resources != nullptr,
@@ -104,16 +125,36 @@ void TestUnbasedFlatCacheHitMaterializes() {
auto prior_resources = auto prior_resources =
std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>(); std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>();
ShaderStageRuntime stage{prior_program, prior_resources}; ShaderStageRuntime stage{prior_program, prior_resources};
std::string error; Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage),
Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error), error.c_str()); "unbased FLAT stage materialization failed");
Check(stage.program == cached_program && stage.resources != prior_resources, Check(stage.program == cached_program && stage.resources != prior_resources,
"unbased FLAT cache hit did not publish its BDA stage snapshot"); "unbased FLAT cache hit did not publish its BDA stage snapshot");
} }
void TestFailedMaterializationPreservesPriorStage() {
using namespace Libs::Graphics;
auto cached_program = UserDataBufferProgram();
auto prior_program = std::make_shared<const ShaderRecompiler::IR::Program>();
auto prior_resources =
std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>();
ShaderStageRuntime stage{prior_program, prior_resources};
Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, stage),
"missing runtime user data did not reject the cached stage");
Check(stage.program == prior_program && stage.resources == prior_resources,
"failed cache materialization replaced the prior stage");
}
} // namespace } // namespace
namespace Common { namespace Common {
int DbgExitHandler(const char *, int, std::string_view) { std::abort(); }
int DbgExitHandler(const char *, int, fmt::text_style, std::string_view) {
std::abort();
}
int DbgExitIfHandler(const char *, const char *, int) { return 1; } int DbgExitIfHandler(const char *, const char *, int) { return 1; }
void DbgExit(int) { std::abort(); } void DbgExit(int) { std::abort(); }
@@ -123,6 +164,7 @@ void DbgExit(int) { std::abort(); }
int main() { int main() {
TestMappedSrtUsesDirectReaderByDefault(); TestMappedSrtUsesDirectReaderByDefault();
TestUnbasedFlatCacheHitMaterializes(); TestUnbasedFlatCacheHitMaterializes();
TestFailedMaterializationPreservesPriorStage();
std::puts("ShaderStageRuntimeTests: all cases passed"); std::puts("ShaderStageRuntimeTests: all cases passed");
return 0; return 0;
} }
@@ -130,7 +172,7 @@ int main() {
// Keep this focused standalone target self-contained by amalgamating its small // Keep this focused standalone target self-contained by amalgamating its small
// typed-IR implementation set. // typed-IR implementation set.
#include "graphics/shader/recompiler/ir/Block.cpp" #include "graphics/shader/recompiler/ir/Block.cpp"
#include "graphics/shader/recompiler/ir/Program.cpp"
#include "graphics/shader/recompiler/ir/Type.cpp" #include "graphics/shader/recompiler/ir/Type.cpp"
#include "graphics/shader/recompiler/ir/Value.cpp" #include "graphics/shader/recompiler/ir/Value.cpp"
#include "graphics/shader/recompiler/ir/Program.cpp"
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
+629 -1261
View File
File diff suppressed because it is too large. Load diff