gremling/KytyPS5
1
0 Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-28 08:13:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

shader_recompiler: add SSA

Browse Source
This commit is contained in:
nmzik committed 2026-08-18 03:53:41 +02:00
1 parent bc2f077c37
commit b42380ced6
113 files changed
+40415 -32616
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/graphics/shader/recompiler/ShaderRecompiler.cpp
+171 -72
View File
@@ -2,18 +2,21 @@
#include "common/assert.h"
#include "common/logging/log.h"
#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
#include "graphics/shader/recompiler/ir/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ReadLaneElimination.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/translate/Translate.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/ir/SrtPatcher.h"
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
#include <algorithm>
#include <array>
@@ -43,7 +46,7 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
std::string dump = "CFG:\n";
dump += CFG::GraphToString(cfg);
dump += "\nIR:\n";
dump += IR::ProgramToString(ir);
dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir);
return dump;
}
@@ -56,10 +59,22 @@ const char* StageName(ShaderType stage) {
}
}
std::string FormatCfgFailure(const CFG::Graph& cfg, const CompileOptions& options,
const std::string& reason) {
const auto block_id = cfg.failure_block != UINT32_MAX ? cfg.failure_block : cfg.entry_block;
return CFG::FormatBlockDiagnostic(cfg, block_id, StageName(options.stage), reason);
void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase,
const std::string& reason) {
const auto* block = cfg.FindBlock(cfg.failure_block);
const auto start = block != nullptr ? block->start_pc : UINT32_MAX;
const auto end = block != nullptr ? block->end_pc : UINT32_MAX;
const auto predecessors = block != nullptr ? block->predecessors.size() : 0u;
const auto successors = block != nullptr ? block->successors.size() : 0u;
LOGF("%s CFG dispatcher fallback: stage=%s hash=0x%016" PRIx64
" phase=%s failure=%s block=%" PRIu32 " pc=0x%08" PRIx32 "..0x%08" PRIx32 " preds=%" PRIu64
" succs=%" PRIu64 " blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64
" reason=%s\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash, phase,
CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
}
bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
@@ -94,21 +109,31 @@ bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
case IR::Opcode::FlatStoreByte:
case IR::Opcode::FlatStoreShort:
case IR::Opcode::FlatStoreDword:
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32:
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32:
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32:
case IR::Opcode::DsWriteAddtidB32:
case IR::Opcode::DsReadAddtidB32:
case IR::Opcode::DsAppend:
case IR::Opcode::DsConsume:
case IR::Opcode::ImageLoad:
case IR::Opcode::ImageStore:
case IR::Opcode::ImageSample:
case IR::Opcode::ImageGather4:
case IR::Opcode::Export: return true;
default: return false;
}
}
bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* reason) {
bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, uint32_t* failure_block,
std::string* reason) {
for (const auto& block: ir.blocks) {
if (!block.terminator.loop_header) {
continue;
@@ -122,6 +147,9 @@ bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string*
"lowering emits internal control flow: {}",
block.id, IR::InstructionToString(inst).c_str());
}
if (failure_block != nullptr) {
*failure_block = block.id;
}
return true;
}
}
@@ -697,27 +725,25 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
bool dispatcher_fallback = false;
std::string dispatcher_reason;
if (cfg.irreducible) {
dispatcher_fallback = true;
dispatcher_reason = cfg.unsupported_reason;
const auto diagnostic = FormatCfgFailure(cfg, options, cfg.unsupported_reason);
LOGF("%s irreducible CFG detected: %s\n", GetDumpLabel(options), diagnostic.c_str());
dispatcher_fallback = true;
dispatcher_reason = cfg.unsupported_reason;
LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
} else {
std::string structure_error;
const auto unstructured_cfg = cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!CFG::Structurize(cfg, &structure_error)) {
const auto diagnostic = FormatCfgFailure(cfg, options, structure_error);
LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str());
dispatcher_fallback = true;
dispatcher_reason = structure_error;
const auto failure_kind = cfg.failure_kind;
const auto failure_block = cfg.failure_block;
cfg = unstructured_cfg;
cfg.unsupported = true;
cfg.failure_kind = failure_kind;
cfg.failure_block = failure_block;
cfg.unsupported_reason = structure_error;
LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
cfg = unstructured_cfg;
cfg.unsupported = true;
cfg.failure_kind = failure_kind;
cfg.failure_block = failure_block;
cfg.unsupported_reason = structure_error;
} else {
LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
static_cast<uint64_t>(cfg.blocks.size()));
@@ -732,13 +758,84 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
IR::Program ir;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash);
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, ir, error)) {
const auto lower_program = [&](IR::Program& output) {
output = {};
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
return false;
}
output.lane_mask_mode = options.lane_mask_mode;
output.shader_hash = options.shader_hash;
output.user_data_base = options.user_data_base;
output.user_data_count = options.user_data_count;
return true;
};
if (!lower_program(ir)) {
return false;
}
ir.lane_mask_mode = options.lane_mask_mode;
ir.shader_hash = options.shader_hash;
ir.user_data_base = options.user_data_base;
ir.user_data_count = options.user_data_count;
if (!dispatcher_fallback) {
const auto select_dispatcher = [&](uint32_t failure_block, const std::string& reason,
const CFG::Graph& diagnostic_cfg) {
dispatcher_fallback = true;
dispatcher_reason = reason;
cfg.failure_kind = CFG::FailureKind::StructuredControlFlow;
cfg.failure_block = failure_block;
cfg.unsupported_reason = reason;
ir.cfg_failure_kind = CFG::FailureKind::StructuredControlFlow;
ir.fallback_reason = reason;
LogDispatcherFallback(options, diagnostic_cfg, "emit", reason);
};
const auto retry_budget = cfg.natural_loops.size();
for (size_t retry = 0;; retry++) {
std::string emitter_reason;
uint32_t failure_block = UINT32_MAX;
if (!NeedsDispatcherForStructuredLoopHeader(ir, &failure_block, &emitter_reason)) {
break;
}
if (retry >= retry_budget) {
select_dispatcher(failure_block,
emitter_reason + "; no-clone loop-header isolation exhausted",
cfg);
break;
}
const auto* source_block = cfg.FindBlock(failure_block);
const auto source_pc = source_block != nullptr ? source_block->start_pc : UINT32_MAX;
const auto previous_cfg = cfg;
const auto blocks_before = cfg.blocks.size();
std::string isolation_error;
if (!CFG::IsolateLoopHeader(cfg, failure_block, &isolation_error)) {
const auto failed_cfg = cfg;
cfg = previous_cfg;
select_dispatcher(failure_block,
fmt::format("{}; no-clone loop-header isolation failed: {}",
emitter_reason, isolation_error),
failed_cfg);
break;
}
IR::Program isolated_ir;
if (!lower_program(isolated_ir)) {
const auto lower_error = error != nullptr ? *error : "IR lowering failed";
if (error != nullptr) {
error->clear();
}
cfg = previous_cfg;
select_dispatcher(failure_block,
fmt::format("{}; isolated loop-header lowering failed: {}",
emitter_reason, lower_error),
cfg);
break;
}
ir = std::move(isolated_ir);
LOGF("%s CFG isolated semantic loop header: stage=%s hash=0x%016" PRIx64
" source_block=%" PRIu32 " pc=0x%08" PRIx32 " blocks=%" PRIu64 "->%" PRIu64
" reason=%s\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
failure_block, source_pc, static_cast<uint64_t>(blocks_before),
static_cast<uint64_t>(cfg.blocks.size()), emitter_reason.c_str());
}
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
" elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
@@ -757,9 +854,41 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
rewritten);
}
}
if (!IR::BuildScalarProvenance(ir, error)) {
ir.dispatcher_fallback = dispatcher_fallback;
if (!dispatcher_reason.empty()) {
ir.fallback_reason = dispatcher_reason;
}
ShaderVertexInputInfo default_vertex {};
ShaderPixelInputInfo default_pixel {};
ShaderComputeInputInfo default_compute {};
const auto* vertex =
options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex;
const auto* pixel =
options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel;
const auto* compute =
options.compute_input_info != nullptr ? options.compute_input_info : &default_compute;
ir.values = std::make_shared<IR::ValueProgram>();
if (!Frontend::TranslateProgram(ir, *ir.values, vertex, pixel, compute, error)) {
return false;
}
ir.blocks.clear();
IR::RewriteToSsa(ir.values->blocks);
IR::ConstantPropagationPass(ir.values->blocks);
for (auto& info: ir.values->block_info) {
info.condition = info.condition.Resolve();
info.indirect_target = info.indirect_target.Resolve();
}
IR::RemoveIdentities(ir.values->blocks);
IR::EliminateDeadCode(ir.values->blocks);
const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size);
if (read_lane_stats.rewritten_reads != 0) {
LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
read_lane_stats.rewritten_reads);
IR::ConstantPropagationPass(ir.values->blocks);
IR::RemoveIdentities(ir.values->blocks);
IR::EliminateDeadCode(ir.values->blocks);
}
std::string srt_error;
if (!IR::BuildSrtPlan(ir, &srt_error)) {
LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str());
@@ -768,29 +897,13 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
}
return false;
}
ir.dispatcher_fallback = dispatcher_fallback;
if (!dispatcher_reason.empty()) {
ir.fallback_reason = dispatcher_reason;
}
if (!IR::PatchSrtReads(ir, error) || !IR::TrackResources(ir, error)) {
IR::EliminateDeadCode(ir.values->blocks);
if (!IR::TrackResources(ir, error)) {
return false;
}
if (options.stage == ShaderType::Vertex) {
ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
}
if (!dispatcher_fallback) {
std::string emitter_reason;
if (NeedsDispatcherForStructuredLoopHeader(ir, &emitter_reason)) {
dispatcher_fallback = true;
dispatcher_reason = emitter_reason;
ir.dispatcher_fallback = true;
ir.cfg_failure_kind = CFG::FailureKind::StructuredControlFlow;
ir.fallback_reason = emitter_reason;
LOGF("%s structured CFG emitter fallback: %s\n", GetDumpLabel(options),
emitter_reason.c_str());
}
}
IR::ResourceSnapshot resources;
if (options.resource_snapshot != nullptr) {
@@ -822,16 +935,10 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
return false;
}
ShaderVertexInputInfo default_vertex {};
ShaderPixelInputInfo default_pixel {};
ShaderComputeInputInfo default_compute {};
IR::ShaderInfoOptions info_options;
info_options.vertex =
options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex;
info_options.pixel =
options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel;
info_options.compute =
options.compute_input_info != nullptr ? options.compute_input_info : &default_compute;
IR::ShaderInfoOptions info_options;
info_options.vertex = vertex;
info_options.pixel = pixel;
info_options.compute = compute;
if (!IR::CollectShaderInfo(ir, info_options, error)) {
return false;
}
@@ -841,11 +948,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
if (!IR::AllocateBindings(ir, layout_options, error)) {
return false;
}
const auto read_lane_stats = IR::EliminateReadLane(ir);
if (read_lane_stats.rewritten_reads != 0) {
LOGF("%s read-lane elimination: reads=%" PRIu32 " shadow_writes=%" PRIu32 "\n",
GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.shadow_writes);
}
std::string ir_dump;
if (options.dump_ir) {
ir_dump = MakeIrDump(cfg, ir);
@@ -859,7 +961,8 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info,
options.compute_input_info, spirv, &emit_error)) {
options.compute_input_info, spirv, &emit_error, options.dump_ir)) {
LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str());
if (dispatcher_fallback && error != nullptr) {
*error = fmt::format("dispatcher fallback failed after {}: {}",
dispatcher_reason.c_str(), emit_error.c_str());
@@ -873,10 +976,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
" elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(spirv.size()), phase_ms());
if (dispatcher_fallback) {
LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str());
}
result.spirv = std::move(spirv);
result.program = std::move(ir);
result.resources = std::move(resources);
src/graphics/shader/recompiler/ShaderRecompiler.h
+1 -1
View File
@@ -3,7 +3,7 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/shader.h"
#include <optional>
src/graphics/shader/recompiler/emitter/SpirvBuilder.cpp → src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include <cstdio>
#include <cstring>
src/graphics/shader/recompiler/emitter/SpirvBuilder.h → src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h
View File
File renamed without changes.
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+396
View File
@@ -0,0 +1,396 @@
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cstdio>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
namespace {
bool Fail(std::string* error, const std::string& message) {
if (error != nullptr) {
*error = message;
}
return false;
}
bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
using Kind = IR::DescriptorBindingKind;
using Dim = Decoder::ImageDimension;
if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
const bool integer = image.kind == IR::ResourceKind::ImageUint;
switch (image.dimension) {
case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
case Dim::Dim1DArray:
kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
return true;
case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
case Dim::Dim2DMsaa:
kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
return true;
case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
case Dim::Dim2DArray:
kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
return true;
case Dim::Dim2DMsaaArray:
kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
return true;
case Dim::Unknown: return false;
}
}
const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
return false;
}
switch (image.dimension) {
case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
case Dim::Dim1DArray:
kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
return true;
case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
case Dim::Dim2DArray:
kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
return true;
case Dim::Dim2DMsaa:
case Dim::Dim2DMsaaArray: return false;
case Dim::Unknown: return false;
}
return false;
}
bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
using Kind = IR::DescriptorBindingKind;
constexpr auto KindCount = static_cast<size_t>(Kind::Count);
std::array<std::vector<uint32_t>, KindCount> expected;
std::array<bool, KindCount> present {};
const auto Dense = [](size_t size) {
std::vector<uint32_t> values(size);
for (uint32_t i = 0; i < values.size(); i++) {
values[i] = i;
}
return values;
};
auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
const auto index = static_cast<size_t>(kind);
present[index] = true;
expected[index] = std::move(resources);
};
if (!program.info.buffers.empty()) {
Expect(Kind::Buffers, Dense(program.info.buffers.size()));
}
for (uint32_t i = 0; i < program.info.images.size(); i++) {
Kind kind;
if (!ImageBinding(program.info.images[i], kind)) {
return Fail(error, "native shader plan has an invalid image class");
}
present[static_cast<size_t>(kind)] = true;
expected[static_cast<size_t>(kind)].push_back(i);
}
if (!program.info.samplers.empty()) {
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
}
const auto uses_gds = program.values != nullptr &&
std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
return std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
});
});
if (uses_gds) {
Expect(Kind::Gds);
}
if (!program.info.addresses.empty()) {
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
}
if (program.values != nullptr && !program.values->srt_reads.empty()) {
Expect(Kind::FlattenedSrt);
}
if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
Expect(Kind::UserData);
}
std::array<bool, KindCount> seen {};
for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
const auto& binding = program.bindings.descriptors[i];
const auto kind = static_cast<size_t>(binding.kind);
if (kind >= KindCount || seen[kind] || !present[kind] ||
binding.resources != expected[kind]) {
return Fail(error, "native descriptor groups do not match shader topology");
}
for (uint32_t j = 0; j < i; j++) {
if (program.bindings.descriptors[j].binding == binding.binding) {
return Fail(error, "native descriptor binding numbers are not unique");
}
}
seen[kind] = true;
}
for (size_t i = 0; i < KindCount; i++) {
if (present[i] != seen[i]) {
return Fail(error, "native shader plan is missing a required descriptor group");
}
}
const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
program.bindings.buffer_offset_count != program.info.buffers.size() ||
(!has_user_storage &&
program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
(has_user_storage && program.bindings.push_constant_size != 0) ||
!std::is_sorted(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) ||
std::adjacent_find(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) !=
program.bindings.user_data_registers.end()) {
return Fail(error, "native user-data layout is inconsistent");
}
if (program.values == nullptr) {
return Fail(error, "native shader plan has no typed SSA");
}
const auto planning_only_handle = [&](const IR::Inst& handle) {
return !handle.Uses().empty() &&
std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
const auto op = use.user->GetOpcode();
if (op != IR::ValueOpcode::LoadAddressU32 &&
op != IR::ValueOpcode::ReadConstBuffer) {
return false;
}
const auto index = use.user->Flags<IR::MemoryFlags>().index;
return index < program.values->memory_info.size() &&
program.values->memory_info[index].planning_only;
});
};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
const auto dense = inst.Flags<uint32_t>();
switch (inst.GetOpcode()) {
case IR::ValueOpcode::GetBufferResource:
if (planning_only_handle(inst)) {
break;
}
if (dense >= program.info.buffers.size()) {
return Fail(error, "typed buffer handle has an invalid dense resource");
}
break;
case IR::ValueOpcode::GetAddressResource:
if (planning_only_handle(inst)) {
break;
}
if (dense >= program.info.addresses.size()) {
return Fail(error, "typed address handle has an invalid dense resource");
}
break;
case IR::ValueOpcode::GetImageResource:
if (dense >= program.info.images.size()) {
return Fail(error, "typed image handle has an invalid dense resource");
}
break;
case IR::ValueOpcode::GetSamplerResource:
if (dense >= program.info.samplers.size()) {
return Fail(error, "typed sampler handle has an invalid dense resource");
}
break;
case IR::ValueOpcode::ReadConst: {
const auto slot = inst.Arg(1).Resolve();
if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
slot.U32() >= program.values->srt_reads.size()) {
return Fail(error, "flattened SRT read has an invalid dense slot");
}
break;
}
default: break;
}
}
}
return true;
}
} // namespace
static constexpr size_t InitialEmitterVectorReserve = 4096;
static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
size_t size) {
if (size <= InitialEmitterVectorReserve) {
return;
}
std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
" stage=%u vector=%s size=%zu reserve=%zu\n",
program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
InitialEmitterVectorReserve);
}
void CollectValueRegisterMirrors(const IR::ValueProgram& program,
std::vector<Emitter::RegisterBinding>& registers) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case IR::ValueOpcode::SetScalarRegister:
case IR::ValueOpcode::SetThreadBitScalarRegister:
if (block != program.blocks.front()) {
Emitter::CollectRegister(
registers,
{IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())});
}
break;
case IR::ValueOpcode::SetVectorRegister:
Emitter::CollectRegister(
registers,
{IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())});
break;
case IR::ValueOpcode::SetM0:
if (block != program.blocks.front()) {
Emitter::CollectRegister(registers, {IR::RegisterFile::M0, 0});
}
break;
case IR::ValueOpcode::SetExecLo:
case IR::ValueOpcode::SetExecHi:
if (block != program.blocks.front()) {
Emitter::CollectRegister(
registers, {IR::RegisterFile::Exec,
inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u});
}
break;
case IR::ValueOpcode::SetVccLo:
case IR::ValueOpcode::SetVccHi:
if (block != program.blocks.front()) {
Emitter::CollectRegister(
registers, {IR::RegisterFile::Vcc,
inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u});
}
break;
default: break;
}
}
}
}
bool ValueProgramHasOpcode(const IR::ValueProgram& program, IR::ValueOpcode opcode) {
return std::ranges::any_of(program.blocks, [opcode](const IR::Block* block) {
return std::ranges::any_of(
*block, [opcode](const IR::Inst& inst) { return inst.GetOpcode() == opcode; });
});
}
bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
if (program.values == nullptr) {
return false;
}
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot:
case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WqmMask:
case IR::ValueOpcode::ReadFirstLane:
case IR::ValueOpcode::ReadLane:
case IR::ValueOpcode::WriteLane:
case IR::ValueOpcode::Permlane16U32:
case IR::ValueOpcode::SwizzleU32:
case IR::ValueOpcode::DataAppend:
case IR::ValueOpcode::DataConsume:
case IR::ValueOpcode::GdsDataAppend:
case IR::ValueOpcode::GdsDataConsume: return true;
default: break;
}
}
}
return false;
}
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
std::string* error, bool preserve_debug_values) {
using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) {
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
return false;
}
if (!program.srt_plan_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete) {
SetError(error, "SPIR-V emitter requires a fully planned native shader program");
return false;
}
if (!IR::ValidateResourceSnapshot(program, resources, error)) {
return false;
}
if (!IR::ValidateResourceSpecialization(program, resources, error)) {
return false;
}
if (!ValidateNativeProgram(program, error)) {
return false;
}
if (program.values == nullptr) {
SetError(error, "SPIR-V emitter requires planned typed SSA");
return false;
}
ShaderVertexInputInfo default_vertex {};
ShaderPixelInputInfo default_pixel {};
ShaderComputeInputInfo default_compute {};
const auto* vertex = vertex_input_info != nullptr ? vertex_input_info : &default_vertex;
const auto* pixel = pixel_input_info != nullptr ? pixel_input_info : &default_pixel;
const auto* compute = compute_input_info != nullptr ? compute_input_info : &default_compute;
(void)preserve_debug_values;
const auto& value_program = *program.values;
if (!IR::ValidateValueProgram(value_program, true, error)) {
return false;
}
EmitterState state(program, resources);
state.vertex_input_info = vertex;
state.pixel_input_info = pixel;
state.compute_input_info = compute;
state.stage = program.stage;
state.wave_size = program.wave_size;
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
state.exact_subgroup_operations =
!state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
state.registers.reserve(InitialEmitterVectorReserve);
state.inputs.reserve(InitialEmitterVectorReserve);
state.outputs.reserve(InitialEmitterVectorReserve);
state.interface_variables.reserve(InitialEmitterVectorReserve);
state.reachable_blocks.reserve(InitialEmitterVectorReserve);
CollectValueRegisterMirrors(value_program, state.registers);
CopyProgramInputsAndOutputs(state, program);
state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program) ||
(!state.per_invocation_masks &&
ValueProgramHasOpcode(value_program, IR::ValueOpcode::Ballot));
state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program);
state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program);
state.needs_function_lds = ProgramNeedsFunctionLds(program);
state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program);
AllocateInputVariables(state);
AllocateOutputVariables(state);
AllocateDescriptorVariables(state);
EmitHeaderAndTypes(state);
AllocateRegisterVariables(state);
if (!EmitValueProgram(state, value_program, error)) {
return false;
}
ReportReserveExceeded(program, "registers", state.registers.size());
ReportReserveExceeded(program, "inputs", state.inputs.size());
ReportReserveExceeded(program, "outputs", state.outputs.size());
ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
auto binary = state.builder.Build();
if (binary.empty()) {
SetError(error, "SPIR-V builder returned an empty module");
return false;
}
spirv = std::move(binary);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/emitter/SpirvEmitter.h → src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h
+2 -2
View File
@@ -3,7 +3,7 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include <vector>
@@ -15,7 +15,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
std::string* error);
std::string* error, bool preserve_debug_values = false);
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/emitter/spirvEmitterAluOps.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
src/graphics/shader/recompiler/emitter/spirvEmitterAnalysis.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp
+97 -236
View File
@@ -1,7 +1,8 @@
#include "common/assert.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -75,168 +76,6 @@ bool IsSccOperand(const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
}
void CollectMaskStateRegisters(std::vector<RegisterBinding>& registers) {
CollectRegister(registers, {IR::RegisterFile::Exec, 0});
CollectRegister(registers, {IR::RegisterFile::Exec, 1});
CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
CollectRegister(registers, {IR::RegisterFile::Vcc, 1});
CollectRegister(registers, SccRegister());
}
void CollectSequentialRegisters(std::vector<RegisterBinding>& registers, const IR::Operand& base,
uint32_t count) {
if (base.kind != IR::OperandKind::Register) {
return;
}
for (uint32_t i = 0; i < count; i++) {
auto reg = base.reg;
reg.index += i;
CollectRegister(registers, reg);
}
}
uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& registers) {
uint32_t max_end = 0;
for (const auto& binding: registers) {
if (binding.reg.file == IR::RegisterFile::Vector) {
max_end = std::max(max_end, binding.reg.index + 1u);
}
}
return std::min(max_end, 256u);
}
void CollectMoveRelSourceRegisters(const IR::Program& program,
std::vector<RegisterBinding>& registers) {
const auto max_vector_end = MaxCollectedVectorRegisterEnd(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != IR::Opcode::MoveRelSourceU32 ||
inst.src[0].kind != IR::OperandKind::Register ||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
continue;
}
const auto base_index = inst.src[0].reg.index;
const auto limit = std::min(std::max(max_vector_end, base_index + 1u), 256u);
CollectSequentialRegisters(registers, inst.src[0], limit - base_index);
}
}
}
bool IsPairDwordOpcode(IR::Opcode op) {
switch (op) {
case IR::Opcode::MoveU64:
case IR::Opcode::WqmB64:
case IR::Opcode::SaveexecB64:
case IR::Opcode::BitwiseAndU64:
case IR::Opcode::BitwiseAndNotU64:
case IR::Opcode::BitwiseOrU64:
case IR::Opcode::BitwiseOrNotU64:
case IR::Opcode::BitwiseXorU64:
case IR::Opcode::BitwiseNandU64:
case IR::Opcode::BitwiseNorU64:
case IR::Opcode::BitwiseXnorU64:
case IR::Opcode::BitwiseNotU64:
case IR::Opcode::BitFieldMaskU64:
case IR::Opcode::BitFieldExtractU64:
case IR::Opcode::BitReplicateB64B32:
case IR::Opcode::ShiftLeftLogicalU64:
case IR::Opcode::ShiftRightLogicalU64:
case IR::Opcode::SelectU64: return true;
default: return false;
}
}
uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
switch (op) {
case IR::Opcode::BitFieldMaskU64: return 0;
case IR::Opcode::BitReplicateB64B32: return 0;
case IR::Opcode::BitFieldExtractU64: return 1;
case IR::Opcode::ShiftLeftLogicalU64:
case IR::Opcode::ShiftRightLogicalU64:
case IR::Opcode::BitwiseNotU64:
case IR::Opcode::SaveexecB64:
case IR::Opcode::WqmB64:
case IR::Opcode::MoveU64: return std::min<uint32_t>(src_count, 1u);
default: return src_count;
}
}
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>& registers) {
CollectMaskStateRegisters(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.dst.kind == IR::OperandKind::Register) {
CollectRegister(registers, inst.dst.reg);
}
if (inst.dst2.kind == IR::OperandKind::Register) {
CollectRegister(registers, inst.dst2.reg);
}
for (uint32_t i = 0; i < inst.src_count; i++) {
if (inst.src[i].kind == IR::OperandKind::Register) {
CollectRegister(registers, inst.src[i].reg);
}
}
if (IsPairDwordOpcode(inst.op)) {
CollectSequentialRegisters(registers, inst.dst, 2);
const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u;
const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count);
for (uint32_t i = first_pair_src; i < pair_src_count; i++) {
CollectSequentialRegisters(registers, inst.src[i], 2);
}
}
if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) {
CollectSequentialRegisters(registers, inst.src[0], 2);
}
if (IR::IsCompare64Opcode(inst.op)) {
CollectSequentialRegisters(registers, inst.src[0], 2);
CollectSequentialRegisters(registers, inst.src[1], 2);
}
if (IR::IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file != IR::RegisterFile::Scc) {
CollectSequentialRegisters(registers, inst.dst, 2);
}
if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) {
CollectSequentialRegisters(registers, inst.dst2, 2);
}
if (inst.op == IR::Opcode::UMadU64U32) {
CollectSequentialRegisters(registers, inst.dst, 2);
CollectSequentialRegisters(registers, inst.dst2, 2);
CollectSequentialRegisters(registers, inst.src[2], 2);
}
if (inst.op == IR::Opcode::SBufferLoadDword &&
inst.memory.kind == IR::ResourceKind::ScalarBuffer) {
CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u});
}
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) &&
inst.memory.image_address_components > 1u) {
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[0],
inst.memory.image_address_components);
}
if (inst.op == IR::Opcode::ImageGetLod) {
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[0],
inst.memory.image_address_components);
}
if (inst.op == IR::Opcode::ImageLoad) {
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[0],
inst.memory.image_address_components);
}
if (inst.op == IR::Opcode::ImageStore) {
CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[1],
inst.memory.image_address_components);
}
if (inst.memory.kind == IR::ResourceKind::StorageImageUint) {
CollectSequentialRegisters(registers, inst.src[1],
inst.memory.image_address_components);
}
}
}
CollectMoveRelSourceRegisters(program, registers);
}
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
uint32_t index) {
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
@@ -276,9 +115,12 @@ uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo
}
bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::ImageGetLod) {
if (program.values == nullptr) {
return false;
}
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == IR::ValueOpcode::ImageQueryLod) {
return true;
}
}
@@ -287,9 +129,12 @@ bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
}
bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::ImageGather4) {
if (program.values == nullptr) {
return false;
}
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == IR::ValueOpcode::ImageGatherRaw) {
return true;
}
}
@@ -297,34 +142,43 @@ bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
return false;
}
bool IsLdsOpcode(IR::Opcode op) {
switch (op) {
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32:
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32:
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32:
case IR::Opcode::DsSwizzleB32:
case IR::Opcode::DsWriteAddtidB32:
case IR::Opcode::DsReadAddtidB32: return true;
default: return false;
}
}
bool ProgramNeedsFunctionLds(const IR::Program& program) {
if (program.stage == ShaderType::Compute) {
if (program.stage == ShaderType::Compute || program.values == nullptr) {
return false;
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (IsLdsOpcode(inst.op)) {
return true;
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case IR::ValueOpcode::SwizzleU32: return true;
case IR::ValueOpcode::LoadSharedU8:
case IR::ValueOpcode::LoadSharedU16:
case IR::ValueOpcode::LoadSharedU32:
case IR::ValueOpcode::WriteSharedU8:
case IR::ValueOpcode::WriteSharedU16:
case IR::ValueOpcode::WriteSharedU32:
case IR::ValueOpcode::SharedAtomicFMin32:
case IR::ValueOpcode::SharedAtomicFMax32:
case IR::ValueOpcode::SharedAtomicSwap32:
case IR::ValueOpcode::SharedAtomicIAdd32:
case IR::ValueOpcode::SharedAtomicISub32:
case IR::ValueOpcode::SharedAtomicSMin32:
case IR::ValueOpcode::SharedAtomicUMin32:
case IR::ValueOpcode::SharedAtomicSMax32:
case IR::ValueOpcode::SharedAtomicUMax32:
case IR::ValueOpcode::SharedAtomicAnd32:
case IR::ValueOpcode::SharedAtomicOr32:
case IR::ValueOpcode::SharedAtomicXor32:
case IR::ValueOpcode::DataAppend:
case IR::ValueOpcode::DataConsume: {
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index < program.values->memory_info.size() &&
program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
return true;
}
break;
}
default: break;
}
}
}
@@ -332,13 +186,18 @@ bool ProgramNeedsFunctionLds(const IR::Program& program) {
}
bool ProgramNeedsPixelValidMask(const IR::Program& program) {
if (program.stage != ShaderType::Pixel) {
if (program.stage != ShaderType::Pixel || program.values == nullptr) {
return false;
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::Export && inst.export_info.vm) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != IR::ValueOpcode::SetAttribute) {
continue;
}
const auto index = inst.Flags<IR::ExportFlags>().index;
if (index < program.values->export_info.size() &&
program.values->export_info[index].vm) {
return true;
}
}
@@ -346,20 +205,14 @@ bool ProgramNeedsPixelValidMask(const IR::Program& program) {
return false;
}
bool InstructionHasDppSource(const IR::Instruction& inst) {
for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) {
if (inst.src[i].dpp) {
return true;
}
}
return false;
}
bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
if (program.values == nullptr) {
return false;
}
if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::WqmB64) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == IR::ValueOpcode::WqmMask) {
return true;
}
}
@@ -369,13 +222,22 @@ bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
}
bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 ||
inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 ||
inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume ||
inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) {
return true;
if (program.values == nullptr) {
return false;
}
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::ReadFirstLane:
case IR::ValueOpcode::ReadLane:
case IR::ValueOpcode::Permlane16U32:
case IR::ValueOpcode::DataAppend:
case IR::ValueOpcode::DataConsume:
case IR::ValueOpcode::GdsDataAppend:
case IR::ValueOpcode::GdsDataConsume:
case IR::ValueOpcode::SwizzleU32: return true;
default: break;
}
}
}
@@ -383,25 +245,24 @@ bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
}
bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 ||
inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend ||
inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 ||
inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export ||
InstructionHasDppSource(inst)) {
return true;
}
if (IR::IsCompareOpcode(inst.op)) {
return true;
}
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Vector) {
return true;
}
if (inst.dst2.kind == IR::OperandKind::Register &&
inst.dst2.reg.file == IR::RegisterFile::Vector) {
return true;
if (program.values == nullptr) {
return false;
}
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WqmMask:
case IR::ValueOpcode::LaneId:
case IR::ValueOpcode::WriteLane:
case IR::ValueOpcode::Permlane16U32:
case IR::ValueOpcode::DataAppend:
case IR::ValueOpcode::DataConsume:
case IR::ValueOpcode::GdsDataAppend:
case IR::ValueOpcode::GdsDataConsume:
case IR::ValueOpcode::SwizzleU32: return true;
default: break;
}
}
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp
+52
View File
@@ -0,0 +1,52 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <algorithm>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
if (reg.file != IR::RegisterFile::Scalar) {
return false;
}
const auto& registers = state.program.bindings.user_data_registers;
const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index);
if (found == registers.end() || *found != reg.index) {
return false;
}
dword_index = static_cast<uint32_t>(found - registers.begin());
return true;
}
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
if (state.push_constant_variable != 0) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index)});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
if (state.vsharp_storage_variable != 0) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
state.vsharp_storage_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index)});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
return ConstantU32(state, 0);
}
uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
if (reg.file != IR::RegisterFile::Exec) {
return 0;
}
if (state.per_invocation_masks) {
return reg.index == 0 ? 1u : 0u;
}
return reg.index == 0 || state.wave_size == 64u ? 0xffffffffu : 0u;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/emitter/spirvEmitterExport.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
src/graphics/shader/recompiler/emitter/spirvEmitterFlowOps.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
src/graphics/shader/recompiler/emitter/spirvEmitterImageOps.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h → src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+66 -62
View File
@@ -4,10 +4,11 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
#include "graphics/shader/recompiler/ir/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include <algorithm>
#include <array>
@@ -17,6 +18,8 @@
#include <cstring>
#include <iterator>
#include <map>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
@@ -129,6 +132,7 @@ enum : uint32_t {
OpTypeFunction = 33,
OpConstant = 43,
OpConstantComposite = 44,
OpUndef = 1,
OpFunction = 54,
OpFunctionEnd = 56,
OpVariable = 59,
@@ -142,6 +146,7 @@ enum : uint32_t {
OpVectorShuffle = 79,
OpCompositeConstruct = 80,
OpCompositeExtract = 81,
OpCopyObject = 83,
OpSampledImage = 86,
OpImageSampleImplicitLod = 87,
OpImageSampleExplicitLod = 88,
@@ -153,6 +158,7 @@ enum : uint32_t {
OpImageWrite = 99,
OpImageQuerySizeLod = 103,
OpImageQueryLod = 105,
OpImageQuerySize = 104,
OpImageQueryLevels = 106,
OpConvertFToU = 109,
OpConvertFToS = 110,
@@ -339,6 +345,11 @@ struct EmitterState {
uint32_t vec3_float_type = 0;
uint32_t vec4_float_type = 0;
uint32_t ptr_func_uint = 0;
uint32_t ptr_func_bool = 0;
uint32_t ptr_func_float = 0;
uint32_t ptr_func_uint_pair = 0;
uint32_t ptr_func_vec4_uint = 0;
uint32_t ptr_func_vec2_float = 0;
uint32_t ptr_input_float = 0;
uint32_t ptr_input_bool = 0;
uint32_t ptr_input_int = 0;
@@ -428,6 +439,37 @@ struct EmitterState {
std::map<uint32_t, uint32_t> float_constants;
};
struct ValueEmitContext {
ValueEmitContext(EmitterState& state_, const IR::ValueProgram& program_)
: state(state_), program(program_) {}
uint32_t Def(IR::Value value);
uint32_t Arg(const IR::Inst& inst, size_t index);
uint32_t Result(const IR::Inst& inst);
uint32_t TypeId(IR::Type type) const;
uint32_t Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
std::initializer_list<uint32_t> args);
uint32_t Define(const IR::Inst& inst, uint32_t value);
uint32_t ResourceIndex(IR::Value value, IR::ValueOpcode opcode);
const IR::Inst* ImageAddress(IR::Value value);
const IR::MemoryInfo& Memory(const IR::Inst& inst) const;
const IR::ExportInfo& Export(const IR::Inst& inst) const;
uint32_t Label(const IR::Block* block) const;
size_t BlockIndex(const IR::Block* block) const;
void Fail(const IR::Inst& inst, const char* reason);
EmitterState& state;
const IR::ValueProgram& program;
std::unordered_map<const IR::Inst*, uint32_t> definitions;
std::unordered_map<const IR::Block*, uint32_t> labels;
std::unordered_map<const IR::Inst*, uint32_t> phi_variables;
std::unordered_map<const IR::Inst*, uint32_t> dispatcher_variables;
const IR::Block* current_block = nullptr;
uint32_t scratch_u32_variable = 0;
bool failed = false;
std::string error;
};
enum class VertexInputScalarKind { Float, Sint, Uint };
constexpr uint32_t NoImageComponent = 0xffffffffu;
@@ -614,22 +656,6 @@ bool IsMaskRegisterFile(IR::RegisterFile file);
bool IsSccOperand(const IR::Operand& operand);
void CollectMaskStateRegisters(std::vector<RegisterBinding>& registers);
void CollectSequentialRegisters(std::vector<RegisterBinding>& registers, const IR::Operand& base,
uint32_t count);
uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& registers);
void CollectMoveRelSourceRegisters(const IR::Program& program,
std::vector<RegisterBinding>& registers);
bool IsPairDwordOpcode(IR::Opcode op);
uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count);
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>& registers);
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
@@ -640,14 +666,10 @@ bool ProgramNeedsComputeDerivatives(const IR::Program& program);
bool ProgramNeedsImageGatherExtended(const IR::Program& program);
bool IsLdsOpcode(IR::Opcode op);
bool ProgramNeedsFunctionLds(const IR::Program& program);
bool ProgramNeedsPixelValidMask(const IR::Program& program);
bool InstructionHasDppSource(const IR::Instruction& inst);
bool ProgramNeedsSubgroupBallot(const IR::Program& program);
bool ProgramNeedsSubgroupShuffle(const IR::Program& program);
@@ -785,6 +807,11 @@ uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location);
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location);
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
uint32_t component);
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component);
uint32_t EmitLocalInvocationIndex(EmitterState& state);
@@ -887,6 +914,8 @@ uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t tr
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index);
uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src,
uint32_t src_count);
@@ -1511,49 +1540,17 @@ void EmitUpdatePixelValidMask(EmitterState& state);
void EmitKillIfPixelValidMaskInactive(EmitterState& state);
void ComputeReachableBlocks(EmitterState& state, const IR::Program& program);
void AllocateBlockLabels(EmitterState& state, const IR::Program& program);
void AllocateDispatcherState(EmitterState& state, const IR::Program& program);
uint32_t BlockLabel(const EmitterState& state, uint32_t block_id);
uint32_t EmitBranchCondition(EmitterState& state, CFG::BranchCondition condition);
void EmitStructuredPrefix(EmitterState& state, const CFG::Terminator& term);
void EmitReturn(EmitterState& state);
void EmitTerminator(EmitterState& state, const CFG::Terminator& term);
uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg);
void EmitRegisterVariables(EmitterState& state);
bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitComputeInputRegisters(EmitterState& state);
bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitPixelInputRegisters(EmitterState& state);
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitInstruction(EmitterState& state, const IR::Instruction& inst);
bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t DispatcherTargetValue(EmitterState& state, uint32_t block_id);
void EmitDispatcherStoreTarget(EmitterState& state, uint32_t block_id);
void EmitDispatcherExit(EmitterState& state);
void EmitDispatcherTerminator(EmitterState& state, const CFG::Terminator& term);
void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program);
void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program);
void EmitDispatcherLoopTail(EmitterState& state);
void EmitDispatcherFunction(EmitterState& state, const IR::Program& program);
void EmitFunction(EmitterState& state, const IR::Program& program);
bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error);
// These templates accept local lambdas from several emitter translation units.
template <typename Fn>
@@ -1574,7 +1571,8 @@ void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
}
template <typename Fn>
uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
uint32_t default_value, Fn&& fn) {
if (condition == 0) {
return fn();
}
@@ -1595,10 +1593,16 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
state.builder.AddFunction({OpLabel, merge_label});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpPhi, state.uint_type, value, then_value, then_exit, ConstantU32(state, 0), else_label});
{OpPhi, type, value, then_value, then_exit, default_value, else_label});
return value;
}
template <typename Fn>
uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
return EmitValueOrDefaultIfCondition(state, condition, state.uint_type, ConstantU32(state, 0),
std::forward<Fn>(fn));
}
template <typename Fn>
void EmitGuardedByExec(EmitterState& state, Fn&& fn) {
const auto then_label = state.builder.AllocateId();
src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+17 -2
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -80,6 +80,21 @@ uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const I
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shifted, part, ConstantU32(state, offset)});
}
if (dst.sdwa_dst_unused == 0u) {
return shifted;
}
if (dst.sdwa_dst_unused == 1u) {
const auto extended = state.builder.AllocateId();
const auto result = offset != 0u ? state.builder.AllocateId() : extended;
state.builder.AddFunction({OpBitFieldSExtract, state.uint_type, extended, value,
ConstantU32(state, 0), ConstantU32(state, width)});
if (offset != 0u) {
state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, result, extended,
ConstantU32(state, offset)});
}
return result;
}
EXIT_IF(dst.sdwa_dst_unused != 2u);
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, kept, old_value, ConstantU32(state, ~field_mask)});
@@ -367,7 +382,7 @@ uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst,
}
if (mem.kind == IR::ResourceKind::Flat ||
((mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) &&
state.program.info.addresses[mem.resource].source == IR::ScalarProvenance::Unknown)) {
state.program.info.addresses[mem.resource].unbased)) {
return EmitFlatVirtualAddress(state, inst, first_src, src_count);
}
if (mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) {
src/graphics/shader/recompiler/emitter/spirvEmitterModule.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+16 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -413,6 +413,11 @@ void EmitHeaderAndTypes(EmitterState& state) {
state.vec3_float_type = state.builder.AllocateId();
state.vec4_float_type = state.builder.AllocateId();
state.ptr_func_uint = state.builder.AllocateId();
state.ptr_func_bool = state.builder.AllocateId();
state.ptr_func_float = state.builder.AllocateId();
state.ptr_func_uint_pair = state.builder.AllocateId();
state.ptr_func_vec4_uint = state.builder.AllocateId();
state.ptr_func_vec2_float = state.builder.AllocateId();
state.ptr_input_float = state.builder.AllocateId();
state.ptr_input_bool = state.builder.AllocateId();
state.ptr_input_int = state.builder.AllocateId();
@@ -568,6 +573,16 @@ void EmitHeaderAndTypes(EmitterState& state) {
state.builder.AddType({OpTypeVector, state.vec4_float_type, state.float_type, 4});
state.builder.AddType(
{OpTypePointer, state.ptr_func_uint, StorageClassFunction, state.uint_type});
state.builder.AddType(
{OpTypePointer, state.ptr_func_bool, StorageClassFunction, state.bool_type});
state.builder.AddType(
{OpTypePointer, state.ptr_func_float, StorageClassFunction, state.float_type});
state.builder.AddType(
{OpTypePointer, state.ptr_func_uint_pair, StorageClassFunction, state.uint_pair_type});
state.builder.AddType(
{OpTypePointer, state.ptr_func_vec4_uint, StorageClassFunction, state.vec4_uint_type});
state.builder.AddType(
{OpTypePointer, state.ptr_func_vec2_float, StorageClassFunction, state.vec2_float_type});
state.builder.AddType(
{OpTypePointer, state.ptr_input_float, StorageClassInput, state.float_type});
state.builder.AddType(
src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp
+541
View File
@@ -0,0 +1,541 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <algorithm>
#include <bit>
#include <fmt/format.h>
#include <functional>
#include <unordered_set>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t BoolConstant(EmitterState& state, bool value) {
const auto id = state.builder.AllocateId();
state.builder.AddType({value ? 41u : 42u, state.bool_type, id});
return id;
}
uint32_t ConstantPair(EmitterState& state, uint64_t value) {
const auto id = state.builder.AllocateId();
state.builder.AddType({OpConstantComposite, state.uint_pair_type, id,
ConstantU32(state, static_cast<uint32_t>(value)),
ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
return id;
}
uint32_t PhiPointerType(const EmitterState& state, IR::Type type) {
switch (type) {
case IR::Type::U1: return state.ptr_func_bool;
case IR::Type::U8:
case IR::Type::U16:
case IR::Type::U32:
case IR::Type::F16: return state.ptr_func_uint;
case IR::Type::F32: return state.ptr_func_float;
case IR::Type::U64:
case IR::Type::U32x2: return state.ptr_func_uint_pair;
case IR::Type::U32x4: return state.ptr_func_vec4_uint;
case IR::Type::F32x2: return state.ptr_func_vec2_float;
default: return 0;
}
}
void StorePhiEdge(ValueEmitContext& ctx, const IR::Block* from, const IR::Block* to) {
if (to == nullptr) {
return;
}
for (const auto& phi: *to) {
if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
continue;
}
for (size_t index = 0; index < phi.NumArgs(); index++) {
if (phi.PhiBlock(index) == from) {
ctx.state.builder.AddFunction(
{OpStore, ctx.phi_variables.at(&phi), ctx.Def(phi.Arg(index))});
break;
}
}
}
}
const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) {
const auto found = std::ranges::find_if(
program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; });
if (found == program.block_info.end()) {
return nullptr;
}
return program.blocks[static_cast<size_t>(found - program.block_info.begin())];
}
void EmitReturn(ValueEmitContext& ctx) {
EmitKillIfPixelValidMaskInactive(ctx.state);
ctx.state.builder.AddFunction({OpReturn});
}
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
const IR::ValueBlockInfo& info) {
const auto& term = info.terminator;
const auto emit_merge = [&]() {
if (term.loop_header) {
const auto* merge = TargetBlock(ctx.program, term.merge_block);
const auto* cont = TargetBlock(ctx.program, term.continue_block);
if (merge != nullptr && cont != nullptr) {
ctx.state.builder.AddFunction(
{OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone});
}
} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
term.merge_block != UINT32_MAX) {
if (const auto* merge = TargetBlock(ctx.program, term.merge_block); merge != nullptr) {
ctx.state.builder.AddFunction(
{OpSelectionMerge, ctx.Label(merge), SelectionControlNone});
}
}
};
switch (term.kind) {
case CFG::TerminatorKind::Branch: {
const auto* target = TargetBlock(ctx.program, term.true_block);
if (target == nullptr) {
EmitReturn(ctx);
return;
}
StorePhiEdge(ctx, block, target);
emit_merge();
ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
return;
}
case CFG::TerminatorKind::ConditionalBranch: {
const auto* true_block = TargetBlock(ctx.program, term.true_block);
const auto* false_block = TargetBlock(ctx.program, term.false_block);
if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) {
EmitReturn(ctx);
return;
}
StorePhiEdge(ctx, block, true_block);
StorePhiEdge(ctx, block, false_block);
const auto condition = ctx.Def(info.condition);
emit_merge();
ctx.state.builder.AddFunction(
{OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
return;
}
default: EmitReturn(ctx); return;
}
}
void EmitDispatcherTarget(ValueEmitContext& ctx, const IR::Block* from, uint32_t target) {
const auto* block = TargetBlock(ctx.program, target);
if (block != nullptr) {
StorePhiEdge(ctx, from, block);
}
}
void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
const IR::ValueBlockInfo& info, uint32_t exit_label) {
const auto& term = info.terminator;
switch (term.kind) {
case CFG::TerminatorKind::Branch:
EmitDispatcherTarget(ctx, block, term.true_block);
ctx.state.builder.AddFunction(
{OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, term.true_block)});
break;
case CFG::TerminatorKind::ConditionalBranch: {
EmitDispatcherTarget(ctx, block, term.true_block);
EmitDispatcherTarget(ctx, block, term.false_block);
const auto selected = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, selected,
ctx.Def(info.condition),
ConstantU32(ctx.state, term.true_block),
ConstantU32(ctx.state, term.false_block)});
ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
break;
}
case CFG::TerminatorKind::IndirectBranch: {
for (const auto target: term.indirect_targets) {
EmitDispatcherTarget(ctx, block, target);
}
for (const auto target: term.indirect_selector_targets) {
EmitDispatcherTarget(ctx, block, target);
}
uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
if (!info.indirect_target.IsEmpty()) {
const auto selector = ctx.Def(info.indirect_target);
const auto& values = term.indirect_selector_code != UINT32_MAX
? term.indirect_selector_values
: term.indirect_target_pcs;
const auto& targets = term.indirect_selector_code != UINT32_MAX
? term.indirect_selector_targets
: term.indirect_targets;
for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
const auto match = ctx.state.builder.AllocateId();
const auto next = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpIEqual, ctx.state.bool_type, match, selector,
ConstantU32(ctx.state, values[index])});
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, next, match,
ConstantU32(ctx.state, targets[index]),
selected});
selected = next;
}
}
ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
break;
}
default:
ctx.state.builder.AddFunction(
{OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
break;
}
ctx.state.builder.AddFunction({OpBranch, exit_label});
}
bool EmitInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto finish = [&]() {
if (const auto found = ctx.dispatcher_variables.find(&inst);
found != ctx.dispatcher_variables.end()) {
ctx.state.builder.AddFunction(
{OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
}
return true;
};
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
ctx.state.builder.AddFunction(
{OpLoad, ctx.TypeId(inst.GetType()), ctx.Result(inst), ctx.phi_variables.at(&inst)});
return finish();
}
if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
EmitValueImage(ctx, inst)) {
return finish();
}
ctx.Fail(inst, "has no direct SPIR-V emitter");
return false;
}
void EmitBlock(ValueEmitContext& ctx, const IR::Block* block) {
ctx.current_block = block;
ctx.state.builder.AddFunction({OpLabel, ctx.Label(block)});
for (const auto& inst: *block) {
if (!EmitInstruction(ctx, inst)) {
return;
}
}
}
void EmitStructuredFunction(ValueEmitContext& ctx) {
ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
EmitBlock(ctx, ctx.program.blocks[index]);
EmitStructuredTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index]);
}
}
void EmitDispatcherFunction(ValueEmitContext& ctx) {
auto& state = ctx.state;
const auto* entry = ctx.program.blocks.front();
state.builder.AddFunction({OpBranch, ctx.Label(entry)});
EmitBlock(ctx, entry);
EmitDispatcherTerminator(ctx, entry, ctx.program.block_info.front(),
state.dispatch_header_label);
state.builder.AddFunction({OpLabel, state.dispatch_header_label});
const auto pc = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, pc, state.dispatch_pc_variable});
const auto done = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, state.bool_type, done, pc, ConstantU32(ctx.state, UINT32_MAX)});
state.builder.AddFunction(
{OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone});
state.builder.AddFunction(
{OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
state.builder.AddFunction({OpLabel, state.dispatch_select_label});
state.builder.AddFunction(
{OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
std::vector<uint32_t> words {OpSwitch, pc, state.dispatch_default_label};
for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
words.push_back(ctx.program.block_info[index].id);
words.push_back(ctx.Label(ctx.program.blocks[index]));
}
state.builder.AddFunction(words);
state.builder.AddFunction({OpLabel, state.dispatch_default_label});
state.builder.AddFunction(
{OpStore, state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) {
EmitBlock(ctx, ctx.program.blocks[index]);
EmitDispatcherTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index],
state.dispatch_after_switch_label);
}
state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
state.builder.AddFunction({OpBranch, state.dispatch_header_label});
state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
EmitReturn(ctx);
}
} // namespace
uint32_t ValueEmitContext::TypeId(IR::Type type) const {
switch (type) {
case IR::Type::U1: return state.bool_type;
case IR::Type::U8:
case IR::Type::U16:
case IR::Type::U32:
case IR::Type::F16: return state.uint_type;
case IR::Type::U64:
case IR::Type::U32x2: return state.uint_pair_type;
case IR::Type::F32: return state.float_type;
case IR::Type::U32x4: return state.vec4_uint_type;
case IR::Type::F32x2: return state.vec2_float_type;
default: return 0;
}
}
uint32_t ValueEmitContext::Def(IR::Value value) {
value = value.Resolve();
if (value.IsImmediate()) {
switch (value.GetType()) {
case IR::Type::U1: return BoolConstant(state, value.U1());
case IR::Type::U8: return ConstantU32(state, value.U8());
case IR::Type::U16: return ConstantU32(state, value.U16());
case IR::Type::U32: return ConstantU32(state, value.U32());
case IR::Type::U64: return ConstantPair(state, value.U64());
case IR::Type::F16: return ConstantU32(state, value.F16Bits());
case IR::Type::F32:
return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
default: break;
}
}
const auto* inst = value.ResolveInstruction();
if (inst == nullptr) {
failed = true;
error = "direct SPIR-V emitter received a non-value argument";
return ConstantU32(state, 0);
}
if (program.dispatcher_fallback && current_block != nullptr &&
inst->Parent() != current_block) {
if (const auto found = dispatcher_variables.find(inst);
found != dispatcher_variables.end()) {
const auto id = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
return id;
}
}
if (const auto found = definitions.find(inst); found != definitions.end()) {
return found->second;
}
const auto id = state.builder.AllocateId();
definitions.emplace(inst, id);
return id;
}
uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
return Def(inst.Arg(index));
}
uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
if (const auto found = definitions.find(&inst); found != definitions.end()) {
return found->second;
}
const auto id = state.builder.AllocateId();
definitions.emplace(&inst, id);
return id;
}
uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
std::initializer_list<uint32_t> args) {
std::vector<uint32_t> words {opcode, TypeId(type), Result(inst)};
words.insert(words.end(), args.begin(), args.end());
state.builder.AddFunction(words);
return Result(inst);
}
uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
if (const auto found = definitions.find(&inst); found != definitions.end()) {
if (found->second != value) {
state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value});
}
return found->second;
}
definitions.emplace(&inst, value);
return value;
}
uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) {
const auto* inst = value.ResolveInstruction();
if (inst == nullptr || inst->GetOpcode() != opcode) {
failed = true;
error = "typed resource handle has the wrong producer";
return 0;
}
return inst->Flags<uint32_t>();
}
const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) {
const auto* inst = value.ResolveInstruction();
if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) {
failed = true;
error = "typed image address was not constructed by MakeImageAddress";
return nullptr;
}
return inst;
}
const IR::MemoryInfo& ValueEmitContext::Memory(const IR::Inst& inst) const {
return program.memory_info.at(inst.Flags<IR::MemoryFlags>().index);
}
const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const {
return program.export_info.at(inst.Flags<IR::ExportFlags>().index);
}
uint32_t ValueEmitContext::Label(const IR::Block* block) const {
return labels.at(block);
}
size_t ValueEmitContext::BlockIndex(const IR::Block* block) const {
return static_cast<size_t>(std::ranges::find(program.blocks, block) - program.blocks.begin());
}
void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
failed = true;
error = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
}
bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) {
ValueEmitContext ctx(state, program);
for (const auto* block: program.blocks) {
ctx.labels.emplace(block, state.builder.AllocateId());
for (const auto& inst: *block) {
if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
continue;
}
const auto ptr_type = PhiPointerType(state, inst.GetType());
if (ptr_type == 0) {
ctx.failed = true;
ctx.error = fmt::format("typed opcode Phi of type {} cannot be spilled across a "
"control-flow edge",
IR::TypeName(inst.GetType()));
break;
}
ctx.phi_variables.emplace(&inst, state.builder.AllocateId());
}
}
if (ctx.failed) {
SetError(error, ctx.error.c_str());
return false;
}
if (program.dispatcher_fallback) {
const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
value = value.Resolve();
const auto* definition = value.TryInstruction();
if (definition == nullptr || definition->Parent() == consumer ||
PhiPointerType(state, definition->GetType()) == 0) {
return;
}
if (!ctx.dispatcher_variables.contains(definition)) {
ctx.dispatcher_variables.emplace(definition, state.builder.AllocateId());
}
};
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
for (size_t index = 0; index < inst.NumArgs(); index++) {
const auto* consumer =
inst.GetOpcode() == IR::ValueOpcode::Phi ? inst.PhiBlock(index) : block;
mark_cross_block(inst.Arg(index), consumer);
}
}
}
for (size_t index = 0; index < program.blocks.size(); index++) {
mark_cross_block(program.block_info[index].condition, program.blocks[index]);
mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
}
state.dispatcher_fallback = true;
state.dispatch_pc_variable = state.builder.AllocateId();
state.dispatch_header_label = state.builder.AllocateId();
state.dispatch_select_label = state.builder.AllocateId();
state.dispatch_default_label = state.builder.AllocateId();
state.dispatch_after_switch_label = state.builder.AllocateId();
state.dispatch_continue_label = state.builder.AllocateId();
state.dispatch_merge_label = state.builder.AllocateId();
}
for (const auto* block: program.blocks) {
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
})) {
ctx.scratch_u32_variable = state.builder.AllocateId();
break;
}
}
state.builder.AddFunction(
{OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
state.builder.AddFunction({OpLabel, state.entry_label});
if (state.needs_function_lds) {
state.builder.AddFunction(
{OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
}
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable,
StorageClassFunction});
}
if (program.dispatcher_fallback) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
}
for (const auto& binding: state.registers) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
}
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
continue;
}
state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
ctx.phi_variables.at(&inst), StorageClassFunction});
}
}
if (program.dispatcher_fallback) {
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (const auto found = ctx.dispatcher_variables.find(&inst);
found != ctx.dispatcher_variables.end()) {
state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
found->second, StorageClassFunction});
}
}
}
}
if (ctx.scratch_u32_variable != 0) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, ctx.scratch_u32_variable, StorageClassFunction});
}
for (const auto& binding: state.registers) {
state.builder.AddFunction({OpStore, binding.pointer_id,
ConstantU32(state, InitialRegisterValue(state, binding.reg))});
}
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction(
{OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
}
EmitStorageBufferOffsets(state);
if (program.blocks.empty()) {
EmitReturn(ctx);
} else if (program.dispatcher_fallback) {
EmitDispatcherFunction(ctx);
} else {
EmitStructuredFunction(ctx);
}
state.builder.AddFunction({OpFunctionEnd});
if (ctx.failed) {
SetError(error, ctx.error.c_str());
return false;
}
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAlu.cpp
+677
View File
@@ -0,0 +1,677 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <array>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
struct Pair {
uint32_t low = 0;
uint32_t high = 0;
};
uint32_t NewUnary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, value});
return result;
}
uint32_t NewBinary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
uint32_t rhs) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, lhs, rhs});
return result;
}
uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
return result;
}
Pair ExtractPair(EmitterState& state, uint32_t value) {
Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.low, value, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.high, value, 1});
return result;
}
uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.uint_pair_type, result, low, high});
return result;
}
uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
uint32_t high_compare, uint32_t low_compare, bool equal_compare = false,
bool not_equal = false) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto high_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.high, rhs.high);
const auto low_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.low, rhs.low);
if (equal_compare) {
return NewBinary(
state, not_equal ? OpLogicalOr : OpLogicalAnd, state.bool_type,
not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, high_equal) : high_equal,
not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, low_equal) : low_equal);
}
const auto high_result = NewBinary(state, high_compare, state.bool_type, lhs.high, rhs.high);
const auto low_result = NewBinary(state, low_compare, state.bool_type, lhs.low, rhs.low);
const auto low_path = NewBinary(state, OpLogicalAnd, state.bool_type, high_equal, low_result);
return NewBinary(state, OpLogicalOr, state.bool_type, high_result, low_path);
}
uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0));
const auto high0 = NewBinary(state, OpIAdd, state.uint_type, lhs.high, rhs.high);
const auto high = NewBinary(state, OpIAdd, state.uint_type, high0, carry.carry);
return MakePair(state, carry.sum, high);
}
uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = NewBinary(state, OpISub, state.uint_type, lhs.low, rhs.low);
const auto borrow = NewBinary(state, OpULessThan, state.bool_type, lhs.low, rhs.low);
const auto borrow_u32 =
NewSelect(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = NewBinary(state, OpISub, state.uint_type, lhs.high, rhs.high);
const auto high = NewBinary(state, OpISub, state.uint_type, high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
const auto operand_type = signed_value ? state.int_type : state.uint_type;
const auto pair_type = signed_value ? state.int_pair_type : state.uint_pair_type;
uint32_t lhs_operand = lhs;
uint32_t rhs_operand = rhs;
if (signed_value) {
lhs_operand = NewUnary(state, OpBitcast, state.int_type, lhs);
rhs_operand = NewUnary(state, OpBitcast, state.int_type, rhs);
}
const auto extended = state.builder.AllocateId();
state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended,
lhs_operand, rhs_operand});
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1});
return signed_value ? NewUnary(state, OpBitcast, state.uint_type, high) : high;
}
uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.high);
const auto high2 = NewBinary(state, OpIMul, state.uint_type, lhs.high, rhs.low);
return MakePair(state, low,
NewBinary(state, OpIAdd, state.uint_type,
NewBinary(state, OpIAdd, state.uint_type, high0, high1), high2));
}
uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value,
uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
return MakePair(state, NewBinary(state, opcode, state.uint_type, lhs.low, rhs.low),
NewBinary(state, opcode, state.uint_type, lhs.high, rhs.high));
}
uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value);
uint32_t low = 0;
uint32_t high = 0;
if (opcode == OpShiftLeftLogical) {
EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else if (opcode == OpShiftRightLogical) {
EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else {
const auto logical = EmitShift64(state, OpShiftRightLogical, value, shift);
const auto logical_pair = ExtractPair(state, logical);
const auto sign =
NewBinary(state, OpSLessThan, state.bool_type, pair.high, ConstantU32(state, 0));
const auto inverse =
NewBinary(state, OpISub, state.uint_type, ConstantU32(state, 64), shift);
const auto fill = NewBinary(state, OpShiftLeftLogical, state.uint_type,
ConstantU32(state, 0xffffffffu), inverse);
const auto high_shift =
NewBinary(state, OpISub, state.uint_type, shift, ConstantU32(state, 32));
const auto high_fill = NewSelect(
state, state.uint_type,
NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, ConstantU32(state, 32)),
ConstantU32(state, 0xffffffffu), fill);
const auto fill_low =
NewSelect(state, state.uint_type, sign,
NewSelect(state, state.uint_type,
NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift,
ConstantU32(state, 32)),
NewBinary(state, OpShiftLeftLogical, state.uint_type,
ConstantU32(state, 0xffffffffu), high_shift),
ConstantU32(state, 0)),
ConstantU32(state, 0));
const auto fill_high =
NewSelect(state, state.uint_type, sign, high_fill, ConstantU32(state, 0));
return MakePair(
state, NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.low, fill_low),
NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.high, fill_high));
}
return MakePair(state, low, high);
}
uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
const auto pair = ExtractPair(state, value);
const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.int_type, high_i, state.glsl_std450, GlslFindUMsb, pair.high});
state.builder.AddFunction(
{OpExtInst, state.int_type, low_i, state.glsl_std450, GlslFindUMsb, pair.low});
const auto high = NewUnary(state, OpBitcast, state.uint_type, high_i);
const auto low = NewUnary(state, OpBitcast, state.uint_type, low_i);
const auto high_nonzero =
NewBinary(state, OpINotEqual, state.bool_type, pair.high, ConstantU32(state, 0));
return NewSelect(state, state.uint_type, high_nonzero,
NewBinary(state, OpIAdd, state.uint_type, high, ConstantU32(state, 32)), low);
}
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
bool max_value) {
const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
: EmitMinMaxU32Value(state, a, b, max_value);
return signed_value ? EmitMinMaxI32Value(state, ab, c, max_value)
: EmitMinMaxU32Value(state, ab, c, max_value);
}
uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
const auto ab = NewBinary(state, OpIAdd, state.uint_type, a, b);
const auto abc = NewBinary(state, OpIAdd, state.uint_type, ab, c);
return NewBinary(state, OpISub, state.uint_type,
NewBinary(state, OpISub, state.uint_type, abc, minimum), maximum);
}
uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
}
uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
const auto min_ab = EmitMinMaxF32Value(state, a, b, false);
const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false);
const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
const auto nan_ab = NewBinary(state, OpLogicalOr, state.bool_type,
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
const auto any_nan =
NewBinary(state, OpLogicalOr, state.bool_type, nan_ab, EmitClassifyF32(state, c).nan);
return NewSelect(state, state.float_type, any_nan, min3, median);
}
uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
std::initializer_list<uint32_t> args) {
const auto result = state.builder.AllocateId();
std::vector<uint32_t> words {OpExtInst, type, result, state.glsl_std450, opcode};
words.insert(words.end(), args.begin(), args.end());
state.builder.AddFunction(words);
return result;
}
uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
const auto trunc = EmitTruncF32Value(state, src);
const auto converted_raw = state.builder.AllocateId();
if (signed_value) {
const auto converted_i = state.builder.AllocateId();
state.builder.AddFunction({OpConvertFToS, state.int_type, converted_i, trunc});
state.builder.AddFunction({OpBitcast, state.uint_type, converted_raw, converted_i});
} else {
state.builder.AddFunction({OpConvertFToU, state.uint_type, converted_raw, trunc});
}
const auto nan = EmitClassifyF32(state, src).nan;
if (signed_value) {
const auto below = NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src,
ConstantF32(state, 0xcf000000u));
const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
ConstantF32(state, 0x4f000000u));
const auto high = NewSelect(state, state.uint_type, above, ConstantU32(state, 0x7fffffffu),
converted_raw);
const auto low =
NewSelect(state, state.uint_type, below, ConstantU32(state, 0x80000000u), high);
return NewSelect(state, state.uint_type, nan, ConstantU32(state, 0), low);
}
const auto below =
NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, ConstantF32(state, 0));
const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
ConstantF32(state, 0x4f800000u));
const auto zero = NewBinary(state, OpLogicalOr, state.bool_type, nan, below);
const auto high =
NewSelect(state, state.uint_type, above, ConstantU32(state, 0xffffffffu), converted_raw);
return NewSelect(state, state.uint_type, zero, ConstantU32(state, 0), high);
}
uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
return EmitExt(state, state.uint_type, GlslPackHalf2x16, {src});
}
uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) {
const auto pair = EmitExt(state, state.vec2_float_type, GlslUnpackHalf2x16, {bits});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.float_type, result, pair, 0});
return result;
}
} // namespace
bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto op = inst.GetOpcode();
const auto unary = [&](uint32_t spirv, IR::Type type) {
ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0)});
return true;
};
const auto binary = [&](uint32_t spirv, IR::Type type) {
ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
};
const auto ext_unary = [&](uint32_t ext) {
ctx.Define(inst, EmitExt(state, state.float_type, ext, {ctx.Arg(inst, 0)}));
return true;
};
switch (op) {
case IR::ValueOpcode::BitCastU16F16:
case IR::ValueOpcode::BitCastF16U16:
case IR::ValueOpcode::ConvertU32U16:
case IR::ValueOpcode::ConvertU32U8:
case IR::ValueOpcode::PackUint2x32:
case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
case IR::ValueOpcode::ConvertU16U32:
ctx.Emit(inst, OpBitwiseAnd, IR::Type::U16,
{ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)});
return true;
case IR::ValueOpcode::ConvertU8U32:
ctx.Emit(inst, OpBitwiseAnd, IR::Type::U8,
{ctx.Arg(inst, 0), ConstantU32(state, 0xffu)});
return true;
case IR::ValueOpcode::ConvertF16F32: {
const auto pair = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair,
ctx.Arg(inst, 0), ConstantF32(state, 0)});
ctx.Define(inst, EmitPackHalf(state, pair));
return true;
}
case IR::ValueOpcode::ConvertF32F16:
ctx.Define(inst, EmitUnpackHalf(state, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::ConvertS32F32:
ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), true));
return true;
case IR::ValueOpcode::ConvertU32F32:
ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
return true;
case IR::ValueOpcode::ConvertF32S32: {
const auto signed_value = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0));
ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value});
return true;
}
case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
case IR::ValueOpcode::CompositeConstructU32x2:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
case IR::ValueOpcode::CompositeConstructF32x2:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
case IR::ValueOpcode::CompositeConstructU32x4:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
return true;
case IR::ValueOpcode::CompositeExtractU32x2:
case IR::ValueOpcode::CompositeExtractU32x4:
ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
{ctx.Arg(inst, 0), inst.Arg(1).U32()});
return true;
case IR::ValueOpcode::PackHalf2x16:
ctx.Define(inst, EmitExt(state, state.uint_type, GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
return true;
case IR::ValueOpcode::PackSnorm2x16:
ctx.Define(inst,
EmitExt(state, state.uint_type, GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
return true;
case IR::ValueOpcode::PackUnorm2x16:
ctx.Define(inst,
EmitExt(state, state.uint_type, GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
return true;
case IR::ValueOpcode::PackFloat2x16Rtz: {
const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
const auto high =
NewBinary(state, OpShiftLeftLogical, state.uint_type,
EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
ctx.Define(inst, NewBinary(state, OpBitwiseOr, state.uint_type, low, high));
return true;
}
case IR::ValueOpcode::FPAbs32:
ctx.Define(inst, EmitFAbsValue(state, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::FPNeg32:
ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::FPSaturate32:
ctx.Define(inst, EmitExt(state, state.float_type, GlslFClamp,
{ctx.Arg(inst, 0), ConstantF32(state, 0),
ConstantF32(state, 0x3f800000u)}));
return true;
case IR::ValueOpcode::BitFieldInsert:
ctx.Emit(inst, OpBitFieldInsert, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
return true;
case IR::ValueOpcode::BitFieldUExtract:
ctx.Emit(inst, OpBitFieldUExtract, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true;
case IR::ValueOpcode::BitFieldSExtract:
ctx.Emit(inst, OpBitFieldSExtract, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true;
case IR::ValueOpcode::SelectU1:
ctx.Emit(inst, OpSelect, IR::Type::U1,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true;
case IR::ValueOpcode::SelectU32:
ctx.Emit(inst, OpSelect, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true;
case IR::ValueOpcode::SelectF32:
ctx.Emit(inst, OpSelect, IR::Type::F32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true;
case IR::ValueOpcode::IAdd32: return binary(OpIAdd, IR::Type::U32);
case IR::ValueOpcode::ISub32: return binary(OpISub, IR::Type::U32);
case IR::ValueOpcode::IMul32: return binary(OpIMul, IR::Type::U32);
case IR::ValueOpcode::UDiv32: return binary(OpUDiv, IR::Type::U32);
case IR::ValueOpcode::IAdd64:
ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::ISub64:
ctx.Define(inst, EmitSub64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::IMul64:
ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::IAddCarry32:
ctx.Emit(inst, OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
case IR::ValueOpcode::SMulHi:
ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
return true;
case IR::ValueOpcode::UMulHi:
ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
return true;
case IR::ValueOpcode::IAbs32: {
const auto value = ctx.Arg(inst, 0);
const auto neg = NewUnary(state, OpSNegate, state.uint_type, value);
const auto negative =
NewBinary(state, OpSLessThan, state.bool_type, value, ConstantU32(state, 0));
ctx.Define(inst, NewSelect(state, state.uint_type, negative, neg, value));
return true;
}
case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32);
case IR::ValueOpcode::ShiftRightLogical32:
return binary(OpShiftRightLogical, IR::Type::U32);
case IR::ValueOpcode::ShiftRightArithmetic32:
return binary(OpShiftRightArithmetic, IR::Type::U32);
case IR::ValueOpcode::ShiftLeftLogical64:
ctx.Define(inst,
EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::ShiftRightLogical64:
ctx.Define(inst,
EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::ShiftRightArithmetic64:
ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
case IR::ValueOpcode::BitwiseOr32: return binary(OpBitwiseOr, IR::Type::U32);
case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
case IR::ValueOpcode::BitwiseAnd64:
ctx.Define(inst,
EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::BitwiseOr64:
ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
case IR::ValueOpcode::BitCount64: {
const auto pair = ExtractPair(state, ctx.Arg(inst, 0));
ctx.Define(inst, NewBinary(state, OpIAdd, state.uint_type,
NewUnary(state, OpBitCount, state.uint_type, pair.low),
NewUnary(state, OpBitCount, state.uint_type, pair.high)));
return true;
}
case IR::ValueOpcode::FindILsb32: {
const auto value = EmitExt(state, state.int_type, GlslFindILsb, {ctx.Arg(inst, 0)});
ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
return true;
}
case IR::ValueOpcode::FindUMsb32: {
const auto value = EmitExt(state, state.int_type, GlslFindUMsb, {ctx.Arg(inst, 0)});
ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
return true;
}
case IR::ValueOpcode::FindUMsb64:
ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::FindSMsb32: {
const auto value =
EmitExt(state, state.int_type, 74u,
{NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0))});
ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
return true;
}
case IR::ValueOpcode::SMin32:
ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
return true;
case IR::ValueOpcode::SMax32:
ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
return true;
case IR::ValueOpcode::UMin32:
ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
return true;
case IR::ValueOpcode::UMax32:
ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
return true;
case IR::ValueOpcode::SMinTri32:
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
ctx.Arg(inst, 2), true, false));
return true;
case IR::ValueOpcode::SMaxTri32:
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
ctx.Arg(inst, 2), true, true));
return true;
case IR::ValueOpcode::UMinTri32:
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
ctx.Arg(inst, 2), false, false));
return true;
case IR::ValueOpcode::UMaxTri32:
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
ctx.Arg(inst, 2), false, true));
return true;
case IR::ValueOpcode::SMedTri32:
ctx.Define(inst,
EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), true));
return true;
case IR::ValueOpcode::UMedTri32:
ctx.Define(
inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false));
return true;
case IR::ValueOpcode::SLessThan32: return binary(OpSLessThan, IR::Type::U1);
case IR::ValueOpcode::ULessThan32: return binary(OpULessThan, IR::Type::U1);
case IR::ValueOpcode::IEqual32: return binary(OpIEqual, IR::Type::U1);
case IR::ValueOpcode::SLessThanEqual32: return binary(OpSLessThanEqual, IR::Type::U1);
case IR::ValueOpcode::ULessThanEqual32: return binary(OpULessThanEqual, IR::Type::U1);
case IR::ValueOpcode::SGreaterThan32: return binary(OpSGreaterThan, IR::Type::U1);
case IR::ValueOpcode::UGreaterThan32: return binary(OpUGreaterThan, IR::Type::U1);
case IR::ValueOpcode::INotEqual32: return binary(OpINotEqual, IR::Type::U1);
case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::IEqual64:
ctx.Define(inst,
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false));
return true;
case IR::ValueOpcode::INotEqual64:
ctx.Define(inst,
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true));
return true;
case IR::ValueOpcode::ULessThan64:
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan,
OpULessThan));
return true;
case IR::ValueOpcode::SLessThan64:
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan,
OpULessThan));
return true;
case IR::ValueOpcode::UGreaterThan64:
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan,
OpUGreaterThan));
return true;
case IR::ValueOpcode::SGreaterThan64:
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan,
OpUGreaterThan));
return true;
case IR::ValueOpcode::ULessThanEqual64: {
const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
OpUGreaterThan, OpUGreaterThan);
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
return true;
}
case IR::ValueOpcode::SLessThanEqual64: {
const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
OpSGreaterThan, OpUGreaterThan);
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
return true;
}
case IR::ValueOpcode::UGreaterThanEqual64: {
const auto less =
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan);
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
return true;
}
case IR::ValueOpcode::SGreaterThanEqual64: {
const auto less =
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan);
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
return true;
}
case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);
case IR::ValueOpcode::LogicalNot: return unary(OpLogicalNot, IR::Type::U1);
case IR::ValueOpcode::FPOrdEqual32: return binary(OpFOrdEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordEqual32: return binary(OpFUnordEqual, IR::Type::U1);
case IR::ValueOpcode::FPOrdNotEqual32: return binary(OpFOrdNotEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordNotEqual32: return binary(OpFUnordNotEqual, IR::Type::U1);
case IR::ValueOpcode::FPOrdLessThan32: return binary(OpFOrdLessThan, IR::Type::U1);
case IR::ValueOpcode::FPUnordLessThan32: return binary(OpFUnordLessThan, IR::Type::U1);
case IR::ValueOpcode::FPOrdGreaterThan32: return binary(OpFOrdGreaterThan, IR::Type::U1);
case IR::ValueOpcode::FPUnordGreaterThan32:
return binary(OpFUnordGreaterThan, IR::Type::U1);
case IR::ValueOpcode::FPOrdLessThanEqual32:
return binary(OpFOrdLessThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordLessThanEqual32:
return binary(OpFUnordLessThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPOrdGreaterThanEqual32:
return binary(OpFOrdGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordGreaterThanEqual32:
return binary(OpFUnordGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPIsNan32:
ctx.Emit(inst, OpFUnordNotEqual, IR::Type::U1, {ctx.Arg(inst, 0), ctx.Arg(inst, 0)});
return true;
case IR::ValueOpcode::FPCmpClass32:
ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::FPAdd32: return binary(OpFAdd, IR::Type::F32);
case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32);
case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32);
case IR::ValueOpcode::FPFma32:
ctx.Define(inst, EmitExt(state, state.float_type, GlslFma,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
return true;
case IR::ValueOpcode::FPMin32:
ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
return true;
case IR::ValueOpcode::FPMax32:
ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
return true;
case IR::ValueOpcode::FPMinTri32:
ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
ctx.Arg(inst, 2), false));
return true;
case IR::ValueOpcode::FPMaxTri32:
ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
ctx.Arg(inst, 2), true));
return true;
case IR::ValueOpcode::FPMedTri32:
ctx.Define(inst,
EmitFMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)));
return true;
case IR::ValueOpcode::FPRecip32: {
const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
ConstantF32(state, 0x3f800000u), source));
return true;
}
case IR::ValueOpcode::FPRecipIFlag32:
// Integer-to-float inputs used by IFLAG cannot be denormal.
ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::FPRecipSqrt32:
ctx.Define(inst, EmitExt(state, state.float_type, GlslInverseSqrt,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true;
case IR::ValueOpcode::FPSqrt:
ctx.Define(inst, EmitExt(state, state.float_type, GlslSqrt,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true;
case IR::ValueOpcode::FPSin:
case IR::ValueOpcode::FPCos: {
auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
source =
NewBinary(state, OpFMul, state.float_type, source, ConstantF32(state, 0x40c90fdbu));
ctx.Define(inst, EmitExt(state, state.float_type,
op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source}));
return true;
}
case IR::ValueOpcode::FPExp2:
ctx.Define(inst, EmitExt(state, state.float_type, GlslExp2,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true;
case IR::ValueOpcode::FPLog2:
ctx.Define(inst, EmitExt(state, state.float_type, GlslLog2,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true;
case IR::ValueOpcode::FPLdexp: {
const auto exponent = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 1));
ctx.Define(inst,
EmitExt(state, state.float_type, GlslLdexp, {ctx.Arg(inst, 0), exponent}));
return true;
}
case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven);
case IR::ValueOpcode::FPFloor32: return ext_unary(GlslFloor);
case IR::ValueOpcode::FPCeil32: return ext_unary(GlslCeil);
case IR::ValueOpcode::FPTrunc32: return ext_unary(GlslTrunc);
case IR::ValueOpcode::FPFract32: return ext_unary(GlslFract);
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp
+557
View File
@@ -0,0 +1,557 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <algorithm>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value,
ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
return result;
}
void StoreRegisterMirror(ValueEmitContext& ctx, IR::Register reg, uint32_t value) {
if (const auto pointer = PointerForRegister(ctx.state, reg); pointer != 0) {
if (IsInactiveWave32ExecHigh(ctx.state, reg)) {
value = ConstantU32(ctx.state, 0);
}
ctx.state.builder.AddFunction({OpStore, pointer, value});
}
}
bool IsPrologue(const ValueEmitContext& ctx) {
return !ctx.program.blocks.empty() && ctx.current_block == ctx.program.blocks.front();
}
uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
auto& state = ctx.state;
if (kind == IR::StageInputKind::LocalInvocationIndex) {
return EmitLocalInvocationIndex(state);
}
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
}
if (kind == IR::StageInputKind::FrontFacing) {
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.bool_type, value, variable});
state.builder.AddFunction(
{OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
return bits;
}
if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) {
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.int_type, value, variable});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
return bits;
}
if (kind == IR::StageInputKind::FragCoord) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_input_float, pointer, variable,
ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
return bits;
}
return EmitInputComponentU32(state, kind, component);
}
uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
auto& state = ctx.state;
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), active});
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
const auto wqm_low = EmitWqmLaneU32(state, low);
const auto wqm_high = EmitWqmLaneU32(state, high);
const auto lane = EmitSubgroupLocalInvocationId(state);
const auto upper = state.builder.AllocateId();
const auto mask = state.builder.AllocateId();
const auto bit_lane = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, mask, upper, wqm_high, wqm_low});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bit_lane, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_lane});
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, result, hit, ConstantU32(state, 0)});
return result;
}
uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
uint32_t exec) {
auto& state = ctx.state;
const auto lane = EmitSubgroupLocalInvocationId(state);
const auto bank_shift = state.builder.AllocateId();
const auto row_shift = state.builder.AllocateId();
const auto bank = state.builder.AllocateId();
const auto row = state.builder.AllocateId();
const auto bank_bit = state.builder.AllocateId();
const auto row_bit = state.builder.AllocateId();
const auto bank_hit = state.builder.AllocateId();
const auto row_hit = state.builder.AllocateId();
const auto bank_ok = state.builder.AllocateId();
const auto row_ok = state.builder.AllocateId();
const auto masks_ok = state.builder.AllocateId();
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, bank_shift, lane, ConstantU32(state, 2)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, row_shift, lane, ConstantU32(state, 4)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, flags.row_mask), row_bit});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, bank_ok, bank_hit, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, row_ok, row_hit, ConstantU32(state, 0)});
state.builder.AddFunction({OpLogicalAnd, state.bool_type, masks_ok, bank_ok, row_ok});
uint32_t writable = masks_ok;
if (!flags.bound_control) {
const auto target = EmitDppTargetLane(state, flags.control);
const auto bounded = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalAnd, state.bool_type, bounded, writable, target.valid});
writable = bounded;
}
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalAnd, state.bool_type, result, exec, writable});
return result;
}
uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
auto& state = ctx.state;
const auto* input = InputBindingForParameter(state, attr);
if (input == nullptr || input->variable_id == 0) {
return ConstantU32(state, 0);
}
if (state.stage == ShaderType::Vertex) {
return EmitVertexParameterComponentU32(state, *input, chan & 3u);
}
const auto vector = state.builder.AllocateId();
const auto component = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.vec4_float_type, vector, input->variable_id});
state.builder.AddFunction({OpCompositeExtract, state.float_type, component, vector, chan & 3u});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
return bits;
}
bool MrtUsesUint(const EmitterState& state, const IR::ExportInfo& exp) {
return exp.kind == IR::ExportTargetKind::Mrt && state.pixel_input_info != nullptr &&
exp.index < std::size(state.pixel_input_info->target_output_mode) &&
state.pixel_input_info->target_output_mode[exp.index] == 7u;
}
uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpCompositeExtract, ctx.state.uint_type, value, vector, component});
return value;
}
uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp,
bool uint_output) {
auto& state = ctx.state;
if (exp.compr && !uint_output) {
uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
ConstantF32(state, 0x3f800000u)};
for (uint32_t pair = 0; pair < 2u; pair++) {
if ((exp.en & (3u << (pair * 2u))) == 0u) {
continue;
}
const auto packed = state.builder.AllocateId();
const auto unpacked = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, packed, data, pair});
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
state.glsl_std450, GlslUnpackHalf2x16, packed});
for (uint32_t lane = 0; lane < 2u; lane++) {
const auto component = pair * 2u + lane;
if (((exp.en >> component) & 1u) != 0u) {
f32[component] = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, f32[component], unpacked, lane});
}
}
}
const auto vector = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
return vector;
}
uint32_t raw[4] = {
ConstantU32(state, 0),
ConstantU32(state, 0),
ConstantU32(state, 0),
ConstantU32(state, uint_output ? 1u : 0x3f800000u),
};
if (exp.compr) {
for (uint32_t pair = 0; pair < 2u; pair++) {
if ((exp.en & (3u << (pair * 2u))) == 0u) {
continue;
}
const auto packed = ExportRawComponent(ctx, data, pair);
for (uint32_t lane = 0; lane < 2u; lane++) {
const auto component = pair * 2u + lane;
if (((exp.en >> component) & 1u) == 0u) {
continue;
}
raw[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, raw[component],
packed, ConstantU32(state, lane * 16u),
ConstantU32(state, 16)});
}
}
} else {
for (uint32_t component = 0; component < 4u; component++) {
if (((exp.en >> component) & 1u) != 0u) {
raw[component] = ExportRawComponent(ctx, data, component);
}
}
}
if (uint_output) {
const auto vector = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeConstruct, state.vec4_uint_type, vector, raw[0], raw[1], raw[2], raw[3]});
return vector;
}
uint32_t f32[4] {};
for (uint32_t component = 0; component < 4u; component++) {
f32[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32[component], raw[component]});
}
const auto vector = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
return vector;
}
void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto& exp = ctx.Export(inst);
const auto exec = ctx.Arg(inst, 1);
if (state.stage == ShaderType::Pixel && exp.vm && state.needs_pixel_valid_mask &&
state.pixel_valid_mask_variable != 0) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
}
if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
exp.en == 0u) {
return;
}
EmitIfCondition(state, exec, [&]() {
const auto data = ctx.Arg(inst, 0);
if (exp.kind == IR::ExportTargetKind::MrtZ) {
if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
const auto raw = ExportRawComponent(ctx, data, 0);
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, raw});
state.builder.AddFunction({OpStore, state.depth_variable, f32});
}
if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
const auto raw = ExportRawComponent(ctx, data, 2);
const auto value = state.builder.AllocateId();
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, value, raw});
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, pointer,
state.sample_mask_variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, pointer, value});
}
return;
}
const auto variable = OutputVariableForExport(state, exp);
if (variable == 0) {
return;
}
const bool uint_output = MrtUsesUint(state, exp);
const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
auto value = ExportVector(ctx, data, exp, uint_output);
if (exp.kind == IR::ExportTargetKind::Mrt && state.pixel_input_info != nullptr &&
exp.index < state.pixel_input_info->target_export_mapping.size()) {
const auto mapping = state.pixel_input_info->target_export_mapping[exp.index];
if (!mapping.IsIdentity()) {
const auto mapped = state.builder.AllocateId();
state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
mapping.Map(0), mapping.Map(1), mapping.Map(2),
mapping.Map(3)});
value = mapped;
}
}
if (exp.kind == IR::ExportTargetKind::Position) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer,
variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, pointer, value});
} else {
state.builder.AddFunction({OpStore, variable, value});
}
});
}
} // namespace
bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Identity:
case IR::ValueOpcode::ConditionRef: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
case IR::ValueOpcode::SetScalarRegister:
StoreRegisterMirror(
ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
ctx.Arg(inst, 1));
return true;
case IR::ValueOpcode::SetThreadBitScalarRegister:
StoreRegisterMirror(
ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
BoolAsU32(ctx, ctx.Arg(inst, 1)));
return true;
case IR::ValueOpcode::SetVectorRegister:
StoreRegisterMirror(
ctx, {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())},
ctx.Arg(inst, 1));
return true;
case IR::ValueOpcode::SetScc: ctx.Arg(inst, 0); return true;
case IR::ValueOpcode::SetExec: ctx.Arg(inst, 0); return true;
case IR::ValueOpcode::SetExecLo:
case IR::ValueOpcode::SetExecHi:
if (!IsPrologue(ctx)) {
StoreRegisterMirror(ctx,
{IR::RegisterFile::Exec,
inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u},
ctx.Arg(inst, 0));
}
return true;
case IR::ValueOpcode::SetVcc: ctx.Arg(inst, 0); return true;
case IR::ValueOpcode::SetVccLo:
case IR::ValueOpcode::SetVccHi:
if (!IsPrologue(ctx)) {
StoreRegisterMirror(ctx,
{IR::RegisterFile::Vcc,
inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u},
ctx.Arg(inst, 0));
}
return true;
case IR::ValueOpcode::SetM0:
StoreRegisterMirror(ctx, {IR::RegisterFile::M0, 0}, ctx.Arg(inst, 0));
return true;
case IR::ValueOpcode::GetScalarRegister:
case IR::ValueOpcode::GetThreadBitScalarRegister:
case IR::ValueOpcode::GetVectorRegister:
case IR::ValueOpcode::GetScc:
case IR::ValueOpcode::GetExec:
case IR::ValueOpcode::GetExecLo:
case IR::ValueOpcode::GetExecHi:
case IR::ValueOpcode::GetVcc:
case IR::ValueOpcode::GetVccLo:
case IR::ValueOpcode::GetVccHi:
case IR::ValueOpcode::GetM0: return true;
case IR::ValueOpcode::Void:
case IR::ValueOpcode::Reference:
case IR::ValueOpcode::ReferenceU32:
case IR::ValueOpcode::Prologue:
case IR::ValueOpcode::Epilogue:
case IR::ValueOpcode::ControlNop:
case IR::ValueOpcode::Waitcnt:
case IR::ValueOpcode::Sendmsg:
case IR::ValueOpcode::TtraceData:
case IR::ValueOpcode::InstPrefetch: return true;
case IR::ValueOpcode::Barrier: {
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, semantics)});
return true;
}
case IR::ValueOpcode::GetUserData: {
const auto reg = inst.Arg(0).ScalarRegister();
uint32_t dword = 0;
if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) {
ctx.Define(inst, ConstantU32(state, 0));
} else {
ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
}
return true;
}
case IR::ValueOpcode::GetBuiltin:
ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
inst.Arg(1).U32()));
return true;
case IR::ValueOpcode::UndefU1:
case IR::ValueOpcode::UndefU8:
case IR::ValueOpcode::UndefU16:
case IR::ValueOpcode::UndefU32:
case IR::ValueOpcode::UndefU64:
state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
return true;
case IR::ValueOpcode::DppMoveU32: {
const auto flags = inst.Flags<IR::DppMoveFlags>();
IR::Operand operand;
operand.dpp = true;
operand.dpp_ctrl = flags.control;
operand.dpp_row_mask = flags.row_mask;
operand.dpp_bank_mask = flags.bank_mask;
operand.dpp_fetch_inactive = flags.fetch_inactive;
operand.dpp_bound_ctrl = flags.bound_control;
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
target.lane});
if (operand.dpp_fetch_inactive) {
ctx.Define(inst, shuffled);
return true;
}
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
const auto can_fetch = state.builder.AllocateId();
state.builder.AddFunction(
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
return true;
}
case IR::ValueOpcode::DppUpdateU32: {
const auto flags = inst.Flags<IR::DppMoveFlags>();
const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
}
case IR::ValueOpcode::WqmMask:
ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
return true;
case IR::ValueOpcode::LaneId:
ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
return true;
case IR::ValueOpcode::Ballot:
if (state.per_invocation_masks) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result,
BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
ConstantU32(state, 0), ConstantU32(state, 0)});
ctx.Define(inst, result);
} else {
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
}
return true;
case IR::ValueOpcode::ReadFirstLane: {
const auto ballot = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, lane,
ConstantU32(state, ScopeSubgroup), ballot});
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
return true;
}
case IR::ValueOpcode::ReadLane:
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true;
case IR::ValueOpcode::WriteLane: {
const auto hit = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, hit,
EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
return true;
}
case IR::ValueOpcode::Permlane16U32: {
const auto flags = inst.Flags<IR::PermlaneFlags>();
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto row = state.builder.AllocateId();
const auto row_value = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto lane8 = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto upper = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto shifted = state.builder.AllocateId();
const auto index = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
if (flags.x16) {
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
} else {
state.builder.AddFunction({OpCopyObject, state.uint_type, row_value, row});
}
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
state.builder.AddFunction(
{OpSelect, state.uint_type, selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, shifted, selected, shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, index, shifted, ConstantU32(state, 15)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index});
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
target});
uint32_t result = shuffled;
if (!flags.fetch_inactive) {
const auto source_exec = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
target});
result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, state.uint_type, result, source_exec, shuffled,
ConstantU32(state, 0)});
}
ctx.Define(inst, result);
return true;
}
case IR::ValueOpcode::GetAttribute:
ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
return true;
case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
case IR::ValueOpcode::GetShaderBase:
// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
return true;
case IR::ValueOpcode::GetSrtResource:
case IR::ValueOpcode::GetBufferResource:
case IR::ValueOpcode::GetAddressResource:
case IR::ValueOpcode::GetImageResource:
case IR::ValueOpcode::GetSamplerResource:
case IR::ValueOpcode::GetLdsResource:
case IR::ValueOpcode::GetGdsResource:
case IR::ValueOpcode::MakeImageAddress: return true;
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp
+516
View File
@@ -0,0 +1,516 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <algorithm>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, value});
return result;
}
uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, lhs, rhs});
return result;
}
bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {
return (mem.image_sample_flags & flag) != 0u;
}
bool UsesA16(const IR::MemoryInfo& mem) {
return HasFlag(mem, Decoder::ImageSampleFlagA16);
}
bool ComponentUsesA16(const IR::MemoryInfo& mem, uint32_t component) {
if (!UsesA16(mem)) return false;
uint32_t cursor = 0;
if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) {
if (component == cursor) return false;
cursor++;
}
if (HasFlag(mem, Decoder::ImageSampleFlagBias)) {
if (component == cursor) return true;
cursor++;
}
if (HasFlag(mem, Decoder::ImageSampleFlagCompare) && component == cursor) return false;
return true;
}
uint32_t HalfComponent(const IR::MemoryInfo& mem, uint32_t component) {
if (!UsesA16(mem)) return component * 2u;
uint32_t half = 0;
for (uint32_t index = 0; index < component; index++) {
const auto width = ComponentUsesA16(mem, index) ? 1u : 2u;
if (width == 2u && (half & 1u) != 0u) half++;
half += width;
}
if (!ComponentUsesA16(mem, component) && (half & 1u) != 0u) half++;
return half;
}
uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
uint32_t component) {
const auto half = HalfComponent(mem, component);
const auto packed = half / 2u;
if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0);
auto value = ctx.Def(address.Arg(packed));
if (ComponentUsesA16(mem, component)) {
if ((half & 1u) != 0u) {
value = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, value,
ConstantU32(ctx.state, 16));
}
value = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, value,
ConstantU32(ctx.state, 0xffffu));
}
return value;
}
uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
uint32_t component) {
const auto value = AddressU32(ctx, mem, address, component);
return ComponentUsesA16(mem, component)
? EmitF16BitsToF32(ctx.state, value)
: Unary(ctx.state, OpBitcast, ctx.state.float_type, value);
}
ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) {
ImageSampleLayout layout;
uint32_t cursor = 0;
if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++;
if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++;
if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++;
if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
layout.grad_x = cursor;
cursor += ImageViewSpatialComponents(view);
layout.grad_y = cursor;
cursor += ImageViewSpatialComponents(view);
}
layout.coord = cursor;
cursor += ImageViewCoordinateComponents(view);
if (HasFlag(mem, Decoder::ImageSampleFlagLod)) layout.lod = cursor++;
return layout;
}
uint32_t ZeroF32(EmitterState& state) {
return ConstantF32(state, 0);
}
uint32_t ZeroU32x4(EmitterState& state) {
const auto zero = ConstantU32(state, 0);
const auto value = state.builder.AllocateId();
state.builder.AddType(
{OpConstantComposite, state.vec4_uint_type, value, zero, zero, zero, zero});
return value;
}
uint32_t CubeAxis(EmitterState& state, uint32_t value) {
return Binary(state, OpFSub, state.float_type, value, ConstantF32(state, 0x3f800000u));
}
uint32_t CubeLayer(EmitterState& state, uint32_t value) {
const auto guest = state.builder.AllocateId();
state.builder.AddFunction({OpConvertFToU, state.uint_type, guest, value});
const auto padding =
Binary(state, OpShiftLeftLogical, state.uint_type,
Binary(state, OpShiftRightLogical, state.uint_type, guest, ConstantU32(state, 3)),
ConstantU32(state, 1));
const auto host = Binary(state, OpISub, state.uint_type, guest, padding);
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, state.float_type, result, host});
return result;
}
uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
uint32_t first, uint32_t components) {
auto x = AddressF32(ctx, mem, address, first);
if (components == 1u) return x;
auto y = mem.image_address_components > first + 1u ? AddressF32(ctx, mem, address, first + 1u)
: ZeroF32(ctx.state);
if (mem.image_cube) {
x = CubeAxis(ctx.state, x);
y = CubeAxis(ctx.state, y);
}
const auto result = ctx.state.builder.AllocateId();
if (components == 3u) {
auto z = mem.image_address_components > first + 2u
? AddressF32(ctx, mem, address, first + 2u)
: ZeroF32(ctx.state);
if (mem.image_cube) z = CubeLayer(ctx.state, z);
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec3_float_type, result, x, y, z});
} else {
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec2_float_type, result, x, y});
}
return result;
}
uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
ImageViewKind view) {
const auto components = ImageViewCoordinateComponents(view);
const auto x = AddressU32(ctx, mem, address, 0);
if (components == 1u) return x;
const auto y = mem.image_address_components > 1u ? AddressU32(ctx, mem, address, 1)
: ConstantU32(ctx.state, 0);
const auto result = ctx.state.builder.AllocateId();
if (components == 3u) {
const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
: ConstantU32(ctx.state, 0);
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec3_uint_type, result, x, y, z});
} else {
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec2_uint_type, result, x, y});
}
return result;
}
uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
ImageViewKind view) {
const auto component = ImageViewCoordinateComponents(view);
return mem.image_has_mip && mem.image_address_components > component
? AddressU32(ctx, mem, address, component)
: ConstantU32(ctx.state, 0);
}
uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
return Unary(ctx.state, OpBitcast, ctx.state.uint_type, value);
}
uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) {
uint32_t component[4] {};
for (uint32_t index = 0; index < 4u; index++) {
if (dref) {
component[index] = index == 0u ? FloatBits(ctx, value) : ConstantU32(ctx.state, 0);
continue;
}
const auto scalar = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeExtract,
integer ? ctx.state.uint_type : ctx.state.float_type, scalar,
value, index});
component[index] = integer ? scalar : FloatBits(ctx, scalar);
}
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, result,
component[0], component[1], component[2], component[3]});
return result;
}
uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const IR::Inst& address) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
const auto view = SampledImageViewKind(ctx.state, mem, pc);
const auto image = LoadSampledImageDescriptor(ctx.state, mem, pc, view);
const auto size = ctx.state.builder.AllocateId();
if (ImageSpirvMultisampled(view) != 0u) {
ctx.state.builder.AddFunction(
{OpImageQuerySize, ImageViewSizeType(ctx.state, view), size, image});
} else {
ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, view),
size, image, AddressU32(ctx, mem, address, 0)});
}
const auto components = ImageViewCoordinateComponents(view);
uint32_t result[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
if (components == 1u) {
result[0] = size;
} else {
for (uint32_t index = 0; index < components; index++) {
result[index] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpCompositeExtract, ctx.state.uint_type, result[index], size, index});
}
}
if (ImageSpirvMultisampled(view) == 0u) {
result[3] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpImageQueryLevels, ctx.state.uint_type, result[3], image});
}
const auto vector = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, vector,
result[0], result[1], result[2], result[3]});
return vector;
}
uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
const ImageSampleLayout& layout, ImageViewKind view) {
const auto components = ImageViewSpatialComponents(view);
const auto zero = ConstantI32(ctx.state, 0);
if (layout.offset == NoImageComponent || mem.image_address_components <= layout.offset) {
if (components == 1u) return zero;
const auto result = ctx.state.builder.AllocateId();
if (components == 3u) {
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec3_int_type, result, zero, zero, zero});
} else {
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec2_int_type, result, zero, zero});
}
return result;
}
const auto packed = Unary(ctx.state, OpBitcast, ctx.state.int_type,
AddressU32(ctx, mem, address, layout.offset));
uint32_t values[3] = {zero, zero, zero};
for (uint32_t index = 0; index < components; index++) {
values[index] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpBitFieldSExtract, ctx.state.int_type, values[index],
packed, ConstantU32(ctx.state, index * 8u),
ConstantU32(ctx.state, 6)});
}
if (components == 1u) return values[0];
const auto result = ctx.state.builder.AllocateId();
if (components == 3u) {
ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec3_int_type, result,
values[0], values[1], values[2]});
} else {
ctx.state.builder.AddFunction(
{OpCompositeConstruct, ctx.state.vec2_int_type, result, values[0], values[1]});
}
return result;
}
uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) {
const auto components = ImageViewSpatialComponents(view);
const auto array_type = state.builder.AllocateId();
const auto count = ConstantU32(state, 4);
const auto element_type = components == 1u ? state.int_type : state.vec2_int_type;
state.builder.AddType({OpTypeArray, array_type, element_type, count});
uint32_t offsets[4] {};
for (uint32_t index = 0; index < 4u; index++) {
const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
if (components == 1u) {
offsets[index] = x;
} else {
offsets[index] = state.builder.AllocateId();
state.builder.AddType({OpConstantComposite, state.vec2_int_type, offsets[index], x,
ConstantI32(state, 0)});
}
}
const auto value = state.builder.AllocateId();
state.builder.AddType(
{OpConstantComposite, array_type, value, offsets[0], offsets[1], offsets[2], offsets[3]});
return value;
}
uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
for (uint32_t source = 0; source < 4u; source++) {
if (((swizzle >> (source * 3u)) & 7u) == 4u + component) return source;
}
return UINT32_MAX;
}
uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) {
const auto swizzle = ctx.state.program.info.images[mem.resource].storage_swizzle;
uint32_t values[4] {};
const auto dmask = mem.dmask != 0u ? mem.dmask : 1u;
for (uint32_t component = 0; component < 4u; component++) {
const auto source = InverseSwizzle(swizzle, component);
uint32_t raw = ConstantU32(ctx.state, 0);
if (source < 4u && ((dmask >> source) & 1u) != 0u) {
const auto packed_index = DmaskComponentIndex(dmask, source);
raw = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpCompositeExtract, ctx.state.uint_type, raw, data, packed_index});
}
values[component] = integer ? raw : Unary(ctx.state, OpBitcast, ctx.state.float_type, raw);
}
const auto texel = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct,
integer ? ctx.state.vec4_uint_type : ctx.state.vec4_float_type,
texel, values[0], values[1], values[2], values[3]});
return texel;
}
uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
switch (opcode) {
case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
default: return 0;
}
}
} // namespace
bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
if (op < IR::ValueOpcode::ImageQueryDimensions || op > IR::ValueOpcode::ImageAtomicXor32) {
return false;
}
auto& state = ctx.state;
const auto& mem = ctx.Memory(inst);
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
const auto image_arg = inst.Arg(0);
ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
const bool sampled = op == IR::ValueOpcode::ImageQueryLod ||
op == IR::ValueOpcode::ImageSampleRaw ||
op == IR::ValueOpcode::ImageGatherRaw;
const auto* address = ctx.ImageAddress(inst.Arg(sampled ? 2 : 1));
if (address == nullptr) return true;
if (op == IR::ValueOpcode::ImageQueryDimensions) {
ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address));
return true;
}
if (op == IR::ValueOpcode::ImageQueryLod) {
const auto view = SampledImageViewKind(state, mem, pc);
const auto sampled = MakeSampledImage(state, mem, pc, view);
const auto lod = state.builder.AllocateId();
state.builder.AddFunction(
{OpImageQueryLod, state.vec2_float_type, lod, sampled,
CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))});
uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
ConstantU32(state, 0)};
for (uint32_t index = 0; index < 2u; index++) {
const auto component = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, component, lod, index});
values[index] = FloatBits(ctx, component);
}
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, values[0],
values[1], values[2], values[3]});
ctx.Define(inst, result);
return true;
}
if (op == IR::ValueOpcode::ImageRead) {
const auto view = SampledImageViewKind(state, mem, pc);
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
const auto condition = ctx.Arg(inst, 2);
ctx.Define(
inst,
EmitValueOrDefaultIfCondition(
state, condition, state.vec4_uint_type, ZeroU32x4(state), [&]() {
const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
const auto color = state.builder.AllocateId();
const auto coord = CoordU32(ctx, mem, *address, view);
if (ImageSpirvMultisampled(view) != 0u) {
state.builder.AddFunction(
{OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
color, image, coord, ImageOperandsSampleMask,
AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))});
} else {
state.builder.AddFunction(
{OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
color, image, coord, ImageOperandsLodMask,
LodU32(ctx, mem, *address, view)});
}
return ResultVector(ctx, color, integer, false);
}));
return true;
}
if (op == IR::ValueOpcode::ImageWrite) {
const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint;
const auto view = StorageImageViewKind(state, mem, uint_image, pc);
EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
state.builder.AddFunction(
{OpImageWrite,
LoadStorageImageDescriptor(state, mem.resource, uint_image, pc, view),
CoordU32(ctx, mem, *address, view),
StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image)});
});
return true;
}
if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
const auto view = SampledImageViewKind(state, mem, pc);
const auto layout = Layout(mem, view);
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
const bool dref = HasFlag(mem, Decoder::ImageSampleFlagCompare);
const auto sampled = MakeSampledImage(state, mem, pc, view);
const auto coord =
CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view));
const auto sample = state.builder.AllocateId();
if (op == IR::ValueOpcode::ImageGatherRaw) {
std::vector<uint32_t> words =
dref ? std::vector<uint32_t> {OpImageDrefGather,
state.vec4_float_type,
sample,
sampled,
coord,
layout.dref != NoImageComponent
? AddressF32(ctx, mem, *address, layout.dref)
: ZeroF32(state)}
: std::vector<uint32_t> {
OpImageGather, integer ? state.vec4_uint_type : state.vec4_float_type,
sample, sampled,
coord, ConstantU32(state, ImageGatherComponent(mem.dmask))};
if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
words.push_back(ImageOperandsConstOffsetsMask);
words.push_back(HorizontalOffsets(state, view));
} else if (layout.offset != NoImageComponent) {
words.push_back(ImageOperandsOffsetMask);
words.push_back(PackedOffset(ctx, mem, *address, layout, view));
}
state.builder.AddFunction(words);
ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false));
return true;
}
const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
HasFlag(mem, Decoder::ImageSampleFlagLod) ||
HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
state.stage != ShaderType::Pixel;
const auto opcode = explicit_lod
? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod)
: (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod);
const auto result_type =
dref ? state.float_type : (integer ? state.vec4_uint_type : state.vec4_float_type);
std::vector<uint32_t> words {opcode, result_type, sample, sampled, coord};
if (dref) {
words.push_back(layout.dref != NoImageComponent
? AddressF32(ctx, mem, *address, layout.dref)
: ZeroF32(state));
}
uint32_t operand_mask = 0;
std::vector<uint32_t> operands;
if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
operand_mask |= ImageOperandsGradMask;
operands.push_back(
CoordF32(ctx, mem, *address, layout.grad_x, ImageViewSpatialComponents(view)));
operands.push_back(
CoordF32(ctx, mem, *address, layout.grad_y, ImageViewSpatialComponents(view)));
} else if (explicit_lod) {
operand_mask |= ImageOperandsLodMask;
operands.push_back(HasFlag(mem, Decoder::ImageSampleFlagLod) &&
layout.lod != NoImageComponent
? AddressF32(ctx, mem, *address, layout.lod)
: ZeroF32(state));
} else if (layout.bias != NoImageComponent) {
operand_mask |= ImageOperandsBiasMask;
operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
}
if (operand_mask != 0u) {
words.push_back(operand_mask);
words.insert(words.end(), operands.begin(), operands.end());
}
state.builder.AddFunction(words);
ctx.Define(inst, ResultVector(ctx, sample, integer, dref));
return true;
}
const auto atomic_opcode = ImageAtomicOpcode(op);
if (atomic_opcode != 0u) {
const auto view = StorageImageViewKind(state, mem, true, pc);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpImageTexelPointer, state.ptr_image_uint, pointer,
StorageImageDescriptorPointer(state, mem.resource, true, pc, view),
CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)});
const auto old = state.builder.AllocateId();
state.builder.AddFunction({atomic_opcode, state.uint_type, old, pointer,
ConstantU32(state, ScopeDevice),
ConstantU32(state, MemorySemanticsNone),
ctx.Arg(inst, 2)});
EmitDeviceAtomicMemoryBarrier(state);
return old;
}));
return true;
}
return false;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp
+713
View File
@@ -0,0 +1,713 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <algorithm>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, value});
return result;
}
uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, lhs, rhs});
return result;
}
uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
return result;
}
uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
if (lhs == 0) return rhs;
if (rhs == 0) return lhs;
return Binary(state, OpLogicalAnd, state.bool_type, lhs, rhs);
}
uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t index, uint32_t offset, uint32_t soffset) {
auto& state = ctx.state;
const auto packed =
ConstantU32(state, StorageBufferPackedStride(state, mem, inst.Flags<IR::MemoryFlags>().pc));
if (mem.formatted && !mem.typed && inst.GetOpcode() == IR::ValueOpcode::StoreBufferU32) {
const auto add_tid_bit = Binary(
state, OpBitwiseAnd, state.uint_type,
Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 20)),
ConstantU32(state, 1));
const auto add_tid =
Binary(state, OpINotEqual, state.bool_type, add_tid_bit, ConstantU32(state, 0));
const auto lane = Binary(state, OpBitwiseAnd, state.uint_type,
EmitLocalInvocationIndex(state), ConstantU32(state, 63));
index = Select(state, state.uint_type, add_tid,
Binary(state, OpIAdd, state.uint_type, index, lane), index);
}
const auto stride =
Binary(state, OpBitwiseAnd, state.uint_type, packed, ConstantU32(state, 0x3fffu));
const auto swizzle =
Binary(state, OpBitwiseAnd, state.uint_type,
Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 14)),
ConstantU32(state, 1));
const auto use_swizzle =
Binary(state, OpLogicalAnd, state.bool_type,
Binary(state, OpINotEqual, state.bool_type, stride, ConstantU32(state, 0)),
Binary(state, OpINotEqual, state.bool_type, swizzle, ConstantU32(state, 0)));
const auto stride_enum =
Binary(state, OpBitwiseAnd, state.uint_type,
Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 16)),
ConstantU32(state, 3));
const auto index_shift =
Binary(state, OpIAdd, state.uint_type, stride_enum, ConstantU32(state, 3));
const auto index_stride =
Binary(state, OpShiftLeftLogical, state.uint_type, ConstantU32(state, 8), stride_enum);
const auto index_mask =
Binary(state, OpISub, state.uint_type, index_stride, ConstantU32(state, 1));
offset = Binary(state, OpIAdd, state.uint_type, offset, ConstantU32(state, mem.offset));
const auto linear = Binary(state, OpIAdd, state.uint_type,
Binary(state, OpIMul, state.uint_type, index, stride), offset);
const auto index_msb = Binary(state, OpShiftRightLogical, state.uint_type, index, index_shift);
const auto index_lsb = Binary(state, OpBitwiseAnd, state.uint_type, index, index_mask);
const auto offset_msb =
Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, ~3u));
const auto offset_lsb =
Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, 3));
const auto msb =
Binary(state, OpIMul, state.uint_type,
Binary(state, OpIAdd, state.uint_type,
Binary(state, OpIMul, state.uint_type, index_msb, stride), offset_msb),
index_stride);
const auto lsb =
Binary(state, OpIAdd, state.uint_type,
Binary(state, OpShiftLeftLogical, state.uint_type, index_lsb, ConstantU32(state, 2)),
offset_lsb);
const auto address = Select(state, state.uint_type, use_swizzle,
Binary(state, OpIAdd, state.uint_type, msb, lsb), linear);
return Binary(state, OpIAdd, state.uint_type, address, soffset);
}
uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low,
uint32_t add_high, uint32_t& out_high) {
const auto result = Binary(state, OpIAdd, state.uint_type, low, add_low);
const auto carry = Binary(state, OpULessThan, state.bool_type, result, low);
out_high = Binary(
state, OpIAdd, state.uint_type, Binary(state, OpIAdd, state.uint_type, high, add_high),
Select(state, state.uint_type, carry, ConstantU32(state, 1), ConstantU32(state, 0)));
return result;
}
uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t low, uint32_t high) {
auto& state = ctx.state;
const auto immediate = static_cast<int32_t>(mem.offset);
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
const auto base = state.program.info.addresses[mem.resource].specialized_base;
if (mem.kind == IR::ResourceKind::Scratch) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid = Binary(state, OpIEqual, state.bool_type, high, ConstantU32(state, 0));
return Select(state, state.uint_type, valid, low, ConstantU32(state, UINT32_MAX));
}
if (mem.kind == IR::ResourceKind::Flat ||
(mem.kind == IR::ResourceKind::Global &&
state.program.info.addresses[mem.resource].unbased)) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto base_low = ConstantU32(state, static_cast<uint32_t>(base));
const auto base_high = ConstantU32(state, static_cast<uint32_t>(base >> 32u));
const auto relative = Binary(state, OpISub, state.uint_type, low, base_low);
const auto borrow = Binary(state, OpULessThan, state.bool_type, low, base_low);
const auto relative_high = Binary(
state, OpISub, state.uint_type, Binary(state, OpISub, state.uint_type, high, base_high),
Select(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0)));
const auto valid =
Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0));
return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX));
}
const auto initial = base + static_cast<uint64_t>(static_cast<int64_t>(immediate));
auto relative_high = ConstantU32(state, static_cast<uint32_t>(initial >> 32u));
const auto relative = AddU64Low(state, ConstantU32(state, static_cast<uint32_t>(initial)),
relative_high, low, ConstantU32(state, 0), relative_high);
const auto valid =
Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0));
return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX));
}
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
if (mem.kind == IR::ResourceKind::Buffer || mem.kind == IR::ResourceKind::ScalarBuffer) {
if (mem.kind == IR::ResourceKind::ScalarBuffer) {
return Binary(ctx.state, OpIAdd, ctx.state.uint_type, ctx.Arg(inst, 1),
ConstantU32(ctx.state, mem.offset));
}
return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2),
ctx.Arg(inst, 3));
}
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
return ctx.Arg(inst, 1);
}
return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
}
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
return Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, ByteAddress(ctx, inst, mem),
ConstantU32(ctx.state, 2));
}
uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
if (mem.kind == IR::ResourceKind::Lds) {
return 0;
}
if (mem.kind == IR::ResourceKind::Gds) {
return EmitGdsElementInBounds(ctx.state, index);
}
return EmitStorageBufferElementInBounds(ctx.state, mem, index, pc);
}
uint32_t Pointer(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
return EmitMemoryElementPointer(ctx.state, mem, index, pc);
}
uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, ctx.state.uint_type, value, Pointer(ctx, mem, index, pc)});
return value;
});
});
}
uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
bool sign_extend, uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto address = ByteAddress(ctx, inst, mem);
const auto index = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address,
ConstantU32(ctx.state, 2));
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto word = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, ctx.state.uint_type, word, Pointer(ctx, mem, index, pc)});
const auto byte = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address,
ConstantU32(ctx.state, 3));
const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, byte,
ConstantU32(ctx.state, 3));
const auto value =
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type,
Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, word, shift),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
if (!sign_extend) return value;
const auto left = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, value,
ConstantU32(ctx.state, 32u - bits));
return Binary(ctx.state, OpShiftRightArithmetic, ctx.state.uint_type, left,
ConstantU32(ctx.state, 32u - bits));
});
});
}
Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem) {
return mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
: StorageBufferFormat(ctx.state, mem, inst.Flags<IR::MemoryFlags>().pc);
}
IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format,
uint32_t component) {
if (!mem.typed && component != 0u) {
const auto component_slot = component * 4u;
mem.offset = mem.offset >= component_slot ? mem.offset - component_slot : 0u;
mem.offset += Format::GetFormatComponentByteOffset(format, component);
}
return mem;
}
uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
const auto format = BufferFormat(ctx, inst, mem);
if (!Format::IsKnownFormat(format)) return LoadWord(ctx, inst, mem);
const auto info = Format::GetFormatInfo(format);
const auto component = mem.component_index;
if (component >= info.component_count) return ConstantU32(ctx.state, 0);
uint32_t raw = 0;
const auto bits = info.component_bits[component];
if (info.packed_bitfield) {
raw = LoadWord(ctx, inst, mem);
const auto type =
IsSignedFormatComponent(info.type) ? ctx.state.int_type : ctx.state.uint_type;
const auto source =
type == ctx.state.int_type ? Unary(ctx.state, OpBitcast, type, raw) : raw;
const auto extracted = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type,
extracted, source, ConstantU32(ctx.state, info.component_bit_offset[component]),
ConstantU32(ctx.state, bits)});
raw = type == ctx.state.int_type
? Unary(ctx.state, OpBitcast, ctx.state.uint_type, extracted)
: extracted;
} else if (bits == 32u) {
raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component));
} else {
raw = LoadSubword(ctx, inst, RebaseFormattedComponent(mem, format, component), bits,
IsSignedFormatComponent(info.type));
}
return NormalizeFormatComponent(ctx.state, info, component, raw);
}
template <typename Fn>
uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory
: MemorySemanticsUniformMemory;
const auto preheader = state.builder.AllocateId();
const auto header = state.builder.AllocateId();
const auto cont = state.builder.AllocateId();
const auto merge = state.builder.AllocateId();
const auto initial = state.builder.AllocateId();
const auto observed = state.builder.AllocateId();
const auto exchanged = state.builder.AllocateId();
state.builder.AddFunction({OpBranch, preheader});
state.builder.AddFunction({OpLabel, preheader});
state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
state.builder.AddFunction({OpBranch, header});
state.builder.AddFunction({OpLabel, header});
state.builder.AddFunction(
{OpPhi, state.uint_type, observed, initial, preheader, exchanged, cont});
const auto next = desired(observed);
state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
ConstantU32(state, MemorySemanticsNone), next, observed});
const auto success = Binary(state, OpIEqual, state.bool_type, exchanged, observed);
state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone});
state.builder.AddFunction({OpBranchConditional, success, merge, cont});
state.builder.AddFunction({OpLabel, cont});
state.builder.AddFunction({OpBranch, header});
state.builder.AddFunction({OpLabel, merge});
state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope),
ConstantU32(state, MemorySemanticsAcquireRelease | memory)});
return observed;
}
void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto address = ByteAddress(ctx, inst, mem);
const auto index = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address,
ConstantU32(ctx.state, 2));
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto pointer = Pointer(ctx, mem, index, pc);
const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address,
ConstantU32(ctx.state, 3)),
ConstantU32(ctx.state, 3));
const auto mask = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
const auto value = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type,
ctx.Arg(inst, inst.NumArgs() - 2),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
shift);
const auto merge = [&](uint32_t old) {
return Binary(ctx.state, OpBitwiseOr, ctx.state.uint_type,
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, old,
Unary(ctx.state, OpNot, ctx.state.uint_type, mask)),
value);
};
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
AtomicUpdate(ctx.state, pointer, mem.kind, merge);
} else {
const auto old = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpLoad, ctx.state.uint_type, old, pointer});
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
}
});
});
}
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
uint32_t extra_offset = 0) {
mem.offset += extra_offset;
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
ctx.state.builder.AddFunction(
{OpStore, Pointer(ctx, mem, index, pc), ctx.Arg(inst, inst.NumArgs() - 2)});
});
});
}
void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
const auto format = BufferFormat(ctx, inst, mem);
if (!Format::IsKnownFormat(format)) {
StoreWord(ctx, inst, mem);
return;
}
const auto info = Format::GetFormatInfo(format);
if (mem.component_index >= info.component_count) return;
const auto bits = info.component_bits[mem.component_index];
const auto component_mem = RebaseFormattedComponent(mem, format, mem.component_index);
if (bits == 8u || bits == 16u) {
StoreSubword(ctx, inst, component_mem, bits);
} else {
StoreWord(ctx, inst, component_mem);
}
}
uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
switch (opcode) {
case IR::ValueOpcode::BufferAtomicSwap32:
case IR::ValueOpcode::SharedAtomicSwap32:
case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange;
case IR::ValueOpcode::BufferAtomicIAdd32:
case IR::ValueOpcode::SharedAtomicIAdd32:
case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd;
case IR::ValueOpcode::BufferAtomicISub32:
case IR::ValueOpcode::SharedAtomicISub32:
case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub;
case IR::ValueOpcode::BufferAtomicSMin32:
case IR::ValueOpcode::SharedAtomicSMin32:
case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin;
case IR::ValueOpcode::BufferAtomicUMin32:
case IR::ValueOpcode::SharedAtomicUMin32:
case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin;
case IR::ValueOpcode::BufferAtomicSMax32:
case IR::ValueOpcode::SharedAtomicSMax32:
case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax;
case IR::ValueOpcode::BufferAtomicUMax32:
case IR::ValueOpcode::SharedAtomicUMax32:
case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax;
case IR::ValueOpcode::BufferAtomicAnd32:
case IR::ValueOpcode::SharedAtomicAnd32:
case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd;
case IR::ValueOpcode::BufferAtomicOr32:
case IR::ValueOpcode::SharedAtomicOr32:
case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr;
case IR::ValueOpcode::BufferAtomicXor32:
case IR::ValueOpcode::SharedAtomicXor32:
case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor;
default: return 0;
}
}
uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool return_value) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
const auto result =
EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
const auto old = ctx.state.builder.AllocateId();
const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
ctx.state.builder.AddFunction({AtomicOpcode(inst.GetOpcode()), ctx.state.uint_type,
old, Pointer(ctx, mem, index, pc),
ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, MemorySemanticsNone), value});
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
ctx.state.builder.AddFunction({OpMemoryBarrier,
ConstantU32(ctx.state, ScopeWorkgroup),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
});
return return_value ? result : ConstantU32(ctx.state, 0);
}
uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
bool max_value) {
struct OrderedBits {
uint32_t nan;
uint32_t zero;
uint32_t key;
};
const auto classify = [&](uint32_t bits) {
const auto value = Unary(state, OpBitcast, state.float_type, bits);
const auto cls = EmitClassifyF32(state, value);
const auto negative = Binary(
state, OpINotEqual, state.bool_type,
Binary(state, OpBitwiseAnd, state.uint_type, bits, ConstantU32(state, 0x80000000u)),
ConstantU32(state, 0));
const auto negative_key = Unary(state, OpNot, state.uint_type, bits);
const auto positive_key =
Binary(state, OpBitwiseXor, state.uint_type, bits, ConstantU32(state, 0x80000000u));
return OrderedBits {cls.nan, cls.zero,
Select(state, state.uint_type, negative, negative_key, positive_key)};
};
const auto source_class = classify(source);
const auto old_class = classify(old);
const auto unordered =
Binary(state, OpLogicalOr, state.bool_type,
Binary(state, OpLogicalOr, state.bool_type, source_class.nan, old_class.nan),
Binary(state, OpLogicalAnd, state.bool_type, source_class.zero, old_class.zero));
const auto ordered = Unary(state, OpLogicalNot, state.bool_type, unordered);
const auto compare = Binary(state, max_value ? OpUGreaterThan : OpULessThan, state.bool_type,
source_class.key, old_class.key);
return Select(state, state.uint_type,
Binary(state, OpLogicalAnd, state.bool_type, ordered, compare), source, old);
}
uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
const auto pointer = Pointer(ctx, mem, index, pc);
const auto source = ctx.Arg(inst, inst.NumArgs() - 2);
return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) {
return FloatAtomicReplacement(ctx.state, old, source, max_value);
});
});
});
}
uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, ctx.state.uint_type, data, ctx.scratch_u32_variable});
AtomicUpdate(ctx.state, Pointer(ctx, mem, index, pc), mem.kind, [&](uint32_t old) {
const auto old_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, old);
const auto compare_f =
Unary(ctx.state, OpBitcast, ctx.state.float_type, ctx.Arg(inst, 3));
const auto data_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, data);
const auto compare = Binary(
ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, ctx.state.bool_type,
max_value ? old_f : compare_f, max_value ? compare_f : old_f);
return Unary(ctx.state, OpBitcast, ctx.state.uint_type,
Select(ctx.state, ctx.state.float_type, compare, data_f, old_f));
});
});
});
return 0;
}
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) {
auto& state = ctx.state;
const auto m0 = ctx.Arg(inst, 1);
const auto base =
Binary(state, OpShiftRightLogical, state.uint_type, m0, ConstantU32(state, 16));
const auto size = Binary(state, OpBitwiseAnd, state.uint_type, m0, ConstantU32(state, 0xffffu));
const auto address =
Binary(state, OpIAdd, state.uint_type, base, ConstantU32(state, ctx.Memory(inst).offset));
const auto index =
Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2));
const auto exec = ctx.Arg(inst, 2);
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), exec});
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 3);
const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4);
const auto count =
Binary(state, OpIAdd, state.uint_type, Unary(state, OpBitCount, state.uint_type, count_low),
Unary(state, OpBitCount, state.uint_type, count_high));
const auto first = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first,
ConstantU32(state, ScopeSubgroup), ballot});
const auto is_first =
Binary(state, OpIEqual, state.bool_type, EmitSubgroupLocalInvocationId(state), first);
const auto storage_bounds = gds ? EmitGdsElementInBounds(state, index) : 0u;
const auto m0_bounds =
gds ? Binary(state, OpINotEqual, state.bool_type, size, ConstantU32(state, 0))
: Binary(state, OpULessThan, state.bool_type,
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{append ? OpAtomicIAdd : OpAtomicISub, state.uint_type, value,
gds ? EmitGdsElementPointer(state, index) : EmitLdsElementPointer(state, index),
ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
ConstantU32(state, MemorySemanticsNone), count});
return value;
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, result,
ConstantU32(state, ScopeSubgroup), atomic, first});
return result;
}
} // namespace
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto op = inst.GetOpcode();
if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
ctx.Memory(inst).planning_only) {
return true;
}
if (op == IR::ValueOpcode::ReadConst) {
if (state.flattened_srt_variable == 0) {
ctx.Fail(inst, "requires the flattened SRT descriptor");
return true;
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
state.flattened_srt_variable, ConstantU32(state, 0),
ctx.Arg(inst, 1)});
ctx.Emit(inst, OpLoad, IR::Type::U32, {pointer});
return true;
}
if (op == IR::ValueOpcode::ReadConstBuffer) {
auto mem = ctx.Memory(inst);
mem.kind = IR::ResourceKind::ScalarBuffer;
const auto address = Binary(state, OpIAdd, state.uint_type, ctx.Arg(inst, 1),
ConstantU32(state, mem.offset));
const auto index =
Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2));
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
const auto condition = EmitStorageBufferElementInBounds(state, mem, index, pc);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, state.uint_type, value,
EmitStorageBufferElementPointer(state, mem, index, pc)});
return value;
}));
return true;
}
const bool load_address = op == IR::ValueOpcode::LoadAddressU8 ||
op == IR::ValueOpcode::LoadAddressU16 ||
op == IR::ValueOpcode::LoadAddressU32;
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadBufferU32;
const bool load_shared =
op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 ||
op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 ||
op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32;
if (load_address || load_buffer || load_shared) {
auto mem = ctx.Memory(inst);
if (load_shared)
mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 ||
op == IR::ValueOpcode::LoadGdsU32
? IR::ResourceKind::Gds
: IR::ResourceKind::Lds;
uint32_t value = 0;
if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
value = FormattedLoad(ctx, inst, mem);
else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8)
value = LoadSubword(ctx, inst, mem, 8, false);
else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16)
value = LoadSubword(ctx, inst, mem, 16, false);
else
value = LoadWord(ctx, inst, mem);
ctx.Define(inst, value);
return true;
}
const bool store_address = op == IR::ValueOpcode::StoreAddressU8 ||
op == IR::ValueOpcode::StoreAddressU16 ||
op == IR::ValueOpcode::StoreAddressU32;
const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::StoreBufferU32;
const bool store_shared =
op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 ||
op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 ||
op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32;
if (store_address || store_buffer || store_shared) {
auto mem = ctx.Memory(inst);
if (store_shared)
mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 ||
op == IR::ValueOpcode::WriteGdsU32
? IR::ResourceKind::Gds
: IR::ResourceKind::Lds;
if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
FormattedStore(ctx, inst, mem);
else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8)
StoreSubword(ctx, inst, mem, 8);
else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16)
StoreSubword(ctx, inst, mem, 16);
else
StoreWord(ctx, inst, mem);
return true;
}
const auto atomic_opcode = AtomicOpcode(op);
if (atomic_opcode != 0) {
auto mem = ctx.Memory(inst);
if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32)
mem.kind = IR::ResourceKind::Lds;
else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32)
mem.kind = IR::ResourceKind::Gds;
ctx.Define(inst, EmitAtomic(ctx, inst, mem, true));
return true;
}
if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst),
op == IR::ValueOpcode::BufferAtomicFMax32));
return true;
}
if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 ||
op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) {
auto mem = ctx.Memory(inst);
mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32
? IR::ResourceKind::Gds
: IR::ResourceKind::Lds;
SharedFloatAtomic(ctx, inst, mem,
op == IR::ValueOpcode::SharedAtomicFMax32 ||
op == IR::ValueOpcode::GdsAtomicFMax32);
return true;
}
if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume ||
op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) {
ctx.Define(inst, AppendConsume(ctx, inst,
op == IR::ValueOpcode::GdsDataAppend ||
op == IR::ValueOpcode::GdsDataConsume,
op == IR::ValueOpcode::DataAppend ||
op == IR::ValueOpcode::GdsDataAppend));
return true;
}
if (op == IR::ValueOpcode::SwizzleU32) {
ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)});
const auto source = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{OpLoad, ctx.state.uint_type, source, ctx.scratch_u32_variable});
const auto target =
EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), source, target});
const auto source_exec = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 2), target});
const auto source_active =
AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target));
ctx.Define(inst,
Select(state, state.uint_type, source_active, shuffled, ConstantU32(state, 0)));
return true;
}
return false;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/emitter/spirvEmitterValues.cpp → src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp
+3 -2
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -190,7 +190,8 @@ uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
uint32_t value = 0;
switch (operand.kind) {
case IR::OperandKind::ImmediateU32:
case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
case IR::OperandKind::PcRelativeU32:
case IR::OperandKind::PcRelativeHighU32: value = ConstantU32(state, operand.imm); break;
case IR::OperandKind::Register: {
const auto pointer = PointerForRegister(state, operand.reg);
if (pointer == 0) {
src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp
-525
View File
@@ -1,525 +0,0 @@
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cstdio>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
namespace {
bool Fail(std::string* error, const std::string& message) {
if (error != nullptr) {
*error = message;
}
return false;
}
bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
using Kind = IR::DescriptorBindingKind;
using Dim = Decoder::ImageDimension;
if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
const bool integer = image.kind == IR::ResourceKind::ImageUint;
switch (image.dimension) {
case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
case Dim::Dim1DArray:
kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
return true;
case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
case Dim::Dim2DMsaa:
kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
return true;
case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
case Dim::Dim2DArray:
kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
return true;
case Dim::Dim2DMsaaArray:
kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
return true;
case Dim::Unknown: return false;
}
}
const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
return false;
}
switch (image.dimension) {
case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
case Dim::Dim1DArray:
kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
return true;
case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
case Dim::Dim2DArray:
kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
return true;
case Dim::Dim2DMsaa:
case Dim::Dim2DMsaaArray: return false;
case Dim::Unknown: return false;
}
return false;
}
bool NeedsSampler(IR::Opcode op) {
return op == IR::Opcode::ImageSample || op == IR::Opcode::ImageGather4 ||
op == IR::Opcode::ImageGetLod;
}
bool IsBufferLoad(IR::Opcode op) {
switch (op) {
case IR::Opcode::BufferLoadUbyte:
case IR::Opcode::BufferLoadSbyte:
case IR::Opcode::BufferLoadUshort:
case IR::Opcode::BufferLoadSshort:
case IR::Opcode::BufferLoadDword: return true;
default: return false;
}
}
bool IsBufferStore(IR::Opcode op) {
return op == IR::Opcode::BufferStoreByte || op == IR::Opcode::BufferStoreShort ||
op == IR::Opcode::BufferStoreDword;
}
bool IsFlatLoad(IR::Opcode op) {
return op == IR::Opcode::FlatLoadUbyte || op == IR::Opcode::FlatLoadSbyte ||
op == IR::Opcode::FlatLoadUshort || op == IR::Opcode::FlatLoadSshort ||
op == IR::Opcode::FlatLoadDword;
}
bool IsFlatStore(IR::Opcode op) {
return op == IR::Opcode::FlatStoreByte || op == IR::Opcode::FlatStoreShort ||
op == IR::Opcode::FlatStoreDword;
}
bool IsDsKind(IR::ResourceKind kind) {
return kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds;
}
bool IsDsRead(IR::Opcode op) {
switch (op) {
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32: return true;
default: return false;
}
}
bool IsDsWrite(IR::Opcode op) {
switch (op) {
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32: return true;
default: return false;
}
}
bool IsAtomic(IR::Opcode op) {
switch (op) {
case IR::Opcode::AtomicSwapU32:
case IR::Opcode::AtomicAddU32:
case IR::Opcode::AtomicSubU32:
case IR::Opcode::AtomicSMinI32:
case IR::Opcode::AtomicUMinU32:
case IR::Opcode::AtomicSMaxI32:
case IR::Opcode::AtomicUMaxU32:
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32:
case IR::Opcode::AtomicFMinF32:
case IR::Opcode::AtomicFMaxF32: return true;
default: return false;
}
}
uint32_t BufferAddressOperandCount(const IR::Instruction& inst) {
return 1u + (inst.memory.idxen ? 1u : 0u) + (inst.memory.offen ? 1u : 0u);
}
bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error) {
if (static_cast<size_t>(inst.op) >= static_cast<size_t>(IR::Opcode::Count)) {
return Fail(error, "IR instruction opcode is out of range");
}
if (inst.src_count > std::size(inst.src)) {
return Fail(error, "instruction source count exceeds fixed IR storage");
}
const auto kind = inst.memory.kind;
const auto flat_kind = kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global ||
kind == IR::ResourceKind::Scratch;
const auto buffer_address_count = BufferAddressOperandCount(inst);
if ((IsBufferLoad(inst.op) &&
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count)) ||
(IsBufferStore(inst.op) &&
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) ||
(inst.op == IR::Opcode::SLoadDword &&
(kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
(inst.op == IR::Opcode::SBufferLoadDword &&
(kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
(IsFlatLoad(inst.op) && (!flat_kind || inst.src_count != 2)) ||
(IsFlatStore(inst.op) && (!flat_kind || inst.src_count != 3))) {
return Fail(error, "memory instruction opcode/kind/operand contract is invalid");
}
const auto ds_kind = IsDsKind(kind);
const auto ds_resource = !ds_kind || inst.memory.resource == 0;
if ((IsDsRead(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register)) ||
(IsDsWrite(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 2 ||
inst.dst.kind != IR::OperandKind::Null)) ||
((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
(!ds_kind || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register)) ||
((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
(!ds_kind || !ds_resource || inst.src_count != 3 ||
inst.dst.kind != IR::OperandKind::Null)) ||
(inst.op == IR::Opcode::DsWriteAddtidB32 &&
(kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 2 ||
inst.dst.kind != IR::OperandKind::Null)) ||
(inst.op == IR::Opcode::DsReadAddtidB32 &&
(kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register)) ||
(inst.op == IR::Opcode::DsSwizzleB32 &&
(kind != IR::ResourceKind::None || inst.src_count != 2 ||
inst.dst.kind != IR::OperandKind::Register))) {
return Fail(error, "DS instruction has an invalid resource/operand contract");
}
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4 ||
inst.op == IR::Opcode::ImageGetLod || inst.op == IR::Opcode::ImageLoad ||
inst.op == IR::Opcode::ImageGetResinfo) &&
((kind != IR::ResourceKind::Image && kind != IR::ResourceKind::ImageUint) ||
inst.src_count != 1)) {
return Fail(error, "sampled image instruction has an invalid resource class");
}
if (inst.op == IR::Opcode::ImageGather4 &&
(inst.memory.image_dimension == Decoder::ImageDimension::Dim1D ||
inst.memory.image_dimension == Decoder::ImageDimension::Dim1DArray)) {
return Fail(error, "1D image gather is not supported by SPIR-V");
}
if (inst.op == IR::Opcode::ImageStore &&
((kind != IR::ResourceKind::StorageImage && kind != IR::ResourceKind::StorageImageUint) ||
inst.src_count != 2)) {
return Fail(error, "storage image instruction has an invalid resource class");
}
const auto float_atomic =
inst.op == IR::Opcode::AtomicFMinF32 || inst.op == IR::Opcode::AtomicFMaxF32;
if (float_atomic &&
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) {
return Fail(error, "floating-point atomic instruction must target a buffer");
}
if (IsAtomic(inst.op) &&
!((kind == IR::ResourceKind::Buffer && inst.src_count == buffer_address_count + 1u) ||
((kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds) &&
inst.memory.resource == 0 && inst.src_count == 2 &&
(inst.dst.kind == IR::OperandKind::Null ||
inst.dst.kind == IR::OperandKind::Register)) ||
(kind == IR::ResourceKind::StorageImageUint && inst.src_count == 2))) {
return Fail(error, "atomic instruction has an invalid resource/operand contract");
}
return true;
}
bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
using Kind = IR::DescriptorBindingKind;
constexpr auto KindCount = static_cast<size_t>(Kind::Count);
std::array<std::vector<uint32_t>, KindCount> expected;
std::array<bool, KindCount> present {};
const auto Dense = [](size_t size) {
std::vector<uint32_t> values(size);
for (uint32_t i = 0; i < values.size(); i++) {
values[i] = i;
}
return values;
};
auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
const auto index = static_cast<size_t>(kind);
present[index] = true;
expected[index] = std::move(resources);
};
if (!program.info.buffers.empty()) {
Expect(Kind::Buffers, Dense(program.info.buffers.size()));
}
for (uint32_t i = 0; i < program.info.images.size(); i++) {
Kind kind;
if (!ImageBinding(program.info.images[i], kind)) {
return Fail(error, "native shader plan has an invalid image class");
}
present[static_cast<size_t>(kind)] = true;
expected[static_cast<size_t>(kind)].push_back(i);
}
if (!program.info.samplers.empty()) {
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
}
const auto uses_gds =
std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) {
return std::any_of(
block.instructions.begin(), block.instructions.end(),
[](const auto& inst) { return inst.memory.kind == IR::ResourceKind::Gds; });
});
if (uses_gds) {
Expect(Kind::Gds);
}
if (!program.info.addresses.empty()) {
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
}
if (!program.srt.reads.empty()) {
Expect(Kind::FlattenedSrt);
}
if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
Expect(Kind::UserData);
}
std::array<bool, KindCount> seen {};
for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
const auto& binding = program.bindings.descriptors[i];
const auto kind = static_cast<size_t>(binding.kind);
if (kind >= KindCount || seen[kind] || !present[kind] ||
binding.resources != expected[kind]) {
return Fail(error, "native descriptor groups do not match shader topology");
}
for (uint32_t j = 0; j < i; j++) {
if (program.bindings.descriptors[j].binding == binding.binding) {
return Fail(error, "native descriptor binding numbers are not unique");
}
}
seen[kind] = true;
}
for (size_t i = 0; i < KindCount; i++) {
if (present[i] != seen[i]) {
return Fail(error, "native shader plan is missing a required descriptor group");
}
}
const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
program.bindings.buffer_offset_count != program.info.buffers.size() ||
(!has_user_storage &&
program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
(has_user_storage && program.bindings.push_constant_size != 0) ||
!std::is_sorted(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) ||
std::adjacent_find(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) !=
program.bindings.user_data_registers.end()) {
return Fail(error, "native user-data layout is inconsistent");
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (!ValidateInstructionContract(inst, error)) {
return false;
}
const auto memory = inst.memory.kind;
if ((memory == IR::ResourceKind::Buffer || (memory == IR::ResourceKind::ScalarBuffer &&
inst.op == IR::Opcode::SBufferLoadDword)) &&
inst.memory.resource >= program.info.buffers.size()) {
return Fail(error, "buffer instruction has an invalid dense resource");
}
if ((memory == IR::ResourceKind::Image || memory == IR::ResourceKind::ImageUint ||
memory == IR::ResourceKind::StorageImage ||
memory == IR::ResourceKind::StorageImageUint) &&
(inst.memory.resource >= program.info.images.size() ||
program.info.images[inst.memory.resource].kind != memory ||
program.info.images[inst.memory.resource].dimension !=
inst.memory.image_dimension)) {
return Fail(error, "image instruction has an invalid dense resource");
}
const bool address =
inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat ||
memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch;
if (address && (inst.memory.resource >= program.info.addresses.size() ||
program.info.addresses[inst.memory.resource].kind != memory)) {
return Fail(error, "address instruction has an invalid dense resource");
}
if (NeedsSampler(inst.op) && inst.memory.sampler >= program.info.samplers.size()) {
return Fail(error, "sample instruction has an invalid dense sampler");
}
if (inst.op == IR::Opcode::LoadSrtDword &&
(inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32 ||
inst.src[0].imm >= program.srt.reads.size())) {
return Fail(error, "flattened SRT instruction has an invalid dense slot");
}
}
}
return true;
}
} // namespace
static constexpr size_t InitialEmitterVectorReserve = 4096;
static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
size_t size) {
if (size <= InitialEmitterVectorReserve) {
return;
}
std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
" stage=%u vector=%s size=%zu reserve=%zu\n",
program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
InitialEmitterVectorReserve);
}
static bool IsExecDestination(const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Exec;
}
static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruction& inst) {
if (inst.op != IR::Opcode::BitFieldMaskU64 || !IsExecDestination(inst.dst) ||
inst.src_count < 1u) {
return false;
}
uint32_t count = 0;
return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], count) &&
(count & 63u) == 0u;
}
static bool ProducesLaneMask(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::IAddCarryU32:
case IR::Opcode::ISubBorrowU32:
case IR::Opcode::UMadU64U32: return inst.dst2.kind != IR::OperandKind::Null;
default: return false;
}
}
bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.terminator.kind == CFG::TerminatorKind::ConditionalBranch) {
switch (block.terminator.condition) {
case CFG::BranchCondition::VccZero:
case CFG::BranchCondition::VccNonZero:
case CFG::BranchCondition::ExecZero:
case CFG::BranchCondition::ExecNonZero: return true;
default: break;
}
}
for (const auto& inst: block.instructions) {
if (Emitter::InstructionHasDppSource(inst)) {
return true;
}
if (IR::IsCompareOpcode(inst.op) && !Emitter::IsSccOperand(inst.dst)) {
return true;
}
if ((IsExecDestination(inst.dst) || IsExecDestination(inst.dst2)) &&
!IsUniformZeroExecWrite(program, inst)) {
return true;
}
if (ProducesLaneMask(inst)) {
return true;
}
if (inst.op == IR::Opcode::SelectMaskU32 || inst.op == IR::Opcode::SelectMaskF32Bits) {
return true;
}
if (inst.op == IR::Opcode::SaveexecB32 || inst.op == IR::Opcode::SaveexecB64) {
return true;
}
switch (inst.op) {
case IR::Opcode::WqmB64:
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file != IR::RegisterFile::Scalar) {
return true;
}
break;
case IR::Opcode::MaskedBitCountLowU32:
case IR::Opcode::MaskedBitCountHighU32:
case IR::Opcode::ReadFirstLaneU32:
case IR::Opcode::ReadLaneU32:
case IR::Opcode::WriteLaneU32:
case IR::Opcode::Permlane16B32:
case IR::Opcode::Permlanex16B32:
case IR::Opcode::DsSwizzleB32:
case IR::Opcode::DsConsume:
case IR::Opcode::DsAppend: return true;
default: break;
}
}
}
return false;
}
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
std::string* error) {
using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) {
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
return false;
}
if (!program.srt_patching_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete) {
SetError(error, "SPIR-V emitter requires a fully planned native shader program");
return false;
}
if (!IR::ValidateResourceSnapshot(program, resources, error)) {
return false;
}
if (!IR::ValidateResourceSpecialization(program, resources, error)) {
return false;
}
if (!ValidateNativeProgram(program, error)) {
return false;
}
EmitterState state(program, resources);
state.vertex_input_info = vertex_input_info;
state.pixel_input_info = pixel_input_info;
state.compute_input_info = compute_input_info;
state.stage = program.stage;
state.wave_size = program.wave_size;
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
state.exact_subgroup_operations =
!state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
state.registers.reserve(InitialEmitterVectorReserve);
state.inputs.reserve(InitialEmitterVectorReserve);
state.outputs.reserve(InitialEmitterVectorReserve);
state.interface_variables.reserve(InitialEmitterVectorReserve);
state.reachable_blocks.reserve(InitialEmitterVectorReserve);
CollectRegisters(program, state.registers);
CopyProgramInputsAndOutputs(state, program);
state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program);
state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program);
state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program);
state.needs_function_lds = ProgramNeedsFunctionLds(program);
state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program);
ComputeReachableBlocks(state, program);
AllocateInputVariables(state);
AllocateOutputVariables(state);
AllocateDescriptorVariables(state);
EmitHeaderAndTypes(state);
AllocateRegisterVariables(state);
AllocateBlockLabels(state, program);
AllocateDispatcherState(state, program);
EmitFunction(state, program);
if (state.unsupported_ir_instruction) {
char pc[9] {};
std::snprintf(pc, sizeof(pc), "%08x", state.unsupported_ir_pc);
return Fail(error, "SPIR-V emitter has no handler for IR opcode " +
std::string(IR::OpcodeName(state.unsupported_ir_opcode)) +
" at pc 0x" + pc);
}
ReportReserveExceeded(program, "registers", state.registers.size());
ReportReserveExceeded(program, "inputs", state.inputs.size());
ReportReserveExceeded(program, "outputs", state.outputs.size());
ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
auto binary = state.builder.Build();
if (binary.empty()) {
SetError(error, "SPIR-V builder returned an empty module");
return false;
}
spirv = std::move(binary);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp
-1138
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/cfg/ShaderCFG.cpp → src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp
+330 -131
View File
@@ -1,6 +1,7 @@
#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
#include <algorithm>
#include <array>
#include <fmt/format.h>
#include <iterator>
#include <map>
@@ -888,6 +889,30 @@ uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) {
return graph.blocks.back().id;
}
bool IsolateSemanticLoopHeader(Graph& graph, uint32_t old_header) {
const auto* header = graph.FindBlock(old_header);
if (header == nullptr || header->inst_begin == header->inst_end) {
return false;
}
const auto predecessors = header->predecessors;
const auto new_header = AppendSyntheticBranchBlock(graph, old_header);
for (auto pred: predecessors) {
auto* block = graph.FindBlock(pred);
if (block != nullptr) {
ReplaceValue(block->successors, old_header, new_header);
ReplaceTerminatorTarget(block->terminator, old_header, new_header);
}
}
if (graph.entry_block == old_header) {
graph.entry_block = new_header;
}
MoveBlockBefore(graph, new_header, old_header);
RebuildPredecessors(graph);
RecomputeAnalyses(graph);
return true;
}
bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t merge) {
const auto* block = graph.FindBlock(block_id);
return block != nullptr && block->inst_begin == block->inst_end &&
@@ -913,6 +938,43 @@ bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t
graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id);
}
bool IsLoopControlGateway(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
const auto* block = graph.FindBlock(block_id);
if (block == nullptr || block->terminator.kind != TerminatorKind::ConditionalBranch) {
return false;
}
const auto is_control_target = [&](uint32_t target) {
return target == loop.merge || target == loop.continue_block;
};
return is_control_target(block->terminator.true_block) &&
is_control_target(block->terminator.false_block);
}
uint32_t FindSelectionMerge(const Graph& graph, const BasicBlock& block) {
const auto global_merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
block.terminator.false_block);
const auto* loop = FindInnermostContainingLoop(graph, block.id);
if (loop == nullptr) {
return global_merge;
}
// A selection can join at the loop-control gateway even when another arm breaks directly to
// the enclosing loop merge. Global post-dominance sees that break and incorrectly expands the
// selection through the backedge.
const auto true_target = block.terminator.true_block;
const auto false_target = block.terminator.false_block;
if (IsLoopControlGateway(graph, *loop, true_target) && graph.Dominates(block.id, true_target) &&
IsInsideLoopConstruct(graph, *loop, false_target)) {
return true_target;
}
if (IsLoopControlGateway(graph, *loop, false_target) &&
graph.Dominates(block.id, false_target) &&
IsInsideLoopConstruct(graph, *loop, true_target)) {
return false_target;
}
return global_merge;
}
bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) {
if (block.terminator.kind != TerminatorKind::ConditionalBranch) {
return false;
@@ -993,22 +1055,7 @@ bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
continue;
}
const auto old_header = loop.header;
const auto predecessors = header->predecessors;
const auto new_header = AppendSyntheticBranchBlock(graph, old_header);
for (auto pred: predecessors) {
auto* block = graph.FindBlock(pred);
if (block != nullptr) {
ReplaceValue(block->successors, old_header, new_header);
ReplaceTerminatorTarget(block->terminator, old_header, new_header);
}
}
if (graph.entry_block == old_header) {
graph.entry_block = new_header;
}
MoveBlockBefore(graph, new_header, old_header);
RebuildPredecessors(graph);
RecomputeAnalyses(graph);
IsolateSemanticLoopHeader(graph, loop.header);
changed = true;
break;
}
@@ -1101,12 +1148,13 @@ bool SplitOneLoopMerge(Graph& graph) {
std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& header,
uint32_t merge) {
std::vector<uint32_t> region;
std::vector<uint32_t> pending = {header.terminator.true_block,
header.terminator.false_block};
std::vector<uint32_t> pending = {header.terminator.true_block, header.terminator.false_block};
const auto* loop = FindInnermostContainingLoop(graph, header.id);
while (!pending.empty()) {
const auto block_id = pending.back();
pending.pop_back();
if (block_id == merge || Contains(region, block_id)) {
if (block_id == merge || Contains(region, block_id) ||
(loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block))) {
continue;
}
const auto* block = graph.FindBlock(block_id);
@@ -1120,80 +1168,7 @@ std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& head
return region;
}
bool DuplicateSelectionRegion(Graph& graph, uint32_t header_id, uint32_t merge,
const std::vector<uint32_t>& region, uint32_t block_budget) {
std::vector<uint32_t> cloned_blocks;
for (auto block_id: region) {
if (!graph.Dominates(header_id, block_id)) {
cloned_blocks.push_back(block_id);
}
}
if (cloned_blocks.empty() || graph.FindBlock(header_id) == nullptr || header_id >= merge ||
graph.blocks.size() + cloned_blocks.size() + 1u > block_budget) {
return false;
}
const auto first_clone = static_cast<uint32_t>(graph.blocks.size());
std::map<uint32_t, uint32_t> clones;
for (uint32_t i = 0; i < cloned_blocks.size(); i++) {
clones.emplace(cloned_blocks[i], first_clone + i);
}
for (auto block_id: cloned_blocks) {
BasicBlock clone = *graph.FindBlock(block_id);
clone.id = clones.at(block_id);
clone.predecessors.clear();
clone.dominators.clear();
clone.post_dominators.clear();
graph.blocks.push_back(std::move(clone));
}
const auto remap_block = [&](BasicBlock& block) {
const auto remap_target = [&](uint32_t& target) {
if (const auto it = clones.find(target); it != clones.end()) {
target = it->second;
}
};
for (auto& successor: block.successors) {
remap_target(successor);
}
remap_target(block.terminator.true_block);
remap_target(block.terminator.false_block);
remap_target(block.terminator.merge_block);
remap_target(block.terminator.continue_block);
for (auto& target: block.terminator.indirect_targets) {
remap_target(target);
}
};
for (auto block_id: region) {
const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
remap_block(*graph.FindBlock(owned_id));
}
const auto private_merge = AppendSyntheticBranchBlock(graph, merge);
auto& header = *graph.FindBlock(header_id);
remap_block(header);
for (auto block_id: region) {
const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
auto* block = graph.FindBlock(owned_id);
if (block != nullptr) {
ReplaceValue(block->successors, merge, private_merge);
ReplaceTerminatorTarget(block->terminator, merge, private_merge);
}
}
ReplaceValue(header.successors, merge, private_merge);
ReplaceTerminatorTarget(header.terminator, merge, private_merge);
for (uint32_t i = 0; i <= cloned_blocks.size(); i++) {
MoveBlockBefore(graph, first_clone + i, merge + i);
}
RebuildPredecessors(graph);
RecomputeAnalyses(graph);
return true;
}
bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) {
bool SplitOneSelectionMerge(Graph& graph, std::string* error) {
std::vector<uint32_t> loop_headers;
loop_headers.reserve(graph.natural_loops.size());
for (const auto& loop: graph.natural_loops) {
@@ -1211,19 +1186,22 @@ bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) {
continue;
}
const auto merge = graph.FindNearestCommonPostDominator(block->terminator.true_block,
block->terminator.false_block);
const auto merge = FindSelectionMerge(graph, *block);
if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) {
continue;
}
const auto region = SelectionRegion(graph, *block, merge);
if (std::any_of(region.begin(), region.end(),
[&](uint32_t member) { return !graph.Dominates(block_id, member); })) {
if (graph.natural_loops.empty() &&
DuplicateSelectionRegion(graph, block_id, merge, region, block_budget)) {
return true;
}
continue;
const auto region = SelectionRegion(graph, *block, merge);
const auto external = std::find_if(region.begin(), region.end(), [&](uint32_t member) {
return !graph.Dominates(block_id, member);
});
if (external != region.end()) {
SetFailure(
graph, FailureKind::StructuredControlFlow, block_id,
fmt::format("selection header block {} has externally entered region block {}; "
"semantic block cloning is disabled",
block_id, *external),
error);
return false;
}
const auto construct_blocks = DominatedBlocks(graph, block_id, merge);
const auto force_split = MergeLeavesContainingLoop(graph, block_id, merge);
@@ -1236,13 +1214,11 @@ bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) {
bool SplitSharedMergeBlocks(Graph& graph, std::string* error) {
const auto original_block_count = static_cast<uint32_t>(graph.blocks.size());
const auto split_budget = std::max<uint32_t>(
16u, std::min<uint32_t>(128u, original_block_count * 4u));
const auto block_budget = std::max<uint32_t>(
32u, std::min<uint32_t>(512u, original_block_count * 8u));
const auto split_budget =
std::max<uint32_t>(16u, std::min<uint32_t>(128u, original_block_count * 4u));
for (uint32_t splits = 0; splits < split_budget; splits++) {
if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, block_budget)) {
return true;
if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, error)) {
return !graph.unsupported;
}
RebuildPredecessors(graph);
RecomputeAnalyses(graph);
@@ -1339,6 +1315,182 @@ uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_
return common.empty() ? UINT32_MAX : common.front();
}
namespace {
struct GotoRouteBlocks {
uint32_t before = UINT32_MAX;
std::array<uint32_t, 5> blocks {};
};
void PlaceGotoRouteBlocks(Graph& graph, uint32_t original_block_count,
const GotoRouteBlocks& route) {
std::vector<BasicBlock> old_blocks = std::move(graph.blocks);
std::vector<BasicBlock> new_blocks;
new_blocks.reserve(old_blocks.size());
for (uint32_t block_id = 0; block_id < original_block_count; block_id++) {
if (block_id == route.before) {
for (const auto route_block: route.blocks) {
new_blocks.push_back(std::move(old_blocks[route_block]));
}
}
new_blocks.push_back(std::move(old_blocks[block_id]));
}
std::vector<uint32_t> id_map(new_blocks.size(), UINT32_MAX);
for (uint32_t block_id = 0; block_id < new_blocks.size(); block_id++) {
id_map[new_blocks[block_id].id] = block_id;
}
graph.blocks = std::move(new_blocks);
graph.entry_block = RemapId(graph.entry_block, id_map);
for (auto& block: graph.blocks) {
block.id = RemapId(block.id, id_map);
RemapIds(block.predecessors, id_map);
RemapIds(block.successors, id_map);
RemapIds(block.dominators, id_map);
RemapIds(block.post_dominators, id_map);
block.terminator.true_block = RemapId(block.terminator.true_block, id_map);
block.terminator.false_block = RemapId(block.terminator.false_block, id_map);
block.terminator.merge_block = RemapId(block.terminator.merge_block, id_map);
block.terminator.continue_block = RemapId(block.terminator.continue_block, id_map);
}
}
uint32_t AppendGotoSelectBlock(Graph& graph, uint32_t route_variable, uint32_t true_target,
uint32_t false_target) {
const auto* target = graph.FindBlock(false_target);
BasicBlock block;
block.id = static_cast<uint32_t>(graph.blocks.size());
block.start_pc = target != nullptr ? target->start_pc : 0u;
block.end_pc = block.start_pc;
block.inst_begin = target != nullptr ? target->inst_begin : 0u;
block.inst_end = block.inst_begin;
block.successors = {true_target, false_target};
block.terminator.kind = TerminatorKind::ConditionalBranch;
block.terminator.condition = BranchCondition::GotoVariable;
block.terminator.true_block = true_target;
block.terminator.false_block = false_target;
block.terminator.goto_variable = route_variable;
graph.blocks.push_back(std::move(block));
return graph.blocks.back().id;
}
uint32_t AppendGotoSetBlock(Graph& graph, uint32_t route_variable, bool value, uint32_t target) {
const auto block_id = AppendSyntheticBranchBlock(graph, target);
auto* block = graph.FindBlock(block_id);
block->terminator.goto_variable = route_variable;
block->terminator.goto_value = value ? 1 : 0;
return block_id;
}
// Turn H0 -> shared/H1, H1 -> shared/other into nested selections whose empty
// forwarding blocks set one typed SSA route value. Guest semantic blocks remain unique.
bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t outer_id,
uint32_t route_variable, GotoRouteBlocks& route) {
auto* outer = graph.FindBlock(outer_id);
if (outer == nullptr || outer->inst_begin == outer->inst_end ||
outer->terminator.kind != TerminatorKind::ConditionalBranch ||
std::ranges::any_of(graph.natural_loops, [outer_id](const auto& loop) {
return Contains(loop.body_blocks, outer_id);
})) {
return false;
}
for (const bool inner_is_true: {true, false}) {
const auto inner_id =
inner_is_true ? outer->terminator.true_block : outer->terminator.false_block;
const auto shared =
inner_is_true ? outer->terminator.false_block : outer->terminator.true_block;
const auto* inner = graph.FindBlock(inner_id);
if (inner == nullptr || inner->inst_begin == inner->inst_end ||
inner->predecessors != std::vector<uint32_t> {outer_id} ||
inner->terminator.kind != TerminatorKind::ConditionalBranch) {
continue;
}
const bool true_is_shared = inner->terminator.true_block == shared;
const bool false_is_shared = inner->terminator.false_block == shared;
if (true_is_shared == false_is_shared) {
continue;
}
const auto other =
true_is_shared ? inner->terminator.false_block : inner->terminator.true_block;
if (other == outer_id || other == inner_id || other == shared) {
continue;
}
const auto first_arm = std::min(shared, other);
if (first_arm >= original_block_count || outer_id >= inner_id || inner_id >= first_arm) {
continue;
}
const auto* shared_block = graph.FindBlock(shared);
const auto* other_block = graph.FindBlock(other);
if (shared_block == nullptr || other_block == nullptr ||
shared_block->terminator.kind != TerminatorKind::Branch ||
other_block->terminator.kind != TerminatorKind::Branch ||
shared_block->successors.size() != 1u ||
shared_block->successors != other_block->successors) {
continue;
}
const auto in_loop = [&](uint32_t block_id) {
return std::ranges::any_of(graph.natural_loops, [&](const auto& loop) {
return Contains(loop.body_blocks, block_id);
});
};
if (in_loop(inner_id) || in_loop(shared) || in_loop(other)) {
continue;
}
const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared);
const auto inner_merge = AppendSyntheticBranchBlock(graph, route_select);
const auto outer_shared = AppendGotoSetBlock(graph, route_variable, false, route_select);
const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge);
const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge);
outer = graph.FindBlock(outer_id);
auto* mutable_inner = graph.FindBlock(inner_id);
outer->terminator.true_block = inner_is_true ? inner_id : outer_shared;
outer->terminator.false_block = inner_is_true ? outer_shared : inner_id;
outer->successors = {outer->terminator.true_block, outer->terminator.false_block};
mutable_inner->terminator.true_block = true_is_shared ? inner_shared : inner_other;
mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared;
mutable_inner->successors = {mutable_inner->terminator.true_block,
mutable_inner->terminator.false_block};
route = {first_arm, {inner_shared, inner_other, inner_merge, outer_shared, route_select}};
return true;
}
return false;
}
bool RouteTerminalSharedSelectionArm(Graph& graph) {
if (graph.irreducible) {
return false;
}
const auto original_block_count = static_cast<uint32_t>(graph.blocks.size());
GotoRouteBlocks route;
bool found = false;
for (uint32_t block_id = 0; block_id < original_block_count; block_id++) {
GotoRouteBlocks next;
if (RouteOneSharedArm(graph, original_block_count, block_id, 0u, next)) {
if (found) {
// Multiple interacting overlaps remain on the dispatcher path.
return false;
}
found = true;
route = next;
RebuildPredecessors(graph);
}
}
if (!found) {
return false;
}
PlaceGotoRouteBlocks(graph, original_block_count, route);
RebuildPredecessors(graph);
RecomputeAnalyses(graph);
return true;
}
} // namespace
bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error) {
graph = {};
@@ -1554,9 +1706,14 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
ComputeComponents(graph);
if (indirect_setpc) {
graph.irreducible = true;
graph.unsupported = false;
graph.failure_kind = FailureKind::IrreducibleControlFlow;
graph.irreducible = true;
graph.unsupported = false;
graph.failure_kind = FailureKind::IrreducibleControlFlow;
const auto indirect =
std::find_if(graph.blocks.begin(), graph.blocks.end(), [](const auto& block) {
return block.terminator.kind == TerminatorKind::IndirectBranch;
});
graph.failure_block = indirect != graph.blocks.end() ? indirect->id : graph.entry_block;
graph.unsupported_reason = "indirect S_SETPC_B64 jump table requires dispatcher fallback";
}
@@ -1567,7 +1724,9 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro
return true;
}
bool Structurize(Graph& graph, std::string* error) {
namespace {
bool StructurizeImpl(Graph& graph, std::string* error) {
if (graph.unsupported || graph.irreducible) {
if (graph.unsupported_reason.empty()) {
graph.unsupported_reason = "unsupported CFG";
@@ -1625,8 +1784,7 @@ bool Structurize(Graph& graph, std::string* error) {
continue;
}
const auto merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
block.terminator.false_block);
const auto merge = FindSelectionMerge(graph, block);
if (merge == UINT32_MAX) {
SetFailure(graph, FailureKind::StructuredControlFlow, block.id,
fmt::format("conditional block {} has no structured merge", block.id),
@@ -1643,6 +1801,57 @@ bool Structurize(Graph& graph, std::string* error) {
return true;
}
} // namespace
bool Structurize(Graph& graph, std::string* error) {
Graph original = graph;
if (StructurizeImpl(graph, error)) {
return true;
}
if (!RouteTerminalSharedSelectionArm(original)) {
return false;
}
Graph failed_graph = std::move(graph);
std::string failed_error = error != nullptr ? *error : std::string {};
graph = std::move(original);
if (error != nullptr) {
error->clear();
}
if (StructurizeImpl(graph, error)) {
return true;
}
graph = std::move(failed_graph);
if (error != nullptr) {
*error = std::move(failed_error);
}
return false;
}
bool IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error) {
const auto loop =
std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(),
[header_id](const auto& value) { return value.header == header_id; });
if (loop == graph.natural_loops.end()) {
SetFailure(graph, FailureKind::StructuredControlFlow, header_id,
fmt::format("block {} is not a natural loop header", header_id), error);
return false;
}
ClearStructuredTerminators(graph);
if (!IsolateSemanticLoopHeader(graph, header_id)) {
SetFailure(graph, FailureKind::StructuredControlFlow, header_id,
fmt::format("loop header block {} has no semantic instructions", header_id),
error);
return false;
}
graph.unsupported = false;
graph.failure_kind = FailureKind::None;
graph.failure_block = UINT32_MAX;
graph.unsupported_reason.clear();
return Structurize(graph, error);
}
std::string BranchConditionToString(BranchCondition condition) {
switch (condition) {
case BranchCondition::Always: return "always";
@@ -1652,6 +1861,7 @@ std::string BranchConditionToString(BranchCondition condition) {
case BranchCondition::VccNonZero: return "vccnz";
case BranchCondition::ExecZero: return "execz";
case BranchCondition::ExecNonZero: return "execnz";
case BranchCondition::GotoVariable: return "goto_variable";
default: return "unknown";
}
}
@@ -1670,19 +1880,6 @@ std::string FailureKindToString(FailureKind kind) {
}
}
std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage,
const std::string& reason) {
const auto* block = graph.FindBlock(block_id);
const char* stage_text = stage != nullptr ? stage : "unknown";
if (block == nullptr) {
return fmt::format("stage={} block={} reason={}", stage_text, block_id, reason.c_str());
}
return fmt::format("stage={} block={} pc=0x{:08x}..0x{:08x} preds={} succs={} reason={}",
stage_text, block->id, block->start_pc, block->end_pc,
static_cast<uint32_t>(block->predecessors.size()),
static_cast<uint32_t>(block->successors.size()), reason.c_str());
}
std::string GraphToString(const Graph& graph) {
std::string text;
text +=
@@ -1706,14 +1903,16 @@ std::string GraphToString(const Graph& graph) {
VectorToString(block.post_dominators).c_str());
text += fmt::format(
" terminator={} condition={} true={} false={} merge={} continue={} loop_header={} "
"indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}]\n",
"indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}] "
"goto_variable={} goto_value={}\n",
static_cast<uint32_t>(block.terminator.kind),
BranchConditionToString(block.terminator.condition).c_str(),
block.terminator.true_block, block.terminator.false_block, block.terminator.merge_block,
block.terminator.continue_block, block.terminator.loop_header ? 1u : 0u,
block.terminator.indirect_pc_sgpr, block.terminator.indirect_selector_code,
VectorToString(block.terminator.indirect_targets).c_str(),
VectorToString(block.terminator.indirect_selector_values).c_str());
VectorToString(block.terminator.indirect_selector_values).c_str(),
block.terminator.goto_variable, block.terminator.goto_value);
}
for (const auto& edge: graph.back_edges) {
src/graphics/shader/recompiler/cfg/ShaderCFG.h → src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h
+6 -4
View File
@@ -3,7 +3,7 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include <vector>
@@ -17,6 +17,7 @@ enum class BranchCondition {
VccNonZero,
ExecZero,
ExecNonZero,
GotoVariable,
Unknown
};
@@ -46,7 +47,9 @@ struct Terminator {
std::vector<uint32_t> indirect_targets;
std::vector<uint32_t> indirect_selector_values;
std::vector<uint32_t> indirect_selector_targets;
bool loop_header = false;
uint32_t goto_variable = UINT32_MAX;
int32_t goto_value = -1;
bool loop_header = false;
};
struct BasicBlock {
@@ -107,10 +110,9 @@ struct Graph {
bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error);
bool Structurize(Graph& graph, std::string* error);
bool IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error);
std::string BranchConditionToString(BranchCondition condition);
std::string FailureKindToString(FailureKind kind);
std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage,
const std::string& reason);
std::string GraphToString(const Graph& graph);
} // namespace Libs::Graphics::ShaderRecompiler::CFG
src/graphics/shader/recompiler/decompiler/ExportOps.cpp → src/graphics/shader/recompiler/frontend/decode/ExportOps.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/decompiler/ExportOps.h"
#include "graphics/shader/recompiler/frontend/decode/ExportOps.h"
#include <fmt/format.h>
src/graphics/shader/recompiler/decompiler/ExportOps.h → src/graphics/shader/recompiler/frontend/decode/ExportOps.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
src/graphics/shader/recompiler/decompiler/ImageOps.cpp → src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/decompiler/ImageOps.h"
#include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
#include <algorithm>
#include <fmt/format.h>
src/graphics/shader/recompiler/decompiler/ImageOps.h → src/graphics/shader/recompiler/frontend/decode/ImageOps.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
src/graphics/shader/recompiler/decompiler/MemoryOps.cpp → src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
#include "graphics/shader/recompiler/decompiler/MemoryOps.h"
#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h"
#include <fmt/format.h>
#include <iterator>
src/graphics/shader/recompiler/decompiler/MemoryOps.h → src/graphics/shader/recompiler/frontend/decode/MemoryOps.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
src/graphics/shader/recompiler/decompiler/OpcodeTable.h → src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include <cstddef>
src/graphics/shader/recompiler/decompiler/ScalarAluOps.cpp → src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp
+2 -2
View File
@@ -1,6 +1,6 @@
#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h"
#include "graphics/shader/recompiler/frontend/decode/ScalarAluOps.h"
#include "graphics/shader/recompiler/decompiler/OpcodeTable.h"
#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
namespace {
src/graphics/shader/recompiler/decompiler/ScalarAluOps.h → src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
src/graphics/shader/recompiler/decompiler/ShaderDecoder.cpp → src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp
+7 -6
View File
@@ -1,10 +1,10 @@
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "graphics/shader/recompiler/decompiler/ExportOps.h"
#include "graphics/shader/recompiler/decompiler/ImageOps.h"
#include "graphics/shader/recompiler/decompiler/MemoryOps.h"
#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h"
#include "graphics/shader/recompiler/decompiler/VectorAluOps.h"
#include "graphics/shader/recompiler/frontend/decode/ExportOps.h"
#include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h"
#include "graphics/shader/recompiler/frontend/decode/ScalarAluOps.h"
#include "graphics/shader/recompiler/frontend/decode/VectorAluOps.h"
#include <algorithm>
#include <bit>
@@ -556,6 +556,7 @@ std::string OpcodeToString(Opcode opcode) {
case Opcode::VCvtF32Ubyte2: return "v_cvt_f32_ubyte2";
case Opcode::VCvtF32Ubyte3: return "v_cvt_f32_ubyte3";
case Opcode::VRcpF32: return "v_rcp_f32";
case Opcode::VRcpIflagF32: return "v_rcp_iflag_f32";
case Opcode::VFractF32: return "v_fract_f32";
case Opcode::VTruncF32: return "v_trunc_f32";
case Opcode::VCeilF32: return "v_ceil_f32";
src/graphics/shader/recompiler/decompiler/ShaderDecoder.h → src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h
+24 -21
View File
@@ -154,6 +154,7 @@ enum class Opcode {
VCvtF32Ubyte2,
VCvtF32Ubyte3,
VRcpF32,
VRcpIflagF32,
VFractF32,
VTruncF32,
VCeilF32,
@@ -587,27 +588,29 @@ constexpr uint32_t MaxInstructionRawWords = 5u;
constexpr uint32_t MaxImageNsaAddressComponents = 12u;
struct Operand {
OperandKind kind = OperandKind::Unknown;
uint32_t value = 0;
int32_t signed_val = 0;
float float_val = 0.0f;
uint32_t reg = 0;
uint32_t sdwa_sel = 6;
uint32_t sdwa_dst_unused = 0;
uint32_t omod = 0;
uint32_t dpp_ctrl = 0;
uint32_t dpp_row_mask = 0xf;
uint32_t dpp_bank_mask = 0xf;
bool sdwa_sext = false;
bool dpp_fetch_inactive = false;
bool dpp_bound_ctrl = false;
bool op_sel = false;
bool op_sel_hi = false;
bool negate = false;
bool negate_hi = false;
bool absolute = false;
bool clamp = false;
bool dpp = false;
OperandKind kind = OperandKind::Unknown;
uint32_t value = 0;
int32_t signed_val = 0;
float float_val = 0.0f;
uint32_t reg = 0;
uint32_t sdwa_sel = 6;
// Native 16-bit destinations use the same selector fields internally but preserve the
// untouched half. Real SDWA encodings overwrite this with their decoded DST_U value.
uint32_t sdwa_dst_unused = 2;
uint32_t omod = 0;
uint32_t dpp_ctrl = 0;
uint32_t dpp_row_mask = 0xf;
uint32_t dpp_bank_mask = 0xf;
bool sdwa_sext = false;
bool dpp_fetch_inactive = false;
bool dpp_bound_ctrl = false;
bool op_sel = false;
bool op_sel_hi = false;
bool negate = false;
bool negate_hi = false;
bool absolute = false;
bool clamp = false;
bool dpp = false;
};
struct Instruction {
src/graphics/shader/recompiler/decompiler/VectorAluOps.cpp → src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp
+6 -4
View File
@@ -1,6 +1,6 @@
#include "graphics/shader/recompiler/decompiler/VectorAluOps.h"
#include "graphics/shader/recompiler/frontend/decode/VectorAluOps.h"
#include "graphics/shader/recompiler/decompiler/OpcodeTable.h"
#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h"
#include <fmt/format.h>
@@ -111,7 +111,7 @@ constexpr OpcodeMap VOP1_OPS[] = {
{0x24u, Opcode::VFloorF32},
{0x25u, Opcode::VExpF32},
{0x27u, Opcode::VLogF32},
{0x2bu, Opcode::VRcpF32},
{0x2bu, Opcode::VRcpIflagF32},
{0x2eu, Opcode::VRsqF32},
{0x33u, Opcode::VSqrtF32},
{0x35u, Opcode::VSinF32},
@@ -157,7 +157,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPS[] = {
{0x24u, Opcode::VFloorF32},
{0x25u, Opcode::VExpF32},
{0x27u, Opcode::VLogF32},
{0x2bu, Opcode::VRcpF32},
{0x2bu, Opcode::VRcpIflagF32},
{0x2eu, Opcode::VRsqF32},
{0x33u, Opcode::VSqrtF32},
{0x35u, Opcode::VSinF32},
@@ -397,6 +397,7 @@ bool IsVop1FloatSourceOpcode(Opcode opcode) {
case Opcode::VCvtFlrI32F32:
case Opcode::VCvtF16F32:
case Opcode::VRcpF32:
case Opcode::VRcpIflagF32:
case Opcode::VFractF32:
case Opcode::VTruncF32:
case Opcode::VCeilF32:
@@ -743,6 +744,7 @@ bool IsVop1FloatResultOpcode(Opcode opcode) {
case Opcode::VCvtF32Ubyte2:
case Opcode::VCvtF32Ubyte3:
case Opcode::VRcpF32:
case Opcode::VRcpIflagF32:
case Opcode::VFractF32:
case Opcode::VTruncF32:
case Opcode::VCeilF32:
src/graphics/shader/recompiler/decompiler/VectorAluOps.h → src/graphics/shader/recompiler/frontend/decode/VectorAluOps.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
src/graphics/shader/recompiler/frontend/translate/Attribute.cpp
+35
View File
@@ -0,0 +1,35 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include <algorithm>
#include <array>
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) {
const auto index = static_cast<uint32_t>(value_program.export_info.size());
value_program.export_info.push_back(inst.export_info);
return {.index = index, .pc = inst.pc};
}
bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::LoadInputF32) {
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::GetAttribute,
{IR::Value(inst.input_info.attr), IR::Value(inst.input_info.chan)}));
return true;
}
if (inst.op == IR::Opcode::Export) {
std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u),
IR::Value(0u)};
for (uint32_t index = 0; index < std::min(inst.src_count, 4u); index++) {
components[index] = ReadRawU32(PlainOperand(inst.src[index]));
}
const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
{components[0], components[1], components[2], components[3]});
ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
return true;
}
return false;
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Compare.cpp
+195
View File
@@ -0,0 +1,195 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateIntegerCompare(const IR::Instruction& inst) {
IR::ValueOpcode opcode {};
IR::Type type = IR::Type::U32;
switch (inst.op) {
case IR::Opcode::CompareFalse:
WriteCompareResult(inst.dst, IR::U1(IR::Value(false)));
return true;
case IR::Opcode::CompareTrue:
WriteCompareResult(inst.dst, IR::U1(IR::Value(true)));
return true;
case IR::Opcode::CompareEqU32:
case IR::Opcode::CompareMaskEqU32:
case IR::Opcode::CompareEqI32:
case IR::Opcode::CompareMaskEqI32: opcode = IR::ValueOpcode::IEqual32; break;
case IR::Opcode::CompareNeU32:
case IR::Opcode::CompareMaskNeU32:
case IR::Opcode::CompareNeI32:
case IR::Opcode::CompareMaskNeI32: opcode = IR::ValueOpcode::INotEqual32; break;
case IR::Opcode::CompareGtU32:
case IR::Opcode::CompareMaskGtU32: opcode = IR::ValueOpcode::UGreaterThan32; break;
case IR::Opcode::CompareGeU32:
case IR::Opcode::CompareMaskGeU32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break;
case IR::Opcode::CompareLtU32:
case IR::Opcode::CompareMaskLtU32: opcode = IR::ValueOpcode::ULessThan32; break;
case IR::Opcode::CompareLeU32:
case IR::Opcode::CompareMaskLeU32: opcode = IR::ValueOpcode::ULessThanEqual32; break;
case IR::Opcode::CompareGtI32:
case IR::Opcode::CompareMaskGtI32: opcode = IR::ValueOpcode::SGreaterThan32; break;
case IR::Opcode::CompareGeI32:
case IR::Opcode::CompareMaskGeI32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break;
case IR::Opcode::CompareLtI32:
case IR::Opcode::CompareMaskLtI32: opcode = IR::ValueOpcode::SLessThan32; break;
case IR::Opcode::CompareLeI32:
case IR::Opcode::CompareMaskLeI32: opcode = IR::ValueOpcode::SLessThanEqual32; break;
case IR::Opcode::CompareEqU64:
opcode = IR::ValueOpcode::IEqual64;
type = IR::Type::U64;
break;
case IR::Opcode::CompareNeU64:
opcode = IR::ValueOpcode::INotEqual64;
type = IR::Type::U64;
break;
default: return false;
}
const auto lhs = ReadOperand(inst.src[0], type);
const auto rhs = ReadOperand(inst.src[1], type);
WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs})));
return true;
}
bool Translator::TranslateInteger16Compare(const IR::Instruction& inst) {
IR::ValueOpcode opcode {};
bool signed_value = false;
switch (inst.op) {
case IR::Opcode::CompareEqU16:
case IR::Opcode::CompareEqI16: opcode = IR::ValueOpcode::IEqual32; break;
case IR::Opcode::CompareNeU16:
case IR::Opcode::CompareNeI16: opcode = IR::ValueOpcode::INotEqual32; break;
case IR::Opcode::CompareGtU16: opcode = IR::ValueOpcode::UGreaterThan32; break;
case IR::Opcode::CompareGeU16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break;
case IR::Opcode::CompareLtU16: opcode = IR::ValueOpcode::ULessThan32; break;
case IR::Opcode::CompareLeU16: opcode = IR::ValueOpcode::ULessThanEqual32; break;
case IR::Opcode::CompareGtI16:
opcode = IR::ValueOpcode::SGreaterThan32;
signed_value = true;
break;
case IR::Opcode::CompareGeI16:
opcode = IR::ValueOpcode::SGreaterThanEqual32;
signed_value = true;
break;
case IR::Opcode::CompareLtI16:
opcode = IR::ValueOpcode::SLessThan32;
signed_value = true;
break;
case IR::Opcode::CompareLeI16:
opcode = IR::ValueOpcode::SLessThanEqual32;
signed_value = true;
break;
default: return false;
}
if (inst.op == IR::Opcode::CompareEqI16 || inst.op == IR::Opcode::CompareNeI16) {
signed_value = true;
}
WriteCompareResult(inst.dst,
IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src[0], signed_value),
ReadU16AsU32(inst.src[1], signed_value)})));
return true;
}
bool Translator::TranslateFloatCompare(const IR::Instruction& inst) {
IR::ValueOpcode opcode {};
bool half = false;
switch (inst.op) {
case IR::Opcode::CompareEqF32:
case IR::Opcode::CompareMaskEqF32: opcode = IR::ValueOpcode::FPOrdEqual32; break;
case IR::Opcode::CompareNeF32:
case IR::Opcode::CompareMaskNeF32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break;
case IR::Opcode::CompareGtF32:
case IR::Opcode::CompareMaskGtF32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break;
case IR::Opcode::CompareGeF32:
case IR::Opcode::CompareMaskGeF32: opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; break;
case IR::Opcode::CompareLtF32:
case IR::Opcode::CompareMaskLtF32: opcode = IR::ValueOpcode::FPOrdLessThan32; break;
case IR::Opcode::CompareLeF32:
case IR::Opcode::CompareMaskLeF32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break;
case IR::Opcode::CompareUnordEqF32:
case IR::Opcode::CompareMaskUnordEqF32: opcode = IR::ValueOpcode::FPUnordEqual32; break;
case IR::Opcode::CompareUnordNeF32:
case IR::Opcode::CompareMaskUnordNeF32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break;
case IR::Opcode::CompareUnordGtF32:
case IR::Opcode::CompareMaskUnordGtF32:
opcode = IR::ValueOpcode::FPUnordGreaterThan32;
break;
case IR::Opcode::CompareUnordGeF32:
case IR::Opcode::CompareMaskUnordGeF32:
opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32;
break;
case IR::Opcode::CompareUnordLtF32:
case IR::Opcode::CompareMaskUnordLtF32: opcode = IR::ValueOpcode::FPUnordLessThan32; break;
case IR::Opcode::CompareUnordLeF32:
case IR::Opcode::CompareMaskUnordLeF32:
opcode = IR::ValueOpcode::FPUnordLessThanEqual32;
break;
case IR::Opcode::CompareEqF16:
case IR::Opcode::CompareMaskEqF16:
opcode = IR::ValueOpcode::FPOrdEqual32;
half = true;
break;
case IR::Opcode::CompareNeF16:
case IR::Opcode::CompareMaskNeF16:
opcode = IR::ValueOpcode::FPOrdNotEqual32;
half = true;
break;
case IR::Opcode::CompareGtF16:
case IR::Opcode::CompareMaskGtF16:
opcode = IR::ValueOpcode::FPOrdGreaterThan32;
half = true;
break;
case IR::Opcode::CompareGeF16:
case IR::Opcode::CompareMaskGeF16:
opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32;
half = true;
break;
case IR::Opcode::CompareLtF16:
case IR::Opcode::CompareMaskLtF16:
opcode = IR::ValueOpcode::FPOrdLessThan32;
half = true;
break;
case IR::Opcode::CompareLeF16:
case IR::Opcode::CompareMaskLeF16:
opcode = IR::ValueOpcode::FPOrdLessThanEqual32;
half = true;
break;
case IR::Opcode::CompareUnordNeF16:
case IR::Opcode::CompareMaskUnordNeF16:
opcode = IR::ValueOpcode::FPUnordNotEqual32;
half = true;
break;
case IR::Opcode::CompareMaskUnordGeF16:
opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32;
half = true;
break;
case IR::Opcode::CompareOrderedF32:
case IR::Opcode::CompareUnorderedF32: {
const auto lhs = IR::F32(ReadOperand(inst.src[0], IR::Type::F32));
const auto rhs = IR::F32(ReadOperand(inst.src[1], IR::Type::F32));
const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})),
IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs})));
const auto result =
inst.op == IR::Opcode::CompareUnorderedF32 ? unordered : ir.LogicalNot(unordered);
WriteCompareResult(inst.dst, result);
return true;
}
case IR::Opcode::CompareClassF32: {
const auto value = ReadOperand(inst.src[0], IR::Type::F32);
const auto mask = ReadOperand(inst.src[1], IR::Type::U32);
WriteCompareResult(inst.dst,
IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})));
return true;
}
default: return false;
}
const auto lhs =
half ? IR::Value(ReadF16AsF32(inst.src[0])) : ReadOperand(inst.src[0], IR::Type::F32);
const auto rhs =
half ? IR::Value(ReadF16AsF32(inst.src[1])) : ReadOperand(inst.src[1], IR::Type::F32);
WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs})));
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Control.cpp
+260
View File
@@ -0,0 +1,260 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
const auto zero = IR::U32(IR::Value(0u));
switch (inst.op) {
case IR::Opcode::SaveexecB32:
case IR::Opcode::SaveexecB64: {
const auto old_low = ir.GetExecLo();
const auto old_high = ir.GetExecHi();
const auto src_low = ReadRawU32(inst.src[0]);
const auto src_high = inst.op == IR::Opcode::SaveexecB64
? ReadRawU32(OffsetOperand(inst.src[0], 1))
: IR::U32(IR::Value(0u));
const auto apply = [&](IR::U32 old_value, IR::U32 src_value) {
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value);
case IR::SaveexecMode::Andn1:
return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value));
case IR::SaveexecMode::Orn2:
return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value);
}
return old_value;
};
const auto result_low = apply(old_low, src_low);
const auto result_high =
inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
if (inst.op == IR::Opcode::SaveexecB64) {
WriteOperand(inst.dst, ir.PackUint2x32(old_low, old_high));
} else {
WriteOperand(inst.dst, old_low);
}
ir.SetExecLo(result_low);
ir.SetExecHi(result_high);
ir.SetExec(ThreadBit(result_low, result_high));
const auto nonzero = inst.op == IR::Opcode::SaveexecB64
? ir.BitwiseOr(result_low, result_high)
: result_low;
ir.SetScc(ir.INotEqual(nonzero, zero));
return true;
}
case IR::Opcode::ScalarAddCarryU32:
case IR::Opcode::IAddCarryU32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const auto carry_in = ConditionBit(inst.src[2]);
const auto add0 = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs});
const auto partial = ir.CompositeExtract(add0, 0);
const auto carry0 = ir.CompositeExtract(add0, 1);
const auto add1 = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in});
const auto result = ir.CompositeExtract(add1, 0);
const auto carry1 = ir.CompositeExtract(add1, 1);
const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), IR::U32(IR::Value(0u)));
WriteOperand(inst.dst, result);
if (inst.op == IR::Opcode::ScalarAddCarryU32) {
ir.SetScc(carry);
} else {
WriteMask(inst.dst2, carry);
}
return true;
}
case IR::Opcode::ScalarSubBorrowU32:
case IR::Opcode::ISubBorrowU32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const auto result = ir.ISub(lhs, rhs);
const auto borrow = ir.UGreaterThan(rhs, lhs);
WriteOperand(inst.dst, result);
if (inst.op == IR::Opcode::ScalarSubBorrowU32) {
ir.SetScc(borrow);
} else {
WriteMask(inst.dst2, borrow);
}
return true;
}
case IR::Opcode::ScalarSubBorrowCarryU32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const auto borrow_in = ConditionBit(inst.src[2]);
const auto partial = ir.ISub(lhs, rhs);
const auto result = ir.ISub(partial, borrow_in);
const auto borrow0 = ir.UGreaterThan(rhs, lhs);
const auto borrow1 = ir.UGreaterThan(borrow_in, partial);
WriteOperand(inst.dst, result);
ir.SetScc(ir.LogicalOr(borrow0, borrow1));
return true;
}
case IR::Opcode::ScalarSignedAddOverflowI32:
case IR::Opcode::ScalarSignedSubOverflowI32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const bool subtract = inst.op == IR::Opcode::ScalarSignedSubOverflowI32;
const auto result = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs);
const auto shift = IR::U32(IR::Value(31u));
const auto lhs_sign = ir.ShiftRightLogical(lhs, shift);
const auto rhs_sign = ir.ShiftRightLogical(rhs, shift);
const auto out_sign = ir.ShiftRightLogical(result, shift);
const auto inputs =
subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign);
const auto changed = ir.INotEqual(lhs_sign, out_sign);
WriteOperand(inst.dst, result);
ir.SetScc(ir.LogicalAnd(inputs, changed));
return true;
}
case IR::Opcode::ScalarShiftLeftAddCarryU32: {
const auto lhs = ReadU32(inst.src[0]);
const auto shift = ReadU32(inst.src[1]);
const auto rhs = ReadU32(inst.src[2]);
const auto shifted = ir.ShiftLeftLogical(lhs, shift);
const auto result = ir.IAdd(shifted, rhs);
const auto add_carry = ir.ULessThan(result, shifted);
const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift);
const auto shifted_out = ir.ShiftRightLogical(lhs, inverse_shift);
const auto shift_carry = ir.INotEqual(shifted_out, zero);
WriteOperand(inst.dst, result);
ir.SetScc(ir.LogicalOr(add_carry, shift_carry));
return true;
}
default: return false;
}
}
bool Translator::TranslateControlOperation(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::ControlNop: ir.Emit(IR::ValueOpcode::ControlNop); return true;
case IR::Opcode::Waitcnt: ir.Emit(IR::ValueOpcode::Waitcnt); return true;
case IR::Opcode::Barrier: ir.Emit(IR::ValueOpcode::Barrier); return true;
case IR::Opcode::Sendmsg: ir.Emit(IR::ValueOpcode::Sendmsg); return true;
case IR::Opcode::TtraceData: ir.Emit(IR::ValueOpcode::TtraceData); return true;
case IR::Opcode::InstPrefetch: ir.Emit(IR::ValueOpcode::InstPrefetch); return true;
default: return false;
}
}
bool Translator::TranslateMove(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::MoveU32:
if (inst.src[0].kind == IR::OperandKind::Register &&
(inst.src[0].reg.file == IR::RegisterFile::Exec ||
inst.src[0].reg.file == IR::RegisterFile::Vcc) &&
inst.dst.kind == IR::OperandKind::Register &&
(inst.dst.reg.file == IR::RegisterFile::Exec ||
inst.dst.reg.file == IR::RegisterFile::Vcc)) {
WriteMask(inst.dst, ReadMask(inst.src[0]));
} else {
WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U32));
}
return true;
case IR::Opcode::MoveF32Bits:
WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
return true;
case IR::Opcode::MoveU64: {
const bool scalar_pair = inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar;
WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U64));
if (scalar_pair) {
ir.SetThreadBitScalarReg(
static_cast<IR::ScalarReg>(inst.dst.reg.index),
ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
}
return true;
}
case IR::Opcode::WqmB64: {
if (!current_per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto result =
ExpandWholeQuadMask(IR::U64(ReadOperand(inst.src[0], IR::Type::U64)));
WriteOperand(inst.dst, result);
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
return true;
}
const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
WriteMask64(inst.dst, result);
ir.SetScc(result);
return true;
}
default: return false;
}
}
bool Translator::TranslateLaneOperation(const IR::Instruction& inst) {
const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u));
switch (inst.op) {
case IR::Opcode::MoveRelSourceU32: {
EXIT_IF(inst.src[0].kind != IR::OperandKind::Register ||
inst.src[0].reg.file != IR::RegisterFile::Vector);
const auto base = inst.src[0].reg.index;
const auto m0 = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu)));
auto selected = ir.GetVectorReg(static_cast<IR::VectorReg>(base));
for (uint32_t index = base + 1u; index < current_vector_limit; index++) {
const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
selected =
ir.Select(match, ir.GetVectorReg(static_cast<IR::VectorReg>(index)), selected);
}
WriteOperand(inst.dst, selected);
return true;
}
case IR::Opcode::MoveRelDestU32: {
EXIT_IF(inst.dst.kind != IR::OperandKind::Register ||
inst.dst.reg.file != IR::RegisterFile::Vector);
const auto base = inst.dst.reg.index;
const auto value = ReadU32(inst.src[0]);
const auto m0 = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu)));
for (uint32_t index = base; index < current_vector_limit; index++) {
const auto reg = static_cast<IR::VectorReg>(index);
const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
const auto write = ir.LogicalAnd(ir.GetExec(), match);
ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg)));
}
return true;
}
case IR::Opcode::ReadFirstLaneU32: {
const auto result =
ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src[0]), ir.GetExec()});
WriteOperand(inst.dst, result);
return true;
}
case IR::Opcode::ReadLaneU32: {
const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask);
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src[0]), lane}));
return true;
}
case IR::Opcode::WriteLaneU32: {
EXIT_IF(inst.dst.kind != IR::OperandKind::Register ||
inst.dst.reg.file != IR::RegisterFile::Vector);
const auto reg = static_cast<IR::VectorReg>(inst.dst.reg.index);
const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask);
const auto result = IR::U32(ir.Emit(
IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src[0]), lane}));
// V_WRITELANE_B32 ignores EXEC; bypass the ordinary VGPR destination path.
ir.SetVectorReg(reg, result);
return true;
}
case IR::Opcode::Permlane16B32:
case IR::Opcode::Permlanex16B32: {
const IR::PermlaneFlags flags {
.x16 = inst.op == IR::Opcode::Permlanex16B32,
.fetch_inactive = inst.dst.op_sel,
.bound_control = inst.dst.op_sel_hi,
};
const auto result = ir.Emit(
IR::ValueOpcode::Permlane16U32,
{ReadU32(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]), ir.GetExec()},
flags);
auto dst = inst.dst;
dst.op_sel = false;
dst.op_sel_hi = false;
WriteOperand(dst, result);
return true;
}
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Convert.cpp
+158
View File
@@ -0,0 +1,158 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
IR::F32 Translator::SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value) {
return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
}
IR::U32 Translator::ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
uint32_t upper_result) {
const auto zero = IR::F32(IR::Value::F32(0.0f));
const auto nan = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value}));
const auto low = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, zero}));
const auto high = IR::U1(
ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)}));
const auto truncated = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value}));
const auto safe_low = SelectF32(ir.LogicalOr(nan, low), zero, truncated);
const auto safe = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low);
const auto converted = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32F32, {safe}));
return ir.Select(high, IR::U32(IR::Value(upper_result)), converted);
}
IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound,
float safe_upper, uint32_t lower_result,
uint32_t upper_result) {
const auto nan = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value}));
const auto low = IR::U1(
ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, IR::Value::F32(lower_bound)}));
const auto high = IR::U1(
ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)}));
const auto truncated = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value}));
const auto safe_low = SelectF32(low, IR::F32(IR::Value::F32(lower_bound)), truncated);
const auto safe_high = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low);
const auto safe = SelectF32(nan, IR::F32(IR::Value::F32(0.0f)), safe_high);
const auto converted = IR::U32(ir.Emit(IR::ValueOpcode::ConvertS32F32, {safe}));
const auto clamped_high = ir.Select(high, IR::U32(IR::Value(upper_result)), converted);
const auto clamped = ir.Select(low, IR::U32(IR::Value(lower_result)), clamped_high);
return ir.Select(nan, IR::U32(IR::Value(0u)), clamped);
}
bool Translator::TranslateConversion(const IR::Instruction& inst) {
const auto f32 = [&](uint32_t index) {
return IR::F32(ReadOperand(inst.src[index], IR::Type::F32));
};
const auto u32 = [&](uint32_t index) { return ReadU32(inst.src[index]); };
const auto convert_u32 = [&](IR::F32 value) {
return ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu);
};
const auto convert_i32 = [&](IR::F32 value) {
return ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f,
0x80000000u, 0x7fffffffu);
};
switch (inst.op) {
case IR::Opcode::ConvertByteU32ToF32: {
const auto index =
inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm & 3u : 0u;
const auto byte =
ir.BitwiseAnd(ir.ShiftRightLogical(u32(0), IR::U32(IR::Value(index * 8u))),
IR::U32(IR::Value(0xffu)));
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte}));
return true;
}
case IR::Opcode::ConvertU32ToF32:
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)}));
return true;
case IR::Opcode::ConvertI32ToF32:
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)}));
return true;
case IR::Opcode::ConvertF32ToU32: WriteOperand(inst.dst, convert_u32(f32(0))); return true;
case IR::Opcode::ConvertF32ToI32: WriteOperand(inst.dst, convert_i32(f32(0))); return true;
case IR::Opcode::ConvertF32ToF16: WriteF16(inst.dst, f32(0)); return true;
case IR::Opcode::ConvertF16ToF32:
WriteOperand(inst.dst, ReadF16AsF32(inst.src[0]));
return true;
case IR::Opcode::ConvertU16ToF16: {
const auto value = IR::F32(
ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src[0], false)}));
WriteF16(inst.dst, value);
return true;
}
case IR::Opcode::ConvertI16ToF16: {
const auto value =
IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src[0], true)}));
WriteF16(inst.dst, value);
return true;
}
case IR::Opcode::ConvertF16ToU16: {
const auto converted =
ConvertF32ToU32Saturated(ReadF16AsF32(inst.src[0]), 65536.0f, 65535.0f, 0xffffu);
WriteU16(inst.dst, converted);
return true;
}
case IR::Opcode::ConvertF16ToI16: {
const auto converted = ConvertF32ToI32Saturated(
ReadF16AsF32(inst.src[0]), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu);
WriteU16(inst.dst, ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu))));
return true;
}
case IR::Opcode::ConvertRoundPlusInfF32ToI32: {
const auto biased =
IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {f32(0), IR::Value::F32(0.5f)}));
const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased}));
WriteOperand(inst.dst, convert_i32(rounded));
return true;
}
case IR::Opcode::ConvertFloorF32ToI32: {
const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {f32(0)}));
WriteOperand(inst.dst, convert_i32(rounded));
return true;
}
case IR::Opcode::ConvertI4ToOffsetF32: {
const auto nibble = IR::U32(
ir.Emit(IR::ValueOpcode::BitFieldSExtract, {u32(0), IR::Value(0u), IR::Value(4u)}));
const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble}));
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)}));
return true;
}
case IR::Opcode::PackF32ToF16Rtz:
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)}));
return true;
case IR::Opcode::PackSnorm2x16F32:
case IR::Opcode::PackUnorm2x16F32: {
const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {f32(0), f32(1)});
const auto opcode = inst.op == IR::Opcode::PackSnorm2x16F32
? IR::ValueOpcode::PackSnorm2x16
: IR::ValueOpcode::PackUnorm2x16;
WriteOperand(inst.dst, ir.Emit(opcode, {pair}));
return true;
}
case IR::Opcode::PackU8F32: {
const auto byte = ConvertF32ToU32Saturated(f32(0), 255.0f, 255.0f, 255u);
const auto index = ir.BitwiseAnd(u32(1), IR::U32(IR::Value(3u)));
const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u)));
const auto mask = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift);
const auto base = ir.BitwiseAnd(u32(2), ir.BitwiseNot(mask));
WriteOperand(inst.dst, ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift)));
return true;
}
case IR::Opcode::PackB32F16: {
const auto low = ReadF16LaneBits(inst.src[0], false);
const auto high =
ir.ShiftLeftLogical(ReadF16LaneBits(inst.src[1], false), IR::U32(IR::Value(16u)));
WriteOperand(inst.dst, ir.BitwiseOr(low, high));
return true;
}
default: return false;
}
}
IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
const auto mask = IR::U32(IR::Value(0xffffu));
return ir.BitwiseOr(ir.BitwiseAnd(low, mask),
ir.ShiftLeftLogical(ir.BitwiseAnd(high, mask), IR::U32(IR::Value(16u))));
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp
+40
View File
@@ -0,0 +1,40 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
if (current_per_invocation_masks) {
return TranslatePerInvocationU64Mask(inst);
}
return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64
? TranslateSimpleInteger(inst)
: TranslateComposedInteger(inst);
}
bool Translator::TranslateInstruction(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot) {
switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
case IR::LoweringClass::Control: return TranslateControlOperation(inst);
case IR::LoweringClass::Move: return TranslateMove(inst);
case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
case IR::LoweringClass::State: return TranslateStateOperation(inst);
case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst, address_snapshot);
case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);
case IR::LoweringClass::FloatCompare: return TranslateFloatCompare(inst);
case IR::LoweringClass::Conversion: return TranslateConversion(inst);
case IR::LoweringClass::Integer16: return TranslateInteger16Operation(inst);
case IR::LoweringClass::PackedInteger16: return TranslatePackedInteger16(inst);
case IR::LoweringClass::PackedFloat16: return TranslatePackedFloat16(inst);
case IR::LoweringClass::Float16: return TranslateFloat16Operation(inst);
case IR::LoweringClass::Float: return TranslateFloatOperation(inst);
case IR::LoweringClass::U64Mask: return TranslateU64MaskOperation(inst);
case IR::LoweringClass::SimpleInteger: return TranslateSimpleInteger(inst);
case IR::LoweringClass::ComposedInteger: return TranslateComposedInteger(inst);
case IR::LoweringClass::ExtendedInteger: return TranslateExtendedInteger(inst);
}
return false;
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Float.cpp
+212
View File
@@ -0,0 +1,212 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include <array>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) {
IR::ValueOpcode opcode;
switch (inst.op) {
case IR::Opcode::PackedAddF16: opcode = IR::ValueOpcode::FPAdd32; break;
case IR::Opcode::PackedMulF16: opcode = IR::ValueOpcode::FPMul32; break;
case IR::Opcode::PackedMinF16: opcode = IR::ValueOpcode::FPMin32; break;
case IR::Opcode::PackedMaxF16: opcode = IR::ValueOpcode::FPMax32; break;
case IR::Opcode::PackedFmaF16: opcode = IR::ValueOpcode::FPFma32; break;
default: return false;
}
const auto translate_lane = [&](bool high) {
const auto lhs = ReadF16LaneAsF32(inst.src[0], high);
const auto rhs = ReadF16LaneAsF32(inst.src[1], high);
IR::F32 result;
if (inst.src_count == 3u) {
result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high)}));
} else {
result = IR::F32(ir.Emit(opcode, {lhs, rhs}));
}
return ApplyF32ResultModifiers(inst.dst, result);
};
auto raw = inst.dst;
raw.omod = 0u;
raw.clamp = false;
auto result = PackHalf2x16(translate_lane(false), translate_lane(true));
if (inst.op == IR::Opcode::PackedMinF16 || inst.op == IR::Opcode::PackedMaxF16) {
const auto quiet_snan = [&](const IR::Operand& operand, bool high) {
const auto bits = ReadU16LaneAsU32(operand, high, false);
const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u)));
const auto payload = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu)));
const auto snan = ir.LogicalAnd(ir.IEqual(exponent, IR::U32(IR::Value(0x7c00u))),
ir.INotEqual(payload, IR::U32(IR::Value(0u))));
return std::pair {snan, ir.BitwiseOr(bits, IR::U32(IR::Value(0x0200u)))};
};
const auto override_lane = [&](bool high) {
const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src[0], high);
const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src[1], high);
const auto normal = high ? ir.ShiftRightLogical(result, IR::U32(IR::Value(16u)))
: ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu)));
return ir.Select(lhs_snan, lhs_quiet, ir.Select(rhs_snan, rhs_quiet, normal));
};
result = PackU16Lanes(override_lane(false), override_lane(true));
}
WriteOperand(raw, result);
return true;
}
bool Translator::TranslateFloat16Operation(const IR::Instruction& inst) {
IR::ValueOpcode opcode;
switch (inst.op) {
case IR::Opcode::AddF16: opcode = IR::ValueOpcode::FPAdd32; break;
case IR::Opcode::SubF16: opcode = IR::ValueOpcode::FPSub32; break;
case IR::Opcode::MulF16: opcode = IR::ValueOpcode::FPMul32; break;
case IR::Opcode::MinF16: opcode = IR::ValueOpcode::FPMin32; break;
case IR::Opcode::MaxF16: opcode = IR::ValueOpcode::FPMax32; break;
case IR::Opcode::FmaF16:
case IR::Opcode::MadMixF16: opcode = IR::ValueOpcode::FPFma32; break;
case IR::Opcode::RcpF16: opcode = IR::ValueOpcode::FPRecip32; break;
case IR::Opcode::SqrtF16: opcode = IR::ValueOpcode::FPSqrt; break;
case IR::Opcode::InverseSqrtF16: opcode = IR::ValueOpcode::FPRecipSqrt32; break;
case IR::Opcode::Log2F16: opcode = IR::ValueOpcode::FPLog2; break;
case IR::Opcode::Exp2F16: opcode = IR::ValueOpcode::FPExp2; break;
case IR::Opcode::FloorF16: opcode = IR::ValueOpcode::FPFloor32; break;
case IR::Opcode::CeilF16: opcode = IR::ValueOpcode::FPCeil32; break;
case IR::Opcode::TruncF16: opcode = IR::ValueOpcode::FPTrunc32; break;
case IR::Opcode::RoundEvenF16: opcode = IR::ValueOpcode::FPRoundEven32; break;
case IR::Opcode::Min3F16: opcode = IR::ValueOpcode::FPMinTri32; break;
case IR::Opcode::Max3F16: opcode = IR::ValueOpcode::FPMaxTri32; break;
case IR::Opcode::Med3F16: opcode = IR::ValueOpcode::FPMedTri32; break;
default: return false;
}
std::array<IR::Value, 3> args;
for (uint32_t index = 0; index < inst.src_count; index++) {
args[index] = inst.op == IR::Opcode::MadMixF16 ? IR::Value(ReadMixF32(inst.src[index]))
: IR::Value(ReadF16AsF32(inst.src[index]));
}
IR::F32 result;
switch (inst.src_count) {
case 1: result = IR::F32(ir.Emit(opcode, {args[0]})); break;
case 2: result = IR::F32(ir.Emit(opcode, {args[0], args[1]})); break;
case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break;
default: EXIT("invalid half-float source count: %u", inst.src_count);
}
WriteF16(inst.dst, result);
return true;
}
bool Translator::TranslateFloatOperation(const IR::Instruction& inst) {
IR::ValueOpcode opcode {};
switch (inst.op) {
case IR::Opcode::RcpF32: opcode = IR::ValueOpcode::FPRecip32; break;
case IR::Opcode::RcpIflagF32: opcode = IR::ValueOpcode::FPRecipIFlag32; break;
case IR::Opcode::FractF32: opcode = IR::ValueOpcode::FPFract32; break;
case IR::Opcode::TruncF32: opcode = IR::ValueOpcode::FPTrunc32; break;
case IR::Opcode::CeilF32: opcode = IR::ValueOpcode::FPCeil32; break;
case IR::Opcode::RoundEvenF32: opcode = IR::ValueOpcode::FPRoundEven32; break;
case IR::Opcode::FloorF32: opcode = IR::ValueOpcode::FPFloor32; break;
case IR::Opcode::Exp2F32: opcode = IR::ValueOpcode::FPExp2; break;
case IR::Opcode::Log2F32: opcode = IR::ValueOpcode::FPLog2; break;
case IR::Opcode::InverseSqrtF32: opcode = IR::ValueOpcode::FPRecipSqrt32; break;
case IR::Opcode::SqrtF32: opcode = IR::ValueOpcode::FPSqrt; break;
case IR::Opcode::SinF32: opcode = IR::ValueOpcode::FPSin; break;
case IR::Opcode::CosF32: opcode = IR::ValueOpcode::FPCos; break;
case IR::Opcode::FAddF32: opcode = IR::ValueOpcode::FPAdd32; break;
case IR::Opcode::FSubF32: opcode = IR::ValueOpcode::FPSub32; break;
case IR::Opcode::FMulF32: opcode = IR::ValueOpcode::FPMul32; break;
case IR::Opcode::FMinF32: opcode = IR::ValueOpcode::FPMin32; break;
case IR::Opcode::FMaxF32: opcode = IR::ValueOpcode::FPMax32; break;
case IR::Opcode::FMadF32: opcode = IR::ValueOpcode::FPFma32; break;
case IR::Opcode::FMin3F32: opcode = IR::ValueOpcode::FPMinTri32; break;
case IR::Opcode::FMax3F32: opcode = IR::ValueOpcode::FPMaxTri32; break;
case IR::Opcode::FMed3F32: opcode = IR::ValueOpcode::FPMedTri32; break;
case IR::Opcode::LdexpF32: opcode = IR::ValueOpcode::FPLdexp; break;
default: break;
}
if (opcode != IR::ValueOpcode {}) {
std::array<IR::Value, 3> args;
for (uint32_t index = 0; index < inst.src_count; index++) {
const auto arg_type = IR::ArgTypeOf(opcode, index);
args[index] =
arg_type == IR::Type::F32
? (inst.op == IR::Opcode::FMadF32 ? IR::Value(ReadMixF32(inst.src[index]))
: ReadOperand(inst.src[index], IR::Type::F32))
: ReadOperand(inst.src[index], arg_type);
}
IR::Value result;
switch (inst.src_count) {
case 1: result = ir.Emit(opcode, {args[0]}); break;
case 2: result = ir.Emit(opcode, {args[0], args[1]}); break;
case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break;
default: EXIT("invalid floating-point source count: %u", inst.src_count);
}
WriteOperand(inst.dst, result);
return true;
}
const auto select_f32 = [&](IR::U1 condition, IR::F32 true_value, IR::F32 false_value) {
return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
};
if (inst.op == IR::Opcode::Dot2AccF32F16) {
auto a = inst.src[0];
a.op_sel = false;
a.op_sel_hi = true;
auto b = inst.src[1];
b.op_sel = false;
b.op_sel_hi = true;
const auto a_lo = ReadF16LaneAsF32(a, false);
const auto a_hi = ReadF16LaneAsF32(a, true);
const auto b_lo = ReadF16LaneAsF32(b, false);
const auto b_hi = ReadF16LaneAsF32(b, true);
const auto acc = ReadMixF32(inst.src[2]);
const auto lo = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc}));
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo}));
return true;
}
if (inst.op != IR::Opcode::CubeIdF32 && inst.op != IR::Opcode::CubeScF32 &&
inst.op != IR::Opcode::CubeTcF32 && inst.op != IR::Opcode::CubeMaF32) {
return false;
}
const auto x = ReadMixF32(inst.src[0]);
const auto y = ReadMixF32(inst.src[1]);
const auto z = ReadMixF32(inst.src[2]);
const auto nx = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {x}));
const auto ny = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {y}));
const auto nz = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {z}));
const auto ax = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {x}));
const auto ay = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {y}));
const auto az = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {z}));
const auto z_face =
ir.LogicalAnd(IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {az, ax})),
IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {az, ay})));
const auto y_face = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {ay, ax}));
const auto x_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {x, IR::Value::F32(0.0f)}));
const auto y_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {y, IR::Value::F32(0.0f)}));
const auto z_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {z, IR::Value::F32(0.0f)}));
const auto select_face = [&](IR::F32 x_value, IR::F32 y_value, IR::F32 z_value) {
return select_f32(z_face, z_value, select_f32(y_face, y_value, x_value));
};
IR::F32 result;
switch (inst.op) {
case IR::Opcode::CubeIdF32:
result = select_face(
select_f32(x_neg, IR::F32(IR::Value::F32(1.0f)), IR::F32(IR::Value::F32(0.0f))),
select_f32(y_neg, IR::F32(IR::Value::F32(3.0f)), IR::F32(IR::Value::F32(2.0f))),
select_f32(z_neg, IR::F32(IR::Value::F32(5.0f)), IR::F32(IR::Value::F32(4.0f))));
break;
case IR::Opcode::CubeScF32:
result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x));
break;
case IR::Opcode::CubeTcF32: result = select_face(ny, select_f32(y_neg, nz, z), ny); break;
case IR::Opcode::CubeMaF32: {
const auto two = IR::F32(IR::Value::F32(2.0f));
result = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})),
IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})),
IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {z, two})));
break;
}
default: return false;
}
WriteOperand(inst.dst, result);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Integer.cpp
+630
View File
@@ -0,0 +1,630 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include <array>
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateInteger16Operation(const IR::Instruction& inst) {
const auto mask16 = [&](IR::U32 value) {
return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
};
IR::Value result;
switch (inst.op) {
case IR::Opcode::ShiftLeftLogicalU16:
case IR::Opcode::ShiftRightLogicalU16:
case IR::Opcode::ShiftRightArithmeticI16: {
const bool arithmetic = inst.op == IR::Opcode::ShiftRightArithmeticI16;
const auto value = ReadU16AsU32(inst.src[0], arithmetic);
const auto count =
ir.BitwiseAnd(ReadU16AsU32(inst.src[1], false), IR::U32(IR::Value(15u)));
if (inst.op == IR::Opcode::ShiftLeftLogicalU16) {
result = ir.ShiftLeftLogical(value, count);
} else if (arithmetic) {
result = ir.ShiftRightArithmetic(value, count);
} else {
result = ir.ShiftRightLogical(value, count);
}
break;
}
case IR::Opcode::IAddU16:
result = ir.IAdd(ReadU16AsU32(inst.src[0], false), ReadU16AsU32(inst.src[1], false));
break;
case IR::Opcode::ISubI16:
result = ir.ISub(ReadU16AsU32(inst.src[0], false), ReadU16AsU32(inst.src[1], false));
break;
case IR::Opcode::IMinI16:
case IR::Opcode::IMaxI16:
case IR::Opcode::UMinU16:
case IR::Opcode::UMaxU16: {
const bool sign = inst.op == IR::Opcode::IMinI16 || inst.op == IR::Opcode::IMaxI16;
IR::ValueOpcode opcode;
switch (inst.op) {
case IR::Opcode::IMinI16: opcode = IR::ValueOpcode::SMin32; break;
case IR::Opcode::IMaxI16: opcode = IR::ValueOpcode::SMax32; break;
case IR::Opcode::UMinU16: opcode = IR::ValueOpcode::UMin32; break;
default: opcode = IR::ValueOpcode::UMax32; break;
}
result =
ir.Emit(opcode, {ReadU16AsU32(inst.src[0], sign), ReadU16AsU32(inst.src[1], sign)});
break;
}
default: return false;
}
WriteU16(inst.dst, mask16(IR::U32(result)));
return true;
}
bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) {
const auto lane = [&](const IR::Operand& operand, bool high, bool sign) {
return ReadU16LaneAsU32(operand, high, sign);
};
const auto mask_count = [&](IR::U32 value) {
return ir.BitwiseAnd(value, IR::U32(IR::Value(15u)));
};
const auto translate_lane = [&](bool high) -> IR::U32 {
if (inst.op == IR::Opcode::PackedLshlrevB16 || inst.op == IR::Opcode::PackedLshrrevB16 ||
inst.op == IR::Opcode::PackedAshrrevI16) {
const auto count = mask_count(lane(inst.src[0], high, false));
const bool arithmetic = inst.op == IR::Opcode::PackedAshrrevI16;
const auto value = lane(inst.src[1], high, arithmetic);
if (inst.op == IR::Opcode::PackedLshlrevB16) {
return ir.ShiftLeftLogical(value, count);
}
return arithmetic ? ir.ShiftRightArithmetic(value, count)
: ir.ShiftRightLogical(value, count);
}
const bool signed_minmax =
inst.op == IR::Opcode::PackedMaxI16 || inst.op == IR::Opcode::PackedMinI16;
const auto lhs = lane(inst.src[0], high, signed_minmax);
const auto rhs = lane(inst.src[1], high, signed_minmax);
switch (inst.op) {
case IR::Opcode::PackedMadI16:
case IR::Opcode::PackedMadU16:
return ir.IAdd(ir.IMul(lhs, rhs),
lane(inst.src[2], high, inst.op == IR::Opcode::PackedMadI16));
case IR::Opcode::PackedMulLoU16: return ir.IMul(lhs, rhs);
case IR::Opcode::PackedAddI16:
case IR::Opcode::PackedAddU16: return ir.IAdd(lhs, rhs);
case IR::Opcode::PackedSubI16:
case IR::Opcode::PackedSubU16: return ir.ISub(lhs, rhs);
case IR::Opcode::PackedMaxI16:
return IR::U32(ir.Emit(IR::ValueOpcode::SMax32, {lhs, rhs}));
case IR::Opcode::PackedMinI16:
return IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {lhs, rhs}));
case IR::Opcode::PackedMaxU16:
return IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs}));
case IR::Opcode::PackedMinU16:
return IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs}));
default: EXIT("invalid packed integer opcode");
}
};
switch (inst.op) {
case IR::Opcode::PackedMadI16:
case IR::Opcode::PackedMulLoU16:
case IR::Opcode::PackedAddI16:
case IR::Opcode::PackedSubI16:
case IR::Opcode::PackedLshlrevB16:
case IR::Opcode::PackedLshrrevB16:
case IR::Opcode::PackedAshrrevI16:
case IR::Opcode::PackedMaxI16:
case IR::Opcode::PackedMinI16:
case IR::Opcode::PackedMadU16:
case IR::Opcode::PackedAddU16:
case IR::Opcode::PackedSubU16:
case IR::Opcode::PackedMaxU16:
case IR::Opcode::PackedMinU16:
WriteOperand(inst.dst, PackU16Lanes(translate_lane(false), translate_lane(true)));
return true;
default: return false;
}
}
bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) {
if (!current_per_invocation_masks) {
return false;
}
const auto lhs = [&] { return ReadMask(inst.src[0]); };
const auto rhs = [&] { return ReadMask(inst.src[1]); };
IR::U1 result;
switch (inst.op) {
case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break;
case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break;
case IR::Opcode::BitwiseXorU64:
result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
break;
case IR::Opcode::BitwiseAndNotU64:
result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
break;
case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break;
case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break;
case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break;
case IR::Opcode::BitwiseXnorU64:
result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
break;
case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break;
default: return false;
}
WriteMask64(inst.dst, result);
return true;
}
bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
IR::ValueOpcode opcode {};
IR::Type type = IR::Type::U32;
switch (inst.op) {
case IR::Opcode::AbsI32: opcode = IR::ValueOpcode::IAbs32; break;
case IR::Opcode::IAddU32: opcode = IR::ValueOpcode::IAdd32; break;
case IR::Opcode::ISubU32: opcode = IR::ValueOpcode::ISub32; break;
case IR::Opcode::IMulU32: opcode = IR::ValueOpcode::IMul32; break;
case IR::Opcode::UMulHighU32: opcode = IR::ValueOpcode::UMulHi; break;
case IR::Opcode::SMulHighI32: opcode = IR::ValueOpcode::SMulHi; break;
case IR::Opcode::IMinI32: opcode = IR::ValueOpcode::SMin32; break;
case IR::Opcode::IMaxI32: opcode = IR::ValueOpcode::SMax32; break;
case IR::Opcode::UMinU32: opcode = IR::ValueOpcode::UMin32; break;
case IR::Opcode::UMaxU32: opcode = IR::ValueOpcode::UMax32; break;
case IR::Opcode::IMin3I32: opcode = IR::ValueOpcode::SMinTri32; break;
case IR::Opcode::IMax3I32: opcode = IR::ValueOpcode::SMaxTri32; break;
case IR::Opcode::IMed3I32: opcode = IR::ValueOpcode::SMedTri32; break;
case IR::Opcode::UMin3U32: opcode = IR::ValueOpcode::UMinTri32; break;
case IR::Opcode::UMax3U32: opcode = IR::ValueOpcode::UMaxTri32; break;
case IR::Opcode::UMed3U32: opcode = IR::ValueOpcode::UMedTri32; break;
case IR::Opcode::BitwiseAndU32: opcode = IR::ValueOpcode::BitwiseAnd32; break;
case IR::Opcode::BitwiseOrU32: opcode = IR::ValueOpcode::BitwiseOr32; break;
case IR::Opcode::BitwiseXorU32: opcode = IR::ValueOpcode::BitwiseXor32; break;
case IR::Opcode::BitwiseNotU32: opcode = IR::ValueOpcode::BitwiseNot32; break;
case IR::Opcode::BitReverseU32: opcode = IR::ValueOpcode::BitReverse32; break;
case IR::Opcode::BitCountU32: opcode = IR::ValueOpcode::BitCount32; break;
case IR::Opcode::BitCountU64:
opcode = IR::ValueOpcode::BitCount64;
type = IR::Type::U64;
break;
case IR::Opcode::FindLsbU32: opcode = IR::ValueOpcode::FindILsb32; break;
case IR::Opcode::BitwiseAndU64:
opcode = IR::ValueOpcode::BitwiseAnd64;
type = IR::Type::U64;
break;
case IR::Opcode::BitwiseOrU64:
opcode = IR::ValueOpcode::BitwiseOr64;
type = IR::Type::U64;
break;
case IR::Opcode::ShiftLeftLogicalU32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break;
case IR::Opcode::ShiftRightLogicalU32: opcode = IR::ValueOpcode::ShiftRightLogical32; break;
case IR::Opcode::ShiftRightArithmeticI32:
opcode = IR::ValueOpcode::ShiftRightArithmetic32;
break;
case IR::Opcode::ShiftLeftLogicalU64:
opcode = IR::ValueOpcode::ShiftLeftLogical64;
type = IR::Type::U64;
break;
case IR::Opcode::ShiftRightLogicalU64:
opcode = IR::ValueOpcode::ShiftRightLogical64;
type = IR::Type::U64;
break;
default: return false;
}
std::array<IR::Value, 3> args;
for (uint32_t index = 0; index < inst.src_count; index++) {
const auto arg_type = IR::ArgTypeOf(opcode, index);
args[index] = ReadOperand(inst.src[index], arg_type == IR::Type::Void ? type : arg_type);
if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical32 ||
opcode == IR::ValueOpcode::ShiftRightLogical32 ||
opcode == IR::ValueOpcode::ShiftRightArithmetic32)) {
args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u)));
} else if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical64 ||
opcode == IR::ValueOpcode::ShiftRightLogical64 ||
opcode == IR::ValueOpcode::ShiftRightArithmetic64)) {
args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(63u)));
}
}
IR::Value result;
switch (inst.src_count) {
case 1: result = ir.Emit(opcode, {args[0]}); break;
case 2: result = ir.Emit(opcode, {args[0], args[1]}); break;
case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break;
default: EXIT("invalid simple integer source count: %u", inst.src_count);
}
WriteOperand(inst.dst, result);
if (type == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
IR::U1 mask;
if (inst.op == IR::Opcode::BitwiseAndU64) {
mask = ir.LogicalAnd(ReadMask(inst.src[0]), ReadMask(inst.src[1]));
} else if (inst.op == IR::Opcode::BitwiseOrU64) {
mask = ir.LogicalOr(ReadMask(inst.src[0]), ReadMask(inst.src[1]));
}
if (!mask.IsEmpty()) {
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index), mask);
}
}
return true;
}
bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
const auto binary_u32 = [&](auto operation) {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
return operation(lhs, rhs);
};
const auto binary_u64 = [&](auto operation) {
const auto lhs = UnpackU64(ReadU64(inst.src[0]));
const auto rhs = UnpackU64(ReadU64(inst.src[1]));
return ir.PackUint2x32(operation(lhs[0], rhs[0]), operation(lhs[1], rhs[1]));
};
IR::Value result;
switch (inst.op) {
case IR::Opcode::BitwiseAndNotU32:
result = binary_u32(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); });
break;
case IR::Opcode::BitwiseOrNotU32:
result = binary_u32(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); });
break;
case IR::Opcode::BitwiseNandU32:
result = binary_u32(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); });
break;
case IR::Opcode::BitwiseNorU32:
result = binary_u32(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); });
break;
case IR::Opcode::BitwiseXnorU32:
result = binary_u32(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); });
break;
case IR::Opcode::BitwiseXorU64:
result = binary_u64([&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseXor(lhs, rhs); });
break;
case IR::Opcode::BitwiseAndNotU64:
result = binary_u64(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); });
break;
case IR::Opcode::BitwiseOrNotU64:
result = binary_u64(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); });
break;
case IR::Opcode::BitwiseNandU64:
result = binary_u64(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); });
break;
case IR::Opcode::BitwiseNorU64:
result = binary_u64(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); });
break;
case IR::Opcode::BitwiseXnorU64:
result = binary_u64(
[&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); });
break;
case IR::Opcode::BitwiseNotU64: {
const auto source = UnpackU64(ReadU64(inst.src[0]));
result = ir.PackUint2x32(ir.BitwiseNot(source[0]), ir.BitwiseNot(source[1]));
break;
}
case IR::Opcode::BitwiseAndOrU32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const auto add = ReadU32(inst.src[2]);
result = ir.BitwiseOr(ir.BitwiseAnd(lhs, rhs), add);
break;
}
case IR::Opcode::BitwiseOr3U32:
result = ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src[0]), ReadU32(inst.src[1])),
ReadU32(inst.src[2]));
break;
case IR::Opcode::BitwiseXor3U32:
result = ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src[0]), ReadU32(inst.src[1])),
ReadU32(inst.src[2]));
break;
case IR::Opcode::FindMsbFromHighU32: {
const auto source = ReadU32(inst.src[0]);
const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {source}));
const auto position = ir.ISub(IR::U32(IR::Value(31u)), msb);
result = ir.Select(ir.INotEqual(source, IR::U32(IR::Value(0u))), position,
IR::U32(IR::Value(0xffffffffu)));
break;
}
case IR::Opcode::FindMsbFromHighU64: {
const auto source = ReadU64(inst.src[0]);
const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source}));
const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb);
const auto nonzero = IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64,
{source, ir.PackUint2x32(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu)));
break;
}
default: return false;
}
WriteOperand(inst.dst, result);
if (result.GetType() == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto lhs = ReadMask(inst.src[0]);
IR::U1 mask;
switch (inst.op) {
case IR::Opcode::BitwiseXorU64:
mask = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])}));
break;
case IR::Opcode::BitwiseAndNotU64:
mask = ir.LogicalAnd(lhs, ir.LogicalNot(ReadMask(inst.src[1])));
break;
case IR::Opcode::BitwiseOrNotU64:
mask = ir.LogicalOr(lhs, ir.LogicalNot(ReadMask(inst.src[1])));
break;
case IR::Opcode::BitwiseNandU64:
mask = ir.LogicalNot(ir.LogicalAnd(lhs, ReadMask(inst.src[1])));
break;
case IR::Opcode::BitwiseNorU64:
mask = ir.LogicalNot(ir.LogicalOr(lhs, ReadMask(inst.src[1])));
break;
case IR::Opcode::BitwiseXnorU64:
mask = ir.LogicalNot(
IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])})));
break;
case IR::Opcode::BitwiseNotU64: mask = ir.LogicalNot(lhs); break;
default: break;
}
if (!mask.IsEmpty()) {
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index), mask);
}
}
return true;
}
bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
const auto imm = [](uint32_t value) { return IR::U32(IR::Value(value)); };
const auto mask = [&](IR::U32 value, uint32_t bits) { return ir.BitwiseAnd(value, imm(bits)); };
const auto extract = [&](IR::U32 value, IR::U32 offset, IR::U32 width, bool sign) {
return IR::U32(
ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract,
{value, offset, width}));
};
const auto right_mask32 = [&](IR::U32 count) {
return IR::U32(
ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), imm(0), count}));
};
const auto select_u64 = [&](IR::U1 condition, IR::U64 true_value, IR::U64 false_value) {
const auto lhs = UnpackU64(true_value);
const auto rhs = UnpackU64(false_value);
return ir.PackUint2x32(ir.Select(condition, lhs[0], rhs[0]),
ir.Select(condition, lhs[1], rhs[1]));
};
const auto right_mask64 = [&](IR::U32 count) {
const auto below32 = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, imm(32)}));
const auto above32 = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, imm(32)}));
const auto low_count = ir.Select(below32, count, imm(32));
const auto high_count = ir.Select(above32, ir.ISub(count, imm(32)), imm(0));
return ir.PackUint2x32(right_mask32(low_count), right_mask32(high_count));
};
IR::Value result;
switch (inst.op) {
case IR::Opcode::IMadI24U32:
case IR::Opcode::UMadU24U32:
case IR::Opcode::IMulI24U32:
case IR::Opcode::UMulU24U32: {
const bool sign =
inst.op == IR::Opcode::IMadI24U32 || inst.op == IR::Opcode::IMulI24U32;
const auto lhs = extract(ReadU32(inst.src[0]), imm(0), imm(24), sign);
const auto rhs = extract(ReadU32(inst.src[1]), imm(0), imm(24), sign);
auto value = ir.IMul(lhs, rhs);
if (inst.src_count == 3) {
value = ir.IAdd(value, ReadU32(inst.src[2]));
}
result = value;
break;
}
case IR::Opcode::UMadU64U32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const auto add = UnpackU64(ReadU64(inst.src[2]));
const auto mul_low = ir.IMul(lhs, rhs);
const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs}));
const auto low = ir.IAdd(mul_low, add[0]);
const auto carry_low = ir.ULessThan(low, mul_low);
const auto high0 = ir.IAdd(mul_high, add[1]);
const auto carry0 = ir.ULessThan(high0, mul_high);
const auto high = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0)));
const auto carry1 = ir.ULessThan(high, high0);
WriteOperand(inst.dst, ir.PackUint2x32(low, high));
if (inst.dst2.kind != IR::OperandKind::Null) {
WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
}
return true;
}
case IR::Opcode::SadU32: {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
const auto lo = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs}));
const auto hi = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs}));
result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src[2]));
break;
}
case IR::Opcode::IAdd3U32:
result =
ir.IAdd(ir.IAdd(ReadU32(inst.src[0]), ReadU32(inst.src[1])), ReadU32(inst.src[2]));
break;
case IR::Opcode::BitClearU32:
case IR::Opcode::BitSetU32: {
const auto bit = ir.ShiftLeftLogical(imm(1), mask(ReadU32(inst.src[1]), 31u));
result = inst.op == IR::Opcode::BitClearU32
? IR::Value(ir.BitwiseAnd(ReadU32(inst.src[0]), ir.BitwiseNot(bit)))
: IR::Value(ir.BitwiseOr(ReadU32(inst.src[0]), bit));
break;
}
case IR::Opcode::BitCountAddU32:
result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src[0])})),
ReadU32(inst.src[1]));
break;
case IR::Opcode::MaskedBitCountLowU32:
case IR::Opcode::MaskedBitCountHighU32: {
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
const auto local = mask(lane, 31u);
const auto below = ir.ISub(ir.ShiftLeftLogical(imm(1), local), imm(1));
const auto high_lane =
IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, imm(32)}));
const auto thread_mask = inst.op == IR::Opcode::MaskedBitCountLowU32
? ir.Select(high_lane, imm(0xffffffffu), below)
: ir.Select(high_lane, below, imm(0));
const auto active = ir.BitwiseAnd(ReadU32(inst.src[0]), thread_mask);
result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active})),
ReadU32(inst.src[1]));
break;
}
case IR::Opcode::BitReplicateB64B32: {
const auto replicate = [&](IR::U32 value) {
auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, imm(8)));
bits = ir.BitwiseAnd(bits, imm(0x00ff00ffu));
bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(4)));
bits = ir.BitwiseAnd(bits, imm(0x0f0f0f0fu));
bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(2)));
bits = ir.BitwiseAnd(bits, imm(0x33333333u));
bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1)));
bits = ir.BitwiseAnd(bits, imm(0x55555555u));
return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1)));
};
const auto source = ReadU32(inst.src[0]);
result = ir.PackUint2x32(replicate(mask(source, 0xffffu)),
replicate(ir.ShiftRightLogical(source, imm(16))));
break;
}
case IR::Opcode::BitFieldMaskU32: {
const auto count = mask(ReadU32(inst.src[0]), 31u);
const auto offset = mask(ReadU32(inst.src[1]), 31u);
result =
ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), offset, count});
break;
}
case IR::Opcode::BitFieldMaskU64: {
const auto count = mask(ReadU32(inst.src[0]), 63u);
const auto offset = mask(ReadU32(inst.src[1]), 63u);
result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {right_mask64(count), offset});
break;
}
case IR::Opcode::BitFieldExtractU32: {
const auto source = ReadU32(inst.src[0]);
const auto field = ReadU32(inst.src[1]);
const auto offset = extract(field, imm(0), imm(5), false);
const auto count =
IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {extract(field, imm(16), imm(7), false),
ir.ISub(imm(32), offset)}));
result = ir.Emit(IR::ValueOpcode::BitFieldUExtract, {source, offset, count});
break;
}
case IR::Opcode::BitFieldExtractU64: {
const auto source = ReadU64(inst.src[0]);
const auto field = ReadU32(inst.src[1]);
const auto offset = extract(field, imm(0), imm(6), false);
const auto raw_count = extract(field, imm(16), imm(7), false);
const auto available = ir.ISub(imm(64), offset);
const auto count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {raw_count, available}));
const auto shifted =
IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {source, offset}));
result = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, right_mask64(count)});
break;
}
case IR::Opcode::BitFieldExtract3U32:
case IR::Opcode::BitFieldExtract3I32: {
const auto source = ReadU32(inst.src[0]);
const auto offset = mask(ReadU32(inst.src[1]), 31u);
const auto count =
IR::U32(ir.Emit(IR::ValueOpcode::UMin32,
{mask(ReadU32(inst.src[2]), 31u), ir.ISub(imm(32), offset)}));
if (inst.op == IR::Opcode::BitFieldExtract3I32) {
result =
ir.BitwiseAnd(ir.ShiftRightArithmetic(source, offset), right_mask32(count));
} else {
result = ir.Emit(IR::ValueOpcode::BitFieldUExtract, {source, offset, count});
}
break;
}
case IR::Opcode::BitFieldInsertSelectU32: {
const auto bits = ReadU32(inst.src[0]);
const auto insert = ReadU32(inst.src[1]);
const auto base = ReadU32(inst.src[2]);
result =
ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base));
break;
}
case IR::Opcode::BitCompare0B32:
case IR::Opcode::BitCompare1B32: {
const auto value = ReadU32(inst.src[0]);
const auto bit = extract(value, mask(ReadU32(inst.src[1]), 31u), imm(1), false);
const auto expected = imm(inst.op == IR::Opcode::BitCompare1B32 ? 1u : 0u);
WriteOperand(inst.dst, ir.IEqual(bit, expected));
return true;
}
case IR::Opcode::AlignBitU32: {
const auto hi = ReadU32(inst.src[0]);
const auto lo = ReadU32(inst.src[1]);
const auto shift = mask(ReadU32(inst.src[2]), 31u);
const auto lo_part = ir.ShiftRightLogical(lo, shift);
const auto hi_part_raw = ir.ShiftLeftLogical(hi, mask(ir.ISub(imm(32), shift), 31u));
const auto hi_part = ir.Select(ir.INotEqual(shift, imm(0)), hi_part_raw, imm(0));
result = ir.BitwiseOr(lo_part, hi_part);
break;
}
case IR::Opcode::ShiftLeftAddU32:
result =
ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src[0]), mask(ReadU32(inst.src[1]), 31u)),
ReadU32(inst.src[2]));
break;
case IR::Opcode::AddShiftLeftU32:
result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src[0]), ReadU32(inst.src[1])),
mask(ReadU32(inst.src[2]), 31u));
break;
case IR::Opcode::XorAddU32:
result = ir.IAdd(ir.BitwiseXor(ReadU32(inst.src[0]), ReadU32(inst.src[1])),
ReadU32(inst.src[2]));
break;
case IR::Opcode::ShiftLeftOrU32:
result = ir.BitwiseOr(
ir.ShiftLeftLogical(ReadU32(inst.src[0]), mask(ReadU32(inst.src[1]), 31u)),
ReadU32(inst.src[2]));
break;
case IR::Opcode::SelectU32:
result =
ir.Select(ReadCondition(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]));
break;
case IR::Opcode::SelectMaskU32:
result = ir.Select(ReadMask(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]));
break;
case IR::Opcode::SelectF32Bits:
case IR::Opcode::SelectMaskF32Bits: {
const auto condition = inst.op == IR::Opcode::SelectMaskF32Bits
? ReadMask(inst.src[0])
: ReadCondition(inst.src[0]);
result = ir.Emit(IR::ValueOpcode::SelectF32,
{condition, ReadOperand(inst.src[1], IR::Type::F32),
ReadOperand(inst.src[2], IR::Type::F32)});
break;
}
case IR::Opcode::SelectU64:
result =
select_u64(ReadCondition(inst.src[0]), ReadU64(inst.src[1]), ReadU64(inst.src[2]));
break;
case IR::Opcode::PackLowLowU16:
case IR::Opcode::PackLowHighU16:
case IR::Opcode::PackHighHighU16:
case IR::Opcode::PackU16U32: {
const bool high0 = inst.op == IR::Opcode::PackHighHighU16;
const bool high1 = inst.op == IR::Opcode::PackLowHighU16 || high0;
const auto lo =
high0 ? ir.ShiftRightLogical(ReadU32(inst.src[0]), imm(16)) : ReadU32(inst.src[0]);
const auto hi =
high1 ? ir.ShiftRightLogical(ReadU32(inst.src[1]), imm(16)) : ReadU32(inst.src[1]);
result =
ir.BitwiseOr(mask(lo, 0xffffu), ir.ShiftLeftLogical(mask(hi, 0xffffu), imm(16)));
break;
}
default: return false;
}
WriteOperand(inst.dst, result);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Memory.cpp
+648
View File
@@ -0,0 +1,648 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include <algorithm>
#include <array>
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
IR::MemoryFlags Translator::AddMemoryInfo(const IR::Instruction& inst) {
const auto index = static_cast<uint32_t>(value_program.memory_info.size());
value_program.memory_info.push_back(inst.memory);
return {.index = index, .pc = inst.pc};
}
IR::U32 Translator::GetResourceDword(uint32_t index, uint32_t dword) {
return ReadScalarCode(index * 4u + dword);
}
IR::Value Translator::GetBufferResource(const IR::MemoryInfo& memory) {
return ir.Emit(IR::ValueOpcode::GetBufferResource,
{GetResourceDword(memory.resource, 0), GetResourceDword(memory.resource, 1),
GetResourceDword(memory.resource, 2), GetResourceDword(memory.resource, 3)});
}
IR::Value Translator::GetAddressResource(IR::Value low, IR::Value high) {
return ir.Emit(IR::ValueOpcode::GetAddressResource, {low, high});
}
Translator::AddressOperands Translator::ReadAddressOperands(const IR::Instruction& inst,
uint32_t first_source) {
const auto low = ReadU32(inst.src[first_source]);
const auto& high_or_base = inst.src[first_source + 1u];
if (inst.memory.kind == IR::ResourceKind::Scratch) {
const auto offset = high_or_base.kind == IR::OperandKind::Register &&
high_or_base.reg.file != IR::RegisterFile::Vector
? ReadU32(high_or_base)
: low;
return {GetAddressResource(offset, IR::Value(0u)), offset, IR::Value(0u)};
}
if (inst.memory.kind == IR::ResourceKind::Global &&
high_or_base.kind == IR::OperandKind::Register &&
high_or_base.reg.file != IR::RegisterFile::Vector) {
const auto base_low = ReadU32(high_or_base);
const auto base_high = ReadU32(OffsetOperand(high_or_base, 1u));
return {GetAddressResource(base_low, base_high), low, IR::Value(0u)};
}
const auto high = ReadU32(high_or_base);
return {GetAddressResource(low, high), low, high};
}
IR::Value Translator::GetScalarAddressResource(uint32_t base) {
return GetAddressResource(ReadScalarCode(base), ReadScalarCode(base + 1u));
}
IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) {
return ir.Emit(IR::ValueOpcode::GetImageResource,
{GetResourceDword(memory.resource, 0), GetResourceDword(memory.resource, 1),
GetResourceDword(memory.resource, 2), GetResourceDword(memory.resource, 3),
GetResourceDword(memory.resource, 4), GetResourceDword(memory.resource, 5),
GetResourceDword(memory.resource, 6), GetResourceDword(memory.resource, 7)});
}
IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) {
return ir.Emit(IR::ValueOpcode::GetSamplerResource,
{GetResourceDword(memory.sampler, 0), GetResourceDword(memory.sampler, 1),
GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)});
}
IR::Value Translator::GetSharedResource(IR::ResourceKind kind) {
return ir.Emit(kind == IR::ResourceKind::Gds ? IR::ValueOpcode::GetGdsResource
: IR::ValueOpcode::GetLdsResource);
}
IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) {
std::array<IR::Value, 13> components {};
components[0] = ReadRawU32(PlainOperand(base));
const auto nsa_components =
std::min(inst.memory.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
for (uint32_t index = 1; index < components.size(); index++) {
if (index - 1u < nsa_components) {
components[index] =
ir.GetVectorReg(static_cast<IR::VectorReg>(inst.memory.image_nsa_addr[index - 1u]));
} else {
components[index] = ReadRawU32(OffsetOperand(PlainOperand(base), index));
}
}
return ir.Emit(IR::ValueOpcode::MakeImageAddress,
{components[0], components[1], components[2], components[3], components[4],
components[5], components[6], components[7], components[8], components[9],
components[10], components[11], components[12]});
}
IR::Value Translator::ConstructU32x4(const IR::Operand& base, uint32_t count) {
std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u),
IR::Value(0u)};
for (uint32_t index = 0; index < std::min(count, 4u); index++) {
components[index] = ReadRawU32(OffsetOperand(PlainOperand(base), index));
}
return ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
{components[0], components[1], components[2], components[3]});
}
void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
uint32_t component_limit) {
const auto mask = dmask != 0u ? dmask : 1u;
uint32_t dst_index = 0;
for (uint32_t component = 0; component < component_limit; component++) {
if (((mask >> component) & 1u) == 0u) {
continue;
}
WriteOperand(
OffsetOperand(dst, dst_index++),
ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {value, IR::Value(component)}));
}
}
IR::ValueOpcode Translator::ImageAtomicOpcode(IR::Opcode opcode) {
switch (opcode) {
case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::ImageAtomicIAdd32;
case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::ImageAtomicUMin32;
case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::ImageAtomicUMax32;
case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::ImageAtomicAnd32;
case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::ImageAtomicOr32;
case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::ImageAtomicXor32;
default: EXIT("invalid image atomic opcode");
}
}
BufferAddressValues Translator::ReadBufferAddress(const IR::Instruction& inst,
uint32_t first_source) {
uint32_t cursor = first_source;
const auto next = [&]() {
return cursor < inst.src_count ? ReadU32(inst.src[cursor++]) : IR::U32(IR::Value(0u));
};
const auto index = inst.memory.idxen ? next() : IR::U32(IR::Value(0u));
const auto offset = inst.memory.offen ? next() : IR::U32(IR::Value(0u));
const auto soffset = next();
return {index, offset, soffset};
}
IR::U32 Translator::WidenSubdword(IR::Value value, uint32_t bits, bool sign) {
IR::U32 widened = bits == 8u ? IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U8, {value}))
: IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {value}));
if (sign) {
widened = IR::U32(
ir.Emit(IR::ValueOpcode::BitFieldSExtract, {widened, IR::Value(0u), IR::Value(bits)}));
}
return widened;
}
IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) {
return bits == 8u ? ir.Emit(IR::ValueOpcode::ConvertU8U32, {value})
: ir.Emit(IR::ValueOpcode::ConvertU16U32, {value});
}
IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) {
switch (opcode) {
case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::BufferAtomicSwap32;
case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::BufferAtomicIAdd32;
case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::BufferAtomicISub32;
case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::BufferAtomicSMin32;
case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::BufferAtomicUMin32;
case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::BufferAtomicSMax32;
case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::BufferAtomicUMax32;
case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::BufferAtomicAnd32;
case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::BufferAtomicOr32;
case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::BufferAtomicXor32;
case IR::Opcode::AtomicFMinF32: return IR::ValueOpcode::BufferAtomicFMin32;
case IR::Opcode::AtomicFMaxF32: return IR::ValueOpcode::BufferAtomicFMax32;
default: EXIT("invalid buffer atomic opcode");
}
}
IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) {
if (gds) {
switch (opcode) {
case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::GdsAtomicSwap32;
case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::GdsAtomicIAdd32;
case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::GdsAtomicISub32;
case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::GdsAtomicSMin32;
case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::GdsAtomicUMin32;
case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::GdsAtomicSMax32;
case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::GdsAtomicUMax32;
case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::GdsAtomicAnd32;
case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::GdsAtomicOr32;
case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::GdsAtomicXor32;
default: EXIT("invalid GDS atomic opcode");
}
}
switch (opcode) {
case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32;
case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32;
case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::SharedAtomicISub32;
case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::SharedAtomicSMin32;
case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::SharedAtomicUMin32;
case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::SharedAtomicSMax32;
case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::SharedAtomicUMax32;
case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32;
case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32;
case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32;
default: EXIT("invalid LDS atomic opcode");
}
}
bool Translator::IsBufferLoadOperation(IR::Opcode opcode) {
return opcode == IR::Opcode::BufferLoadUbyte || opcode == IR::Opcode::BufferLoadSbyte ||
opcode == IR::Opcode::BufferLoadUshort || opcode == IR::Opcode::BufferLoadSshort ||
opcode == IR::Opcode::BufferLoadDword;
}
bool Translator::TranslateScalarMemory(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::LoadSrtDword: {
const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource);
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::ReadConst, {resource, ReadU32(inst.src[0])}));
return true;
}
case IR::Opcode::SBufferLoadDword: {
const auto resource = GetBufferResource(inst.memory);
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ReadConstBuffer,
{resource, ReadU32(inst.src[0])}, AddMemoryInfo(inst)));
return true;
}
case IR::Opcode::SLoadDword: {
const auto resource = GetScalarAddressResource(inst.memory.resource);
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::LoadAddressU32,
{resource, ReadU32(inst.src[0]), IR::Value(0u), IR::Value(true)},
AddMemoryInfo(inst)));
return true;
}
default: return false;
}
}
bool Translator::TranslateBufferLoad(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot) {
IR::ValueOpcode opcode;
uint32_t bits;
bool sign;
switch (inst.op) {
case IR::Opcode::BufferLoadUbyte:
case IR::Opcode::BufferLoadSbyte:
opcode = IR::ValueOpcode::LoadBufferU8;
bits = 8u;
sign = inst.op == IR::Opcode::BufferLoadSbyte;
break;
case IR::Opcode::BufferLoadUshort:
case IR::Opcode::BufferLoadSshort:
opcode = IR::ValueOpcode::LoadBufferU16;
bits = 16u;
sign = inst.op == IR::Opcode::BufferLoadSshort;
break;
case IR::Opcode::BufferLoadDword:
opcode = IR::ValueOpcode::LoadBufferU32;
bits = 32u;
sign = false;
break;
default: return false;
}
const auto resource = GetBufferResource(inst.memory);
const auto address =
address_snapshot != nullptr ? *address_snapshot : ReadBufferAddress(inst, 0);
const auto loaded =
ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, ir.GetExec()},
AddMemoryInfo(inst));
WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
return true;
}
bool Translator::TranslateBufferStore(const IR::Instruction& inst) {
const auto resource = GetBufferResource(inst.memory);
const auto address = ReadBufferAddress(inst, 1);
const auto data = ReadU32(inst.src[0]);
IR::ValueOpcode opcode;
IR::Value value;
switch (inst.op) {
case IR::Opcode::BufferStoreByte:
opcode = IR::ValueOpcode::StoreBufferU8;
value = NarrowSubdword(data, 8u);
break;
case IR::Opcode::BufferStoreShort:
opcode = IR::ValueOpcode::StoreBufferU16;
value = NarrowSubdword(data, 16u);
break;
case IR::Opcode::BufferStoreDword:
opcode = IR::ValueOpcode::StoreBufferU32;
value = data;
break;
default: return false;
}
ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, value, ir.GetExec()},
AddMemoryInfo(inst));
return true;
}
bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) {
IR::Value result;
switch (inst.memory.kind) {
case IR::ResourceKind::Buffer: {
const auto resource = GetBufferResource(inst.memory);
const auto address = ReadBufferAddress(inst, 1);
result = ir.Emit(BufferAtomicOpcode(inst.op),
{resource, address.index, address.offset, address.soffset,
ReadU32(inst.src[0]), ir.GetExec()},
AddMemoryInfo(inst));
break;
}
case IR::ResourceKind::Image:
case IR::ResourceKind::ImageUint:
case IR::ResourceKind::StorageImage:
case IR::ResourceKind::StorageImageUint: {
const auto resource = GetImageResource(inst.memory);
const auto address = MakeImageAddress(inst, inst.src[1]);
const auto flags = AddMemoryInfo(inst);
result = ir.Emit(ImageAtomicOpcode(inst.op),
{resource, address, ReadU32(inst.src[0]), ir.GetExec()}, flags);
break;
}
case IR::ResourceKind::Lds:
case IR::ResourceKind::Gds: {
const bool gds = inst.memory.kind == IR::ResourceKind::Gds;
const auto resource = GetSharedResource(inst.memory.kind);
const auto address =
ir.IAdd(ReadU32(inst.src[1]), IR::U32(IR::Value(inst.memory.offset)));
result = ir.Emit(SharedAtomicOpcode(inst.op, gds),
{resource, address, ReadU32(inst.src[0]), ir.GetExec()},
AddMemoryInfo(inst));
break;
}
default: return false;
}
if (inst.dst.kind != IR::OperandKind::Null) {
WriteOperand(inst.dst, result);
}
return true;
}
bool Translator::TranslateFlatLoad(const IR::Instruction& inst) {
IR::ValueOpcode opcode;
uint32_t bits;
bool sign;
switch (inst.op) {
case IR::Opcode::FlatLoadUbyte:
case IR::Opcode::FlatLoadSbyte:
opcode = IR::ValueOpcode::LoadAddressU8;
bits = 8u;
sign = inst.op == IR::Opcode::FlatLoadSbyte;
break;
case IR::Opcode::FlatLoadUshort:
case IR::Opcode::FlatLoadSshort:
opcode = IR::ValueOpcode::LoadAddressU16;
bits = 16u;
sign = inst.op == IR::Opcode::FlatLoadSshort;
break;
case IR::Opcode::FlatLoadDword:
opcode = IR::ValueOpcode::LoadAddressU32;
bits = 32u;
sign = false;
break;
default: return false;
}
const auto address = ReadAddressOperands(inst, 0);
const auto active = ir.GetExec();
const auto count = bits == 32u ? std::min(inst.memory.data_dwords, 4u) : 1u;
for (uint32_t index = 0; index < count; index++) {
auto component = inst;
component.memory.offset += index * 4u;
component.memory.data_dwords = 1u;
component.memory.component_index = index;
const auto loaded = ir.Emit(opcode, {address.resource, address.low, address.high, active},
AddMemoryInfo(component));
WriteOperand(OffsetOperand(inst.dst, index),
bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
}
return true;
}
bool Translator::TranslateFlatStore(const IR::Instruction& inst) {
const auto data = ReadU32(inst.src[0]);
const auto address = ReadAddressOperands(inst, 1);
IR::ValueOpcode opcode;
IR::Value value;
switch (inst.op) {
case IR::Opcode::FlatStoreByte:
opcode = IR::ValueOpcode::StoreAddressU8;
value = NarrowSubdword(data, 8u);
break;
case IR::Opcode::FlatStoreShort:
opcode = IR::ValueOpcode::StoreAddressU16;
value = NarrowSubdword(data, 16u);
break;
case IR::Opcode::FlatStoreDword:
opcode = IR::ValueOpcode::StoreAddressU32;
value = data;
break;
default: return false;
}
ir.Emit(opcode, {address.resource, address.low, address.high, value, ir.GetExec()},
AddMemoryInfo(inst));
return true;
}
bool Translator::TranslateImageMemory(const IR::Instruction& inst) {
const bool image = inst.memory.kind == IR::ResourceKind::Image ||
inst.memory.kind == IR::ResourceKind::ImageUint ||
inst.memory.kind == IR::ResourceKind::StorageImage ||
inst.memory.kind == IR::ResourceKind::StorageImageUint;
if (!image) {
return false;
}
const auto resource = GetImageResource(inst.memory);
const auto address =
MakeImageAddress(inst, inst.op == IR::Opcode::ImageStore ? inst.src[1] : inst.src[0]);
const auto flags = AddMemoryInfo(inst);
switch (inst.op) {
case IR::Opcode::ImageGetResinfo: {
const auto result =
ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, flags);
WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
return true;
}
case IR::Opcode::ImageGetLod: {
const auto sampler = GetSamplerResource(inst.memory);
const auto result =
ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, flags);
WriteImageComponents(inst.dst, result, inst.memory.dmask, 2u);
return true;
}
case IR::Opcode::ImageLoad: {
const auto result =
ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, flags);
WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
return true;
}
case IR::Opcode::ImageStore: {
const auto data = ConstructU32x4(inst.src[0], inst.memory.data_dwords);
ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()}, flags);
return true;
}
case IR::Opcode::ImageSample:
case IR::Opcode::ImageGather4: {
const auto sampler = GetSamplerResource(inst.memory);
const auto opcode = inst.op == IR::Opcode::ImageSample
? IR::ValueOpcode::ImageSampleRaw
: IR::ValueOpcode::ImageGatherRaw;
const auto result = ir.Emit(opcode, {resource, sampler, address}, flags);
const bool dref =
(inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u;
if (inst.op == IR::Opcode::ImageSample && dref) {
const auto component =
ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)});
for (uint32_t index = 0; index < inst.memory.data_dwords; index++) {
WriteOperand(OffsetOperand(inst.dst, index), component);
}
} else if (inst.op == IR::Opcode::ImageGather4) {
for (uint32_t index = 0; index < inst.memory.data_dwords; index++) {
WriteOperand(OffsetOperand(inst.dst, index),
ir.Emit(IR::ValueOpcode::CompositeExtractU32x4,
{result, IR::Value(index)}));
}
} else {
WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u);
}
return true;
}
default: return false;
}
}
bool Translator::TranslateSharedMemory(const IR::Instruction& inst) {
const bool shared =
inst.memory.kind == IR::ResourceKind::Lds || inst.memory.kind == IR::ResourceKind::Gds;
if (!shared && inst.op != IR::Opcode::DsSwizzleB32) {
return false;
}
const bool gds = inst.memory.kind == IR::ResourceKind::Gds;
const auto resource = shared ? GetSharedResource(inst.memory.kind) : IR::Value {};
const auto shared_address = [&](uint32_t source) {
return ir.IAdd(ReadU32(inst.src[source]), IR::U32(IR::Value(inst.memory.offset)));
};
switch (inst.op) {
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32: {
IR::ValueOpcode opcode;
uint32_t bits;
bool sign;
if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) {
opcode = gds ? IR::ValueOpcode::LoadGdsU8 : IR::ValueOpcode::LoadSharedU8;
bits = 8u;
sign = inst.op == IR::Opcode::DsReadSbyte;
} else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) {
opcode = gds ? IR::ValueOpcode::LoadGdsU16 : IR::ValueOpcode::LoadSharedU16;
bits = 16u;
sign = inst.op == IR::Opcode::DsReadSshort;
} else {
opcode = gds ? IR::ValueOpcode::LoadGdsU32 : IR::ValueOpcode::LoadSharedU32;
bits = 32u;
sign = false;
}
const auto loaded =
ir.Emit(opcode, {resource, shared_address(0), ir.GetExec()}, AddMemoryInfo(inst));
WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
return true;
}
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32: {
const auto data = ReadU32(inst.src[0]);
const auto address = shared_address(1);
IR::ValueOpcode opcode;
IR::Value value;
if (inst.op == IR::Opcode::DsWriteByte) {
opcode = gds ? IR::ValueOpcode::WriteGdsU8 : IR::ValueOpcode::WriteSharedU8;
value = NarrowSubdword(data, 8u);
} else if (inst.op == IR::Opcode::DsWriteShort) {
opcode = gds ? IR::ValueOpcode::WriteGdsU16 : IR::ValueOpcode::WriteSharedU16;
value = NarrowSubdword(data, 16u);
} else {
opcode = gds ? IR::ValueOpcode::WriteGdsU32 : IR::ValueOpcode::WriteSharedU32;
value = data;
}
ir.Emit(opcode, {resource, address, value, ir.GetExec()}, AddMemoryInfo(inst));
return true;
}
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32: {
const auto opcode =
gds ? (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::GdsAtomicFMin32
: IR::ValueOpcode::GdsAtomicFMax32)
: (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::SharedAtomicFMin32
: IR::ValueOpcode::SharedAtomicFMax32);
ir.Emit(opcode,
{resource, shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]),
ir.GetExec()},
AddMemoryInfo(inst));
return true;
}
case IR::Opcode::DsAppend:
case IR::Opcode::DsConsume: {
const bool append = inst.op == IR::Opcode::DsAppend;
const auto opcode =
gds ? (append ? IR::ValueOpcode::GdsDataAppend : IR::ValueOpcode::GdsDataConsume)
: (append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume);
WriteOperand(inst.dst, ir.Emit(opcode,
{resource, ReadU32(inst.src[0]), ir.GetExec(),
ir.GetExecLo(), ir.GetExecHi()},
AddMemoryInfo(inst)));
return true;
}
case IR::Opcode::DsWriteAddtidB32:
case IR::Opcode::DsReadAddtidB32: {
const auto m0_index = inst.op == IR::Opcode::DsWriteAddtidB32 ? 1u : 0u;
const auto base =
ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu)));
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
const auto address = ir.IAdd(ir.IAdd(base, IR::U32(IR::Value(inst.memory.offset))),
ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u))));
if (inst.op == IR::Opcode::DsWriteAddtidB32) {
ir.Emit(IR::ValueOpcode::WriteSharedU32,
{resource, address, ReadU32(inst.src[0]), ir.GetExec()},
AddMemoryInfo(inst));
} else {
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::LoadSharedU32,
{resource, address, ir.GetExec()}, AddMemoryInfo(inst)));
}
return true;
}
case IR::Opcode::DsSwizzleB32:
WriteOperand(inst.dst,
ir.Emit(IR::ValueOpcode::SwizzleU32,
{ReadU32(inst.src[0]), ReadU32(inst.src[1]), ir.GetExec()}));
return true;
default: return false;
}
}
bool Translator::TranslateMemoryOperation(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot) {
switch (inst.op) {
case IR::Opcode::LoadSrtDword:
case IR::Opcode::SLoadDword:
case IR::Opcode::SBufferLoadDword: return TranslateScalarMemory(inst);
case IR::Opcode::BufferLoadUbyte:
case IR::Opcode::BufferLoadSbyte:
case IR::Opcode::BufferLoadUshort:
case IR::Opcode::BufferLoadSshort:
case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst, address_snapshot);
case IR::Opcode::BufferStoreByte:
case IR::Opcode::BufferStoreShort:
case IR::Opcode::BufferStoreDword: return TranslateBufferStore(inst);
case IR::Opcode::AtomicSwapU32:
case IR::Opcode::AtomicAddU32:
case IR::Opcode::AtomicSubU32:
case IR::Opcode::AtomicSMinI32:
case IR::Opcode::AtomicUMinU32:
case IR::Opcode::AtomicSMaxI32:
case IR::Opcode::AtomicUMaxU32:
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32:
case IR::Opcode::AtomicFMinF32:
case IR::Opcode::AtomicFMaxF32: return TranslateAtomicMemory(inst);
case IR::Opcode::FlatLoadUbyte:
case IR::Opcode::FlatLoadSbyte:
case IR::Opcode::FlatLoadUshort:
case IR::Opcode::FlatLoadSshort:
case IR::Opcode::FlatLoadDword: return TranslateFlatLoad(inst);
case IR::Opcode::FlatStoreByte:
case IR::Opcode::FlatStoreShort:
case IR::Opcode::FlatStoreDword: return TranslateFlatStore(inst);
case IR::Opcode::ImageGetResinfo:
case IR::Opcode::ImageGetLod:
case IR::Opcode::ImageLoad:
case IR::Opcode::ImageStore:
case IR::Opcode::ImageSample:
case IR::Opcode::ImageGather4: return TranslateImageMemory(inst);
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32:
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32:
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32:
case IR::Opcode::DsSwizzleB32:
case IR::Opcode::DsConsume:
case IR::Opcode::DsAppend:
case IR::Opcode::DsWriteAddtidB32:
case IR::Opcode::DsReadAddtidB32: return TranslateSharedMemory(inst);
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/frontend/translate/Translate.cpp
+927
View File
@@ -0,0 +1,927 @@
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
#include <algorithm>
#include <array>
#include <fmt/format.h>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::Frontend {
namespace Detail {
bool Fail(std::string* error, const std::string& message) {
if (error != nullptr) {
*error = message;
}
return false;
}
IR::Operand Translator::OffsetOperand(const IR::Operand& operand, uint32_t offset) {
if (offset == 0 || operand.kind != IR::OperandKind::Register) {
return operand;
}
auto result = operand;
result.reg.index += offset;
result.sdwa_sel = 6;
result.sdwa_dst_unused = 2;
result.omod = 0;
result.sdwa_sext = false;
result.op_sel = false;
result.op_sel_hi = false;
result.negate = false;
result.negate_hi = false;
result.absolute = false;
result.clamp = false;
result.dpp_ctrl = 0;
result.dpp_row_mask = 0xf;
result.dpp_bank_mask = 0xf;
result.dpp_fetch_inactive = false;
result.dpp_bound_ctrl = false;
result.dpp = false;
return result;
}
IR::Operand Translator::PlainOperand(const IR::Operand& operand) {
auto result = operand;
result.sdwa_sel = 6;
result.sdwa_dst_unused = 2;
result.omod = 0;
result.sdwa_sext = false;
result.op_sel = false;
result.op_sel_hi = false;
result.negate = false;
result.negate_hi = false;
result.absolute = false;
result.clamp = false;
result.dpp_ctrl = 0;
result.dpp_row_mask = 0xf;
result.dpp_bank_mask = 0xf;
result.dpp_fetch_inactive = false;
result.dpp_bound_ctrl = false;
result.dpp = false;
return result;
}
IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) {
EXIT_IF(operand.kind != IR::OperandKind::PcRelativeU32 &&
operand.kind != IR::OperandKind::PcRelativeHighU32);
const auto address =
ir.Emit(IR::ValueOpcode::IAdd64, {ir.Emit(IR::ValueOpcode::GetShaderBase),
IR::Value(static_cast<uint64_t>(operand.imm))});
const auto pair = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {address});
return IR::U32(
ir.Emit(IR::ValueOpcode::CompositeExtractU32x2,
{pair, IR::Value(operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u)}));
}
std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
if (current_per_invocation_masks) {
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value});
const auto low =
IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)}));
// RDNA2 wave32 masks have one architectural dword; the upper half is unused.
const auto high =
current_wave_size == 32u
? IR::U32(IR::Value(0u))
: IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)}));
return {low, high};
}
IR::U32 Translator::ReadRawU32(const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::ImmediateU32) {
return IR::U32(IR::Value(operand.imm));
}
if (operand.kind == IR::OperandKind::PcRelativeU32 ||
operand.kind == IR::OperandKind::PcRelativeHighU32) {
return ReadPcRelativeU32(operand);
}
if (operand.kind == IR::OperandKind::Null) {
return IR::U32(IR::Value(0u));
}
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
return ir.GetScalarReg(static_cast<IR::ScalarReg>(operand.reg.index));
case IR::RegisterFile::Vector:
return ir.GetVectorReg(static_cast<IR::VectorReg>(operand.reg.index));
case IR::RegisterFile::Vcc: return operand.reg.index == 0 ? ir.GetVccLo() : ir.GetVccHi();
case IR::RegisterFile::M0: return ir.GetM0();
case IR::RegisterFile::Exec:
return operand.reg.index == 0 ? ir.GetExecLo() : ir.GetExecHi();
case IR::RegisterFile::Scc:
return ir.Select(ir.GetScc(), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)));
}
EXIT("invalid register file used as a raw U32 source");
}
// Scalar operands share one encoded namespace with VCC, M0, and EXEC aliases.
IR::U32 Translator::ReadScalarCode(uint32_t code) {
if (code < 106u) {
return ir.GetScalarReg(static_cast<IR::ScalarReg>(code));
}
switch (code) {
case 106u: return ir.GetVccLo();
case 107u: return ir.GetVccHi();
case 124u: return ir.GetM0();
case 126u:
case 127u: {
const auto mask = BallotMask(ir.GetExec());
return mask[code - 126u];
}
default: return IR::U32(IR::Value(0u));
}
}
IR::U32 Translator::ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value) {
if (operand.dpp) {
const IR::DppMoveFlags flags {
.control = static_cast<uint16_t>(operand.dpp_ctrl),
.row_mask = static_cast<uint8_t>(operand.dpp_row_mask),
.bank_mask = static_cast<uint8_t>(operand.dpp_bank_mask),
.fetch_inactive = operand.dpp_fetch_inactive,
.bound_control = operand.dpp_bound_ctrl,
};
value = IR::U32(ir.Emit(IR::ValueOpcode::DppMoveU32, {value, ir.GetExec()}, flags));
}
if (operand.sdwa_sel != 6u) {
uint32_t offset = 0;
uint32_t width = 0;
if (operand.sdwa_sel <= 3u) {
offset = operand.sdwa_sel * 8u;
width = 8u;
} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
offset = operand.sdwa_sel == 5u ? 16u : 0u;
width = 16u;
} else {
EXIT("invalid SDWA source selector");
}
const auto opcode = operand.sdwa_sext ? IR::ValueOpcode::BitFieldSExtract
: IR::ValueOpcode::BitFieldUExtract;
value = IR::U32(ir.Emit(opcode, {value, IR::Value(offset), IR::Value(width)}));
}
return value;
}
IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) {
if (type == IR::Type::U16) {
return ir.Emit(IR::ValueOpcode::ConvertU16U32,
{ApplyBitSourceModifiers(operand, ReadRawU32(operand))});
}
if (type == IR::Type::F16) {
const auto bits = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32,
{ApplyBitSourceModifiers(operand, ReadRawU32(operand))}));
return ir.Emit(IR::ValueOpcode::BitCastF16U16, {bits});
}
if (type == IR::Type::U1) {
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
case IR::RegisterFile::Scc: return ir.GetScc();
case IR::RegisterFile::Exec: return ir.GetExec();
case IR::RegisterFile::Vcc: return ir.GetVcc();
default: break;
}
return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u)));
}
if (type == IR::Type::U64) {
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Exec) {
return ir.PackUint2x32(ir.GetExecLo(), ir.GetExecHi());
}
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Vcc) {
return ir.PackUint2x32(ir.GetVccLo(), ir.GetVccHi());
}
const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
IR::U32 high(IR::Value(0u));
if (operand.kind == IR::OperandKind::Register) {
high = ReadRawU32(OffsetOperand(operand, 1));
} else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) {
high = IR::U32(IR::Value(0xffffffffu));
}
return ir.PackUint2x32(low, high);
}
auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) {
auto value = ir.BitCastF32(bits);
if (operand.absolute) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
}
if (operand.negate) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
}
return value;
}
if (operand.absolute) {
bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7fffffffu)));
}
if (operand.negate) {
bits = ir.BitwiseXor(bits, IR::U32(IR::Value(0x80000000u)));
}
if (!TypesOverlap(type, IR::Type::U32)) {
EXIT("opcode %s at 0x%08x requested unsupported operand type %s",
IR::OpcodeName(current_opcode).data(), current_pc, IR::TypeName(type).c_str());
}
return bits;
}
void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
if (operand.kind == IR::OperandKind::Null) {
return;
}
EXIT_IF(operand.kind != IR::OperandKind::Register);
if (operand.sdwa_sel != 6u) {
uint32_t offset = 0;
uint32_t width = 0;
if (operand.sdwa_sel <= 3u) {
offset = operand.sdwa_sel * 8u;
width = 8u;
} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
offset = operand.sdwa_sel == 5u ? 16u : 0u;
width = 16u;
} else {
EXIT("invalid SDWA destination selector");
}
switch (operand.sdwa_dst_unused) {
case 0:
value =
IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert,
{IR::Value(0u), value, IR::Value(offset), IR::Value(width)}));
break;
case 1: {
const auto extended = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldSExtract,
{value, IR::Value(0u), IR::Value(width)}));
value = ir.ShiftLeftLogical(extended, IR::U32(IR::Value(offset)));
break;
}
case 2: {
const uint32_t field_mask = width == 32u ? 0xffffffffu : (1u << width) - 1u;
const auto inserted =
ir.ShiftLeftLogical(ir.BitwiseAnd(value, IR::U32(IR::Value(field_mask))),
IR::U32(IR::Value(offset)));
const auto cleared = ir.BitwiseAnd(ReadRawU32(PlainOperand(operand)),
IR::U32(IR::Value(~(field_mask << offset))));
value = ir.BitwiseOr(cleared, inserted);
} break;
default: EXIT("reserved SDWA DST_U mode");
}
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
ir.SetScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), value);
break;
case IR::RegisterFile::Vector: {
const auto reg = static_cast<IR::VectorReg>(operand.reg.index);
const auto old = ir.GetVectorReg(reg);
if (operand.dpp) {
const IR::DppMoveFlags flags {
.control = static_cast<uint16_t>(operand.dpp_ctrl),
.row_mask = static_cast<uint8_t>(operand.dpp_row_mask),
.bank_mask = static_cast<uint8_t>(operand.dpp_bank_mask),
.fetch_inactive = operand.dpp_fetch_inactive,
.bound_control = operand.dpp_bound_ctrl,
};
value = IR::U32(
ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()}, flags));
} else {
value = ir.Select(ir.GetExec(), value, old);
}
ir.SetVectorReg(reg, value);
break;
}
case IR::RegisterFile::Vcc:
if (operand.reg.index == 0) {
ir.SetVccLo(IR::U32(value));
ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi()));
} else {
ir.SetVccHi(IR::U32(value));
ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value)));
}
break;
case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break;
case IR::RegisterFile::Exec:
if (operand.reg.index == 0) {
ir.SetExecLo(IR::U32(value));
ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi()));
} else {
ir.SetExecHi(IR::U32(value));
ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value)));
}
break;
case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break;
}
}
IR::U64 Translator::ExpandWholeQuadMask(IR::U64 value) {
const auto shifted1 =
IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(1u)}));
const auto shifted2 =
IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(2u)}));
const auto shifted3 =
IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(3u)}));
const auto merged1 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {value, shifted1});
const auto merged2 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {merged1, shifted2});
const auto merged3 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {merged2, shifted3});
const auto low_bits =
ir.Emit(IR::ValueOpcode::BitwiseAnd64, {merged3, IR::Value(0x1111111111111111ull)});
return IR::U64(ir.Emit(IR::ValueOpcode::IMul64, {low_bits, IR::Value(uint64_t {0xfull})}));
}
IR::F32 Translator::ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value) {
if (operand.omod != 0u) {
float multiplier = 0.5f;
switch (operand.omod) {
case 1u: multiplier = 2.0f; break;
case 2u: multiplier = 4.0f; break;
default: break;
}
value = IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(multiplier)}));
}
if (operand.clamp) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPSaturate32, {value}));
}
return value;
}
void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) {
if (operand.kind == IR::OperandKind::Null) {
return;
}
auto type = value.GetType();
if (type == IR::Type::F32) {
value = ApplyF32ResultModifiers(operand, IR::F32(value));
type = IR::Type::F32;
}
if (type == IR::Type::Opaque) {
EXIT("opcode %s at 0x%08x produced an untyped value", IR::OpcodeName(current_opcode).data(),
current_pc);
}
if (type == IR::Type::U1) {
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
case IR::RegisterFile::Scc: ir.SetScc(IR::U1(value)); return;
case IR::RegisterFile::Exec: {
const auto mask = BallotMask(IR::U1(value));
ir.SetExec(IR::U1(value));
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
return;
}
case IR::RegisterFile::Vcc: {
const auto mask = BallotMask(IR::U1(value));
ir.SetVcc(IR::U1(value));
ir.SetVccLo(mask[0]);
ir.SetVccHi(mask[1]);
return;
}
default:
WriteRawU32(operand, ir.Select(IR::U1(value), IR::U32(IR::Value(1u)),
IR::U32(IR::Value(0u))));
return;
}
}
if (type == IR::Type::U16) {
WriteRawU32(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {IR::U16(value)})));
return;
}
if (type == IR::Type::F16) {
const auto bits = IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {value}));
WriteRawU32(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {bits})));
return;
}
if (type == IR::Type::U64) {
const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value});
const auto low = ir.CompositeExtract(unpacked, 0);
const auto high = ir.CompositeExtract(unpacked, 1);
if (operand.kind == IR::OperandKind::Register) {
const auto thread_bit = ThreadBit(low, high);
switch (operand.reg.file) {
case IR::RegisterFile::Exec:
ir.SetExec(thread_bit);
ir.SetExecLo(low);
ir.SetExecHi(high);
return;
case IR::RegisterFile::Vcc:
ir.SetVcc(thread_bit);
ir.SetVccLo(low);
ir.SetVccHi(high);
return;
case IR::RegisterFile::Scalar:
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index),
thread_bit);
break;
default: break;
}
}
WriteRawU32(operand, low);
WriteRawU32(OffsetOperand(operand, 1), high);
return;
}
if (type == IR::Type::F32) {
WriteRawU32(operand, ir.BitCastU32(IR::F32(value)));
return;
}
EXIT_IF(type != IR::Type::U32);
WriteRawU32(operand, IR::U32(value));
}
IR::U32 Translator::PackHalf2x16(IR::F32 low, IR::F32 high) {
const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {low, high});
return IR::U32(ir.Emit(IR::ValueOpcode::PackHalf2x16, {pair}));
}
void Translator::WriteF16(const IR::Operand& operand, IR::F32 value) {
value = ApplyF32ResultModifiers(operand, value);
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
const auto bits = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16,
{IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {half}))}));
auto merged = bits;
if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert,
{ReadRawU32(PlainOperand(operand)), bits,
IR::Value(operand.sdwa_sel == 5u ? 16u : 0u), IR::Value(16u)}));
}
auto raw = operand;
raw.sdwa_sel = 6u;
raw.omod = 0u;
raw.op_sel = false;
raw.clamp = false;
WriteRawU32(raw, merged);
}
void Translator::WriteU16(const IR::Operand& operand, IR::U32 value) {
auto merged = ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu)));
if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert,
{ReadRawU32(PlainOperand(operand)), merged,
IR::Value(operand.sdwa_sel == 5u ? 16u : 0u), IR::Value(16u)}));
}
auto raw = operand;
raw.sdwa_sel = 6u;
raw.omod = 0u;
raw.op_sel = false;
raw.clamp = false;
WriteRawU32(raw, merged);
}
IR::U32 Translator::ReadU32(const IR::Operand& operand) {
return IR::U32(ReadOperand(operand, IR::Type::U32));
}
IR::U64 Translator::ReadU64(const IR::Operand& operand) {
return IR::U64(ReadOperand(operand, IR::Type::U64));
}
IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane) {
auto raw_operand = operand;
raw_operand.sdwa_sel = 6;
raw_operand.sdwa_sext = false;
const auto bits = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand));
uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
offset = operand.sdwa_sel == 5u ? 16u : 0u;
}
const auto half_u32 = IR::U32(
ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(16u)}));
const auto half_u16 = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, {half_u32}));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::BitCastF16U16, {half_u16}));
auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
if (operand.absolute) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
}
if (high_lane ? operand.negate_hi : operand.negate) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value}));
}
return value;
}
IR::F32 Translator::ReadF16AsF32(const IR::Operand& operand) {
return ReadF16LaneAsF32(operand, false);
}
IR::F32 Translator::ReadMixF32(const IR::Operand& operand) {
if (operand.op_sel_hi) {
return ReadF16AsF32(operand);
}
auto value_operand = operand;
value_operand.op_sel = false;
value_operand.op_sel_hi = false;
value_operand.negate_hi = false;
return IR::F32(ReadOperand(value_operand, IR::Type::F32));
}
IR::U32 Translator::ReadU16LaneRaw(const IR::Operand& operand, bool high_lane) {
auto raw_operand = operand;
raw_operand.sdwa_sel = 6;
raw_operand.sdwa_sext = false;
const auto bits = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand));
uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u;
uint32_t width = 16u;
if (operand.sdwa_sel <= 3u) {
offset = operand.sdwa_sel * 8u;
width = 8u;
} else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) {
offset = operand.sdwa_sel == 5u ? 16u : 0u;
}
return IR::U32(
ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)}));
}
IR::U32 Translator::ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend) {
auto value = ReadU16LaneRaw(operand, high_lane);
if (high_lane ? operand.negate_hi : operand.negate) {
value = ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(0u)), value), IR::U32(IR::Value(0xffffu)));
}
if (sign_extend || operand.sdwa_sext) {
value = IR::U32(
ir.Emit(IR::ValueOpcode::BitFieldSExtract, {value, IR::Value(0u), IR::Value(16u)}));
}
return value;
}
IR::U32 Translator::ReadU16AsU32(const IR::Operand& operand, bool sign_extend) {
return ReadU16LaneAsU32(operand, false, sign_extend);
}
IR::U32 Translator::ReadF16LaneBits(const IR::Operand& operand, bool high_lane) {
auto value = ReadU16LaneRaw(operand, high_lane);
if (operand.absolute) {
value = ir.BitwiseAnd(value, IR::U32(IR::Value(0x7fffu)));
}
if (high_lane ? operand.negate_hi : operand.negate) {
value = ir.BitwiseXor(value, IR::U32(IR::Value(0x8000u)));
}
return value;
}
std::array<IR::U32, 2> Translator::UnpackU64(IR::U64 value) {
const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value});
return {ir.CompositeExtract(unpacked, 0), ir.CompositeExtract(unpacked, 1)};
}
IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) {
if (current_per_invocation_masks) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
auto word = low;
if (current_wave_size == 64u) {
const auto high_lane =
IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)}));
word = ir.Select(high_lane, high, low);
}
const auto bit =
ir.BitwiseAnd(ir.ShiftRightLogical(word, ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)))),
IR::U32(IR::Value(1u)));
return ir.INotEqual(bit, IR::U32(IR::Value(0u)));
}
IR::U1 Translator::ReadCondition(const IR::Operand& operand) {
return IR::U1(ReadOperand(operand, IR::Type::U1));
}
IR::U32 Translator::ConditionBit(const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::ImmediateU32) {
return IR::U32(IR::Value(operand.imm & 1u));
}
return ir.Select(ReadCondition(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)));
}
IR::U1 Translator::ReadMask(const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::ImmediateU32) {
return IR::U1(IR::Value(operand.imm != 0u || operand.sext_64));
}
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto high =
current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs
? ir.GetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u))
: IR::U32(IR::Value(0u));
return ThreadBit(ir.GetScalarReg(reg), high);
}
case IR::RegisterFile::Exec: return ir.GetExec();
case IR::RegisterFile::Vcc: return ir.GetVcc();
case IR::RegisterFile::Scc: return ir.GetScc();
default: return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u)));
}
}
void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarReg(reg, mask[0]);
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
}
return;
}
case IR::RegisterFile::Exec: {
const auto mask = BallotMask(value);
ir.SetExec(value);
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
return;
}
case IR::RegisterFile::Vcc: {
const auto mask = BallotMask(value);
ir.SetVcc(value);
ir.SetVccLo(mask[0]);
ir.SetVccHi(mask[1]);
return;
}
case IR::RegisterFile::Scc: ir.SetScc(value); return;
default:
WriteRawU32(operand, ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))));
WriteRawU32(OffsetOperand(operand, 1), IR::U32(IR::Value(0u)));
return;
}
}
void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) {
if (operand.kind != IR::OperandKind::Register || operand.reg.file != IR::RegisterFile::Scalar) {
WriteMask(operand, value);
return;
}
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarReg(reg, mask[0]);
if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
}
}
void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) {
if (operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc) {
WriteOperand(operand, value);
return;
}
// VALU instructions only write lanes enabled by the incoming EXEC mask. This is
// especially important for V_CMPX: replacing EXEC with the ungated comparison would
// reactivate lanes disabled by an enclosing divergent region.
WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
}
bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) {
BufferAddressValues buffer_address_snapshot {};
uint32_t buffer_address_pc = 0;
bool has_buffer_address_snapshot = false;
for (const auto& source_inst: source.instructions) {
current_opcode = source_inst.op;
current_pc = source_inst.pc;
const bool grouped_buffer_load =
IsBufferLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u;
if (grouped_buffer_load &&
(!has_buffer_address_snapshot || buffer_address_pc != source_inst.pc ||
source_inst.memory.component_index == 0u)) {
buffer_address_snapshot = ReadBufferAddress(source_inst, 0);
buffer_address_pc = source_inst.pc;
has_buffer_address_snapshot = true;
} else if (!grouped_buffer_load) {
has_buffer_address_snapshot = false;
}
if (static_cast<size_t>(source_inst.op) >= static_cast<size_t>(IR::Opcode::Count)) {
return Fail(error, "value IR input opcode is out of range");
}
if (TranslateInstruction(source_inst,
grouped_buffer_load ? &buffer_address_snapshot : nullptr)) {
continue;
}
return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering",
IR::OpcodeName(source_inst.op), source_inst.pc));
}
return true;
}
bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
std::string* error) {
if (source.terminator.goto_value >= 0) {
if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error, fmt::format("block {} sets an invalid goto variable", source.id));
}
ir.SetGotoVariable(source.terminator.goto_variable,
IR::U1(IR::Value(source.terminator.goto_value != 0)));
}
if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) {
if (source.terminator.indirect_selector_code != UINT32_MAX) {
info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code);
} else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) {
info.indirect_target =
ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr));
} else {
return Fail(error, fmt::format("block {} has no indirect branch selector", source.id));
}
ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target});
return true;
}
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
return true;
}
IR::U1 condition;
const auto mask_non_zero = [&](IR::U1 value) {
const auto mask = BallotMask(value);
return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)));
};
switch (source.terminator.condition) {
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
case CFG::BranchCondition::VccZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetVcc()));
break;
case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break;
case CFG::BranchCondition::ExecZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetExec()));
break;
case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break;
case CFG::BranchCondition::GotoVariable:
if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error,
fmt::format("block {} reads an invalid goto variable", source.id));
}
condition = ir.GetGotoVariable(source.terminator.goto_variable);
break;
case CFG::BranchCondition::Unknown:
return Fail(error, fmt::format("block {} has an unknown branch condition", source.id));
}
info.condition = condition;
ir.Emit(IR::ValueOpcode::Reference, {condition});
return true;
}
} // namespace Detail
bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::string* error) {
result = {};
result.dispatcher_fallback = source.dispatcher_fallback;
const uint32_t wave_size = source.wave_size;
const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Vector) {
vector_limit =
std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + 1u));
}
};
for (const auto& block: source.blocks) {
for (const auto& inst: block.instructions) {
include_vector(inst.dst);
include_vector(inst.dst2);
for (uint32_t index = 0; index < inst.src_count; index++) {
include_vector(inst.src[index]);
}
}
}
result.block_storage.reserve(source.blocks.size() + 1u);
result.blocks.reserve(source.blocks.size() + 1u);
result.block_info.reserve(source.blocks.size() + 1u);
if (!source.blocks.empty()) {
const auto max_id =
std::ranges::max_element(source.blocks, {}, [](const IR::BasicBlock& block) {
return block.id;
})->id;
if (max_id == UINT32_MAX) {
return Detail::Fail(error, "cannot allocate a typed prologue block id");
}
CFG::Terminator terminator;
terminator.kind = CFG::TerminatorKind::Branch;
terminator.true_block = source.blocks.front().id;
result.block_storage.push_back(std::make_unique<IR::Block>());
result.blocks.push_back(result.block_storage.back().get());
result.block_info.push_back({max_id + 1u, source.blocks.front().start_pc,
source.blocks.front().start_pc, std::move(terminator)});
}
for (const auto& source_block: source.blocks) {
result.block_storage.push_back(std::make_unique<IR::Block>());
result.blocks.push_back(result.block_storage.back().get());
result.block_info.push_back(
{source_block.id, source_block.start_pc, source_block.end_pc, source_block.terminator});
}
for (size_t index = 0; index < source.blocks.size(); index++) {
for (const auto successor: source.blocks[index].successors) {
if (successor >= source.blocks.size()) {
return Detail::Fail(error, "value IR block successor is out of range");
}
result.blocks[index + 1u]->AddBranch(result.blocks[successor + 1u]);
}
}
if (!result.blocks.empty()) {
result.blocks.front()->AddBranch(result.blocks[1]);
IR::IREmitter prologue(result.blocks.front());
const auto builtin = [&](IR::StageInputKind kind, uint32_t component = 0u) {
return IR::U32(
prologue.Emit(IR::ValueOpcode::GetBuiltin,
{IR::Value(static_cast<uint32_t>(kind)), IR::Value(component)}));
};
prologue.Emit(IR::ValueOpcode::Prologue);
const IR::U32 zero(IR::Value(0u));
for (uint32_t index = 0; index < IR::NumScalarRegs; index++) {
prologue.SetScalarReg(static_cast<IR::ScalarReg>(index), zero);
}
for (uint32_t index = 0; index < vector_limit; index++) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(index), zero);
}
for (uint32_t index = 0; index < source.user_data_count; index++) {
const auto reg = static_cast<IR::ScalarReg>(source.user_data_base + index);
if (IR::RegIndex(reg) >= IR::NumScalarRegs) {
break;
}
prologue.SetScalarReg(reg, prologue.GetUserData(reg));
}
prologue.SetScc(IR::U1(IR::Value(false)));
prologue.SetExec(IR::U1(IR::Value(true)));
prologue.SetExecLo(IR::U32(IR::Value(0xffffffffu)));
prologue.SetExecHi(IR::U32(IR::Value(source.wave_size > 32u ? 0xffffffffu : 0u)));
prologue.SetVcc(IR::U1(IR::Value(false)));
prologue.SetVccLo(IR::U32(IR::Value(0u)));
prologue.SetVccHi(IR::U32(IR::Value(0u)));
prologue.SetM0(IR::U32(IR::Value(0u)));
if (source.stage == ShaderType::Compute && compute_input_info != nullptr) {
const auto* cs = compute_input_info;
const auto thread_ids =
cs->thread_ids_num > 0 ? std::min<uint32_t>(cs->thread_ids_num, 3u) : 0u;
for (uint32_t index = 0; index < thread_ids; index++) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(index),
builtin(IR::StageInputKind::LocalInvocationId, index));
}
uint32_t reg_offset = 0;
for (uint32_t index = 0; index < 3u; index++) {
if (cs->group_id[index]) {
prologue.SetScalarReg(
static_cast<IR::ScalarReg>(cs->workgroup_register + reg_offset++),
builtin(IR::StageInputKind::WorkgroupId, index));
}
}
if (cs->tg_size_en) {
const auto wave_size = cs->wave_size != 0u ? cs->wave_size : 64u;
const auto total_threads = std::max(cs->threads_num[0], 1u) *
std::max(cs->threads_num[1], 1u) *
std::max(cs->threads_num[2], 1u);
const auto waves = std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu);
const auto local_index = builtin(IR::StageInputKind::LocalInvocationIndex);
const auto wave_id = IR::U32(
prologue.Emit(IR::ValueOpcode::UDiv32, {local_index, IR::Value(wave_size)}));
const auto wave_bits = prologue.ShiftLeftLogical(wave_id, IR::U32(IR::Value(20u)));
const auto first_bit =
prologue.Select(prologue.IEqual(wave_id, IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0x80000000u)), IR::U32(IR::Value(0u)));
prologue.SetScalarReg(
static_cast<IR::ScalarReg>(cs->workgroup_register + reg_offset),
prologue.BitwiseOr(prologue.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))),
first_bit));
}
} else if (source.stage == ShaderType::Pixel && pixel_input_info != nullptr) {
const auto* ps = pixel_input_info;
uint32_t reg = ps->ps_system_input_base;
if (ps->ps_pos_x) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
builtin(IR::StageInputKind::FragCoord, 0));
}
if (ps->ps_pos_y) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
builtin(IR::StageInputKind::FragCoord, 1));
}
if (ps->ps_pos_z) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
builtin(IR::StageInputKind::FragCoord, 2));
}
if (ps->ps_pos_w) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(reg++),
builtin(IR::StageInputKind::FragCoord, 3));
}
if (ps->ps_front_face) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(reg),
builtin(IR::StageInputKind::FrontFacing));
}
} else if (source.stage == ShaderType::Vertex && vertex_input_info != nullptr) {
prologue.SetVectorReg(static_cast<IR::VectorReg>(5),
builtin(IR::StageInputKind::VertexIndex));
prologue.SetVectorReg(static_cast<IR::VectorReg>(8),
builtin(IR::StageInputKind::InstanceIndex));
}
}
for (size_t index = 0; index < source.blocks.size(); index++) {
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size,
per_invocation_masks);
if (!translator.TranslateBlock(source.blocks[index], error) ||
!translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u],
error)) {
return false;
}
}
return IR::ValidateValueProgram(result, false, error);
}
} // namespace Libs::Graphics::ShaderRecompiler::Frontend
src/graphics/shader/recompiler/frontend/translate/Translate.h
+12
View File
@@ -0,0 +1,12 @@
#pragma once
#include "graphics/shader/recompiler/ir/ValueProgram.h"
namespace Libs::Graphics::ShaderRecompiler::Frontend {
bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Frontend
src/graphics/shader/recompiler/frontend/translate/Translator.h
+140
View File
@@ -0,0 +1,140 @@
#pragma once
#include "graphics/shader/recompiler/frontend/translate/Translate.h"
#include "graphics/shader/recompiler/ir/IREmitter.h"
#include <array>
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
struct BufferAddressValues {
IR::U32 index;
IR::U32 offset;
IR::U32 soffset;
};
bool Fail(std::string* error, const std::string& message);
class Translator {
public:
Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
uint32_t wave_size, bool per_invocation_masks)
: value_program(program), ir(block), current_vector_limit(vector_limit),
current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {}
bool TranslateBlock(const IR::BasicBlock& source, std::string* error);
bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
std::string* error);
private:
IR::Operand OffsetOperand(const IR::Operand& operand, uint32_t offset);
IR::Operand PlainOperand(const IR::Operand& operand);
IR::U32 ReadPcRelativeU32(const IR::Operand& operand);
std::array<IR::U32, 2> BallotMask(IR::U1 value);
IR::U32 ReadRawU32(const IR::Operand& operand);
IR::U32 ReadScalarCode(uint32_t code);
IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value);
IR::Value ReadOperand(const IR::Operand& operand, IR::Type type);
IR::U1 ThreadBit(IR::U32 low, IR::U32 high);
void WriteRawU32(const IR::Operand& operand, IR::U32 value);
IR::U64 ExpandWholeQuadMask(IR::U64 value);
IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value);
void WriteOperand(const IR::Operand& operand, IR::Value value);
IR::U32 PackHalf2x16(IR::F32 low, IR::F32 high);
void WriteF16(const IR::Operand& operand, IR::F32 value);
void WriteU16(const IR::Operand& operand, IR::U32 value);
IR::U32 ReadU32(const IR::Operand& operand);
IR::U64 ReadU64(const IR::Operand& operand);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
std::array<IR::U32, 2> UnpackU64(IR::U64 value);
IR::U1 ReadCondition(const IR::Operand& operand);
IR::U32 ConditionBit(const IR::Operand& operand);
IR::U1 ReadMask(const IR::Operand& operand);
void WriteMask(const IR::Operand& operand, IR::U1 value);
void WriteMask64(const IR::Operand& operand, IR::U1 value);
void WriteCompareResult(const IR::Operand& operand, IR::U1 value);
IR::MemoryFlags AddMemoryInfo(const IR::Instruction& inst);
IR::ExportFlags AddExportInfo(const IR::Instruction& inst);
struct AddressOperands {
IR::Value resource;
IR::Value low;
IR::Value high;
};
AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source);
IR::U32 GetResourceDword(uint32_t index, uint32_t dword);
IR::Value GetBufferResource(const IR::MemoryInfo& memory);
IR::Value GetAddressResource(IR::Value low, IR::Value high);
IR::Value GetScalarAddressResource(uint32_t base);
IR::Value GetImageResource(const IR::MemoryInfo& memory);
IR::Value GetSamplerResource(const IR::MemoryInfo& memory);
IR::Value GetSharedResource(IR::ResourceKind kind);
IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count);
void WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask,
uint32_t component_limit);
IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
BufferAddressValues ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
IR::Value NarrowSubdword(IR::U32 value, uint32_t bits);
IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode);
IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds);
bool IsBufferLoadOperation(IR::Opcode opcode);
bool TranslateScalarMemory(const IR::Instruction& inst);
bool TranslateBufferLoad(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot);
bool TranslateBufferStore(const IR::Instruction& inst);
bool TranslateAtomicMemory(const IR::Instruction& inst);
bool TranslateFlatLoad(const IR::Instruction& inst);
bool TranslateFlatStore(const IR::Instruction& inst);
bool TranslateImageMemory(const IR::Instruction& inst);
bool TranslateSharedMemory(const IR::Instruction& inst);
IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
uint32_t high_result);
IR::U32 ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound,
float safe_upper, uint32_t lower_result,
uint32_t upper_result);
IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
bool TranslateInstruction(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst,
const BufferAddressValues* address_snapshot = nullptr);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
bool TranslatePerInvocationU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
IR::ValueProgram& value_program;
IR::IREmitter ir;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
bool current_per_invocation_masks = false;
};
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/ir/Block.cpp
+84
View File
@@ -0,0 +1,84 @@
#include "graphics/shader/recompiler/ir/Block.h"
#include <algorithm>
#include <limits>
namespace Libs::Graphics::ShaderRecompiler::IR {
Block::~Block() {
// Instructions own reverse-use links. Unlink every argument while all definitions are still
// alive, then let the list destroy the detached instructions.
for (auto& inst: instructions) {
inst.Invalidate();
}
}
Inst& Block::AppendNewInst(ValueOpcode opcode, std::initializer_list<Value> args, uint64_t flags) {
return *PrependNewInst(end(), opcode, args, flags);
}
Block::iterator Block::PrependNewInst(iterator insertion_point, ValueOpcode opcode,
std::initializer_list<Value> args, uint64_t flags) {
const auto expected = NumArgsOf(opcode);
EXIT_IF(expected != std::numeric_limits<size_t>::max() && expected != args.size());
const auto inst = instructions.emplace(insertion_point, opcode, flags);
inst->SetParent(this);
size_t index = 0;
for (const auto arg: args) {
inst->SetArg(index++, arg);
}
return inst;
}
void Block::AddBranch(Block* block) {
EXIT_IF(block == nullptr || std::ranges::find(successors, block) != successors.end());
EXIT_IF(std::ranges::find(block->predecessors, this) != block->predecessors.end());
successors.push_back(block);
block->predecessors.push_back(this);
}
Block::InstructionList& Block::Instructions() {
return instructions;
}
const Block::InstructionList& Block::Instructions() const {
return instructions;
}
std::span<Block* const> Block::ImmPredecessors() const {
return predecessors;
}
std::span<Block* const> Block::ImmSuccessors() const {
return successors;
}
void Block::SsaSeal() {
ssa_sealed = true;
}
bool Block::IsSsaSealed() const {
return ssa_sealed;
}
Block::iterator Block::begin() {
return instructions.begin();
}
Block::const_iterator Block::begin() const {
return instructions.begin();
}
Block::iterator Block::end() {
return instructions.end();
}
Block::const_iterator Block::end() const {
return instructions.end();
}
bool Block::empty() const {
return instructions.empty();
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/Block.h
+63
View File
@@ -0,0 +1,63 @@
#pragma once
#include "graphics/shader/recompiler/ir/Reg.h"
#include "graphics/shader/recompiler/ir/Value.h"
#include <array>
#include <initializer_list>
#include <list>
#include <span>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
// A standard list provides stable instruction addresses without adding an intrusive-container
// dependency.
class Block {
public:
using InstructionList = std::list<Inst>;
using iterator = InstructionList::iterator;
using const_iterator = InstructionList::const_iterator;
Block() = default;
~Block();
Block(const Block&) = delete;
Block& operator=(const Block&) = delete;
Block(Block&&) = delete;
Block& operator=(Block&&) = delete;
Inst& AppendNewInst(ValueOpcode opcode, std::initializer_list<Value> args = {},
uint64_t flags = 0);
iterator PrependNewInst(iterator insertion_point, ValueOpcode opcode,
std::initializer_list<Value> args = {}, uint64_t flags = 0);
void AddBranch(Block* block);
[[nodiscard]] InstructionList& Instructions();
[[nodiscard]] const InstructionList& Instructions() const;
[[nodiscard]] std::span<Block* const> ImmPredecessors() const;
[[nodiscard]] std::span<Block* const> ImmSuccessors() const;
void SsaSeal();
[[nodiscard]] bool IsSsaSealed() const;
iterator begin();
const_iterator begin() const;
iterator end();
const_iterator end() const;
bool empty() const;
std::array<Value, NumScalarRegs> ssa_sreg_values {};
std::array<Value, NumScalarRegs> ssa_thread_bit_sreg_values {};
std::array<Value, NumVectorRegs> ssa_vreg_values {};
private:
InstructionList instructions;
std::vector<Block*> predecessors;
std::vector<Block*> successors;
bool ssa_sealed = false;
};
using BlockList = std::vector<Block*>;
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/IREmitter.cpp
+218
View File
@@ -0,0 +1,218 @@
#include "graphics/shader/recompiler/ir/IREmitter.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
IREmitter::IREmitter(Block* value_block): block(value_block) {
EXIT_IF(block == nullptr);
}
void IREmitter::SetBlock(Block* value_block) {
EXIT_IF(value_block == nullptr);
block = value_block;
}
Value IREmitter::Emit(ValueOpcode opcode, std::initializer_list<Value> args, uint64_t flags) {
auto& inst = block->AppendNewInst(opcode, args, flags);
return TypeOf(opcode) == Type::Void ? Value {} : Value(&inst);
}
U32 IREmitter::GetUserData(ScalarReg reg) {
return U32(Emit(ValueOpcode::GetUserData, {Value(reg)}));
}
U32 IREmitter::GetScalarReg(ScalarReg reg) {
return U32(Emit(ValueOpcode::GetScalarRegister, {Value(reg)}));
}
void IREmitter::SetScalarReg(ScalarReg reg, U32 value) {
Emit(ValueOpcode::SetScalarRegister, {Value(reg), value});
}
U1 IREmitter::GetThreadBitScalarReg(ScalarReg reg) {
return U1(Emit(ValueOpcode::GetThreadBitScalarRegister, {Value(reg)}));
}
void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value});
}
U32 IREmitter::GetVectorReg(VectorReg reg) {
return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)}));
}
void IREmitter::SetVectorReg(VectorReg reg, U32 value) {
Emit(ValueOpcode::SetVectorRegister, {Value(reg), value});
}
U1 IREmitter::GetGotoVariable(uint32_t id) {
return U1(Emit(ValueOpcode::GetGotoVariable, {Value(id)}));
}
void IREmitter::SetGotoVariable(uint32_t id, U1 value) {
Emit(ValueOpcode::SetGotoVariable, {Value(id), value});
}
U1 IREmitter::GetScc() {
return U1(Emit(ValueOpcode::GetScc));
}
void IREmitter::SetScc(U1 value) {
Emit(ValueOpcode::SetScc, {value});
}
U1 IREmitter::GetExec() {
return U1(Emit(ValueOpcode::GetExec));
}
void IREmitter::SetExec(U1 value) {
Emit(ValueOpcode::SetExec, {value});
}
U32 IREmitter::GetExecLo() {
return U32(Emit(ValueOpcode::GetExecLo));
}
void IREmitter::SetExecLo(U32 value) {
Emit(ValueOpcode::SetExecLo, {value});
}
U32 IREmitter::GetExecHi() {
return U32(Emit(ValueOpcode::GetExecHi));
}
void IREmitter::SetExecHi(U32 value) {
Emit(ValueOpcode::SetExecHi, {value});
}
U1 IREmitter::GetVcc() {
return U1(Emit(ValueOpcode::GetVcc));
}
void IREmitter::SetVcc(U1 value) {
Emit(ValueOpcode::SetVcc, {value});
}
U32 IREmitter::GetVccLo() {
return U32(Emit(ValueOpcode::GetVccLo));
}
void IREmitter::SetVccLo(U32 value) {
Emit(ValueOpcode::SetVccLo, {value});
}
U32 IREmitter::GetVccHi() {
return U32(Emit(ValueOpcode::GetVccHi));
}
void IREmitter::SetVccHi(U32 value) {
Emit(ValueOpcode::SetVccHi, {value});
}
U32 IREmitter::GetM0() {
return U32(Emit(ValueOpcode::GetM0));
}
void IREmitter::SetM0(U32 value) {
Emit(ValueOpcode::SetM0, {value});
}
F32 IREmitter::BitCastF32(U32 value) {
return F32(Emit(ValueOpcode::BitCastF32U32, {value}));
}
U32 IREmitter::BitCastU32(F32 value) {
return U32(Emit(ValueOpcode::BitCastU32F32, {value}));
}
F16 IREmitter::BitCastF16(U32 value) {
const auto half = U16(Emit(ValueOpcode::ConvertU16U32, {value}));
return F16(Emit(ValueOpcode::BitCastF16U16, {half}));
}
U32 IREmitter::BitCastU32(F16 value) {
const auto half = U16(Emit(ValueOpcode::BitCastU16F16, {value}));
return U32(Emit(ValueOpcode::ConvertU32U16, {half}));
}
U64 IREmitter::PackUint2x32(U32 low, U32 high) {
const auto pair = Emit(ValueOpcode::CompositeConstructU32x2, {low, high});
return U64(Emit(ValueOpcode::PackUint2x32, {pair}));
}
U32 IREmitter::CompositeExtract(Value composite, uint32_t index) {
return U32(Emit(ValueOpcode::CompositeExtractU32x2, {composite, Value(index)}));
}
U32 IREmitter::IAdd(U32 lhs, U32 rhs) {
return U32(Emit(ValueOpcode::IAdd32, {lhs, rhs}));
}
U32 IREmitter::ISub(U32 lhs, U32 rhs) {
return U32(Emit(ValueOpcode::ISub32, {lhs, rhs}));
}
U32 IREmitter::IMul(U32 lhs, U32 rhs) {
return U32(Emit(ValueOpcode::IMul32, {lhs, rhs}));
}
U32 IREmitter::ShiftLeftLogical(U32 value, U32 shift) {
return U32(Emit(ValueOpcode::ShiftLeftLogical32, {value, shift}));
}
U32 IREmitter::ShiftRightLogical(U32 value, U32 shift) {
return U32(Emit(ValueOpcode::ShiftRightLogical32, {value, shift}));
}
U32 IREmitter::ShiftRightArithmetic(U32 value, U32 shift) {
return U32(Emit(ValueOpcode::ShiftRightArithmetic32, {value, shift}));
}
U32 IREmitter::BitwiseAnd(U32 lhs, U32 rhs) {
return U32(Emit(ValueOpcode::BitwiseAnd32, {lhs, rhs}));
}
U32 IREmitter::BitwiseOr(U32 lhs, U32 rhs) {
return U32(Emit(ValueOpcode::BitwiseOr32, {lhs, rhs}));
}
U32 IREmitter::BitwiseXor(U32 lhs, U32 rhs) {
return U32(Emit(ValueOpcode::BitwiseXor32, {lhs, rhs}));
}
U32 IREmitter::BitwiseNot(U32 value) {
return U32(Emit(ValueOpcode::BitwiseNot32, {value}));
}
U32 IREmitter::Select(U1 condition, U32 true_value, U32 false_value) {
return U32(Emit(ValueOpcode::SelectU32, {condition, true_value, false_value}));
}
U1 IREmitter::IEqual(U32 lhs, U32 rhs) {
return U1(Emit(ValueOpcode::IEqual32, {lhs, rhs}));
}
U1 IREmitter::INotEqual(U32 lhs, U32 rhs) {
return U1(Emit(ValueOpcode::INotEqual32, {lhs, rhs}));
}
U1 IREmitter::ULessThan(U32 lhs, U32 rhs) {
return U1(Emit(ValueOpcode::ULessThan32, {lhs, rhs}));
}
U1 IREmitter::UGreaterThan(U32 lhs, U32 rhs) {
return U1(Emit(ValueOpcode::UGreaterThan32, {lhs, rhs}));
}
U1 IREmitter::LogicalAnd(U1 lhs, U1 rhs) {
return U1(Emit(ValueOpcode::LogicalAnd, {lhs, rhs}));
}
U1 IREmitter::LogicalOr(U1 lhs, U1 rhs) {
return U1(Emit(ValueOpcode::LogicalOr, {lhs, rhs}));
}
U1 IREmitter::LogicalNot(U1 value) {
return U1(Emit(ValueOpcode::LogicalNot, {value}));
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/IREmitter.h
+83
View File
@@ -0,0 +1,83 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
#include <cstring>
#include <initializer_list>
#include <type_traits>
namespace Libs::Graphics::ShaderRecompiler::IR {
class IREmitter {
public:
explicit IREmitter(Block* block);
void SetBlock(Block* block);
Value Emit(ValueOpcode opcode, std::initializer_list<Value> args = {}, uint64_t flags = 0);
template <typename T>
requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v<T>)
Value Emit(ValueOpcode opcode, std::initializer_list<Value> args, T flags) {
uint64_t bits = 0;
std::memcpy(&bits, &flags, sizeof(flags));
return Emit(opcode, args, bits);
}
U32 GetUserData(ScalarReg reg);
U32 GetScalarReg(ScalarReg reg);
void SetScalarReg(ScalarReg reg, U32 value);
U1 GetThreadBitScalarReg(ScalarReg reg);
void SetThreadBitScalarReg(ScalarReg reg, U1 value);
U32 GetVectorReg(VectorReg reg);
void SetVectorReg(VectorReg reg, U32 value);
U1 GetGotoVariable(uint32_t id);
void SetGotoVariable(uint32_t id, U1 value);
U1 GetScc();
void SetScc(U1 value);
U1 GetExec();
void SetExec(U1 value);
U32 GetExecLo();
void SetExecLo(U32 value);
U32 GetExecHi();
void SetExecHi(U32 value);
U1 GetVcc();
void SetVcc(U1 value);
U32 GetVccLo();
void SetVccLo(U32 value);
U32 GetVccHi();
void SetVccHi(U32 value);
U32 GetM0();
void SetM0(U32 value);
F32 BitCastF32(U32 value);
U32 BitCastU32(F32 value);
F16 BitCastF16(U32 value);
U32 BitCastU32(F16 value);
U64 PackUint2x32(U32 low, U32 high);
U32 CompositeExtract(Value composite, uint32_t index);
U32 IAdd(U32 lhs, U32 rhs);
U32 ISub(U32 lhs, U32 rhs);
U32 IMul(U32 lhs, U32 rhs);
U32 ShiftLeftLogical(U32 value, U32 shift);
U32 ShiftRightLogical(U32 value, U32 shift);
U32 ShiftRightArithmetic(U32 value, U32 shift);
U32 BitwiseAnd(U32 lhs, U32 rhs);
U32 BitwiseOr(U32 lhs, U32 rhs);
U32 BitwiseXor(U32 lhs, U32 rhs);
U32 BitwiseNot(U32 value);
U32 Select(U1 condition, U32 true_value, U32 false_value);
U1 IEqual(U32 lhs, U32 rhs);
U1 INotEqual(U32 lhs, U32 rhs);
U1 ULessThan(U32 lhs, U32 rhs);
U1 UGreaterThan(U32 lhs, U32 rhs);
U1 LogicalAnd(U1 lhs, U1 rhs);
U1 LogicalOr(U1 lhs, U1 rhs);
U1 LogicalNot(U1 value);
private:
Block* block = nullptr;
};
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ReadLaneElimination.cpp
-323
View File
@@ -1,323 +0,0 @@
#include "graphics/shader/recompiler/ir/ReadLaneElimination.h"
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include <algorithm>
#include <iterator>
#include <map>
#include <set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint32_t FirstTemporaryScalarRegister = 128;
struct LaneKey {
uint32_t reg = 0;
uint32_t lane = 0;
auto operator<=>(const LaneKey&) const = default;
};
using LaneSet = std::set<LaneKey>;
bool PairDwordOpcode(Opcode op) {
switch (op) {
case Opcode::MoveU64:
case Opcode::WqmB64:
case Opcode::SaveexecB64:
case Opcode::BitwiseAndU64:
case Opcode::BitwiseAndNotU64:
case Opcode::BitwiseOrU64:
case Opcode::BitwiseOrNotU64:
case Opcode::BitwiseXorU64:
case Opcode::BitwiseNandU64:
case Opcode::BitwiseNorU64:
case Opcode::BitwiseXnorU64:
case Opcode::BitwiseNotU64:
case Opcode::BitFieldMaskU64:
case Opcode::BitFieldExtractU64:
case Opcode::BitReplicateB64B32:
case Opcode::ShiftLeftLogicalU64:
case Opcode::ShiftRightLogicalU64:
case Opcode::SelectU64: return true;
default: return false;
}
}
bool ResolveLane(const Program& program, const Instruction& inst, uint32_t source_index,
uint32_t& lane) {
if (source_index >= inst.src_count || (program.wave_size != 32 && program.wave_size != 64)) {
return false;
}
const auto& selector = inst.src[source_index];
if (selector.kind == OperandKind::ImmediateU32) {
lane = selector.imm % program.wave_size;
return true;
}
uint32_t folded = 0;
if (!FoldScalarConstant(program.provenance, inst.scalar_sources[source_index], folded)) {
return false;
}
lane = folded % program.wave_size;
return true;
}
bool UniformWriteSource(const Instruction& inst) {
if (inst.src_count == 0) {
return false;
}
const auto& source = inst.src[0];
if (source.kind == OperandKind::ImmediateU32 || source.kind == OperandKind::PcRelativeU32) {
return true;
}
return source.kind == OperandKind::Register &&
(source.reg.file == RegisterFile::Scalar || source.reg.file == RegisterFile::Scc ||
source.reg.file == RegisterFile::M0);
}
bool WriteLaneKey(const Program& program, const Instruction& inst, LaneKey& key) {
if (inst.op != Opcode::WriteLaneU32 || inst.dst.kind != OperandKind::Register ||
inst.dst.reg.file != RegisterFile::Vector || !UniformWriteSource(inst)) {
return false;
}
uint32_t lane = 0;
if (!ResolveLane(program, inst, 1, lane)) {
return false;
}
key = {inst.dst.reg.index, lane};
return true;
}
bool ReadLaneKey(const Program& program, const Instruction& inst, LaneKey& key) {
if (inst.op != Opcode::ReadLaneU32 || inst.src_count < 2 ||
inst.src[0].kind != OperandKind::Register || inst.src[0].reg.file != RegisterFile::Vector) {
return false;
}
uint32_t lane = 0;
if (!ResolveLane(program, inst, 1, lane)) {
return false;
}
key = {inst.src[0].reg.index, lane};
return true;
}
void InvalidateRegister(LaneSet& valid, uint32_t reg) {
const auto first = valid.lower_bound({reg, 0});
const auto last = valid.lower_bound({reg + 1u, 0});
valid.erase(first, last);
}
void ApplyInstruction(const Program& program, const Instruction& inst, LaneSet& valid) {
if (inst.op == Opcode::WriteLaneU32 && inst.dst.kind == OperandKind::Register &&
inst.dst.reg.file == RegisterFile::Vector) {
LaneKey key;
if (WriteLaneKey(program, inst, key)) {
valid.insert(key);
return;
}
uint32_t lane = 0;
if (ResolveLane(program, inst, 1, lane)) {
valid.erase({inst.dst.reg.index, lane});
} else {
InvalidateRegister(valid, inst.dst.reg.index);
}
return;
}
if (inst.op == Opcode::MoveRelDestU32 && inst.dst.kind == OperandKind::Register &&
inst.dst.reg.file == RegisterFile::Vector) {
valid.clear();
return;
}
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Vector) {
uint32_t dwords = std::max(inst.memory.data_dwords, 1u);
if (PairDwordOpcode(inst.op) || inst.op == Opcode::UMadU64U32) {
dwords = std::max(dwords, 2u);
}
for (uint32_t i = 0; i < dwords && inst.dst.reg.index <= UINT32_MAX - i; i++) {
InvalidateRegister(valid, inst.dst.reg.index + i);
}
}
if (inst.dst2.kind == OperandKind::Register && inst.dst2.reg.file == RegisterFile::Vector) {
InvalidateRegister(valid, inst.dst2.reg.index);
}
}
LaneSet TransferBlock(const Program& program, const BasicBlock& block, LaneSet state) {
for (const auto& inst: block.instructions) {
ApplyInstruction(program, inst, state);
}
return state;
}
LaneSet Intersect(const LaneSet& left, const LaneSet& right) {
LaneSet result;
std::set_intersection(left.begin(), left.end(), right.begin(), right.end(),
std::inserter(result, result.end()));
return result;
}
uint32_t NextTemporaryScalarRegister(const Program& program) {
uint32_t next = FirstTemporaryScalarRegister;
const auto consider = [&next](const Operand& operand) {
if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::Scalar &&
operand.reg.index >= next && operand.reg.index != UINT32_MAX) {
next = operand.reg.index + 1u;
}
};
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
consider(inst.dst);
consider(inst.dst2);
for (uint32_t i = 0; i < inst.src_count; i++) {
consider(inst.src[i]);
}
}
}
return next;
}
Operand ScalarRegisterOperand(uint32_t reg) {
Operand operand;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Scalar;
operand.reg.index = reg;
return operand;
}
Instruction ShadowWrite(const Instruction& write, uint32_t temporary) {
Instruction shadow;
shadow.pc = write.pc;
shadow.op = Opcode::MoveU32;
shadow.dst = ScalarRegisterOperand(temporary);
shadow.src[0] = write.src[0];
shadow.src_count = 1;
return shadow;
}
Instruction ShadowRead(const Instruction& read, uint32_t temporary) {
Instruction rewritten;
rewritten.pc = read.pc;
rewritten.op = Opcode::MoveU32;
rewritten.dst = read.dst;
rewritten.src[0] = ScalarRegisterOperand(temporary);
rewritten.src_count = 1;
return rewritten;
}
} // namespace
ReadLaneEliminationStats EliminateReadLane(Program& program) {
ReadLaneEliminationStats stats;
if (program.blocks.empty() || (program.wave_size != 32 && program.wave_size != 64)) {
return stats;
}
LaneSet universe;
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
LaneKey key;
if (WriteLaneKey(program, inst, key)) {
universe.insert(key);
}
}
}
if (universe.empty()) {
return stats;
}
const size_t block_count = program.blocks.size();
std::vector<LaneSet> entry(block_count, universe);
std::vector<LaneSet> exit(block_count, universe);
entry[0].clear();
for (size_t block = 0; block < block_count; block++) {
exit[block] = TransferBlock(program, program.blocks[block], entry[block]);
}
bool changed = true;
while (changed) {
changed = false;
for (size_t block_index = 0; block_index < block_count; block_index++) {
LaneSet next_entry;
const auto& block = program.blocks[block_index];
if (block_index != 0 && !block.predecessors.empty()) {
next_entry = universe;
for (const auto predecessor: block.predecessors) {
if (predecessor >= block_count) {
next_entry.clear();
break;
}
next_entry = Intersect(next_entry, exit[predecessor]);
}
}
auto next_exit = TransferBlock(program, block, next_entry);
if (next_entry != entry[block_index] || next_exit != exit[block_index]) {
entry[block_index] = std::move(next_entry);
exit[block_index] = std::move(next_exit);
changed = true;
}
}
}
LaneSet forwarded;
for (size_t block_index = 0; block_index < block_count; block_index++) {
auto state = entry[block_index];
for (const auto& inst: program.blocks[block_index].instructions) {
LaneKey key;
if (ReadLaneKey(program, inst, key) && state.contains(key)) {
forwarded.insert(key);
}
ApplyInstruction(program, inst, state);
}
}
if (forwarded.empty()) {
return stats;
}
std::map<LaneKey, uint32_t> temporaries;
auto next_temporary = NextTemporaryScalarRegister(program);
for (const auto& key: forwarded) {
if (next_temporary == UINT32_MAX) {
return {};
}
temporaries.emplace(key, next_temporary++);
}
for (size_t block_index = 0; block_index < block_count; block_index++) {
const auto original = std::move(program.blocks[block_index].instructions);
auto& rewritten = program.blocks[block_index].instructions;
rewritten.clear();
rewritten.reserve(original.size() + temporaries.size());
auto state = entry[block_index];
for (const auto& inst: original) {
LaneKey read_key;
if (ReadLaneKey(program, inst, read_key) && state.contains(read_key)) {
const auto temporary = temporaries.find(read_key);
if (temporary != temporaries.end()) {
rewritten.push_back(ShadowRead(inst, temporary->second));
stats.rewritten_reads++;
ApplyInstruction(program, inst, state);
continue;
}
}
rewritten.push_back(inst);
LaneKey write_key;
if (WriteLaneKey(program, inst, write_key)) {
const auto temporary = temporaries.find(write_key);
if (temporary != temporaries.end()) {
rewritten.push_back(ShadowWrite(inst, temporary->second));
stats.shadow_writes++;
}
}
ApplyInstruction(program, inst, state);
}
}
return stats;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ReadLaneElimination.h
-20
View File
@@ -1,20 +0,0 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_READLANEELIMINATION_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_READLANEELIMINATION_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct ReadLaneEliminationStats {
uint32_t rewritten_reads = 0;
uint32_t shadow_writes = 0;
};
// Replaces fixed-lane ReadLane operations that are reached by a matching WriteLane on every
// control-flow path. A synthetic scalar register snapshots the value at WriteLane execution time,
// so the rewrite remains valid when the source SGPR is subsequently overwritten.
[[nodiscard]] ReadLaneEliminationStats EliminateReadLane(Program& program);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_READLANEELIMINATION_H_ */
src/graphics/shader/recompiler/ir/Reg.h
+24
View File
@@ -0,0 +1,24 @@
#pragma once
#include <cstdint>
namespace Libs::Graphics::ShaderRecompiler::IR {
enum class ScalarReg : uint16_t {};
enum class VectorReg : uint16_t {};
// Architectural SGPRs occupy the low range. The current shader-only migration also accepts the
// scalar shadow registers produced by Kyty's pre-existing read-lane pass; that pass disappears
// once read-lane elimination operates on the SSA graph.
constexpr uint32_t NumScalarRegs = 512;
constexpr uint32_t NumVectorRegs = 256;
constexpr uint32_t RegIndex(ScalarReg reg) {
return static_cast<uint32_t>(reg);
}
constexpr uint32_t RegIndex(VectorReg reg) {
return static_cast<uint32_t>(reg);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ResourceTracking.cpp
-464
View File
@@ -1,464 +0,0 @@
#include "graphics/shader/recompiler/ir/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
#include <algorithm>
#include <fmt/format.h>
#include <functional>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
bool IsAtomic(Opcode op) {
switch (op) {
case Opcode::AtomicSwapU32:
case Opcode::AtomicAddU32:
case Opcode::AtomicSubU32:
case Opcode::AtomicSMinI32:
case Opcode::AtomicUMinU32:
case Opcode::AtomicSMaxI32:
case Opcode::AtomicUMaxU32:
case Opcode::AtomicAndU32:
case Opcode::AtomicOrU32:
case Opcode::AtomicXorU32:
case Opcode::AtomicFMinF32:
case Opcode::AtomicFMaxF32: return true;
default: return false;
}
}
bool IsWrite(Opcode op) {
switch (op) {
case Opcode::BufferStoreByte:
case Opcode::BufferStoreShort:
case Opcode::BufferStoreDword:
case Opcode::ImageStore: return true;
default: return IsAtomic(op);
}
}
bool NeedsSampler(Opcode op) {
return op == Opcode::ImageSample || op == Opcode::ImageGather4 || op == Opcode::ImageGetLod;
}
bool IsDepthCompare(const Instruction& inst) {
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0;
}
ImageMipMode MipMode(const Instruction& inst) {
const bool storage = inst.memory.kind == ResourceKind::StorageImage ||
inst.memory.kind == ResourceKind::StorageImageUint;
return storage && inst.memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
}
bool IsBuffer(const Instruction& inst) {
return inst.memory.kind == ResourceKind::Buffer ||
(inst.memory.kind == ResourceKind::ScalarBuffer && inst.op == Opcode::SBufferLoadDword);
}
bool IsImage(const Instruction& inst) {
return inst.memory.kind == ResourceKind::Image || inst.memory.kind == ResourceKind::ImageUint ||
inst.memory.kind == ResourceKind::StorageImage ||
inst.memory.kind == ResourceKind::StorageImageUint;
}
bool IsAddress(const Instruction& inst) {
return inst.op == Opcode::SLoadDword || inst.memory.kind == ResourceKind::Flat ||
inst.memory.kind == ResourceKind::Global || inst.memory.kind == ResourceKind::Scratch;
}
uint32_t ByteExtent(const Instruction& inst) {
const auto bytes = std::max((inst.memory.data_bits + 7u) / 8u, 1u);
const auto count = std::max(inst.memory.data_dwords, 1u);
const auto end =
static_cast<uint64_t>(inst.memory.offset) + static_cast<uint64_t>(bytes) * count;
return end > UINT32_MAX ? UINT32_MAX : static_cast<uint32_t>(end);
}
bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>& visited,
std::vector<uint32_t>& path) {
path.push_back(id);
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return true;
}
if (visited[id] != 0) {
path.pop_back();
return false;
}
visited[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (ContainsUnknown(provenance, arg, visited, path)) {
return true;
}
}
path.pop_back();
return false;
}
const auto args = ScalarValueArgCount(value.op);
for (uint32_t i = 0; i < args; i++) {
if (ContainsUnknown(provenance, value.args[i], visited, path)) {
return true;
}
}
path.pop_back();
return false;
}
bool IsLoopInvariantValue(const ScalarProvenance& provenance, uint32_t id,
std::vector<uint8_t>& visiting) {
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return false;
}
if (visiting[id] != 0) {
return true;
}
visiting[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
uint32_t invariant = ScalarProvenance::Undefined;
for (const auto arg: value.phi_args) {
if (arg == id) {
continue;
}
if (invariant == ScalarProvenance::Undefined) {
invariant = arg;
} else if (arg != invariant) {
return false;
}
}
return invariant != ScalarProvenance::Undefined &&
IsLoopInvariantValue(provenance, invariant, visiting);
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (!IsLoopInvariantValue(provenance, value.args[i], visiting)) {
return false;
}
}
return true;
}
bool IsLoopInvariantDescriptor(const ScalarProvenance& provenance,
const DescriptorValue& descriptor) {
std::vector<uint8_t> visiting(provenance.values.size());
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (!IsLoopInvariantValue(provenance, descriptor.dwords[i], visiting)) {
return false;
}
}
return true;
}
class Tracker {
public:
explicit Tracker(Program& program): m_program(program) {}
bool Run(std::string* error) {
if (m_program.resource_tracking_complete) {
return Fail(0, error, "resources already tracked");
}
if (!m_program.srt_plan_complete) {
return Fail(0, error, "SRT plan is not ready");
}
if (!m_program.srt_patching_complete) {
return Fail(0, error, "SRT reads were not patched");
}
for (auto& block: m_program.blocks) {
for (auto& inst: block.instructions) {
if (!Collect(inst, error)) {
return false;
}
}
}
LinkImageAliases();
m_program.info = std::move(m_info);
for (const auto& patch: m_patches) {
auto& inst = patch.inst.get();
inst.memory.resource = patch.resource;
inst.memory.sampler = patch.sampler;
inst.memory.resource_source = ScalarProvenance::Undefined;
inst.memory.sampler_source = ScalarProvenance::Undefined;
}
m_program.resource_tracking_complete = true;
return true;
}
private:
struct Patch {
std::reference_wrapper<Instruction> inst;
uint32_t resource = 0;
uint32_t sampler = 0;
};
bool Fail(uint32_t pc, std::string* error, const std::string& reason) const {
if (error != nullptr) {
*error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}",
m_program.shader_hash, StageName(m_program.stage), pc, reason);
}
return false;
}
bool ValidateSource(uint32_t source, uint32_t dwords, uint32_t pc, std::string* error) const {
const auto* descriptor = GetDescriptorSource(m_program, source);
if (descriptor == nullptr || descriptor->dword_count != dwords) {
return Fail(pc, error,
fmt::format("descriptor source {} is missing or has wrong width", source));
}
std::vector<uint8_t> visited(m_program.provenance.values.size());
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
std::vector<uint32_t> path;
if (ContainsUnknown(m_program.provenance, descriptor->dwords[i], visited, path)) {
const auto value = descriptor->dwords[i];
std::string chain;
for (const auto id: path) {
const auto op = id < m_program.provenance.values.size()
? static_cast<uint32_t>(m_program.provenance.values[id].op)
: UINT32_MAX;
chain += fmt::format("{}{}:{}({})", chain.empty() ? "" : " -> ", id, op,
ScalarValueToString(m_program.provenance, id));
}
return Fail(
pc, error,
fmt::format(
"descriptor source {} dword {} contains an unknown value {} ({}) path {}",
source, i, value, ScalarValueToString(m_program.provenance, value), chain));
}
}
const auto dynamic =
std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(),
source) != m_program.srt.dynamic_sources.end();
if (dynamic && !IsLoopInvariantDescriptor(m_program.provenance, *descriptor)) {
std::string detail;
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
const auto id = descriptor->dwords[i];
detail +=
fmt::format(" d{}={}({})", i, id,
id < m_program.provenance.values.size()
? static_cast<uint32_t>(m_program.provenance.values[id].op)
: UINT32_MAX);
if (id < m_program.provenance.values.size()) {
for (const auto arg: m_program.provenance.values[id].phi_args) {
detail += fmt::format(
"/{}({})", arg,
arg < m_program.provenance.values.size()
? static_cast<uint32_t>(m_program.provenance.values[arg].op)
: UINT32_MAX);
}
}
}
return Fail(pc, error,
fmt::format("descriptor source {} requires unsupported GPU selection{}",
source, detail));
}
if (!DescriptorSourceResolved(m_program, source) && !dynamic) {
return Fail(pc, error, fmt::format("descriptor source {} is unresolved", source));
}
return true;
}
uint32_t AddBuffer(const Instruction& inst) {
for (uint32_t i = 0; i < m_info.buffers.size(); i++) {
if (m_info.buffers[i].source == inst.memory.resource_source) {
Merge(&m_info.buffers[i], inst);
return i;
}
}
if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) {
return UINT32_MAX;
}
BufferResource resource;
resource.source = inst.memory.resource_source;
resource.first_use_pc = inst.pc;
Merge(&resource, inst);
m_info.buffers.push_back(resource);
return static_cast<uint32_t>(m_info.buffers.size() - 1);
}
void Merge(BufferResource* resource, const Instruction& inst) const {
resource->first_use_pc = std::min(resource->first_use_pc, inst.pc);
resource->max_byte_extent = std::max(resource->max_byte_extent, ByteExtent(inst));
resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op);
resource->written = resource->written || IsWrite(inst.op);
resource->atomic = resource->atomic || IsAtomic(inst.op);
resource->formatted = resource->formatted || inst.memory.formatted;
resource->scalar = resource->scalar || inst.memory.kind == ResourceKind::ScalarBuffer;
}
void LinkImageAliases() {
for (auto& buffer: m_info.buffers) {
const auto* buffer_source = GetDescriptorSource(m_program, buffer.source);
if (buffer_source == nullptr || buffer_source->dword_count != 4) {
continue;
}
for (uint32_t i = 0; i < m_info.images.size(); i++) {
const auto* image_source = GetDescriptorSource(m_program, m_info.images[i].source);
if (image_source != nullptr && image_source->dword_count == 8 &&
std::equal(buffer_source->dwords.begin(), buffer_source->dwords.begin() + 4,
image_source->dwords.begin())) {
buffer.image_alias = i;
break;
}
}
}
}
uint32_t AddImage(const Instruction& inst) {
for (uint32_t i = 0; i < m_info.images.size(); i++) {
auto& image = m_info.images[i];
if (image.source == inst.memory.resource_source && image.kind == inst.memory.kind &&
image.dimension == inst.memory.image_dimension && image.mip_mode == MipMode(inst) &&
image.depth_compare == IsDepthCompare(inst)) {
Merge(&image, inst);
return i;
}
}
if (m_info.images.size() >= ShaderInfo::MaxImages) {
return UINT32_MAX;
}
ImageResource image;
image.source = inst.memory.resource_source;
image.first_use_pc = inst.pc;
image.kind = inst.memory.kind;
image.dimension = inst.memory.image_dimension;
image.mip_mode = MipMode(inst);
image.depth_compare = IsDepthCompare(inst);
Merge(&image, inst);
m_info.images.push_back(image);
return static_cast<uint32_t>(m_info.images.size() - 1);
}
uint32_t AddAddress(const Instruction& inst) {
auto immediate = static_cast<int32_t>(inst.memory.offset);
if (inst.memory.kind == ResourceKind::ScalarBuffer) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto min_offset =
inst.memory.resource_source == ScalarProvenance::Unknown ? 0 : std::min(immediate, 0);
for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
auto& address = m_info.addresses[i];
if (address.source == inst.memory.resource_source && address.kind == inst.memory.kind) {
address.first_use_pc = std::min(address.first_use_pc, inst.pc);
address.min_offset = std::min(address.min_offset, min_offset);
address.read = address.read || !IsWrite(inst.op) || IsAtomic(inst.op);
address.written = address.written || IsWrite(inst.op);
address.atomic = address.atomic || IsAtomic(inst.op);
return i;
}
}
if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
return UINT32_MAX;
}
AddressResource address {inst.memory.resource_source, inst.pc, inst.memory.kind,
min_offset};
address.read = !IsWrite(inst.op) || IsAtomic(inst.op);
address.written = IsWrite(inst.op);
address.atomic = IsAtomic(inst.op);
m_info.addresses.push_back(address);
return static_cast<uint32_t>(m_info.addresses.size() - 1);
}
void Merge(ImageResource* resource, const Instruction& inst) const {
resource->first_use_pc = std::min(resource->first_use_pc, inst.pc);
resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op);
resource->written = resource->written || IsWrite(inst.op);
resource->atomic = resource->atomic || IsAtomic(inst.op);
}
uint32_t AddSampler(const Instruction& inst) {
for (uint32_t i = 0; i < m_info.samplers.size(); i++) {
if (m_info.samplers[i].source == inst.memory.sampler_source) {
m_info.samplers[i].first_use_pc =
std::min(m_info.samplers[i].first_use_pc, inst.pc);
return i;
}
}
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
return UINT32_MAX;
}
m_info.samplers.push_back({inst.memory.sampler_source, inst.pc});
return static_cast<uint32_t>(m_info.samplers.size() - 1);
}
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) {
pair.first_use_pc = std::min(pair.first_use_pc, pc);
return true;
}
}
if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) {
return Fail(pc, error, "sampled image/sampler pair limit exceeded");
}
m_info.sampled_pairs.push_back({image, sampler, pc});
return true;
}
bool Collect(Instruction& inst, std::string* error) {
if (IsAddress(inst)) {
const bool unbased = inst.memory.resource_source == ScalarProvenance::Unknown;
if ((!unbased && !ValidateSource(inst.memory.resource_source, 2, inst.pc, error)) ||
(unbased && inst.memory.kind != ResourceKind::Flat &&
inst.memory.kind != ResourceKind::Global &&
inst.memory.kind != ResourceKind::Scratch)) {
return unbased ? Fail(inst.pc, error, "scalar memory base is unresolved") : false;
}
const auto resource = AddAddress(inst);
if (resource == UINT32_MAX) {
return Fail(inst.pc, error, "address resource limit exceeded");
}
m_patches.push_back({std::ref(inst), resource, 0});
return true;
}
if (!IsBuffer(inst) && !IsImage(inst)) {
return true;
}
if (!ValidateSource(inst.memory.resource_source, IsBuffer(inst) ? 4u : 8u, inst.pc,
error)) {
return false;
}
const auto resource = IsBuffer(inst) ? AddBuffer(inst) : AddImage(inst);
if (resource == UINT32_MAX) {
return Fail(inst.pc, error,
IsBuffer(inst) ? "buffer resource limit exceeded"
: "image resource limit exceeded");
}
uint32_t sampler = 0;
if (NeedsSampler(inst.op)) {
if (!ValidateSource(inst.memory.sampler_source, 4, inst.pc, error)) {
return false;
}
sampler = AddSampler(inst);
if (sampler == UINT32_MAX) {
return Fail(inst.pc, error, "sampler resource limit exceeded");
}
if (!AddSampledPair(resource, sampler, inst.pc, error)) {
return false;
}
}
m_patches.push_back({std::ref(inst), resource, sampler});
return true;
}
Program& m_program;
ShaderInfo m_info;
std::vector<Patch> m_patches;
};
} // namespace
bool TrackResources(Program& program, std::string* error) {
return Tracker(program).Run(error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ScalarProvenance.cpp
-963
View File
@@ -1,963 +0,0 @@
#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
#include <algorithm>
#include <array>
#include <compare>
#include <deque>
#include <fmt/format.h>
#include <map>
namespace Libs::Graphics::ShaderRecompiler::IR {
uint32_t ScalarValueArgCount(ScalarValueOp op) {
switch (op) {
case ScalarValueOp::Not: return 1;
case ScalarValueOp::Add:
case ScalarValueOp::Sub:
case ScalarValueOp::Mul:
case ScalarValueOp::And:
case ScalarValueOp::AndNot:
case ScalarValueOp::Or:
case ScalarValueOp::OrNot:
case ScalarValueOp::Xor:
case ScalarValueOp::ShiftLeft:
case ScalarValueOp::ShiftRight:
case ScalarValueOp::ShiftRightArithmetic:
case ScalarValueOp::BitFieldMaskU32:
case ScalarValueOp::BitFieldMaskU64Low:
case ScalarValueOp::BitFieldMaskU64High: return 2;
case ScalarValueOp::AddCarry:
case ScalarValueOp::Carry:
case ScalarValueOp::SubBorrow:
case ScalarValueOp::Borrow:
case ScalarValueOp::Add3:
case ScalarValueOp::ShiftLeftAdd:
case ScalarValueOp::ShiftLeftAddCarry:
case ScalarValueOp::AddShiftLeft:
case ScalarValueOp::XorAdd:
case ScalarValueOp::ShiftLeftOr:
case ScalarValueOp::ReadConst: return 3;
case ScalarValueOp::ReadConstBuffer: return 5;
default: return 0;
}
}
namespace {
constexpr uint32_t ScalarRegisters = 128;
constexpr uint32_t VectorRegisters = 256;
// Clamp X/Y/Z are consecutive three-bit fields; the high bit of each selects a border mode.
constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
struct ScalarState {
std::array<uint32_t, ScalarRegisters> regs = {};
std::array<uint32_t, VectorRegisters> address_bases = {};
uint32_t scc = ScalarProvenance::Undefined;
uint32_t m0 = ScalarProvenance::Undefined;
std::map<uint64_t, uint32_t> vector_lanes;
void Fill(uint32_t value) {
regs.fill(value);
address_bases.fill(value);
scc = value;
m0 = value;
vector_lanes.clear();
}
bool operator==(const ScalarState&) const = default;
};
struct ValueKey {
ScalarValueOp op = ScalarValueOp::Undefined;
uint32_t imm = 0;
std::array<uint32_t, 6> args = {};
auto operator<=>(const ValueKey&) const = default;
};
struct DescriptorKey {
std::array<uint32_t, 8> dwords = {};
uint32_t dword_count = 0;
auto operator<=>(const DescriptorKey&) const = default;
};
bool ScalarRegister(const Operand& operand, uint32_t& reg) {
if (operand.kind != OperandKind::Register) {
return false;
}
if (operand.reg.file == RegisterFile::Scalar && operand.reg.index < ScalarRegisters) {
reg = operand.reg.index;
return true;
}
if (operand.reg.file == RegisterFile::Vcc && operand.reg.index < 2) {
reg = 106u + operand.reg.index;
return true;
}
return false;
}
bool VectorRegister(const Operand& operand, uint32_t& reg) {
if (operand.kind != OperandKind::Register || operand.reg.file != RegisterFile::Vector ||
operand.reg.index >= VectorRegisters) {
return false;
}
reg = operand.reg.index;
return true;
}
bool FlatStore(Opcode op) {
switch (op) {
case Opcode::FlatStoreByte:
case Opcode::FlatStoreShort:
case Opcode::FlatStoreDword: return true;
default: return false;
}
}
bool PairDwordOpcode(Opcode op) {
switch (op) {
case Opcode::MoveU64:
case Opcode::WqmB64:
case Opcode::SaveexecB64:
case Opcode::BitwiseAndU64:
case Opcode::BitwiseAndNotU64:
case Opcode::BitwiseOrU64:
case Opcode::BitwiseOrNotU64:
case Opcode::BitwiseXorU64:
case Opcode::BitwiseNandU64:
case Opcode::BitwiseNorU64:
case Opcode::BitwiseXnorU64:
case Opcode::BitwiseNotU64:
case Opcode::BitFieldMaskU64:
case Opcode::BitFieldExtractU64:
case Opcode::BitReplicateB64B32:
case Opcode::ShiftLeftLogicalU64:
case Opcode::ShiftRightLogicalU64:
case Opcode::SelectU64: return true;
default: return false;
}
}
bool ScalarMemoryLoad(Opcode op) {
return op == Opcode::SLoadDword || op == Opcode::SBufferLoadDword;
}
bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>& visited) {
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return false;
}
if (visited[id] == 1) {
return false;
}
if (visited[id] == 2) {
return true;
}
if (visited[id] == 3) {
return false;
}
visited[id] = 1;
const auto& value = provenance.values[id];
bool resolved = true;
if (value.op == ScalarValueOp::Phi) {
resolved = !value.phi_args.empty() &&
std::all_of(value.phi_args.begin(), value.phi_args.end(), [&](uint32_t arg) {
return ValueResolved(provenance, arg, visited);
});
} else {
const auto args = ScalarValueArgCount(value.op);
for (uint32_t i = 0; i < args && resolved; i++) {
resolved = ValueResolved(provenance, value.args[i], visited);
}
}
visited[id] = resolved ? 2 : 3;
return resolved;
}
class Builder {
public:
explicit Builder(Program& program): m_program(program), m_graph(program.provenance) {}
bool Run(std::string* error) {
m_graph = {};
m_graph.values.resize(2);
m_graph.values[ScalarProvenance::Undefined].op = ScalarValueOp::Undefined;
m_graph.values[ScalarProvenance::Unknown].op = ScalarValueOp::Unknown;
const auto block_count = m_program.blocks.size();
m_entry.resize(block_count);
m_exit.resize(block_count);
m_phi.resize(block_count);
m_exit_ready.assign(block_count, false);
m_queued.assign(block_count, false);
for (auto& state: m_entry) {
state.Fill(ScalarProvenance::Undefined);
}
for (auto& state: m_exit) {
state.Fill(ScalarProvenance::Undefined);
}
for (auto& phis: m_phi) {
phis.Fill(ScalarProvenance::Undefined);
}
m_user_data.Fill(ScalarProvenance::Undefined);
for (uint32_t reg = 0; reg < ScalarRegisters; reg++) {
m_user_data.regs[reg] =
reg >= m_program.user_data_base &&
reg - m_program.user_data_base < m_program.user_data_count
? InternValue({ScalarValueOp::UserData, 0, reg})
: ScalarProvenance::Unknown;
}
m_user_data.scc = ScalarProvenance::Unknown;
m_user_data.m0 = ScalarProvenance::Unknown;
if (block_count == 0) {
return true;
}
for (const auto& block: m_program.blocks) {
for (const auto predecessor: block.predecessors) {
if (predecessor >= block_count) {
return Fail(error, fmt::format("scalar provenance has invalid predecessor {}",
predecessor));
}
}
for (const auto successor: block.successors) {
if (successor >= block_count) {
return Fail(error, fmt::format("scalar provenance has invalid successor {}",
successor));
}
}
if (!ValidateScalarMemoryGroups(block, error)) {
return false;
}
}
Queue(0);
while (!m_work.empty()) {
const auto block_index = m_work.front();
m_work.pop_front();
m_queued[block_index] = false;
auto next_entry = MergeEntry(block_index);
const bool entry_changed = next_entry != m_entry[block_index];
m_entry[block_index] = next_entry;
auto next_exit = Execute(m_program.blocks[block_index], next_entry, false);
const bool was_ready = m_exit_ready[block_index];
if (was_ready && !entry_changed && next_exit == m_exit[block_index]) {
continue;
}
m_exit[block_index] = next_exit;
m_exit_ready[block_index] = true;
for (const auto successor: m_program.blocks[block_index].successors) {
Queue(successor);
}
}
for (size_t block = 0; block < block_count; block++) {
Execute(m_program.blocks[block], m_entry[block], true);
}
return true;
}
private:
bool Fail(std::string* error, std::string text) const {
if (error != nullptr) {
*error = std::move(text);
}
return false;
}
uint32_t AddValue(ScalarValue value) {
m_graph.values.push_back(std::move(value));
return static_cast<uint32_t>(m_graph.values.size() - 1);
}
bool ValidateScalarMemoryGroups(const BasicBlock& block, std::string* error) const {
for (size_t index = 0; index < block.instructions.size();) {
const auto& first = block.instructions[index];
if (!ScalarMemoryLoad(first.op) || first.memory.component_count == 1) {
index++;
continue;
}
const auto count = first.memory.component_count;
uint32_t first_dst = 0;
if (count == 0 || first.memory.component_index != 0 ||
index + count > block.instructions.size() ||
!ScalarRegister(first.dst, first_dst)) {
return Fail(error,
fmt::format("malformed scalar-memory group at pc=0x{:x}", first.pc));
}
for (uint32_t component = 0; component < count; component++) {
const auto& inst = block.instructions[index + component];
uint32_t dst = 0;
if (inst.op != first.op || inst.pc != first.pc ||
inst.memory.component_count != count ||
inst.memory.component_index != component ||
inst.memory.resource != first.memory.resource ||
inst.memory.offset != first.memory.offset + component * 4u ||
inst.src_count != first.src_count ||
!std::equal(std::begin(inst.src), std::end(inst.src), std::begin(first.src)) ||
!ScalarRegister(inst.dst, dst) || dst != first_dst + component) {
return Fail(error,
fmt::format("malformed scalar-memory component {} at pc=0x{:x}",
component, first.pc));
}
}
index += count;
}
return true;
}
uint32_t InternValue(ScalarValue value) {
const ValueKey key {value.op, value.imm, value.args};
if (const auto found = m_values.find(key); found != m_values.end()) {
return found->second;
}
const auto id = AddValue(std::move(value));
m_values.emplace(key, id);
return id;
}
uint32_t Constant(uint32_t value) { return InternValue({ScalarValueOp::Constant, 0, value}); }
static uint64_t VectorLaneKey(uint32_t reg, uint32_t lane) {
return (static_cast<uint64_t>(reg) << 32u) | lane;
}
uint32_t OperandValue(const Operand& operand, const ScalarState& state) {
uint32_t reg = 0;
if (ScalarRegister(operand, reg)) {
return state.regs[reg];
}
if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::Scc) {
return state.scc;
}
if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::M0) {
return state.m0;
}
if (operand.kind == OperandKind::ImmediateU32 ||
operand.kind == OperandKind::PcRelativeU32) {
if (operand.kind == OperandKind::PcRelativeU32) {
return InternValue({ScalarValueOp::PcRelativeLow, 0, operand.imm});
}
return Constant(operand.imm);
}
return ScalarProvenance::Unknown;
}
uint32_t Define(const Instruction& inst, ScalarValueOp op, const ScalarState& state,
uint32_t imm = 0) {
ScalarValue node;
node.op = op;
node.pc = inst.pc;
node.imm = imm;
for (uint32_t i = 0; i < inst.src_count && i < node.args.size(); i++) {
node.args[i] = OperandValue(inst.src[i], state);
}
return InternValue(std::move(node));
}
uint32_t DefineBorrow(const Instruction& inst, const ScalarState& state) {
ScalarValue node;
node.op = ScalarValueOp::Borrow;
node.pc = inst.pc;
node.args[0] =
inst.src_count > 0 ? OperandValue(inst.src[0], state) : ScalarProvenance::Unknown;
node.args[1] =
inst.src_count > 1 ? OperandValue(inst.src[1], state) : ScalarProvenance::Unknown;
node.args[2] = inst.src_count > 2 ? OperandValue(inst.src[2], state) : Constant(0);
return InternValue(std::move(node));
}
bool ConstantOperand(const Operand& operand, const ScalarState& state, uint32_t& value) {
const auto id = OperandValue(operand, state);
if (id >= m_graph.values.size() || m_graph.values[id].op != ScalarValueOp::Constant) {
return false;
}
value = m_graph.values[id].imm;
return true;
}
uint32_t ReadVectorLane(const Instruction& inst, const ScalarState& state) {
if (inst.src_count < 2 || inst.src[0].kind != OperandKind::Register ||
inst.src[0].reg.file != RegisterFile::Vector ||
(m_program.wave_size != 32 && m_program.wave_size != 64)) {
return ScalarProvenance::Unknown;
}
uint32_t lane = 0;
if (!ConstantOperand(inst.src[1], state, lane)) {
return ScalarProvenance::Unknown;
}
const auto found = state.vector_lanes.find(
VectorLaneKey(inst.src[0].reg.index, lane % m_program.wave_size));
return found != state.vector_lanes.end() ? found->second : ScalarProvenance::Unknown;
}
void ClearVectorLanes(uint32_t reg, ScalarState& state) {
const auto first = state.vector_lanes.lower_bound(VectorLaneKey(reg, 0));
const auto last = state.vector_lanes.lower_bound((static_cast<uint64_t>(reg) + 1u) << 32u);
state.vector_lanes.erase(first, last);
}
void WriteVectorDestination(const Instruction& inst, ScalarState& state) {
if (inst.dst.kind != OperandKind::Register || inst.dst.reg.file != RegisterFile::Vector) {
return;
}
const uint32_t reg = inst.dst.reg.index;
uint32_t address_base = ScalarProvenance::Unknown;
if (inst.op == Opcode::MoveU32 || inst.op == Opcode::IAddU32 ||
inst.op == Opcode::IAddCarryU32 || inst.op == Opcode::ISubU32) {
for (uint32_t i = 0; i < inst.src_count; i++) {
uint32_t src_reg = 0;
if (VectorRegister(inst.src[i], src_reg) &&
state.address_bases[src_reg] > ScalarProvenance::Unknown) {
address_base = state.address_bases[src_reg];
break;
}
}
if (address_base == ScalarProvenance::Unknown) {
for (uint32_t i = 0; i < inst.src_count; i++) {
uint32_t src_reg = 0;
if (ScalarRegister(inst.src[i], src_reg) &&
state.regs[src_reg] > ScalarProvenance::Unknown) {
address_base = state.regs[src_reg];
break;
}
}
}
}
state.address_bases[reg] = ScalarProvenance::Unknown;
if (inst.op == Opcode::MoveRelDestU32) {
state.vector_lanes.clear();
return;
}
if (inst.op != Opcode::WriteLaneU32) {
uint32_t dwords = std::max(inst.memory.data_dwords, 1u);
if (PairDwordOpcode(inst.op) || inst.op == Opcode::UMadU64U32) {
dwords = std::max(dwords, 2u);
}
for (uint32_t i = 0; i < dwords && reg <= UINT32_MAX - i; i++) {
ClearVectorLanes(reg + i, state);
if (reg + i < state.address_bases.size()) {
state.address_bases[reg + i] = ScalarProvenance::Unknown;
}
}
state.address_bases[reg] = address_base;
return;
}
if (inst.src_count < 2 || (m_program.wave_size != 32 && m_program.wave_size != 64)) {
ClearVectorLanes(reg, state);
return;
}
uint32_t lane = 0;
if (!ConstantOperand(inst.src[1], state, lane)) {
ClearVectorLanes(reg, state);
return;
}
state.vector_lanes[VectorLaneKey(reg, lane % m_program.wave_size)] =
OperandValue(inst.src[0], state);
}
uint32_t ReadConst(const Instruction& inst, const ScalarState& state, bool buffer) {
ScalarValue node;
node.op = buffer ? ScalarValueOp::ReadConstBuffer : ScalarValueOp::ReadConst;
node.pc = inst.pc;
const auto base = buffer ? inst.memory.resource * 4u : inst.memory.resource;
if (base + 1u >= ScalarRegisters) {
return ScalarProvenance::Unknown;
}
node.imm = inst.memory.offset;
if (buffer) {
if (base + 3u >= ScalarRegisters) {
return ScalarProvenance::Unknown;
}
for (uint32_t i = 0; i < 4; i++) {
node.args[i] = state.regs[base + i];
}
node.args[4] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0);
} else {
node.args[0] = state.regs[base];
node.args[1] = state.regs[base + 1u];
node.args[2] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0);
}
return InternValue(std::move(node));
}
ScalarValueOp Operation(Opcode op) const {
switch (op) {
case Opcode::IAddU32:
case Opcode::ScalarSignedAddOverflowI32: return ScalarValueOp::Add;
case Opcode::IAddCarryU32:
case Opcode::ScalarAddCarryU32: return ScalarValueOp::AddCarry;
case Opcode::ISubU32:
case Opcode::ScalarSubBorrowU32:
case Opcode::ScalarSignedSubOverflowI32: return ScalarValueOp::Sub;
case Opcode::ScalarSubBorrowCarryU32: return ScalarValueOp::SubBorrow;
case Opcode::IMulU32: return ScalarValueOp::Mul;
case Opcode::BitwiseAndU32: return ScalarValueOp::And;
case Opcode::BitwiseAndNotU32: return ScalarValueOp::AndNot;
case Opcode::BitwiseOrU32: return ScalarValueOp::Or;
case Opcode::BitwiseOrNotU32: return ScalarValueOp::OrNot;
case Opcode::BitwiseXorU32: return ScalarValueOp::Xor;
case Opcode::BitwiseNotU32: return ScalarValueOp::Not;
case Opcode::ShiftLeftLogicalU32: return ScalarValueOp::ShiftLeft;
case Opcode::ShiftRightLogicalU32: return ScalarValueOp::ShiftRight;
case Opcode::ShiftRightArithmeticI32: return ScalarValueOp::ShiftRightArithmetic;
case Opcode::BitFieldMaskU32: return ScalarValueOp::BitFieldMaskU32;
case Opcode::IAdd3U32: return ScalarValueOp::Add3;
case Opcode::ScalarShiftLeftAddCarryU32:
case Opcode::ShiftLeftAddU32: return ScalarValueOp::ShiftLeftAdd;
case Opcode::AddShiftLeftU32: return ScalarValueOp::AddShiftLeft;
case Opcode::XorAddU32: return ScalarValueOp::XorAdd;
case Opcode::ShiftLeftOrU32: return ScalarValueOp::ShiftLeftOr;
default: return ScalarValueOp::Unknown;
}
}
void UpdateScc(const Instruction& inst, const ScalarState& before, ScalarState& state) {
switch (inst.op) {
case Opcode::ScalarAddCarryU32:
state.scc = Define(inst, ScalarValueOp::Carry, before);
break;
case Opcode::ScalarSubBorrowU32:
case Opcode::ScalarSubBorrowCarryU32: state.scc = DefineBorrow(inst, before); break;
case Opcode::ScalarShiftLeftAddCarryU32:
state.scc = Define(inst, ScalarValueOp::ShiftLeftAddCarry, before);
break;
default: break;
}
}
void WriteDestination(const Instruction& inst, ScalarState& state) {
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Scc) {
state.scc = inst.op == Opcode::MoveU32 && inst.src_count != 0
? OperandValue(inst.src[0], state)
: ScalarProvenance::Unknown;
return;
}
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::M0) {
const auto before = state;
const auto op = Operation(inst.op);
state.m0 = ScalarProvenance::Unknown;
if (inst.op == Opcode::MoveU32 && inst.src_count != 0) {
state.m0 = OperandValue(inst.src[0], before);
} else if (op != ScalarValueOp::Unknown) {
state.m0 = Define(inst, op, before);
}
UpdateScc(inst, before, state);
return;
}
uint32_t dst = 0;
if (!ScalarRegister(inst.dst, dst)) {
return;
}
const auto before = state;
state.regs[dst] = ScalarProvenance::Unknown;
if (PairDwordOpcode(inst.op) && dst + 1 < ScalarRegisters) {
state.regs[dst + 1] = ScalarProvenance::Unknown;
}
uint32_t value = ScalarProvenance::Unknown;
switch (inst.op) {
case Opcode::MoveU32:
case Opcode::MoveF32Bits:
if (inst.src_count != 0) {
if (inst.src[0].kind == OperandKind::PcRelativeU32) {
value =
InternValue({ScalarValueOp::PcRelativeLow, inst.pc, inst.src[0].imm});
} else {
value = OperandValue(inst.src[0], before);
}
}
if (inst.op == Opcode::MoveU32 && inst.src_count != 0 &&
inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].imm == 0 &&
dst != 0) {
const auto low = before.regs[dst - 1];
if (low < m_graph.values.size() &&
m_graph.values[low].op == ScalarValueOp::PcRelativeLow &&
m_graph.values[low].pc == inst.pc) {
value = InternValue(
{ScalarValueOp::PcRelativeHigh, inst.pc, m_graph.values[low].imm});
}
}
break;
case Opcode::MoveU64: {
uint32_t src = 0;
if (inst.src_count != 0 && dst + 1 < ScalarRegisters) {
if (ScalarRegister(inst.src[0], src) && src + 1 < ScalarRegisters) {
value = before.regs[src];
state.regs[dst + 1] = before.regs[src + 1];
} else {
value = OperandValue(inst.src[0], before);
state.regs[dst + 1] = Constant(
inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].sext_64
? UINT32_MAX
: 0u);
}
}
break;
}
case Opcode::BitFieldMaskU64:
value = Define(inst, ScalarValueOp::BitFieldMaskU64Low, before);
if (dst + 1 < ScalarRegisters) {
state.regs[dst + 1] = Define(inst, ScalarValueOp::BitFieldMaskU64High, before);
}
break;
case Opcode::SLoadDword: value = ReadConst(inst, before, false); break;
case Opcode::SBufferLoadDword: value = ReadConst(inst, before, true); break;
case Opcode::ReadLaneU32: value = ReadVectorLane(inst, before); break;
default: {
const auto op = Operation(inst.op);
if (op != ScalarValueOp::Unknown) {
value = Define(inst, op, before);
}
break;
}
}
state.regs[dst] = value;
UpdateScc(inst, before, state);
uint32_t dst2 = 0;
if (ScalarRegister(inst.dst2, dst2)) {
state.regs[dst2] =
inst.src_count > 1 ? OperandValue(inst.src[1], before) : ScalarProvenance::Unknown;
}
}
uint32_t AddDescriptor(DescriptorValue descriptor) {
const DescriptorKey key {descriptor.dwords, descriptor.dword_count};
if (const auto found = m_descriptors.find(key); found != m_descriptors.end()) {
return found->second;
}
m_graph.descriptors.push_back(descriptor);
const auto id = static_cast<uint32_t>(m_graph.descriptors.size() - 1) + 2u;
m_descriptors.emplace(key, id);
return id;
}
uint32_t AddDescriptor(const ScalarState& state, uint32_t base, uint32_t dwords) {
if (base >= ScalarRegisters || dwords == 0 || dwords > 8 ||
base + dwords > ScalarRegisters) {
return ScalarProvenance::Unknown;
}
DescriptorValue descriptor;
descriptor.dword_count = dwords;
for (uint32_t i = 0; i < dwords; i++) {
descriptor.dwords[i] = state.regs[base + i];
}
return AddDescriptor(descriptor);
}
uint32_t AddSamplerDescriptor(const ScalarState& state, uint32_t base) {
if (base >= ScalarRegisters || 4u > ScalarRegisters - base) {
return ScalarProvenance::Unknown;
}
DescriptorValue descriptor;
descriptor.dword_count = 4;
for (uint32_t i = 0; i < 4; i++) {
descriptor.dwords[i] = state.regs[base + i];
}
if (auto d0 = descriptor.dwords[0];
d0 < m_graph.values.size() && m_graph.values[d0].op == ScalarValueOp::Constant &&
(m_graph.values[d0].imm & SamplerBorderClampMask) == 0) {
// Without a border clamp, the border color and table index in dword 3 are unused.
descriptor.dwords[3] = Constant(0);
}
return AddDescriptor(descriptor);
}
uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) {
const uint32_t first = FlatStore(inst.op) ? 1u : 0u;
if (inst.src_count < first + 2u) {
return ScalarProvenance::Unknown;
}
uint32_t low = 0;
uint32_t high = 0;
if (!VectorRegister(inst.src[first], low) || !VectorRegister(inst.src[first + 1u], high) ||
state.address_bases[low] <= ScalarProvenance::Unknown ||
state.address_bases[high] <= ScalarProvenance::Unknown) {
return ScalarProvenance::Unknown;
}
DescriptorValue descriptor;
descriptor.dword_count = 2;
descriptor.dwords[0] = state.address_bases[low];
descriptor.dwords[1] = state.address_bases[high];
return AddDescriptor(descriptor);
}
void AttachSources(Instruction& inst, const ScalarState& state) {
inst.memory.resource_source = 0;
inst.memory.sampler_source = 0;
if (inst.op == Opcode::SLoadDword) {
inst.memory.resource_source = AddDescriptor(state, inst.memory.resource, 2);
return;
}
if (inst.memory.kind == ResourceKind::Flat || inst.memory.kind == ResourceKind::Global ||
inst.memory.kind == ResourceKind::Scratch) {
inst.memory.resource_source = ScalarProvenance::Unknown;
if (inst.memory.kind == ResourceKind::Flat) {
inst.memory.resource_source = AddFlatAddressDescriptor(inst, state);
}
for (uint32_t i = 0; i < inst.src_count; i++) {
const auto& source = inst.src[i];
if (inst.memory.resource_source == ScalarProvenance::Unknown &&
source.kind == OperandKind::Register &&
source.reg.file == RegisterFile::Scalar) {
inst.memory.resource_source = AddDescriptor(state, source.reg.index, 2);
break;
}
}
return;
}
if (inst.memory.kind == ResourceKind::Buffer ||
(inst.memory.kind == ResourceKind::ScalarBuffer &&
inst.op == Opcode::SBufferLoadDword)) {
const auto base = inst.memory.resource * 4u;
inst.memory.resource_source = AddDescriptor(state, base, 4);
return;
}
if (inst.memory.kind == ResourceKind::Image ||
inst.memory.kind == ResourceKind::ImageUint ||
inst.memory.kind == ResourceKind::StorageImage ||
inst.memory.kind == ResourceKind::StorageImageUint) {
inst.memory.resource_source = AddDescriptor(state, inst.memory.resource * 4u, 8);
if (inst.op == Opcode::ImageSample || inst.op == Opcode::ImageGather4 ||
inst.op == Opcode::ImageGetLod) {
inst.memory.sampler_source = AddSamplerDescriptor(state, inst.memory.sampler * 4u);
}
}
}
void AttachScalarSources(Instruction& inst, const ScalarState& state) {
std::fill(std::begin(inst.scalar_sources), std::end(inst.scalar_sources),
ScalarProvenance::Undefined);
for (uint32_t i = 0; i < inst.src_count; i++) {
const auto& source = inst.src[i];
if (source.kind == OperandKind::Register && source.reg.file != RegisterFile::Vector &&
source.reg.file != RegisterFile::Exec) {
inst.scalar_sources[i] = OperandValue(source, state);
}
}
}
ScalarState Execute(BasicBlock& block, ScalarState state, bool attach_sources) {
for (size_t index = 0; index < block.instructions.size();) {
auto& inst = block.instructions[index];
uint32_t first_dst = 0;
if (ScalarMemoryLoad(inst.op) && inst.memory.component_count > 1 &&
ScalarRegister(inst.dst, first_dst)) {
const auto before = state;
for (uint32_t component = 0; component < inst.memory.component_count; component++) {
auto& component_inst = block.instructions[index + component];
if (attach_sources) {
component_inst.scalar_value = ScalarProvenance::Undefined;
AttachScalarSources(component_inst, before);
AttachSources(component_inst, before);
}
auto component_state = before;
WriteDestination(component_inst, component_state);
uint32_t dst = 0;
ScalarRegister(component_inst.dst, dst);
state.regs[dst] = component_state.regs[dst];
if (attach_sources) {
component_inst.scalar_value = component_state.regs[dst];
}
}
index += inst.memory.component_count;
continue;
}
if (attach_sources) {
inst.scalar_value = ScalarProvenance::Undefined;
AttachScalarSources(inst, state);
AttachSources(inst, state);
}
WriteVectorDestination(inst, state);
WriteDestination(inst, state);
if (attach_sources) {
uint32_t dst = 0;
if (ScalarRegister(inst.dst, dst)) {
inst.scalar_value = state.regs[dst];
}
}
index++;
}
return state;
}
ScalarState MergeEntry(size_t block_index) {
ScalarState result;
result.Fill(ScalarProvenance::Undefined);
const auto& block = m_program.blocks[block_index];
const auto merge = [&](uint32_t initial, auto predecessor_value, uint32_t* phi) {
std::vector<uint32_t> incoming;
if (initial != ScalarProvenance::Undefined) {
incoming.push_back(initial);
}
for (const auto predecessor: block.predecessors) {
const auto value = predecessor_value(predecessor, m_exit[predecessor]);
if (value != ScalarProvenance::Undefined &&
std::find(incoming.begin(), incoming.end(), value) == incoming.end()) {
incoming.push_back(value);
}
}
if (incoming.empty()) {
return ScalarProvenance::Undefined;
}
if (*phi == ScalarProvenance::Undefined) {
if (incoming.size() == 1) {
return incoming[0];
}
*phi = AddValue({ScalarValueOp::Phi, block.start_pc});
}
m_graph.values[*phi].phi_args = std::move(incoming);
return *phi;
};
for (uint32_t reg = 0; reg < ScalarRegisters; reg++) {
const auto initial =
block_index == 0 ? m_user_data.regs[reg] : ScalarProvenance::Undefined;
result.regs[reg] = merge(
initial, [reg](uint32_t, const ScalarState& state) { return state.regs[reg]; },
&m_phi[block_index].regs[reg]);
}
for (uint32_t reg = 0; reg < VectorRegisters; reg++) {
result.address_bases[reg] = merge(
block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined,
[reg](uint32_t, const ScalarState& state) { return state.address_bases[reg]; },
&m_phi[block_index].address_bases[reg]);
}
result.scc = merge(
block_index == 0 ? m_user_data.scc : ScalarProvenance::Undefined,
[](uint32_t, const ScalarState& state) { return state.scc; }, &m_phi[block_index].scc);
result.m0 = merge(
block_index == 0 ? m_user_data.m0 : ScalarProvenance::Undefined,
[](uint32_t, const ScalarState& state) { return state.m0; }, &m_phi[block_index].m0);
std::vector<uint64_t> vector_lanes;
for (const auto predecessor: block.predecessors) {
for (const auto& [lane, value]: m_exit[predecessor].vector_lanes) {
(void)value;
vector_lanes.push_back(lane);
}
}
std::sort(vector_lanes.begin(), vector_lanes.end());
vector_lanes.erase(std::unique(vector_lanes.begin(), vector_lanes.end()),
vector_lanes.end());
for (const auto lane: vector_lanes) {
auto& phi = m_phi[block_index].vector_lanes[lane];
result.vector_lanes[lane] = merge(
block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined,
[this, lane](uint32_t predecessor, const ScalarState& state) {
if (!m_exit_ready[predecessor]) {
return ScalarProvenance::Undefined;
}
const auto found = state.vector_lanes.find(lane);
return found != state.vector_lanes.end() ? found->second
: ScalarProvenance::Unknown;
},
&phi);
}
return result;
}
void Queue(uint32_t block) {
if (!m_queued[block]) {
m_queued[block] = true;
m_work.push_back(block);
}
}
Program& m_program;
ScalarProvenance& m_graph;
ScalarState m_user_data;
std::vector<ScalarState> m_entry;
std::vector<ScalarState> m_exit;
std::vector<ScalarState> m_phi;
std::vector<bool> m_exit_ready;
std::vector<bool> m_queued;
std::deque<uint32_t> m_work;
std::map<ValueKey, uint32_t> m_values;
std::map<DescriptorKey, uint32_t> m_descriptors;
};
} // namespace
bool BuildScalarProvenance(Program& program, std::string* error) {
if (program.resource_tracking_complete) {
if (error != nullptr) {
*error = "cannot rebuild scalar provenance after resource tracking";
}
return false;
}
if (program.user_data_count > 64) {
if (error != nullptr) {
*error = "scalar provenance user-data count exceeds 64 SGPRs";
}
return false;
}
if (program.user_data_base > ScalarRegisters ||
program.user_data_count > ScalarRegisters - program.user_data_base) {
if (error != nullptr) {
*error = "scalar provenance user-data window is out of range";
}
return false;
}
if (program.srt_patching_complete) {
if (error != nullptr) {
*error = "cannot rebuild scalar provenance after SRT patching";
}
return false;
}
program.srt = {};
program.srt_plan_complete = false;
return Builder(program).Run(error);
}
const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source) {
if (source < 2 || source - 2 >= program.provenance.descriptors.size()) {
return nullptr;
}
return &program.provenance.descriptors[source - 2];
}
bool DescriptorSourceResolved(const Program& program, uint32_t source) {
const auto* descriptor = GetDescriptorSource(program, source);
if (descriptor == nullptr) {
return false;
}
std::vector<uint8_t> visited(program.provenance.values.size());
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!ValueResolved(program.provenance, descriptor->dwords[i], visited)) {
return false;
}
}
return true;
}
std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value) {
if (value >= provenance.values.size()) {
return "invalid";
}
const auto& node = provenance.values[value];
switch (node.op) {
case ScalarValueOp::Undefined: return "undefined";
case ScalarValueOp::Unknown: return "unknown";
case ScalarValueOp::UserData: return fmt::format("ud{}", node.imm);
case ScalarValueOp::Constant: return fmt::format("0x{:08x}", node.imm);
case ScalarValueOp::PcRelativeLow: return fmt::format("pc_lo+0x{:x}", node.imm);
case ScalarValueOp::PcRelativeHigh: return fmt::format("pc_hi+0x{:x}", node.imm);
case ScalarValueOp::ReadConst:
return fmt::format("read_const({}, {}, {}, +{})", node.args[0], node.args[1],
node.args[2], node.imm);
case ScalarValueOp::ReadConstBuffer:
return fmt::format("read_const_buffer({}, {}, {}, {}, {}, +{})", node.args[0],
node.args[1], node.args[2], node.args[3], node.args[4], node.imm);
case ScalarValueOp::Phi: return fmt::format("phi{}", value);
case ScalarValueOp::BitFieldMaskU32:
return fmt::format("bfm_u32({}, {})", node.args[0], node.args[1]);
default: return fmt::format("value{}", value);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ScalarProvenance.h
-20
View File
@@ -1,20 +0,0 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
// Builds scalar reaching definitions and attaches descriptor sources to individual memory uses.
bool BuildScalarProvenance(Program& program, std::string* error);
uint32_t ScalarValueArgCount(ScalarValueOp op);
const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source);
bool DescriptorSourceResolved(const Program& program, uint32_t source);
std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ */
src/graphics/shader/recompiler/ir/ShaderIR.cpp
+27 -14
View File
@@ -32,6 +32,13 @@ Operand MakePcRelativeU32(uint32_t value) {
return operand;
}
Operand MakePcRelativeHighU32(uint32_t value) {
Operand operand;
operand.kind = OperandKind::PcRelativeHighU32;
operand.imm = value;
return operand;
}
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
if (!ScalarResultWritesSccNonZero(decoded.opcode)) {
@@ -74,6 +81,7 @@ uint32_t VectorByteConvertIndex(Decoder::Opcode opcode) {
void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand) {
operand.sdwa_sel = decoded.sdwa_sel;
operand.sdwa_dst_unused = decoded.sdwa_dst_unused;
operand.omod = decoded.omod;
operand.sdwa_sext = decoded.sdwa_sext;
operand.op_sel = decoded.op_sel;
@@ -214,17 +222,20 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
mem.image_nsa_addr[i] = decoded.image_nsa_addr[i];
}
mem.memory_segment = decoded.memory_segment;
mem.data_signed = decoded.data_signed;
mem.typed = decoded.typed;
mem.formatted = decoded.formatted;
mem.image_has_mip = decoded.opcode == Decoder::Opcode::ImageLoadMip ||
decoded.opcode == Decoder::Opcode::ImageStoreMip;
mem.glc = decoded.glc;
mem.slc = decoded.slc;
mem.idxen = decoded.idxen;
mem.offen = decoded.offen;
mem.resource = ResourceIndexFromOperand(decoded.src1);
mem.sampler = ResourceIndexFromOperand(decoded.src2);
mem.address_is_full =
kind == ResourceKind::Flat || kind == ResourceKind::Scratch ||
(kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.data_signed = decoded.data_signed;
mem.typed = decoded.typed;
mem.formatted = decoded.formatted;
mem.image_has_mip = decoded.opcode == Decoder::Opcode::ImageLoadMip ||
decoded.opcode == Decoder::Opcode::ImageStoreMip;
mem.glc = decoded.glc;
mem.slc = decoded.slc;
mem.idxen = decoded.idxen;
mem.offen = decoded.offen;
mem.resource = ResourceIndexFromOperand(decoded.src1);
mem.sampler = ResourceIndexFromOperand(decoded.src2);
if (kind == ResourceKind::Lds || kind == ResourceKind::Gds) {
mem.resource = 0;
mem.sampler = 0;
@@ -246,6 +257,7 @@ uint32_t RawScalarLoadBase(const Decoder::Operand& operand) {
void ClearRegisterOffsetModifiers(Decoder::Operand& operand) {
operand.sdwa_sel = 6;
operand.sdwa_dst_unused = 2;
operand.omod = 0;
operand.sdwa_sext = false;
operand.op_sel = false;
@@ -988,9 +1000,10 @@ bool LowerScalarGetpcB64(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
for (uint32_t i = 0; i < 2u; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::MoveU32;
inst.src[0] = i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakeImmediateU32(0u);
inst.pc = decoded.pc;
inst.op = Opcode::MoveU32;
inst.src[0] =
i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakePcRelativeHighU32(decoded.pc + 4u);
inst.src_count = 1;
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error)) {
return false;
src/graphics/shader/recompiler/ir/ShaderIR.h
+38 -148
View File
@@ -4,52 +4,19 @@
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
#include "graphics/shader/recompiler/ir/opcodes/Opcodes.h"
#include "graphics/shader/shader.h"
#include <array>
#include <memory>
#include <string_view>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
enum class Opcode {
#define IR_OPCODE(name, category) name,
#include "graphics/shader/recompiler/ir/ShaderIROpcodes.inc"
#undef IR_OPCODE
Count,
};
enum class OpcodeClass { General, Compare, Compare64 };
struct OpcodeInfo {
std::string_view name;
OpcodeClass opcode_class = OpcodeClass::General;
};
inline constexpr OpcodeInfo OPCODE_INFO[] = {
#define IR_OPCODE(name, category) {#name, OpcodeClass::category},
#include "graphics/shader/recompiler/ir/ShaderIROpcodes.inc"
#undef IR_OPCODE
};
static_assert(sizeof(OPCODE_INFO) / sizeof(OPCODE_INFO[0]) == static_cast<size_t>(Opcode::Count));
constexpr const OpcodeInfo& GetOpcodeInfo(Opcode opcode) {
return OPCODE_INFO[static_cast<size_t>(opcode)];
}
constexpr std::string_view OpcodeName(Opcode opcode) {
return GetOpcodeInfo(opcode).name;
}
constexpr bool IsCompareOpcode(Opcode opcode) {
return GetOpcodeInfo(opcode).opcode_class != OpcodeClass::General;
}
constexpr bool IsCompare64Opcode(Opcode opcode) {
return GetOpcodeInfo(opcode).opcode_class == OpcodeClass::Compare64;
}
struct ValueProgram;
enum class RegisterFile { Scalar, Vector, Vcc, Exec, Scc, M0 };
@@ -62,7 +29,7 @@ struct Register {
}
};
enum class OperandKind { Register, ImmediateU32, PcRelativeU32, Null };
enum class OperandKind { Register, ImmediateU32, PcRelativeU32, PcRelativeHighU32, Null };
struct Operand {
OperandKind kind = OperandKind::Null;
@@ -70,6 +37,7 @@ struct Operand {
uint32_t imm = 0;
bool sext_64 = false;
uint32_t sdwa_sel = 6;
uint32_t sdwa_dst_unused = 2;
uint32_t omod = 0;
bool sdwa_sext = false;
bool op_sel = false;
@@ -123,19 +91,17 @@ struct MemoryInfo {
uint32_t image_nsa_dwords = 0;
uint32_t image_nsa_addr[Decoder::MaxImageNsaAddressComponents] = {};
uint32_t memory_segment = 0;
// Dense resource indices are assigned by resource tracking. These source IDs refer to the
// exact scalar definitions reaching this instruction before that patching step.
uint32_t resource_source = 0;
uint32_t sampler_source = 0;
bool data_signed = false;
bool typed = false;
bool formatted = false;
bool image_has_mip = false;
bool image_cube = false;
bool glc = false;
bool slc = false;
bool idxen = false;
bool offen = false;
bool address_is_full = false;
bool data_signed = false;
bool typed = false;
bool formatted = false;
bool image_has_mip = false;
bool image_cube = false;
bool glc = false;
bool slc = false;
bool idxen = false;
bool offen = false;
bool planning_only = false;
bool operator==(const MemoryInfo& other) const = default;
};
@@ -169,9 +135,7 @@ struct Instruction {
Operand dst;
Operand dst2;
Operand src[4];
uint32_t src_count = 0;
uint32_t scalar_sources[4] = {};
uint32_t scalar_value = 0;
uint32_t src_count = 0;
MemoryInfo memory;
ExportInfo export_info;
InputInfo input_info;
@@ -192,53 +156,6 @@ struct BasicBlock {
std::vector<Instruction> instructions;
};
enum class ScalarValueOp {
Undefined,
Unknown,
UserData,
Constant,
PcRelativeLow,
PcRelativeHigh,
Add,
AddCarry,
Carry,
Sub,
SubBorrow,
Borrow,
Mul,
And,
AndNot,
Or,
OrNot,
Xor,
Not,
ShiftLeft,
ShiftRight,
ShiftRightArithmetic,
BitFieldMaskU32,
BitFieldMaskU64Low,
BitFieldMaskU64High,
Add3,
ShiftLeftAdd,
ShiftLeftAddCarry,
AddShiftLeft,
XorAdd,
ShiftLeftOr,
ReadConst,
ReadConstBuffer,
Phi,
};
struct ScalarValue {
ScalarValueOp op = ScalarValueOp::Undefined;
uint32_t pc = 0;
uint32_t imm = 0;
std::array<uint32_t, 6> args = {};
std::vector<uint32_t> phi_args;
bool operator==(const ScalarValue& other) const = default;
};
struct DescriptorValue {
std::array<uint32_t, 8> dwords = {};
uint32_t dword_count = 0;
@@ -248,32 +165,6 @@ struct DescriptorValue {
}
};
struct ScalarProvenance {
static constexpr uint32_t Undefined = 0;
static constexpr uint32_t Unknown = 1;
std::vector<ScalarValue> values;
std::vector<DescriptorValue> descriptors;
bool operator==(const ScalarProvenance& other) const = default;
};
struct SrtRead {
uint32_t value = 0;
uint32_t flat_offset = 0;
uint32_t use_pc = 0;
bool operator==(const SrtRead& other) const = default;
};
struct SrtPlan {
std::vector<SrtRead> reads;
std::vector<uint32_t> dynamic_reads;
std::vector<uint32_t> dynamic_sources;
bool operator==(const SrtPlan& other) const = default;
};
struct BufferResource {
static constexpr uint32_t NoImageAlias = UINT32_MAX;
@@ -328,11 +219,12 @@ struct SampledResourcePair {
};
struct AddressResource {
uint32_t source = 0;
uint32_t source = UINT32_MAX;
uint32_t first_use_pc = 0;
ResourceKind kind = ResourceKind::Flat;
int32_t min_offset = 0;
uint64_t specialized_base = 0;
bool unbased = true;
bool read = false;
bool written = false;
bool atomic = false;
@@ -451,25 +343,23 @@ struct ShaderInfo {
};
struct Program {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
bool dispatcher_fallback = false;
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
std::string fallback_reason;
std::vector<BasicBlock> blocks;
ScalarProvenance provenance;
SrtPlan srt;
bool srt_plan_complete = false;
bool srt_patching_complete = false;
ShaderInfo info;
bool resource_tracking_complete = false;
bool shader_info_complete = false;
BindingLayout bindings;
bool binding_layout_complete = false;
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
bool dispatcher_fallback = false;
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
std::string fallback_reason;
std::vector<BasicBlock> blocks;
std::shared_ptr<ValueProgram> values;
bool srt_plan_complete = false;
ShaderInfo info;
bool resource_tracking_complete = false;
bool shader_info_complete = false;
BindingLayout bindings;
bool binding_layout_complete = false;
};
bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage,
src/graphics/shader/recompiler/ir/ShaderIRInternal.h
+1
View File
@@ -18,6 +18,7 @@ bool ScalarResultIs64Bit(Decoder::Opcode opcode);
Operand MakeSccOperand();
Operand MakeImmediateU32(uint32_t value);
Operand MakePcRelativeU32(uint32_t value);
Operand MakePcRelativeHighU32(uint32_t value);
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error);
uint32_t VectorByteConvertIndex(Decoder::Opcode opcode);
src/graphics/shader/recompiler/ir/ShaderIRLog.cpp
+3
View File
@@ -39,6 +39,9 @@ std::string OperandToString(const Operand& operand) {
case OperandKind::Register: text = RegisterToString(operand.reg); break;
case OperandKind::ImmediateU32: text = fmt::format("0x{:08x}", operand.imm); break;
case OperandKind::PcRelativeU32: text = fmt::format("pc_rel(0x{:08x})", operand.imm); break;
case OperandKind::PcRelativeHighU32:
text = fmt::format("pc_rel_hi(0x{:08x})", operand.imm);
break;
case OperandKind::Null: text = "null"; break;
default: text = "unknown"; break;
}
src/graphics/shader/recompiler/ir/ShaderIROpcodes.inc
-342
View File
@@ -1,342 +0,0 @@
// Central IR opcode catalog. Keep semantic categories here so enum identity, logging,
// and generic analysis cannot drift apart.
IR_OPCODE(ControlNop, General)
IR_OPCODE(Waitcnt, General)
IR_OPCODE(Barrier, General)
IR_OPCODE(Sendmsg, General)
IR_OPCODE(TtraceData, General)
IR_OPCODE(InstPrefetch, General)
IR_OPCODE(MoveU32, General)
IR_OPCODE(MoveF32Bits, General)
IR_OPCODE(MoveRelDestU32, General)
IR_OPCODE(MoveRelSourceU32, General)
IR_OPCODE(MoveU64, General)
IR_OPCODE(WqmB64, General)
IR_OPCODE(SaveexecB32, General)
IR_OPCODE(SaveexecB64, General)
IR_OPCODE(ReadFirstLaneU32, General)
IR_OPCODE(ReadLaneU32, General)
IR_OPCODE(WriteLaneU32, General)
IR_OPCODE(Permlane16B32, General)
IR_OPCODE(Permlanex16B32, General)
IR_OPCODE(AbsI32, General)
IR_OPCODE(IAddU32, General)
IR_OPCODE(IAddCarryU32, General)
IR_OPCODE(ISubBorrowU32, General)
IR_OPCODE(ScalarAddCarryU32, General)
IR_OPCODE(ScalarSubBorrowU32, General)
IR_OPCODE(ScalarSubBorrowCarryU32, General)
IR_OPCODE(ScalarSignedAddOverflowI32, General)
IR_OPCODE(ScalarSignedSubOverflowI32, General)
IR_OPCODE(ScalarShiftLeftAddCarryU32, General)
IR_OPCODE(ISubU32, General)
IR_OPCODE(IMulU32, General)
IR_OPCODE(UMulHighU32, General)
IR_OPCODE(SMulHighI32, General)
IR_OPCODE(IMadI24U32, General)
IR_OPCODE(UMadU24U32, General)
IR_OPCODE(UMadU64U32, General)
IR_OPCODE(SadU32, General)
IR_OPCODE(IAdd3U32, General)
IR_OPCODE(IMulI24U32, General)
IR_OPCODE(UMulU24U32, General)
IR_OPCODE(IMinI32, General)
IR_OPCODE(IMaxI32, General)
IR_OPCODE(IMin3I32, General)
IR_OPCODE(IMax3I32, General)
IR_OPCODE(IMed3I32, General)
IR_OPCODE(UMinU32, General)
IR_OPCODE(UMaxU32, General)
IR_OPCODE(UMin3U32, General)
IR_OPCODE(UMax3U32, General)
IR_OPCODE(UMed3U32, General)
IR_OPCODE(BitwiseAndU32, General)
IR_OPCODE(BitwiseAndU64, General)
IR_OPCODE(BitwiseAndNotU32, General)
IR_OPCODE(BitwiseAndNotU64, General)
IR_OPCODE(BitwiseOrU32, General)
IR_OPCODE(BitwiseOrU64, General)
IR_OPCODE(BitwiseOrNotU32, General)
IR_OPCODE(BitwiseOrNotU64, General)
IR_OPCODE(BitwiseAndOrU32, General)
IR_OPCODE(BitwiseOr3U32, General)
IR_OPCODE(BitwiseXorU32, General)
IR_OPCODE(BitwiseXorU64, General)
IR_OPCODE(BitwiseXor3U32, General)
IR_OPCODE(BitwiseNandU32, General)
IR_OPCODE(BitwiseNandU64, General)
IR_OPCODE(BitwiseNorU32, General)
IR_OPCODE(BitwiseNorU64, General)
IR_OPCODE(BitwiseXnorU32, General)
IR_OPCODE(BitwiseXnorU64, General)
IR_OPCODE(BitwiseNotU32, General)
IR_OPCODE(BitwiseNotU64, General)
IR_OPCODE(BitClearU32, General)
IR_OPCODE(BitSetU32, General)
IR_OPCODE(BitReverseU32, General)
IR_OPCODE(BitCountU32, General)
IR_OPCODE(BitCountU64, General)
IR_OPCODE(BitReplicateB64B32, General)
IR_OPCODE(BitCountAddU32, General)
IR_OPCODE(MaskedBitCountLowU32, General)
IR_OPCODE(MaskedBitCountHighU32, General)
IR_OPCODE(FindLsbU32, General)
IR_OPCODE(FindMsbFromHighU32, General)
IR_OPCODE(FindMsbFromHighU64, General)
IR_OPCODE(BitFieldMaskU32, General)
IR_OPCODE(BitFieldMaskU64, General)
IR_OPCODE(BitFieldExtractU32, General)
IR_OPCODE(BitFieldExtractU64, General)
IR_OPCODE(BitFieldExtract3U32, General)
IR_OPCODE(BitFieldExtract3I32, General)
IR_OPCODE(BitFieldInsertSelectU32, General)
IR_OPCODE(BitCompare0B32, General)
IR_OPCODE(BitCompare1B32, General)
IR_OPCODE(AlignBitU32, General)
IR_OPCODE(ShiftLeftAddU32, General)
IR_OPCODE(AddShiftLeftU32, General)
IR_OPCODE(XorAddU32, General)
IR_OPCODE(ShiftLeftOrU32, General)
IR_OPCODE(ShiftLeftLogicalU32, General)
IR_OPCODE(ShiftLeftLogicalU64, General)
IR_OPCODE(ShiftLeftLogicalU16, General)
IR_OPCODE(ShiftRightLogicalU32, General)
IR_OPCODE(ShiftRightLogicalU64, General)
IR_OPCODE(ShiftRightLogicalU16, General)
IR_OPCODE(ShiftRightArithmeticI32, General)
IR_OPCODE(ShiftRightArithmeticI16, General)
IR_OPCODE(SelectU32, General)
IR_OPCODE(SelectMaskU32, General)
IR_OPCODE(SelectF32Bits, General)
IR_OPCODE(SelectMaskF32Bits, General)
IR_OPCODE(SelectU64, General)
IR_OPCODE(PackLowLowU16, General)
IR_OPCODE(PackLowHighU16, General)
IR_OPCODE(PackHighHighU16, General)
IR_OPCODE(CompareFalse, Compare)
IR_OPCODE(CompareTrue, Compare)
IR_OPCODE(CompareEqU32, Compare)
IR_OPCODE(CompareNeU32, Compare)
IR_OPCODE(CompareGtU32, Compare)
IR_OPCODE(CompareGeU32, Compare)
IR_OPCODE(CompareLtU32, Compare)
IR_OPCODE(CompareLeU32, Compare)
IR_OPCODE(CompareEqU64, Compare64)
IR_OPCODE(CompareNeU64, Compare64)
IR_OPCODE(CompareMaskEqU32, Compare)
IR_OPCODE(CompareMaskNeU32, Compare)
IR_OPCODE(CompareMaskGtU32, Compare)
IR_OPCODE(CompareMaskGeU32, Compare)
IR_OPCODE(CompareMaskLtU32, Compare)
IR_OPCODE(CompareMaskLeU32, Compare)
IR_OPCODE(CompareEqI32, Compare)
IR_OPCODE(CompareNeI32, Compare)
IR_OPCODE(CompareGtI32, Compare)
IR_OPCODE(CompareGeI32, Compare)
IR_OPCODE(CompareLtI32, Compare)
IR_OPCODE(CompareLeI32, Compare)
IR_OPCODE(CompareEqI16, Compare)
IR_OPCODE(CompareNeI16, Compare)
IR_OPCODE(CompareGtI16, Compare)
IR_OPCODE(CompareGeI16, Compare)
IR_OPCODE(CompareLtI16, Compare)
IR_OPCODE(CompareLeI16, Compare)
IR_OPCODE(CompareMaskEqI32, Compare)
IR_OPCODE(CompareMaskNeI32, Compare)
IR_OPCODE(CompareMaskGtI32, Compare)
IR_OPCODE(CompareMaskGeI32, Compare)
IR_OPCODE(CompareMaskLtI32, Compare)
IR_OPCODE(CompareMaskLeI32, Compare)
IR_OPCODE(CompareEqU16, Compare)
IR_OPCODE(CompareNeU16, Compare)
IR_OPCODE(CompareGtU16, Compare)
IR_OPCODE(CompareGeU16, Compare)
IR_OPCODE(CompareLtU16, Compare)
IR_OPCODE(CompareLeU16, Compare)
IR_OPCODE(CompareEqF32, Compare)
IR_OPCODE(CompareNeF32, Compare)
IR_OPCODE(CompareGtF32, Compare)
IR_OPCODE(CompareGeF32, Compare)
IR_OPCODE(CompareLtF32, Compare)
IR_OPCODE(CompareLeF32, Compare)
IR_OPCODE(CompareOrderedF32, Compare)
IR_OPCODE(CompareUnorderedF32, Compare)
IR_OPCODE(CompareUnordEqF32, Compare)
IR_OPCODE(CompareUnordNeF32, Compare)
IR_OPCODE(CompareUnordGtF32, Compare)
IR_OPCODE(CompareUnordGeF32, Compare)
IR_OPCODE(CompareUnordLtF32, Compare)
IR_OPCODE(CompareUnordLeF32, Compare)
IR_OPCODE(CompareClassF32, Compare)
IR_OPCODE(CompareEqF16, Compare)
IR_OPCODE(CompareNeF16, Compare)
IR_OPCODE(CompareGtF16, Compare)
IR_OPCODE(CompareGeF16, Compare)
IR_OPCODE(CompareLtF16, Compare)
IR_OPCODE(CompareLeF16, Compare)
IR_OPCODE(CompareUnordNeF16, Compare)
IR_OPCODE(CompareMaskEqF16, Compare)
IR_OPCODE(CompareMaskNeF16, Compare)
IR_OPCODE(CompareMaskGtF16, Compare)
IR_OPCODE(CompareMaskGeF16, Compare)
IR_OPCODE(CompareMaskLtF16, Compare)
IR_OPCODE(CompareMaskLeF16, Compare)
IR_OPCODE(CompareMaskUnordNeF16, Compare)
IR_OPCODE(CompareMaskUnordGeF16, Compare)
IR_OPCODE(CompareMaskEqF32, Compare)
IR_OPCODE(CompareMaskNeF32, Compare)
IR_OPCODE(CompareMaskGtF32, Compare)
IR_OPCODE(CompareMaskGeF32, Compare)
IR_OPCODE(CompareMaskLtF32, Compare)
IR_OPCODE(CompareMaskLeF32, Compare)
IR_OPCODE(CompareMaskUnordEqF32, Compare)
IR_OPCODE(CompareMaskUnordNeF32, Compare)
IR_OPCODE(CompareMaskUnordGtF32, Compare)
IR_OPCODE(CompareMaskUnordGeF32, Compare)
IR_OPCODE(CompareMaskUnordLtF32, Compare)
IR_OPCODE(CompareMaskUnordLeF32, Compare)
IR_OPCODE(ConvertByteU32ToF32, General)
IR_OPCODE(ConvertU32ToF32, General)
IR_OPCODE(ConvertI32ToF32, General)
IR_OPCODE(ConvertF32ToU32, General)
IR_OPCODE(ConvertF32ToI32, General)
IR_OPCODE(ConvertF32ToF16, General)
IR_OPCODE(ConvertF16ToF32, General)
IR_OPCODE(ConvertU16ToF16, General)
IR_OPCODE(ConvertF16ToU16, General)
IR_OPCODE(ConvertI16ToF16, General)
IR_OPCODE(ConvertF16ToI16, General)
IR_OPCODE(ConvertRoundPlusInfF32ToI32, General)
IR_OPCODE(ConvertFloorF32ToI32, General)
IR_OPCODE(ConvertI4ToOffsetF32, General)
IR_OPCODE(LdexpF32, General)
IR_OPCODE(PackF32ToF16Rtz, General)
IR_OPCODE(PackSnorm2x16F32, General)
IR_OPCODE(PackUnorm2x16F32, General)
IR_OPCODE(PackU16U32, General)
IR_OPCODE(PackU8F32, General)
IR_OPCODE(PackB32F16, General)
IR_OPCODE(PackedMadI16, General)
IR_OPCODE(PackedMulLoU16, General)
IR_OPCODE(PackedAddI16, General)
IR_OPCODE(PackedSubI16, General)
IR_OPCODE(PackedLshlrevB16, General)
IR_OPCODE(PackedLshrrevB16, General)
IR_OPCODE(PackedAshrrevI16, General)
IR_OPCODE(PackedMaxI16, General)
IR_OPCODE(PackedMinI16, General)
IR_OPCODE(PackedMadU16, General)
IR_OPCODE(PackedAddU16, General)
IR_OPCODE(PackedSubU16, General)
IR_OPCODE(PackedMaxU16, General)
IR_OPCODE(PackedMinU16, General)
IR_OPCODE(PackedAddF16, General)
IR_OPCODE(PackedMulF16, General)
IR_OPCODE(PackedMinF16, General)
IR_OPCODE(PackedMaxF16, General)
IR_OPCODE(PackedFmaF16, General)
IR_OPCODE(AddF16, General)
IR_OPCODE(SubF16, General)
IR_OPCODE(MulF16, General)
IR_OPCODE(MinF16, General)
IR_OPCODE(MaxF16, General)
IR_OPCODE(FmaF16, General)
IR_OPCODE(MadMixF16, General)
IR_OPCODE(IAddU16, General)
IR_OPCODE(ISubI16, General)
IR_OPCODE(IMinI16, General)
IR_OPCODE(IMaxI16, General)
IR_OPCODE(UMinU16, General)
IR_OPCODE(UMaxU16, General)
IR_OPCODE(RcpF32, General)
IR_OPCODE(FractF32, General)
IR_OPCODE(TruncF32, General)
IR_OPCODE(CeilF32, General)
IR_OPCODE(RoundEvenF32, General)
IR_OPCODE(FloorF32, General)
IR_OPCODE(Exp2F32, General)
IR_OPCODE(Log2F32, General)
IR_OPCODE(InverseSqrtF32, General)
IR_OPCODE(SqrtF32, General)
IR_OPCODE(RcpF16, General)
IR_OPCODE(SqrtF16, General)
IR_OPCODE(InverseSqrtF16, General)
IR_OPCODE(Log2F16, General)
IR_OPCODE(Exp2F16, General)
IR_OPCODE(FloorF16, General)
IR_OPCODE(CeilF16, General)
IR_OPCODE(TruncF16, General)
IR_OPCODE(RoundEvenF16, General)
IR_OPCODE(SinF32, General)
IR_OPCODE(CosF32, General)
IR_OPCODE(CubeIdF32, General)
IR_OPCODE(CubeScF32, General)
IR_OPCODE(CubeTcF32, General)
IR_OPCODE(CubeMaF32, General)
IR_OPCODE(FAddF32, General)
IR_OPCODE(FSubF32, General)
IR_OPCODE(FMulF32, General)
IR_OPCODE(FMinF32, General)
IR_OPCODE(FMaxF32, General)
IR_OPCODE(FMadF32, General)
IR_OPCODE(Dot2AccF32F16, General)
IR_OPCODE(FMin3F32, General)
IR_OPCODE(FMax3F32, General)
IR_OPCODE(FMed3F32, General)
IR_OPCODE(Min3F16, General)
IR_OPCODE(Max3F16, General)
IR_OPCODE(Med3F16, General)
IR_OPCODE(LoadSrtDword, General)
IR_OPCODE(SLoadDword, General)
IR_OPCODE(SBufferLoadDword, General)
IR_OPCODE(BufferLoadUbyte, General)
IR_OPCODE(BufferLoadSbyte, General)
IR_OPCODE(BufferLoadUshort, General)
IR_OPCODE(BufferLoadSshort, General)
IR_OPCODE(BufferLoadDword, General)
IR_OPCODE(BufferStoreByte, General)
IR_OPCODE(BufferStoreShort, General)
IR_OPCODE(BufferStoreDword, General)
IR_OPCODE(AtomicSwapU32, General)
IR_OPCODE(AtomicAddU32, General)
IR_OPCODE(AtomicSubU32, General)
IR_OPCODE(AtomicSMinI32, General)
IR_OPCODE(AtomicUMinU32, General)
IR_OPCODE(AtomicSMaxI32, General)
IR_OPCODE(AtomicUMaxU32, General)
IR_OPCODE(AtomicAndU32, General)
IR_OPCODE(AtomicOrU32, General)
IR_OPCODE(AtomicXorU32, General)
IR_OPCODE(AtomicFMinF32, General)
IR_OPCODE(AtomicFMaxF32, General)
IR_OPCODE(FlatLoadUbyte, General)
IR_OPCODE(FlatLoadSbyte, General)
IR_OPCODE(FlatLoadUshort, General)
IR_OPCODE(FlatLoadSshort, General)
IR_OPCODE(FlatLoadDword, General)
IR_OPCODE(FlatStoreByte, General)
IR_OPCODE(FlatStoreShort, General)
IR_OPCODE(FlatStoreDword, General)
IR_OPCODE(DsReadUbyte, General)
IR_OPCODE(DsReadSbyte, General)
IR_OPCODE(DsReadUshort, General)
IR_OPCODE(DsReadSshort, General)
IR_OPCODE(DsReadB32, General)
IR_OPCODE(DsWriteByte, General)
IR_OPCODE(DsWriteShort, General)
IR_OPCODE(DsWriteB32, General)
IR_OPCODE(DsMinF32, General)
IR_OPCODE(DsMaxF32, General)
IR_OPCODE(DsSwizzleB32, General)
IR_OPCODE(DsConsume, General)
IR_OPCODE(DsAppend, General)
IR_OPCODE(DsWriteAddtidB32, General)
IR_OPCODE(DsReadAddtidB32, General)
IR_OPCODE(ImageGetResinfo, General)
IR_OPCODE(ImageGetLod, General)
IR_OPCODE(ImageLoad, General)
IR_OPCODE(ImageStore, General)
IR_OPCODE(ImageSample, General)
IR_OPCODE(ImageGather4, General)
IR_OPCODE(LoadInputF32, General)
IR_OPCODE(Export, General)
src/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp
-235
View File
@@ -1,235 +0,0 @@
#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h"
#include <algorithm>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
bool HasInput(const ShaderInfo& info, StageInputKind kind, uint32_t location) {
return std::any_of(info.inputs.begin(), info.inputs.end(), [=](const auto& input) {
return input.kind == kind && input.location == location;
});
}
void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components,
std::string name) {
if (!HasInput(info, kind, location)) {
info.inputs.push_back({kind, location, components, std::move(name)});
}
}
bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) {
return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) {
return output.kind == kind && output.index == index;
});
}
void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t location,
std::string name) {
if (!HasOutput(info, kind, index)) {
info.outputs.push_back({kind, index, location, std::move(name)});
}
}
bool NeedsLocalInvocationIndex(const Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::DsWriteAddtidB32 || inst.op == Opcode::DsReadAddtidB32) {
return true;
}
}
}
return false;
}
bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) {
auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
switch (program.stage) {
case ShaderType::Vertex:
if (options.vertex == nullptr) {
return Fail("vertex shader info requires vertex metadata");
}
if (options.vertex->resources_num < 0 ||
options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) {
return Fail("vertex resource count is out of range");
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::LoadInputF32 &&
(inst.input_info.chan >= 4 ||
inst.input_info.attr >=
static_cast<uint32_t>(options.vertex->resources_num))) {
return Fail("vertex input reference is out of range");
}
}
}
return true;
case ShaderType::Pixel:
if (options.pixel == nullptr) {
return Fail("pixel shader info requires pixel metadata");
}
return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) ||
Fail("pixel input count is out of range");
case ShaderType::Compute:
if (options.compute == nullptr) {
return Fail("compute shader info requires compute metadata");
}
return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) ||
Fail("compute thread ID count is out of range");
default: return Fail("unsupported shader stage for info collection");
}
}
void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex,
ShaderInfo& info) {
AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex");
AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex");
if (vertex == nullptr) {
return;
}
uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {};
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::LoadInputF32 &&
inst.input_info.attr < static_cast<uint32_t>(vertex->resources_num) &&
inst.input_info.attr < ShaderVertexInputInfo::RES_MAX) {
used_components[inst.input_info.attr] =
std::max(used_components[inst.input_info.attr], inst.input_info.chan + 1u);
}
}
}
for (uint32_t attr = 0; attr < static_cast<uint32_t>(vertex->resources_num) &&
attr < ShaderVertexInputInfo::RES_MAX;
attr++) {
if (used_components[attr] != 0) {
AddInput(info, StageInputKind::Parameter, attr, used_components[attr],
fmt::format("in_attr_{}", attr));
}
}
}
void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
if (pixel == nullptr) {
return;
}
if (pixel->HasPositionInput()) {
AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord");
}
if (pixel->ps_front_face) {
AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing");
}
for (uint32_t input = 0; input < pixel->input_num; input++) {
AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input));
}
}
void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo* compute,
ShaderInfo& info) {
if (compute != nullptr) {
if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) {
AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID");
}
if (compute->thread_ids_num > 0) {
AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID");
}
if (compute->thread_ids_num > 0 || compute->tg_size_en) {
AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex");
}
if (compute->dispatch_thread_dimensions) {
AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID");
}
}
if (NeedsLocalInvocationIndex(program)) {
AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex");
}
}
void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != Opcode::Export) {
continue;
}
if (inst.export_info.kind == ExportTargetKind::MrtZ) {
if (pixel != nullptr && (inst.export_info.en & 0x1u) != 0 &&
pixel->ps_depth_export_enable) {
AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth");
}
if (pixel != nullptr && (inst.export_info.en & 0x4u) != 0 &&
pixel->ps_sample_mask_export_enable) {
AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask");
}
continue;
}
if (inst.export_info.en == 0) {
continue;
}
switch (inst.export_info.kind) {
case ExportTargetKind::Position:
AddOutput(info, StageOutputKind::Position, inst.export_info.index, 0,
"out_position");
break;
case ExportTargetKind::Parameter:
AddOutput(info, StageOutputKind::Parameter, inst.export_info.index,
inst.export_info.index,
fmt::format("out_param_{}", inst.export_info.index));
break;
case ExportTargetKind::Mrt:
AddOutput(info, StageOutputKind::Mrt, inst.export_info.index,
inst.export_info.index,
fmt::format("out_mrt_{}", inst.export_info.index));
break;
default: break;
}
}
}
}
} // namespace
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) {
if (!program.resource_tracking_complete || program.shader_info_complete) {
if (error != nullptr) {
*error = !program.resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected";
}
return false;
}
if (!ValidateOptions(program, options, error)) {
return false;
}
auto next = program.info;
next.inputs.clear();
next.outputs.clear();
next.has_bitwise_xor =
std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) {
return std::any_of(block.instructions.begin(), block.instructions.end(),
[](const auto& inst) {
switch (inst.op) {
case Opcode::BitwiseXorU32:
case Opcode::BitwiseXor3U32:
case Opcode::XorAddU32: return true;
default: return false;
}
});
});
switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break;
case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(program, options.compute, next); break;
default: return false;
}
CollectOutputs(program, options.pixel, next);
program.info = std::move(next);
program.shader_info_complete = true;
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/SrtPatcher.cpp
-138
View File
@@ -1,138 +0,0 @@
#include "graphics/shader/recompiler/ir/SrtPatcher.h"
#include <fmt/format.h>
#include <functional>
#include <map>
#include <set>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
bool PatchSrtReads(Program& program, std::string* error) {
if (!program.srt_plan_complete || program.srt_patching_complete ||
program.resource_tracking_complete) {
if (error != nullptr) {
if (!program.srt_plan_complete) {
*error = "SRT plan is not ready";
} else if (program.srt_patching_complete) {
*error = "SRT reads already patched";
} else {
*error = "resources already tracked";
}
}
return false;
}
std::map<uint32_t, uint32_t> offsets;
std::map<uint32_t, uint32_t> values;
for (const auto& read: program.srt.reads) {
if (read.value >= program.provenance.values.size() ||
read.flat_offset >= program.srt.reads.size() ||
(program.provenance.values[read.value].op != ScalarValueOp::ReadConst &&
program.provenance.values[read.value].op != ScalarValueOp::ReadConstBuffer)) {
if (error != nullptr) {
*error = fmt::format("invalid SRT read value={} flat_offset={}", read.value,
read.flat_offset);
}
return false;
}
const auto [it, inserted] = offsets.emplace(read.value, read.flat_offset);
if (!inserted && it->second != read.flat_offset) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has conflicting flat offsets", read.value);
}
return false;
}
const auto [value, unique] = values.emplace(read.flat_offset, read.value);
if (!unique && value->second != read.value) {
if (error != nullptr) {
*error = fmt::format("SRT flat offset {} has conflicting values", read.flat_offset);
}
return false;
}
}
if (offsets.size() != program.srt.reads.size() || values.size() != program.srt.reads.size()) {
if (error != nullptr) {
*error = "SRT reads do not form a dense value-to-offset bijection";
}
return false;
}
std::set<uint32_t> dynamic_reads;
for (const auto value: program.srt.dynamic_reads) {
if (value >= program.provenance.values.size() || offsets.contains(value) ||
(program.provenance.values[value].op != ScalarValueOp::ReadConst &&
program.provenance.values[value].op != ScalarValueOp::ReadConstBuffer) ||
!dynamic_reads.insert(value).second) {
if (error != nullptr) {
*error = fmt::format("invalid dynamic SRT read value={}", value);
}
return false;
}
}
std::set<uint32_t> dynamic_sources;
for (const auto source: program.srt.dynamic_sources) {
if (source == ScalarProvenance::Undefined ||
(source > ScalarProvenance::Unknown &&
source - 2u >= program.provenance.descriptors.size()) ||
!dynamic_sources.insert(source).second) {
if (error != nullptr) {
*error = fmt::format("invalid dynamic descriptor source={}", source);
}
return false;
}
}
struct Patch {
std::reference_wrapper<Instruction> inst;
uint32_t flat_offset = 0;
};
std::vector<Patch> patches;
std::map<uint32_t, bool> matched;
for (const auto& [value, offset]: offsets) {
matched.emplace(value, false);
}
for (auto& block: program.blocks) {
for (auto& inst: block.instructions) {
if (inst.op != Opcode::SLoadDword && inst.op != Opcode::SBufferLoadDword) {
continue;
}
if (const auto found = offsets.find(inst.scalar_value); found != offsets.end()) {
const auto value_op = program.provenance.values[found->first].op;
if ((inst.op == Opcode::SLoadDword) != (value_op == ScalarValueOp::ReadConst)) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has the wrong scalar-load producer",
found->first);
}
return false;
}
patches.push_back({std::ref(inst), found->second});
matched[found->first] = true;
}
}
}
for (const auto& [value, found]: matched) {
if (!found) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has no scalar-load producer", value);
}
return false;
}
}
for (const auto& patch: patches) {
auto& inst = patch.inst.get();
inst.op = Opcode::LoadSrtDword;
inst.src_count = 1;
for (auto& src: inst.src) {
src = {};
}
inst.src[0].kind = OperandKind::ImmediateU32;
inst.src[0].imm = patch.flat_offset;
inst.memory = {};
}
program.srt_patching_complete = true;
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/SrtPatcher.h
-14
View File
@@ -1,14 +0,0 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
// Replaces each immediate ReadConst producer selected by the SRT plan with a dense flat-buffer
// dword load. Dynamic-offset reads remain explicit. On failure no instruction is changed.
bool PatchSrtReads(Program& program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ */
src/graphics/shader/recompiler/ir/SrtWalker.cpp
-624
View File
@@ -1,624 +0,0 @@
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
#include <algorithm>
#include <array>
#include <cstring>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) {
return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
StageName(program.stage), pc, message);
}
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
if (base > AddressMask) {
return false;
}
if (offset < 0) {
const auto magnitude = uint64_t {0} - static_cast<uint64_t>(offset);
if (magnitude > base) {
return false;
}
result = base - magnitude;
return true;
}
const auto magnitude = static_cast<uint64_t>(offset);
if (magnitude > AddressMask - base) {
return false;
}
result = base + magnitude;
return true;
}
bool ApplyOperation(ScalarValueOp op, const std::array<uint32_t, 3>& args, uint32_t& result) {
const auto shift = args[1] & 31u;
switch (op) {
case ScalarValueOp::Add: result = args[0] + args[1]; break;
case ScalarValueOp::AddCarry: result = args[0] + args[1] + (args[2] & 1u); break;
case ScalarValueOp::Carry:
result = static_cast<uint32_t>(
(static_cast<uint64_t>(args[0]) + args[1] + (args[2] & 1u)) >> 32u);
break;
case ScalarValueOp::Sub: result = args[0] - args[1]; break;
case ScalarValueOp::SubBorrow: result = args[0] - args[1] - (args[2] & 1u); break;
case ScalarValueOp::Borrow:
result = static_cast<uint64_t>(args[1]) + (args[2] & 1u) > args[0] ? 1u : 0u;
break;
case ScalarValueOp::Mul: result = args[0] * args[1]; break;
case ScalarValueOp::And: result = args[0] & args[1]; break;
case ScalarValueOp::AndNot: result = args[0] & ~args[1]; break;
case ScalarValueOp::Or: result = args[0] | args[1]; break;
case ScalarValueOp::OrNot: result = args[0] | ~args[1]; break;
case ScalarValueOp::Xor: result = args[0] ^ args[1]; break;
case ScalarValueOp::Not: result = ~args[0]; break;
case ScalarValueOp::ShiftLeft: result = args[0] << shift; break;
case ScalarValueOp::ShiftRight: result = args[0] >> shift; break;
case ScalarValueOp::ShiftRightArithmetic:
result = static_cast<uint32_t>(static_cast<int32_t>(args[0]) >> shift);
break;
case ScalarValueOp::BitFieldMaskU32: {
const auto count = args[0] & 31u;
const auto offset = args[1] & 31u;
result = ((uint32_t {1} << count) - 1u) << offset;
break;
}
case ScalarValueOp::BitFieldMaskU64Low:
case ScalarValueOp::BitFieldMaskU64High: {
const auto count = args[0] & 63u;
const auto value = ((uint64_t {1} << count) - 1u) << (args[1] & 63u);
result = op == ScalarValueOp::BitFieldMaskU64Low ? static_cast<uint32_t>(value)
: static_cast<uint32_t>(value >> 32u);
break;
}
case ScalarValueOp::Add3: result = args[0] + args[1] + args[2]; break;
case ScalarValueOp::ShiftLeftAdd: result = (args[0] << shift) + args[2]; break;
case ScalarValueOp::ShiftLeftAddCarry:
result =
static_cast<uint32_t>(((static_cast<uint64_t>(args[0]) << shift) + args[2]) >> 32u);
break;
case ScalarValueOp::AddShiftLeft: result = (args[0] + args[1]) << (args[2] & 31u); break;
case ScalarValueOp::XorAdd: result = (args[0] ^ args[1]) + args[2]; break;
case ScalarValueOp::ShiftLeftOr: result = (args[0] << shift) | args[2]; break;
default: return false;
}
return true;
}
class ConstantFolder {
public:
explicit ConstantFolder(const ScalarProvenance& provenance)
: m_provenance(provenance), m_values(provenance.values.size()),
m_state(provenance.values.size()) {}
bool Fold(uint32_t id, uint32_t& result) {
if (id >= m_provenance.values.size() || m_state[id] == 1 || m_state[id] == 3) {
return false;
}
if (m_state[id] == 2) {
result = m_values[id];
return true;
}
m_state[id] = 1;
const auto& value = m_provenance.values[id];
uint32_t out = 0;
switch (value.op) {
case ScalarValueOp::Constant: out = value.imm; break;
case ScalarValueOp::Phi:
if (value.phi_args.empty() || !Fold(value.phi_args[0], out)) {
return Dynamic(id);
}
for (size_t i = 1; i < value.phi_args.size(); i++) {
uint32_t other = 0;
if (!Fold(value.phi_args[i], other) || other != out) {
return Dynamic(id);
}
}
break;
default: {
const auto count = ScalarValueArgCount(value.op);
if (count == 0 || count > 3 || value.op == ScalarValueOp::ReadConst) {
return Dynamic(id);
}
std::array<uint32_t, 3> args {};
for (uint32_t i = 0; i < count; i++) {
if (!Fold(value.args[i], args[i])) {
return Dynamic(id);
}
}
if (!ApplyOperation(value.op, args, out)) {
return Dynamic(id);
}
break;
}
}
m_state[id] = 2;
m_values[id] = out;
result = out;
return true;
}
private:
bool Dynamic(uint32_t id) {
m_state[id] = 3;
return false;
}
const ScalarProvenance& m_provenance;
std::vector<uint32_t> m_values;
std::vector<uint8_t> m_state;
};
class PlanBuilder {
public:
explicit PlanBuilder(Program& program): m_program(program), m_folder(program.provenance) {
m_state.resize(program.provenance.values.size());
}
bool Run(std::string* error) {
m_program.srt = {};
for (const auto& block: m_program.blocks) {
for (const auto& inst: block.instructions) {
if (!CollectDescriptor(inst.memory.resource_source, inst.pc, error) ||
!CollectDescriptor(inst.memory.sampler_source, inst.pc, error)) {
return false;
}
}
}
for (const auto& block: m_program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::SLoadDword && inst.scalar_value < m_state.size() &&
m_state[inst.scalar_value] == 0) {
const auto& value = m_program.provenance.values[inst.scalar_value];
uint32_t ignored = 0;
if (value.op == ScalarValueOp::ReadConst &&
m_folder.Fold(value.args[2], ignored) &&
!CollectValue(inst.scalar_value, inst.pc, error)) {
return false;
}
}
}
}
return true;
}
private:
bool Fail(uint32_t pc, std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, pc, message);
}
return false;
}
bool CollectDescriptor(uint32_t source, uint32_t use_pc, std::string* error) {
const auto* descriptor = GetDescriptorSource(m_program, source);
if (descriptor == nullptr) {
if (source <= ScalarProvenance::Unknown) {
return source == ScalarProvenance::Undefined || AddDynamicSource(source);
}
return Fail(use_pc, error, fmt::format("invalid descriptor source {}", source));
}
if (!DescriptorSourceResolved(m_program, source) ||
DescriptorNeedsControlFlow(*descriptor)) {
MarkDescriptor(*descriptor);
return AddDynamicSource(source);
}
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!CollectValue(descriptor->dwords[i], use_pc, error)) {
return false;
}
}
return true;
}
void MarkDescriptor(const DescriptorValue& descriptor) {
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
MarkValue(descriptor.dwords[i]);
}
}
void MarkValue(uint32_t id) {
if (id >= m_state.size() || m_state[id] != 0) {
return;
}
m_state[id] = 2;
const auto& value = m_program.provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
MarkValue(arg);
}
return;
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
MarkValue(value.args[i]);
}
}
bool DescriptorNeedsControlFlow(const DescriptorValue& descriptor) {
std::vector<uint8_t> visited(m_program.provenance.values.size());
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (ValueNeedsControlFlow(descriptor.dwords[i], visited)) {
return true;
}
}
return false;
}
bool ValueNeedsControlFlow(uint32_t id, std::vector<uint8_t>& visited) {
if (id >= m_program.provenance.values.size() || visited[id] != 0) {
return false;
}
visited[id] = 1;
const auto& value = m_program.provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
uint32_t ignored = 0;
return !m_folder.Fold(id, ignored);
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (ValueNeedsControlFlow(value.args[i], visited)) {
return true;
}
}
return false;
}
bool AddDynamicSource(uint32_t source) {
if (std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(),
source) == m_program.srt.dynamic_sources.end()) {
m_program.srt.dynamic_sources.push_back(source);
}
return true;
}
bool CollectValue(uint32_t id, uint32_t use_pc, std::string* error) {
if (id >= m_program.provenance.values.size()) {
return Fail(use_pc, error, fmt::format("invalid scalar value {}", id));
}
if (m_state[id] == 2) {
return true;
}
if (m_state[id] == 1) {
return Fail(use_pc, error, fmt::format("cyclic scalar value {}", id));
}
m_state[id] = 1;
const auto& value = m_program.provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (!CollectValue(arg, use_pc, error)) {
return false;
}
}
} else {
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (!CollectValue(value.args[i], use_pc, error)) {
return false;
}
}
}
if (value.op == ScalarValueOp::ReadConst || value.op == ScalarValueOp::ReadConstBuffer) {
const auto offset_arg = value.op == ScalarValueOp::ReadConst ? 2u : 4u;
uint32_t ignored = 0;
if (m_folder.Fold(value.args[offset_arg], ignored)) {
m_program.srt.reads.push_back(
{id, static_cast<uint32_t>(m_program.srt.reads.size()), use_pc});
} else {
m_program.srt.dynamic_reads.push_back(id);
}
}
m_state[id] = 2;
return true;
}
Program& m_program;
ConstantFolder m_folder;
std::vector<uint8_t> m_state;
};
class Evaluator {
public:
Evaluator(const Program& program, const SrtRuntime& runtime, uint32_t use_pc)
: m_program(program), m_runtime(runtime), m_use_pc(use_pc),
m_values(program.provenance.values.size()), m_state(program.provenance.values.size()) {}
void SetUsePc(uint32_t use_pc) { m_use_pc = use_pc; }
bool Evaluate(uint32_t id, uint32_t& result, std::string* error) {
if (id >= m_program.provenance.values.size()) {
return Fail(error, fmt::format("invalid scalar value {}", id));
}
if (m_state[id] == 2) {
result = m_values[id];
return true;
}
if (m_state[id] == 1) {
return Fail(error, fmt::format("cyclic scalar value {}", id));
}
m_state[id] = 1;
const auto& value = m_program.provenance.values[id];
uint32_t out = 0;
switch (value.op) {
case ScalarValueOp::UserData:
if (value.imm < m_program.user_data_base ||
value.imm - m_program.user_data_base >= m_runtime.user_data.size()) {
return Fail(error, fmt::format("user SGPR {} is unavailable", value.imm));
}
out = m_runtime.user_data[value.imm - m_program.user_data_base];
break;
case ScalarValueOp::Constant: out = value.imm; break;
case ScalarValueOp::PcRelativeLow:
out = static_cast<uint32_t>(m_runtime.shader_base + value.imm);
break;
case ScalarValueOp::PcRelativeHigh:
out = static_cast<uint32_t>((m_runtime.shader_base + value.imm) >> 32u);
break;
case ScalarValueOp::Phi: {
const auto first = std::find_if(value.phi_args.begin(), value.phi_args.end(),
[id](uint32_t arg) { return arg != id; });
if (first == value.phi_args.end()) {
return Fail(error, fmt::format("empty scalar phi {}", id));
}
if (!Evaluate(*first, out, error)) {
return false;
}
for (const auto arg: value.phi_args) {
if (arg == id || arg == *first) {
continue;
}
uint32_t other = 0;
if (!Evaluate(arg, other, error)) {
return false;
}
if (other != out) {
return Fail(error,
fmt::format("scalar phi {} has runtime-dependent values", id));
}
}
break;
}
case ScalarValueOp::ReadConst:
case ScalarValueOp::ReadConstBuffer:
if (!Read(value, out, error)) {
return false;
}
break;
case ScalarValueOp::Undefined:
case ScalarValueOp::Unknown:
return Fail(error, fmt::format("scalar value {} is unresolved", id));
default:
if (!EvaluateOperation(value, out, error)) {
return false;
}
break;
}
m_state[id] = 2;
m_values[id] = out;
result = out;
return true;
}
private:
bool Fail(std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, m_use_pc, message);
}
return false;
}
bool EvaluateOperation(const ScalarValue& value, uint32_t& result, std::string* error) {
const auto count = ScalarValueArgCount(value.op);
if (count == 0 || count > 3) {
return Fail(error, fmt::format("unsupported scalar operation {}",
static_cast<uint32_t>(value.op)));
}
std::array<uint32_t, 3> args {};
for (uint32_t i = 0; i < count; i++) {
if (!Evaluate(value.args[i], args[i], error)) {
return false;
}
}
return ApplyOperation(value.op, args, result) ||
Fail(error, fmt::format("unsupported scalar operation {}",
static_cast<uint32_t>(value.op)));
}
bool Read(const ScalarValue& value, uint32_t& result, std::string* error) {
const bool buffer = value.op == ScalarValueOp::ReadConstBuffer;
uint32_t lo = 0;
uint32_t hi = 0;
uint32_t offset = 0;
if (!Evaluate(value.args[0], lo, error) || !Evaluate(value.args[1], hi, error) ||
!Evaluate(value.args[buffer ? 4u : 2u], offset, error)) {
return false;
}
const auto base = (static_cast<uint64_t>(hi) << 32u | lo) & AddressMask;
const auto immediate = static_cast<int64_t>(static_cast<int32_t>(value.imm));
uint64_t address = 0;
if (buffer) {
uint32_t num_records = 0;
uint32_t ignored = 0;
if (!Evaluate(value.args[2], num_records, error) ||
!Evaluate(value.args[3], ignored, error)) {
return false;
}
if (immediate < 0) {
return Fail(error,
fmt::format("ReadConstBuffer pc=0x{:08x} has negative immediate {}",
value.pc, immediate));
}
const auto byte_offset = static_cast<uint64_t>(immediate) + offset;
const auto aligned_offset = byte_offset & ~uint64_t {3};
const auto stride = (hi >> 16u) & 0x3fffu;
const auto size = stride == 0 ? static_cast<uint64_t>(num_records)
: static_cast<uint64_t>(stride) * num_records;
if (aligned_offset > size || size - aligned_offset < sizeof(uint32_t)) {
return Fail(error,
fmt::format("ReadConstBuffer pc=0x{:08x} offset={} exceeds size={}",
value.pc, aligned_offset, size));
}
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
} else {
const auto base_aligned = base & ~uint64_t {3};
const auto immediate_aligned = immediate & ~int64_t {3};
const auto relative = immediate_aligned + static_cast<int64_t>(offset & ~uint32_t {3});
if (!AddSignedAddress(base_aligned, relative, address)) {
return Fail(error,
fmt::format("ReadConst pc=0x{:08x} address base=0x{:016x} offset={} "
"is outside the 48-bit address space",
value.pc, base_aligned, relative));
}
}
if (m_runtime.read_memory != nullptr &&
!m_runtime.read_memory(m_runtime.userdata, address, &result)) {
return Fail(
error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address));
}
if (m_runtime.read_memory == nullptr) {
std::memcpy(&result, reinterpret_cast<const void*>(address), sizeof(result));
}
return true;
}
const Program& m_program;
const SrtRuntime& m_runtime;
uint32_t m_use_pc;
std::vector<uint32_t> m_values;
std::vector<uint8_t> m_state;
};
} // namespace
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t& result) {
ConstantFolder folder(provenance);
return folder.Fold(value, result);
}
bool BuildSrtPlan(Program& program, std::string* error) {
if (program.resource_tracking_complete) {
if (error != nullptr) {
*error = "cannot rebuild SRT plan after resource tracking";
}
return false;
}
if (program.srt_patching_complete) {
if (error != nullptr) {
*error = "cannot rebuild SRT plan after SRT patching";
}
return false;
}
program.srt_plan_complete = false;
if (!PlanBuilder(program).Run(error)) {
return false;
}
program.srt_plan_complete = true;
return true;
}
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue& result,
std::string* error) {
const DescriptorSourceRequest request {source, use_pc};
std::vector<DescriptorValue> results;
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) {
return false;
}
result = results[0];
return true;
}
static bool EvaluateRuntimeSourcesImpl(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime,
std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, bool evaluate_flat,
std::string* error) {
if (!program.srt_plan_complete) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "SRT plan is not ready");
}
return false;
}
std::vector<DescriptorValue> evaluated;
evaluated.reserve(requests.size());
std::vector<uint32_t> flattened(evaluate_flat ? program.srt.reads.size() : 0u);
Evaluator evaluator(program, runtime, 0);
for (const auto& request: requests) {
const auto* descriptor = GetDescriptorSource(program, request.source);
const auto dynamic =
std::find(program.srt.dynamic_sources.begin(), program.srt.dynamic_sources.end(),
request.source) != program.srt.dynamic_sources.end();
if (descriptor == nullptr || dynamic) {
if (error != nullptr) {
*error = Diagnostic(program, request.use_pc,
fmt::format("descriptor source {} is {}", request.source,
dynamic ? "GPU-dynamic" : "unresolved"));
}
return false;
}
evaluator.SetUsePc(request.use_pc);
auto value = *descriptor;
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!evaluator.Evaluate(descriptor->dwords[i], value.dwords[i], error)) {
return false;
}
}
evaluated.push_back(value);
}
if (evaluate_flat) {
for (const auto& read: program.srt.reads) {
evaluator.SetUsePc(read.use_pc);
if (read.flat_offset >= flattened.size()) {
if (error != nullptr) {
*error = Diagnostic(program, read.use_pc, "invalid flat SRT offset");
}
return false;
}
if (!evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) {
return false;
}
}
}
results = std::move(evaluated);
if (evaluate_flat) {
flat = std::move(flattened);
}
return true;
}
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::string* error) {
std::vector<uint32_t> ignored;
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error);
}
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::string* error) {
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error);
}
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
std::string* error) {
std::vector<DescriptorValue> ignored;
return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/Type.cpp
+55
View File
@@ -0,0 +1,55 @@
#include "graphics/shader/recompiler/ir/Type.h"
#include <array>
namespace Libs::Graphics::ShaderRecompiler::IR {
bool AreTypesCompatible(Type lhs, Type rhs) {
return lhs == rhs || lhs == Type::Opaque || rhs == Type::Opaque || TypesOverlap(lhs, rhs);
}
std::string TypeName(Type type) {
static constexpr std::array names = {
"Opaque",
"ScalarReg",
"VectorReg",
"U1",
"U8",
"U16",
"U32",
"U64",
"F16",
"F32",
"U32x2",
"U32x3",
"U32x4",
"F32x2",
"F32x3",
"F32x4",
"SrtResource",
"BufferResource",
"AddressResource",
"ImageResource",
"SamplerResource",
"LdsResource",
"GdsResource",
"ImageAddress",
};
const auto bits = static_cast<uint32_t>(type);
if (bits == 0) {
return "Void";
}
std::string result;
for (uint32_t index = 0; index < names.size(); index++) {
if ((bits & (1u << index)) == 0) {
continue;
}
if (!result.empty()) {
result += '|';
}
result += names[index];
}
return result;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/Type.h
+51
View File
@@ -0,0 +1,51 @@
#pragma once
#include <cstdint>
#include <string>
namespace Libs::Graphics::ShaderRecompiler::IR {
enum class Type : uint32_t {
Void = 0,
Opaque = 1u << 0u,
ScalarReg = 1u << 1u,
VectorReg = 1u << 2u,
U1 = 1u << 3u,
U8 = 1u << 4u,
U16 = 1u << 5u,
U32 = 1u << 6u,
U64 = 1u << 7u,
F16 = 1u << 8u,
F32 = 1u << 9u,
U32x2 = 1u << 10u,
U32x3 = 1u << 11u,
U32x4 = 1u << 12u,
F32x2 = 1u << 13u,
F32x3 = 1u << 14u,
F32x4 = 1u << 15u,
SrtResource = 1u << 16u,
BufferResource = 1u << 17u,
AddressResource = 1u << 18u,
ImageResource = 1u << 19u,
SamplerResource = 1u << 20u,
LdsResource = 1u << 21u,
GdsResource = 1u << 22u,
ImageAddress = 1u << 23u,
};
constexpr Type operator|(Type lhs, Type rhs) {
return static_cast<Type>(static_cast<uint32_t>(lhs) | static_cast<uint32_t>(rhs));
}
constexpr Type operator&(Type lhs, Type rhs) {
return static_cast<Type>(static_cast<uint32_t>(lhs) & static_cast<uint32_t>(rhs));
}
constexpr bool TypesOverlap(Type lhs, Type rhs) {
return (lhs & rhs) != Type::Void;
}
[[nodiscard]] bool AreTypesCompatible(Type lhs, Type rhs);
[[nodiscard]] std::string TypeName(Type type);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/Value.cpp
+277
View File
@@ -0,0 +1,277 @@
#include "graphics/shader/recompiler/ir/Value.h"
#include <algorithm>
#include <cstring>
#include <limits>
namespace Libs::Graphics::ShaderRecompiler::IR {
Value::Value(Inst* value): type(Type::Opaque), inst(value) {}
Value::Value(ScalarReg value): type(Type::ScalarReg), scalar_reg(value) {}
Value::Value(VectorReg value): type(Type::VectorReg), vector_reg(value) {}
Value::Value(bool value): type(Type::U1), imm_u1(value) {}
Value::Value(uint8_t value): type(Type::U8), imm_u8(value) {}
Value::Value(uint16_t value): type(Type::U16), imm_u16(value) {}
Value::Value(uint32_t value): type(Type::U32), imm_u32(value) {}
Value::Value(uint64_t value): type(Type::U64), imm_u64(value) {}
Value::Value(Type value_type, uint64_t bits): type(value_type), imm_u64(bits) {}
Value Value::F16(uint16_t bits) {
return Value(Type::F16, bits);
}
Value Value::F32(float value) {
return Value(Type::F32, std::bit_cast<uint32_t>(value));
}
bool Value::IsEmpty() const {
return type == Type::Void;
}
bool Value::IsImmediate() const {
return type != Type::Opaque;
}
bool Value::IsIdentity() const {
return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Identity;
}
bool Value::IsPhi() const {
return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Phi;
}
Type Value::GetType() const {
if (IsPhi()) {
return inst->Flags<Type>();
}
if (IsIdentity()) {
return inst->Arg(0).GetType();
}
return type == Type::Opaque ? inst->GetType() : type;
}
Inst* Value::Instruction() const {
EXIT_IF(type != Type::Opaque);
return inst;
}
Inst* Value::TryInstruction() const {
return type == Type::Opaque ? inst : nullptr;
}
Inst* Value::ResolveInstruction() const {
EXIT_IF(type != Type::Opaque);
return IsIdentity() ? inst->Arg(0).ResolveInstruction() : inst;
}
Value Value::Resolve() const {
return IsIdentity() ? inst->Arg(0).Resolve() : *this;
}
ScalarReg Value::ScalarRegister() const {
EXIT_IF(type != Type::ScalarReg);
return scalar_reg;
}
VectorReg Value::VectorRegister() const {
EXIT_IF(type != Type::VectorReg);
return vector_reg;
}
bool Value::U1() const {
EXIT_IF(type != Type::U1);
return imm_u1;
}
uint8_t Value::U8() const {
EXIT_IF(type != Type::U8);
return imm_u8;
}
uint16_t Value::U16() const {
EXIT_IF(type != Type::U16);
return imm_u16;
}
uint32_t Value::U32() const {
EXIT_IF(type != Type::U32);
return imm_u32;
}
uint64_t Value::U64() const {
EXIT_IF(type != Type::U64);
return imm_u64;
}
uint16_t Value::F16Bits() const {
EXIT_IF(type != Type::F16);
return imm_u16;
}
float Value::F32Value() const {
EXIT_IF(type != Type::F32);
return std::bit_cast<float>(imm_u32);
}
bool Value::operator==(const Value& other) const {
if (type != other.type) {
return false;
}
switch (type) {
case Type::Void: return true;
case Type::Opaque: return inst == other.inst;
case Type::ScalarReg: return scalar_reg == other.scalar_reg;
case Type::VectorReg: return vector_reg == other.vector_reg;
case Type::U1: return imm_u1 == other.imm_u1;
case Type::U8: return imm_u8 == other.imm_u8;
case Type::U16:
case Type::F16: return imm_u16 == other.imm_u16;
case Type::U32:
case Type::F32: return imm_u32 == other.imm_u32;
case Type::U64: return imm_u64 == other.imm_u64;
default: return false;
}
}
Inst::Inst(ValueOpcode value_opcode, uint64_t value_flags)
: opcode(value_opcode), flags(value_flags) {
const auto count = NumArgsOf(opcode);
if (count != std::numeric_limits<size_t>::max()) {
args.resize(count);
}
}
Inst::~Inst() {
ClearArgs();
}
ValueOpcode Inst::GetOpcode() const {
return opcode;
}
Type Inst::GetType() const {
if (opcode == ValueOpcode::Phi) {
return static_cast<Type>(flags);
}
if (opcode == ValueOpcode::Identity && !args.empty()) {
return args.front().GetType();
}
return TypeOf(opcode);
}
bool Inst::MayHaveSideEffects() const {
return HasSideEffects(opcode);
}
bool Inst::HasUses() const {
return !uses.empty();
}
size_t Inst::UseCount() const {
return uses.size();
}
size_t Inst::NumArgs() const {
return args.size();
}
Value Inst::Arg(size_t index) const {
EXIT_IF(index >= args.size());
return args[index];
}
Block* Inst::PhiBlock(size_t index) const {
EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_blocks.size());
return phi_blocks[index];
}
Block* Inst::Parent() const {
return parent;
}
const std::vector<Use>& Inst::Uses() const {
return uses;
}
void Inst::SetParent(Block* block) {
parent = block;
}
void Inst::SetArg(size_t index, Value value) {
if (index >= args.size()) {
EXIT_IF(NumArgsOf(opcode) != std::numeric_limits<size_t>::max());
args.resize(index + 1);
}
const auto old = args[index];
if (auto* old_inst = old.TryInstruction(); old_inst != nullptr) {
RemoveUse(old_inst, index);
}
args[index] = value;
if (auto* new_inst = value.TryInstruction(); new_inst != nullptr) {
AddUse(new_inst, index);
}
}
void Inst::AddPhiOperand(Block* predecessor, Value value) {
EXIT_IF(opcode != ValueOpcode::Phi);
const auto index = args.size();
args.push_back(value);
phi_blocks.push_back(predecessor);
if (auto* value_inst = value.TryInstruction(); value_inst != nullptr) {
AddUse(value_inst, index);
}
}
void Inst::ReplaceUsesWith(Value replacement, bool preserve) {
const auto old_uses = uses;
for (const auto& use: old_uses) {
use.user->SetArg(use.operand, replacement);
}
Invalidate();
if (preserve) {
ReplaceOpcode(ValueOpcode::Identity);
args.resize(1);
SetArg(0, replacement);
}
}
void Inst::ReplaceOpcode(ValueOpcode value_opcode) {
opcode = value_opcode;
const auto count = NumArgsOf(opcode);
if (count != std::numeric_limits<size_t>::max()) {
EXIT_IF(!args.empty() && args.size() != count);
args.resize(count);
}
}
void Inst::Invalidate() {
ClearArgs();
opcode = ValueOpcode::Void;
}
void Inst::AddUse(Inst* used, size_t operand) {
const auto found = std::ranges::find_if(
used->uses, [&](const Use& use) { return use.user == this && use.operand == operand; });
EXIT_IF(found != used->uses.end());
used->uses.push_back({this, operand});
}
void Inst::RemoveUse(Inst* used, size_t operand) {
const auto found = std::ranges::find_if(
used->uses, [&](const Use& use) { return use.user == this && use.operand == operand; });
EXIT_IF(found == used->uses.end());
used->uses.erase(found);
}
void Inst::ClearArgs() {
for (size_t index = 0; index < args.size(); index++) {
if (auto* value_inst = args[index].TryInstruction(); value_inst != nullptr) {
RemoveUse(value_inst, index);
}
}
args.clear();
phi_blocks.clear();
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/Value.h
+159
View File
@@ -0,0 +1,159 @@
#pragma once
#include "common/assert.h"
#include "graphics/shader/recompiler/ir/Reg.h"
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h"
#include <bit>
#include <cstdint>
#include <cstring>
#include <type_traits>
#include <utility>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
class Block;
class Inst;
class Value {
public:
Value() = default;
explicit Value(Inst* value);
explicit Value(ScalarReg value);
explicit Value(VectorReg value);
explicit Value(bool value);
explicit Value(uint8_t value);
explicit Value(uint16_t value);
explicit Value(uint32_t value);
explicit Value(uint64_t value);
static Value F16(uint16_t bits);
static Value F32(float value);
[[nodiscard]] bool IsEmpty() const;
[[nodiscard]] bool IsImmediate() const;
[[nodiscard]] bool IsIdentity() const;
[[nodiscard]] bool IsPhi() const;
[[nodiscard]] Type GetType() const;
[[nodiscard]] Inst* Instruction() const;
[[nodiscard]] Inst* TryInstruction() const;
[[nodiscard]] Inst* ResolveInstruction() const;
[[nodiscard]] Value Resolve() const;
[[nodiscard]] ScalarReg ScalarRegister() const;
[[nodiscard]] VectorReg VectorRegister() const;
[[nodiscard]] bool U1() const;
[[nodiscard]] uint8_t U8() const;
[[nodiscard]] uint16_t U16() const;
[[nodiscard]] uint32_t U32() const;
[[nodiscard]] uint64_t U64() const;
[[nodiscard]] uint16_t F16Bits() const;
[[nodiscard]] float F32Value() const;
bool operator==(const Value& other) const;
private:
Type type = Type::Void;
union {
Inst* inst;
ScalarReg scalar_reg;
VectorReg vector_reg;
bool imm_u1;
uint8_t imm_u8;
uint16_t imm_u16;
uint32_t imm_u32;
uint64_t imm_u64;
};
explicit Value(Type type, uint64_t bits);
};
static_assert(std::is_trivially_copyable_v<Value>);
template <Type type_>
class TypedValue: public Value {
public:
TypedValue() = default;
template <Type other_type>
requires(TypesOverlap(type_, other_type))
TypedValue(const TypedValue<other_type>& value): Value(value) {}
explicit TypedValue(Value value): Value(value) {
EXIT_IF(!AreTypesCompatible(value.GetType(), type_));
}
};
using U1 = TypedValue<Type::U1>;
using U8 = TypedValue<Type::U8>;
using U16 = TypedValue<Type::U16>;
using U32 = TypedValue<Type::U32>;
using U64 = TypedValue<Type::U64>;
using F16 = TypedValue<Type::F16>;
using F32 = TypedValue<Type::F32>;
using U32F32 = TypedValue<Type::U32 | Type::F32>;
struct Use {
Inst* user = nullptr;
size_t operand = 0;
bool operator==(const Use&) const = default;
};
class Inst {
public:
explicit Inst(ValueOpcode opcode, uint64_t flags = 0);
~Inst();
Inst(const Inst&) = delete;
Inst& operator=(const Inst&) = delete;
Inst(Inst&&) = delete;
Inst& operator=(Inst&&) = delete;
[[nodiscard]] ValueOpcode GetOpcode() const;
[[nodiscard]] Type GetType() const;
[[nodiscard]] bool MayHaveSideEffects() const;
[[nodiscard]] bool HasUses() const;
[[nodiscard]] size_t UseCount() const;
[[nodiscard]] size_t NumArgs() const;
[[nodiscard]] Value Arg(size_t index) const;
[[nodiscard]] Block* PhiBlock(size_t index) const;
[[nodiscard]] Block* Parent() const;
[[nodiscard]] const std::vector<Use>& Uses() const;
void SetParent(Block* block);
void SetArg(size_t index, Value value);
void AddPhiOperand(Block* predecessor, Value value);
void ReplaceUsesWith(Value replacement, bool preserve = true);
void ReplaceOpcode(ValueOpcode opcode);
void Invalidate();
template <typename T>
requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v<T>)
[[nodiscard]] T Flags() const {
T result {};
std::memcpy(&result, &flags, sizeof(result));
return result;
}
template <typename T>
requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v<T>)
void SetFlags(T value) {
flags = 0;
std::memcpy(&flags, &value, sizeof(value));
}
private:
void AddUse(Inst* used, size_t operand);
void RemoveUse(Inst* used, size_t operand);
void ClearArgs();
ValueOpcode opcode;
uint64_t flags;
Block* parent = nullptr;
std::vector<Value> args;
std::vector<Block*> phi_blocks;
std::vector<Use> uses;
};
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ValueProgram.cpp
+241
View File
@@ -0,0 +1,241 @@
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <fmt/format.h>
#include <map>
#include <unordered_set>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
bool Fail(std::string* error, const std::string& message) {
if (error != nullptr) {
*error = message;
}
return false;
}
bool IsRegisterGet(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::GetThreadBitScalarRegister:
case ValueOpcode::GetScalarRegister:
case ValueOpcode::GetVectorRegister:
case ValueOpcode::GetGotoVariable:
case ValueOpcode::GetScc:
case ValueOpcode::GetExec:
case ValueOpcode::GetExecLo:
case ValueOpcode::GetExecHi:
case ValueOpcode::GetVcc:
case ValueOpcode::GetVccLo:
case ValueOpcode::GetVccHi:
case ValueOpcode::GetM0: return true;
default: return false;
}
}
bool IsRuntimeRead(ValueOpcode opcode) {
return opcode == ValueOpcode::LoadAddressU32 || opcode == ValueOpcode::ReadConstBuffer;
}
bool EquivalentValue(const ValueProgram& program, Value left, Value right,
std::vector<std::pair<const Inst*, const Inst*>>& visited) {
left = left.Resolve();
right = right.Resolve();
if (left == right) {
return true;
}
if (left.IsImmediate() || right.IsImmediate() || left.GetType() != right.GetType()) {
return false;
}
const auto* lhs = left.TryInstruction();
const auto* rhs = right.TryInstruction();
if (lhs == nullptr || rhs == nullptr || lhs->GetOpcode() != rhs->GetOpcode() ||
lhs->NumArgs() != rhs->NumArgs()) {
return false;
}
if (std::ranges::find(visited, std::pair {lhs, rhs}) != visited.end()) {
return true;
}
visited.emplace_back(lhs, rhs);
if (IsRuntimeRead(lhs->GetOpcode())) {
const auto li = lhs->Flags<MemoryFlags>().index;
const auto ri = rhs->Flags<MemoryFlags>().index;
if (li >= program.memory_info.size() || ri >= program.memory_info.size() ||
program.memory_info[li] != program.memory_info[ri]) {
return false;
}
} else if (lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
return false;
}
for (size_t index = 0; index < lhs->NumArgs(); index++) {
if (lhs->GetOpcode() == ValueOpcode::Phi && lhs->PhiBlock(index) != rhs->PhiBlock(index)) {
return false;
}
if (!EquivalentValue(program, lhs->Arg(index), rhs->Arg(index), visited)) {
return false;
}
}
return true;
}
} // namespace
ValueProgram::~ValueProgram() {
// Values may cross block boundaries. Detach all arguments before any block starts destroying
// its instruction storage so reverse-use links always point to live definitions.
for (auto* block: blocks) {
for (auto& inst: *block) {
inst.Invalidate();
}
}
}
bool EquivalentValue(const ValueProgram& program, Value left, Value right) {
std::vector<std::pair<const Inst*, const Inst*>> visited;
return EquivalentValue(program, left, right, visited);
}
Value ResolveInvariantPhi(const ValueProgram& program, Value value) {
value = value.Resolve();
const auto* root = value.TryInstruction();
if (root == nullptr || root->GetOpcode() != ValueOpcode::Phi) {
return value;
}
Value invariant;
std::vector<Value> pending {value};
std::unordered_set<const Inst*> visited;
while (!pending.empty()) {
const auto current = pending.back().Resolve();
pending.pop_back();
const auto* inst = current.TryInstruction();
if (inst != nullptr && inst->GetOpcode() == ValueOpcode::Phi) {
if (!visited.insert(inst).second) {
continue;
}
for (size_t index = 0; index < inst->NumArgs(); index++) {
pending.push_back(inst->Arg(index));
}
continue;
}
if (invariant.IsEmpty()) {
invariant = current;
} else if (!EquivalentValue(program, invariant, current)) {
return {};
}
}
return invariant;
}
bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error) {
if (program.blocks.size() != program.block_info.size() ||
program.blocks.size() != program.block_storage.size()) {
return Fail(error, "value IR block storage is inconsistent");
}
for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) {
const auto* block = program.blocks[block_index];
if (block != program.block_storage[block_index].get()) {
return Fail(error, "value IR block pointer is inconsistent");
}
for (const auto& inst: *block) {
if (inst.Parent() != block) {
return Fail(error, "value IR instruction has the wrong parent block");
}
if (require_ssa && IsRegisterGet(inst.GetOpcode())) {
return Fail(error, fmt::format("register getter {} survived SSA rewrite",
ValueOpcodeName(inst.GetOpcode())));
}
if (inst.GetOpcode() != ValueOpcode::Phi && inst.GetOpcode() != ValueOpcode::Identity &&
inst.GetType() == Type::Opaque) {
return Fail(error, fmt::format("untyped opcode {} survived translation",
ValueOpcodeName(inst.GetOpcode())));
}
for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) {
const auto arg = inst.Arg(arg_index);
if (arg.IsEmpty()) {
return Fail(error, fmt::format("{} has an empty argument",
ValueOpcodeName(inst.GetOpcode())));
}
if (const auto* definition = arg.TryInstruction();
definition != nullptr && definition->Parent() == nullptr) {
return Fail(error, "value IR argument has a detached definition");
}
if (const auto* definition = arg.TryInstruction(); definition != nullptr) {
const auto& uses = definition->Uses();
const auto use = std::ranges::find_if(uses, [&](const Use& candidate) {
return candidate.user == &inst && candidate.operand == arg_index;
});
if (use == uses.end()) {
return Fail(error,
fmt::format("{} argument {} is absent from {} reverse uses",
ValueOpcodeName(inst.GetOpcode()), arg_index,
ValueOpcodeName(definition->GetOpcode())));
}
}
}
}
}
return true;
}
std::string ValueProgramToString(const ValueProgram& program) {
std::map<const Inst*, size_t> ids;
size_t next_id = 1;
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
ids.emplace(&inst, next_id++);
}
}
const auto value_string = [&](Value value) {
if (value.IsEmpty()) {
return std::string("<null>");
}
if (const auto* inst = value.TryInstruction(); inst != nullptr) {
return fmt::format("%{}", ids.at(inst));
}
switch (value.GetType()) {
case Type::ScalarReg: return fmt::format("s{}", RegIndex(value.ScalarRegister()));
case Type::VectorReg: return fmt::format("v{}", RegIndex(value.VectorRegister()));
case Type::U1: return std::string(value.U1() ? "true" : "false");
case Type::U8: return fmt::format("{}u8", value.U8());
case Type::U16: return fmt::format("{}u16", value.U16());
case Type::U32: return fmt::format("0x{:08x}", value.U32());
case Type::U64: return fmt::format("0x{:016x}", value.U64());
case Type::F16: return fmt::format("f16(0x{:04x})", value.F16Bits());
case Type::F32: return fmt::format("{}f", value.F32Value());
default: return fmt::format("<{}>", TypeName(value.GetType()));
}
};
std::string text =
fmt::format("mode={}\n", program.dispatcher_fallback ? "dispatcher" : "structured");
for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) {
text += fmt::format("Block ${} pc=0x{:08x}..0x{:08x}\n", block_index,
program.block_info[block_index].start_pc,
program.block_info[block_index].end_pc);
for (const auto& inst: *program.blocks[block_index]) {
const auto type = inst.GetType();
if (type != Type::Void) {
text +=
fmt::format(" %{:<5} = {}", ids.at(&inst), ValueOpcodeName(inst.GetOpcode()));
} else {
text += fmt::format(" {}", ValueOpcodeName(inst.GetOpcode()));
}
for (size_t index = 0; index < inst.NumArgs(); index++) {
text += index == 0 ? " " : ", ";
if (inst.GetOpcode() == ValueOpcode::Phi) {
const auto predecessor =
std::ranges::find(program.blocks, inst.PhiBlock(index));
text += fmt::format("[{}, ${}]", value_string(inst.Arg(index)),
std::distance(program.blocks.begin(), predecessor));
} else {
text += value_string(inst.Arg(index));
}
}
text +=
fmt::format(" ({}; uses={})\n", TypeName(Value(const_cast<Inst*>(&inst)).GetType()),
inst.UseCount());
}
}
return text;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ValueProgram.h
+64
View File
@@ -0,0 +1,64 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <memory>
#include <string>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
struct ValueBlockInfo {
uint32_t id = 0;
uint32_t start_pc = 0;
uint32_t end_pc = 0;
CFG::Terminator terminator;
Value condition;
Value indirect_target;
};
struct DescriptorSource {
std::array<Value, 8> dwords {};
uint32_t dword_count = 0;
bool operator==(const DescriptorSource& other) const = default;
};
struct SrtRead {
Value value;
uint32_t flat_offset = 0;
uint32_t use_pc = 0;
bool operator==(const SrtRead& other) const = default;
};
struct ValueProgram {
ValueProgram() = default;
~ValueProgram();
ValueProgram(const ValueProgram&) = delete;
ValueProgram& operator=(const ValueProgram&) = delete;
ValueProgram(ValueProgram&&) = default;
ValueProgram& operator=(ValueProgram&&) = default;
std::vector<std::unique_ptr<Block>> block_storage;
BlockList blocks;
std::vector<ValueBlockInfo> block_info;
// Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and
// storage-image swizzles. Typed memory instructions carry a dense index into these shader-local
// tables until those fields are consumed by emission.
std::vector<MemoryInfo> memory_info;
std::vector<ExportInfo> export_info;
std::vector<DescriptorSource> descriptor_sources;
std::vector<SrtRead> srt_reads;
std::vector<Value> dynamic_reads;
bool dispatcher_fallback = false;
};
bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error);
bool EquivalentValue(const ValueProgram& program, Value left, Value right);
Value ResolveInvariantPhi(const ValueProgram& program, Value value);
std::string ValueProgramToString(const ValueProgram& program);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/Opcodes.h
+71
View File
@@ -0,0 +1,71 @@
#pragma once
#include "graphics/shader/recompiler/ir/Type.h"
#include <array>
#include <cstddef>
#include <string_view>
namespace Libs::Graphics::ShaderRecompiler::IR {
enum class Opcode {
#define IR_OPCODE(name, category, lowering, ...) name,
#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc"
#undef IR_OPCODE
Count,
};
enum class OpcodeClass { General, Compare, Compare64 };
enum class LoweringClass {
Control,
Move,
Lane,
State,
Memory,
Attribute,
IntegerCompare,
Integer16Compare,
FloatCompare,
Conversion,
Integer16,
PackedInteger16,
PackedFloat16,
Float16,
Float,
U64Mask,
SimpleInteger,
ComposedInteger,
ExtendedInteger,
};
struct OpcodeInfo {
std::string_view name;
OpcodeClass opcode_class = OpcodeClass::General;
LoweringClass lowering_class = LoweringClass::Control;
};
inline constexpr OpcodeInfo OPCODE_INFO[] = {
#define IR_OPCODE(name, category, lowering, ...) \
{#name, OpcodeClass::category, LoweringClass::lowering},
#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc"
#undef IR_OPCODE
};
static_assert(std::size(OPCODE_INFO) == static_cast<size_t>(Opcode::Count));
constexpr const OpcodeInfo& GetOpcodeInfo(Opcode opcode) {
return OPCODE_INFO[static_cast<size_t>(opcode)];
}
constexpr std::string_view OpcodeName(Opcode opcode) {
return GetOpcodeInfo(opcode).name;
}
constexpr bool IsCompareOpcode(Opcode opcode) {
return GetOpcodeInfo(opcode).opcode_class != OpcodeClass::General;
}
constexpr bool IsCompare64Opcode(Opcode opcode) {
return GetOpcodeInfo(opcode).opcode_class == OpcodeClass::Compare64;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ShaderIROpcodes.cpp → src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp
+1
View File
@@ -103,6 +103,7 @@ constexpr LowerMap LOWER_OPS[] = {
{Decoder::Opcode::VCvtFlrI32F32, Opcode::ConvertFloorF32ToI32},
{Decoder::Opcode::VCvtOffF32I4, Opcode::ConvertI4ToOffsetF32},
{Decoder::Opcode::VRcpF32, Opcode::RcpF32},
{Decoder::Opcode::VRcpIflagF32, Opcode::RcpIflagF32},
{Decoder::Opcode::VFractF32, Opcode::FractF32},
{Decoder::Opcode::VTruncF32, Opcode::TruncF32},
{Decoder::Opcode::VCeilF32, Opcode::CeilF32},
src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc
+343
View File
@@ -0,0 +1,343 @@
// Semantic operations produced by RDNA2 instruction lowering. The typed frontend consumes this
// baseline set before optimization and resource planning; it must not grow without decoder support.
IR_OPCODE(ControlNop, General, Control, Void)
IR_OPCODE(Waitcnt, General, Control, Void)
IR_OPCODE(Barrier, General, Control, Void)
IR_OPCODE(Sendmsg, General, Control, Void)
IR_OPCODE(TtraceData, General, Control, Void)
IR_OPCODE(InstPrefetch, General, Control, Void)
IR_OPCODE(MoveU32, General, Move)
IR_OPCODE(MoveF32Bits, General, Move)
IR_OPCODE(MoveRelDestU32, General, Lane)
IR_OPCODE(MoveRelSourceU32, General, Lane)
IR_OPCODE(MoveU64, General, Move)
IR_OPCODE(WqmB64, General, Move)
IR_OPCODE(SaveexecB32, General, State)
IR_OPCODE(SaveexecB64, General, State)
IR_OPCODE(ReadFirstLaneU32, General, Lane)
IR_OPCODE(ReadLaneU32, General, Lane)
IR_OPCODE(WriteLaneU32, General, Lane)
IR_OPCODE(Permlane16B32, General, Lane)
IR_OPCODE(Permlanex16B32, General, Lane)
IR_OPCODE(AbsI32, General, SimpleInteger)
IR_OPCODE(IAddU32, General, SimpleInteger)
IR_OPCODE(IAddCarryU32, General, State)
IR_OPCODE(ISubBorrowU32, General, State)
IR_OPCODE(ScalarAddCarryU32, General, State)
IR_OPCODE(ScalarSubBorrowU32, General, State)
IR_OPCODE(ScalarSubBorrowCarryU32, General, State)
IR_OPCODE(ScalarSignedAddOverflowI32, General, State)
IR_OPCODE(ScalarSignedSubOverflowI32, General, State)
IR_OPCODE(ScalarShiftLeftAddCarryU32, General, State)
IR_OPCODE(ISubU32, General, SimpleInteger)
IR_OPCODE(IMulU32, General, SimpleInteger)
IR_OPCODE(UMulHighU32, General, SimpleInteger)
IR_OPCODE(SMulHighI32, General, SimpleInteger)
IR_OPCODE(IMadI24U32, General, ExtendedInteger)
IR_OPCODE(UMadU24U32, General, ExtendedInteger)
IR_OPCODE(UMadU64U32, General, ExtendedInteger)
IR_OPCODE(SadU32, General, ExtendedInteger)
IR_OPCODE(IAdd3U32, General, ExtendedInteger)
IR_OPCODE(IMulI24U32, General, ExtendedInteger)
IR_OPCODE(UMulU24U32, General, ExtendedInteger)
IR_OPCODE(IMinI32, General, SimpleInteger)
IR_OPCODE(IMaxI32, General, SimpleInteger)
IR_OPCODE(IMin3I32, General, SimpleInteger)
IR_OPCODE(IMax3I32, General, SimpleInteger)
IR_OPCODE(IMed3I32, General, SimpleInteger)
IR_OPCODE(UMinU32, General, SimpleInteger)
IR_OPCODE(UMaxU32, General, SimpleInteger)
IR_OPCODE(UMin3U32, General, SimpleInteger)
IR_OPCODE(UMax3U32, General, SimpleInteger)
IR_OPCODE(UMed3U32, General, SimpleInteger)
IR_OPCODE(BitwiseAndU32, General, SimpleInteger)
IR_OPCODE(BitwiseAndU64, General, U64Mask)
IR_OPCODE(BitwiseAndNotU32, General, ComposedInteger)
IR_OPCODE(BitwiseAndNotU64, General, U64Mask)
IR_OPCODE(BitwiseOrU32, General, SimpleInteger)
IR_OPCODE(BitwiseOrU64, General, U64Mask)
IR_OPCODE(BitwiseOrNotU32, General, ComposedInteger)
IR_OPCODE(BitwiseOrNotU64, General, U64Mask)
IR_OPCODE(BitwiseAndOrU32, General, ComposedInteger)
IR_OPCODE(BitwiseOr3U32, General, ComposedInteger)
IR_OPCODE(BitwiseXorU32, General, SimpleInteger)
IR_OPCODE(BitwiseXorU64, General, U64Mask)
IR_OPCODE(BitwiseXor3U32, General, ComposedInteger)
IR_OPCODE(BitwiseNandU32, General, ComposedInteger)
IR_OPCODE(BitwiseNandU64, General, U64Mask)
IR_OPCODE(BitwiseNorU32, General, ComposedInteger)
IR_OPCODE(BitwiseNorU64, General, U64Mask)
IR_OPCODE(BitwiseXnorU32, General, ComposedInteger)
IR_OPCODE(BitwiseXnorU64, General, U64Mask)
IR_OPCODE(BitwiseNotU32, General, SimpleInteger)
IR_OPCODE(BitwiseNotU64, General, U64Mask)
IR_OPCODE(BitClearU32, General, ExtendedInteger)
IR_OPCODE(BitSetU32, General, ExtendedInteger)
IR_OPCODE(BitReverseU32, General, SimpleInteger)
IR_OPCODE(BitCountU32, General, SimpleInteger)
IR_OPCODE(BitCountU64, General, SimpleInteger)
IR_OPCODE(BitReplicateB64B32, General, ExtendedInteger)
IR_OPCODE(BitCountAddU32, General, ExtendedInteger)
IR_OPCODE(MaskedBitCountLowU32, General, ExtendedInteger)
IR_OPCODE(MaskedBitCountHighU32, General, ExtendedInteger)
IR_OPCODE(FindLsbU32, General, SimpleInteger)
IR_OPCODE(FindMsbFromHighU32, General, ComposedInteger)
IR_OPCODE(FindMsbFromHighU64, General, ComposedInteger)
IR_OPCODE(BitFieldMaskU32, General, ExtendedInteger)
IR_OPCODE(BitFieldMaskU64, General, ExtendedInteger)
IR_OPCODE(BitFieldExtractU32, General, ExtendedInteger)
IR_OPCODE(BitFieldExtractU64, General, ExtendedInteger)
IR_OPCODE(BitFieldExtract3U32, General, ExtendedInteger)
IR_OPCODE(BitFieldExtract3I32, General, ExtendedInteger)
IR_OPCODE(BitFieldInsertSelectU32, General, ExtendedInteger)
IR_OPCODE(BitCompare0B32, General, ExtendedInteger)
IR_OPCODE(BitCompare1B32, General, ExtendedInteger)
IR_OPCODE(AlignBitU32, General, ExtendedInteger)
IR_OPCODE(ShiftLeftAddU32, General, ExtendedInteger)
IR_OPCODE(AddShiftLeftU32, General, ExtendedInteger)
IR_OPCODE(XorAddU32, General, ExtendedInteger)
IR_OPCODE(ShiftLeftOrU32, General, ExtendedInteger)
IR_OPCODE(ShiftLeftLogicalU32, General, SimpleInteger)
IR_OPCODE(ShiftLeftLogicalU64, General, SimpleInteger)
IR_OPCODE(ShiftLeftLogicalU16, General, Integer16)
IR_OPCODE(ShiftRightLogicalU32, General, SimpleInteger)
IR_OPCODE(ShiftRightLogicalU64, General, SimpleInteger)
IR_OPCODE(ShiftRightLogicalU16, General, Integer16)
IR_OPCODE(ShiftRightArithmeticI32, General, SimpleInteger)
IR_OPCODE(ShiftRightArithmeticI16, General, Integer16)
IR_OPCODE(SelectU32, General, ExtendedInteger, U32, U1, U32, U32)
IR_OPCODE(SelectMaskU32, General, ExtendedInteger)
IR_OPCODE(SelectF32Bits, General, ExtendedInteger)
IR_OPCODE(SelectMaskF32Bits, General, ExtendedInteger)
IR_OPCODE(SelectU64, General, ExtendedInteger)
IR_OPCODE(PackLowLowU16, General, ExtendedInteger)
IR_OPCODE(PackLowHighU16, General, ExtendedInteger)
IR_OPCODE(PackHighHighU16, General, ExtendedInteger)
IR_OPCODE(CompareFalse, Compare, IntegerCompare)
IR_OPCODE(CompareTrue, Compare, IntegerCompare)
IR_OPCODE(CompareEqU32, Compare, IntegerCompare)
IR_OPCODE(CompareNeU32, Compare, IntegerCompare)
IR_OPCODE(CompareGtU32, Compare, IntegerCompare)
IR_OPCODE(CompareGeU32, Compare, IntegerCompare)
IR_OPCODE(CompareLtU32, Compare, IntegerCompare)
IR_OPCODE(CompareLeU32, Compare, IntegerCompare)
IR_OPCODE(CompareEqU64, Compare64, IntegerCompare)
IR_OPCODE(CompareNeU64, Compare64, IntegerCompare)
IR_OPCODE(CompareMaskEqU32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskNeU32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskGtU32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskGeU32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskLtU32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskLeU32, Compare, IntegerCompare)
IR_OPCODE(CompareEqI32, Compare, IntegerCompare)
IR_OPCODE(CompareNeI32, Compare, IntegerCompare)
IR_OPCODE(CompareGtI32, Compare, IntegerCompare)
IR_OPCODE(CompareGeI32, Compare, IntegerCompare)
IR_OPCODE(CompareLtI32, Compare, IntegerCompare)
IR_OPCODE(CompareLeI32, Compare, IntegerCompare)
IR_OPCODE(CompareEqI16, Compare, Integer16Compare)
IR_OPCODE(CompareNeI16, Compare, Integer16Compare)
IR_OPCODE(CompareGtI16, Compare, Integer16Compare)
IR_OPCODE(CompareGeI16, Compare, Integer16Compare)
IR_OPCODE(CompareLtI16, Compare, Integer16Compare)
IR_OPCODE(CompareLeI16, Compare, Integer16Compare)
IR_OPCODE(CompareMaskEqI32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskNeI32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskGtI32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskGeI32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskLtI32, Compare, IntegerCompare)
IR_OPCODE(CompareMaskLeI32, Compare, IntegerCompare)
IR_OPCODE(CompareEqU16, Compare, Integer16Compare)
IR_OPCODE(CompareNeU16, Compare, Integer16Compare)
IR_OPCODE(CompareGtU16, Compare, Integer16Compare)
IR_OPCODE(CompareGeU16, Compare, Integer16Compare)
IR_OPCODE(CompareLtU16, Compare, Integer16Compare)
IR_OPCODE(CompareLeU16, Compare, Integer16Compare)
IR_OPCODE(CompareEqF32, Compare, FloatCompare)
IR_OPCODE(CompareNeF32, Compare, FloatCompare)
IR_OPCODE(CompareGtF32, Compare, FloatCompare)
IR_OPCODE(CompareGeF32, Compare, FloatCompare)
IR_OPCODE(CompareLtF32, Compare, FloatCompare)
IR_OPCODE(CompareLeF32, Compare, FloatCompare)
IR_OPCODE(CompareOrderedF32, Compare, FloatCompare)
IR_OPCODE(CompareUnorderedF32, Compare, FloatCompare)
IR_OPCODE(CompareUnordEqF32, Compare, FloatCompare)
IR_OPCODE(CompareUnordNeF32, Compare, FloatCompare)
IR_OPCODE(CompareUnordGtF32, Compare, FloatCompare)
IR_OPCODE(CompareUnordGeF32, Compare, FloatCompare)
IR_OPCODE(CompareUnordLtF32, Compare, FloatCompare)
IR_OPCODE(CompareUnordLeF32, Compare, FloatCompare)
IR_OPCODE(CompareClassF32, Compare, FloatCompare)
IR_OPCODE(CompareEqF16, Compare, FloatCompare)
IR_OPCODE(CompareNeF16, Compare, FloatCompare)
IR_OPCODE(CompareGtF16, Compare, FloatCompare)
IR_OPCODE(CompareGeF16, Compare, FloatCompare)
IR_OPCODE(CompareLtF16, Compare, FloatCompare)
IR_OPCODE(CompareLeF16, Compare, FloatCompare)
IR_OPCODE(CompareUnordNeF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskEqF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskNeF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskGtF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskGeF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskLtF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskLeF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordNeF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordGeF16, Compare, FloatCompare)
IR_OPCODE(CompareMaskEqF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskNeF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskGtF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskGeF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskLtF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskLeF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordEqF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordNeF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordGtF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordGeF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordLtF32, Compare, FloatCompare)
IR_OPCODE(CompareMaskUnordLeF32, Compare, FloatCompare)
IR_OPCODE(ConvertByteU32ToF32, General, Conversion)
IR_OPCODE(ConvertU32ToF32, General, Conversion)
IR_OPCODE(ConvertI32ToF32, General, Conversion)
IR_OPCODE(ConvertF32ToU32, General, Conversion)
IR_OPCODE(ConvertF32ToI32, General, Conversion)
IR_OPCODE(ConvertF32ToF16, General, Conversion)
IR_OPCODE(ConvertF16ToF32, General, Conversion)
IR_OPCODE(ConvertU16ToF16, General, Conversion)
IR_OPCODE(ConvertF16ToU16, General, Conversion)
IR_OPCODE(ConvertI16ToF16, General, Conversion)
IR_OPCODE(ConvertF16ToI16, General, Conversion)
IR_OPCODE(ConvertRoundPlusInfF32ToI32, General, Conversion)
IR_OPCODE(ConvertFloorF32ToI32, General, Conversion)
IR_OPCODE(ConvertI4ToOffsetF32, General, Conversion)
IR_OPCODE(LdexpF32, General, Float)
IR_OPCODE(PackF32ToF16Rtz, General, Conversion)
IR_OPCODE(PackSnorm2x16F32, General, Conversion)
IR_OPCODE(PackUnorm2x16F32, General, Conversion)
IR_OPCODE(PackU16U32, General, ExtendedInteger)
IR_OPCODE(PackU8F32, General, Conversion)
IR_OPCODE(PackB32F16, General, Conversion)
IR_OPCODE(PackedMadI16, General, PackedInteger16)
IR_OPCODE(PackedMulLoU16, General, PackedInteger16)
IR_OPCODE(PackedAddI16, General, PackedInteger16)
IR_OPCODE(PackedSubI16, General, PackedInteger16)
IR_OPCODE(PackedLshlrevB16, General, PackedInteger16)
IR_OPCODE(PackedLshrrevB16, General, PackedInteger16)
IR_OPCODE(PackedAshrrevI16, General, PackedInteger16)
IR_OPCODE(PackedMaxI16, General, PackedInteger16)
IR_OPCODE(PackedMinI16, General, PackedInteger16)
IR_OPCODE(PackedMadU16, General, PackedInteger16)
IR_OPCODE(PackedAddU16, General, PackedInteger16)
IR_OPCODE(PackedSubU16, General, PackedInteger16)
IR_OPCODE(PackedMaxU16, General, PackedInteger16)
IR_OPCODE(PackedMinU16, General, PackedInteger16)
IR_OPCODE(PackedAddF16, General, PackedFloat16)
IR_OPCODE(PackedMulF16, General, PackedFloat16)
IR_OPCODE(PackedMinF16, General, PackedFloat16)
IR_OPCODE(PackedMaxF16, General, PackedFloat16)
IR_OPCODE(PackedFmaF16, General, PackedFloat16)
IR_OPCODE(AddF16, General, Float16)
IR_OPCODE(SubF16, General, Float16)
IR_OPCODE(MulF16, General, Float16)
IR_OPCODE(MinF16, General, Float16)
IR_OPCODE(MaxF16, General, Float16)
IR_OPCODE(FmaF16, General, Float16)
IR_OPCODE(MadMixF16, General, Float16)
IR_OPCODE(IAddU16, General, Integer16)
IR_OPCODE(ISubI16, General, Integer16)
IR_OPCODE(IMinI16, General, Integer16)
IR_OPCODE(IMaxI16, General, Integer16)
IR_OPCODE(UMinU16, General, Integer16)
IR_OPCODE(UMaxU16, General, Integer16)
IR_OPCODE(RcpF32, General, Float)
IR_OPCODE(RcpIflagF32, General, Float)
IR_OPCODE(FractF32, General, Float)
IR_OPCODE(TruncF32, General, Float)
IR_OPCODE(CeilF32, General, Float)
IR_OPCODE(RoundEvenF32, General, Float)
IR_OPCODE(FloorF32, General, Float)
IR_OPCODE(Exp2F32, General, Float)
IR_OPCODE(Log2F32, General, Float)
IR_OPCODE(InverseSqrtF32, General, Float)
IR_OPCODE(SqrtF32, General, Float)
IR_OPCODE(RcpF16, General, Float16)
IR_OPCODE(SqrtF16, General, Float16)
IR_OPCODE(InverseSqrtF16, General, Float16)
IR_OPCODE(Log2F16, General, Float16)
IR_OPCODE(Exp2F16, General, Float16)
IR_OPCODE(FloorF16, General, Float16)
IR_OPCODE(CeilF16, General, Float16)
IR_OPCODE(TruncF16, General, Float16)
IR_OPCODE(RoundEvenF16, General, Float16)
IR_OPCODE(SinF32, General, Float)
IR_OPCODE(CosF32, General, Float)
IR_OPCODE(CubeIdF32, General, Float)
IR_OPCODE(CubeScF32, General, Float)
IR_OPCODE(CubeTcF32, General, Float)
IR_OPCODE(CubeMaF32, General, Float)
IR_OPCODE(FAddF32, General, Float)
IR_OPCODE(FSubF32, General, Float)
IR_OPCODE(FMulF32, General, Float)
IR_OPCODE(FMinF32, General, Float)
IR_OPCODE(FMaxF32, General, Float)
IR_OPCODE(FMadF32, General, Float)
IR_OPCODE(Dot2AccF32F16, General, Float)
IR_OPCODE(FMin3F32, General, Float)
IR_OPCODE(FMax3F32, General, Float)
IR_OPCODE(FMed3F32, General, Float)
IR_OPCODE(Min3F16, General, Float16)
IR_OPCODE(Max3F16, General, Float16)
IR_OPCODE(Med3F16, General, Float16)
IR_OPCODE(LoadSrtDword, General, Memory)
IR_OPCODE(SLoadDword, General, Memory)
IR_OPCODE(SBufferLoadDword, General, Memory)
IR_OPCODE(BufferLoadUbyte, General, Memory)
IR_OPCODE(BufferLoadSbyte, General, Memory)
IR_OPCODE(BufferLoadUshort, General, Memory)
IR_OPCODE(BufferLoadSshort, General, Memory)
IR_OPCODE(BufferLoadDword, General, Memory)
IR_OPCODE(BufferStoreByte, General, Memory)
IR_OPCODE(BufferStoreShort, General, Memory)
IR_OPCODE(BufferStoreDword, General, Memory)
IR_OPCODE(AtomicSwapU32, General, Memory)
IR_OPCODE(AtomicAddU32, General, Memory)
IR_OPCODE(AtomicSubU32, General, Memory)
IR_OPCODE(AtomicSMinI32, General, Memory)
IR_OPCODE(AtomicUMinU32, General, Memory)
IR_OPCODE(AtomicSMaxI32, General, Memory)
IR_OPCODE(AtomicUMaxU32, General, Memory)
IR_OPCODE(AtomicAndU32, General, Memory)
IR_OPCODE(AtomicOrU32, General, Memory)
IR_OPCODE(AtomicXorU32, General, Memory)
IR_OPCODE(AtomicFMinF32, General, Memory)
IR_OPCODE(AtomicFMaxF32, General, Memory)
IR_OPCODE(FlatLoadUbyte, General, Memory)
IR_OPCODE(FlatLoadSbyte, General, Memory)
IR_OPCODE(FlatLoadUshort, General, Memory)
IR_OPCODE(FlatLoadSshort, General, Memory)
IR_OPCODE(FlatLoadDword, General, Memory)
IR_OPCODE(FlatStoreByte, General, Memory)
IR_OPCODE(FlatStoreShort, General, Memory)
IR_OPCODE(FlatStoreDword, General, Memory)
IR_OPCODE(DsReadUbyte, General, Memory)
IR_OPCODE(DsReadSbyte, General, Memory)
IR_OPCODE(DsReadUshort, General, Memory)
IR_OPCODE(DsReadSshort, General, Memory)
IR_OPCODE(DsReadB32, General, Memory)
IR_OPCODE(DsWriteByte, General, Memory)
IR_OPCODE(DsWriteShort, General, Memory)
IR_OPCODE(DsWriteB32, General, Memory)
IR_OPCODE(DsMinF32, General, Memory)
IR_OPCODE(DsMaxF32, General, Memory)
IR_OPCODE(DsSwizzleB32, General, Memory)
IR_OPCODE(DsConsume, General, Memory)
IR_OPCODE(DsAppend, General, Memory)
IR_OPCODE(DsWriteAddtidB32, General, Memory)
IR_OPCODE(DsReadAddtidB32, General, Memory)
IR_OPCODE(ImageGetResinfo, General, Memory)
IR_OPCODE(ImageGetLod, General, Memory)
IR_OPCODE(ImageLoad, General, Memory)
IR_OPCODE(ImageStore, General, Memory)
IR_OPCODE(ImageSample, General, Memory)
IR_OPCODE(ImageGather4, General, Memory)
IR_OPCODE(LoadInputF32, General, Attribute)
IR_OPCODE(Export, General, Attribute)
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp
+144
View File
@@ -0,0 +1,144 @@
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h"
#include <array>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
struct OpcodeMeta {
std::string_view name;
Type type = Type::Void;
std::array<Type, 16> args = {};
size_t num_args = 0;
};
template <typename... Args>
consteval OpcodeMeta MakeMeta(std::string_view name, Type type, Args... args) {
static_assert(sizeof...(Args) <= 16);
OpcodeMeta meta {.name = name, .type = type, .num_args = sizeof...(Args)};
meta.args.fill(Type::Void);
size_t index = 0;
((meta.args[index++] = args), ...);
return meta;
}
constexpr Type Void = Type::Void;
constexpr Type Opaque = Type::Opaque;
constexpr Type ScalarReg = Type::ScalarReg;
constexpr Type VectorReg = Type::VectorReg;
constexpr Type U1 = Type::U1;
constexpr Type U8 = Type::U8;
constexpr Type U16 = Type::U16;
constexpr Type U32 = Type::U32;
constexpr Type U64 = Type::U64;
constexpr Type F16 = Type::F16;
constexpr Type F32 = Type::F32;
constexpr Type U32x2 = Type::U32x2;
constexpr Type U32x4 = Type::U32x4;
constexpr Type F32x2 = Type::F32x2;
constexpr Type SrtResource = Type::SrtResource;
constexpr Type BufferResource = Type::BufferResource;
constexpr Type AddressResource = Type::AddressResource;
constexpr Type ImageResource = Type::ImageResource;
constexpr Type SamplerResource = Type::SamplerResource;
constexpr Type LdsResource = Type::LdsResource;
constexpr Type GdsResource = Type::GdsResource;
constexpr Type ImageAddress = Type::ImageAddress;
constexpr std::array<OpcodeMeta, static_cast<size_t>(ValueOpcode::Count)> MetaTable = {{
#define VALUE_OPCODE(name, ...) MakeMeta(#name __VA_OPT__(, ) __VA_ARGS__),
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc"
#undef VALUE_OPCODE
}};
} // namespace
Type TypeOf(ValueOpcode opcode) {
return MetaTable[static_cast<size_t>(opcode)].type;
}
size_t NumArgsOf(ValueOpcode opcode) {
return MetaTable[static_cast<size_t>(opcode)].num_args;
}
Type ArgTypeOf(ValueOpcode opcode, size_t index) {
const auto& meta = MetaTable[static_cast<size_t>(opcode)];
return meta.args.at(index);
}
std::string_view ValueOpcodeName(ValueOpcode opcode) {
return MetaTable[static_cast<size_t>(opcode)].name;
}
bool HasSideEffects(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::Reference:
case ValueOpcode::ReferenceU32:
case ValueOpcode::Prologue:
case ValueOpcode::Epilogue:
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32:
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32:
case ValueOpcode::WriteSharedU8:
case ValueOpcode::WriteSharedU16:
case ValueOpcode::WriteSharedU32:
case ValueOpcode::WriteGdsU8:
case ValueOpcode::WriteGdsU16:
case ValueOpcode::WriteGdsU32:
case ValueOpcode::SharedAtomicFMin32:
case ValueOpcode::SharedAtomicFMax32:
case ValueOpcode::GdsAtomicFMin32:
case ValueOpcode::GdsAtomicFMax32:
case ValueOpcode::SharedAtomicSwap32:
case ValueOpcode::SharedAtomicIAdd32:
case ValueOpcode::SharedAtomicISub32:
case ValueOpcode::SharedAtomicSMin32:
case ValueOpcode::SharedAtomicUMin32:
case ValueOpcode::SharedAtomicSMax32:
case ValueOpcode::SharedAtomicUMax32:
case ValueOpcode::SharedAtomicAnd32:
case ValueOpcode::SharedAtomicOr32:
case ValueOpcode::SharedAtomicXor32:
case ValueOpcode::GdsAtomicSwap32:
case ValueOpcode::GdsAtomicIAdd32:
case ValueOpcode::GdsAtomicISub32:
case ValueOpcode::GdsAtomicSMin32:
case ValueOpcode::GdsAtomicUMin32:
case ValueOpcode::GdsAtomicSMax32:
case ValueOpcode::GdsAtomicUMax32:
case ValueOpcode::GdsAtomicAnd32:
case ValueOpcode::GdsAtomicOr32:
case ValueOpcode::GdsAtomicXor32:
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
case ValueOpcode::BufferAtomicSMin32:
case ValueOpcode::BufferAtomicUMin32:
case ValueOpcode::BufferAtomicSMax32:
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32:
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
case ValueOpcode::ImageAtomicAnd32:
case ValueOpcode::ImageAtomicOr32:
case ValueOpcode::ImageAtomicXor32:
case ValueOpcode::DataAppend:
case ValueOpcode::DataConsume:
case ValueOpcode::GdsDataAppend:
case ValueOpcode::GdsDataConsume:
case ValueOpcode::ImageWrite:
case ValueOpcode::SetAttribute:
case ValueOpcode::Barrier: return true;
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h
+57
View File
@@ -0,0 +1,57 @@
#pragma once
#include "graphics/shader/recompiler/ir/Type.h"
#include <cstddef>
#include <cstdint>
#include <string_view>
#include <type_traits>
namespace Libs::Graphics::ShaderRecompiler::IR {
enum class ValueOpcode {
#define VALUE_OPCODE(name, ...) name,
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc"
#undef VALUE_OPCODE
Count,
};
struct DppMoveFlags {
uint16_t control = 0;
uint8_t row_mask = 0xf;
uint8_t bank_mask = 0xf;
bool fetch_inactive = false;
bool bound_control = false;
};
static_assert(sizeof(DppMoveFlags) <= sizeof(uint64_t));
static_assert(std::is_trivially_copyable_v<DppMoveFlags>);
struct PermlaneFlags {
bool x16 = false;
bool fetch_inactive = false;
bool bound_control = false;
};
static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t));
static_assert(std::is_trivially_copyable_v<PermlaneFlags>);
struct MemoryFlags {
uint32_t index = 0;
uint32_t pc = 0;
};
static_assert(sizeof(MemoryFlags) == sizeof(uint64_t));
static_assert(std::is_trivially_copyable_v<MemoryFlags>);
struct ExportFlags {
uint32_t index = 0;
uint32_t pc = 0;
};
static_assert(sizeof(ExportFlags) == sizeof(uint64_t));
static_assert(std::is_trivially_copyable_v<ExportFlags>);
[[nodiscard]] Type TypeOf(ValueOpcode opcode);
[[nodiscard]] Type ArgTypeOf(ValueOpcode opcode, size_t index);
[[nodiscard]] size_t NumArgsOf(ValueOpcode opcode);
[[nodiscard]] bool HasSideEffects(ValueOpcode opcode);
[[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+299
View File
@@ -0,0 +1,299 @@
// operation, result, arguments...
VALUE_OPCODE(Phi, Opaque)
VALUE_OPCODE(Identity, Opaque, Opaque)
VALUE_OPCODE(Void, Void)
VALUE_OPCODE(ConditionRef, U1, U1)
VALUE_OPCODE(Reference, Void, U1)
VALUE_OPCODE(ReferenceU32, Void, U32)
// Translation and register-state pseudo operations.
VALUE_OPCODE(Prologue, Void)
VALUE_OPCODE(Epilogue, Void)
VALUE_OPCODE(GetUserData, U32, ScalarReg)
VALUE_OPCODE(GetShaderBase, U64)
VALUE_OPCODE(GetBuiltin, U32, U32, U32)
VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg)
VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarRegister, U32, ScalarReg)
VALUE_OPCODE(SetScalarRegister, Void, ScalarReg, U32)
VALUE_OPCODE(GetVectorRegister, U32, VectorReg)
VALUE_OPCODE(SetVectorRegister, Void, VectorReg, U32)
VALUE_OPCODE(GetGotoVariable, U1, U32)
VALUE_OPCODE(SetGotoVariable, Void, U32, U1)
VALUE_OPCODE(GetScc, U1)
VALUE_OPCODE(SetScc, Void, U1)
VALUE_OPCODE(GetExec, U1)
VALUE_OPCODE(SetExec, Void, U1)
VALUE_OPCODE(GetExecLo, U32)
VALUE_OPCODE(SetExecLo, Void, U32)
VALUE_OPCODE(GetExecHi, U32)
VALUE_OPCODE(SetExecHi, Void, U32)
VALUE_OPCODE(GetVcc, U1)
VALUE_OPCODE(SetVcc, Void, U1)
VALUE_OPCODE(GetVccLo, U32)
VALUE_OPCODE(SetVccLo, Void, U32)
VALUE_OPCODE(GetVccHi, U32)
VALUE_OPCODE(SetVccHi, Void, U32)
VALUE_OPCODE(GetM0, U32)
VALUE_OPCODE(SetM0, Void, U32)
VALUE_OPCODE(UndefU1, U1)
VALUE_OPCODE(UndefU8, U8)
VALUE_OPCODE(UndefU16, U16)
VALUE_OPCODE(UndefU32, U32)
VALUE_OPCODE(UndefU64, U64)
// Value construction and bit-preserving conversion.
VALUE_OPCODE(BitCastU16F16, U16, F16)
VALUE_OPCODE(BitCastF16U16, F16, U16)
VALUE_OPCODE(BitCastU32F32, U32, F32)
VALUE_OPCODE(BitCastF32U32, F32, U32)
VALUE_OPCODE(ConvertU16U32, U16, U32)
VALUE_OPCODE(ConvertU32U16, U32, U16)
VALUE_OPCODE(ConvertU8U32, U8, U32)
VALUE_OPCODE(ConvertU32U8, U32, U8)
VALUE_OPCODE(ConvertF32F16, F32, F16)
VALUE_OPCODE(ConvertF16F32, F16, F32)
VALUE_OPCODE(ConvertS32F32, U32, F32)
VALUE_OPCODE(ConvertU32F32, U32, F32)
VALUE_OPCODE(ConvertF32S32, F32, U32)
VALUE_OPCODE(ConvertF32U32, F32, U32)
VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32)
VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32)
VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32)
VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32)
VALUE_OPCODE(PackUint2x32, U64, U32x2)
VALUE_OPCODE(UnpackUint2x32, U32x2, U64)
VALUE_OPCODE(PackHalf2x16, U32, F32x2)
VALUE_OPCODE(PackSnorm2x16, U32, F32x2)
VALUE_OPCODE(PackUnorm2x16, U32, F32x2)
VALUE_OPCODE(PackFloat2x16Rtz, U32, F32, F32)
// Source/destination modifiers made explicit by translation.
VALUE_OPCODE(FPAbs32, F32, F32)
VALUE_OPCODE(FPNeg32, F32, F32)
VALUE_OPCODE(FPSaturate32, F32, F32)
VALUE_OPCODE(BitFieldInsert, U32, U32, U32, U32, U32)
VALUE_OPCODE(BitFieldUExtract, U32, U32, U32, U32)
VALUE_OPCODE(BitFieldSExtract, U32, U32, U32, U32)
VALUE_OPCODE(DppMoveU32, U32, U32, U1)
VALUE_OPCODE(DppUpdateU32, U32, U32, U32, U1)
VALUE_OPCODE(WqmMask, U1, U1)
// Core value operations used to expand architectural side effects.
VALUE_OPCODE(SelectU1, U1, U1, U1, U1)
VALUE_OPCODE(SelectF32, F32, U1, F32, F32)
VALUE_OPCODE(IAdd32, U32, U32, U32)
VALUE_OPCODE(IAdd64, U64, U64, U64)
VALUE_OPCODE(IAddCarry32, U32x2, U32, U32)
VALUE_OPCODE(ISub32, U32, U32, U32)
VALUE_OPCODE(ISub64, U64, U64, U64)
VALUE_OPCODE(IMul32, U32, U32, U32)
VALUE_OPCODE(IMul64, U64, U64, U64)
VALUE_OPCODE(UDiv32, U32, U32, U32)
VALUE_OPCODE(SMulHi, U32, U32, U32)
VALUE_OPCODE(UMulHi, U32, U32, U32)
VALUE_OPCODE(IAbs32, U32, U32)
VALUE_OPCODE(ShiftLeftLogical32, U32, U32, U32)
VALUE_OPCODE(ShiftLeftLogical64, U64, U64, U32)
VALUE_OPCODE(ShiftRightLogical32, U32, U32, U32)
VALUE_OPCODE(ShiftRightLogical64, U64, U64, U32)
VALUE_OPCODE(ShiftRightArithmetic32, U32, U32, U32)
VALUE_OPCODE(ShiftRightArithmetic64, U64, U64, U32)
VALUE_OPCODE(BitwiseAnd32, U32, U32, U32)
VALUE_OPCODE(BitwiseAnd64, U64, U64, U64)
VALUE_OPCODE(BitwiseOr32, U32, U32, U32)
VALUE_OPCODE(BitwiseOr64, U64, U64, U64)
VALUE_OPCODE(BitwiseXor32, U32, U32, U32)
VALUE_OPCODE(BitwiseNot32, U32, U32)
VALUE_OPCODE(BitReverse32, U32, U32)
VALUE_OPCODE(BitCount32, U32, U32)
VALUE_OPCODE(BitCount64, U32, U64)
VALUE_OPCODE(FindUMsb32, U32, U32)
VALUE_OPCODE(FindUMsb64, U32, U64)
VALUE_OPCODE(FindSMsb32, U32, U32)
VALUE_OPCODE(FindILsb32, U32, U32)
VALUE_OPCODE(SMin32, U32, U32, U32)
VALUE_OPCODE(UMin32, U32, U32, U32)
VALUE_OPCODE(SMax32, U32, U32, U32)
VALUE_OPCODE(UMax32, U32, U32, U32)
VALUE_OPCODE(SMinTri32, U32, U32, U32, U32)
VALUE_OPCODE(UMinTri32, U32, U32, U32, U32)
VALUE_OPCODE(SMaxTri32, U32, U32, U32, U32)
VALUE_OPCODE(UMaxTri32, U32, U32, U32, U32)
VALUE_OPCODE(SMedTri32, U32, U32, U32, U32)
VALUE_OPCODE(UMedTri32, U32, U32, U32, U32)
VALUE_OPCODE(SLessThan32, U1, U32, U32)
VALUE_OPCODE(SLessThan64, U1, U64, U64)
VALUE_OPCODE(ULessThan32, U1, U32, U32)
VALUE_OPCODE(ULessThan64, U1, U64, U64)
VALUE_OPCODE(IEqual32, U1, U32, U32)
VALUE_OPCODE(IEqual64, U1, U64, U64)
VALUE_OPCODE(SLessThanEqual32, U1, U32, U32)
VALUE_OPCODE(SLessThanEqual64, U1, U64, U64)
VALUE_OPCODE(ULessThanEqual32, U1, U32, U32)
VALUE_OPCODE(ULessThanEqual64, U1, U64, U64)
VALUE_OPCODE(SGreaterThan32, U1, U32, U32)
VALUE_OPCODE(SGreaterThan64, U1, U64, U64)
VALUE_OPCODE(UGreaterThan32, U1, U32, U32)
VALUE_OPCODE(UGreaterThan64, U1, U64, U64)
VALUE_OPCODE(INotEqual32, U1, U32, U32)
VALUE_OPCODE(INotEqual64, U1, U64, U64)
VALUE_OPCODE(SGreaterThanEqual32, U1, U32, U32)
VALUE_OPCODE(SGreaterThanEqual64, U1, U64, U64)
VALUE_OPCODE(UGreaterThanEqual32, U1, U32, U32)
VALUE_OPCODE(UGreaterThanEqual64, U1, U64, U64)
VALUE_OPCODE(LogicalOr, U1, U1, U1)
VALUE_OPCODE(LogicalAnd, U1, U1, U1)
VALUE_OPCODE(LogicalXor, U1, U1, U1)
VALUE_OPCODE(LogicalNot, U1, U1)
VALUE_OPCODE(FPOrdEqual32, U1, F32, F32)
VALUE_OPCODE(FPUnordEqual32, U1, F32, F32)
VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32)
VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32)
VALUE_OPCODE(FPOrdLessThan32, U1, F32, F32)
VALUE_OPCODE(FPUnordLessThan32, U1, F32, F32)
VALUE_OPCODE(FPOrdGreaterThan32, U1, F32, F32)
VALUE_OPCODE(FPUnordGreaterThan32, U1, F32, F32)
VALUE_OPCODE(FPOrdLessThanEqual32, U1, F32, F32)
VALUE_OPCODE(FPUnordLessThanEqual32, U1, F32, F32)
VALUE_OPCODE(FPOrdGreaterThanEqual32, U1, F32, F32)
VALUE_OPCODE(FPUnordGreaterThanEqual32, U1, F32, F32)
VALUE_OPCODE(FPIsNan32, U1, F32)
VALUE_OPCODE(FPCmpClass32, U1, F32, U32)
VALUE_OPCODE(FPAdd32, F32, F32, F32)
VALUE_OPCODE(FPSub32, F32, F32, F32)
VALUE_OPCODE(FPFma32, F32, F32, F32, F32)
VALUE_OPCODE(FPMul32, F32, F32, F32)
VALUE_OPCODE(FPMin32, F32, F32, F32)
VALUE_OPCODE(FPMax32, F32, F32, F32)
VALUE_OPCODE(FPMinTri32, F32, F32, F32, F32)
VALUE_OPCODE(FPMaxTri32, F32, F32, F32, F32)
VALUE_OPCODE(FPMedTri32, F32, F32, F32, F32)
VALUE_OPCODE(FPRecip32, F32, F32)
VALUE_OPCODE(FPRecipIFlag32, F32, F32)
VALUE_OPCODE(FPRecipSqrt32, F32, F32)
VALUE_OPCODE(FPSqrt, F32, F32)
VALUE_OPCODE(FPSin, F32, F32)
VALUE_OPCODE(FPCos, F32, F32)
VALUE_OPCODE(FPExp2, F32, F32)
VALUE_OPCODE(FPLog2, F32, F32)
VALUE_OPCODE(FPLdexp, F32, F32, U32)
VALUE_OPCODE(FPRoundEven32, F32, F32)
VALUE_OPCODE(FPFloor32, F32, F32)
VALUE_OPCODE(FPCeil32, F32, F32)
VALUE_OPCODE(FPTrunc32, F32, F32)
VALUE_OPCODE(FPFract32, F32, F32)
VALUE_OPCODE(LaneId, U32)
VALUE_OPCODE(Ballot, U32x4, U1)
VALUE_OPCODE(ReadFirstLane, U32, U32, U1)
VALUE_OPCODE(ReadLane, U32, U32, U32)
VALUE_OPCODE(WriteLane, U32, U32, U32, U32)
VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1)
// Typed shader resources and memory operations. Descriptor state is represented by SSA values;
// dense bindings are assigned by resource tracking.
VALUE_OPCODE(GetSrtResource, SrtResource)
VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32)
VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32)
VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32)
VALUE_OPCODE(GetSamplerResource, SamplerResource, U32, U32, U32, U32)
VALUE_OPCODE(GetLdsResource, LdsResource)
VALUE_OPCODE(GetGdsResource, GdsResource)
VALUE_OPCODE(MakeImageAddress, ImageAddress, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32,
U32, U32)
VALUE_OPCODE(ReadConst, U32, SrtResource, U32)
VALUE_OPCODE(ReadConstBuffer, U32, BufferResource, U32)
VALUE_OPCODE(LoadAddressU8, U8, AddressResource, U32, U32, U1)
VALUE_OPCODE(LoadAddressU16, U16, AddressResource, U32, U32, U1)
VALUE_OPCODE(LoadAddressU32, U32, AddressResource, U32, U32, U1)
VALUE_OPCODE(StoreAddressU8, Void, AddressResource, U32, U32, U8, U1)
VALUE_OPCODE(StoreAddressU16, Void, AddressResource, U32, U32, U16, U1)
VALUE_OPCODE(StoreAddressU32, Void, AddressResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU8, U8, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU16, U16, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(LoadBufferU32, U32, BufferResource, U32, U32, U32, U1)
VALUE_OPCODE(StoreBufferU8, Void, BufferResource, U32, U32, U32, U8, U1)
VALUE_OPCODE(StoreBufferU16, Void, BufferResource, U32, U32, U32, U16, U1)
VALUE_OPCODE(StoreBufferU32, Void, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicFMin32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(BufferAtomicFMax32, U32, BufferResource, U32, U32, U32, U32, U1)
VALUE_OPCODE(LoadSharedU8, U8, LdsResource, U32, U1)
VALUE_OPCODE(LoadSharedU16, U16, LdsResource, U32, U1)
VALUE_OPCODE(LoadSharedU32, U32, LdsResource, U32, U1)
VALUE_OPCODE(WriteSharedU8, Void, LdsResource, U32, U8, U1)
VALUE_OPCODE(WriteSharedU16, Void, LdsResource, U32, U16, U1)
VALUE_OPCODE(WriteSharedU32, Void, LdsResource, U32, U32, U1)
VALUE_OPCODE(LoadGdsU8, U8, GdsResource, U32, U1)
VALUE_OPCODE(LoadGdsU16, U16, GdsResource, U32, U1)
VALUE_OPCODE(LoadGdsU32, U32, GdsResource, U32, U1)
VALUE_OPCODE(WriteGdsU8, Void, GdsResource, U32, U8, U1)
VALUE_OPCODE(WriteGdsU16, Void, GdsResource, U32, U16, U1)
VALUE_OPCODE(WriteGdsU32, Void, GdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMin32, Void, LdsResource, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMax32, Void, LdsResource, U32, U32, U32, U1)
VALUE_OPCODE(GdsAtomicFMin32, Void, GdsResource, U32, U32, U32, U1)
VALUE_OPCODE(GdsAtomicFMax32, Void, GdsResource, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSwap32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicIAdd32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicISub32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSMin32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicUMin32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSMax32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicUMax32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicAnd32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicOr32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(SharedAtomicXor32, U32, LdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicSwap32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicIAdd32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicISub32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicSMin32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicUMin32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicSMax32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicUMax32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicAnd32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicOr32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(GdsAtomicXor32, U32, GdsResource, U32, U32, U1)
VALUE_OPCODE(DataAppend, U32, LdsResource, U32, U1, U32, U32)
VALUE_OPCODE(DataConsume, U32, LdsResource, U32, U1, U32, U32)
VALUE_OPCODE(GdsDataAppend, U32, GdsResource, U32, U1, U32, U32)
VALUE_OPCODE(GdsDataConsume, U32, GdsResource, U32, U1, U32, U32)
VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress)
VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1)
VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1)
VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress)
VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1)
VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1)
VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)
VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1)
VALUE_OPCODE(GetAttribute, U32, U32, U32)
VALUE_OPCODE(SetAttribute, Void, U32x4, U1)
VALUE_OPCODE(ControlNop, Void)
VALUE_OPCODE(Waitcnt, Void)
VALUE_OPCODE(Barrier, Void)
VALUE_OPCODE(Sendmsg, Void)
VALUE_OPCODE(TtraceData, Void)
VALUE_OPCODE(InstPrefetch, Void)
VALUE_OPCODE(SelectU32, U32, U1, U32, U32)
src/graphics/shader/recompiler/ir/BindingLayout.cpp → src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp
+29 -102
View File
@@ -1,6 +1,6 @@
#include "graphics/shader/recompiler/ir/BindingLayout.h"
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <array>
@@ -88,105 +88,21 @@ bool ImageBinding(const ImageResource& image, DescriptorBindingKind& result) {
}
}
bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>& visited,
std::set<uint32_t>& registers) {
if (id <= ScalarProvenance::Unknown) {
return true;
}
if (id >= provenance.values.size()) {
return false;
}
if (visited[id] != 0) {
return true;
}
visited[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::UserData) {
registers.insert(value.imm);
return true;
}
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (!CollectValue(provenance, arg, visited, registers)) {
return false;
}
}
return true;
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (!CollectValue(provenance, value.args[i], visited, registers)) {
return false;
}
}
return true;
}
bool CollectSource(const Program& program, uint32_t source, bool allow_unknown,
std::vector<uint8_t>& visited, std::set<uint32_t>& registers) {
if (allow_unknown && source == ScalarProvenance::Unknown) {
return true;
}
if (source < 2u || source - 2u >= program.provenance.descriptors.size()) {
return false;
}
const auto& descriptor = program.provenance.descriptors[source - 2u];
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (!CollectValue(program.provenance, descriptor.dwords[i], visited, registers)) {
return false;
}
}
return true;
}
bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
std::set<uint32_t> registers;
std::vector<uint8_t> visited(program.provenance.values.size());
for (const auto& buffer: program.info.buffers) {
if (!CollectSource(program, buffer.source, false, visited, registers)) {
return false;
}
}
for (const auto& image: program.info.images) {
if (!CollectSource(program, image.source, false, visited, registers)) {
return false;
}
}
for (const auto& sampler: program.info.samplers) {
if (!CollectSource(program, sampler.source, false, visited, registers)) {
return false;
}
}
for (const auto& address: program.info.addresses) {
if (!CollectSource(program, address.source, true, visited, registers)) {
return false;
}
}
for (const auto& read: program.srt.reads) {
if (!CollectValue(program.provenance, read.value, visited, registers)) {
return false;
}
}
for (const auto read: program.srt.dynamic_reads) {
if (!CollectValue(program.provenance, read, visited, registers)) {
return false;
}
}
for (const auto source: program.srt.dynamic_sources) {
if (!CollectSource(program, source, true, visited, registers)) {
return false;
}
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::SLoadDword && inst.memory.kind == ResourceKind::ScalarBuffer &&
!CollectValue(program.provenance, inst.scalar_value, visited, registers)) {
std::set<uint32_t> registers;
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) {
continue;
}
if (inst.Arg(0).GetType() != Type::ScalarReg) {
return false;
}
for (uint32_t i = 0; i < inst.src_count; i++) {
if (!CollectValue(program.provenance, inst.scalar_sources[i], visited, registers)) {
return false;
}
const auto index = RegIndex(inst.Arg(0).ScalarRegister());
if (index >= NumScalarRegs) {
return false;
}
registers.insert(index);
}
}
result.assign(registers.begin(), registers.end());
@@ -200,11 +116,16 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
}
bool UsesGds(const Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.memory.kind == ResourceKind::Gds) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == ValueOpcode::GetGdsResource && inst.HasUses()) {
return true;
}
for (size_t index = 0; index < inst.NumArgs(); index++) {
if (inst.Arg(index).GetType() == Type::GdsResource) {
return true;
}
}
}
}
return false;
@@ -232,13 +153,19 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std
}
return false;
}
if (program.values == nullptr) {
if (error != nullptr) {
*error = "typed value program is not available";
}
return false;
}
BindingLayout next;
next.descriptor_set = options.descriptor_set;
next.push_constant_offset = options.push_constant_offset;
if (!CollectUserData(program, next.user_data_registers)) {
if (error != nullptr) {
*error = "shader plan contains an invalid scalar provenance reference";
*error = "typed shader contains an invalid user-data register";
}
return false;
}
@@ -294,7 +221,7 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std
}
AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
}
if (!program.srt.reads.empty()) {
if (!program.values->srt_reads.empty()) {
AddBinding(next, DescriptorBindingKind::FlattenedSrt);
}
src/graphics/shader/recompiler/ir/BindingLayout.h → src/graphics/shader/recompiler/ir/passes/BindingLayout.h
View File
File renamed without changes.
src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp
+612
View File
@@ -0,0 +1,612 @@
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
#include <algorithm>
#include <bit>
#include <cstdint>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
Value Arg(const Inst& inst, size_t index) {
return inst.Arg(index).Resolve();
}
bool IsImmediate(Value value, Type type) {
return value.IsImmediate() && value.GetType() == type;
}
void Replace(Inst& inst, Value value) {
inst.ReplaceUsesWith(value.Resolve());
}
template <typename Function>
bool FoldU32(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U32) || !IsImmediate(rhs, Type::U32)) {
return false;
}
Replace(inst, Value(static_cast<uint32_t>(function(lhs.U32(), rhs.U32()))));
return true;
}
template <typename Function>
bool FoldU64(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U64) || !IsImmediate(rhs, Type::U64)) {
return false;
}
Replace(inst, Value(static_cast<uint64_t>(function(lhs.U64(), rhs.U64()))));
return true;
}
template <typename Function>
bool FoldU64Shift(Inst& inst, Function function) {
const auto value = Arg(inst, 0);
const auto shift = Arg(inst, 1);
if (!IsImmediate(value, Type::U64) || !IsImmediate(shift, Type::U32)) {
return false;
}
Replace(inst, Value(static_cast<uint64_t>(function(value.U64(), shift.U32()))));
return true;
}
template <typename Function>
bool FoldU32Compare(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U32) || !IsImmediate(rhs, Type::U32)) {
return false;
}
Replace(inst, Value(function(lhs.U32(), rhs.U32())));
return true;
}
template <typename Function>
bool FoldU64Compare(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U64) || !IsImmediate(rhs, Type::U64)) {
return false;
}
Replace(inst, Value(function(lhs.U64(), rhs.U64())));
return true;
}
template <typename Function>
bool FoldLogical(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U1) || !IsImmediate(rhs, Type::U1)) {
return false;
}
Replace(inst, Value(function(lhs.U1(), rhs.U1())));
return true;
}
bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, type) &&
(type == Type::U32 ? lhs.U32() == identity : lhs.U64() == identity)) {
Replace(inst, rhs);
return true;
}
if (IsImmediate(rhs, type) &&
(type == Type::U32 ? rhs.U32() == identity : rhs.U64() == identity)) {
Replace(inst, lhs);
return true;
}
return false;
}
bool FoldSelect(Inst& inst) {
const auto condition = Arg(inst, 0);
const auto true_value = Arg(inst, 1);
const auto false_value = Arg(inst, 2);
if (IsImmediate(condition, Type::U1)) {
Replace(inst, condition.U1() ? true_value : false_value);
return true;
}
if (true_value == false_value) {
Replace(inst, true_value);
return true;
}
return false;
}
bool FoldPhi(Inst& inst) {
Value same;
for (size_t index = 0; index < inst.NumArgs(); index++) {
const auto value = Arg(inst, index);
if (value.TryInstruction() == &inst) {
continue;
}
if (same.IsEmpty()) {
same = value;
} else if (same != value) {
return false;
}
}
if (same.IsEmpty()) {
return false;
}
Replace(inst, same);
return true;
}
bool FoldBitCast(Inst& inst, ValueOpcode reverse) {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::F32) && inst.GetOpcode() == ValueOpcode::BitCastU32F32) {
Replace(inst, Value(std::bit_cast<uint32_t>(value.F32Value())));
return true;
}
if (IsImmediate(value, Type::U32) && inst.GetOpcode() == ValueOpcode::BitCastF32U32) {
Replace(inst, Value::F32(std::bit_cast<float>(value.U32())));
return true;
}
if (IsImmediate(value, Type::F16) && inst.GetOpcode() == ValueOpcode::BitCastU16F16) {
Replace(inst, Value(value.F16Bits()));
return true;
}
if (IsImmediate(value, Type::U16) && inst.GetOpcode() == ValueOpcode::BitCastF16U16) {
Replace(inst, Value::F16(value.U16()));
return true;
}
if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == reverse) {
Replace(inst, producer->Arg(0));
return true;
}
return false;
}
bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components) {
const auto composite = Arg(inst, 0);
const auto index = Arg(inst, 1);
if (!IsImmediate(index, Type::U32) || index.U32() >= components) {
return false;
}
const auto component = index.U32();
const auto* producer = composite.TryInstruction();
if (producer != nullptr && producer->GetOpcode() == construct) {
Replace(inst, producer->Arg(component));
return true;
}
if (component < 2u && producer != nullptr &&
producer->GetOpcode() == ValueOpcode::IAddCarry32) {
const auto lhs = producer->Arg(0).Resolve();
const auto rhs = producer->Arg(1).Resolve();
if (IsImmediate(lhs, Type::U32) && IsImmediate(rhs, Type::U32)) {
const auto sum = static_cast<uint64_t>(lhs.U32()) + rhs.U32();
Replace(inst, Value(component == 0u ? static_cast<uint32_t>(sum)
: static_cast<uint32_t>(sum >> 32u)));
return true;
}
}
if (component < 2u && producer != nullptr &&
producer->GetOpcode() == ValueOpcode::UnpackUint2x32) {
const auto packed = producer->Arg(0).Resolve();
if (IsImmediate(packed, Type::U64)) {
Replace(inst, Value(static_cast<uint32_t>(packed.U64() >> (component * 32u))));
return true;
}
}
return false;
}
void FoldInstruction(Inst& inst) {
switch (inst.GetOpcode()) {
case ValueOpcode::Phi: FoldPhi(inst); return;
case ValueOpcode::SelectU1:
case ValueOpcode::SelectU32:
case ValueOpcode::SelectF32: FoldSelect(inst); return;
case ValueOpcode::BitFieldInsert: {
const auto base = Arg(inst, 0);
const auto insert = Arg(inst, 1);
const auto offset = Arg(inst, 2);
const auto count = Arg(inst, 3);
if (IsImmediate(base, Type::U32) && IsImmediate(insert, Type::U32) &&
IsImmediate(offset, Type::U32) && IsImmediate(count, Type::U32) &&
offset.U32() <= 32u && count.U32() <= 32u - offset.U32()) {
if (count.U32() == 0u) {
Replace(inst, base);
return;
}
const auto mask = count.U32() == 32u
? UINT32_MAX
: ((uint32_t {1} << count.U32()) - 1u) << offset.U32();
Replace(inst,
Value((base.U32() & ~mask) | ((insert.U32() << offset.U32()) & mask)));
}
return;
}
case ValueOpcode::BitFieldUExtract:
case ValueOpcode::BitFieldSExtract: {
const auto value = Arg(inst, 0);
const auto offset = Arg(inst, 1);
const auto count = Arg(inst, 2);
if (!IsImmediate(value, Type::U32) || !IsImmediate(offset, Type::U32) ||
!IsImmediate(count, Type::U32) || offset.U32() > 32u ||
count.U32() > 32u - offset.U32()) {
return;
}
if (count.U32() == 0u) {
Replace(inst, Value(0u));
} else if (inst.GetOpcode() == ValueOpcode::BitFieldUExtract) {
const auto mask =
count.U32() == 32u ? UINT32_MAX : (uint32_t {1} << count.U32()) - 1u;
Replace(inst, Value((value.U32() >> offset.U32()) & mask));
} else {
const auto left = 32u - offset.U32() - count.U32();
const auto bits = value.U32() << left;
Replace(inst, Value(static_cast<uint32_t>(std::bit_cast<int32_t>(bits) >>
(left + offset.U32()))));
}
return;
}
case ValueOpcode::BitCastU16F16: FoldBitCast(inst, ValueOpcode::BitCastF16U16); return;
case ValueOpcode::BitCastF16U16: FoldBitCast(inst, ValueOpcode::BitCastU16F16); return;
case ValueOpcode::BitCastU32F32: FoldBitCast(inst, ValueOpcode::BitCastF32U32); return;
case ValueOpcode::BitCastF32U32: FoldBitCast(inst, ValueOpcode::BitCastU32F32); return;
case ValueOpcode::ConvertU16U32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(static_cast<uint16_t>(value.U32())));
}
return;
}
case ValueOpcode::ConvertU32U16: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U16)) {
Replace(inst, Value(static_cast<uint32_t>(value.U16())));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::ConvertU16U32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::ConvertU8U32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(static_cast<uint8_t>(value.U32())));
}
return;
}
case ValueOpcode::ConvertU32U8: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U8)) {
Replace(inst, Value(static_cast<uint32_t>(value.U8())));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::ConvertU8U32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::CompositeExtractU32x2:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2);
return;
case ValueOpcode::CompositeExtractU32x4:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4);
return;
case ValueOpcode::PackUint2x32: {
const auto value = Arg(inst, 0);
const auto* producer = value.TryInstruction();
if (producer != nullptr && producer->GetOpcode() == ValueOpcode::UnpackUint2x32) {
Replace(inst, producer->Arg(0));
} else if (producer != nullptr &&
producer->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
const auto low = producer->Arg(0).Resolve();
const auto high = producer->Arg(1).Resolve();
if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) {
Replace(inst, Value(static_cast<uint64_t>(low.U32()) |
(static_cast<uint64_t>(high.U32()) << 32u)));
}
}
return;
}
case ValueOpcode::UnpackUint2x32: {
const auto value = Arg(inst, 0);
const auto* producer = value.TryInstruction();
if (producer != nullptr && producer->GetOpcode() == ValueOpcode::PackUint2x32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::IAdd32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a + b; })) {
ReplaceBinaryIdentity(inst, Type::U32, 0u);
}
return;
case ValueOpcode::IAdd64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a + b; })) {
ReplaceBinaryIdentity(inst, Type::U64, 0u);
}
return;
case ValueOpcode::ISub32: {
if (FoldU32(inst, [](uint32_t a, uint32_t b) { return a - b; })) {
return;
}
const auto rhs = Arg(inst, 1);
if (IsImmediate(rhs, Type::U32) && rhs.U32() == 0u) {
Replace(inst, Arg(inst, 0));
}
return;
}
case ValueOpcode::ISub64: {
if (FoldU64(inst, [](uint64_t a, uint64_t b) { return a - b; })) {
return;
}
const auto rhs = Arg(inst, 1);
if (IsImmediate(rhs, Type::U64) && rhs.U64() == 0u) {
Replace(inst, Arg(inst, 0));
}
return;
}
case ValueOpcode::IMul32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a * b; })) {
ReplaceBinaryIdentity(inst, Type::U32, 1u);
}
return;
case ValueOpcode::IMul64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a * b; })) {
ReplaceBinaryIdentity(inst, Type::U64, 1u);
}
return;
case ValueOpcode::UDiv32: {
const auto rhs = Arg(inst, 1);
if (IsImmediate(rhs, Type::U32) && rhs.U32() == 1u) {
Replace(inst, Arg(inst, 0));
} else if (IsImmediate(rhs, Type::U32) && rhs.U32() != 0u) {
FoldU32(inst, [](uint32_t a, uint32_t b) { return a / b; });
}
return;
}
case ValueOpcode::SMulHi:
FoldU32(inst, [](uint32_t a, uint32_t b) {
const auto product = static_cast<int64_t>(std::bit_cast<int32_t>(a)) *
static_cast<int64_t>(std::bit_cast<int32_t>(b));
return static_cast<uint32_t>(static_cast<uint64_t>(product) >> 32u);
});
return;
case ValueOpcode::UMulHi:
FoldU32(inst, [](uint32_t a, uint32_t b) {
return static_cast<uint32_t>((static_cast<uint64_t>(a) * b) >> 32u);
});
return;
case ValueOpcode::IAbs32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst,
Value((value.U32() & 0x80000000u) != 0u ? 0u - value.U32() : value.U32()));
}
return;
}
case ValueOpcode::ShiftLeftLogical32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a << (b & 31u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightLogical32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a >> (b & 31u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightArithmetic32:
FoldU32(inst, [](uint32_t a, uint32_t b) {
return static_cast<uint32_t>(std::bit_cast<int32_t>(a) >> (b & 31u));
});
return;
case ValueOpcode::ShiftLeftLogical64:
if (!FoldU64Shift(inst, [](uint64_t a, uint32_t b) { return a << (b & 63u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 63u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightLogical64:
if (!FoldU64Shift(inst, [](uint64_t a, uint32_t b) { return a >> (b & 63u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 63u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightArithmetic64:
FoldU64Shift(inst, [](uint64_t a, uint32_t b) {
return static_cast<uint64_t>(std::bit_cast<int64_t>(a) >> (b & 63u));
});
return;
case ValueOpcode::BitwiseAnd32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a & b; })) {
ReplaceBinaryIdentity(inst, Type::U32, 0xffffffffu);
}
return;
case ValueOpcode::BitwiseAnd64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a & b; })) {
ReplaceBinaryIdentity(inst, Type::U64, UINT64_MAX);
}
return;
case ValueOpcode::BitwiseOr32:
case ValueOpcode::BitwiseXor32:
if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b;
})) {
ReplaceBinaryIdentity(inst, Type::U32, 0u);
}
return;
case ValueOpcode::BitwiseOr64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a | b; })) {
ReplaceBinaryIdentity(inst, Type::U64, 0u);
}
return;
case ValueOpcode::BitwiseNot32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(~value.U32()));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::BitwiseNot32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::BitCount32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(static_cast<uint32_t>(std::popcount(value.U32()))));
}
return;
}
case ValueOpcode::BitCount64: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U64)) {
Replace(inst, Value(static_cast<uint32_t>(std::popcount(value.U64()))));
}
return;
}
case ValueOpcode::SMin32:
case ValueOpcode::SMax32:
FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
const auto lhs = std::bit_cast<int32_t>(a);
const auto rhs = std::bit_cast<int32_t>(b);
return opcode == ValueOpcode::SMin32 ? (lhs < rhs ? a : b) : (lhs > rhs ? a : b);
});
return;
case ValueOpcode::UMin32:
FoldU32(inst, [](uint32_t a, uint32_t b) { return std::min(a, b); });
return;
case ValueOpcode::UMax32:
FoldU32(inst, [](uint32_t a, uint32_t b) { return std::max(a, b); });
return;
case ValueOpcode::IEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a == b; });
return;
case ValueOpcode::INotEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a != b; });
return;
case ValueOpcode::ULessThan32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a < b; });
return;
case ValueOpcode::ULessThanEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a <= b; });
return;
case ValueOpcode::UGreaterThan32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a > b; });
return;
case ValueOpcode::UGreaterThanEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a >= b; });
return;
case ValueOpcode::SLessThan32:
case ValueOpcode::SLessThanEqual32:
case ValueOpcode::SGreaterThan32:
case ValueOpcode::SGreaterThanEqual32:
FoldU32Compare(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
const auto lhs = std::bit_cast<int32_t>(a);
const auto rhs = std::bit_cast<int32_t>(b);
switch (opcode) {
case ValueOpcode::SLessThan32: return lhs < rhs;
case ValueOpcode::SLessThanEqual32: return lhs <= rhs;
case ValueOpcode::SGreaterThan32: return lhs > rhs;
default: return lhs >= rhs;
}
});
return;
case ValueOpcode::IEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a == b; });
return;
case ValueOpcode::INotEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a != b; });
return;
case ValueOpcode::ULessThan64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a < b; });
return;
case ValueOpcode::ULessThanEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; });
return;
case ValueOpcode::UGreaterThan64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; });
return;
case ValueOpcode::UGreaterThanEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; });
return;
case ValueOpcode::SLessThan64:
case ValueOpcode::SLessThanEqual64:
case ValueOpcode::SGreaterThan64:
case ValueOpcode::SGreaterThanEqual64:
FoldU64Compare(inst, [opcode = inst.GetOpcode()](uint64_t a, uint64_t b) {
const auto lhs = std::bit_cast<int64_t>(a);
const auto rhs = std::bit_cast<int64_t>(b);
switch (opcode) {
case ValueOpcode::SLessThan64: return lhs < rhs;
case ValueOpcode::SLessThanEqual64: return lhs <= rhs;
case ValueOpcode::SGreaterThan64: return lhs > rhs;
default: return lhs >= rhs;
}
});
return;
case ValueOpcode::LogicalAnd:
if (!FoldLogical(inst, [](bool a, bool b) { return a && b; })) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, Type::U1)) {
Replace(inst, lhs.U1() ? rhs : lhs);
} else if (IsImmediate(rhs, Type::U1)) {
Replace(inst, rhs.U1() ? lhs : rhs);
}
}
return;
case ValueOpcode::LogicalOr:
if (!FoldLogical(inst, [](bool a, bool b) { return a || b; })) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, Type::U1)) {
Replace(inst, lhs.U1() ? lhs : rhs);
} else if (IsImmediate(rhs, Type::U1)) {
Replace(inst, rhs.U1() ? rhs : lhs);
}
}
return;
case ValueOpcode::LogicalXor:
if (!FoldLogical(inst, [](bool a, bool b) { return a != b; })) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, Type::U1) && !lhs.U1()) {
Replace(inst, rhs);
} else if (IsImmediate(rhs, Type::U1) && !rhs.U1()) {
Replace(inst, lhs);
}
}
return;
case ValueOpcode::LogicalNot: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U1)) {
Replace(inst, Value(!value.U1()));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::LogicalNot) {
Replace(inst, producer->Arg(0));
}
return;
}
default: return;
}
}
} // namespace
void ConstantPropagationPass(const BlockList& blocks) {
for (auto* block: blocks) {
for (auto& inst: block->Instructions()) {
FoldInstruction(inst);
}
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h
+9
View File
@@ -0,0 +1,9 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void ConstantPropagationPass(const BlockList& blocks);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp
+40
View File
@@ -0,0 +1,40 @@
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void RemoveIdentities(const BlockList& blocks) {
for (auto* block: blocks) {
auto& instructions = block->Instructions();
for (auto inst = instructions.begin(); inst != instructions.end();) {
if (inst->GetOpcode() != ValueOpcode::Identity) {
inst++;
continue;
}
const auto replacement = inst->Arg(0);
inst->ReplaceUsesWith(replacement, false);
inst = instructions.erase(inst);
}
}
}
void EliminateDeadCode(const BlockList& blocks) {
bool changed;
do {
changed = false;
for (auto block = blocks.rbegin(); block != blocks.rend(); block++) {
auto& instructions = (*block)->Instructions();
auto inst = instructions.end();
while (inst != instructions.begin()) {
--inst;
if (inst->HasUses() || inst->MayHaveSideEffects()) {
continue;
}
inst->Invalidate();
inst = instructions.erase(inst);
changed = true;
}
}
} while (changed);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.h
+10
View File
@@ -0,0 +1,10 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void RemoveIdentities(const BlockList& blocks);
void EliminateDeadCode(const BlockList& blocks);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp
+141
View File
@@ -0,0 +1,141 @@
#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
#include <algorithm>
#include <queue>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
struct ChainResult {
Value value;
Inst* write = nullptr;
};
ChainResult SearchChain(Value value, uint32_t lane, uint32_t wave_size) {
for (;;) {
value = value.Resolve();
auto* inst = value.TryInstruction();
if (inst == nullptr || inst->GetOpcode() != ValueOpcode::WriteLane) {
return {value};
}
const auto selector = inst->Arg(2).Resolve();
if (!selector.IsImmediate() || selector.GetType() != Type::U32) {
return {value};
}
if (selector.U32() % wave_size == lane) {
return {value, inst};
}
value = inst->Arg(0);
}
}
bool IsPossibleToEliminate(Value source, uint32_t lane, uint32_t wave_size) {
std::queue<Value> queue;
std::unordered_set<Inst*> visited;
queue.push(source);
while (!queue.empty()) {
const auto chain = SearchChain(queue.front(), lane, wave_size);
queue.pop();
if (chain.write != nullptr) {
continue;
}
auto* inst = chain.value.TryInstruction();
if (inst == nullptr || inst->GetOpcode() != ValueOpcode::Phi || inst->NumArgs() == 0) {
return false;
}
if (!visited.insert(inst).second) {
continue;
}
for (size_t index = inst->NumArgs(); index-- > 0;) {
queue.push(inst->Arg(index));
}
}
return true;
}
using PhiMap = std::unordered_map<Inst*, Inst*>;
Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_size) {
const auto chain = SearchChain(source, lane, wave_size);
if (chain.write != nullptr) {
return chain.write->Arg(1);
}
auto* inst = chain.value.ResolveInstruction();
EXIT_IF(inst->GetOpcode() != ValueOpcode::Phi);
const auto [entry, is_new] = phi_map.try_emplace(inst);
if (!is_new) {
return Value(entry->second);
}
auto* block = inst->Parent();
auto insertion_point = std::find_if(block->begin(), block->end(),
[&](const Inst& candidate) { return &candidate == inst; });
EXIT_IF(insertion_point == block->end());
auto& phi = *block->PrependNewInst(insertion_point, ValueOpcode::Phi);
phi.SetFlags(Type::U32);
entry->second = &phi;
std::vector<Value> arguments;
arguments.reserve(inst->NumArgs());
for (size_t index = 0; index < inst->NumArgs(); index++) {
arguments.push_back(GetRealValue(phi_map, inst->Arg(index), lane, wave_size));
}
const auto first = arguments.front().Resolve();
if (std::ranges::all_of(arguments,
[&](Value argument) { return argument.Resolve() == first; })) {
phi.ReplaceUsesWith(first);
} else {
for (size_t index = 0; index < arguments.size(); index++) {
phi.AddPhiOperand(inst->PhiBlock(index), arguments[index]);
}
}
return Value(&phi);
}
} // namespace
ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size) {
ReadLaneStats stats;
if (wave_size != 32u && wave_size != 64u) {
return stats;
}
for (auto* block: program.blocks) {
for (auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::ReadLane) {
continue;
}
const auto selector = inst.Arg(1).Resolve();
if (!selector.IsImmediate() || selector.GetType() != Type::U32) {
continue;
}
const auto lane = selector.U32() % wave_size;
const auto chain = SearchChain(inst.Arg(0), lane, wave_size);
if (chain.write != nullptr) {
inst.ReplaceUsesWith(chain.write->Arg(1));
stats.rewritten_reads++;
continue;
}
auto* producer = chain.value.TryInstruction();
if (producer == nullptr || producer->GetOpcode() != ValueOpcode::Phi ||
!IsPossibleToEliminate(chain.value, lane, wave_size)) {
continue;
}
PhiMap phi_map;
inst.ReplaceUsesWith(GetRealValue(phi_map, chain.value, lane, wave_size));
stats.rewritten_reads++;
}
}
return stats;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include "graphics/shader/recompiler/ir/ValueProgram.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct ReadLaneStats {
uint32_t rewritten_reads = 0;
};
[[nodiscard]] ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp → src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
+53 -29
View File
@@ -1,6 +1,7 @@
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include "graphics/shader/shaderBindings.h"
#include <algorithm>
@@ -81,11 +82,29 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou
uint64_t AddressSpecialization(const AddressResource& resource,
const ResourceSnapshot::Address& snapshot) {
return resource.kind == ResourceKind::Flat || resource.source == ScalarProvenance::Unknown
return resource.kind == ResourceKind::Flat || resource.unbased
? snapshot.binding_base
: snapshot.guest_base - snapshot.binding_base;
}
bool IsImageOp(ValueOpcode op) {
switch (op) {
case ValueOpcode::ImageQueryDimensions:
case ValueOpcode::ImageQueryLod:
case ValueOpcode::ImageRead:
case ValueOpcode::ImageWrite:
case ValueOpcode::ImageSampleRaw:
case ValueOpcode::ImageGatherRaw:
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
case ValueOpcode::ImageAtomicAnd32:
case ValueOpcode::ImageAtomicOr32:
case ValueOpcode::ImageAtomicXor32: return true;
default: return false;
}
}
} // namespace
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot,
@@ -105,7 +124,8 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
}
return false;
}
if (snapshot.flattened_srt.size() != program.srt.reads.size()) {
if (program.values == nullptr ||
snapshot.flattened_srt.size() != program.values->srt_reads.size()) {
if (error != nullptr) {
*error = "flattened SRT snapshot does not match the shader plan";
}
@@ -280,7 +300,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
requests.push_back({sampler.source, sampler.first_use_pc});
}
for (const auto& address: program.info.addresses) {
if (address.source != ScalarProvenance::Unknown) {
if (!address.unbased) {
requests.push_back({address.source, address.first_use_pc});
}
}
@@ -311,7 +331,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
next.samplers.assign(cursor, cursor + program.info.samplers.size());
cursor += program.info.samplers.size();
for (const auto& address: program.info.addresses) {
if (address.source != ScalarProvenance::Unknown) {
if (!address.unbased) {
const auto value = *cursor++;
auto base = (static_cast<uint64_t>(value.dwords[0]) |
static_cast<uint64_t>(value.dwords[1]) << 32u) &
@@ -430,38 +450,42 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
}
}
}
struct ImagePatch {
std::reference_wrapper<Instruction> inst;
ResourceKind kind;
Decoder::ImageDimension dimension;
bool cube;
};
std::vector<ImagePatch> patches;
for (auto& block: program.blocks) {
for (auto& inst: block.instructions) {
if (inst.memory.kind != ResourceKind::Image &&
inst.memory.kind != ResourceKind::ImageUint &&
inst.memory.kind != ResourceKind::StorageImage &&
inst.memory.kind != ResourceKind::StorageImageUint) {
if (program.values == nullptr) {
if (error != nullptr) {
*error = "typed SSA is not ready";
}
return false;
}
auto memory_info = program.values->memory_info;
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (!IsImageOp(inst.GetOpcode())) {
continue;
}
if (inst.memory.resource >= next.images.size()) {
const auto index = inst.Flags<MemoryFlags>().index;
if (index >= memory_info.size()) {
if (error != nullptr) {
*error = fmt::format("image instruction at pc 0x{:08x} has invalid resource {}",
inst.pc, inst.memory.resource);
*error = fmt::format("typed image instruction has invalid memory metadata {}",
index);
}
return false;
}
const auto& image = next.images[inst.memory.resource];
patches.push_back({std::ref(inst), image.kind, image.dimension, image.cube});
auto& memory = memory_info[index];
if (memory.resource >= next.images.size()) {
if (error != nullptr) {
*error = fmt::format("typed image instruction has invalid resource {}",
memory.resource);
}
return false;
}
const auto& image = next.images[memory.resource];
memory.kind = image.kind;
memory.image_dimension = image.dimension;
memory.image_cube = image.cube;
}
}
program.info = std::move(next);
for (const auto& patch: patches) {
patch.inst.get().memory.kind = patch.kind;
patch.inst.get().memory.image_dimension = patch.dimension;
patch.inst.get().memory.image_cube = patch.cube;
}
program.info = std::move(next);
program.values->memory_info = std::move(memory_info);
return true;
}
src/graphics/shader/recompiler/ir/ResourceMaterialization.h → src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_
#include "graphics/shader/recompiler/ir/SrtWalker.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+677
View File
@@ -0,0 +1,677 @@
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include <algorithm>
#include <fmt/format.h>
#include <unordered_set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
bool IsBufferAtomic(ValueOpcode op) {
switch (op) {
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
case ValueOpcode::BufferAtomicSMin32:
case ValueOpcode::BufferAtomicUMin32:
case ValueOpcode::BufferAtomicSMax32:
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32: return true;
default: return false;
}
}
bool IsBufferStore(ValueOpcode op) {
switch (op) {
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return true;
default: return IsBufferAtomic(op);
}
}
bool IsBuffer(ValueOpcode op) {
switch (op) {
case ValueOpcode::ReadConstBuffer:
case ValueOpcode::LoadBufferU8:
case ValueOpcode::LoadBufferU16:
case ValueOpcode::LoadBufferU32:
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return true;
default: return IsBufferAtomic(op);
}
}
bool IsAddressStore(ValueOpcode op) {
switch (op) {
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32: return true;
default: return false;
}
}
bool IsAddress(ValueOpcode op) {
switch (op) {
case ValueOpcode::LoadAddressU8:
case ValueOpcode::LoadAddressU16:
case ValueOpcode::LoadAddressU32:
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32: return true;
default: return false;
}
}
bool IsImageAtomic(ValueOpcode op) {
switch (op) {
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
case ValueOpcode::ImageAtomicAnd32:
case ValueOpcode::ImageAtomicOr32:
case ValueOpcode::ImageAtomicXor32: return true;
default: return false;
}
}
bool IsImageStore(ValueOpcode op) {
return op == ValueOpcode::ImageWrite || IsImageAtomic(op);
}
bool IsImage(ValueOpcode op) {
switch (op) {
case ValueOpcode::ImageQueryDimensions:
case ValueOpcode::ImageQueryLod:
case ValueOpcode::ImageRead:
case ValueOpcode::ImageWrite:
case ValueOpcode::ImageSampleRaw:
case ValueOpcode::ImageGatherRaw: return true;
default: return IsImageAtomic(op);
}
}
bool NeedsSampler(ValueOpcode op) {
return op == ValueOpcode::ImageQueryLod || op == ValueOpcode::ImageSampleRaw ||
op == ValueOpcode::ImageGatherRaw;
}
uint32_t ByteExtent(const MemoryInfo& memory) {
const auto bytes = std::max((memory.data_bits + 7u) / 8u, 1u);
const auto count = std::max(memory.data_dwords, 1u);
const auto end = static_cast<uint64_t>(memory.offset) + static_cast<uint64_t>(bytes) * count;
return end > UINT32_MAX ? UINT32_MAX : static_cast<uint32_t>(end);
}
ImageMipMode MipMode(const MemoryInfo& memory) {
const bool storage =
memory.kind == ResourceKind::StorageImage || memory.kind == ResourceKind::StorageImageUint;
return storage && memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
}
class Tracker {
public:
Tracker(Program& program, ValueProgram& values)
: m_program(program), m_values(values), m_info(program.info) {
m_info.buffers.clear();
m_info.addresses.clear();
m_info.images.clear();
m_info.samplers.clear();
m_info.sampled_pairs.clear();
}
bool Run(std::string* error) {
if (m_program.resource_tracking_complete) {
return Fail(0, error, "resources already tracked");
}
if (!m_program.srt_plan_complete) {
return Fail(0, error, "SRT plan is not ready");
}
for (auto* block: m_values.blocks) {
for (auto& inst: *block) {
if (!Collect(inst, error)) {
return false;
}
}
}
LinkImageAliases();
for (const auto& patch: m_handle_patches) {
patch.handle->SetFlags<uint32_t>(patch.resource);
}
for (const auto& patch: m_memory_patches) {
auto& memory = m_values.memory_info[patch.index];
memory.resource = patch.resource;
if (patch.has_sampler) {
memory.sampler = patch.sampler;
}
}
m_values.descriptor_sources = std::move(m_sources);
m_program.info = std::move(m_info);
m_program.resource_tracking_complete = true;
return true;
}
private:
struct HandlePatch {
Inst* handle = nullptr;
uint32_t resource = 0;
};
struct MemoryPatch {
uint32_t index = 0;
uint32_t resource = 0;
uint32_t sampler = 0;
bool has_sampler = false;
};
bool Fail(uint32_t pc, std::string* error, const std::string& reason) const {
if (error != nullptr) {
*error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}",
m_program.shader_hash, StageName(m_program.stage), pc, reason);
}
return false;
}
struct AddressPart {
Value value;
bool rooted = false;
};
AddressPart FindAddressPart(Value value, std::unordered_set<const Inst*>& visiting) const {
value = value.Resolve();
if (value.IsImmediate()) {
return {value, false};
}
const auto* inst = value.TryInstruction();
if (inst == nullptr || !visiting.insert(inst).second) {
return {};
}
const auto finish = [&](AddressPart part) {
visiting.erase(inst);
return part;
};
std::string reason;
if (ValidateRuntimeValue(m_program, value, reason)) {
return finish({value, true});
}
const auto merge = [&](AddressPart left, AddressPart right, bool require_both) {
if (left.value.IsEmpty() || right.value.IsEmpty()) {
return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left);
}
return EquivalentValue(m_values, left.value, right.value)
? AddressPart {left.value, left.rooted || right.rooted}
: AddressPart {};
};
switch (inst->GetOpcode()) {
case ValueOpcode::SelectU32:
return finish(merge(FindAddressPart(inst->Arg(1), visiting),
FindAddressPart(inst->Arg(2), visiting), true));
case ValueOpcode::Phi: {
const auto invariant = ResolveInvariantPhi(m_values, value);
return finish(invariant.IsEmpty() ? AddressPart {}
: FindAddressPart(invariant, visiting));
}
case ValueOpcode::IAdd32:
case ValueOpcode::ISub32: {
const auto left = FindAddressPart(inst->Arg(0), visiting);
const auto right = FindAddressPart(inst->Arg(1), visiting);
if (left.rooted == right.rooted ||
(inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) {
return finish({});
}
return finish(left.rooted ? left : right);
}
case ValueOpcode::CompositeExtractU32x2: {
const auto* source = inst->Arg(0).ResolveInstruction();
if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) {
return finish({});
}
const auto left = FindAddressPart(source->Arg(0), visiting);
const auto right = FindAddressPart(source->Arg(1), visiting);
if (left.rooted == right.rooted) {
return finish({});
}
return finish(left.rooted ? left : right);
}
default: return finish({});
}
}
bool MakeFlatAddressSource(const Inst& handle, DescriptorSource& descriptor) const {
std::unordered_set<const Inst*> visiting;
auto low = FindAddressPart(handle.Arg(0), visiting);
visiting.clear();
auto high = FindAddressPart(handle.Arg(1), visiting);
if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) {
return false;
}
descriptor.dword_count = 2;
descriptor.dwords[0] = low.value;
descriptor.dwords[1] = high.value;
return true;
}
bool MakeSource(const Inst& handle, uint32_t width, bool sampler, DescriptorSource& descriptor,
uint32_t pc, std::string* error) const {
if (handle.NumArgs() != width) {
return Fail(pc, error,
fmt::format("{} has {} descriptor dwords, expected {}",
ValueOpcodeName(handle.GetOpcode()), handle.NumArgs(), width));
}
descriptor.dword_count = width;
for (uint32_t i = 0; i < width; i++) {
descriptor.dwords[i] = handle.Arg(i).Resolve();
}
const auto dword0 = descriptor.dwords[0].Resolve();
if (sampler && dword0.IsImmediate() && dword0.GetType() == Type::U32 &&
(dword0.U32() & SamplerBorderClampMask) == 0) {
// Border color and its table index are unused unless a clamp axis selects border mode.
descriptor.dwords[3] = Value(0u);
}
return true;
}
bool ValidateSource(const DescriptorSource& descriptor, std::string& reason,
uint32_t& bad_dword) const {
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
bad_dword = i;
if (descriptor.dwords[i].Resolve().GetType() != Type::U32) {
reason = "is not U32";
return false;
}
if (!ValidateRuntimeValue(m_program, descriptor.dwords[i], reason)) {
return false;
}
}
return true;
}
uint32_t InternSource(const DescriptorSource& descriptor) {
for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
const auto& current = m_sources[candidate];
if (current.dword_count != descriptor.dword_count) {
continue;
}
bool same = true;
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
same = same && EquivalentValue(m_values, current.dwords[i], descriptor.dwords[i]);
}
if (same) {
return candidate;
}
}
m_sources.push_back(descriptor);
return static_cast<uint32_t>(m_sources.size() - 1);
}
bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle,
uint32_t& source, std::string* error, bool sampler = false) {
handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != expected) {
return Fail(pc, error,
fmt::format("memory operation requires {}", ValueOpcodeName(expected)));
}
DescriptorSource descriptor;
if (!MakeSource(*handle, width, sampler, descriptor, pc, error)) {
return false;
}
std::string reason;
uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) {
return Fail(
pc, error,
fmt::format("{} dword {} {}", ValueOpcodeName(expected), bad_dword, reason));
}
source = InternSource(descriptor);
return true;
}
bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle,
uint32_t& source, bool& unbased, std::string* error) {
handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) {
return Fail(pc, error, "address operation requires GetAddressResource");
}
if (handle->NumArgs() != 2) {
return Fail(pc, error, "GetAddressResource must have two address dwords");
}
DescriptorSource descriptor;
const auto& memory = m_values.memory_info[memory_inst.Flags<MemoryFlags>().index];
if (memory.address_is_full) {
if (memory.kind != ResourceKind::Flat || !MakeFlatAddressSource(*handle, descriptor)) {
unbased = true;
source = UINT32_MAX;
return true;
}
} else if (!MakeSource(*handle, 2, false, descriptor, pc, error)) {
return false;
}
std::string reason;
uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) {
if (memory.address_is_full) {
unbased = true;
source = UINT32_MAX;
return true;
}
return Fail(
pc, error,
fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason));
}
unbased = false;
source = InternSource(descriptor);
return true;
}
uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
for (uint32_t i = 0; i < m_info.buffers.size(); i++) {
if (m_info.buffers[i].source == source) {
Merge(m_info.buffers[i], memory, op, pc);
return i;
}
}
if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) {
return UINT32_MAX;
}
BufferResource resource;
resource.source = source;
resource.first_use_pc = pc;
Merge(resource, memory, op, pc);
m_info.buffers.push_back(resource);
return static_cast<uint32_t>(m_info.buffers.size() - 1);
}
static void Merge(BufferResource& resource, const MemoryInfo& memory, ValueOpcode op,
uint32_t pc) {
const bool atomic = IsBufferAtomic(op);
const bool write = IsBufferStore(op);
resource.first_use_pc = std::min(resource.first_use_pc, pc);
resource.max_byte_extent = std::max(resource.max_byte_extent, ByteExtent(memory));
resource.read = resource.read || !write || atomic;
resource.written = resource.written || write;
resource.atomic = resource.atomic || atomic;
resource.formatted = resource.formatted || memory.formatted;
resource.scalar = resource.scalar || op == ValueOpcode::ReadConstBuffer ||
memory.kind == ResourceKind::ScalarBuffer;
}
uint32_t AddImage(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
const auto mip = MipMode(memory);
const bool depth = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0;
for (uint32_t i = 0; i < m_info.images.size(); i++) {
auto& image = m_info.images[i];
if (image.source == source && image.kind == memory.kind &&
image.dimension == memory.image_dimension && image.mip_mode == mip &&
image.depth_compare == depth) {
Merge(image, op, pc);
return i;
}
}
if (m_info.images.size() >= ShaderInfo::MaxImages) {
return UINT32_MAX;
}
ImageResource image;
image.source = source;
image.first_use_pc = pc;
image.kind = memory.kind;
image.dimension = memory.image_dimension;
image.mip_mode = mip;
image.depth_compare = depth;
Merge(image, op, pc);
m_info.images.push_back(image);
return static_cast<uint32_t>(m_info.images.size() - 1);
}
static void Merge(ImageResource& image, ValueOpcode op, uint32_t pc) {
const bool atomic = IsImageAtomic(op);
const bool write = IsImageStore(op);
image.first_use_pc = std::min(image.first_use_pc, pc);
image.read = image.read || !write || atomic;
image.written = image.written || write;
image.atomic = image.atomic || atomic;
}
uint32_t AddSampler(uint32_t source, uint32_t pc) {
for (uint32_t i = 0; i < m_info.samplers.size(); i++) {
if (m_info.samplers[i].source == source) {
m_info.samplers[i].first_use_pc = std::min(m_info.samplers[i].first_use_pc, pc);
return i;
}
}
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
return UINT32_MAX;
}
m_info.samplers.push_back({source, pc});
return static_cast<uint32_t>(m_info.samplers.size() - 1);
}
uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory, ValueOpcode op,
uint32_t pc) {
auto immediate = static_cast<int32_t>(memory.offset);
if (memory.kind == ResourceKind::ScalarBuffer) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto min_offset = unbased ? 0 : std::min(immediate, 0);
for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
auto& address = m_info.addresses[i];
if (address.source == source && address.unbased == unbased &&
address.kind == memory.kind) {
address.first_use_pc = std::min(address.first_use_pc, pc);
address.min_offset = std::min(address.min_offset, min_offset);
address.read = address.read || !IsAddressStore(op);
address.written = address.written || IsAddressStore(op);
return i;
}
}
if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
return UINT32_MAX;
}
AddressResource address;
address.source = source;
address.first_use_pc = pc;
address.kind = memory.kind;
address.min_offset = min_offset;
address.unbased = unbased;
address.read = !IsAddressStore(op);
address.written = IsAddressStore(op);
m_info.addresses.push_back(address);
return static_cast<uint32_t>(m_info.addresses.size() - 1);
}
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) {
pair.first_use_pc = std::min(pair.first_use_pc, pc);
return true;
}
}
if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) {
return Fail(pc, error, "sampled image/sampler pair limit exceeded");
}
m_info.sampled_pairs.push_back({image, sampler, pc});
return true;
}
bool AddHandlePatch(Inst* handle, uint32_t resource, uint32_t pc, std::string* error) {
for (const auto& patch: m_handle_patches) {
if (patch.handle == handle) {
return patch.resource == resource ||
Fail(pc, error,
fmt::format("{} is reused with incompatible resource classes",
ValueOpcodeName(handle->GetOpcode())));
}
}
m_handle_patches.push_back({handle, resource});
return true;
}
bool AddMemoryPatch(uint32_t index, uint32_t resource, uint32_t sampler, bool has_sampler,
uint32_t pc, std::string* error) {
for (auto& patch: m_memory_patches) {
if (patch.index != index) {
continue;
}
if (patch.resource != resource ||
(has_sampler && patch.has_sampler && patch.sampler != sampler)) {
return Fail(pc, error, "memory metadata is reused with incompatible resources");
}
if (has_sampler) {
patch.sampler = sampler;
patch.has_sampler = true;
}
return true;
}
m_memory_patches.push_back({index, resource, sampler, has_sampler});
return true;
}
bool Collect(Inst& inst, std::string* error) {
const auto op = inst.GetOpcode();
if (!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) {
return true;
}
const auto flags = inst.Flags<MemoryFlags>();
if (flags.index >= m_values.memory_info.size()) {
return Fail(flags.pc, error,
fmt::format("memory metadata index {} is out of range", flags.index));
}
if (inst.NumArgs() == 0) {
return Fail(flags.pc, error, "memory operation has no resource handle");
}
const auto& memory = m_values.memory_info[flags.index];
if (memory.planning_only) {
return true;
}
Inst* handle = nullptr;
uint32_t source = 0;
uint32_t resource = 0;
if (IsBuffer(op)) {
if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source,
error)) {
return false;
}
resource = AddBuffer(source, memory, op, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "buffer resource limit exceeded");
}
return AddHandlePatch(handle, resource, flags.pc, error) &&
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
}
if (IsAddress(op)) {
bool unbased = false;
if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) {
return false;
}
resource = AddAddress(source, unbased, memory, op, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "address resource limit exceeded");
}
return AddHandlePatch(handle, resource, flags.pc, error) &&
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
}
if (!GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, source,
error)) {
return false;
}
resource = AddImage(source, memory, op, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "image resource limit exceeded");
}
if (!AddHandlePatch(handle, resource, flags.pc, error)) {
return false;
}
uint32_t sampler = 0;
if (NeedsSampler(op)) {
if (inst.NumArgs() < 2) {
return Fail(flags.pc, error, "sampled image operation has no sampler handle");
}
Inst* sampler_handle = nullptr;
uint32_t sampler_source = 0;
if (!GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc,
sampler_handle, sampler_source, error, true)) {
return false;
}
sampler = AddSampler(sampler_source, flags.pc);
if (sampler == UINT32_MAX) {
return Fail(flags.pc, error, "sampler resource limit exceeded");
}
if (!AddHandlePatch(sampler_handle, sampler, flags.pc, error) ||
!AddSampledPair(resource, sampler, flags.pc, error)) {
return false;
}
}
return AddMemoryPatch(flags.index, resource, sampler, NeedsSampler(op), flags.pc, error);
}
const DescriptorSource* Source(uint32_t source) const {
return source < m_sources.size() ? &m_sources[source] : nullptr;
}
void LinkImageAliases() {
for (auto& buffer: m_info.buffers) {
const auto* buffer_source = Source(buffer.source);
if (buffer_source == nullptr || buffer_source->dword_count != 4) {
continue;
}
for (uint32_t image = 0; image < m_info.images.size(); image++) {
const auto* image_source = Source(m_info.images[image].source);
if (image_source == nullptr || image_source->dword_count != 8) {
continue;
}
bool alias = true;
for (uint32_t dword = 0; dword < 4; dword++) {
alias = alias && EquivalentValue(m_values, buffer_source->dwords[dword],
image_source->dwords[dword]);
}
if (alias) {
buffer.image_alias = image;
break;
}
}
}
}
Program& m_program;
ValueProgram& m_values;
ShaderInfo m_info;
std::vector<DescriptorSource> m_sources;
std::vector<HandlePatch> m_handle_patches;
std::vector<MemoryPatch> m_memory_patches;
};
} // namespace
bool TrackResources(Program& program, std::string* error) {
if (program.values == nullptr) {
if (error != nullptr) {
*error = "shader resource tracking requires typed IR";
}
return false;
}
return Tracker(program, *program.values).Run(error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ResourceTracking.h → src/graphics/shader/recompiler/ir/passes/ResourceTracking.h
+2 -2
View File
@@ -5,8 +5,8 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
// Collects immutable resource topology, then replaces descriptor operands with dense indices.
// On failure the program is left unpatched.
// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
// descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
bool TrackResources(Program& program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp
+318
View File
@@ -0,0 +1,318 @@
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components,
std::string name) {
const auto input = std::find_if(info.inputs.begin(), info.inputs.end(), [=](const auto& value) {
return value.kind == kind && value.location == location;
});
if (input == info.inputs.end()) {
info.inputs.push_back({kind, location, components, std::move(name)});
} else {
input->component_count = std::max(input->component_count, components);
}
}
bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) {
return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) {
return output.kind == kind && output.index == index;
});
}
void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t location,
std::string name) {
if (!HasOutput(info, kind, index)) {
info.outputs.push_back({kind, index, location, std::move(name)});
}
}
bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) {
auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
switch (program.stage) {
case ShaderType::Vertex:
if (options.vertex == nullptr) {
return Fail("vertex shader info requires vertex metadata");
}
if (options.vertex->resources_num < 0 ||
options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) {
return Fail("vertex resource count is out of range");
}
return true;
case ShaderType::Pixel:
if (options.pixel == nullptr) {
return Fail("pixel shader info requires pixel metadata");
}
return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) ||
Fail("pixel input count is out of range");
case ShaderType::Compute:
if (options.compute == nullptr) {
return Fail("compute shader info requires compute metadata");
}
return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) ||
Fail("compute thread ID count is out of range");
default: return Fail("unsupported shader stage for info collection");
}
}
bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& options,
std::string* error) {
const auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case ValueOpcode::GetAttribute: {
if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 ||
!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) {
return Fail("typed attribute reference is not constant");
}
if (program.stage == ShaderType::Vertex &&
(inst.Arg(1).U32() >= 4u || options.vertex == nullptr ||
inst.Arg(0).U32() >=
static_cast<uint32_t>(options.vertex->resources_num))) {
return Fail("vertex input reference is out of range");
}
break;
}
case ValueOpcode::GetBuiltin: {
if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 ||
!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) {
return Fail("typed builtin reference is not constant");
}
const auto kind = static_cast<StageInputKind>(inst.Arg(0).U32());
const auto component = inst.Arg(1).U32();
switch (kind) {
case StageInputKind::VertexIndex:
case StageInputKind::InstanceIndex:
case StageInputKind::FrontFacing:
case StageInputKind::LocalInvocationIndex:
if (component != 0u) {
return Fail("typed scalar builtin component is out of range");
}
break;
case StageInputKind::FragCoord:
if (component >= 4u) {
return Fail("typed fragment-coordinate component is out of range");
}
break;
case StageInputKind::WorkgroupId:
case StageInputKind::LocalInvocationId:
case StageInputKind::GlobalInvocationId:
if (component >= 3u) {
return Fail("typed invocation builtin component is out of range");
}
break;
case StageInputKind::Parameter:
default: return Fail("typed builtin kind is invalid");
}
break;
}
case ValueOpcode::SetAttribute:
if (inst.Flags<ExportFlags>().index >= program.values->export_info.size()) {
return Fail("typed export metadata index is out of range");
}
break;
default: break;
}
}
}
return true;
}
void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex,
ShaderInfo& info) {
AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex");
AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex");
if (vertex == nullptr) {
return;
}
uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == ValueOpcode::GetAttribute) {
const auto attr = inst.Arg(0).U32();
const auto chan = inst.Arg(1).U32();
used_components[attr] = std::max(used_components[attr], chan + 1u);
}
}
}
for (uint32_t attr = 0; attr < static_cast<uint32_t>(vertex->resources_num) &&
attr < ShaderVertexInputInfo::RES_MAX;
attr++) {
if (used_components[attr] != 0) {
AddInput(info, StageInputKind::Parameter, attr, used_components[attr],
fmt::format("in_attr_{}", attr));
}
}
}
void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
if (pixel == nullptr) {
return;
}
if (pixel->HasPositionInput()) {
AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord");
}
if (pixel->ps_front_face) {
AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing");
}
for (uint32_t input = 0; input < pixel->input_num; input++) {
AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input));
}
}
void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& info) {
if (compute != nullptr) {
if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) {
AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID");
}
if (compute->thread_ids_num > 0) {
AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID");
}
if (compute->thread_ids_num > 0 || compute->tg_size_en) {
AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex");
}
if (compute->dispatch_thread_dimensions) {
AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID");
}
}
}
void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::GetBuiltin) {
continue;
}
const auto kind = static_cast<StageInputKind>(inst.Arg(0).U32());
switch (kind) {
case StageInputKind::VertexIndex:
AddInput(info, kind, 0, 1, "gl_VertexIndex");
break;
case StageInputKind::InstanceIndex:
AddInput(info, kind, 0, 1, "gl_InstanceIndex");
break;
case StageInputKind::FragCoord: AddInput(info, kind, 0, 4, "gl_FragCoord"); break;
case StageInputKind::FrontFacing:
AddInput(info, kind, 0, 1, "gl_FrontFacing");
break;
case StageInputKind::WorkgroupId:
AddInput(info, kind, 0, 3, "gl_WorkGroupID");
break;
case StageInputKind::LocalInvocationId:
AddInput(info, kind, 0, 3, "gl_LocalInvocationID");
break;
case StageInputKind::LocalInvocationIndex:
AddInput(info, kind, 0, 1, "gl_LocalInvocationIndex");
break;
case StageInputKind::GlobalInvocationId:
AddInput(info, kind, 0, 3, "gl_GlobalInvocationID");
break;
case StageInputKind::Parameter: break;
}
}
}
}
void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::SetAttribute) {
continue;
}
const auto& export_info = program.values->export_info[inst.Flags<ExportFlags>().index];
if (export_info.kind == ExportTargetKind::MrtZ) {
if (pixel != nullptr && (export_info.en & 0x1u) != 0 &&
pixel->ps_depth_export_enable) {
AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth");
}
if (pixel != nullptr && (export_info.en & 0x4u) != 0 &&
pixel->ps_sample_mask_export_enable) {
AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask");
}
continue;
}
if (export_info.en == 0) {
continue;
}
switch (export_info.kind) {
case ExportTargetKind::Position:
AddOutput(info, StageOutputKind::Position, export_info.index, 0,
"out_position");
break;
case ExportTargetKind::Parameter:
AddOutput(info, StageOutputKind::Parameter, export_info.index,
export_info.index, fmt::format("out_param_{}", export_info.index));
break;
case ExportTargetKind::Mrt:
AddOutput(info, StageOutputKind::Mrt, export_info.index, export_info.index,
fmt::format("out_mrt_{}", export_info.index));
break;
default: break;
}
}
}
}
} // namespace
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) {
if (!program.resource_tracking_complete || program.shader_info_complete) {
if (error != nullptr) {
*error = !program.resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected";
}
return false;
}
if (program.values == nullptr) {
if (error != nullptr) {
*error = "typed value program is not available";
}
return false;
}
if (!ValidateOptions(program, options, error)) {
return false;
}
if (!ValidateValueReferences(program, options, error)) {
return false;
}
auto next = program.info;
next.inputs.clear();
next.outputs.clear();
next.has_bitwise_xor = std::any_of(
program.values->blocks.begin(), program.values->blocks.end(), [](const auto* block) {
return std::any_of(block->begin(), block->end(), [](const auto& inst) {
return inst.GetOpcode() == ValueOpcode::BitwiseXor32;
});
});
switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break;
case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(options.compute, next); break;
default: return false;
}
CollectBuiltinInputs(program, next);
CollectOutputs(program, options.pixel, next);
program.info = std::move(next);
program.shader_info_complete = true;
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/ShaderInfoCollection.h → src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h
View File
File renamed without changes.
src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp
+951
View File
@@ -0,0 +1,951 @@
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <bit>
#include <cstring>
#include <fmt/format.h>
#include <unordered_map>
#include <unordered_set>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) {
return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
StageName(program.stage), pc, message);
}
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
if (base > AddressMask) {
return false;
}
if (offset < 0) {
const auto magnitude = uint64_t {0} - static_cast<uint64_t>(offset);
if (magnitude > base) {
return false;
}
result = base - magnitude;
return true;
}
const auto magnitude = static_cast<uint64_t>(offset);
if (magnitude > AddressMask - base) {
return false;
}
result = base + magnitude;
return true;
}
bool IsRawRead(const ValueProgram& values, const Inst& inst) {
if (inst.GetOpcode() != ValueOpcode::LoadAddressU32 &&
inst.GetOpcode() != ValueOpcode::ReadConstBuffer) {
return false;
}
const auto index = inst.Flags<MemoryFlags>().index;
return index < values.memory_info.size() &&
values.memory_info[index].kind == ResourceKind::ScalarBuffer;
}
bool IsDescriptorHandle(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::GetBufferResource:
case ValueOpcode::GetAddressResource:
case ValueOpcode::GetImageResource:
case ValueOpcode::GetSamplerResource: return true;
default: return false;
}
}
bool IsRuntimeIntegerOp(ValueOpcode op) {
switch (op) {
case ValueOpcode::CompositeConstructU32x2:
case ValueOpcode::CompositeExtractU32x2:
case ValueOpcode::PackUint2x32:
case ValueOpcode::UnpackUint2x32:
case ValueOpcode::BitFieldInsert:
case ValueOpcode::BitFieldUExtract:
case ValueOpcode::BitFieldSExtract:
case ValueOpcode::IAdd32:
case ValueOpcode::IAdd64:
case ValueOpcode::IAddCarry32:
case ValueOpcode::ISub32:
case ValueOpcode::ISub64:
case ValueOpcode::IMul32:
case ValueOpcode::IMul64:
case ValueOpcode::ShiftLeftLogical32:
case ValueOpcode::ShiftLeftLogical64:
case ValueOpcode::ShiftRightLogical32:
case ValueOpcode::ShiftRightLogical64:
case ValueOpcode::ShiftRightArithmetic32:
case ValueOpcode::ShiftRightArithmetic64:
case ValueOpcode::BitwiseAnd32:
case ValueOpcode::BitwiseAnd64:
case ValueOpcode::BitwiseOr32:
case ValueOpcode::BitwiseOr64:
case ValueOpcode::BitwiseXor32:
case ValueOpcode::BitwiseNot32:
case ValueOpcode::SelectU1:
case ValueOpcode::SelectU32:
case ValueOpcode::ULessThan32:
case ValueOpcode::IEqual32:
case ValueOpcode::UGreaterThan32:
case ValueOpcode::INotEqual32:
case ValueOpcode::LogicalOr:
case ValueOpcode::LogicalAnd:
case ValueOpcode::LogicalXor:
case ValueOpcode::LogicalNot: return true;
default: return false;
}
}
class RuntimeValidator {
public:
RuntimeValidator(const Program& program, const ValueProgram& values)
: m_program(program), m_values(values) {}
bool Run(Value value, std::string& reason) { return Validate(value, reason); }
private:
bool Validate(Value value, std::string& reason) {
value = value.Resolve();
const auto* inst = value.TryInstruction();
if (inst == nullptr) {
switch (value.GetType()) {
case Type::U1:
case Type::U8:
case Type::U16:
case Type::U32:
case Type::U64: return true;
default:
reason = fmt::format("contains a non-integer immediate of type {}",
TypeName(value.GetType()));
return false;
}
}
if (!m_visiting.insert(inst).second) {
reason = fmt::format("contains a cyclic {} value", ValueOpcodeName(inst->GetOpcode()));
return false;
}
const auto finish = [&](bool valid) {
m_visiting.erase(inst);
return valid;
};
const auto op = inst->GetOpcode();
if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU8 ||
op == ValueOpcode::UndefU16 || op == ValueOpcode::UndefU32 ||
op == ValueOpcode::UndefU64 || op == ValueOpcode::Void) {
reason = fmt::format("contains {}", ValueOpcodeName(op));
return finish(false);
}
if (op == ValueOpcode::GetUserData) {
if (inst->NumArgs() != 1 || inst->Arg(0).GetType() != Type::ScalarReg) {
reason = "contains a malformed GetUserData";
return finish(false);
}
const auto reg = RegIndex(inst->Arg(0).ScalarRegister());
if (reg < m_program.user_data_base ||
reg - m_program.user_data_base >= m_program.user_data_count) {
reason = fmt::format("references unavailable user SGPR {}", reg);
return finish(false);
}
return finish(true);
}
if (op == ValueOpcode::GetShaderBase) {
if (inst->NumArgs() != 0) {
reason = "contains a malformed GetShaderBase";
return finish(false);
}
return finish(true);
}
if (op == ValueOpcode::Phi) {
const auto invariant = ResolveInvariantPhi(m_values, value);
if (invariant.IsEmpty()) {
reason = "contains a control-dependent phi";
return finish(false);
}
return finish(Validate(invariant, reason));
}
if (op == ValueOpcode::GetSrtResource) {
if (inst->NumArgs() != 0) {
reason = "contains a malformed GetSrtResource";
return finish(false);
}
return finish(true);
}
if (op == ValueOpcode::ReadConst) {
const auto slot = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
if (inst->NumArgs() != 2 || inst->Arg(0).Resolve().TryInstruction() == nullptr ||
inst->Arg(0).Resolve().TryInstruction()->GetOpcode() !=
ValueOpcode::GetSrtResource ||
!slot.IsImmediate() || slot.GetType() != Type::U32 ||
slot.U32() >= m_values.srt_reads.size()) {
reason = "contains a malformed flattened SRT read";
return finish(false);
}
} else if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
const auto expected = op == ValueOpcode::LoadAddressU32
? ValueOpcode::GetAddressResource
: ValueOpcode::GetBufferResource;
const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr;
if (!IsRawRead(m_values, *inst) || handle == nullptr ||
handle->GetOpcode() != expected) {
reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op));
return finish(false);
}
} else if (op == ValueOpcode::PackUint2x32) {
const auto* pair = inst->NumArgs() == 1 ? inst->Arg(0).ResolveInstruction() : nullptr;
if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2) {
reason = "contains an unsupported packed runtime value";
return finish(false);
}
} else if (op == ValueOpcode::CompositeExtractU32x2) {
const auto* source = inst->NumArgs() == 2 ? inst->Arg(0).ResolveInstruction() : nullptr;
const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
if (source == nullptr || !index.IsImmediate() || index.GetType() != Type::U32 ||
index.U32() >= 2u ||
(source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 &&
source->GetOpcode() != ValueOpcode::UnpackUint2x32 &&
source->GetOpcode() != ValueOpcode::IAddCarry32)) {
reason = "contains an unsupported composite runtime source";
return finish(false);
}
}
if (op == ValueOpcode::GetBufferResource || op == ValueOpcode::GetImageResource ||
op == ValueOpcode::GetSamplerResource || op == ValueOpcode::GetAddressResource) {
const size_t expected = op == ValueOpcode::GetBufferResource ? 4u
: op == ValueOpcode::GetImageResource ? 8u
: op == ValueOpcode::GetSamplerResource ? 4u
: 2u;
if (inst->NumArgs() != expected) {
reason = fmt::format("contains a malformed {}", ValueOpcodeName(op));
return finish(false);
}
} else if (op != ValueOpcode::ReadConst && op != ValueOpcode::ReadConstBuffer &&
op != ValueOpcode::LoadAddressU32 && !IsRuntimeIntegerOp(op)) {
reason =
fmt::format("contains unsupported or control-dependent {}", ValueOpcodeName(op));
return finish(false);
}
for (size_t index = 0; index < inst->NumArgs(); index++) {
if (!Validate(inst->Arg(index), reason)) {
return finish(false);
}
}
return finish(true);
}
const Program& m_program;
const ValueProgram& m_values;
std::unordered_set<const Inst*> m_visiting;
};
class PlanBuilder {
public:
PlanBuilder(Program& program, ValueProgram& values): m_program(program), m_values(values) {}
bool Run(std::string* error) {
m_values.srt_reads.clear();
m_values.dynamic_reads.clear();
for (auto* block: m_values.blocks) {
for (auto& inst: *block) {
if (IsDescriptorHandle(inst.GetOpcode())) {
for (size_t index = 0; index < inst.NumArgs(); index++) {
if (!Collect(inst.Arg(index), 0, error)) {
return false;
}
}
}
}
}
for (auto* block: m_values.blocks) {
for (auto& inst: *block) {
std::string reason;
if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_values, inst) &&
inst.Arg(1).Resolve().IsImmediate() &&
ValidateRuntimeValue(m_program, Value(&inst), reason) &&
!Collect(Value(&inst), inst.Flags<MemoryFlags>().pc, error)) {
return false;
}
}
}
PatchReads();
return true;
}
private:
struct Patch {
Inst* inst = nullptr;
uint32_t slot = 0;
bool keep = false;
};
bool Fail(uint32_t pc, std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, pc, message);
}
return false;
}
bool Collect(Value value, uint32_t use_pc, std::string* error) {
value = value.Resolve();
if (value.IsImmediate()) {
return true;
}
auto* inst = value.TryInstruction();
if (inst == nullptr) {
return Fail(use_pc, error, "invalid typed planning value");
}
if (std::ranges::find(m_visiting, inst) != m_visiting.end()) {
return inst->GetOpcode() == ValueOpcode::Phi;
}
if (std::ranges::find(m_visited, inst) != m_visited.end()) {
return true;
}
m_visiting.push_back(inst);
for (size_t index = 0; index < inst->NumArgs(); index++) {
if (!Collect(inst->Arg(index), use_pc, error)) {
return false;
}
}
m_visiting.pop_back();
m_visited.push_back(inst);
if (!IsRawRead(m_values, *inst)) {
return true;
}
const auto offset = inst->Arg(1).Resolve();
if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
if (std::ranges::find(m_values.dynamic_reads, value) == m_values.dynamic_reads.end()) {
m_values.dynamic_reads.push_back(value);
}
return true;
}
use_pc = inst->Flags<MemoryFlags>().pc;
for (uint32_t slot = 0; slot < m_values.srt_reads.size(); slot++) {
if (EquivalentValue(m_values, value, m_values.srt_reads[slot].value)) {
m_patches.push_back({inst, slot, false});
return true;
}
}
const auto slot = static_cast<uint32_t>(m_values.srt_reads.size());
m_values.srt_reads.push_back({value, slot, use_pc});
m_patches.push_back({inst, slot, true});
return true;
}
void PatchReads() {
for (const auto& patch: m_patches) {
auto* block = patch.inst->Parent();
auto& list = block->Instructions();
auto where =
std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; });
const auto resource =
Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource));
const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst,
{resource, Value(patch.slot)}));
const auto uses = patch.inst->Uses();
for (const auto& use: uses) {
use.user->SetArg(use.operand, flat);
}
for (auto& info: m_values.block_info) {
if (info.condition.Resolve() == Value(patch.inst)) {
info.condition = flat;
}
if (info.indirect_target.Resolve() == Value(patch.inst)) {
info.indirect_target = flat;
}
}
if (patch.keep) {
const auto memory = patch.inst->Flags<MemoryFlags>().index;
if (memory < m_values.memory_info.size()) {
m_values.memory_info[memory].planning_only = true;
}
block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
}
}
}
Program& m_program;
ValueProgram& m_values;
std::vector<Inst*> m_visiting;
std::vector<Inst*> m_visited;
std::vector<Patch> m_patches;
};
class Evaluator {
public:
Evaluator(const Program& program, const SrtRuntime& runtime)
: m_program(program), m_values(*program.values), m_runtime(runtime) {}
void SetUsePc(uint32_t pc) { m_use_pc = pc; }
bool Evaluate(Value value, uint32_t& result, std::string* error) {
uint64_t wide = 0;
if (!EvaluateWide(value, wide, error)) {
return false;
}
result = static_cast<uint32_t>(wide);
return true;
}
private:
bool Fail(std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, m_use_pc, message);
}
return false;
}
bool EvaluateWide(Value value, uint64_t& result, std::string* error) {
value = value.Resolve();
if (value.IsImmediate()) {
switch (value.GetType()) {
case Type::U1: result = value.U1(); return true;
case Type::U8: result = value.U8(); return true;
case Type::U16: result = value.U16(); return true;
case Type::U32: result = value.U32(); return true;
case Type::U64: result = value.U64(); return true;
default: return Fail(error, "non-integer immediate in runtime expression");
}
}
auto* inst = value.TryInstruction();
if (inst == nullptr) {
return Fail(error, "invalid typed runtime value");
}
if (const auto found = m_cache.find(inst); found != m_cache.end()) {
result = found->second;
return true;
}
if (std::ranges::find(m_visiting, inst) != m_visiting.end()) {
return Fail(error, "cyclic typed runtime value");
}
m_visiting.push_back(inst);
uint64_t out = 0;
if (!EvaluateInst(*inst, out, error)) {
return false;
}
m_visiting.pop_back();
m_cache.emplace(inst, out);
result = out;
return true;
}
bool Arg(const Inst& inst, size_t index, uint64_t& result, std::string* error) {
return EvaluateWide(inst.Arg(index), result, error);
}
bool EvaluatePhi(const Inst& inst, uint64_t& result, std::string* error) {
const auto value = ResolveInvariantPhi(m_values, Value(const_cast<Inst*>(&inst)));
return !value.IsEmpty() ? EvaluateWide(value, result, error)
: Fail(error, "typed phi has runtime-dependent values");
}
bool EvaluateExtract(const Inst& inst, uint64_t& result, std::string* error) {
const auto index = inst.Arg(1).Resolve();
if (!index.IsImmediate() || index.GetType() != Type::U32) {
return Fail(error, "dynamic composite extract in runtime expression");
}
const auto component = index.U32();
const auto* source = inst.Arg(0).ResolveInstruction();
if (source == nullptr || component >= 2u) {
return Fail(error, "unsupported composite runtime source");
}
if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
return EvaluateWide(source->Arg(component), result, error);
}
if (source->GetOpcode() == ValueOpcode::UnpackUint2x32) {
uint64_t packed = 0;
if (!Arg(*source, 0, packed, error)) {
return false;
}
result = static_cast<uint32_t>(packed >> (component * 32u));
return true;
}
if (source->GetOpcode() == ValueOpcode::IAddCarry32) {
uint64_t lhs = 0;
uint64_t rhs = 0;
if (!Arg(*source, 0, lhs, error) || !Arg(*source, 1, rhs, error)) {
return false;
}
const auto sum =
static_cast<uint64_t>(static_cast<uint32_t>(lhs)) + static_cast<uint32_t>(rhs);
result =
component == 0u ? static_cast<uint32_t>(sum) : static_cast<uint32_t>(sum >> 32u);
return true;
}
return Fail(error, "unsupported composite runtime source");
}
bool EvaluateRawRead(const Inst& inst, uint64_t& result, std::string* error) {
const auto flags = inst.Flags<MemoryFlags>();
if (flags.index >= m_values.memory_info.size()) {
return Fail(error, "raw scalar read has invalid metadata");
}
const auto& mem = m_values.memory_info[flags.index];
const auto* handle = inst.Arg(0).ResolveInstruction();
if (handle == nullptr) {
return Fail(error, "raw scalar read has no descriptor handle");
}
uint64_t low = 0;
uint64_t high = 0;
uint64_t offset = 0;
if (!Arg(*handle, 0, low, error) || !Arg(*handle, 1, high, error) ||
!Arg(inst, 1, offset, error)) {
return false;
}
const auto base = ((high << 32u) | static_cast<uint32_t>(low)) & AddressMask;
const auto immediate = static_cast<int64_t>(static_cast<int32_t>(mem.offset));
uint64_t address = 0;
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
uint64_t records = 0;
uint64_t word3 = 0;
if (handle->NumArgs() != 4u || !Arg(*handle, 2, records, error) ||
!Arg(*handle, 3, word3, error)) {
return Fail(error, "constant-buffer descriptor has invalid width");
}
if (immediate < 0) {
return Fail(error, "constant-buffer read has a negative immediate offset");
}
const auto byte_offset =
static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
const auto aligned = byte_offset & ~uint64_t {3};
const auto stride = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
const auto size = stride == 0u
? static_cast<uint64_t>(static_cast<uint32_t>(records))
: static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
if (aligned > size || size - aligned < sizeof(uint32_t)) {
return Fail(
error, fmt::format("constant-buffer offset {} exceeds size {}", aligned, size));
}
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
} else {
const auto relative = (immediate & ~int64_t {3}) +
static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) {
return Fail(error, "raw scalar read is outside the 48-bit address space");
}
}
uint32_t word = 0;
if (m_runtime.read_memory != nullptr) {
if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) {
return Fail(error, fmt::format("constant read failed at 0x{:016x}", address));
}
} else {
std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
}
result = word;
return true;
}
bool EvaluateInst(const Inst& inst, uint64_t& result, std::string* error) {
uint64_t a = 0;
uint64_t b = 0;
uint64_t c = 0;
const auto binary = [&]() { return Arg(inst, 0, a, error) && Arg(inst, 1, b, error); };
const auto ternary = [&]() {
return Arg(inst, 0, a, error) && Arg(inst, 1, b, error) && Arg(inst, 2, c, error);
};
switch (inst.GetOpcode()) {
case ValueOpcode::GetUserData: {
const auto reg = RegIndex(inst.Arg(0).ScalarRegister());
if (reg < m_program.user_data_base ||
reg - m_program.user_data_base >= m_runtime.user_data.size()) {
return Fail(error, fmt::format("user SGPR {} is unavailable", reg));
}
result = m_runtime.user_data[reg - m_program.user_data_base];
return true;
}
case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
case ValueOpcode::Phi: return EvaluatePhi(inst, result, error);
case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result, error);
case ValueOpcode::PackUint2x32: {
const auto* pair = inst.Arg(0).ResolveInstruction();
if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2 ||
!Arg(*pair, 0, a, error) || !Arg(*pair, 1, b, error)) {
return Fail(error, "unsupported packed runtime value");
}
result = static_cast<uint32_t>(a) |
(static_cast<uint64_t>(static_cast<uint32_t>(b)) << 32u);
return true;
}
case ValueOpcode::ReadConst: {
const auto slot = inst.Arg(1).Resolve();
if (!slot.IsImmediate() || slot.GetType() != Type::U32 ||
slot.U32() >= m_values.srt_reads.size()) {
return Fail(error, "invalid flattened SRT index");
}
return EvaluateWide(m_values.srt_reads[slot.U32()].value, result, error);
}
case ValueOpcode::LoadAddressU32:
case ValueOpcode::ReadConstBuffer:
if (IsRawRead(m_values, inst)) {
return EvaluateRawRead(inst, result, error);
}
break;
case ValueOpcode::IAdd32:
if (binary()) {
result = static_cast<uint32_t>(a + b);
return true;
}
return false;
case ValueOpcode::IAdd64:
if (binary()) {
result = a + b;
return true;
}
return false;
case ValueOpcode::ISub32:
if (binary()) {
result = static_cast<uint32_t>(a - b);
return true;
}
return false;
case ValueOpcode::ISub64:
if (binary()) {
result = a - b;
return true;
}
return false;
case ValueOpcode::IMul32:
if (binary()) {
result = static_cast<uint32_t>(a * b);
return true;
}
return false;
case ValueOpcode::IMul64:
if (binary()) {
result = a * b;
return true;
}
return false;
case ValueOpcode::BitwiseAnd32:
if (binary()) {
result = static_cast<uint32_t>(a & b);
return true;
}
return false;
case ValueOpcode::BitwiseAnd64:
if (binary()) {
result = a & b;
return true;
}
return false;
case ValueOpcode::BitwiseOr32:
if (binary()) {
result = static_cast<uint32_t>(a | b);
return true;
}
return false;
case ValueOpcode::BitwiseOr64:
if (binary()) {
result = a | b;
return true;
}
return false;
case ValueOpcode::BitwiseXor32:
if (binary()) {
result = static_cast<uint32_t>(a ^ b);
return true;
}
return false;
case ValueOpcode::BitwiseNot32:
if (Arg(inst, 0, a, error)) {
result = ~static_cast<uint32_t>(a);
return true;
}
return false;
case ValueOpcode::ShiftLeftLogical32:
if (binary()) {
result = static_cast<uint32_t>(a) << (b & 31u);
return true;
}
return false;
case ValueOpcode::ShiftLeftLogical64:
if (binary()) {
result = a << (b & 63u);
return true;
}
return false;
case ValueOpcode::ShiftRightLogical32:
if (binary()) {
result = static_cast<uint32_t>(a) >> (b & 31u);
return true;
}
return false;
case ValueOpcode::ShiftRightLogical64:
if (binary()) {
result = a >> (b & 63u);
return true;
}
return false;
case ValueOpcode::ShiftRightArithmetic32:
if (binary()) {
result = static_cast<uint32_t>(
std::bit_cast<int32_t>(static_cast<uint32_t>(a)) >> (b & 31u));
return true;
}
return false;
case ValueOpcode::ShiftRightArithmetic64:
if (binary()) {
result = static_cast<uint64_t>(std::bit_cast<int64_t>(a) >> (b & 63u));
return true;
}
return false;
case ValueOpcode::BitFieldUExtract:
if (ternary()) {
const auto offset = static_cast<uint32_t>(b);
const auto width = static_cast<uint32_t>(c);
if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid unsigned bit-field range");
}
const auto mask = width == 32u ? UINT32_MAX
: width == 0u ? 0u
: (uint32_t {1} << width) - 1u;
result = width == 0u ? 0u : (static_cast<uint32_t>(a) >> offset) & mask;
return true;
}
return false;
case ValueOpcode::BitFieldSExtract:
if (ternary()) {
const auto offset = static_cast<uint32_t>(b);
const auto width = static_cast<uint32_t>(c);
if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid signed bit-field range");
}
if (width == 0u) {
result = 0;
return true;
}
const auto mask = width == 32u ? UINT32_MAX : (uint32_t {1} << width) - 1u;
auto bits = (static_cast<uint32_t>(a) >> offset) & mask;
if (width < 32u && (bits & (uint32_t {1} << (width - 1u))) != 0u) {
bits |= ~mask;
}
result = bits;
return true;
}
return false;
case ValueOpcode::BitFieldInsert: {
uint64_t d = 0;
if (!ternary() || !Arg(inst, 3, d, error)) {
return false;
}
const auto offset = static_cast<uint32_t>(c);
const auto width = static_cast<uint32_t>(d);
if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid inserted bit-field range");
}
if (width == 0u) {
result = static_cast<uint32_t>(a);
return true;
}
const auto mask =
width == 32u ? UINT32_MAX : ((uint32_t {1} << width) - 1u) << offset;
result = (static_cast<uint32_t>(a) & ~mask) |
((static_cast<uint32_t>(b) << offset) & mask);
return true;
}
case ValueOpcode::SelectU32:
case ValueOpcode::SelectU1:
if (ternary()) {
result = a != 0u ? b : c;
return true;
}
return false;
case ValueOpcode::IEqual32:
if (binary()) {
result = static_cast<uint32_t>(a) == static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::INotEqual32:
if (binary()) {
result = static_cast<uint32_t>(a) != static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::ULessThan32:
if (binary()) {
result = static_cast<uint32_t>(a) < static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::UGreaterThan32:
if (binary()) {
result = static_cast<uint32_t>(a) > static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::LogicalAnd:
if (binary()) {
result = (a != 0u) && (b != 0u);
return true;
}
return false;
case ValueOpcode::LogicalOr:
if (binary()) {
result = (a != 0u) || (b != 0u);
return true;
}
return false;
case ValueOpcode::LogicalXor:
if (binary()) {
result = (a != 0u) != (b != 0u);
return true;
}
return false;
case ValueOpcode::LogicalNot:
if (Arg(inst, 0, a, error)) {
result = a == 0u;
return true;
}
return false;
case ValueOpcode::UndefU1:
case ValueOpcode::UndefU8:
case ValueOpcode::UndefU16:
case ValueOpcode::UndefU32:
case ValueOpcode::UndefU64: return Fail(error, "undefined typed runtime value");
default: break;
}
return Fail(error, fmt::format("unsupported typed runtime opcode {}",
ValueOpcodeName(inst.GetOpcode())));
}
const Program& m_program;
const ValueProgram& m_values;
const SrtRuntime& m_runtime;
uint32_t m_use_pc = 0;
std::unordered_map<const Inst*, uint64_t> m_cache;
std::vector<const Inst*> m_visiting;
};
const DescriptorSource* Source(const Program& program, uint32_t source) {
if (program.values == nullptr || source >= program.values->descriptor_sources.size()) {
return nullptr;
}
return &program.values->descriptor_sources[source];
}
bool EvaluateRuntimeSourcesImpl(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, bool evaluate_flat,
std::string* error) {
if (program.values == nullptr || !program.srt_plan_complete) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "typed SRT plan is not ready");
}
return false;
}
Evaluator evaluator(program, runtime);
std::vector<DescriptorValue> evaluated;
evaluated.reserve(requests.size());
for (const auto& request: requests) {
const auto* source = Source(program, request.source);
if (source == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, request.use_pc,
fmt::format("invalid descriptor source {}", request.source));
}
return false;
}
evaluator.SetUsePc(request.use_pc);
DescriptorValue value;
value.dword_count = source->dword_count;
for (uint32_t index = 0; index < source->dword_count; index++) {
if (!evaluator.Evaluate(source->dwords[index], value.dwords[index], error)) {
return false;
}
}
evaluated.push_back(value);
}
std::vector<uint32_t> flattened;
if (evaluate_flat) {
flattened.resize(program.values->srt_reads.size());
for (const auto& read: program.values->srt_reads) {
evaluator.SetUsePc(read.use_pc);
if (read.flat_offset >= flattened.size() ||
!evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) {
return false;
}
}
}
results = std::move(evaluated);
if (evaluate_flat) {
flat = std::move(flattened);
}
return true;
}
} // namespace
bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason) {
if (program.values == nullptr) {
reason = "typed SSA is not ready";
return false;
}
return RuntimeValidator(program, *program.values).Run(value, reason);
}
bool BuildSrtPlan(Program& program, std::string* error) {
if (program.values == nullptr || program.resource_tracking_complete) {
if (error != nullptr) {
*error = program.values == nullptr ? "typed SSA is not ready"
: "cannot rebuild SRT after resource tracking";
}
return false;
}
program.srt_plan_complete = false;
if (!PlanBuilder(program, *program.values).Run(error)) {
return false;
}
program.srt_plan_complete = true;
return true;
}
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue& result,
std::string* error) {
const DescriptorSourceRequest request {source, use_pc};
std::vector<DescriptorValue> results;
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) {
return false;
}
result = results.front();
return true;
}
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::string* error) {
std::vector<uint32_t> ignored;
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error);
}
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::string* error) {
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error);
}
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
std::string* error) {
std::vector<DescriptorValue> ignored;
return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/SrtWalker.h → src/graphics/shader/recompiler/ir/passes/SrtWalker.h
+3 -2
View File
@@ -8,6 +8,8 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
class Value;
using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* value);
struct SrtRuntime {
@@ -23,11 +25,10 @@ struct DescriptorSourceRequest {
uint32_t use_pc = 0;
};
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t& result);
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
// dynamic offsets remain explicit and are never assigned a fake slot.
bool BuildSrtPlan(Program& program, std::string* error);
bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason);
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue& result,
src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp
+377
View File
@@ -0,0 +1,377 @@
#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
#include <algorithm>
#include <map>
#include <unordered_map>
#include <variant>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
struct SccTag {
auto operator<=>(const SccTag&) const = default;
};
struct ThreadBitScalarReg {
ScalarReg reg {};
auto operator<=>(const ThreadBitScalarReg&) const = default;
};
struct ExecTag {
auto operator<=>(const ExecTag&) const = default;
};
struct ExecLoTag {
auto operator<=>(const ExecLoTag&) const = default;
};
struct ExecHiTag {
auto operator<=>(const ExecHiTag&) const = default;
};
struct VccTag {
auto operator<=>(const VccTag&) const = default;
};
struct VccLoTag {
auto operator<=>(const VccLoTag&) const = default;
};
struct VccHiTag {
auto operator<=>(const VccHiTag&) const = default;
};
struct M0Tag {
auto operator<=>(const M0Tag&) const = default;
};
struct GotoVariable {
uint32_t index = 0;
auto operator<=>(const GotoVariable&) const = default;
};
using Variable = std::variant<ScalarReg, ThreadBitScalarReg, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using ValueMap = std::unordered_map<Block*, Value>;
struct DefTable {
const Value& Get(Block* block, ScalarReg reg) {
if (RegIndex(reg) >= NumScalarRegs) {
EXIT("SSA scalar read index is out of range: %u", RegIndex(reg));
}
return block->ssa_sreg_values[RegIndex(reg)];
}
void Set(Block* block, ScalarReg reg, Value value) {
if (RegIndex(reg) >= NumScalarRegs) {
EXIT("SSA scalar write index is out of range: %u", RegIndex(reg));
}
block->ssa_sreg_values[RegIndex(reg)] = value;
}
const Value& Get(Block* block, ThreadBitScalarReg value) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
return block->ssa_thread_bit_sreg_values[RegIndex(value.reg)];
}
void Set(Block* block, ThreadBitScalarReg value, Value definition) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, VectorReg reg) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
return block->ssa_vreg_values[RegIndex(reg)];
}
void Set(Block* block, VectorReg reg, Value value) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
block->ssa_vreg_values[RegIndex(reg)] = value;
}
const Value& Get(Block* block, GotoVariable value) {
return goto_variables[value.index][block];
}
void Set(Block* block, GotoVariable value, Value definition) {
goto_variables[value.index][block] = definition;
}
const Value& Get(Block* block, SccTag) { return scc[block]; }
void Set(Block* block, SccTag, Value value) { scc[block] = value; }
const Value& Get(Block* block, ExecTag) { return exec[block]; }
void Set(Block* block, ExecTag, Value value) { exec[block] = value; }
const Value& Get(Block* block, ExecLoTag) { return exec_lo[block]; }
void Set(Block* block, ExecLoTag, Value value) { exec_lo[block] = value; }
const Value& Get(Block* block, ExecHiTag) { return exec_hi[block]; }
void Set(Block* block, ExecHiTag, Value value) { exec_hi[block] = value; }
const Value& Get(Block* block, VccTag) { return vcc[block]; }
void Set(Block* block, VccTag, Value value) { vcc[block] = value; }
const Value& Get(Block* block, VccLoTag) { return vcc_lo[block]; }
void Set(Block* block, VccLoTag, Value value) { vcc_lo[block] = value; }
const Value& Get(Block* block, VccHiTag) { return vcc_hi[block]; }
void Set(Block* block, VccHiTag, Value value) { vcc_hi[block] = value; }
const Value& Get(Block* block, M0Tag) { return m0[block]; }
void Set(Block* block, M0Tag, Value value) { m0[block] = value; }
ValueMap scc;
ValueMap exec;
ValueMap exec_lo;
ValueMap exec_hi;
ValueMap vcc;
ValueMap vcc_lo;
ValueMap vcc_hi;
ValueMap m0;
std::unordered_map<uint32_t, ValueMap> goto_variables;
};
ValueOpcode UndefOpcode(ScalarReg) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(ThreadBitScalarReg) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VectorReg) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(GotoVariable) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(SccTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ExecTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ExecLoTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(ExecHiTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(VccTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VccLoTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(VccHiTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(M0Tag) {
return ValueOpcode::UndefU32;
}
Value InitialValue(ScalarReg) {
return Value(0u);
}
Value InitialValue(ThreadBitScalarReg) {
return Value(false);
}
Value InitialValue(VectorReg) {
return Value(0u);
}
Value InitialValue(GotoVariable) {
return Value(false);
}
Value InitialValue(SccTag) {
return Value(false);
}
Value InitialValue(ExecTag) {
return Value(false);
}
Value InitialValue(ExecLoTag) {
return Value(0u);
}
Value InitialValue(ExecHiTag) {
return Value(0u);
}
Value InitialValue(VccTag) {
return Value(false);
}
Value InitialValue(VccLoTag) {
return Value(0u);
}
Value InitialValue(VccHiTag) {
return Value(0u);
}
Value InitialValue(M0Tag) {
return Value(0u);
}
enum class ReadStep { Start, SetValue, PushPhiArgument };
template <typename T>
struct ReadState {
Block* block = nullptr;
Value result;
Inst* phi = nullptr;
size_t pred = 0;
ReadStep step = ReadStep::Start;
};
class Pass {
public:
template <typename T>
void Write(T variable, Block* block, Value value) {
definitions.Set(block, variable, value);
}
template <typename T>
Value Read(T variable, Block* root) {
std::vector<ReadState<T>> stack {{}, {.block = root}};
const auto prepare_phi = [&]() {
auto& state = stack.back();
const auto predecessors = state.block->ImmPredecessors();
if (state.pred == predecessors.size()) {
const auto result = TryRemoveTrivialPhi(*state.phi, InitialValue(variable));
const auto block = state.block;
stack.pop_back();
stack.back().result = result;
Write(variable, block, result);
return;
}
state.step = ReadStep::PushPhiArgument;
stack.push_back({.block = predecessors[state.pred]});
};
do {
auto& state = stack.back();
auto* block = state.block;
switch (state.step) {
case ReadStep::Start: {
if (const auto& def = definitions.Get(block, variable); !def.IsEmpty()) {
state.result = def;
} else if (!block->IsSsaSealed()) {
auto& phi = *block->PrependNewInst(block->begin(), ValueOpcode::Phi);
phi.SetFlags(TypeOf(UndefOpcode(variable)));
incomplete_phis[block][Variable(variable)] = &phi;
state.result = Value(&phi);
} else if (const auto predecessors = block->ImmPredecessors();
predecessors.size() == 1) {
state.step = ReadStep::SetValue;
stack.push_back({.block = predecessors.front()});
break;
} else {
auto& phi = *block->PrependNewInst(block->begin(), ValueOpcode::Phi);
phi.SetFlags(TypeOf(UndefOpcode(variable)));
Write(variable, block, Value(&phi));
state.phi = &phi;
prepare_phi();
break;
}
[[fallthrough]];
}
case ReadStep::SetValue: {
const auto result = state.result;
Write(variable, block, result);
stack.pop_back();
stack.back().result = result;
break;
}
case ReadStep::PushPhiArgument: {
const auto predecessors = block->ImmPredecessors();
state.phi->AddPhiOperand(predecessors[state.pred], state.result);
state.pred++;
prepare_phi();
break;
}
}
} while (stack.size() > 1);
return stack.back().result;
}
void Seal(Block* block) {
if (const auto found = incomplete_phis.find(block); found != incomplete_phis.end()) {
for (auto& [variable, phi]: found->second) {
std::visit([&](auto value) { AddPhiOperands(value, *phi, block); }, variable);
}
}
block->SsaSeal();
}
private:
template <typename T>
Value AddPhiOperands(T variable, Inst& phi, Block* block) {
for (auto* predecessor: block->ImmPredecessors()) {
phi.AddPhiOperand(predecessor, Read(variable, predecessor));
}
return TryRemoveTrivialPhi(phi, InitialValue(variable));
}
Value TryRemoveTrivialPhi(Inst& phi, Value initial_value) {
Value same;
for (size_t index = 0; index < phi.NumArgs(); index++) {
const auto operand = phi.Arg(index).Resolve();
if (operand == same.Resolve() || operand == Value(&phi)) {
continue;
}
if (!same.IsEmpty()) {
return Value(&phi);
}
same = operand;
}
if (same.IsEmpty()) {
same = initial_value;
}
const auto users = phi.Uses();
phi.ReplaceUsesWith(same);
for (const auto& use: users) {
if (use.user->GetOpcode() == ValueOpcode::Phi) {
TryRemoveTrivialPhi(*use.user, initial_value);
}
}
return same;
}
std::unordered_map<Block*, std::map<Variable, Inst*>> incomplete_phis;
DefTable definitions;
};
void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
switch (inst.GetOpcode()) {
case ValueOpcode::SetScalarRegister:
pass.Write(inst.Arg(0).ScalarRegister(), block, inst.Arg(1));
break;
case ValueOpcode::SetThreadBitScalarRegister:
pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetVectorRegister:
pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1));
break;
case ValueOpcode::SetGotoVariable:
pass.Write(GotoVariable {inst.Arg(0).U32()}, block, inst.Arg(1));
break;
case ValueOpcode::SetScc: pass.Write(SccTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetExec: pass.Write(ExecTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetExecLo: pass.Write(ExecLoTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetExecHi: pass.Write(ExecHiTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetVcc: pass.Write(VccTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetVccLo: pass.Write(VccLoTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetVccHi: pass.Write(VccHiTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetM0: pass.Write(M0Tag {}, block, inst.Arg(0)); break;
case ValueOpcode::GetScalarRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).ScalarRegister(), block));
break;
case ValueOpcode::GetThreadBitScalarRegister:
inst.ReplaceUsesWith(
pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetVectorRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block));
break;
case ValueOpcode::GetGotoVariable:
inst.ReplaceUsesWith(pass.Read(GotoVariable {inst.Arg(0).U32()}, block));
break;
case ValueOpcode::GetScc: inst.ReplaceUsesWith(pass.Read(SccTag {}, block)); break;
case ValueOpcode::GetExec: inst.ReplaceUsesWith(pass.Read(ExecTag {}, block)); break;
case ValueOpcode::GetExecLo: inst.ReplaceUsesWith(pass.Read(ExecLoTag {}, block)); break;
case ValueOpcode::GetExecHi: inst.ReplaceUsesWith(pass.Read(ExecHiTag {}, block)); break;
case ValueOpcode::GetVcc: inst.ReplaceUsesWith(pass.Read(VccTag {}, block)); break;
case ValueOpcode::GetVccLo: inst.ReplaceUsesWith(pass.Read(VccLoTag {}, block)); break;
case ValueOpcode::GetVccHi: inst.ReplaceUsesWith(pass.Read(VccHiTag {}, block)); break;
case ValueOpcode::GetM0: inst.ReplaceUsesWith(pass.Read(M0Tag {}, block)); break;
default: break;
}
}
} // namespace
void RewriteToSsa(const BlockList& blocks) {
Pass pass;
for (auto* block: blocks) {
for (auto& inst: *block) {
VisitInstruction(pass, block, inst);
}
}
for (auto* block: blocks) {
pass.Seal(block);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ir/passes/SsaRewrite.h
+9
View File
@@ -0,0 +1,9 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void RewriteToSsa(const BlockList& blocks);
} // namespace Libs::Graphics::ShaderRecompiler::IR